diff --git a/go.mod b/go.mod index 3d52704..468c225 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/guoyk93/esbridge go 1.14 require ( - github.com/buger/jsonparser v1.0.0 + github.com/buger/jsonparser v1.1.1 github.com/guoyk93/conc v1.1.1 github.com/guoyk93/esexporter v1.4.0 github.com/guoyk93/iocount v1.0.0 diff --git a/go.sum b/go.sum index 16f85bc..e93aa6f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/buger/jsonparser v1.0.0 h1:etJTGF5ESxjI0Ic2UaLQs2LQQpa8G9ykQScukbh4L8A= github.com/buger/jsonparser v1.0.0/go.mod h1:tgcrVJ81GPSF0mz+0nu1Xaz0fazGPrmmJfJtxjbHhUQ= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,7 +19,6 @@ github.com/guoyk93/esexporter v1.4.0 h1:60XEL+K11bWt56owatMMc2F+1YrEcrPQyr+4SbD2 github.com/guoyk93/esexporter v1.4.0/go.mod h1:Fr5ioXETupYrQczUlzBlVJX6XVKRaJr6kfGDLqXdghg= github.com/guoyk93/iocount v1.0.0 h1:W/wcX2SWvxW7LwfyMTaori1Lfu0Z5KwdpYgpRRE6lL8= github.com/guoyk93/iocount v1.0.0/go.mod h1:liRY2VKoMqynnZEiKVtC4FTl0VX9QQibmm+nHBHZR7s= -github.com/guoyk93/logutil v1.0.0 h1:yoVVh4EuAoN1viBrP7ZnVijvatwrRfIDrm2SGprlHJ0= github.com/guoyk93/logutil v1.0.0/go.mod h1:HszSIxq8EecplH+383axY9UznRsyL+ojVCtmqIsgemU= github.com/guoyk93/logutil v1.0.1 h1:ovHOL3Xmp0qkH02q1NTBm26X6T/K3sPkzpfvbehcmoo= github.com/guoyk93/logutil v1.0.1/go.mod h1:HszSIxq8EecplH+383axY9UznRsyL+ojVCtmqIsgemU= diff --git a/vendor/github.com/buger/jsonparser/.travis.yml b/vendor/github.com/buger/jsonparser/.travis.yml index 8e29f13..dbfb7cf 100644 --- a/vendor/github.com/buger/jsonparser/.travis.yml +++ b/vendor/github.com/buger/jsonparser/.travis.yml @@ -1,4 +1,7 @@ language: go +arch: + - amd64 + - ppc64le go: - 1.7.x - 1.8.x diff --git a/vendor/github.com/buger/jsonparser/README.md b/vendor/github.com/buger/jsonparser/README.md index c449563..d7e0ec3 100644 --- a/vendor/github.com/buger/jsonparser/README.md +++ b/vendor/github.com/buger/jsonparser/README.md @@ -1,5 +1,5 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/buger/jsonparser)](https://goreportcard.com/report/github.com/buger/jsonparser) ![License](https://img.shields.io/dub/l/vibe-d.svg) -# Alternative JSON parser for Go (so far fastest) +# Alternative JSON parser for Go (10x times faster standard library) It does not require you to know the structure of the payload (eg. create structs), and allows accessing fields by providing the path to them. It is up to **10 times faster** than standard `encoding/json` package (depending on payload size and usage), **allocates no memory**. See benchmarks below. diff --git a/vendor/github.com/buger/jsonparser/bytes_unsafe.go b/vendor/github.com/buger/jsonparser/bytes_unsafe.go index d3f523d..589fea8 100644 --- a/vendor/github.com/buger/jsonparser/bytes_unsafe.go +++ b/vendor/github.com/buger/jsonparser/bytes_unsafe.go @@ -6,6 +6,7 @@ import ( "reflect" "strconv" "unsafe" + "runtime" ) // @@ -32,11 +33,12 @@ func bytesToString(b *[]byte) string { } func StringToBytes(s string) []byte { + b := make([]byte, 0, 0) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) - bh := reflect.SliceHeader{ - Data: sh.Data, - Len: sh.Len, - Cap: sh.Len, - } - return *(*[]byte)(unsafe.Pointer(&bh)) + bh.Data = sh.Data + bh.Cap = sh.Len + bh.Len = sh.Len + runtime.KeepAlive(s) + return b } diff --git a/vendor/github.com/buger/jsonparser/fuzz.go b/vendor/github.com/buger/jsonparser/fuzz.go index 84809d8..854bd11 100644 --- a/vendor/github.com/buger/jsonparser/fuzz.go +++ b/vendor/github.com/buger/jsonparser/fuzz.go @@ -7,3 +7,111 @@ func FuzzParseString(data []byte) int { } return 1 } + +func FuzzEachKey(data []byte) int { + paths := [][]string{ + {"name"}, + {"order"}, + {"nested", "a"}, + {"nested", "b"}, + {"nested2", "a"}, + {"nested", "nested3", "b"}, + {"arr", "[1]", "b"}, + {"arrInt", "[3]"}, + {"arrInt", "[5]"}, + {"nested"}, + {"arr", "["}, + {"a\n", "b\n"}, + } + EachKey(data, func(idx int, value []byte, vt ValueType, err error) {}, paths...) + return 1 +} + +func FuzzDelete(data []byte) int { + Delete(data, "test") + return 1 +} + +func FuzzSet(data []byte) int { + _, err := Set(data, []byte(`"new value"`), "test") + if err != nil { + return 0 + } + return 1 +} + +func FuzzObjectEach(data []byte) int { + _ = ObjectEach(data, func(key, value []byte, valueType ValueType, off int) error { + return nil + }) + return 1 +} + +func FuzzParseFloat(data []byte) int { + _, err := ParseFloat(data) + if err != nil { + return 0 + } + return 1 +} + +func FuzzParseInt(data []byte) int { + _, err := ParseInt(data) + if err != nil { + return 0 + } + return 1 +} + +func FuzzParseBool(data []byte) int { + _, err := ParseBoolean(data) + if err != nil { + return 0 + } + return 1 +} + +func FuzzTokenStart(data []byte) int { + _ = tokenStart(data) + return 1 +} + +func FuzzGetString(data []byte) int { + _, err := GetString(data, "test") + if err != nil { + return 0 + } + return 1 +} + +func FuzzGetFloat(data []byte) int { + _, err := GetFloat(data, "test") + if err != nil { + return 0 + } + return 1 +} + +func FuzzGetInt(data []byte) int { + _, err := GetInt(data, "test") + if err != nil { + return 0 + } + return 1 +} + +func FuzzGetBoolean(data []byte) int { + _, err := GetBoolean(data, "test") + if err != nil { + return 0 + } + return 1 +} + +func FuzzGetUnsafeString(data []byte) int { + _, err := GetUnsafeString(data, "test") + if err != nil { + return 0 + } + return 1 +} diff --git a/vendor/github.com/buger/jsonparser/go.mod b/vendor/github.com/buger/jsonparser/go.mod index 4f35768..7ede21f 100644 --- a/vendor/github.com/buger/jsonparser/go.mod +++ b/vendor/github.com/buger/jsonparser/go.mod @@ -1,3 +1,4 @@ module github.com/buger/jsonparser go 1.13 + diff --git a/vendor/github.com/buger/jsonparser/oss-fuzz-build.sh b/vendor/github.com/buger/jsonparser/oss-fuzz-build.sh new file mode 100644 index 0000000..c573b0e --- /dev/null +++ b/vendor/github.com/buger/jsonparser/oss-fuzz-build.sh @@ -0,0 +1,47 @@ +#!/bin/bash -eu + +git clone https://github.com/dvyukov/go-fuzz-corpus +zip corpus.zip go-fuzz-corpus/json/corpus/* + +cp corpus.zip $OUT/fuzzparsestring_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzParseString fuzzparsestring + +cp corpus.zip $OUT/fuzzeachkey_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzEachKey fuzzeachkey + +cp corpus.zip $OUT/fuzzdelete_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzDelete fuzzdelete + +cp corpus.zip $OUT/fuzzset_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzSet fuzzset + +cp corpus.zip $OUT/fuzzobjecteach_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzObjectEach fuzzobjecteach + +cp corpus.zip $OUT/fuzzparsefloat_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzParseFloat fuzzparsefloat + +cp corpus.zip $OUT/fuzzparseint_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzParseInt fuzzparseint + +cp corpus.zip $OUT/fuzzparsebool_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzParseBool fuzzparsebool + +cp corpus.zip $OUT/fuzztokenstart_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzTokenStart fuzztokenstart + +cp corpus.zip $OUT/fuzzgetstring_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzGetString fuzzgetstring + +cp corpus.zip $OUT/fuzzgetfloat_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzGetFloat fuzzgetfloat + +cp corpus.zip $OUT/fuzzgetint_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzGetInt fuzzgetint + +cp corpus.zip $OUT/fuzzgetboolean_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzGetBoolean fuzzgetboolean + +cp corpus.zip $OUT/fuzzgetunsafestring_seed_corpus.zip +compile_go_fuzzer github.com/buger/jsonparser FuzzGetUnsafeString fuzzgetunsafestring + diff --git a/vendor/github.com/buger/jsonparser/parser.go b/vendor/github.com/buger/jsonparser/parser.go index bd318e5..14b80bc 100644 --- a/vendor/github.com/buger/jsonparser/parser.go +++ b/vendor/github.com/buger/jsonparser/parser.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "fmt" - "math" "strconv" ) @@ -309,7 +308,11 @@ func searchKeys(data []byte, keys ...string) int { case '[': // If we want to get array element by index if keyLevel == level && keys[level][0] == '[' { - aIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) + var keyLen = len(keys[level]) + if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + return -1 + } + aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { return -1 } @@ -356,14 +359,6 @@ func searchKeys(data []byte, keys ...string) int { return -1 } -var bitwiseFlags []int64 - -func init() { - for i := 0; i < 63; i++ { - bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i)))) - } -} - func sameTree(p1, p2 []string) bool { minLen := len(p1) if len(p2) < minLen { @@ -380,7 +375,8 @@ func sameTree(p1, p2 []string) bool { } func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int { - var pathFlags int64 + var x struct{} + pathFlags := make([]bool, len(paths)) var level, pathsMatched, i int ln := len(data) @@ -422,13 +418,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str // for unescape: if there are no escape sequences, this is cheap; if there are, it is a // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize var keyUnesc []byte - var stackbuf [unescapeStackBufSize]byte if !keyEscaped { keyUnesc = key - } else if ku, err := Unescape(key, stackbuf[:]); err != nil { - return -1 } else { - keyUnesc = ku + var stackbuf [unescapeStackBufSize]byte + if ku, err := Unescape(key, stackbuf[:]); err != nil { + return -1 + } else { + keyUnesc = ku + } } if maxPath >= level { @@ -439,17 +437,16 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str pathsBuf[level-1] = bytesToString(&keyUnesc) for pi, p := range paths { - if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) { + if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) { continue } match = pi - i++ pathsMatched++ - pathFlags |= bitwiseFlags[pi+1] + pathFlags[pi] = true - v, dt, _, e := Get(data[i:]) + v, dt, _, e := Get(data[i+1:]) cb(pi, v, dt, e) if pathsMatched == len(paths) { @@ -485,8 +482,9 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str case '}': level-- case '[': - var arrIdxFlags int64 - var pIdxFlags int64 + var ok bool + arrIdxFlags := make(map[int]struct{}) + pIdxFlags := make([]bool, len(paths)) if level < 0 { cb(-1, nil, Unknown, MalformedJsonError) @@ -494,31 +492,31 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str } for pi, p := range paths { - if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) { + if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) { continue } if len(p[level]) >= 2 { aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1]) - arrIdxFlags |= bitwiseFlags[aIdx+1] - pIdxFlags |= bitwiseFlags[pi+1] + arrIdxFlags[aIdx] = x + pIdxFlags[pi] = true } } - if arrIdxFlags > 0 { + if len(arrIdxFlags) > 0 { level++ var curIdx int arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 { + if _, ok = arrIdxFlags[curIdx]; ok { for pi, p := range paths { - if pIdxFlags&bitwiseFlags[pi+1] != 0 { + if pIdxFlags[pi] { aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1]) if curIdx == aIdx { of := searchKeys(value, p[level:]...) pathsMatched++ - pathFlags |= bitwiseFlags[pi+1] + pathFlags[pi] = true if of != -1 { v, dt, _, e := Get(value[of:]) @@ -597,48 +595,96 @@ var ( ) func createInsertComponent(keys []string, setValue []byte, comma, object bool) []byte { - var buffer bytes.Buffer isIndex := string(keys[0][0]) == "[" + offset := 0 + lk := calcAllocateSpace(keys, setValue, comma, object) + buffer := make([]byte, lk, lk) if comma { - buffer.WriteString(",") + offset += WriteToBuffer(buffer[offset:], ",") } if isIndex && !comma { - buffer.WriteString("[") + offset += WriteToBuffer(buffer[offset:], "[") } else { if object { - buffer.WriteString("{") + offset += WriteToBuffer(buffer[offset:], "{") } if !isIndex { - buffer.WriteString("\"") - buffer.WriteString(keys[0]) - buffer.WriteString("\":") + offset += WriteToBuffer(buffer[offset:], "\"") + offset += WriteToBuffer(buffer[offset:], keys[0]) + offset += WriteToBuffer(buffer[offset:], "\":") } } for i := 1; i < len(keys); i++ { if string(keys[i][0]) == "[" { - buffer.WriteString("[") + offset += WriteToBuffer(buffer[offset:], "[") } else { - buffer.WriteString("{\"") - buffer.WriteString(keys[i]) - buffer.WriteString("\":") + offset += WriteToBuffer(buffer[offset:], "{\"") + offset += WriteToBuffer(buffer[offset:], keys[i]) + offset += WriteToBuffer(buffer[offset:], "\":") } } - buffer.Write(setValue) + offset += WriteToBuffer(buffer[offset:], string(setValue)) for i := len(keys) - 1; i > 0; i-- { if string(keys[i][0]) == "[" { - buffer.WriteString("]") + offset += WriteToBuffer(buffer[offset:], "]") } else { - buffer.WriteString("}") + offset += WriteToBuffer(buffer[offset:], "}") } } if isIndex && !comma { - buffer.WriteString("]") + offset += WriteToBuffer(buffer[offset:], "]") + } + if object && !isIndex { + offset += WriteToBuffer(buffer[offset:], "}") + } + return buffer +} + +func calcAllocateSpace(keys []string, setValue []byte, comma, object bool) int { + isIndex := string(keys[0][0]) == "[" + lk := 0 + if comma { + // , + lk += 1 + } + if isIndex && !comma { + // [] + lk += 2 + } else { + if object { + // { + lk += 1 + } + if !isIndex { + // "keys[0]" + lk += len(keys[0]) + 3 + } } + + + lk += len(setValue) + for i := 1; i < len(keys); i++ { + if string(keys[i][0]) == "[" { + // [] + lk += 2 + } else { + // {"keys[i]":setValue} + lk += len(keys[i]) + 5 + } + } + if object && !isIndex { - buffer.WriteString("}") + // } + lk += 1 } - return buffer.Bytes() + + return lk +} + +func WriteToBuffer(buffer []byte, str string) int { + copy(buffer, str) + return len(str) } /* @@ -766,7 +812,7 @@ func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error) if endOffset == -1 { firstToken := nextToken(data) // We can't set a top-level key if data isn't an object - if len(data) == 0 || data[firstToken] != '{' { + if firstToken < 0 || data[firstToken] != '{' { return nil, KeyPathNotFoundError } // Don't need a comma if the input is an empty object @@ -916,7 +962,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, value = value[1 : len(value)-1] } - return value, dataType, offset, endOffset, nil + return value[:len(value):len(value)], dataType, offset, endOffset, nil } // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. @@ -930,7 +976,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int return -1, MalformedJsonError } - offset = nT+1 + offset = nT + 1 if len(keys) > 0 { if offset = searchKeys(data, keys...); offset == -1 { @@ -1135,7 +1181,7 @@ func GetString(data []byte, keys ...string) (val string, err error) { return "", fmt.Errorf("Value is not a string: %s", string(v)) } - // If no escapes return raw conten + // If no escapes return raw content if bytes.IndexByte(v, '\\') == -1 { return string(v), nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index a5f1054..082227a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/buger/jsonparser v1.0.0 +# github.com/buger/jsonparser v1.1.1 ## explicit github.com/buger/jsonparser # github.com/davecgh/go-spew v1.1.1