From accee2e68967392931024150dd57ad77f048c503 Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Wed, 9 Jun 2021 15:58:27 +0800 Subject: [PATCH] feat: support UseNumber for ast (#14) Co-authored-by: duanyi.aster --- README.md | 22 ++++--- ast/iterator_test.go | 6 ++ ast/node.go | 138 +++++++++++++++++++++++++++++++++++-------- ast/node_test.go | 121 +++++++++++++++++++++++++++---------- ast/parser.go | 38 ++++++++---- ast/parser_test.go | 101 ++++++++++++++++++++++++++++--- ast/search.go | 1 - ast/search_test.go | 17 ++++++ 8 files changed, 362 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 985fd47..27f4656 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # Sonic -A blazingly fast JSON serializing & deserializing library. +A blazingly fast JSON serializing & deserializing library, accelerated by JIT(just-in-time compiling) and SIMD(single-instruction-multi-data). ## Benchmarks For all sizes of json and all scenes of usage, Sonic performs almost best. - Small (400B, 11 keys, 3 levels) ![small benchmarks](bench-400B.png) -- Medium (110KB, 300+ keys, 3 levels) +- Medium (110KB, 300+ keys, 3 levels, with many quoted-json values) ![medium benchmarks](bench-110KB.png) - Large (550KB, 10000+ key, 6 levels) ![large benchmarks](bench-550KB.png) -For a 13KB [TwitterJson](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19)(cpu i9-9880H, goarch amd64), Sonic is **1.5x** fast than [json-iterator](https://github.com/json-iterator/go) in decoding, **2.5x** fast in encoding. +For a 13KB [TwitterJson](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19)(cpu i9-9880H, goarch amd64), Sonic is **1.5x** faster than [json-iterator](https://github.com/json-iterator/go) in decoding, **2.5x** faster in encoding. ```powershell BenchmarkDecoder_Generic_Sonic-16 10000 54309 ns/op 240.01 MB/s 46149 B/op 303 allocs/op @@ -27,7 +27,7 @@ BenchmarkEncoder_Binding_Sonic-16 10000 73 BenchmarkEncoder_Binding_JsonIter-16 10000 23223 ns/op 561.31 MB/s 9489 B/op 2 allocs/op BenchmarkEncoder_Binding_StdLib-16 10000 19512 ns/op 668.07 MB/s 9477 B/op 1 allocs/op ``` -More detail see [ast/search_test.go](https://github.com/bytedance/sonic/blob/main/ast/search_test.go), [decoder/decoder_test.go](https://github.com/bytedance/sonic/blob/main/decoder/decoder_test.go), [encoder/encoder_test.go](https://github.com/bytedance/sonic/blob/main/encoder/encoder_test.go) +More detail see [ast/search_test.go](https://github.com/bytedance/sonic/blob/main/ast/search_test.go), [decoder/decoder_test.go](https://github.com/bytedance/sonic/blob/main/decoder/decoder_test.go), [encoder/encoder_test.go](https://github.com/bytedance/sonic/blob/main/encoder/encoder_test.go), ## Usage @@ -83,6 +83,14 @@ dc.Decode(&data) // data == json.Number("1") dc = decoder.NewDecoder(input) dc.UseInt64() dc.Decode(&data) // data == int64(1) + +root, err := sonic.GetFromString(input) +// Get json.Number +jn := root.Number() +jm := root.InterfaceUseNumber().(json.Number) // jn == jm +// Get float64 +fn := root.Float64() +fm := root.Interface().(float64) // jn == jm ``` ## Tips @@ -102,7 +110,7 @@ func init() { ``` ### Pass string or []byte? -For alignment to encoding/json, we provide API to pass `[]byte` as arguement, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when origin json is huge. Therefore, you can use `UnmarshalString`, `GetFromString` to pass string, as long as your origin data is string or **nocopy-case** is safe for your []byte. +For alignment to encoding/json, we provide API to pass `[]byte` as arguement, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when origin json is huge. Therefore, you can use `UnmarshalString`, `GetFromString` to pass string, as long as your origin data is string or **nocopy-cast** is safe for your []byte. ### Avoid repeating work `Get()` overlapping pathes from the same root may cause repeating parsing. Instead of using `Get()` several times, you can use parser and searcher together like this: @@ -115,8 +123,8 @@ b = root.GetByPath( "entities","url") c = root.GetByPath( "created_at") ``` No need to worry about the overlaping or overparsing of a, b and c, because the inner parser of their root is lazy-loaded. -### Better performance for `interface{}` (or `map[string]interface{}`) -In most cases of fully-load generic json, `Unmarshal()` performs better than `ast.Loads()`. But if you only want to search a partial json and convert it into `interface{}` (or `Map()` for `map[string]interface{}`, `Array()` for `[]interface{}`), we advise you to combine these two: +### Better performance for generic deserializing +In most cases of fully-load generic json, `Unmarshal()` performs better than `ast.Loads()`. But if you only want to search a partial json and convert it into `interface{}` (or `map[string]interface{}`, `[]interface{}`), we advise you to combine `Get()` and `Unmarshal()`: ```go import "github.com/bytedance/sonic" diff --git a/ast/iterator_test.go b/ast/iterator_test.go index 99e130b..90ab516 100644 --- a/ast/iterator_test.go +++ b/ast/iterator_test.go @@ -53,6 +53,9 @@ func TestIterator(t *testing.T) { if i < int64(loop) && v.Int64() != i { t.Fatalf("exp:%v, got:%v", i, v) } + if i != int64(ai.Pos())-1 || i >= int64(ai.Len()) { + t.Fatal(i) + } i++ } @@ -70,6 +73,9 @@ func TestIterator(t *testing.T) { if i < int64(loop) &&( v.Value.Int64() != i ||v.Key != fmt.Sprintf("k%d", i)) { t.Fatalf("exp:%v, got:%v", i, v.Value.Interface()) } + if i != int64(mi.Pos())-1 || i >= int64(mi.Len()) { + t.Fatal(i) + } i++ } } diff --git a/ast/node.go b/ast/node.go index 9328eda..634486a 100644 --- a/ast/node.go +++ b/ast/node.go @@ -19,7 +19,6 @@ package ast import ( `encoding/json` `fmt` - `strconv` `unsafe` `github.com/bytedance/sonic/internal/native` @@ -29,7 +28,7 @@ import ( const ( _MAP_THRESHOLD = 5 _CAP_BITS = 32 - _LEN_MASK = 1<<_CAP_BITS - 1 + _LEN_MASK = 1 << _CAP_BITS - 1 _APPEND_EXTRA_SIZE = 5 _NODE_SIZE = unsafe.Sizeof(Node{}) @@ -38,6 +37,7 @@ const ( const ( V_RAW native.ValueType = 1 << 4 + V_NUMBER native.ValueType = 10 V_ARRAY_RAW = V_RAW | native.V_ARRAY V_OBJECT_RAW = V_RAW | native.V_OBJECT MASK_RAW = V_RAW - 1 @@ -85,10 +85,9 @@ func (self *Node) Bool() bool { // Int64 as above. func (self *Node) Int64() int64 { switch self.t { + case V_NUMBER : return numberToInt64(self) case native.V_TRUE : return 1 case native.V_FALSE : return 0 - case native.V_DOUBLE : return int64(i64tof(self.v)) - case native.V_INTEGER : return self.v case V_RAW : n := self.parseRaw() return n.Int64() @@ -99,11 +98,10 @@ func (self *Node) Int64() int64 { // Number as above. func (self *Node) Number() json.Number { switch self.t { - case native.V_DOUBLE : return json.Number(strconv.FormatFloat(i64tof(self.v), 'g', -1, 64)) - case native.V_INTEGER : return json.Number(strconv.FormatInt(self.v, 10)) case V_RAW : n := self.parseRaw() return n.Number() + case V_NUMBER : return toNumber(self) default : panic("value cannot be represented as a json.Number") } } @@ -111,12 +109,11 @@ func (self *Node) Number() json.Number { // String as above. func (self *Node) String() string { switch self.t { + case V_NUMBER : return toNumber(self).String() case native.V_NULL : return "null" case native.V_TRUE : return "true" case native.V_FALSE : return "false" case native.V_STRING : return addr2str(self.p, self.v) - case native.V_DOUBLE : return strconv.FormatFloat(i64tof(self.v), 'g', -1, 64) - case native.V_INTEGER : return strconv.FormatInt(self.v, 10) case V_RAW : n := self.parseRaw() return n.String() @@ -127,10 +124,9 @@ func (self *Node) String() string { // Float64 as above. func (self *Node) Float64() float64 { switch self.t { + case V_NUMBER : return numberToFloat64(self) case native.V_TRUE : return 1.0 case native.V_FALSE : return 0.0 - case native.V_DOUBLE : return i64tof(self.v) - case native.V_INTEGER : return float64(self.v) case V_RAW : n := self.parseRaw() return n.Float64() @@ -254,12 +250,14 @@ func (self *Node) Index(idx int) *Node { return self.loadIndex(idx) } +// Values returns iterator for array's children traversal func (self *Node) Values() ListIterator { self.must(native.V_ARRAY, "an array") self.loadAllIndex() return ListIterator{Iterator{p: self}} } +// Values returns iterator for object's children traversal func (self *Node) Properties() ObjectIterator { self.must(native.V_OBJECT, "an object") self.loadAllKey() @@ -275,6 +273,13 @@ func (self *Node) Map() map[string]interface{} { return self.toGenericObject() } +// MapUseNumber loads all keys of an object node, with numeric nodes casted to json.Number +func (self *Node) MapUseNumber() map[string]interface{} { + self.must(native.V_OBJECT, "an object") + self.loadAllKey() + return self.toGenericObjectUseNumber() +} + // Array loads all indexes of an array node func (self *Node) Array() []interface{} { self.must(native.V_ARRAY, "an array") @@ -282,8 +287,16 @@ func (self *Node) Array() []interface{} { return self.toGenericArray() } +// Array loads all indexes of an array node, with numeric nodes casted to json.Number +func (self *Node) ArrayUseNumber() []interface{} { + self.must(native.V_ARRAY, "an array") + self.loadAllIndex() + return self.toGenericArrayUseNumber() +} + // Interface loads all children under all pathes from this node, // and converts itself as generic go type +// all numberic nodes are casted to float64 func (self *Node) Interface() interface{} { switch self.t { case native.V_EOF : panic("invalid value") @@ -293,8 +306,8 @@ func (self *Node) Interface() interface{} { case native.V_ARRAY : return self.toGenericArray() case native.V_OBJECT : return self.toGenericObject() case native.V_STRING : return addr2str(self.p, self.v) - case native.V_DOUBLE : return i64tof(self.v) - case native.V_INTEGER : return self.v + case V_NUMBER : + return numberToFloat64(self) case V_ARRAY_RAW: self.loadAllIndex() return self.toGenericArray() @@ -308,6 +321,32 @@ func (self *Node) Interface() interface{} { } } +// InterfaceUseNumber works same with Interface() +// except numberic nodes are casted to json.Number +func (self *Node) InterfaceUseNumber() interface{} { + switch self.t { + case native.V_EOF : panic("invalid value") + case native.V_NULL : return nil + case native.V_TRUE : return true + case native.V_FALSE : return false + case native.V_ARRAY : return self.toGenericArrayUseNumber() + case native.V_OBJECT : return self.toGenericObjectUseNumber() + case native.V_STRING : return addr2str(self.p, self.v) + case V_NUMBER : + return toNumber(self) + case V_ARRAY_RAW: + self.loadAllIndex() + return self.toGenericArrayUseNumber() + case V_OBJECT_RAW: + self.loadAllKey() + return self.toGenericObjectUseNumber() + case V_RAW : + n := self.parseRaw() + return n.InterfaceUseNumber() + default : panic("not gonna happen") + } +} + /** Internal Helper Methods **/ var ( @@ -652,6 +691,25 @@ func (self *Node) toGenericArray() []interface{} { return ret } +func (self *Node) toGenericArrayUseNumber() []interface{} { + nb := self.Len() + ret := make([]interface{}, nb) + if nb == 0 { + return ret + } + + /* convert each item */ + var p = (*Node)(self.p) + ret[0] = p.InterfaceUseNumber() + for i := 1; i < nb; i++ { + p = p.unsafe_next() + ret[i] = p.InterfaceUseNumber() + } + + /* all done */ + return ret +} + func (self *Node) toGenericObject() map[string]interface{} { nb := self.Len() ret := make(map[string]interface{}, nb) @@ -671,6 +729,26 @@ func (self *Node) toGenericObject() map[string]interface{} { return ret } + +func (self *Node) toGenericObjectUseNumber() map[string]interface{} { + nb := self.Len() + ret := make(map[string]interface{}, nb) + if nb == 0 { + return ret + } + + /* convert each item */ + var p = (*Pair)(self.p) + ret[p.Key] = p.Value.InterfaceUseNumber() + for i := 1; i < nb; i++ { + p = p.unsafe_next() + ret[p.Key] = p.Value.InterfaceUseNumber() + } + + /* all done */ + return ret +} + /** Internal Factory Methods **/ var ( @@ -682,13 +760,34 @@ var ( emptyObjectNode = Node{t: native.V_OBJECT} ) -func newInt64(v int64) Node { +func newNumber(v string) Node { return Node{ - v: v, - t: native.V_INTEGER, + v: int64(len(v)), + p: str2ptr(v), + t: V_NUMBER, } } +func toNumber(node *Node) json.Number { + return json.Number(addr2str(node.p, node.v)) +} + +func numberToFloat64(node *Node) float64 { + ret,err := toNumber(node).Float64() + if err != nil { + panic(err) + } + return ret +} + +func numberToInt64(node *Node) int64 { + ret,err := toNumber(node).Int64() + if err != nil { + panic(err) + } + return ret +} + func newBytes(v []byte) Node { return Node{ t: native.V_STRING, @@ -705,13 +804,6 @@ func newString(v string) Node { } } -func newFloat64(v float64) Node { - return Node{ - t: native.V_DOUBLE, - v: f64toi(v), - } -} - func newArray(v []Node) Node { return Node{ t: native.V_ARRAY, @@ -799,7 +891,7 @@ func newRawNode(str string) Node { } func (self *Node) parseRaw() Node { - raw := self.Raw() + raw := addr2str(self.p, self.v) n, e := NewParser(raw).Parse() if e != 0 { panic(fmt.Sprintf("%v, raw json: %s", e.Error(), raw)) diff --git a/ast/node_test.go b/ast/node_test.go index f0dedab..33d5e31 100644 --- a/ast/node_test.go +++ b/ast/node_test.go @@ -17,15 +17,81 @@ package ast import ( - "strconv" - "testing" + "encoding/json" + "strconv" + "testing" - "github.com/bytedance/sonic/internal/native" - jsoniter "github.com/json-iterator/go" + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" ) var parallelism = 4 +func TestUseNumber(t *testing.T) { + node, err := NewParser("1061346755812312312313").Parse() + if err != 0 { + t.Fatal(err) + } + if node.Type() != V_NUMBER { + t.Fatalf("wrong type: %v", node.Type()) + } + iv := node.InterfaceUseNumber().(json.Number) + if iv.String() != "1061346755812312312313" { + t.Fatalf("exp:%#v, got:%#v", "1061346755812312312313", iv.String()) + } + ix := node.Interface().(float64) + if ix != float64(1061346755812312312313) { + t.Fatalf("exp:%#v, got:%#v", float64(1061346755812312312313), ix) + } + ij,_ := node.Number().Int64() + jj,_ := json.Number("1061346755812312312313").Int64() + if ij != jj { + t.Fatalf("exp:%#v, got:%#v", jj, ij) + } +} + +func TestMap(t *testing.T) { + node, err := NewParser(`{"a":-0, "b":1, "c":-1.2, "d":-1.2e-10}`).Parse() + if err != 0 { + t.Fatal(err) + } + m := node.Map() + assert.Equal(t, m, map[string]interface{}{ + "a": float64(0), + "b": float64(1), + "c": float64(-1.2), + "d": float64(-1.2e-10), + }) + m1 := node.MapUseNumber() + assert.Equal(t, m1, map[string]interface{}{ + "a": json.Number("-0"), + "b": json.Number("1"), + "c": json.Number("-1.2"), + "d": json.Number("-1.2e-10"), + }) +} + +func TestArray(t *testing.T) { + node, err := NewParser(`[-0, 1, -1.2, -1.2e-10]`).Parse() + if err != 0 { + t.Fatal(err) + } + m := node.Array() + assert.Equal(t, m, []interface{}{ + float64(0), + float64(1), + float64(-1.2), + float64(-1.2e-10), + }) + m1 := node.ArrayUseNumber() + assert.Equal(t, m1, []interface{}{ + json.Number("-0"), + json.Number("1"), + json.Number("-1.2"), + json.Number("-1.2e-10"), + }) +} + func TestNodeRaw(t *testing.T) { root, derr := NewSearcher(_TwitterJson).GetByPath("search_metadata") if derr != nil { @@ -145,19 +211,14 @@ func TestNodeSet(t *testing.T) { if derr != 0 { t.Fatalf("decode failed: %v", derr.Error()) } - root.GetByPath("statuses", 3).Set("id_str", Node{ - v: int64(111), - t: native.V_INTEGER, - }) + app,_ := NewParser("111").Parse() + root.GetByPath("statuses", 3).Set("id_str", app) val := root.GetByPath("statuses", 3, "id_str").Int64() if val != 111 { t.Fatalf("exp: %+v, got: %+v", 111, val) } for i := root.GetByPath("statuses", 3).Cap(); i >= 0; i-- { - root.GetByPath("statuses", 3).Set("id_str"+strconv.Itoa(i), Node{ - v: int64(111), - t: native.V_INTEGER, - }) + root.GetByPath("statuses", 3).Set("id_str"+strconv.Itoa(i), app) } val = root.GetByPath("statuses", 3, "id_str0").Int64() if val != 111 { @@ -180,10 +241,8 @@ func TestNodeSetByIndex(t *testing.T) { if derr != 0 { t.Fatalf("decode failed: %v", derr.Error()) } - root.GetByPath("statuses").SetByIndex(0, Node{ - v: int64(111), - t: native.V_INTEGER, - }) + app, _ := NewParser("111").Parse() + root.GetByPath("statuses").SetByIndex(0, app) val := root.GetByPath("statuses", 0).Int64() if val != 111 { t.Fatalf("exp: %+v, got: %+v", 111, val) @@ -205,12 +264,10 @@ func TestNodeAdd(t *testing.T) { if derr != 0 { t.Fatalf("decode failed: %v", derr.Error()) } + app, _ := NewParser("111").Parse() for i := root.GetByPath("statuses").Cap(); i >= 0; i-- { - root.GetByPath("statuses").Add(Node{ - v: int64(111), - t: native.V_INTEGER, - }) + root.GetByPath("statuses").Add(app) } val := root.GetByPath("statuses", 4).Int64() if val != 111 { @@ -286,11 +343,11 @@ func BenchmarkNodeGet(b *testing.B) { b.Fatalf("decode failed: %v", derr.Error()) } node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", newInt64(1)) - node.Set("test2", newInt64(2)) - node.Set("test3", newInt64(3)) - node.Set("test4", newInt64(4)) - node.Set("test5", newInt64(5)) + node.Set("test1", newNumber("1")) + node.Set("test2", newNumber("2")) + node.Set("test3", newNumber("3")) + node.Set("test4", newNumber("4")) + node.Set("test5", newNumber("5")) b.ResetTimer() for i := 0; i < b.N; i++ { node.Get("text").Interface() @@ -303,11 +360,11 @@ func BenchmarkMapGet(b *testing.B) { b.Fatalf("decode failed: %v", derr.Error()) } node := root.Get("statuses").Index(3).Get("entities").Get("hashtags").Index(0) - node.Set("test1", newInt64(1)) - node.Set("test2", newInt64(2)) - node.Set("test3", newInt64(3)) - node.Set("test4", newInt64(4)) - node.Set("test5", newInt64(5)) + node.Set("test1", newNumber("1")) + node.Set("test2", newNumber("2")) + node.Set("test3", newNumber("3")) + node.Set("test4", newNumber("4")) + node.Set("test5", newNumber("5")) m := node.Map() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -324,7 +381,7 @@ func BenchmarkNodeSet(b *testing.B) { b.SetParallelism(parallelism) b.ResetTimer() for i := 0; i < b.N; i++ { - node.Set("test1", newInt64(1)) + node.Set("test1", newNumber("1")) } } @@ -352,7 +409,7 @@ func BenchmarkNodeAdd(b *testing.B) { for i := 0; i < b.N; i++ { root, _ := NewParser(data).Parse() node := root.Get("statuses") - node.Add(newObject([]Pair{{"test", newInt64(1)}})) + node.Add(newObject([]Pair{{"test", newNumber("1")}})) } } diff --git a/ast/parser.go b/ast/parser.go index b9712a0..04c73a1 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -257,19 +257,19 @@ func (self *Parser) Parse() (Node, native.ParsingError) { case native.V_NULL : return nullNode, 0 case native.V_TRUE : return trueNode, 0 case native.V_FALSE : return falseNode, 0 - case native.V_ARRAY: - if self.noLazy { - return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP)) - } - return newRawArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0 - case native.V_OBJECT: - if self.noLazy { - return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP)) - } - return newRawObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0 case native.V_STRING : return self.decodeString(val.Iv, val.Ep) - case native.V_DOUBLE : return newFloat64(val.Dv), 0 - case native.V_INTEGER : return newInt64(val.Iv), 0 + case native.V_ARRAY: + if self.noLazy { + return self.decodeArray(make([]Node, 0, _DEFAULT_NODE_CAP)) + } + return newRawArray(self, make([]Node, 0, _DEFAULT_NODE_CAP)), 0 + case native.V_OBJECT: + if self.noLazy { + return self.decodeObject(make([]Pair, 0, _DEFAULT_NODE_CAP)) + } + return newRawObject(self, make([]Pair, 0, _DEFAULT_NODE_CAP)), 0 + case native.V_DOUBLE : return newNumber(self.s[val.Ep:self.p]), 0 + case native.V_INTEGER : return newNumber(self.s[val.Ep:self.p]), 0 default : return Node{}, native.ParsingError(-val.Vt) } } @@ -287,6 +287,7 @@ func (self *Parser) skip() (int, native.ParsingError) { /** Parser Factory **/ +// Loads parse all json into interface{} func Loads(src string) (int, interface{}, native.ParsingError) { ps := &Parser{s: src} np, err := ps.Parse() @@ -299,6 +300,19 @@ func Loads(src string) (int, interface{}, native.ParsingError) { } } +// Loads parse all json into interface{}, with numeric nodes casted to json.Number +func LoadsUseNumber(src string) (int, interface{}, native.ParsingError) { + ps := &Parser{s: src} + np, err := ps.Parse() + + /* check for errors */ + if err != 0 { + return 0, nil, err + } else { + return ps.Pos(), np.InterfaceUseNumber(), 0 + } +} + func NewParser(src string) *Parser { return &Parser{s: src} } diff --git a/ast/parser_test.go b/ast/parser_test.go index 2ad5ced..9a69109 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -18,8 +18,6 @@ package ast import ( `encoding/json` - `fmt` - `reflect` `testing` jsoniter `github.com/json-iterator/go` @@ -30,20 +28,53 @@ import ( func runDecoderTest(t *testing.T, src string, expect interface{}) { vv, err := NewParser(src).Parse() if err != 0 { panic(err) } - fmt.Printf("%s -> %s :: %v\n", src, reflect.TypeOf(vv), vv) assert.Equal(t, expect, vv.Interface()) } +func runDecoderTestUseNumber(t *testing.T, src string, expect interface{}) { + vv, err := NewParser(src).Parse() + if err != 0 { panic(err) } + vvv := vv.InterfaceUseNumber() + switch vvv.(type) { + case json.Number: + assert.Equal(t, expect, n2f64(vvv.(json.Number))) + case []interface{}: + x := vvv.([]interface{}) + for i, e := range x { + if ev,ok := e.(json.Number);ok { + x[i] = n2f64(ev) + } + } + assert.Equal(t, expect, x) + case map[string]interface{}: + x := vvv.(map[string]interface{}) + for k,v := range x { + if ev, ok := v.(json.Number); ok { + x[k] = n2f64(ev) + } + } + assert.Equal(t, expect, x) + } +} + +func n2f64(i json.Number) float64{ + x, err := i.Float64() + if err != nil { + panic(err) + } + return x +} + func TestParser_Basic(t *testing.T) { runDecoderTest(t, `null`, nil) runDecoderTest(t, `true`, true) runDecoderTest(t, `false`, false) runDecoderTest(t, `"hello, world \\ \/ \b \f \n \r \t \u666f 测试中文"`, "hello, world \\ / \b \f \n \r \t \u666f 测试中文") runDecoderTest(t, `"\ud83d\ude00"`, "😀") - runDecoderTest(t, `0`, int64(0)) - runDecoderTest(t, `-0`, int64(0)) - runDecoderTest(t, `123456`, int64(123456)) - runDecoderTest(t, `-12345`, int64(-12345)) + runDecoderTest(t, `0`, float64(0)) + runDecoderTest(t, `-0`, float64(0)) + runDecoderTest(t, `123456`, float64(123456)) + runDecoderTest(t, `-12345`, float64(-12345)) runDecoderTest(t, `0.2`, 0.2) runDecoderTest(t, `1.2`, 1.2) runDecoderTest(t, `-0.2`, -0.2) @@ -78,6 +109,62 @@ func TestParser_Basic(t *testing.T) { runDecoderTest(t, `{}`, map[string]interface{}{}) runDecoderTest(t, `["asd", "123", true, false, null, 2.4, 1.2e15]`, []interface{}{"asd", "123", true, false, nil, 2.4, 1.2e15}) runDecoderTest(t, `{"asdf": "qwer", "zxcv": true}`, map[string]interface{}{"asdf": "qwer", "zxcv": true}) + runDecoderTest(t, `{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`, map[string]interface{}{"a":"123", "b":true, "c":false, "d":nil, "e":float64(2.4), "f":float64(1.2e15), "g":float64(1)}) + + runDecoderTestUseNumber(t, `null`, nil) + runDecoderTestUseNumber(t, `true`, true) + runDecoderTestUseNumber(t, `false`, false) + runDecoderTestUseNumber(t, `"hello, world \\ \/ \b \f \n \r \t \u666f 测试中文"`, "hello, world \\ / \b \f \n \r \t \u666f 测试中文") + runDecoderTestUseNumber(t, `"\ud83d\ude00"`, "😀") + runDecoderTestUseNumber(t, `0`, float64(0)) + runDecoderTestUseNumber(t, `-0`, float64(0)) + runDecoderTestUseNumber(t, `123456`, float64(123456)) + runDecoderTestUseNumber(t, `-12345`, float64(-12345)) + runDecoderTestUseNumber(t, `0.2`, float64(0.2)) + runDecoderTestUseNumber(t, `1.2`, float64(1.2)) + runDecoderTestUseNumber(t, `-0.2`, float64(-0.2)) + runDecoderTestUseNumber(t, `-1.2`, float64(-1.2)) + runDecoderTestUseNumber(t, `0e12`, float64(0e12)) + runDecoderTestUseNumber(t, `0e+12`, float64(0e+12)) + runDecoderTestUseNumber(t, `0e-12`, float64(0e-12)) + runDecoderTestUseNumber(t, `-0e12`, float64(-0e12)) + runDecoderTestUseNumber(t, `-0e+12`, float64(-0e+12)) + runDecoderTestUseNumber(t, `-0e-12`, float64(-0e-12)) + runDecoderTestUseNumber(t, `2e12`, float64(2e12)) + runDecoderTestUseNumber(t, `2E12`, float64(2e12)) + runDecoderTestUseNumber(t, `2e+12`, float64(2e+12)) + runDecoderTestUseNumber(t, `2e-12`, float64(2e-12)) + runDecoderTestUseNumber(t, `-2e12`, float64(-2e12)) + runDecoderTestUseNumber(t, `-2e+12`, float64(-2e+12)) + runDecoderTestUseNumber(t, `-2e-12`, float64(-2e-12)) + runDecoderTestUseNumber(t, `0.2e12`, float64(0.2e12)) + runDecoderTestUseNumber(t, `0.2e+12`, float64(0.2e+12)) + runDecoderTestUseNumber(t, `0.2e-12`, float64(0.2e-12)) + runDecoderTestUseNumber(t, `-0.2e12`, float64(-0.2e12)) + runDecoderTestUseNumber(t, `-0.2e+12`, float64(-0.2e+12)) + runDecoderTestUseNumber(t, `-0.2e-12`, float64(-0.2e-12)) + runDecoderTestUseNumber(t, `1.2e12`, float64(1.2e12)) + runDecoderTestUseNumber(t, `1.2e+12`, float64(1.2e+12)) + runDecoderTestUseNumber(t, `1.2e-12`, float64(1.2e-12)) + runDecoderTestUseNumber(t, `-1.2e12`, float64(-1.2e12)) + runDecoderTestUseNumber(t, `-1.2e+12`, float64(-1.2e+12)) + runDecoderTestUseNumber(t, `-1.2e-12`, float64(-1.2e-12)) + runDecoderTestUseNumber(t, `-1.2E-12`, float64(-1.2e-12)) + runDecoderTestUseNumber(t, `["asd", "123", true, false, null, 2.4, 1.2e15, 1]`, []interface{}{"asd", "123", true, false, nil, float64(2.4), float64(1.2e15), float64(1)}) + runDecoderTestUseNumber(t, `{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`, map[string]interface{}{"a":"123", "b":true, "c":false, "d":nil, "e":float64(2.4), "f":float64(1.2e15), "g":float64(1)}) +} + +func TestLoads(t *testing.T) { + _,i,e := Loads(`{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`) + if e != 0 { + t.Fatal(e) + } + assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": float64(2.4), "f": float64(1.2e15), "g": float64(1)}, i) + _,i,e = LoadsUseNumber(`{"a": "123", "b": true, "c": false, "d": null, "e": 2.4, "f": 1.2e15, "g": 1}`) + if e != 0 { + t.Fatal(e) + } + assert.Equal(t, map[string]interface{}{"a": "123", "b": true, "c": false, "d": nil, "e": json.Number("2.4"), "f": json.Number("1.2e15"), "g": json.Number("1")}, i) } func BenchmarkParser_StdLib(b *testing.B) { diff --git a/ast/search.go b/ast/search.go index 5b679e1..93784ae 100644 --- a/ast/search.go +++ b/ast/search.go @@ -24,7 +24,6 @@ import ( type Searcher struct { parser Parser - // cache map[string]*Node } func NewSearcher(str string) *Searcher { diff --git a/ast/search_test.go b/ast/search_test.go index 15f58d3..f2bb240 100644 --- a/ast/search_test.go +++ b/ast/search_test.go @@ -20,6 +20,7 @@ import ( `testing` jsoniter `github.com/json-iterator/go` + `github.com/stretchr/testify/assert` `github.com/tidwall/gjson` ) @@ -85,6 +86,22 @@ func TestSearcher_GetByPathErr(t *testing.T) { } } +func TestLoadIndex(t *testing.T) { + node, err := NewSearcher(`{"a":[-0, 1, -1.2, -1.2e-10]}`).GetByPath("a") + if err != nil { + t.Fatal(err) + } + a := node.Index(3).Float64() + assert.Equal(t, float64(-1.2e-10), a) + m := node.Array() + assert.Equal(t, m, []interface{}{ + float64(0), + float64(1), + float64(-1.2), + float64(-1.2e-10), + }) +} + func BenchmarkSearchOne_Gjson(b *testing.B) { b.SetBytes(int64(len(_TwitterJson))) for i := 0; i < b.N; i++ {