XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity
sonic/search_test.go

596 lines
No EOL
17 KiB
Go
Raw Blame History

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/hex`
`encoding/json`
`fmt`
`math/rand`
`reflect`
`strings`
`testing`
`time`
`github.com/bytedance/sonic/ast`
)
func Parse(src string) (*ast.Node, error) {
node, err := ast.NewParser(src).Parse()
if err != 0 {
return &node, fmt.Errorf("parsing error: %v", err)
}
return &node, nil
}
func assertCond(cond bool) {
if !cond {
panic("assertCond failed")
}
}
func TestExampleSearch(t *testing.T) {
data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)
node, e := Get(data, "test", 0)
if e != nil || node.Bool() != true {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = Get(data, "test", 1)
if e != nil || node.Float64() != 0.1 {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = Get(data, "test", 2)
if e != nil || node.String() != "abc" {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = Get(data, "test", 3)
if e != nil || node.Array()[0] != "h" {
t.Fatalf("node: %v, err: %v", node, e)
}
node, e = Get(data, "test", 4, "a")
if e != nil || node.String() != "bc" {
t.Fatalf("node: %v, err: %v", node, e)
}
}
func TestExampleSearchErr(t *testing.T) {
data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)
node, e := Get(data, "zz")
if e == nil {
t.Fatalf("node: %v, err: %v", node, e)
}
fmt.Println(e)
node, e = Get(data, "xx", 4)
if e == nil {
t.Fatalf("node: %v, err: %v", node, e)
}
fmt.Println(e)
node, e = Get(data, "yy", "a")
if e == nil {
t.Fatalf("node: %v, err: %v", node, e)
}
fmt.Println(e)
node, e = Get(data, "test", 4, "x")
if e == nil {
t.Fatalf("node: %v, err: %v", node, e)
}
fmt.Println(e)
}
func TestRandomData(t *testing.T) {
var lstr string
defer func() {
if v := recover(); v != nil {
println("'" + hex.Dump([]byte(lstr)) + "'")
println(lstr)
panic(v)
}
}()
data := []byte(`"<22>-mp<6D>`)
_, err := ast.NewParser(string(data)).Parse()
if err != 0 {
fmt.Println(hex.Dump(data))
fmt.Println(string(data))
}
rand.Seed(time.Now().UnixNano())
b := make([]byte, 200)
for i := 0; i < 1000000; i++ {
n, ee := rand.Read(b[:rand.Int()%len(b)])
if ee != nil {
t.Fatalf("get random bytes failed: %v,", ee)
return
}
lstr = string(b[:n])
_, _ = ast.NewParser(lstr).Parse()
}
}
func TestRandomValidStrings(t *testing.T) {
rand.Seed(time.Now().UnixNano())
b := make([]byte, 200)
for i := 0; i < 1000; i++ {
n, err := rand.Read(b[:rand.Int()%len(b)])
if err != nil {
t.Fatal("get random data failed:", err)
}
sm, err := json.Marshal(string(b[:n]))
if err != nil {
t.Fatal("marshal data failed:",err)
}
var su string
if err := json.Unmarshal(sm, &su); err != nil {
t.Fatal("unmarshal data failed:",err)
}
token, err := GetFromString(`{"str":`+string(sm)+`}`, "str")
if err != nil {
t.Fatal("search data failed:",err)
}
if token.Interface().(string) != su {
t.Fatalf("string mismatch, exp: %v, got: %v", su, token.Interface())
}
}
}
func TestEmoji(t *testing.T) {
var input = []byte(`{"utf8":"Example emoji, KO: \ud83d\udd13, \ud83c\udfc3 ` +
`OK: \u2764\ufe0f "}`)
value, err := Get(input, "utf8")
if err != nil {
t.Fatal(err)
}
var v map[string]interface{}
if err := json.Unmarshal(input, &v); err != nil {
t.Fatal(err)
}
s, _ := v["utf8"].(string)
if value.String() != s {
t.Fatalf("expected '%v', got '%v'", s, value.String())
}
}
func testEscapePath(t *testing.T, json, expect string, path ...interface{}) {
n, e := Get([]byte(json), path...)
if e != nil {
t.Fatal(e)
}
if n.String() != expect {
t.Fatalf("expected '%v', got '%v'", expect, n.Interface())
}
}
func TestEscapePath(t *testing.T) {
data := `{
"test":{
"*":"valZ",
"*v":"val0",
"keyv*":"val1",
"key*v":"val2",
"keyv?":"val3",
"key?v":"val4",
"keyv.":"val5",
"key.v":"val6",
"keyk*":{"key?":"val7"}
}
}`
testEscapePath(t, data, "valZ", "test", "*")
testEscapePath(t, data, "val0", "test", "*v")
testEscapePath(t, data, "val1", "test", "keyv*")
testEscapePath(t, data, "val2", "test", "key*v")
testEscapePath(t, data, "val3", "test", "keyv?")
testEscapePath(t, data, "val4", "test", "key?v")
testEscapePath(t, data, "val5", "test", "keyv.")
testEscapePath(t, data, "val6", "test", "key.v")
testEscapePath(t, data, "val7", "test", "keyk*", "key?")
}
func TestParseAny(t *testing.T) {
n, e := Parse("100")
assertCond(e == nil)
if n == nil {
panic("n is nil")
}
assertCond(n.Float64() == 100)
n, e = Parse("true")
assertCond(e == nil)
if n == nil {
panic("n is nil")
}
assertCond(n.Bool())
n, e = Parse("false")
assertCond(e == nil)
if n == nil {
panic("n is nil")
}
assertCond(n.Bool() == false)
n, e = Parse("yikes")
assertCond(e != nil)
}
func TestTime(t *testing.T) {
data := []byte(`{
"code": 0,
"msg": "",
"data": {
"sz002024": {
"qfqday": [
[
"2014-01-02",
"8.93",
"9.03",
"9.17",
"8.88",
"621143.00"
],
[
"2014-01-03",
"9.03",
"9.30",
"9.47",
"8.98",
"1624438.00"
]
]
}
}
}`)
var num []string
n, e := Get(data, "data", "sz002024", "qfqday", 0)
if e != nil {
t.Fatal(e)
}
for _, v := range n.Array() {
s := v.(string)
num = append(num, s)
}
if fmt.Sprintf("%v", num) != "[2014-01-02 8.93 9.03 9.17 8.88 621143.00]" {
t.Fatalf("invalid result")
}
}
var exampleJSON = `{
"widget": {
"debug": "on",
"window": {
"title": "Sample Konfabulator Widget",
"name": "main_window",
"width": 500,
"height": 500
},
"image": {
"src": "Images/Sun.png",
"hOffset": 250,
"vOffset": 250,
"alignment": "center"
},
"text": {
"data": "Click Here",
"size": 36,
"style": "bold",
"vOffset": 100,
"alignment": "center",
"onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
}
}
}`
func TestUnmarshalMap(t *testing.T) {
n, err := Parse(exampleJSON)
if err != nil || n == nil {
t.Fatal(err)
}
m1 := n.Map()
var m2 map[string]interface{}
if err := json.Unmarshal([]byte(exampleJSON), &m2); err != nil {
t.Fatal(err)
}
b1, err := json.Marshal(m1)
if err != nil {
t.Fatal(err)
}
b2, err := json.Marshal(m2)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b1, b2) {
t.Fatalf("b1 != b2\n b1: %v\nb2: %v", string(b1), string(b2))
}
}
func GetMany(src2 string, path ...string) (ret []string) {
src := []byte(src2)
for _, p := range path {
pathes := strings.Split(p, ".")
if len(pathes) == 0 {
panic(fmt.Sprintf("invalid path: %v", p))
}
ps := make([]interface{}, 0, len(pathes))
for _, p := range pathes {
ps = append(ps, p)
}
n, e := Get(src, ps...)
if e != nil {
ret = append(ret, "")
continue
}
ret = append(ret, fmt.Sprintf("%v", n.Interface()))
}
return
}
func get(str string, path string) *ast.Node {
src := []byte(str)
pathes := strings.Split(path, ".")
if len(pathes) == 0 {
panic(fmt.Sprintf("invalid path: %v", path))
}
ps := make([]interface{}, 0, len(pathes))
for _, p := range pathes {
ps = append(ps, p)
}
n, e := Get(src, ps...)
if e != nil {
return nil
}
return &n
}
func TestSingleArrayValue(t *testing.T) {
var data = []byte(`{"key": "value","key2":[1,2,3,4,"A"]}`)
array := get(string(data), "key2").Array()
if len(array) != 5 {
t.Fatalf("got '%v', expected '%v'", len(array), 5)
}
_, e := Get(data, "key3")
if e == nil {
t.Fatalf("got '%v', expected '%v'", e, nil)
}
}
var manyJSON = ` {
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"hello":"world"
}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},
"position":{"type":"Point","coordinates":[-115.24,33.09]},
"loves":["world peace"],
"name":{"last":"Anderson","first":"Nancy"},
"age":31,
"x":{"a":"emptya","b":"emptyb"},
"name_last":"Yellow",
"name_first":"Cat",
}`
func TestManyBasic(t *testing.T) {
testMany := func(shouldFallback bool, expect string, paths ...string) {
rs := GetMany(
manyJSON,
paths...,
)
// if len(rs) != len(paths) {
// t.Fatalf("expected %v, got %v", len(paths), len(rs))
// }
var results = "[" + strings.Join(rs, " ") + "]"
if results != expect {
fmt.Printf("%v\n", paths)
t.Fatalf("expected %v, got %v", expect, results)
}
}
testMany(false, "[Point]", "position.type")
testMany(false, `[emptya [world peace] 31]`, "x.a", "loves", "age")
testMany(false, `[[world peace]]`, "loves")
testMany(false, `[map[first:Nancy last:Anderson] Nancy]`, "name",
"name.first")
testMany(true, `[]`, strings.Repeat("a.", 40)+"hello")
res := get(manyJSON, strings.Repeat("a.", 48)+"a")
assertCond(res != nil)
testMany(true, "["+fmt.Sprint(res.Interface())+"]", strings.Repeat("a.", 48)+"a")
// these should fallback
testMany(true, `[Cat Nancy]`, "name_first", "name.first")
testMany(true, `[world]`, strings.Repeat("a.", 70)+"hello")
}
func testMany(t *testing.T, json string, paths, expected []string) {
testManyAny(t, json, paths, expected)
testManyAny(t, json, paths, expected)
}
func testManyAny(t *testing.T, json string, paths, expected []string) {
var result []string
for i := 0; i < 2; i++ {
var which string
if i == 0 {
which = "Get"
result = nil
for j := 0; j < len(expected); j++ {
result = append(result, fmt.Sprintf("%v", get(json, paths[j]).Interface()))
}
} else if i == 1 {
which = "GetMany"
result = GetMany(json, paths...)
}
if result == nil {
panic("result is nil")
}
for j := 0; j < len(expected); j++ {
if result[j] != expected[j] {
t.Fatalf("Using key '%s' for '%s'\nexpected '%v', got '%v'",
paths[j], which, expected[j], result[j])
}
}
}
}
func TestNested(t *testing.T) {
data := `{ "name": "FirstName", "name1": "FirstName1", ` +
`"address": "address1", "addressDetails": "address2", }`
paths := []string{"name", "name1", "address", "addressDetails"}
expected := []string{"FirstName", "FirstName1", "address1", "address2"}
t.Run("SingleMany", func(t *testing.T) {
testMany(t, data, paths,
expected)
})
}
func TestMultiLevelFields(t *testing.T) {
data := `{ "Level1Field1":3,
"Level1Field4":4,
"Level1Field2":{ "Level2Field1":[ "value1", "value2" ],
"Level2Field2":{ "Level3Field1":[ { "key1":"value1" } ] } } }`
paths := []string{"Level1Field1", "Level1Field2.Level2Field1",
"Level1Field2.Level2Field2.Level3Field1", "Level1Field4"}
expected := []string{"3", `[value1 value2]`,
`[map[key1:value1]]`, "4"}
t.Run("SingleMany", func(t *testing.T) {
testMany(t, data, paths,
expected)
})
}
func TestRandomMany(t *testing.T) {
var lstr string
defer func() {
if v := recover(); v != nil {
println("'" + hex.EncodeToString([]byte(lstr)) + "'")
println("'" + lstr + "'")
panic(v)
}
}()
rand.Seed(time.Now().UnixNano())
b := make([]byte, 512)
for i := 0; i < 5000; i++ {
n, err := rand.Read(b[:rand.Int()%len(b)])
if err != nil {
t.Fatal(err)
}
lstr = string(b[:n])
paths := make([]string, rand.Int()%64)
for i := range paths {
var b []byte
n := rand.Int() % 5
for j := 0; j < n; j++ {
if j > 0 {
b = append(b, '.')
}
nn := rand.Int() % 10
for k := 0; k < nn; k++ {
b = append(b, 'a'+byte(rand.Int()%26))
}
}
paths[i] = string(b)
}
GetMany(lstr, paths...)
}
}
func TestGetMany(t *testing.T) {
data := `{"bar": {"id": 99, "mybar": "my mybar" }, "foo": ` +
`{"myfoo": [605]}}`
paths := []string{"foo.myfoo", "bar.id", "bar.mybar", "bar.mybarx"}
expected := []string{"[605]", "99", "my mybar", ""}
results := GetMany(data, paths...)
if len(expected) != len(results) {
t.Fatalf("expected %v, got %v", len(expected), len(results))
}
for i, path := range paths {
if results[i] != expected[i] {
t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i],
results[i], path)
}
}
}
func TestGetMany2(t *testing.T) {
data := `{"bar": {"id": 99, "xyz": "my xyz"}, "foo": {"myfoo": [605]}}`
paths := []string{"foo.myfoo", "bar.id", "bar.xyz", "bar.abc"}
expected := []string{"[605]", "99", "my xyz", ""}
results := GetMany(data, paths...)
if len(expected) != len(results) {
t.Fatalf("expected %v, got %v", len(expected), len(results))
}
for i, path := range paths {
if results[i] != expected[i] {
t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i],
results[i], path)
}
}
}
func TestNullArray(t *testing.T) {
n := get(`{"data":null}`, "data").Interface()
if n != nil {
t.Fatalf("expected '%v', got '%v'", nil, n)
}
n = get(`{}`, "data")
if reflect.DeepEqual(n, nil) {
t.Fatalf("expected '%v', got '%v'", nil, n)
}
n = get(`{"data":[]}`, "data")
if reflect.DeepEqual(n, &ast.Node{}) {
t.Fatalf("expected '%v', got '%v'", nil, n)
}
n = len(get(`{"data":[null]}`, "data").Array())
if n != 1 {
t.Fatalf("expected '%v', got '%v'", 1, n)
}
}
func TestGetMany3(t *testing.T) {
var r []string
data := `{"MarketName":null,"Nounce":6115}`
r = GetMany(data, "Nounce", "Buys", "Sells", "Fills")
if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
t.Fatalf("expected '%v', got '%v'", "[6115]",
strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
}
r = GetMany(data, "Nounce", "Buys", "Sells")
if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
t.Fatalf("expected '%v', got '%v'", "[6115]",
strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
}
r = GetMany(data, "Nounce")
if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
t.Fatalf("expected '%v', got '%v'", "[6115]",
strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
}
}
func TestGetMany4(t *testing.T) {
data := `{"one": {"two": 2, "three": 3}, "four": 4, "five": 5}`
results := GetMany(data, "four", "five", "one.two", "one.six")
expected := []string{"4", "5", "2", ""}
for i, r := range results {
if r != expected[i] {
t.Fatalf("expected %v, got %v", expected[i], r)
}
}
}
Reference in a new issue View git blame Copy permalink