fix: validate on demand at first for GetByPath and NewRaw (#389)

* feat(ast): validate the demanded fields for ast Get

* fix: add more tests

* fix: check invalid path type

* fix(arm): make return error when skip invalid exponent

* fix(arm): skip object and array with validate

* fix: check input json in NewRaw

* fix: validate the path in native c

* chore: add licenses

* fix: panic when invalid path

* fix: add Get Full json benchmark

* fix(arm): use fast skip besides the ondemand fields

* chore codes

---------

Co-authored-by: liuqiang <liuqiang.06@bytedance.com>

ast/api_amd64.go

@@ -1,5 +1,20 @@
 // +build amd64,go1.15,!go1.21
 
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package ast
 
@@ -100,7 +115,9 @@ func (self *Parser) skipFast() (int, types.ParsingError) {
 }
 
 func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) {
-    start := native.GetByPath(&self.s, &self.p, &path)
+    fsm := types.NewStateMachine()
+    start := native.GetByPath(&self.s, &self.p, &path, fsm)
+    types.FreeStateMachine(fsm)
     runtime.KeepAlive(path)
     if start < 0 {
         return self.p, types.ParsingError(-start)
@@ -108,7 +125,6 @@ func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) {
     return start, 0
 }
 
-
 func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
     var err types.ParsingError
     var start int
@@ -120,6 +136,9 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
         if err == types.ERR_NOT_FOUND {
             return Node{}, ErrNotExist
         }
+        if err == types.ERR_UNSUPPORT_TYPE {
+            panic("path must be either int(>=0) or string")
+        }
         return Node{}, self.parser.syntaxError(err)
     }
 

ast/api_amd64_test.go

@@ -1,5 +1,21 @@
 // +build amd64,go1.15,!go1.21
 
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
 package ast
 
 import (

ast/api_compat.go

@@ -1,5 +1,21 @@
 // +build !amd64 go1.21
 
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package ast
 
 import (
@@ -24,8 +40,6 @@ func unquote(src string) (string, types.ParsingError) {
     return rt.Mem2Str(out), 0
 }
 
-
-
 func decodeBase64(src string) ([]byte, error) {
     return base64.StdEncoding.DecodeString(src)
 }
@@ -53,7 +67,12 @@ func (self *Parser) skip() (int, types.ParsingError) {
 }
 
 func (self *Parser) skipFast() (int, types.ParsingError) {
-    return self.skip()
+    e, s := skipValueFast(self.s, self.p)
+    if e < 0 {
+        return self.p, types.ParsingError(-e)
+    }
+    self.p = e
+    return s, 0
 }
 
 func (self *Node) encodeInterface(buf *[]byte) error {
@@ -70,17 +89,16 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
 
     var err types.ParsingError
     for _, p := range path {
-        switch p := p.(type) {
-        case int:
-            if err = self.parser.searchIndex(p); err != 0 {
+        if idx, ok := p.(int); ok && idx >= 0 {
+            if err = self.parser.searchIndex(idx); err != 0 {
                 return Node{}, self.parser.ExportError(err)
             }
-        case string:
-            if err = self.parser.searchKey(p); err != 0 {
+        } else if key, ok := p.(string); ok {
+            if err = self.parser.searchKey(key); err != 0 {
                 return Node{}, self.parser.ExportError(err)
             }
-        default:
-            panic("path must be either int or string")
+        } else {
+            panic("path must be either int(>=0) or string")
         }
     }
 

ast/api_test.go

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ast
+
+import (
+    `testing`
+    `strings`
+
+    `github.com/stretchr/testify/assert`
+)
+
+type Path = []interface{}
+
+type testGetApi struct {
+    json      string
+    path      Path
+}
+
+type checkError func(error) bool
+
+func isSyntaxError(err error) bool {
+    if err == nil {
+        return false
+    }
+    return strings.HasPrefix(err.Error(), `"Syntax error at index`)
+}
+
+func isEmptySource(err error) bool {
+    if err == nil {
+        return false
+    }
+    return strings.Contains(err.Error(), "no sources available")
+}
+
+func isErrNotExist(err error) bool {
+    return err == ErrNotExist
+}
+
+func isErrUnsupportType(err error) bool {
+    return err == ErrUnsupportType
+}
+
+func testSyntaxJson(t *testing.T, json string, path ...interface{}) {
+    search := NewSearcher(json)
+    _, err := search.GetByPath(path...)
+    assert.True(t, isSyntaxError(err))
+}
+
+func TestGetFromEmptyJson(t *testing.T) {
+    tests := []testGetApi {
+        { "", nil },
+        { "", Path{}},
+        { "", Path{""}},
+        { "", Path{0}},
+        { "", Path{"", ""}},
+    }
+    for _, test := range tests {
+        f := func(t *testing.T) {
+            search := NewSearcher(test.json)
+            _, err := search.GetByPath(test.path...)
+            assert.True(t, isEmptySource(err))
+        }
+        t.Run(test.json, f)
+    }
+}
+
+func TestGetFromSyntaxError(t *testing.T) {
+    tests := []testGetApi {
+        { " \r\n\f\t", Path{} },
+        { "123.", Path{} },
+        { "+124", Path{} },
+        { "-", Path{} },
+        { "-e123", Path{} },
+        { "-1.e123", Path{} },
+        { "-12e456.1", Path{} },
+        { "-12e.1", Path{} },
+        { "[", Path{} },
+        { "{", Path{} },
+        { "[}", Path{} },
+        { "{]", Path{} },
+        { "{,}", Path{} },
+        { "[,]", Path{} },
+        { "tru", Path{} },
+        { "fals", Path{} },
+        { "nul", Path{} },
+        { `{"a":"`, Path{"a"} },
+        { `{"`, Path{} },
+        { `"`, Path{} },
+        { `"\"`, Path{} },
+        { `"\\\"`, Path{} },
+        { `"hello`, Path{} },
+        { `{{}}`, Path{} },
+        { `{[]}`, Path{} },
+        { `{:,}`, Path{} },
+        { `{test:error}`, Path{} },
+        { `{":true}`, Path{} },
+        { `{"" false}`, Path{} },
+        { `{ "" : "false }`, Path{} },
+        { `{"":"",}`, Path{} },
+        { `{ " test : true}`, Path{} },
+        { `{ "test" : tru }`, Path{} },
+        { `{ "test" : true , }`, Path{} },
+        { `{ {"test" : true , } }`, Path{} },
+        { `{"test":1. }`, Path{} },
+        { `{"\\\""`, Path{} },
+        { `{"\\\"":`, Path{} },
+        { `{"\\\":",""}`, Path{} },
+        { `[{]`, Path{} },
+        { `[tru]`, Path{} },
+        { `[-1.]`, Path{} },
+        { `[[]`, Path{} },
+        { `[[],`, Path{} },
+        { `[ true , false , [ ]`, Path{} },
+        { `[true, false, [],`, Path{} },
+        { `[true, false, [],]`, Path{} },
+        { `{"key": [true, false, []], "key2": {{}}`, Path{} },
+    }
+
+    for _, test := range tests {
+        f := func(t *testing.T) {
+            testSyntaxJson(t, test.json, test.path...)
+            path := append(Path{"key"}, test.path...)
+            testSyntaxJson(t, `{"key":` + test.json, path...)
+            path  = append(Path{""}, test.path...)
+            testSyntaxJson(t, `{"":` + test.json, path...)
+            path  = append(Path{1}, test.path...)
+            testSyntaxJson(t, `["",` + test.json, path...)
+        }
+        t.Run(test.json, f)
+    }
+}
+
+// NOTE: GetByPath API not validate the undemanded fields for performance.
+func TestGetWithInvalidUndemandedField(t *testing.T) {
+    type Any = interface{}
+    tests := []struct {
+        json string
+        path Path
+        exp  Any
+    } {
+        { "-0xyz", Path{}, Any(float64(-0))},
+        { "-12e4xyz", Path{}, Any(float64(-12e4))},
+        { "truex",  Path{}, Any(true)},
+        { "false,", Path{}, Any(false)},
+        { `{"a":{,xxx},"b":true}`, Path{"b"}, Any(true)},
+        { `{"a":[,xxx],"b":true}`, Path{"b"}, Any(true)},
+    }
+
+    for _, test := range tests {
+        f := func(t *testing.T) {
+            search := NewSearcher(test.json)
+            node, err := search.GetByPath(test.path...)
+            assert.NoError(t, err)
+            v, err := node.Interface()
+            assert.NoError(t, err)
+            assert.Equal(t, v, test.exp)
+        }
+        t.Run(test.json, f)
+    }
+}
+
+func TestGet_InvalidPathType(t *testing.T) {
+    assert.Panics(t, assert.PanicTestFunc(func() {
+        data := `{"a":[{"b":true}]}`
+        s := NewSearcher(data)
+        s.GetByPath("a", true)
+
+        s = NewSearcher(data)
+        s.GetByPath("a", nil)
+
+        s = NewSearcher(data)
+        s.GetByPath("a", -1)
+    }))
+}

ast/decode.go

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package ast
 
 import (
@@ -286,7 +302,7 @@ func skipNumber(src string, pos int) (ret int) {
         } else if nextNeedDigit {
             return -int(types.ERR_INVALID_CHAR)
         } else if c == '.' {
-            if !lastIsDigit || pointer || sp == ss {
+            if !lastIsDigit || pointer || exponent || sp == ss {
                 return -int(types.ERR_INVALID_CHAR)
             }
             pointer = true
@@ -408,7 +424,7 @@ func skipPair(src string, pos int, lchar byte, rchar byte) (ret int) {
     return int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
 }
 
-func skipValue(src string, pos int) (ret int, start int) {
+func skipValueFast(src string, pos int) (ret int, start int) {
     pos = skipBlank(src, pos)
     if pos < 0 {
         return pos, -1
@@ -433,3 +449,127 @@ func skipValue(src string, pos int) (ret int, start int) {
     }
     return ret, pos
 }
+
+func skipValue(src string, pos int) (ret int, start int) {
+    pos = skipBlank(src, pos)
+    if pos < 0 {
+        return pos, -1
+    }
+    switch c := src[pos]; c {
+    case 'n':
+        ret = decodeNull(src, pos)
+    case '"':
+        ret, _ = skipString(src, pos)
+    case '{':
+        ret, _ = skipObject(src, pos)
+    case '[':
+        ret, _ = skipArray(src, pos)
+    case 't':
+        ret = decodeTrue(src, pos)
+    case 'f':
+        ret = decodeFalse(src, pos)
+    case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+        ret = skipNumber(src, pos)
+    default:
+        ret = -int(types.ERR_INVALID_CHAR)
+    }
+    return ret, pos
+}
+
+func skipObject(src string, pos int) (ret int, start int) {
+    start = skipBlank(src, pos)
+    if start < 0 {
+        return start, -1
+    }
+
+    if src[start] != '{' {
+        return -int(types.ERR_INVALID_CHAR), -1
+    }
+
+    pos = start + 1
+    pos = skipBlank(src, pos)
+    if pos < 0 {
+        return pos, -1
+    }
+    if src[pos] == '}' {
+        return pos + 1, start
+    }
+
+    for {
+        pos, _ = skipString(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+
+        pos = skipBlank(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+        if src[pos] != ':' {
+            return -int(types.ERR_INVALID_CHAR), -1
+        }
+
+        pos++
+        pos, _ = skipValue(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+
+        pos = skipBlank(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+        if src[pos] == '}' {
+            return pos + 1, start
+        }
+        if src[pos] != ',' {
+            return -int(types.ERR_INVALID_CHAR), -1
+        }
+
+        pos++
+        pos = skipBlank(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+
+    }
+}
+
+func skipArray(src string, pos int) (ret int, start int) {
+    start = skipBlank(src, pos)
+    if start < 0 {
+        return start, -1
+    }
+
+    if src[start] != '[' {
+        return -int(types.ERR_INVALID_CHAR), -1
+    }
+
+    pos = start + 1
+    pos = skipBlank(src, pos)
+    if pos < 0 {
+        return pos, -1
+    }
+    if src[pos] == ']' {
+        return pos + 1, start
+    }
+
+    for {
+        pos, _ = skipValue(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+
+        pos = skipBlank(src, pos)
+        if pos < 0 {
+            return pos, -1
+        }
+        if src[pos] == ']' {
+            return pos + 1, start
+        }
+        if src[pos] != ',' {
+            return -int(types.ERR_INVALID_CHAR), -1
+        }
+        pos++
+    }
+}

ast/node.go

@@ -1541,13 +1541,19 @@ var (
     emptyObjectNode = Node{t: types.V_OBJECT}
 )
 
-// NewRaw creates a node of raw json, and decides its type by first char.
+// NewRaw creates a node of raw json.
+// If the input json is invalid, NewRaw returns a error Node.
 func NewRaw(json string) Node {
-    if json == "" {
-        panic("empty json string")
+    parser := NewParser(json)
+    start, err := parser.skip()
+    if err != 0 {
+        return *newError(err, err.Message()) 
     }
-    it := switchRawType(json[0])
-    return newRawNode(json, it)
+    it := switchRawType(parser.s[start])
+    if it == _V_NONE {
+        return Node{}
+    }
+    return newRawNode(parser.s[start:parser.p], it)
 }
 
 // NewAny creates a node of type V_ANY if any's type isn't Node or *Node, 

ast/node_test.go

@@ -272,7 +272,12 @@ func TestTypeCast(t *testing.T) {
         {"ArrayUseNumber", NewArray([]Node{NewNumber("1")}), []interface{}{json.Number("1")}, nil},
         {"Raw", Node{}, "", ErrNotExist},
         {"Raw", NewAny(""), `""`, nil},
+        {"Raw", NewRaw(" "), "", nonEmptyErr},
+        {"Raw", NewRaw(" [ ] "), "[ ]", nil},
         {"Raw", NewRaw("[ ]"), "[ ]", nil},
+        {"Raw", NewRaw(` { "a" : [ true, -1.2e34 ] } `), `{ "a" : [ true, -1.2e34 ] }`, nil},
+        {"Raw", NewRaw(` { "a" : [ true, -1.2e34 ] `), "", nonEmptyErr},
+        {"Raw", NewRaw(` { "a" : [ true, -1.2e34 }`), "", nonEmptyErr},
         {"Raw", NewBool(true), "true", nil},
         {"Raw", NewNumber("-0.0"), "-0.0", nil},
         {"Raw", NewString(""), `""`, nil},
@@ -593,8 +598,8 @@ func TestUnset(t *testing.T) {
         t.Fatalf("decode failed: %v", derr.Error())
     }
     entities := root.GetByPath("statuses", 0, "entities")
-    if !entities.Exists() {
-        t.Fatal()
+    if !entities.Exists() || entities.Check() != nil {
+        t.Fatal(entities.Check().Error())
     }
     exist, err := entities.Unset("urls")
     if !exist || err != nil {

ast/parser.go

@@ -350,7 +350,7 @@ func (self *Parser) searchKey(match string) types.ParsingError {
 
         /* skip value */
         if key != match {
-            if _, err = self.skip(); err != 0 {
+            if _, err = self.skipFast(); err != 0 {
                 return err
             }
         } else {
@@ -398,7 +398,7 @@ func (self *Parser) searchIndex(idx int) types.ParsingError {
     for i := 0; i < idx; i++ {
 
         /* decode the value */
-        if _, err = self.skip(); err != 0 {
+        if _, err = self.skipFast(); err != 0 {
             return err
         }
 

ast/search_test.go

@@ -325,6 +325,19 @@ func BenchmarkGetOne_Sonic(b *testing.B) {
     }
 }
 
+func BenchmarkGetFull_Sonic(b *testing.B) {
+    ast := NewSearcher(_TwitterJson)
+    b.SetBytes(int64(len(_TwitterJson)))
+    b.ReportAllocs()
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        node, err := ast.GetByPath()
+        if err != nil || node.Type() != V_OBJECT {
+            b.Fatal(err)
+        }
+    }
+}
+
 func BenchmarkGetWithManyCompare_Sonic(b *testing.B) {
     b.SetBytes(int64(len(_LotsCompare)))
     ast := NewSearcher(_LotsCompare)

internal/native/avx/native_amd64.go

@@ -122,7 +122,7 @@ func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
 //go:nosplit
 //go:noescape
 //goland:noinspection GoUnusedParameter
-func __get_by_path(s *string, p *int, path *[]interface{}) (ret int)
+func __get_by_path(s *string, p *int, path *[]interface{}, m *types.StateMachine) (ret int)
 
 //go:nosplit
 //go:noescape

internal/native/avx/native_subr_amd64.go

@@ -9,23 +9,23 @@ package avx
 func __native_entry__() uintptr
 
 var (
-    _subr__f32toa             = __native_entry__() + 28672
+    _subr__f32toa             = __native_entry__() + 28800
     _subr__f64toa             = __native_entry__() + 448
-    _subr__get_by_path        = __native_entry__() + 25568
+    _subr__get_by_path        = __native_entry__() + 25664
     _subr__html_escape        = __native_entry__() + 9296
     _subr__i64toa             = __native_entry__() + 3744
     _subr__lspace             = __native_entry__() + 80
     _subr__quote              = __native_entry__() + 5136
     _subr__skip_array         = __native_entry__() + 18592
-    _subr__skip_number        = __native_entry__() + 22144
-    _subr__skip_object        = __native_entry__() + 20592
-    _subr__skip_one           = __native_entry__() + 22304
-    _subr__skip_one_fast      = __native_entry__() + 22544
+    _subr__skip_number        = __native_entry__() + 22224
+    _subr__skip_object        = __native_entry__() + 20640
+    _subr__skip_one           = __native_entry__() + 22384
+    _subr__skip_one_fast      = __native_entry__() + 22624
     _subr__u64toa             = __native_entry__() + 3856
     _subr__unquote            = __native_entry__() + 6928
-    _subr__validate_one       = __native_entry__() + 22368
-    _subr__validate_utf8      = __native_entry__() + 27424
-    _subr__validate_utf8_fast = __native_entry__() + 28096
+    _subr__validate_one       = __native_entry__() + 22448
+    _subr__validate_utf8      = __native_entry__() + 27552
+    _subr__validate_utf8_fast = __native_entry__() + 28224
     _subr__value              = __native_entry__() + 12480
     _subr__vnumber            = __native_entry__() + 16256
     _subr__vsigned            = __native_entry__() + 17872

internal/native/avx2/native_amd64.go

@@ -122,7 +122,7 @@ func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
 //go:nosplit
 //go:noescape
 //goland:noinspection GoUnusedParameter
-func __get_by_path(s *string, p *int, path *[]interface{}) (ret int)
+func __get_by_path(s *string, p *int, path *[]interface{}, m *types.StateMachine) (ret int)
 
 //go:nosplit
 //go:noescape

internal/native/avx2/native_subr_amd64.go

@@ -9,23 +9,23 @@ package avx2
 func __native_entry__() uintptr
 
 var (
-    _subr__f32toa             = __native_entry__() + 34656
+    _subr__f32toa             = __native_entry__() + 34720
     _subr__f64toa             = __native_entry__() + 736
-    _subr__get_by_path        = __native_entry__() + 29232
+    _subr__get_by_path        = __native_entry__() + 29248
     _subr__html_escape        = __native_entry__() + 10944
     _subr__i64toa             = __native_entry__() + 4032
     _subr__lspace             = __native_entry__() + 224
     _subr__quote              = __native_entry__() + 5520
     _subr__skip_array         = __native_entry__() + 21616
     _subr__skip_number        = __native_entry__() + 25840
-    _subr__skip_object        = __native_entry__() + 23680
+    _subr__skip_object        = __native_entry__() + 23648
     _subr__skip_one           = __native_entry__() + 26000
     _subr__skip_one_fast      = __native_entry__() + 26416
     _subr__u64toa             = __native_entry__() + 4144
     _subr__unquote            = __native_entry__() + 8336
     _subr__validate_one       = __native_entry__() + 26064
-    _subr__validate_utf8      = __native_entry__() + 31088
-    _subr__validate_utf8_fast = __native_entry__() + 32048
+    _subr__validate_utf8      = __native_entry__() + 31152
+    _subr__validate_utf8_fast = __native_entry__() + 32112
     _subr__value              = __native_entry__() + 15472
     _subr__vnumber            = __native_entry__() + 19280
     _subr__vsigned            = __native_entry__() + 20896

internal/native/dispatch_amd64.go

@@ -94,7 +94,7 @@ func SkipOneFast(s *string, p *int) int
 //go:nosplit
 //go:noescape
 //goland:noinspection GoUnusedParameter
-func GetByPath(s *string, p *int, path *[]interface{}) int
+func GetByPath(s *string, p *int, path *[]interface{}, m *types.StateMachine) int
 
 //go:nosplit
 //go:noescape

internal/native/dispatch_amd64.s

@@ -72,7 +72,7 @@ TEXT ·SkipOneFast(SB), NOSPLIT, $0 - 24
     JMP  github·com∕bytedance∕sonic∕internal∕native∕avx·__skip_one_fast(SB)
     JMP  github·com∕bytedance∕sonic∕internal∕native∕sse·__skip_one_fast(SB)
 
-TEXT ·GetByPath(SB), NOSPLIT, $0 - 32
+TEXT ·GetByPath(SB), NOSPLIT, $0 - 40
     CMPB github·com∕bytedance∕sonic∕internal∕cpu·HasAVX2(SB), $0
     JE   2(PC)
     JMP  github·com∕bytedance∕sonic∕internal∕native∕avx2·__get_by_path(SB)

internal/native/native_amd64.tmpl

@@ -120,7 +120,7 @@ func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
 //go:nosplit
 //go:noescape
 //goland:noinspection GoUnusedParameter
-func __get_by_path(s *string, p *int, path *[]interface{}) (ret int)
+func __get_by_path(s *string, p *int, path *[]interface{}, m *types.StateMachine) (ret int)
 
 //go:nosplit
 //go:noescape

internal/native/sse/native_amd64.go

@@ -122,7 +122,7 @@ func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
 //go:nosplit
 //go:noescape
 //goland:noinspection GoUnusedParameter
-func __get_by_path(s *string, p *int, path *[]interface{}) (ret int)
+func __get_by_path(s *string, p *int, path *[]interface{}, m *types.StateMachine) (ret int)
 
 //go:nosplit
 //go:noescape

internal/native/sse/native_subr_amd64.go

@@ -9,23 +9,23 @@ package sse
 func __native_entry__() uintptr
 
 var (
-    _subr__f32toa             = __native_entry__() + 29312
+    _subr__f32toa             = __native_entry__() + 29440
     _subr__f64toa             = __native_entry__() + 448
-    _subr__get_by_path        = __native_entry__() + 26208
+    _subr__get_by_path        = __native_entry__() + 26304
     _subr__html_escape        = __native_entry__() + 9360
     _subr__i64toa             = __native_entry__() + 3712
     _subr__lspace             = __native_entry__() + 80
     _subr__quote              = __native_entry__() + 5152
     _subr__skip_array         = __native_entry__() + 18800
-    _subr__skip_number        = __native_entry__() + 22368
-    _subr__skip_object        = __native_entry__() + 20784
-    _subr__skip_one           = __native_entry__() + 22528
-    _subr__skip_one_fast      = __native_entry__() + 22752
+    _subr__skip_number        = __native_entry__() + 22448
+    _subr__skip_object        = __native_entry__() + 20832
+    _subr__skip_one           = __native_entry__() + 22608
+    _subr__skip_one_fast      = __native_entry__() + 22832
     _subr__u64toa             = __native_entry__() + 3840
     _subr__unquote            = __native_entry__() + 6992
-    _subr__validate_one       = __native_entry__() + 22576
-    _subr__validate_utf8      = __native_entry__() + 28080
-    _subr__validate_utf8_fast = __native_entry__() + 28752
+    _subr__validate_one       = __native_entry__() + 22656
+    _subr__validate_utf8      = __native_entry__() + 28208
+    _subr__validate_utf8_fast = __native_entry__() + 28880
     _subr__value              = __native_entry__() + 12560
     _subr__vnumber            = __native_entry__() + 16416
     _subr__vsigned            = __native_entry__() + 18064

native/native.h

@@ -150,5 +150,5 @@ long validate_utf8(const GoString *src, long *p, StateMachine *m);
 long validate_utf8_fast(const GoString *src); 
 
 long skip_one_fast(const GoString *src, long *p);
-long get_by_path(const GoString *src, long *p, const GoSlice *path);
+long get_by_path(const GoString *src, long *p, const GoSlice *path, StateMachine* sm);
 #endif

native/scanning.c

@@ -1128,7 +1128,7 @@ static inline long fsm_exec(StateMachine *self, const GoString *src, long *p, ui
             case 'f' : FSM_XERR(advance_dword(src, p, 0, *p - 1, VS_ALSE)); break;
             case '[' : FSM_XERR(fsm_push(self, FSM_ARR_0));                 break;
             case '{' : FSM_XERR(fsm_push(self, FSM_OBJ_0));                 break;
-            case '"' : FSM_XERR(skip_string(src, p, flags));                       break;
+            case '"' : FSM_XERR(skip_string(src, p, flags));                break;
             case  0  : return -ERR_EOF;
             default  : return -ERR_INVAL;
         }
@@ -1366,18 +1366,19 @@ long skip_object(const GoString *src, long *p, StateMachine *m, uint64_t flags)
 }
 
 long skip_string(const GoString *src, long *p, uint64_t flags) {
-    int64_t v;
-    ssize_t q = *p - 1;
+    int64_t v = -1;
+    ssize_t q = *p - 1; // start position
     ssize_t e = advance_string(src, *p, &v, flags);
 
-    /* check for errors, and update the position */
-    if (e >= 0) {
-        *p = e;
-        return q;
-    } else {
-        *p = v;
+    /* check for errors */
+    if (e < 0) {
+        *p = e == -ERR_EOF ? src->len : v;
         return e;
     }
+
+    /* update the position */
+    *p = e;
+    return q;
 }
 
 long skip_negative(const GoString *src, long *p) {
@@ -1701,11 +1702,11 @@ static always_inline GoKind kind(const GoIface* iface) {
 }
 
 static always_inline bool is_int(const GoIface* iface) {
-    return kind(iface) == Int;
+    return iface->type != NULL && kind(iface) == Int;
 }
 
 static always_inline bool is_str(const GoIface* iface) {
-    return kind(iface) == String;
+    return iface->type != NULL && kind(iface) == String;
 }
 
 static always_inline GoString get_str(const GoIface* iface) {
@@ -1818,7 +1819,7 @@ static always_inline long match_key(const GoString *src, long *p, const GoString
     return sp == end && kp == ke;
 }
 
-long get_by_path(const GoString *src, long *p, const GoSlice *path) {
+long get_by_path(const GoString *src, long *p, const GoSlice *path, StateMachine* sm) {
     GoIface *ps = (GoIface*)(path->buf);
     GoIface *pe = (GoIface*)(path->buf) + path->len;
     char c = 0;
@@ -1826,9 +1827,9 @@ long get_by_path(const GoString *src, long *p, const GoSlice *path) {
     long found;
 
 query:
-    /* if empty path, skip the whole json */
+    /* to be safer for invalid json, use slower skip for the demanded fields */
     if (ps == pe) {
-        return skip_one_fast(src, p);
+        return skip_one(src, p, sm, 0);
     }
 
     /* match type: should query key in object, query index in array */
@@ -1842,9 +1843,15 @@ query:
         if (c != '[') {
             goto err_inval;
         }
+
+        index = get_int(ps);
+        if (index < 0) {
+            goto err_path;
+        }
+
         goto skip_in_arr;
     } else {
-        goto err_inval;
+        goto err_path;
     }
 
 skip_in_obj:
@@ -1884,8 +1891,6 @@ skip_in_obj:
     goto skip_in_obj;
 
 skip_in_arr:
-    index = get_int(ps);
-
     /* check empty array */
     c = advance_ns(src, p);
     if (c == ']') {
@@ -1913,6 +1918,9 @@ not_found:
 err_inval:
     *p -= 1;
     return -ERR_INVAL;
+err_path:
+    *p -= 1;
+    return -ERR_UNSUPPORT_TYPE;
 }
 
 //