XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-23 01:46:44 +08:00
Code Issues Projects Releases Wiki Activity

fix: parser.ExportError() exceeds slice boundary

Browse source
This commit is contained in:
duanyi.aster 2021-07-20 16:47:59 +08:00
parent a5efd7e8b7
commit 6b4022a19f
5 changed files with 79 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
ast/node.go
View file

@ -514,10 +514,9 @@ func (self *Node) loadAllIndex() {
parser, stack := self.getParserAndArrayStack() parser, stack := self.getParserAndArrayStack()
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeArray(stack.v) *self, err = parser.decodeArray(stack.v)
if err != 0 { if err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.noLazy = old parser.noLazy = old
} }
@ -532,10 +531,9 @@ func (self *Node) skipAllIndex() {
parser.skipValue = true parser.skipValue = true
oldl := parser.noLazy oldl := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeArray(stack.v) *self, err = parser.decodeArray(stack.v)
if err != 0 { if err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.skipValue = olds parser.skipValue = olds
parser.noLazy = oldl parser.noLazy = oldl
@ -549,10 +547,9 @@ func (self *Node) loadAllKey() {
parser, stack := self.getParserAndObjectStack() parser, stack := self.getParserAndObjectStack()
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeObject(stack.v) *self, err = parser.decodeObject(stack.v)
if err != 0 { if err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.noLazy = old parser.noLazy = old
} }
@ -567,10 +564,9 @@ func (self *Node) skipAllKey() {
parser.skipValue = true parser.skipValue = true
oldl := parser.noLazy oldl := parser.noLazy
parser.noLazy = true parser.noLazy = true
start := parser.p
*self, err = parser.decodeObject(stack.v) *self, err = parser.decodeObject(stack.v)
if err != 0 { if err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.skipValue = olds parser.skipValue = olds
parser.noLazy = oldl parser.noLazy = oldl
@ -604,11 +600,10 @@ func (self *Node) loadNextNode() *Node {
parser := &stack.parser parser := &stack.parser
sp := parser.p sp := parser.p
ns := len(parser.s) ns := len(parser.s)
start := sp
/* check for EOF */ /* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns { if parser.p = parser.lspace(sp); parser.p >= ns {
panic(parser.ExportError(types.ERR_EOF, start)) panic(parser.ExportError(types.ERR_EOF))
} }
/* check for empty array */ /* check for empty array */
@ -624,9 +619,8 @@ func (self *Node) loadNextNode() *Node {
/* decode the value */ /* decode the value */
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start = parser.p
if val, err = parser.Parse(); err != 0 { if val, err = parser.Parse(); err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.noLazy = old parser.noLazy = old
@ -636,7 +630,7 @@ func (self *Node) loadNextNode() *Node {
/* check for EOF */ /* check for EOF */
if parser.p >= ns { if parser.p >= ns {
panic(parser.ExportError(types.ERR_EOF, start)) panic(parser.ExportError(types.ERR_EOF))
} }
/* check for the next character */ /* check for the next character */
@ -650,7 +644,7 @@ func (self *Node) loadNextNode() *Node {
self.setArray(ret) self.setArray(ret)
return &ret[len(ret)-1] return &ret[len(ret)-1]
default: default:
panic(parser.ExportError(types.ERR_INVALID_CHAR, start)) panic(parser.ExportError(types.ERR_INVALID_CHAR))
} }
} }
@ -682,11 +676,10 @@ func (self *Node) loadNextPair() (*Pair) {
parser := &stack.parser parser := &stack.parser
sp := parser.p sp := parser.p
ns := len(parser.s) ns := len(parser.s)
start := sp
/* check for EOF */ /* check for EOF */
if parser.p = parser.lspace(sp); parser.p >= ns { if parser.p = parser.lspace(sp); parser.p >= ns {
panic(parser.ExportError(types.ERR_EOF, start)) panic(parser.ExportError(types.ERR_EOF))
} }
/* check for empty object */ /* check for empty object */
@ -702,9 +695,8 @@ func (self *Node) loadNextPair() (*Pair) {
var err types.ParsingError var err types.ParsingError
/* decode the key */ /* decode the key */
start = parser.p
if njs = parser.decodeValue(); njs.Vt != types.V_STRING { if njs = parser.decodeValue(); njs.Vt != types.V_STRING {
panic(parser.ExportError(types.ERR_INVALID_CHAR, start)) panic(parser.ExportError(types.ERR_INVALID_CHAR))
} }
/* extract the key */ /* extract the key */
@ -714,22 +706,20 @@ func (self *Node) loadNextPair() (*Pair) {
/* check for escape sequence */ /* check for escape sequence */
if njs.Ep != -1 { if njs.Ep != -1 {
if key, err = unquote.String(key); err != 0 { if key, err = unquote.String(key); err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
} }
/* expect a ':' delimiter */ /* expect a ':' delimiter */
start = parser.p
if err = parser.delim(); err != 0 { if err = parser.delim(); err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
/* decode the value */ /* decode the value */
old := parser.noLazy old := parser.noLazy
parser.noLazy = true parser.noLazy = true
start = parser.p
if val, err = parser.Parse(); err != 0 { if val, err = parser.Parse(); err != 0 {
panic(parser.ExportError(err, start)) panic(parser.ExportError(err))
} }
parser.noLazy = old parser.noLazy = old
@ -739,7 +729,7 @@ func (self *Node) loadNextPair() (*Pair) {
/* check for EOF */ /* check for EOF */
if parser.p >= ns { if parser.p >= ns {
panic(parser.ExportError(types.ERR_EOF, start)) panic(parser.ExportError(types.ERR_EOF))
} }
/* check for the next character */ /* check for the next character */
@ -753,7 +743,7 @@ func (self *Node) loadNextPair() (*Pair) {
self.setObject(ret) self.setObject(ret)
return &ret[len(ret)-1] return &ret[len(ret)-1]
default: default:
panic(parser.ExportError(types.ERR_INVALID_CHAR, start)) panic(parser.ExportError(types.ERR_INVALID_CHAR))
} }
} }
@ -1015,7 +1005,7 @@ func (self *Node) parseRaw() Node {
parser := NewParser(raw) parser := NewParser(raw)
n, e := parser.Parse() n, e := parser.Parse()
if e != 0 { if e != 0 {
panic(parser.ExportError(e, 0)) panic(parser.ExportError(e))
} }
return n return n
} }

42
ast/parser.go
View file

@ -17,14 +17,15 @@
package ast package ast
import ( import (
`fmt` `fmt`
`sync` `sync`
`unsafe` `unsafe`
`github.com/bytedance/sonic/internal/native` `github.com/bytedance/sonic/decoder`
`github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/rt` `github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/unquote` `github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/unquote`
) )
const _DEFAULT_NODE_CAP int = 16 const _DEFAULT_NODE_CAP int = 16
@ -359,20 +360,27 @@ func NewParser(src string) *Parser {
} }
// ExportError converts types.ParsingError to std Error // ExportError converts types.ParsingError to std Error
func (self *Parser) ExportError(err types.ParsingError, start int) error { func (self *Parser) ExportError(err types.ParsingError) error {
if err == _ERR_NOT_FOUND { if err == _ERR_NOT_FOUND {
return fmt.Errorf("node not exists") return fmt.Errorf("node not exists")
} }
return fmt.Errorf("%v at %d, near '%s'", err, self.p, self.printNear(start)) return fmt.Errorf("%q", decoder.SyntaxError{
Pos: self.p,
Src: self.s,
Code: err,
}.Description())
} }
func (self *Parser) printNear(start int) string { // func (self *Parser) printNear(start int) string {
if start < 0 { // end := self.p + 10
start = 0 // if end > len(self.s) {
} // end = len(self.s)
end := self.p + 10 // }
if end > len(self.s) { // if start > end {
end = len(self.s) // start = end - 1
} // }
return self.s[start:end] // if start < 0 {
} // start = 0
// }
// return self.s[start:end]
// }

14
ast/search.go
View file

@ -43,32 +43,30 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) {
for _, p := range path { for _, p := range path {
switch p.(type) { switch p.(type) {
case int: case int:
start := self.parser.p
if err = self.parser.searchIndex(p.(int)); err != 0 { if err = self.parser.searchIndex(p.(int)); err != 0 {
return Node{}, self.parser.ExportError(err, start) return Node{}, self.parser.ExportError(err)
} }
case string: case string:
start := self.parser.p
if err = self.parser.searchKey(p.(string)); err != 0 { if err = self.parser.searchKey(p.(string)); err != 0 {
return Node{}, self.parser.ExportError(err, start) return Node{}, self.parser.ExportError(err)
} }
default: default:
panic("path must be either int or string") panic("path must be either int or string")
} }
} }
var start int var start = self.parser.p
if start, err = self.parser.skip(); err != 0 { if start, err = self.parser.skip(); err != 0 {
return Node{}, self.parser.ExportError(err, start) return Node{}, self.parser.ExportError(err)
} }
ns := len(self.parser.s) ns := len(self.parser.s)
if self.parser.p > ns || start >= ns { if self.parser.p > ns || start >= ns || start>=self.parser.p {
return Node{}, fmt.Errorf("skip %d char out of json boundary", start) return Node{}, fmt.Errorf("skip %d char out of json boundary", start)
} }
t := switchRawType(self.parser.s[start]) t := switchRawType(self.parser.s[start])
if t == _V_NONE { if t == _V_NONE {
return Node{}, self.parser.ExportError(err, start) return Node{}, self.parser.ExportError(err)
} }
return newRawNode(self.parser.s[start:self.parser.p], t), nil return newRawNode(self.parser.s[start:self.parser.p], t), nil

32
ast/search_test.go
View file

@ -24,6 +24,38 @@ import (
`github.com/tidwall/gjson` `github.com/tidwall/gjson`
) )
func TestExportError(t *testing.T) {
data := `{"a":]`
p := NewSearcher(data)
_, err := p.GetByPath("a")
if err == nil {
t.Fatal()
}
if err.Error() != `"Syntax error at index 6: invalid char\n\n\t{\"a\":]\n\t......^\n"` {
t.Fatal(err)
}
data = `:"b"]`
p = NewSearcher(data)
_, err = p.GetByPath("a")
if err == nil {
t.Fatal()
}
if err.Error() != `"Syntax error at index 0: invalid char\n\n\t:\"b\"]\n\t^....\n"` {
t.Fatal(err)
}
data = `{:"b"]`
p = NewSearcher(data)
_, err = p.GetByPath("a")
if err == nil {
t.Fatal()
}
if err.Error() != `"Syntax error at index 1: invalid char\n\n\t{:\"b\"]\n\t.^....\n"` {
t.Fatal(err)
}
}
func TestSearcher_GetByPath(t *testing.T) { func TestSearcher_GetByPath(t *testing.T) {
s := NewSearcher(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `) s := NewSearcher(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)

21
decoder/generic_test.go
View file

@ -21,7 +21,6 @@ import (
`reflect` `reflect`
`testing` `testing`
`github.com/bytedance/sonic/ast`
`github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/native/types`
`github.com/davecgh/go-spew/spew` `github.com/davecgh/go-spew/spew`
`github.com/stretchr/testify/assert` `github.com/stretchr/testify/assert`
@ -55,15 +54,6 @@ func TestGeneric_DecodeInterface(t *testing.T) {
fmt.Printf("type: %s\n", reflect.TypeOf(v)) fmt.Printf("type: %s\n", reflect.TypeOf(v))
} }
func BenchmarkGeneric_DecodeAST(b *testing.B) {
_, _, _ = ast.Loads(TwitterJson)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _, _ = ast.Loads(TwitterJson)
}
}
func BenchmarkGeneric_DecodeGeneric(b *testing.B) { func BenchmarkGeneric_DecodeGeneric(b *testing.B) {
t := newStack() t := newStack()
_, _, _ = decodeValue(t, TwitterJson, 0, 0) _, _, _ = decodeValue(t, TwitterJson, 0, 0)
@ -75,17 +65,6 @@ func BenchmarkGeneric_DecodeGeneric(b *testing.B) {
freeStack(t) freeStack(t)
} }
func BenchmarkGeneric_Parallel_DecodeAST(b *testing.B) {
_, _, _ = ast.Loads(TwitterJson)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _, _ = ast.Loads(TwitterJson)
}
})
}
func BenchmarkGeneric_Parallel_DecodeGeneric(b *testing.B) { func BenchmarkGeneric_Parallel_DecodeGeneric(b *testing.B) {
_, _, _ = decodeGeneric(TwitterJson, 0, 0) _, _, _ = decodeGeneric(TwitterJson, 0, 0)
b.SetBytes(int64(len(TwitterJson))) b.SetBytes(int64(len(TwitterJson)))