XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-23 09:56:44 +08:00
Code Issues Projects Releases Wiki Activity

fix:(decoder) StreamDecoder unexpectedly stop while skipping incomplete number (#295)

Browse source
This commit is contained in:
Yi Duan 2022-09-07 19:54:13 +08:00 committed by GitHub
parent 0a3d980974
commit 6e979df0d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 70 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
decoder/stream.go
View file

@ -25,7 +25,7 @@ import (
) )
var ( var (
defaultBufferSize uint = 1024 defaultBufferSize uint = 4096
growSliceFactorShift uint = 1 growSliceFactorShift uint = 1
minLeftBufferShift uint = 2 minLeftBufferShift uint = 2
) )
@ -99,7 +99,7 @@ read_more:
self.Decoder.Reset(string(buf)) self.Decoder.Reset(string(buf))
err = self.Decoder.Decode(val) err = self.Decoder.Decode(val)
if err != nil { if err != nil {
if repeat && repeatable(err) { if repeat && self.repeatable(err) {
goto read_more goto read_more
} }
self.err = err self.err = err
@ -125,8 +125,9 @@ read_more:
return err return err
} }
func repeatable(err error) bool { func (self StreamDecoder) repeatable(err error) bool {
if ee, ok := err.(SyntaxError); ok && ee.Code == types.ERR_EOF { if ee, ok := err.(SyntaxError); ok &&
(ee.Code == types.ERR_EOF || (ee.Code == types.ERR_INVALID_CHAR && self.i == len(self.s)-1)) {
return true return true
} }
return false return false

54
decoder/stream_test.go
View file

@ -30,40 +30,40 @@ import (
) )
var ( var (
_Single_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"} { ` _Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} { `
_Double_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"} {"11111":"` + strings.Repeat("2",1024) + `"}` _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} {"11111":"` + strings.Repeat("2", int(defaultBufferSize)) + `"}`
_Triple_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"}{ } {"11111":"` + _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"}{ } {"11111":"` +
strings.Repeat("2",1024)+`"} b {}` strings.Repeat("2", int(defaultBufferSize))+`"} b {}`
) )
func TestStreamError(t *testing.T) { func TestStreamError(t *testing.T) {
var qs = []string{ var qs = []string{
`{`+strings.Repeat(" ", 1024)+`"`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"`,
`{`+strings.Repeat(" ", 1024)+`"}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"}`,
`{`+strings.Repeat(" ", 1024)+`""}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`""}`,
`{`+strings.Repeat(" ", 1024)+`"":}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":}`,
`{`+strings.Repeat(" ", 1024)+`"":]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":]`,
`{`+strings.Repeat(" ", 1024)+`"":1x`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x`,
`{`+strings.Repeat(" ", 1024)+`"":1x}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x}`,
`{`+strings.Repeat(" ", 1024)+`"":1x]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x]`,
`{`+strings.Repeat(" ", 1024)+`"":t`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t`,
`{`+strings.Repeat(" ", 1024)+`"":t}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t}`,
`{`+strings.Repeat(" ", 1024)+`"":true]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":true]`,
`{`+strings.Repeat(" ", 1024)+`"":f`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f`,
`{`+strings.Repeat(" ", 1024)+`"":f}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f}`,
`{`+strings.Repeat(" ", 1024)+`"":false]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":false]`,
`{`+strings.Repeat(" ", 1024)+`"":n`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n`,
`{`+strings.Repeat(" ", 1024)+`"":n}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n}`,
`{`+strings.Repeat(" ", 1024)+`"":null]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":null]`,
`{`+strings.Repeat(" ", 1024)+`"":"`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"`,
`{`+strings.Repeat(" ", 1024)+`"":"a`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a`,
`{`+strings.Repeat(" ", 1024)+`"":"a}`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a}`,
`{`+strings.Repeat(" ", 1024)+`"":"a"`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"`,
`{`+strings.Repeat(" ", 1024)+`"":"a"]`, `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"]`,
} }
for i, q := range qs { for i, q := range qs {
var qq = []byte(q[:1024]+strings.Repeat(" ", i*100)+q[1024:]) var qq = []byte(q[:int(defaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(defaultBufferSize):])
var obj interface{} var obj interface{}
require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj)) require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj))
} }

38
issue_test/issue293_test.go Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package issue_test
import (
`encoding/json`
`strings`
`testing`
`github.com/bytedance/sonic/decoder`
)
func TestIssue293(t *testing.T) {
left := `{"a":`
var data = left+strings.Repeat(" ", 4096 - len(left)-3) + "33.0}"
sd := decoder.NewStreamDecoder(strings.NewReader(data))
var v = struct{
A json.RawMessage
}{}
err := sd.Decode(&v)
if err != nil {
t.Fatal(err)
}
}