From 6e979df0d37353ab72cecaeb32f9a2d79e4cdaff Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Wed, 7 Sep 2022 19:54:13 +0800 Subject: [PATCH] fix:(decoder) `StreamDecoder` unexpectedly stop while skipping incomplete number (#295) --- decoder/stream.go | 9 ++++--- decoder/stream_test.go | 54 ++++++++++++++++++------------------- issue_test/issue293_test.go | 38 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 issue_test/issue293_test.go diff --git a/decoder/stream.go b/decoder/stream.go index 3a0cb5d..32c11cd 100644 --- a/decoder/stream.go +++ b/decoder/stream.go @@ -25,7 +25,7 @@ import ( ) var ( - defaultBufferSize uint = 1024 + defaultBufferSize uint = 4096 growSliceFactorShift uint = 1 minLeftBufferShift uint = 2 ) @@ -99,7 +99,7 @@ read_more: self.Decoder.Reset(string(buf)) err = self.Decoder.Decode(val) if err != nil { - if repeat && repeatable(err) { + if repeat && self.repeatable(err) { goto read_more } self.err = err @@ -125,8 +125,9 @@ read_more: return err } -func repeatable(err error) bool { - if ee, ok := err.(SyntaxError); ok && ee.Code == types.ERR_EOF { +func (self StreamDecoder) repeatable(err error) bool { + if ee, ok := err.(SyntaxError); ok && + (ee.Code == types.ERR_EOF || (ee.Code == types.ERR_INVALID_CHAR && self.i == len(self.s)-1)) { return true } return false diff --git a/decoder/stream_test.go b/decoder/stream_test.go index 35372ee..d490ad2 100644 --- a/decoder/stream_test.go +++ b/decoder/stream_test.go @@ -30,40 +30,40 @@ import ( ) var ( - _Single_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"} { ` - _Double_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"} {"11111":"` + strings.Repeat("2",1024) + `"}` - _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b",1024) + `"}{ } {"11111":"` + - strings.Repeat("2",1024)+`"} b {}` + _Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} { ` + _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} {"11111":"` + strings.Repeat("2", int(defaultBufferSize)) + `"}` + _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"}{ } {"11111":"` + + strings.Repeat("2", int(defaultBufferSize))+`"} b {}` ) func TestStreamError(t *testing.T) { var qs = []string{ - `{`+strings.Repeat(" ", 1024)+`"`, - `{`+strings.Repeat(" ", 1024)+`"}`, - `{`+strings.Repeat(" ", 1024)+`""}`, - `{`+strings.Repeat(" ", 1024)+`"":}`, - `{`+strings.Repeat(" ", 1024)+`"":]`, - `{`+strings.Repeat(" ", 1024)+`"":1x`, - `{`+strings.Repeat(" ", 1024)+`"":1x}`, - `{`+strings.Repeat(" ", 1024)+`"":1x]`, - `{`+strings.Repeat(" ", 1024)+`"":t`, - `{`+strings.Repeat(" ", 1024)+`"":t}`, - `{`+strings.Repeat(" ", 1024)+`"":true]`, - `{`+strings.Repeat(" ", 1024)+`"":f`, - `{`+strings.Repeat(" ", 1024)+`"":f}`, - `{`+strings.Repeat(" ", 1024)+`"":false]`, - `{`+strings.Repeat(" ", 1024)+`"":n`, - `{`+strings.Repeat(" ", 1024)+`"":n}`, - `{`+strings.Repeat(" ", 1024)+`"":null]`, - `{`+strings.Repeat(" ", 1024)+`"":"`, - `{`+strings.Repeat(" ", 1024)+`"":"a`, - `{`+strings.Repeat(" ", 1024)+`"":"a}`, - `{`+strings.Repeat(" ", 1024)+`"":"a"`, - `{`+strings.Repeat(" ", 1024)+`"":"a"]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`""}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":true]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":false]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":null]`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a}`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"`, + `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"]`, } for i, q := range qs { - var qq = []byte(q[:1024]+strings.Repeat(" ", i*100)+q[1024:]) + var qq = []byte(q[:int(defaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(defaultBufferSize):]) var obj interface{} require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj)) } diff --git a/issue_test/issue293_test.go b/issue_test/issue293_test.go new file mode 100644 index 0000000..eb1ca76 --- /dev/null +++ b/issue_test/issue293_test.go @@ -0,0 +1,38 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package issue_test + +import ( + `encoding/json` + `strings` + `testing` + + `github.com/bytedance/sonic/decoder` +) + +func TestIssue293(t *testing.T) { + left := `{"a":` + var data = left+strings.Repeat(" ", 4096 - len(left)-3) + "33.0}" + sd := decoder.NewStreamDecoder(strings.NewReader(data)) + var v = struct{ + A json.RawMessage + }{} + err := sd.Decode(&v) + if err != nil { + t.Fatal(err) + } +} \ No newline at end of file