fix:(native) StreamDecoder returns error when run out of buffer while skiping value (#267)

* fix:(native) StreamDecoder return error when run out of input buffer while skiping value

* refactor

decoder/stream.go

@@ -94,11 +94,12 @@ read_more:
     }
     first = false
 
-    if len(buf) > 0 {
+    l := len(buf)
+    if l > 0 {
         self.Decoder.Reset(string(buf))
         err = self.Decoder.Decode(val)
         if err != nil {
-            if ee, ok := err.(SyntaxError); repeat && ok && ee.Code == types.ERR_EOF {
+            if repeat && repeatable(err) {
                 goto read_more
             }
             self.err = err
@@ -109,7 +110,7 @@ read_more:
         self.scanp = 0
     }
     
-    if len(buf) > p {
+    if l > p {
         // remain undecoded bytes, so copy them into self.buf
         self.buf = append(self.buf[:0], buf[p:]...)
     } else {
@@ -124,6 +125,13 @@ read_more:
     return err
 }
 
+func repeatable(err error) bool {
+    if ee, ok := err.(SyntaxError); ok && ee.Code == types.ERR_EOF {
+        return true
+    }
+    return false
+}
+
 // InputOffset returns the input stream byte offset of the current decoder position. 
 // The offset gives the location of the end of the most recently returned token and the beginning of the next token.
 func (self *StreamDecoder) InputOffset() int64 {

decoder/stream_test.go

@@ -17,6 +17,7 @@
 package decoder
 
 import (
+    `bytes`
     `encoding/json`
     `io`
     `io/ioutil`
@@ -35,6 +36,39 @@ var (
     strings.Repeat("2",1024)+`"} b {}`
 )
 
+func TestStreamError(t *testing.T) {
+    var qs = []string{
+        `{`+strings.Repeat(" ", 1024)+`"`,
+        `{`+strings.Repeat(" ", 1024)+`"}`,
+        `{`+strings.Repeat(" ", 1024)+`""}`,
+        `{`+strings.Repeat(" ", 1024)+`"":}`,
+        `{`+strings.Repeat(" ", 1024)+`"":]`,
+        `{`+strings.Repeat(" ", 1024)+`"":1x`,
+        `{`+strings.Repeat(" ", 1024)+`"":1x}`,
+        `{`+strings.Repeat(" ", 1024)+`"":1x]`,
+        `{`+strings.Repeat(" ", 1024)+`"":t`,
+        `{`+strings.Repeat(" ", 1024)+`"":t}`,
+        `{`+strings.Repeat(" ", 1024)+`"":true]`,
+        `{`+strings.Repeat(" ", 1024)+`"":f`,
+        `{`+strings.Repeat(" ", 1024)+`"":f}`,
+        `{`+strings.Repeat(" ", 1024)+`"":false]`,
+        `{`+strings.Repeat(" ", 1024)+`"":n`,
+        `{`+strings.Repeat(" ", 1024)+`"":n}`,
+        `{`+strings.Repeat(" ", 1024)+`"":null]`,
+        `{`+strings.Repeat(" ", 1024)+`"":"`,
+        `{`+strings.Repeat(" ", 1024)+`"":"a`,
+        `{`+strings.Repeat(" ", 1024)+`"":"a}`,
+        `{`+strings.Repeat(" ", 1024)+`"":"a"`,
+        `{`+strings.Repeat(" ", 1024)+`"":"a"]`,
+    }
+
+    for i, q := range qs {
+        var qq = []byte(q[:1024]+strings.Repeat(" ", i*100)+q[1024:])
+        var obj interface{}
+        require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj))
+    }
+}
+
 func TestDecodeEmpty(t *testing.T) {
     var str = ``
     var r1 = strings.NewReader(str)

internal/native/avx/native_subr_amd64.go

@@ -9,45 +9,45 @@ package avx
 func __native_entry__() uintptr
 
 var (
-    _subr__f64toa       = __native_entry__() + 570
-    _subr__html_escape  = __native_entry__() + 8834
-    _subr__i64toa       = __native_entry__() + 3653
-    _subr__lspace       = __native_entry__() + 251
+    _subr__f64toa       = __native_entry__() + 630
+    _subr__html_escape  = __native_entry__() + 8581
+    _subr__i64toa       = __native_entry__() + 3642
+    _subr__lspace       = __native_entry__() + 301
     _subr__lzero        = __native_entry__() + 13
-    _subr__quote        = __native_entry__() + 4970
-    _subr__skip_array   = __native_entry__() + 18226
-    _subr__skip_number  = __native_entry__() + 21165
-    _subr__skip_object  = __native_entry__() + 18263
-    _subr__skip_one     = __native_entry__() + 16378
-    _subr__u64toa       = __native_entry__() + 3748
-    _subr__unquote      = __native_entry__() + 6540
-    _subr__validate_one = __native_entry__() + 21282
-    _subr__value        = __native_entry__() + 11437
-    _subr__vnumber      = __native_entry__() + 14392
-    _subr__vsigned      = __native_entry__() + 15822
-    _subr__vstring      = __native_entry__() + 13345
-    _subr__vunsigned    = __native_entry__() + 16102
+    _subr__quote        = __native_entry__() + 4955
+    _subr__skip_array   = __native_entry__() + 17819
+    _subr__skip_number  = __native_entry__() + 20937
+    _subr__skip_object  = __native_entry__() + 17856
+    _subr__skip_one     = __native_entry__() + 16120
+    _subr__u64toa       = __native_entry__() + 3735
+    _subr__unquote      = __native_entry__() + 6426
+    _subr__validate_one = __native_entry__() + 21054
+    _subr__value        = __native_entry__() + 11301
+    _subr__vnumber      = __native_entry__() + 14278
+    _subr__vsigned      = __native_entry__() + 15592
+    _subr__vstring      = __native_entry__() + 13243
+    _subr__vunsigned    = __native_entry__() + 15851
 )
 
 const (
     _stack__f64toa = 120
-    _stack__html_escape = 64
+    _stack__html_escape = 72
     _stack__i64toa = 24
     _stack__lspace = 8
     _stack__lzero = 8
-    _stack__quote = 80
-    _stack__skip_array = 152
-    _stack__skip_number = 88
-    _stack__skip_object = 152
-    _stack__skip_one = 152
+    _stack__quote = 56
+    _stack__skip_array = 144
+    _stack__skip_number = 96
+    _stack__skip_object = 144
+    _stack__skip_one = 144
     _stack__u64toa = 8
-    _stack__unquote = 80
-    _stack__validate_one = 152
-    _stack__value = 424
-    _stack__vnumber = 320
+    _stack__unquote = 88
+    _stack__validate_one = 144
+    _stack__value = 416
+    _stack__vnumber = 312
     _stack__vsigned = 16
-    _stack__vstring = 120
-    _stack__vunsigned = 16
+    _stack__vstring = 128
+    _stack__vunsigned = 8
 )
 
 var (

internal/native/avx2/native_subr_amd64.go

@@ -9,24 +9,24 @@ package avx2
 func __native_entry__() uintptr
 
 var (
-    _subr__f64toa       = __native_entry__() + 825
-    _subr__html_escape  = __native_entry__() + 10493
-    _subr__i64toa       = __native_entry__() + 3908
-    _subr__lspace       = __native_entry__() + 379
+    _subr__f64toa       = __native_entry__() + 903
+    _subr__html_escape  = __native_entry__() + 10249
+    _subr__i64toa       = __native_entry__() + 3915
+    _subr__lspace       = __native_entry__() + 429
     _subr__lzero        = __native_entry__() + 13
-    _subr__quote        = __native_entry__() + 5325
-    _subr__skip_array   = __native_entry__() + 21311
-    _subr__skip_number  = __native_entry__() + 24831
-    _subr__skip_object  = __native_entry__() + 21348
-    _subr__skip_one     = __native_entry__() + 19468
-    _subr__u64toa       = __native_entry__() + 4003
-    _subr__unquote      = __native_entry__() + 7998
-    _subr__validate_one = __native_entry__() + 24948
-    _subr__value        = __native_entry__() + 14390
-    _subr__vnumber      = __native_entry__() + 17482
-    _subr__vsigned      = __native_entry__() + 18912
-    _subr__vstring      = __native_entry__() + 16575
-    _subr__vunsigned    = __native_entry__() + 19192
+    _subr__quote        = __native_entry__() + 5328
+    _subr__skip_array   = __native_entry__() + 21867
+    _subr__skip_number  = __native_entry__() + 25515
+    _subr__skip_object  = __native_entry__() + 21904
+    _subr__skip_one     = __native_entry__() + 19172
+    _subr__u64toa       = __native_entry__() + 4008
+    _subr__unquote      = __native_entry__() + 7794
+    _subr__validate_one = __native_entry__() + 25632
+    _subr__value        = __native_entry__() + 14495
+    _subr__vnumber      = __native_entry__() + 17330
+    _subr__vsigned      = __native_entry__() + 18644
+    _subr__vstring      = __native_entry__() + 16453
+    _subr__vunsigned    = __native_entry__() + 18903
 )
 
 const (
@@ -35,19 +35,19 @@ const (
     _stack__i64toa = 24
     _stack__lspace = 8
     _stack__lzero = 8
-    _stack__quote = 72
-    _stack__skip_array = 160
+    _stack__quote = 56
+    _stack__skip_array = 152
     _stack__skip_number = 96
-    _stack__skip_object = 160
-    _stack__skip_one = 160
+    _stack__skip_object = 152
+    _stack__skip_one = 152
     _stack__u64toa = 8
     _stack__unquote = 72
-    _stack__validate_one = 160
-    _stack__value = 424
-    _stack__vnumber = 320
+    _stack__validate_one = 152
+    _stack__value = 408
+    _stack__vnumber = 312
     _stack__vsigned = 16
     _stack__vstring = 112
-    _stack__vunsigned = 16
+    _stack__vunsigned = 8
 )
 
 var (

issue_test/issue263_test.go

@@ -0,0 +1,49 @@
+package issue_test
+
+import (
+    `bytes`
+    `strings`
+    `testing`
+
+    `github.com/bytedance/sonic/decoder`
+    `github.com/stretchr/testify/require`
+)
+
+type Response struct {
+    Menu Menu `json:"menu"`
+}
+
+type Menu struct {
+    Items []*Item `json:"items"`
+}
+
+type Item struct {
+    ID string `json:"id"`
+}
+
+func (i *Item) UnmarshalJSON(buf []byte) error {    
+	return nil
+}
+
+func TestIssue263(t *testing.T) {
+    q := `{
+		"menu": {
+			"items": [
+				{`+strings.Repeat(" ", 1024)+`}
+			]
+		}
+	}`
+
+    var response Response
+    require.Nil(t, decoder.NewStreamDecoder(bytes.NewReader([]byte(q))).Decode(&response))
+
+    q = `{
+        "menu": {
+            "items": [
+                {"a":"`+strings.Repeat("b", 2048)+`"}
+            ]
+        }
+    }`
+    
+    require.Nil(t, decoder.NewStreamDecoder(bytes.NewReader([]byte(q))).Decode(&response))
+}

native/scanning.c

@@ -1003,6 +1003,9 @@ static inline long fsm_exec(StateMachine *self, const GoString *src, long *p, in
     /* run until no more nested values */
     while (self->sp) {
         ch = advance_ns(src, p);
+        if (ch  == 0) {
+            return -ERR_EOF;
+        }
         vt = self->vt[self->sp - 1];
 
         /* set the start address if any */
@@ -1115,7 +1118,6 @@ static inline long fsm_exec(StateMachine *self, const GoString *src, long *p, in
                 }
                 break;
             }
-            case  0  : return -ERR_EOF;
             default  : return -ERR_INVAL;
         }
     }