fix: decoder and encoder support fallback (#430)

decoder/decoder_amd64.go

@@ -0,0 +1,50 @@
+// +build amd64,go1.15,!go1.21
+
+/*
+* Copyright 2023 ByteDance Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package decoder
+
+import (
+    `github.com/bytedance/sonic/internal/decoder`
+)
+
+type Decoder = decoder.Decoder
+
+type MismatchTypeError = decoder.MismatchTypeError
+
+type Options = decoder.Options
+
+const (
+    OptionUseInt64         Options = decoder.OptionUseInt64
+    OptionUseNumber        Options = decoder.OptionUseNumber
+    OptionUseUnicodeErrors Options = decoder.OptionUseUnicodeErrors
+    OptionDisableUnknown   Options = decoder.OptionDisableUnknown
+    OptionCopyString       Options = decoder.OptionCopyString
+    OptionValidateString   Options = decoder.OptionValidateString
+)
+
+type StreamDecoder = decoder.StreamDecoder
+
+type SyntaxError = decoder.SyntaxError
+
+var Pretouch = decoder.Pretouch
+
+var Skip = decoder.Skip
+
+var NewDecoder = decoder.NewDecoder
+
+var NewStreamDecoder = decoder.NewStreamDecoder

decoder/decoder_amd64_test.go

@@ -0,0 +1,276 @@
+// +build amd64,go1.15,!go1.21
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package decoder
+
+import (
+    `encoding/json`
+    `strings`
+    `testing`
+    `reflect`
+
+    `github.com/bytedance/sonic/internal/rt`
+    `github.com/stretchr/testify/assert`
+)
+
+func TestSkipMismatchTypeAmd64Error(t *testing.T) {
+    t.Run("struct", func(t *testing.T) {
+        println("TestSkipError")
+        type skiptype struct {
+            A int `json:"a"`
+            B string `json:"b"`
+
+            Pass *int `json:"pass"`
+
+            C struct{
+
+                Pass4 interface{} `json:"pass4"`
+
+                D struct{
+                    E float32 `json:"e"`
+                } `json:"d"`
+
+                Pass2 int `json:"pass2"`
+
+            } `json:"c"`
+
+            E bool `json:"e"`
+            F []int `json:"f"`
+            G map[string]int `json:"g"`
+            H bool `json:"h,string"`
+
+            Pass3 int `json:"pass2"`
+
+            I json.Number `json:"i"`
+        }
+        var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
+        var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+        if te, ok := err.(*MismatchTypeError); ok {
+            assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
+            assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
+            println(err.Error())
+        } else {
+            t.Fatal("invalid error")
+        }
+    })
+    t.Run("short array", func(t *testing.T) {
+        var obj, obj2 = &[]int{}, &[]int{}
+        var data = `[""]`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        // println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("int ", func(t *testing.T) {
+        var obj int = 123
+        var obj2 int = 123
+        var data = `[""]`
+        d := NewDecoder(data)
+        err := d.Decode(&obj)
+        err2 := json.Unmarshal([]byte(data), &obj2)
+        println(err.Error(), obj, obj2)
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("array", func(t *testing.T) {
+        var obj, obj2 = &[]int{}, &[]int{}
+        var data = `["",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        // println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("map", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { },"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+    t.Run("map error", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { ],"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        println(err.Error())
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        // assert.Equal(t, obj2, obj)
+    })
+}
+
+func TestCopyString(t *testing.T) {
+    var data []byte
+    var dc *Decoder
+    var err error
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    var obj struct{
+        A string
+        B string
+    }
+    err = dc.Decode(&obj)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if obj.A != "0" {
+        t.Fatal(obj)
+    }
+    data[14] = '0'
+    if obj.B != "1" {
+        t.Fatal(obj)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    err = dc.Decode(&obj)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if obj.A != "1" {
+        t.Fatal(obj)
+    }
+    data[14] = '0'
+    if obj.B != "0" {
+        t.Fatal(obj)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    m := map[string]interface{}{}
+    err = dc.Decode(&m)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[2] = 'C'
+    data[6] = '1'
+    if m["A"] != "0" {
+        t.Fatal(m)
+    }
+    data[10] = 'D'
+    data[14] = '0'
+    if m["B"] != "1" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    m = map[string]interface{}{}
+    err = dc.Decode(&m)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if m["A"] != "1" {
+        t.Fatal(m)
+    }
+    data[14] = '0'
+    if m["B"] != "0" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    var x interface{}
+    err = dc.Decode(&x)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[2] = 'C'
+    data[6] = '1'
+    m = x.(map[string]interface{})
+    if m["A"] != "0" {
+        t.Fatal(m)
+    }
+    data[10] = 'D'
+    data[14] = '0'
+    if m["B"] != "1" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    var y interface{}
+    err = dc.Decode(&y)
+    if err != nil {
+        t.Fatal(err)
+    }
+    m = y.(map[string]interface{})
+    data[6] = '1'
+    if m["A"] != "1" {
+        t.Fatal(m)
+    }
+    data[14] = '0'
+    if m["B"] != "0" {
+        t.Fatal(m)
+    }
+}
+
+func TestDecoder_SetOption(t *testing.T) {
+    var v interface{}
+    d := NewDecoder("123")
+    d.SetOptions(OptionUseInt64)
+    err := d.Decode(&v)
+    assert.NoError(t, err)
+    assert.Equal(t, v, int64(123))
+}
+
+func BenchmarkSkip_Sonic(b *testing.B) {
+    var data = rt.Str2Mem(TwitterJson)
+    if ret, _ := Skip(data); ret < 0 {
+        b.Fatal()
+    }
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i:=0; i<b.N; i++ {
+        _, _ = Skip(data)
+    }
+}

decoder/decoder_compat.go

@@ -0,0 +1,196 @@
+// +build !amd64 go1.21
+
+/*
+* Copyright 2023 ByteDance Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package decoder
+
+import (
+     `encoding/json`
+     `bytes`
+     `reflect`
+     `github.com/bytedance/sonic/internal/native/types`
+     `github.com/bytedance/sonic/option`
+     `io`
+)
+
+const (
+     _F_use_int64 = iota
+     _F_use_number
+     _F_disable_urc
+     _F_disable_unknown
+     _F_copy_string
+     _F_validate_string
+
+     _F_allow_control = 31
+)
+
+type Options uint64
+
+const (
+     OptionUseInt64         Options = 1 << _F_use_int64
+     OptionUseNumber        Options = 1 << _F_use_number
+     OptionUseUnicodeErrors Options = 1 << _F_disable_urc
+     OptionDisableUnknown   Options = 1 << _F_disable_unknown
+     OptionCopyString       Options = 1 << _F_copy_string
+     OptionValidateString   Options = 1 << _F_validate_string
+)
+
+func (self *Decoder) SetOptions(opts Options) {
+     if (opts & OptionUseNumber != 0) && (opts & OptionUseInt64 != 0) {
+         panic("can't set OptionUseInt64 and OptionUseNumber both!")
+     }
+     self.f = uint64(opts)
+}
+
+
+// Decoder is the decoder context object
+type Decoder struct {
+     i int
+     f uint64
+     s string
+}
+
+// NewDecoder creates a new decoder instance.
+func NewDecoder(s string) *Decoder {
+     return &Decoder{s: s}
+}
+
+// Pos returns the current decoding position.
+func (self *Decoder) Pos() int {
+     return self.i
+}
+
+func (self *Decoder) Reset(s string) {
+     self.s = s
+     self.i = 0
+     // self.f = 0
+}
+
+// NOTE: api fallback do nothing
+func (self *Decoder) CheckTrailings() error {
+     pos := self.i
+     buf := self.s
+     /* skip all the trailing spaces */
+     if pos != len(buf) {
+         for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 {
+             pos++
+         }
+     }
+
+     /* then it must be at EOF */
+     if pos == len(buf) {
+         return nil
+     }
+
+     /* junk after JSON value */
+     return nil
+}
+
+
+// Decode parses the JSON-encoded data from current position and stores the result
+// in the value pointed to by val.
+func (self *Decoder) Decode(val interface{}) error {
+    r := bytes.NewBufferString(self.s)
+   dec := json.NewDecoder(r)
+   if (self.f | uint64(OptionUseNumber)) != 0  {
+       dec.UseNumber()
+   }
+   if (self.f | uint64(OptionDisableUnknown)) != 0  {
+       dec.DisallowUnknownFields()
+   }
+   return dec.Decode(val)
+}
+
+// UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
+// int64 instead of as a float64.
+func (self *Decoder) UseInt64() {
+     self.f  |= 1 << _F_use_int64
+     self.f &^= 1 << _F_use_number
+}
+
+// UseNumber indicates the Decoder to unmarshal a number into an interface{} as a
+// json.Number instead of as a float64.
+func (self *Decoder) UseNumber() {
+     self.f &^= 1 << _F_use_int64
+     self.f  |= 1 << _F_use_number
+}
+
+// UseUnicodeErrors indicates the Decoder to return an error when encounter invalid
+// UTF-8 escape sequences.
+func (self *Decoder) UseUnicodeErrors() {
+     self.f |= 1 << _F_disable_urc
+}
+
+// DisallowUnknownFields indicates the Decoder to return an error when the destination
+// is a struct and the input contains object keys which do not match any
+// non-ignored, exported fields in the destination.
+func (self *Decoder) DisallowUnknownFields() {
+     self.f |= 1 << _F_disable_unknown
+}
+
+// CopyString indicates the Decoder to decode string values by copying instead of referring.
+func (self *Decoder) CopyString() {
+     self.f |= 1 << _F_copy_string
+}
+
+// ValidateString causes the Decoder to validate string values when decoding string value 
+// in JSON. Validation is that, returning error when unescaped control chars(0x00-0x1f) or
+// invalid UTF-8 chars in the string value of JSON.
+func (self *Decoder) ValidateString() {
+     self.f |= 1 << _F_validate_string
+}
+
+// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
+// order to reduce the first-hit latency.
+//
+// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
+// a compile option to set the depth of recursive compile for the nested struct type.
+func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
+     return nil
+}
+
+type StreamDecoder struct {
+   r       io.Reader
+   buf     []byte
+   scanp   int
+   scanned int64
+   err     error
+   Decoder
+}
+
+// NewStreamDecoder adapts to encoding/json.NewDecoder API.
+//
+// NewStreamDecoder returns a new decoder that reads from r.
+func NewStreamDecoder(r io.Reader) *StreamDecoder {
+   return &StreamDecoder{r : r}
+}
+
+// Decode decodes input stream into val with corresponding data. 
+// Redundantly bytes may be read and left in its buffer, and can be used at next call.
+// Either io error from underlying io.Reader (except io.EOF) 
+// or syntax error from data will be recorded and stop subsequently decoding.
+func (self *StreamDecoder) Decode(val interface{}) (err error) {
+   dec := json.NewDecoder(self.r)
+   if (self.f | uint64(OptionUseNumber)) != 0  {
+       dec.UseNumber()
+   }
+   if (self.f | uint64(OptionDisableUnknown)) != 0  {
+       dec.DisallowUnknownFields()
+   }
+   return dec.Decode(val)
+}
+

decoder/decoder_test.go

@@ -24,9 +24,7 @@ import (
     `sync`
     `testing`
     `time`
-    `reflect`
 
-    `github.com/bytedance/sonic/internal/rt`
     `github.com/davecgh/go-spew/spew`
     `github.com/stretchr/testify/assert`
     `github.com/stretchr/testify/require`
@@ -126,11 +124,7 @@ func TestSkipMismatchTypeError(t *testing.T) {
         assert.Equal(t, err2 == nil, err == nil)
         // assert.Equal(t, len(data), d.i)
         assert.Equal(t, obj2, obj)
-        if te, ok := err.(*MismatchTypeError); ok {
+        if err == nil {
-            assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
-            assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
-            println(err.Error())
-        } else {
             t.Fatal("invalid error")
         }
     })
@@ -237,125 +231,7 @@ func decode(s string, v interface{}, copy bool) (int, error) {
     if err != nil {
         return 0, err
     }
-    return d.i, err
+    return len(s), err
-}
-
-func TestCopyString(t *testing.T) {
-    var data []byte
-    var dc *Decoder
-    var err error
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    dc.CopyString()
-    var obj struct{
-        A string
-        B string
-    }
-    err = dc.Decode(&obj)
-    if err != nil {
-        t.Fatal(err)
-    }
-    data[6] = '1'
-    if obj.A != "0" {
-        t.Fatal(obj)
-    }
-    data[14] = '0'
-    if obj.B != "1" {
-        t.Fatal(obj)
-    }
-
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    err = dc.Decode(&obj)
-    if err != nil {
-        t.Fatal(err)
-    }
-    data[6] = '1'
-    if obj.A != "1" {
-        t.Fatal(obj)
-    }
-    data[14] = '0'
-    if obj.B != "0" {
-        t.Fatal(obj)
-    }
-
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    dc.CopyString()
-    m := map[string]interface{}{}
-    err = dc.Decode(&m)
-    if err != nil {
-        t.Fatal(err)
-    }
-    data[2] = 'C'
-    data[6] = '1'
-    if m["A"] != "0" {
-        t.Fatal(m)
-    }
-    data[10] = 'D'
-    data[14] = '0'
-    if m["B"] != "1" {
-        t.Fatal(m)
-    }
-
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    m = map[string]interface{}{}
-    err = dc.Decode(&m)
-    if err != nil {
-        t.Fatal(err)
-    }
-    data[6] = '1'
-    if m["A"] != "1" {
-        t.Fatal(m)
-    }
-    data[14] = '0'
-    if m["B"] != "0" {
-        t.Fatal(m)
-    }
-
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    dc.CopyString()
-    var x interface{}
-    err = dc.Decode(&x)
-    if err != nil {
-        t.Fatal(err)
-    }
-    data[2] = 'C'
-    data[6] = '1'
-    m = x.(map[string]interface{})
-    if m["A"] != "0" {
-        t.Fatal(m)
-    }
-    data[10] = 'D'
-    data[14] = '0'
-    if m["B"] != "1" {
-        t.Fatal(m)
-    }
-
-    data = []byte(`{"A":"0","B":"1"}`)
-    dc = NewDecoder(rt.Mem2Str(data))
-    dc.UseNumber()
-    var y interface{}
-    err = dc.Decode(&y)
-    if err != nil {
-        t.Fatal(err)
-    }
-    m = y.(map[string]interface{})
-    data[6] = '1'
-    if m["A"] != "1" {
-        t.Fatal(m)
-    }
-    data[14] = '0'
-    if m["B"] != "0" {
-        t.Fatal(m)
-    }
 }
 
 func TestDecoder_Basic(t *testing.T) {
@@ -382,21 +258,10 @@ func TestDecoder_Binding(t *testing.T) {
     spew.Dump(v)
 }
 
-func TestDecoder_SetOption(t *testing.T) {
-    var v interface{}
-    d := NewDecoder("123")
-    d.SetOptions(OptionUseInt64)
-    err := d.Decode(&v)
-    assert.NoError(t, err)
-    assert.Equal(t, v, int64(123))
-}
 
 func TestDecoder_MapWithIndirectElement(t *testing.T) {
     var v map[string]struct { A [129]byte }
     _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false)
-    if x, ok := err.(SyntaxError); ok {
-        println(x.Description())
-    }
     require.NoError(t, err)
     assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A)
 }
@@ -548,15 +413,3 @@ func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {
         }
     })
 }
-
-func BenchmarkSkip_Sonic(b *testing.B) {
-    var data = rt.Str2Mem(TwitterJson)
-    if ret, _ := Skip(data); ret < 0 {
-        b.Fatal()
-    }
-    b.SetBytes(int64(len(TwitterJson)))
-    b.ResetTimer()
-    for i:=0; i<b.N; i++ {
-        _, _ = Skip(data)
-    }
-}

decoder/testdata_test.go

@@ -16,6 +16,16 @@
 
 package decoder
 
+import (
+  `os`
+)
+
+var (
+  debugSyncGC  = os.Getenv("SONIC_SYNC_GC") != ""
+  debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
+)
+
+
 const TwitterJson = `{
   "statuses": [
     {

encoder/encoder_amd64.go

@@ -0,0 +1,72 @@
+// +build amd64,go1.15,!go1.21
+
+/*
+ * Copyright 2023 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `github.com/bytedance/sonic/internal/encoder`
+)
+
+var (
+    Encode = encoder.Encode
+    EncodeIndented = encoder.EncodeIndented
+    EncodeInto = encoder.EncodeInto
+    HTMLEscape = encoder.HTMLEscape
+    Pretouch = encoder.Pretouch
+    Quote = encoder.Quote
+    Valid = encoder.Valid
+)
+
+type Encoder = encoder.Encoder
+
+type Options = encoder.Options
+
+const (
+    // SortMapKeys indicates that the keys of a map needs to be sorted
+    // before serializing into JSON.
+    // WARNING: This hurts performance A LOT, USE WITH CARE.
+    SortMapKeys Options = encoder.SortMapKeys
+
+    // EscapeHTML indicates encoder to escape all HTML characters
+    // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
+    // WARNING: This hurts performance A LOT, USE WITH CARE.
+    EscapeHTML Options = encoder.EscapeHTML
+
+    // CompactMarshaler indicates that the output JSON from json.Marshaler
+    // is always compact and needs no validation
+    CompactMarshaler Options = encoder.CompactMarshaler
+
+    // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
+    // is always escaped string and needs no quoting
+    NoQuoteTextMarshaler Options = encoder.NoQuoteTextMarshaler
+
+    // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
+    // instead of 'null'
+    NoNullSliceOrMap Options = encoder.NoNullSliceOrMap
+
+    // ValidateString indicates that encoder should validate the input string
+    // before encoding it into JSON.
+    ValidateString Options = encoder.ValidateString
+
+    // CompatibleWithStd is used to be compatible with std encoder.
+    CompatibleWithStd Options = encoder.CompatibleWithStd
+)
+
+type StreamEncoder = encoder.StreamEncoder
+
+var NewStreamEncoder = encoder.NewStreamEncoder

encoder/encoder_amd64_test.go

@@ -0,0 +1,120 @@
+// +build amd64,go1.15,!go1.21
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `encoding/json`
+    `testing`
+
+    `github.com/stretchr/testify/require`
+)
+
+func TestOptionSliceOrMapNoNull(t *testing.T) {
+    obj := sample{}
+    out, err := Encode(obj, NoNullSliceOrMap)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+
+    obj2 := sample{}
+    out, err = Encode(obj2, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+}
+
+func TestEncoder_Marshaler(t *testing.T) {
+    v := MarshalerStruct{V: MarshalerImpl{X: 12345}}
+    ret, err := Encode(&v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":12345    }`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":{"X":12345}}`, string(ret))
+
+    ret2, err2 := Encode(&v, 0)
+    require.NoError(t, err2)
+    require.Equal(t, `{"V":12345    }`, string(ret2))
+    ret3, err3 := Encode(v, CompactMarshaler)
+    require.NoError(t, err3)
+    require.Equal(t, `{"V":{"X":12345}}`, string(ret3))
+}
+
+func TestMarshalerError(t *testing.T) {
+    v := MarshalerErrorStruct{}
+    ret, err := Encode(&v, 0)
+    require.EqualError(t, err, `invalid Marshaler output json syntax at 5: "[\"\"] {"`)
+    require.Equal(t, []byte(nil), ret)
+}
+
+
+func TestEncoder_RawMessage(t *testing.T) {
+    rms := RawMessageStruct{
+        X: json.RawMessage("123456    "),
+    }
+    ret, err := Encode(&rms, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"X":123456    }`, string(ret))
+
+    ret, err = Encode(&rms, CompactMarshaler)
+    require.NoError(t, err)
+    require.Equal(t, `{"X":123456}`, string(ret))
+}
+
+
+func TestEncoder_TextMarshaler(t *testing.T) {
+    v := TextMarshalerStruct{V: TextMarshalerImpl{X: (`{"a"}`)}}
+    ret, err := Encode(&v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":"{\"a\"}"}`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret))
+
+    ret2, err2 := Encode(&v, NoQuoteTextMarshaler)
+    require.NoError(t, err2)
+    require.Equal(t, `{"V":{"a"}}`, string(ret2))
+    ret3, err3 := Encode(v, NoQuoteTextMarshaler)
+    require.NoError(t, err3)
+    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret3))
+}
+
+func TestEncoder_Marshal_EscapeHTML(t *testing.T) {
+    v := map[string]TextMarshalerImpl{"&&":{"<>"}}
+    ret, err := Encode(v, EscapeHTML)
+    require.NoError(t, err)
+    require.Equal(t, `{"\u0026\u0026":{"X":"\u003c\u003e"}}`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"&&":{"X":"<>"}}`, string(ret))
+
+    // “ is \xe2\x80\x9c, and ” is \xe2\x80\x9d,
+    // similar as HTML escaped chars \u2028(\xe2\x80\xa8) and \u2029(\xe2\x80\xa9)
+    m := map[string]string{"test": "“123”"}
+    ret, err = Encode(m, EscapeHTML)
+    require.Equal(t, string(ret), `{"test":"“123”"}`)
+    require.NoError(t, err)
+
+    m = map[string]string{"K": "\u2028\u2028\xe2"}
+    ret, err = Encode(m, EscapeHTML)
+    require.Equal(t, string(ret), "{\"K\":\"\\u2028\\u2028\xe2\"}")
+    require.NoError(t, err)
+}

encoder/encoder_compat.go

@@ -0,0 +1,234 @@
+// +build !amd64 go1.21
+
+/*
+* Copyright 2023 ByteDance Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package encoder
+
+import (
+   `io`
+    `bytes`
+    `encoding/json`
+    `reflect`
+
+    `github.com/bytedance/sonic/option`
+)
+
+// Options is a set of encoding options.
+type Options uint64
+
+const (
+    bitSortMapKeys          = iota
+    bitEscapeHTML          
+    bitCompactMarshaler
+    bitNoQuoteTextMarshaler
+    bitNoNullSliceOrMap
+    bitValidateString
+
+    // used for recursive compile
+    bitPointerValue = 63
+)
+
+const (
+    // SortMapKeys indicates that the keys of a map needs to be sorted 
+    // before serializing into JSON.
+    // WARNING: This hurts performance A LOT, USE WITH CARE.
+    SortMapKeys          Options = 1 << bitSortMapKeys
+
+    // EscapeHTML indicates encoder to escape all HTML characters 
+    // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).
+    // WARNING: This hurts performance A LOT, USE WITH CARE.
+    EscapeHTML           Options = 1 << bitEscapeHTML
+
+    // CompactMarshaler indicates that the output JSON from json.Marshaler 
+    // is always compact and needs no validation 
+    CompactMarshaler     Options = 1 << bitCompactMarshaler
+
+    // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler 
+    // is always escaped string and needs no quoting
+    NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
+
+    // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
+    // instead of 'null'
+    NoNullSliceOrMap     Options = 1 << bitNoNullSliceOrMap
+
+    // ValidateString indicates that encoder should validate the input string
+    // before encoding it into JSON.
+    ValidateString       Options = 1 << bitValidateString
+  
+    // CompatibleWithStd is used to be compatible with std encoder.
+    CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
+)
+
+// Encoder represents a specific set of encoder configurations.
+type Encoder struct {
+    Opts Options
+    prefix string
+    indent string
+}
+
+// Encode returns the JSON encoding of v.
+func (self *Encoder) Encode(v interface{}) ([]byte, error) {
+    if self.indent != "" || self.prefix != "" { 
+        return EncodeIndented(v, self.prefix, self.indent, self.Opts)
+    }
+    return Encode(v, self.Opts)
+}
+
+// SortKeys enables the SortMapKeys option.
+func (self *Encoder) SortKeys() *Encoder {
+    self.Opts |= SortMapKeys
+    return self
+}
+
+// SetEscapeHTML specifies if option EscapeHTML opens
+func (self *Encoder) SetEscapeHTML(f bool) {
+    if f {
+        self.Opts |= EscapeHTML
+    } else {
+        self.Opts &= ^EscapeHTML
+    }
+}
+
+// SetValidateString specifies if option ValidateString opens
+func (self *Encoder) SetValidateString(f bool) {
+    if f {
+        self.Opts |= ValidateString
+    } else {
+        self.Opts &= ^ValidateString
+    }
+}
+
+// SetCompactMarshaler specifies if option CompactMarshaler opens
+func (self *Encoder) SetCompactMarshaler(f bool) {
+    if f {
+        self.Opts |= CompactMarshaler
+    } else {
+        self.Opts &= ^CompactMarshaler
+    }
+}
+
+// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens
+func (self *Encoder) SetNoQuoteTextMarshaler(f bool) {
+    if f {
+        self.Opts |= NoQuoteTextMarshaler
+    } else {
+        self.Opts &= ^NoQuoteTextMarshaler
+    }
+}
+
+// SetIndent instructs the encoder to format each subsequent encoded
+// value as if indented by the package-level function EncodeIndent().
+// Calling SetIndent("", "") disables indentation.
+func (enc *Encoder) SetIndent(prefix, indent string) {
+    enc.prefix = prefix
+    enc.indent = indent
+}
+
+// Quote returns the JSON-quoted version of s.
+func Quote(s string) string {
+    /* check for empty string */
+    if s == "" {
+        return `""`
+    }
+
+    out, _ := json.Marshal(s)
+    return string(out)
+}
+
+// Encode returns the JSON encoding of val, encoded with opts.
+func Encode(val interface{}, opts Options) ([]byte, error) {
+   return json.Marshal(val)
+}
+
+// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating
+// a new one.
+func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
+   if buf == nil {
+       panic("user-supplied buffer buf is nil")
+   }
+   w := bytes.NewBuffer(*buf)
+   enc := json.NewEncoder(w)
+   enc.SetEscapeHTML((opts & EscapeHTML) != 0)
+   err := enc.Encode(val)
+   *buf = w.Bytes()
+   return err
+}
+
+// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029
+// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029
+// so that the JSON will be safe to embed inside HTML <script> tags.
+// For historical reasons, web browsers don't honor standard HTML
+// escaping within <script> tags, so an alternative JSON encoding must
+// be used.
+func HTMLEscape(dst []byte, src []byte) []byte {
+   d := bytes.NewBuffer(dst)
+   json.HTMLEscape(d, src)
+   return d.Bytes()
+}
+
+// EncodeIndented is like Encode but applies Indent to format the output.
+// Each JSON element in the output will begin on a new line beginning with prefix
+// followed by one or more copies of indent according to the indentation nesting.
+func EncodeIndented(val interface{}, prefix string, indent string, opts Options) ([]byte, error) {
+   w := bytes.NewBuffer([]byte{})
+   enc := json.NewEncoder(w)
+   enc.SetEscapeHTML((opts & EscapeHTML) != 0)
+   enc.SetIndent(prefix, indent)
+   err := enc.Encode(val)
+   out := w.Bytes()
+   return out, err
+}
+
+// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
+// order to reduce the first-hit latency.
+//
+// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
+// a compile option to set the depth of recursive compile for the nested struct type.
+func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
+   return nil
+}
+
+// Valid validates json and returns first non-blank character position,
+// if it is only one valid json value.
+// Otherwise returns invalid character position using start.
+//
+// Note: it does not check for the invalid UTF-8 characters.
+func Valid(data []byte) (ok bool, start int) {
+   return json.Valid(data), 0
+}
+
+// StreamEncoder uses io.Writer as 
+type StreamEncoder struct {
+   w io.Writer
+   Encoder
+}
+
+// NewStreamEncoder adapts to encoding/json.NewDecoder API.
+//
+// NewStreamEncoder returns a new encoder that write to w.
+func NewStreamEncoder(w io.Writer) *StreamEncoder {
+   return &StreamEncoder{w: w}
+}
+
+// Encode encodes interface{} as JSON to io.Writer
+func (enc *StreamEncoder) Encode(val interface{}) (err error) {
+   jenc := json.NewEncoder(enc.w)
+   jenc.SetEscapeHTML((enc.Opts & EscapeHTML) != 0)
+   jenc.SetIndent(enc.prefix, enc.indent)
+   err = jenc.Encode(val)
+   return err
+}

encoder/encoder_test.go

@@ -84,22 +84,6 @@ type sample struct {
     AP *[0]interface{}
 }
 
-func TestOptionSliceOrMapNoNull(t *testing.T) {
-    obj := sample{}
-    out, err := Encode(obj, NoNullSliceOrMap)
-    if err != nil {
-        t.Fatal(err)
-    }
-    require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
-
-    obj2 := sample{}
-    out, err = Encode(obj2, 0)
-    if err != nil {
-        t.Fatal(err)
-    }
-    require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
-}
-
 func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
     b.Run("true", func (b *testing.B) {
         obj := sample{}
@@ -162,7 +146,6 @@ func TestEncodeErrorAndScratchBuf(t *testing.T) {
     buf := make([]byte, 0, 10)
     _ = EncodeInto(&buf, obj, 0)
     if len(buf) < 0 || len(buf) > 10 {
-        println(buf)
         t.Fatal()
     }
 }
@@ -180,23 +163,6 @@ type MarshalerStruct struct {
     V MarshalerImpl
 }
 
-func TestEncoder_Marshaler(t *testing.T) {
-    v := MarshalerStruct{V: MarshalerImpl{X: 12345}}
-    ret, err := Encode(&v, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"V":12345    }`, string(ret))
-    ret, err = Encode(v, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"V":{"X":12345}}`, string(ret))
-
-    ret2, err2 := Encode(&v, 0)
-    require.NoError(t, err2)
-    require.Equal(t, `{"V":12345    }`, string(ret2))
-    ret3, err3 := Encode(v, CompactMarshaler)
-    require.NoError(t, err3)
-    require.Equal(t, `{"V":{"X":12345}}`, string(ret3))
-}
-
 type MarshalerErrorStruct struct {
     V MarshalerImpl
 }
@@ -205,30 +171,10 @@ func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) {
     return []byte(`[""] {`), nil
 }
 
-func TestMarshalerError(t *testing.T) {
-    v := MarshalerErrorStruct{}
-    ret, err := Encode(&v, 0)
-    require.EqualError(t, err, `invalid Marshaler output json syntax at 5: "[\"\"] {"`)
-    require.Equal(t, []byte(nil), ret)
-}
-
 type RawMessageStruct struct {
     X json.RawMessage
 }
 
-func TestEncoder_RawMessage(t *testing.T) {
-    rms := RawMessageStruct{
-        X: json.RawMessage("123456    "),
-    }
-    ret, err := Encode(&rms, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"X":123456    }`, string(ret))
-
-    ret, err = Encode(&rms, CompactMarshaler)
-    require.NoError(t, err)
-    require.Equal(t, `{"X":123456}`, string(ret))
-}
-
 type TextMarshalerImpl struct {
     X string
 }
@@ -249,23 +195,6 @@ type TextMarshalerStruct struct {
     V TextMarshalerImpl
 }
 
-func TestEncoder_TextMarshaler(t *testing.T) {
-    v := TextMarshalerStruct{V: TextMarshalerImpl{X: (`{"a"}`)}}
-    ret, err := Encode(&v, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"V":"{\"a\"}"}`, string(ret))
-    ret, err = Encode(v, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret))
-
-    ret2, err2 := Encode(&v, NoQuoteTextMarshaler)
-    require.NoError(t, err2)
-    require.Equal(t, `{"V":{"a"}}`, string(ret2))
-    ret3, err3 := Encode(v, NoQuoteTextMarshaler)
-    require.NoError(t, err3)
-    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret3))
-}
-
 func TestTextMarshalTextKey_SortKeys(t *testing.T) {
     v := map[*TextMarshalerImpl]string{
         {"b"}: "b",
@@ -295,28 +224,6 @@ func TestTextMarshalTextKey_SortKeys(t *testing.T) {
     require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
 }
 
-func TestEncoder_Marshal_EscapeHTML(t *testing.T) {
-    v := map[string]TextMarshalerImpl{"&&":{"<>"}}
-    ret, err := Encode(v, EscapeHTML)
-    require.NoError(t, err)
-    require.Equal(t, `{"\u0026\u0026":{"X":"\u003c\u003e"}}`, string(ret))
-    ret, err = Encode(v, 0)
-    require.NoError(t, err)
-    require.Equal(t, `{"&&":{"X":"<>"}}`, string(ret))
-
-    // “ is \xe2\x80\x9c, and ” is \xe2\x80\x9d,
-    // similar as HTML escaped chars \u2028(\xe2\x80\xa8) and \u2029(\xe2\x80\xa9)
-    m := map[string]string{"test": "“123”"}
-    ret, err = Encode(m, EscapeHTML)
-    require.Equal(t, string(ret), `{"test":"“123”"}`)
-    require.NoError(t, err)
-
-    m = map[string]string{"K": "\u2028\u2028\xe2"}
-    ret, err = Encode(m, EscapeHTML)
-    require.Equal(t, string(ret), "{\"K\":\"\\u2028\\u2028\xe2\"}")
-    require.NoError(t, err)
-}
-
 func TestEncoder_EscapeHTML(t *testing.T) {
     // test data from libfuzzer
     test := []string{

encoder/testdata_test.go

@@ -16,6 +16,15 @@
 
 package encoder
 
+import (
+  `os`
+)
+
+var (
+  debugSyncGC  = os.Getenv("SONIC_SYNC_GC") != ""
+  debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
+)
+
 const TwitterJson = `{
   "statuses": [
     {

internal/decoder/decoder_test.go

@@ -0,0 +1,562 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package decoder
+
+import (
+    `encoding/json`
+    `runtime`
+    `runtime/debug`
+    `strings`
+    `sync`
+    `testing`
+    `time`
+    `reflect`
+
+    `github.com/bytedance/sonic/internal/rt`
+    `github.com/davecgh/go-spew/spew`
+    `github.com/stretchr/testify/assert`
+    `github.com/stretchr/testify/require`
+)
+
+func TestMain(m *testing.M) {
+    go func ()  {
+        if !debugAsyncGC {
+            return
+        }
+        println("Begin GC looping...")
+        for {
+           runtime.GC()
+           debug.FreeOSMemory() 
+        }
+        println("stop GC looping!")
+    }()
+    time.Sleep(time.Millisecond)
+    m.Run()
+}
+
+func TestGC(t *testing.T) {
+    if debugSyncGC {
+        return 
+    }
+    var w interface{}
+    out, err := decode(TwitterJson, &w, true)
+    if err != nil {
+        t.Fatal(err)
+    }
+    if out != len(TwitterJson) {
+        t.Fatal(out)
+    }
+    wg := &sync.WaitGroup{}
+    N := 10000
+    for i:=0; i<N; i++ {
+        wg.Add(1)
+        go func (wg *sync.WaitGroup)  {
+            defer wg.Done()
+            var w interface{}
+            out, err := decode(TwitterJson, &w, true)
+            if err != nil {
+                t.Fatal(err)
+            }
+            if out != len(TwitterJson) {
+                t.Fatal(out)
+            }
+            runtime.GC()
+        }(wg)
+    }
+    wg.Wait()
+}
+
+var _BindingValue TwitterStruct
+
+func init() {
+    _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
+}
+
+
+func TestSkipMismatchTypeError(t *testing.T) {
+    t.Run("struct", func(t *testing.T) {
+        println("TestSkipError")
+        type skiptype struct {
+            A int `json:"a"`
+            B string `json:"b"`
+
+            Pass *int `json:"pass"`
+
+            C struct{
+
+                Pass4 interface{} `json:"pass4"`
+
+                D struct{
+                    E float32 `json:"e"`
+                } `json:"d"`
+
+                Pass2 int `json:"pass2"`
+
+            } `json:"c"`
+
+            E bool `json:"e"`
+            F []int `json:"f"`
+            G map[string]int `json:"g"`
+            H bool `json:"h,string"`
+
+            Pass3 int `json:"pass2"`
+
+            I json.Number `json:"i"`
+        }
+        var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
+        var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+        if te, ok := err.(*MismatchTypeError); ok {
+            assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
+            assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
+            println(err.Error())
+        } else {
+            t.Fatal("invalid error")
+        }
+    })
+    t.Run("short array", func(t *testing.T) {
+        var obj, obj2 = &[]int{}, &[]int{}
+        var data = `[""]`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        // println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("int ", func(t *testing.T) {
+        var obj int = 123
+        var obj2 int = 123
+        var data = `[""]`
+        d := NewDecoder(data)
+        err := d.Decode(&obj)
+        err2 := json.Unmarshal([]byte(data), &obj2)
+        println(err.Error(), obj, obj2)
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("array", func(t *testing.T) {
+        var obj, obj2 = &[]int{}, &[]int{}
+        var data = `["",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        // println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+
+    t.Run("map", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { },"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        assert.Equal(t, obj2, obj)
+    })
+    t.Run("map error", func(t *testing.T) {
+        var obj, obj2 = &map[int]int{}, &map[int]int{}
+        var data = `{"true" : { ],"1":1,"2" : true,"3":3}`
+        d := NewDecoder(data)
+        err := d.Decode(obj)
+        err2 := json.Unmarshal([]byte(data), obj2)
+        println(err.Error())
+        println(err2.Error())
+        assert.Equal(t, err2 == nil, err == nil)
+        // assert.Equal(t, len(data), d.i)
+        // assert.Equal(t, obj2, obj)
+    })
+}
+
+func TestDecodeCorrupt(t *testing.T) {
+    var ds = []string{
+        `{,}`,
+        `{,"a"}`,
+        `{"a":}`,
+        `{"a":1,}`,
+        `{"a":1,"b"}`,
+        `{"a":1,"b":}`,
+        `{,"a":1 "b":2}`,
+        `{"a",:1 "b":2}`,
+        `{"a":,1 "b":2}`,
+        `{"a":1 "b",:2}`,
+        `{"a":1 "b":,2}`,
+        `{"a":1 "b":2,}`,
+        `{"a":1 "b":2}`,
+        `[,]`,
+        `[,1]`,
+        `[1,]`,
+        `[,1,2]`,
+        `[1,2,]`,
+    }
+    for _, d := range ds {
+        var o interface{}
+        _, err := decode(d, &o, false)
+        if err == nil {
+            t.Fatalf("%#v", d)
+        }
+        if !strings.Contains(err.Error(), "invalid char"){
+            t.Fatal(err.Error())
+        }
+    }
+}
+
+func decode(s string, v interface{}, copy bool) (int, error) {
+    d := NewDecoder(s)
+    if copy {
+        d.CopyString()
+    }
+    err := d.Decode(v)
+    if err != nil {
+        return 0, err
+    }
+    return d.i, err
+}
+
+func TestCopyString(t *testing.T) {
+    var data []byte
+    var dc *Decoder
+    var err error
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    var obj struct{
+        A string
+        B string
+    }
+    err = dc.Decode(&obj)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if obj.A != "0" {
+        t.Fatal(obj)
+    }
+    data[14] = '0'
+    if obj.B != "1" {
+        t.Fatal(obj)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    err = dc.Decode(&obj)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if obj.A != "1" {
+        t.Fatal(obj)
+    }
+    data[14] = '0'
+    if obj.B != "0" {
+        t.Fatal(obj)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    m := map[string]interface{}{}
+    err = dc.Decode(&m)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[2] = 'C'
+    data[6] = '1'
+    if m["A"] != "0" {
+        t.Fatal(m)
+    }
+    data[10] = 'D'
+    data[14] = '0'
+    if m["B"] != "1" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    m = map[string]interface{}{}
+    err = dc.Decode(&m)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[6] = '1'
+    if m["A"] != "1" {
+        t.Fatal(m)
+    }
+    data[14] = '0'
+    if m["B"] != "0" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    dc.CopyString()
+    var x interface{}
+    err = dc.Decode(&x)
+    if err != nil {
+        t.Fatal(err)
+    }
+    data[2] = 'C'
+    data[6] = '1'
+    m = x.(map[string]interface{})
+    if m["A"] != "0" {
+        t.Fatal(m)
+    }
+    data[10] = 'D'
+    data[14] = '0'
+    if m["B"] != "1" {
+        t.Fatal(m)
+    }
+
+    data = []byte(`{"A":"0","B":"1"}`)
+    dc = NewDecoder(rt.Mem2Str(data))
+    dc.UseNumber()
+    var y interface{}
+    err = dc.Decode(&y)
+    if err != nil {
+        t.Fatal(err)
+    }
+    m = y.(map[string]interface{})
+    data[6] = '1'
+    if m["A"] != "1" {
+        t.Fatal(m)
+    }
+    data[14] = '0'
+    if m["B"] != "0" {
+        t.Fatal(m)
+    }
+}
+
+func TestDecoder_Basic(t *testing.T) {
+    var v int
+    pos, err := decode("12345", &v, false)
+    assert.NoError(t, err)
+    assert.Equal(t, 5, pos)
+    assert.Equal(t, 12345, v)
+}
+
+func TestDecoder_Generic(t *testing.T) {
+    var v interface{}
+    pos, err := decode(TwitterJson, &v, false)
+    assert.NoError(t, err)
+    assert.Equal(t, len(TwitterJson), pos)
+}
+
+func TestDecoder_Binding(t *testing.T) {
+    var v TwitterStruct
+    pos, err := decode(TwitterJson, &v, false)
+    assert.NoError(t, err)
+    assert.Equal(t, len(TwitterJson), pos)
+    assert.Equal(t, _BindingValue, v, 0)
+    spew.Dump(v)
+}
+
+func TestDecoder_SetOption(t *testing.T) {
+    var v interface{}
+    d := NewDecoder("123")
+    d.SetOptions(OptionUseInt64)
+    err := d.Decode(&v)
+    assert.NoError(t, err)
+    assert.Equal(t, v, int64(123))
+}
+
+func TestDecoder_MapWithIndirectElement(t *testing.T) {
+    var v map[string]struct { A [129]byte }
+    _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false)
+    if x, ok := err.(SyntaxError); ok {
+        println(x.Description())
+    }
+    require.NoError(t, err)
+    assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A)
+}
+
+func BenchmarkDecoder_Generic_Sonic(b *testing.B) {
+    var w interface{}
+    _, _ = decode(TwitterJson, &w, true)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v interface{}
+        _, _ = decode(TwitterJson, &v, true)
+    }
+}
+
+func BenchmarkDecoder_Generic_Sonic_Fast(b *testing.B) {
+    var w interface{}
+    _, _ = decode(TwitterJson, &w, false)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v interface{}
+        _, _ = decode(TwitterJson, &v, false)
+    }
+}
+
+func BenchmarkDecoder_Generic_StdLib(b *testing.B) {
+    var w interface{}
+    m := []byte(TwitterJson)
+    _ = json.Unmarshal(m, &w)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v interface{}
+        _ = json.Unmarshal(m, &v)
+    }
+}
+
+func BenchmarkDecoder_Binding_Sonic(b *testing.B) {
+    var w TwitterStruct
+    _, _ = decode(TwitterJson, &w, true)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v TwitterStruct
+        _, _ = decode(TwitterJson, &v, true)
+    }
+}
+
+func BenchmarkDecoder_Binding_Sonic_Fast(b *testing.B) {
+    var w TwitterStruct
+    _, _ = decode(TwitterJson, &w, false)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v TwitterStruct
+        _, _ = decode(TwitterJson, &v, false)
+    }
+}
+
+func BenchmarkDecoder_Binding_StdLib(b *testing.B) {
+    var w TwitterStruct
+    m := []byte(TwitterJson)
+    _ = json.Unmarshal(m, &w)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        var v TwitterStruct
+        _ = json.Unmarshal(m, &v)
+    }
+}
+
+func BenchmarkDecoder_Parallel_Generic_Sonic(b *testing.B) {
+    var w interface{}
+    _, _ = decode(TwitterJson, &w, true)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v interface{}
+            _, _ = decode(TwitterJson, &v, true)
+        }
+    })
+}
+
+func BenchmarkDecoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
+    var w interface{}
+    _, _ = decode(TwitterJson, &w, false)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v interface{}
+            _, _ = decode(TwitterJson, &v, false)
+        }
+    })
+}
+
+func BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) {
+    var w interface{}
+    m := []byte(TwitterJson)
+    _ = json.Unmarshal(m, &w)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v interface{}
+            _ = json.Unmarshal(m, &v)
+        }
+    })
+}
+
+func BenchmarkDecoder_Parallel_Binding_Sonic(b *testing.B) {
+    var w TwitterStruct
+    _, _ = decode(TwitterJson, &w, true)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v TwitterStruct
+            _, _ = decode(TwitterJson, &v, true)
+        }
+    })
+}
+
+func BenchmarkDecoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
+    var w TwitterStruct
+    _, _ = decode(TwitterJson, &w, false)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v TwitterStruct
+            _, _ = decode(TwitterJson, &v, false)
+        }
+    })
+}
+
+func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {
+    var w TwitterStruct
+    m := []byte(TwitterJson)
+    _ = json.Unmarshal(m, &w)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            var v TwitterStruct
+            _ = json.Unmarshal(m, &v)
+        }
+    })
+}
+
+func BenchmarkSkip_Sonic(b *testing.B) {
+    var data = rt.Str2Mem(TwitterJson)
+    if ret, _ := Skip(data); ret < 0 {
+        b.Fatal()
+    }
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i:=0; i<b.N; i++ {
+        _, _ = Skip(data)
+    }
+}

internal/decoder/testdata_test.go

@@ -0,0 +1,551 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package decoder
+
+const TwitterJson = `{
+  "statuses": [
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Mon Sep 24 03:35:21 +0000 2012",
+      "id_str": "250075927172759552",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Aggressive Ponytail #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 250075927172759552,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "DDEEF6",
+        "profile_sidebar_border_color": "C0DEED",
+        "profile_background_tile": false,
+        "name": "Sean Cummings",
+        "profile_image_url": "https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
+        "created_at": "Mon Apr 26 06:01:55 +0000 2010",
+        "location": "LA, CA",
+        "follow_request_sent": null,
+        "profile_link_color": "0084B4",
+        "is_translator": false,
+        "id_str": "137238150",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "",
+                "indices": [
+                  0,
+                  0
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": true,
+        "contributors_enabled": false,
+        "favourites_count": 0,
+        "url": null,
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
+        "utc_offset": -28800,
+        "id": 137238150,
+        "profile_use_background_image": true,
+        "listed_count": 2,
+        "profile_text_color": "333333",
+        "lang": "en",
+        "followers_count": 70,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png",
+        "profile_background_color": "C0DEED",
+        "verified": false,
+        "geo_enabled": true,
+        "time_zone": "Pacific Time (US & Canada)",
+        "description": "Born 330 Live 310",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/images/themes/theme1/bg.png",
+        "statuses_count": 579,
+        "friends_count": 110,
+        "following": null,
+        "show_all_inline_media": false,
+        "screen_name": "sean_cummings"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "<a href=\"//itunes.apple.com/us/app/twitter/id409789998?mt=12%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for Mac</a>",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 23:40:54 +0000 2012",
+      "id_str": "249292149810667520",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "FreeBandNames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Thee Namaste Nerdz. #FreeBandNames",
+      "metadata": {
+        "iso_language_code": "pl",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249292149810667520,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "DDFFCC",
+        "profile_sidebar_border_color": "BDDCAD",
+        "profile_background_tile": true,
+        "name": "Chaz Martenstein",
+        "profile_image_url": "https://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
+        "created_at": "Tue Apr 07 19:05:07 +0000 2009",
+        "location": "Durham, NC",
+        "follow_request_sent": null,
+        "profile_link_color": "0084B4",
+        "is_translator": false,
+        "id_str": "29516238",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "https://bullcityrecords.com/wnng/",
+                "indices": [
+                  0,
+                  32
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 8,
+        "url": "https://bullcityrecords.com/wnng/",
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
+        "utc_offset": -18000,
+        "id": 29516238,
+        "profile_use_background_image": true,
+        "listed_count": 118,
+        "profile_text_color": "333333",
+        "lang": "en",
+        "followers_count": 2052,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp",
+        "profile_background_color": "9AE4E8",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Eastern Time (US & Canada)",
+        "description": "You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/profile_background_images/9423277/background_tile.bmp",
+        "statuses_count": 7579,
+        "friends_count": 348,
+        "following": null,
+        "show_all_inline_media": true,
+        "screen_name": "bullcityrecords"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "web",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 23:30:20 +0000 2012",
+      "id_str": "249289491129438208",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              29,
+              43
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Mexican Heaven, Mexican Hell #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249289491129438208,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "99CC33",
+        "profile_sidebar_border_color": "829D5E",
+        "profile_background_tile": false,
+        "name": "Thomas John Wakeman",
+        "profile_image_url": "https://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
+        "created_at": "Tue Sep 01 21:21:35 +0000 2009",
+        "location": "Kingston New York",
+        "follow_request_sent": null,
+        "profile_link_color": "D02B55",
+        "is_translator": false,
+        "id_str": "70789458",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "",
+                "indices": [
+                  0,
+                  0
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 19,
+        "url": null,
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
+        "utc_offset": -18000,
+        "id": 70789458,
+        "profile_use_background_image": true,
+        "listed_count": 1,
+        "profile_text_color": "3E4415",
+        "lang": "en",
+        "followers_count": 63,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif",
+        "profile_background_color": "352726",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Eastern Time (US & Canada)",
+        "description": "Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/images/themes/theme5/bg.gif",
+        "statuses_count": 1048,
+        "friends_count": 63,
+        "following": null,
+        "show_all_inline_media": false,
+        "screen_name": "MonkiesFist"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "web",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 22:51:18 +0000 2012",
+      "id_str": "249279667666817024",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "The Foolish Mortals #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249279667666817024,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "BFAC83",
+        "profile_sidebar_border_color": "615A44",
+        "profile_background_tile": true,
+        "name": "Marty Elmer",
+        "profile_image_url": "https://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
+        "created_at": "Mon May 04 00:05:00 +0000 2009",
+        "location": "Wisconsin, USA",
+        "follow_request_sent": null,
+        "profile_link_color": "3B2A26",
+        "is_translator": false,
+        "id_str": "37539828",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "https://www.omnitarian.me",
+                "indices": [
+                  0,
+                  24
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 647,
+        "url": "https://www.omnitarian.me",
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
+        "utc_offset": -21600,
+        "id": 37539828,
+        "profile_use_background_image": true,
+        "listed_count": 52,
+        "profile_text_color": "000000",
+        "lang": "en",
+        "followers_count": 608,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png",
+        "profile_background_color": "EEE3C4",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Central Time (US & Canada)",
+        "description": "Cartoonist, Illustrator, and T-Shirt connoisseur",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/profile_background_images/106455659/rect6056-9.png",
+        "statuses_count": 3575,
+        "friends_count": 249,
+        "following": null,
+        "show_all_inline_media": true,
+        "screen_name": "Omnitarian"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "<a href=\"//twitter.com/download/iphone%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for iPhone</a>",
+      "in_reply_to_status_id": null
+    }
+  ],
+  "search_metadata": {
+    "max_id": 250126199840518145,
+    "since_id": 24012619984051000,
+    "refresh_url": "?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1",
+    "next_results": "?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed",
+    "count": 4,
+    "completed_in": 0.035,
+    "since_id_str": "24012619984051000",
+    "query": "%23freebandnames",
+    "max_id_str": "250126199840518145"
+  }
+}`
+
+type TwitterStruct struct {
+    Statuses       []Statuses     `json:"statuses"`
+    SearchMetadata SearchMetadata `json:"search_metadata"`
+}
+
+type Hashtags struct {
+    Text    string `json:"text"`
+    Indices []int  `json:"indices"`
+}
+
+type Entities struct {
+    Urls         []interface{} `json:"urls"`
+    Hashtags     []Hashtags    `json:"hashtags"`
+    UserMentions []interface{} `json:"user_mentions"`
+}
+
+type Metadata struct {
+    IsoLanguageCode string `json:"iso_language_code"`
+    ResultType      string `json:"result_type"`
+}
+
+type Urls struct {
+    ExpandedURL interface{} `json:"expanded_url"`
+    URL         string      `json:"url"`
+    Indices     []int       `json:"indices"`
+}
+
+type URL struct {
+    Urls []Urls `json:"urls"`
+}
+
+type Description struct {
+    Urls []interface{} `json:"urls"`
+}
+
+type UserEntities struct {
+    URL         URL         `json:"url"`
+    Description Description `json:"description"`
+}
+
+type User struct {
+    ProfileSidebarFillColor        string       `json:"profile_sidebar_fill_color"`
+    ProfileSidebarBorderColor      string       `json:"profile_sidebar_border_color"`
+    ProfileBackgroundTile          bool         `json:"profile_background_tile"`
+    Name                           string       `json:"name"`
+    ProfileImageURL                string       `json:"profile_image_url"`
+    CreatedAt                      string       `json:"created_at"`
+    Location                       string       `json:"location"`
+    FollowRequestSent              interface{}  `json:"follow_request_sent"`
+    ProfileLinkColor               string       `json:"profile_link_color"`
+    IsTranslator                   bool         `json:"is_translator"`
+    IDStr                          string       `json:"id_str"`
+    Entities                       UserEntities `json:"entities"`
+    DefaultProfile                 bool         `json:"default_profile"`
+    ContributorsEnabled            bool         `json:"contributors_enabled"`
+    FavouritesCount                int          `json:"favourites_count"`
+    URL                            interface{}  `json:"url"`
+    ProfileImageURLHTTPS           string       `json:"profile_image_url_https"`
+    UtcOffset                      int          `json:"utc_offset"`
+    ID                             int          `json:"id"`
+    ProfileUseBackgroundImage      bool         `json:"profile_use_background_image"`
+    ListedCount                    int          `json:"listed_count"`
+    ProfileTextColor               string       `json:"profile_text_color"`
+    Lang                           string       `json:"lang"`
+    FollowersCount                 int          `json:"followers_count"`
+    Protected                      bool         `json:"protected"`
+    Notifications                  interface{}  `json:"notifications"`
+    ProfileBackgroundImageURLHTTPS string       `json:"profile_background_image_url_https"`
+    ProfileBackgroundColor         string       `json:"profile_background_color"`
+    Verified                       bool         `json:"verified"`
+    GeoEnabled                     bool         `json:"geo_enabled"`
+    TimeZone                       string       `json:"time_zone"`
+    Description                    string       `json:"description"`
+    DefaultProfileImage            bool         `json:"default_profile_image"`
+    ProfileBackgroundImageURL      string       `json:"profile_background_image_url"`
+    StatusesCount                  int          `json:"statuses_count"`
+    FriendsCount                   int          `json:"friends_count"`
+    Following                      interface{}  `json:"following"`
+    ShowAllInlineMedia             bool         `json:"show_all_inline_media"`
+    ScreenName                     string       `json:"screen_name"`
+}
+
+type Statuses struct {
+    Coordinates          interface{} `json:"coordinates"`
+    Favorited            bool        `json:"favorited"`
+    Truncated            bool        `json:"truncated"`
+    CreatedAt            string      `json:"created_at"`
+    IDStr                string      `json:"id_str"`
+    Entities             Entities    `json:"entities"`
+    InReplyToUserIDStr   interface{} `json:"in_reply_to_user_id_str"`
+    Contributors         interface{} `json:"contributors"`
+    Text                 string      `json:"text"`
+    Metadata             Metadata    `json:"metadata"`
+    RetweetCount         int         `json:"retweet_count"`
+    InReplyToStatusIDStr interface{} `json:"in_reply_to_status_id_str"`
+    ID                   int64       `json:"id"`
+    Geo                  interface{} `json:"geo"`
+    Retweeted            bool        `json:"retweeted"`
+    InReplyToUserID      interface{} `json:"in_reply_to_user_id"`
+    Place                interface{} `json:"place"`
+    User                 User        `json:"user"`
+    InReplyToScreenName  interface{} `json:"in_reply_to_screen_name"`
+    Source               string      `json:"source"`
+    InReplyToStatusID    interface{} `json:"in_reply_to_status_id"`
+}
+
+type SearchMetadata struct {
+    MaxID       int64   `json:"max_id"`
+    SinceID     int64   `json:"since_id"`
+    RefreshURL  string  `json:"refresh_url"`
+    NextResults string  `json:"next_results"`
+    Count       int     `json:"count"`
+    CompletedIn float64 `json:"completed_in"`
+    SinceIDStr  string  `json:"since_id_str"`
+    Query       string  `json:"query"`
+    MaxIDStr    string  `json:"max_id_str"`
+}

internal/encoder/encoder_test.go

@@ -0,0 +1,639 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `bytes`
+    `encoding`
+    `encoding/json`
+    `runtime`
+    `runtime/debug`
+    `strconv`
+    `sync`
+    `testing`
+    `time`
+
+    `github.com/bytedance/sonic/internal/rt`
+    `github.com/stretchr/testify/require`
+)
+
+func TestMain(m *testing.M) {
+    go func ()  {
+        if !debugAsyncGC {
+            return
+        }
+        println("Begin GC looping...")
+        for {
+            runtime.GC()
+            debug.FreeOSMemory()
+        }
+        println("stop GC looping!")
+    }()
+    time.Sleep(time.Millisecond)
+    m.Run()
+}
+
+func TestGC(t *testing.T) {
+    if debugSyncGC {
+        return
+    }
+    out, err := Encode(_GenericValue, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    n := len(out)
+    wg := &sync.WaitGroup{}
+    N := 10000
+    for i:=0; i<N; i++ {
+        wg.Add(1)
+        go func (wg *sync.WaitGroup, size int)  {
+            defer wg.Done()
+            out, err := Encode(_GenericValue, 0)
+            if err != nil {
+                t.Fatal(err)
+            }
+            if len(out) != size {
+                t.Fatal(len(out), size)
+            }
+            runtime.GC()
+        }(wg, n)
+    }
+    wg.Wait()
+}
+
+type sample struct {
+    M map[string]interface{}
+    S []interface{}
+    A [0]interface{}
+    MP *map[string]interface{}
+    SP *[]interface{}
+    AP *[0]interface{}
+}
+
+func TestOptionSliceOrMapNoNull(t *testing.T) {
+    obj := sample{}
+    out, err := Encode(obj, NoNullSliceOrMap)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+
+    obj2 := sample{}
+    out, err = Encode(obj2, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+}
+
+func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
+    b.Run("true", func (b *testing.B) {
+        obj := sample{}
+        _, err := Encode(obj, NoNullSliceOrMap)
+        if err != nil {
+            b.Fatal(err)
+        }
+        b.ResetTimer()
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj, NoNullSliceOrMap)
+        }
+    })
+
+    b.Run("false", func (b *testing.B) {
+        obj2 := sample{}
+        _, err := Encode(obj2, 0)
+        if err != nil {
+            b.Fatal(err)
+        }
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj2, 0)
+        }
+    })
+}
+
+func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
+    require.Equal(t, exp, fn(arg))
+}
+
+func TestEncoder_String(t *testing.T) {
+    runEncoderTest(t, Quote, `""`                                                 , "")
+    runEncoderTest(t, Quote, `"hello, world"`                                     , "hello, world")
+    runEncoderTest(t, Quote, `"hello啊啊啊aa"`                                    , "hello啊啊啊aa")
+    runEncoderTest(t, Quote, `"hello\\\"world"`                                   , "hello\\\"world")
+    runEncoderTest(t, Quote, `"hello\n\tworld"`                                   , "hello\n\tworld")
+    runEncoderTest(t, Quote, `"hello\u0000\u0001world"`                           , "hello\x00\x01world")
+    runEncoderTest(t, Quote, `"hello\u0000\u0001world"`                           , "hello\x00\x01world")
+    runEncoderTest(t, Quote, `"Cartoonist, Illustrator, and T-Shirt connoisseur"` , "Cartoonist, Illustrator, and T-Shirt connoisseur")
+}
+
+type StringStruct struct {
+    X *int        `json:"x,string,omitempty"`
+    Y []int       `json:"y"`
+    Z json.Number `json:"z,string"`
+    W string      `json:"w,string"`
+}
+
+func TestEncoder_FieldStringize(t *testing.T) {
+    x := 12345
+    v := StringStruct{X: &x, Y: []int{1, 2, 3}, Z: "4567456", W: "asdf"}
+    r, e := Encode(v, 0)
+    require.NoError(t, e)
+    println(string(r))
+}
+
+func TestEncodeErrorAndScratchBuf(t *testing.T) {
+    var obj = map[string]interface{}{
+        "a": json.RawMessage(" [} "),
+    }
+    buf := make([]byte, 0, 10)
+    _ = EncodeInto(&buf, obj, 0)
+    if len(buf) < 0 || len(buf) > 10 {
+        println(buf)
+        t.Fatal()
+    }
+}
+
+type MarshalerImpl struct {
+    X int
+}
+
+func (self *MarshalerImpl) MarshalJSON() ([]byte, error) {
+    ret := []byte(strconv.Itoa(self.X))
+    return append(ret, "    "...), nil
+}
+
+type MarshalerStruct struct {
+    V MarshalerImpl
+}
+
+func TestEncoder_Marshaler(t *testing.T) {
+    v := MarshalerStruct{V: MarshalerImpl{X: 12345}}
+    ret, err := Encode(&v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":12345    }`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":{"X":12345}}`, string(ret))
+
+    ret2, err2 := Encode(&v, 0)
+    require.NoError(t, err2)
+    require.Equal(t, `{"V":12345    }`, string(ret2))
+    ret3, err3 := Encode(v, CompactMarshaler)
+    require.NoError(t, err3)
+    require.Equal(t, `{"V":{"X":12345}}`, string(ret3))
+}
+
+type MarshalerErrorStruct struct {
+    V MarshalerImpl
+}
+
+func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) {
+    return []byte(`[""] {`), nil
+}
+
+func TestMarshalerError(t *testing.T) {
+    v := MarshalerErrorStruct{}
+    ret, err := Encode(&v, 0)
+    require.EqualError(t, err, `invalid Marshaler output json syntax at 5: "[\"\"] {"`)
+    require.Equal(t, []byte(nil), ret)
+}
+
+type RawMessageStruct struct {
+    X json.RawMessage
+}
+
+func TestEncoder_RawMessage(t *testing.T) {
+    rms := RawMessageStruct{
+        X: json.RawMessage("123456    "),
+    }
+    ret, err := Encode(&rms, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"X":123456    }`, string(ret))
+
+    ret, err = Encode(&rms, CompactMarshaler)
+    require.NoError(t, err)
+    require.Equal(t, `{"X":123456}`, string(ret))
+}
+
+type TextMarshalerImpl struct {
+    X string
+}
+
+func (self *TextMarshalerImpl) MarshalText() ([]byte, error) {
+    return []byte(self.X), nil
+}
+
+type TextMarshalerImplV struct {
+    X string
+}
+
+func (self TextMarshalerImplV) MarshalText() ([]byte, error) {
+    return []byte(self.X), nil
+}
+
+type TextMarshalerStruct struct {
+    V TextMarshalerImpl
+}
+
+func TestEncoder_TextMarshaler(t *testing.T) {
+    v := TextMarshalerStruct{V: TextMarshalerImpl{X: (`{"a"}`)}}
+    ret, err := Encode(&v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":"{\"a\"}"}`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret))
+
+    ret2, err2 := Encode(&v, NoQuoteTextMarshaler)
+    require.NoError(t, err2)
+    require.Equal(t, `{"V":{"a"}}`, string(ret2))
+    ret3, err3 := Encode(v, NoQuoteTextMarshaler)
+    require.NoError(t, err3)
+    require.Equal(t, `{"V":{"X":"{\"a\"}"}}`, string(ret3))
+}
+
+func TestTextMarshalTextKey_SortKeys(t *testing.T) {
+    v := map[*TextMarshalerImpl]string{
+        {"b"}: "b",
+        {"c"}: "c",
+        {"a"}: "a",
+    }
+    ret, err := Encode(v, SortMapKeys)
+    require.NoError(t, err)
+    require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
+
+    v2 := map[TextMarshalerImplV]string{
+        {"b"}: "b",
+        {"c"}: "c",
+        {"a"}: "a",
+    }
+    ret, err = Encode(v2, SortMapKeys)
+    require.NoError(t, err)
+    require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
+
+    v3 := map[encoding.TextMarshaler]string{
+        TextMarshalerImplV{"b"}: "b",
+        &TextMarshalerImpl{"c"}: "c",
+        TextMarshalerImplV{"a"}: "a",
+    }
+    ret, err = Encode(v3, SortMapKeys)
+    require.NoError(t, err)
+    require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
+}
+
+func TestEncoder_Marshal_EscapeHTML(t *testing.T) {
+    v := map[string]TextMarshalerImpl{"&&":{"<>"}}
+    ret, err := Encode(v, EscapeHTML)
+    require.NoError(t, err)
+    require.Equal(t, `{"\u0026\u0026":{"X":"\u003c\u003e"}}`, string(ret))
+    ret, err = Encode(v, 0)
+    require.NoError(t, err)
+    require.Equal(t, `{"&&":{"X":"<>"}}`, string(ret))
+
+    // “ is \xe2\x80\x9c, and ” is \xe2\x80\x9d,
+    // similar as HTML escaped chars \u2028(\xe2\x80\xa8) and \u2029(\xe2\x80\xa9)
+    m := map[string]string{"test": "“123”"}
+    ret, err = Encode(m, EscapeHTML)
+    require.Equal(t, string(ret), `{"test":"“123”"}`)
+    require.NoError(t, err)
+
+    m = map[string]string{"K": "\u2028\u2028\xe2"}
+    ret, err = Encode(m, EscapeHTML)
+    require.Equal(t, string(ret), "{\"K\":\"\\u2028\\u2028\xe2\"}")
+    require.NoError(t, err)
+}
+
+func TestEncoder_EscapeHTML(t *testing.T) {
+    // test data from libfuzzer
+    test := []string{
+        "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&",
+        "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}",
+    }
+    for _, s := range(test) {
+        data := []byte(s)
+        sdst := HTMLEscape(nil, data)
+        var dst bytes.Buffer
+        json.HTMLEscape(&dst, data)
+        require.Equal(t, string(sdst), dst.String())
+    }
+}
+
+func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) {
+    buf1, err1 := Encode(&_BindingValue, SortMapKeys | EscapeHTML)
+    require.NoError(t, err1)
+    buf2, err2 :=json.Marshal(&_BindingValue)
+    require.NoError(t, err2)
+    require.Equal(t, buf1, buf2)
+}
+
+var _GenericValue interface{}
+var _BindingValue TwitterStruct
+
+func init() {
+    _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue)
+    _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
+}
+
+func TestEncoder_Generic(t *testing.T) {
+    v, e := Encode(_GenericValue, 0)
+    require.NoError(t, e)
+    println(string(v))
+}
+
+func TestEncoder_Binding(t *testing.T) {
+    v, e := Encode(_BindingValue, 0)
+    require.NoError(t, e)
+    println(string(v))
+}
+
+func TestEncoder_MapSortKey(t *testing.T) {
+    m := map[string]string {
+        "C": "third",
+        "D": "forth",
+        "A": "first",
+        "F": "sixth",
+        "E": "fifth",
+        "B": "second",
+    }
+    v, e := Encode(m, SortMapKeys)
+    require.NoError(t, e)
+    require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v))
+}
+
+func BenchmarkEncoder_Generic_Sonic(b *testing.B) {
+    _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    }
+}
+
+func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) {
+    _, _ = Encode(_GenericValue, 0)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = Encode(_GenericValue, 0)
+    }
+}
+
+func BenchmarkEncoder_Generic_StdLib(b *testing.B) {
+    _, _ = json.Marshal(_GenericValue)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = json.Marshal(_GenericValue)
+    }
+}
+
+func BenchmarkEncoder_Binding_Sonic(b *testing.B) {
+    _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    }
+}
+
+func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) {
+    _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
+    }
+}
+
+func BenchmarkEncoder_Binding_StdLib(b *testing.B) {
+    _, _ = json.Marshal(&_BindingValue)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i := 0; i < b.N; i++ {
+        _, _ = json.Marshal(&_BindingValue)
+    }
+}
+
+func BenchmarkEncoder_Parallel_Generic_Sonic(b *testing.B) {
+    _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+        }
+    })
+}
+
+func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
+    _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
+        }
+    })
+}
+
+func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) {
+    _, _ = json.Marshal(_GenericValue)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = json.Marshal(_GenericValue)
+        }
+    })
+}
+
+func BenchmarkEncoder_Parallel_Binding_Sonic(b *testing.B) {
+    _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
+        }
+    })
+}
+
+func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
+    _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
+        }
+    })
+}
+
+func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) {
+    _, _ = json.Marshal(&_BindingValue)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    b.RunParallel(func(pb *testing.PB) {
+        for pb.Next() {
+            _, _ = json.Marshal(&_BindingValue)
+        }
+    })
+}
+
+func BenchmarkHTMLEscape_Sonic(b *testing.B) {
+    jsonByte := []byte(TwitterJson)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    var buf []byte
+    for i := 0; i < b.N; i++ {
+        buf = HTMLEscape(nil, jsonByte)
+    }
+    _ = buf
+}
+
+func BenchmarkHTMLEscape_StdLib(b *testing.B) {
+    jsonByte := []byte(TwitterJson)
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    var buf []byte
+    for i := 0; i < b.N; i++ {
+        out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson) * 6 / 5))
+        json.HTMLEscape(out, jsonByte)
+        buf = out.Bytes()
+    }
+    _ = buf
+}
+
+
+func BenchmarkValidate_Sonic(b *testing.B) {
+    var data = rt.Str2Mem(TwitterJson)
+    ok, s := Valid(data)
+    if !ok {
+        b.Fatal(s)
+    }
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i:=0; i<b.N; i++ {
+        _, _ = Valid(data)
+    }
+}
+
+func BenchmarkValidate_Std(b *testing.B) {
+    var data = rt.Str2Mem(TwitterJson)
+    if !json.Valid(data) {
+        b.Fatal()
+    }
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i:=0; i<b.N; i++ {
+        _ = json.Valid(data)
+    }
+}
+
+func BenchmarkCompact_Std(b *testing.B) {
+    var data = rt.Str2Mem(TwitterJson)
+    var dst = bytes.NewBuffer(nil)
+    if err := json.Compact(dst, data); err != nil {
+        b.Fatal(err)
+    }
+    b.SetBytes(int64(len(TwitterJson)))
+    b.ResetTimer()
+    for i:=0; i<b.N; i++ {
+        dst.Reset()
+        _ = json.Compact(dst, data)
+    }
+}
+
+type f64Bench struct {
+    name    string
+    float   float64
+}
+func BenchmarkEncode_Float64(b *testing.B) {
+    var bench = []f64Bench{
+        {"Zero", 0},
+        {"ShortDecimal", 1000},
+        {"Decimal", 33909},
+        {"Float", 339.7784},
+        {"Exp", -5.09e75},
+        {"NegExp", -5.11e-95},
+        {"LongExp", 1.234567890123456e-78},
+        {"Big", 123456789123456789123456789},
+    
+    }
+    maxUint := "18446744073709551615"
+    for i := 1; i <= len(maxUint); i++ {
+        name := strconv.FormatInt(int64(i), 10) + "-Digs"
+        num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
+        bench = append(bench, f64Bench{name, float64(num)})
+    }
+    for _, c := range bench {
+        libs := []struct {
+            name string
+            test func(*testing.B)
+        }{{
+            name: "StdLib",
+            test: func(b *testing.B) {  _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
+        }, {
+            name: "Sonic",
+            test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
+        }}
+        for _, lib := range libs {
+            name := lib.name + "_" + c.name
+            b.Run(name, lib.test)
+        }  
+    }
+}
+
+type f32Bench struct {
+    name    string
+    float   float32
+}
+func BenchmarkEncode_Float32(b *testing.B) {
+    var bench = []f32Bench{
+        {"Zero", 0},
+        {"ShortDecimal", 1000},
+        {"Decimal", 33909},
+        {"ExactFraction", 3.375},
+        {"Point", 339.7784},
+        {"Exp", -5.09e25},
+        {"NegExp", -5.11e-25},
+        {"Shortest", 1.234567e-8},
+    }
+
+    maxUint := "18446744073709551615"
+    for i := 1; i <= len(maxUint); i++ {
+        name := strconv.FormatInt(int64(i), 10) + "-Digs"
+        num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
+        bench = append(bench, f32Bench{name, float32(num)})
+    }
+    for _, c := range bench {
+        libs := []struct {
+            name string
+            test func(*testing.B)
+        }{{
+            name: "StdLib",
+            test: func(b *testing.B) {  _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
+        }, {
+            name: "Sonic",
+            test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
+        }}
+        for _, lib := range libs {
+            name := lib.name + "_" + c.name
+            b.Run(name, lib.test)
+        }  
+    }
+}

internal/encoder/testdata_test.go

@@ -0,0 +1,551 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+const TwitterJson = `{
+  "statuses": [
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Mon Sep 24 03:35:21 +0000 2012",
+      "id_str": "250075927172759552",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Aggressive Ponytail #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 250075927172759552,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "DDEEF6",
+        "profile_sidebar_border_color": "C0DEED",
+        "profile_background_tile": false,
+        "name": "Sean Cummings",
+        "profile_image_url": "https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
+        "created_at": "Mon Apr 26 06:01:55 +0000 2010",
+        "location": "LA, CA",
+        "follow_request_sent": null,
+        "profile_link_color": "0084B4",
+        "is_translator": false,
+        "id_str": "137238150",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "",
+                "indices": [
+                  0,
+                  0
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": true,
+        "contributors_enabled": false,
+        "favourites_count": 0,
+        "url": null,
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
+        "utc_offset": -28800,
+        "id": 137238150,
+        "profile_use_background_image": true,
+        "listed_count": 2,
+        "profile_text_color": "333333",
+        "lang": "en",
+        "followers_count": 70,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png",
+        "profile_background_color": "C0DEED",
+        "verified": false,
+        "geo_enabled": true,
+        "time_zone": "Pacific Time (US & Canada)",
+        "description": "Born 330 Live 310",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/images/themes/theme1/bg.png",
+        "statuses_count": 579,
+        "friends_count": 110,
+        "following": null,
+        "show_all_inline_media": false,
+        "screen_name": "sean_cummings"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "<a href=\"//itunes.apple.com/us/app/twitter/id409789998?mt=12%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for Mac</a>",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 23:40:54 +0000 2012",
+      "id_str": "249292149810667520",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "FreeBandNames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Thee Namaste Nerdz. #FreeBandNames",
+      "metadata": {
+        "iso_language_code": "pl",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249292149810667520,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "DDFFCC",
+        "profile_sidebar_border_color": "BDDCAD",
+        "profile_background_tile": true,
+        "name": "Chaz Martenstein",
+        "profile_image_url": "https://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
+        "created_at": "Tue Apr 07 19:05:07 +0000 2009",
+        "location": "Durham, NC",
+        "follow_request_sent": null,
+        "profile_link_color": "0084B4",
+        "is_translator": false,
+        "id_str": "29516238",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "https://bullcityrecords.com/wnng/",
+                "indices": [
+                  0,
+                  32
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 8,
+        "url": "https://bullcityrecords.com/wnng/",
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
+        "utc_offset": -18000,
+        "id": 29516238,
+        "profile_use_background_image": true,
+        "listed_count": 118,
+        "profile_text_color": "333333",
+        "lang": "en",
+        "followers_count": 2052,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp",
+        "profile_background_color": "9AE4E8",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Eastern Time (US & Canada)",
+        "description": "You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/profile_background_images/9423277/background_tile.bmp",
+        "statuses_count": 7579,
+        "friends_count": 348,
+        "following": null,
+        "show_all_inline_media": true,
+        "screen_name": "bullcityrecords"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "web",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 23:30:20 +0000 2012",
+      "id_str": "249289491129438208",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              29,
+              43
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "Mexican Heaven, Mexican Hell #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249289491129438208,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "99CC33",
+        "profile_sidebar_border_color": "829D5E",
+        "profile_background_tile": false,
+        "name": "Thomas John Wakeman",
+        "profile_image_url": "https://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
+        "created_at": "Tue Sep 01 21:21:35 +0000 2009",
+        "location": "Kingston New York",
+        "follow_request_sent": null,
+        "profile_link_color": "D02B55",
+        "is_translator": false,
+        "id_str": "70789458",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "",
+                "indices": [
+                  0,
+                  0
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 19,
+        "url": null,
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
+        "utc_offset": -18000,
+        "id": 70789458,
+        "profile_use_background_image": true,
+        "listed_count": 1,
+        "profile_text_color": "3E4415",
+        "lang": "en",
+        "followers_count": 63,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif",
+        "profile_background_color": "352726",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Eastern Time (US & Canada)",
+        "description": "Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/images/themes/theme5/bg.gif",
+        "statuses_count": 1048,
+        "friends_count": 63,
+        "following": null,
+        "show_all_inline_media": false,
+        "screen_name": "MonkiesFist"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "web",
+      "in_reply_to_status_id": null
+    },
+    {
+      "coordinates": null,
+      "favorited": false,
+      "truncated": false,
+      "created_at": "Fri Sep 21 22:51:18 +0000 2012",
+      "id_str": "249279667666817024",
+      "entities": {
+        "urls": [
+ 
+        ],
+        "hashtags": [
+          {
+            "text": "freebandnames",
+            "indices": [
+              20,
+              34
+            ]
+          }
+        ],
+        "user_mentions": [
+ 
+        ]
+      },
+      "in_reply_to_user_id_str": null,
+      "contributors": null,
+      "text": "The Foolish Mortals #freebandnames",
+      "metadata": {
+        "iso_language_code": "en",
+        "result_type": "recent"
+      },
+      "retweet_count": 0,
+      "in_reply_to_status_id_str": null,
+      "id": 249279667666817024,
+      "geo": null,
+      "retweeted": false,
+      "in_reply_to_user_id": null,
+      "place": null,
+      "user": {
+        "profile_sidebar_fill_color": "BFAC83",
+        "profile_sidebar_border_color": "615A44",
+        "profile_background_tile": true,
+        "name": "Marty Elmer",
+        "profile_image_url": "https://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
+        "created_at": "Mon May 04 00:05:00 +0000 2009",
+        "location": "Wisconsin, USA",
+        "follow_request_sent": null,
+        "profile_link_color": "3B2A26",
+        "is_translator": false,
+        "id_str": "37539828",
+        "entities": {
+          "url": {
+            "urls": [
+              {
+                "expanded_url": null,
+                "url": "https://www.omnitarian.me",
+                "indices": [
+                  0,
+                  24
+                ]
+              }
+            ]
+          },
+          "description": {
+            "urls": [
+ 
+            ]
+          }
+        },
+        "default_profile": false,
+        "contributors_enabled": false,
+        "favourites_count": 647,
+        "url": "https://www.omnitarian.me",
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
+        "utc_offset": -21600,
+        "id": 37539828,
+        "profile_use_background_image": true,
+        "listed_count": 52,
+        "profile_text_color": "000000",
+        "lang": "en",
+        "followers_count": 608,
+        "protected": false,
+        "notifications": null,
+        "profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png",
+        "profile_background_color": "EEE3C4",
+        "verified": false,
+        "geo_enabled": false,
+        "time_zone": "Central Time (US & Canada)",
+        "description": "Cartoonist, Illustrator, and T-Shirt connoisseur",
+        "default_profile_image": false,
+        "profile_background_image_url": "https://a0.twimg.com/profile_background_images/106455659/rect6056-9.png",
+        "statuses_count": 3575,
+        "friends_count": 249,
+        "following": null,
+        "show_all_inline_media": true,
+        "screen_name": "Omnitarian"
+      },
+      "in_reply_to_screen_name": null,
+      "source": "<a href=\"//twitter.com/download/iphone%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for iPhone</a>",
+      "in_reply_to_status_id": null
+    }
+  ],
+  "search_metadata": {
+    "max_id": 250126199840518145,
+    "since_id": 24012619984051000,
+    "refresh_url": "?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1",
+    "next_results": "?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed",
+    "count": 4,
+    "completed_in": 0.035,
+    "since_id_str": "24012619984051000",
+    "query": "%23freebandnames",
+    "max_id_str": "250126199840518145"
+  }
+}`
+
+type TwitterStruct struct {
+    Statuses       []Statuses     `json:"statuses"`
+    SearchMetadata SearchMetadata `json:"search_metadata"`
+}
+
+type Hashtags struct {
+    Text    string `json:"text"`
+    Indices []int  `json:"indices"`
+}
+
+type Entities struct {
+    Urls         []interface{} `json:"urls"`
+    Hashtags     []Hashtags    `json:"hashtags"`
+    UserMentions []interface{} `json:"user_mentions"`
+}
+
+type Metadata struct {
+    IsoLanguageCode string `json:"iso_language_code"`
+    ResultType      string `json:"result_type"`
+}
+
+type Urls struct {
+    ExpandedURL interface{} `json:"expanded_url"`
+    URL         string      `json:"url"`
+    Indices     []int       `json:"indices"`
+}
+
+type URL struct {
+    Urls []Urls `json:"urls"`
+}
+
+type Description struct {
+    Urls []interface{} `json:"urls"`
+}
+
+type UserEntities struct {
+    URL         URL         `json:"url"`
+    Description Description `json:"description"`
+}
+
+type User struct {
+    ProfileSidebarFillColor        string       `json:"profile_sidebar_fill_color"`
+    ProfileSidebarBorderColor      string       `json:"profile_sidebar_border_color"`
+    ProfileBackgroundTile          bool         `json:"profile_background_tile"`
+    Name                           string       `json:"name"`
+    ProfileImageURL                string       `json:"profile_image_url"`
+    CreatedAt                      string       `json:"created_at"`
+    Location                       string       `json:"location"`
+    FollowRequestSent              interface{}  `json:"follow_request_sent"`
+    ProfileLinkColor               string       `json:"profile_link_color"`
+    IsTranslator                   bool         `json:"is_translator"`
+    IDStr                          string       `json:"id_str"`
+    Entities                       UserEntities `json:"entities"`
+    DefaultProfile                 bool         `json:"default_profile"`
+    ContributorsEnabled            bool         `json:"contributors_enabled"`
+    FavouritesCount                int          `json:"favourites_count"`
+    URL                            interface{}  `json:"url"`
+    ProfileImageURLHTTPS           string       `json:"profile_image_url_https"`
+    UtcOffset                      int          `json:"utc_offset"`
+    ID                             int          `json:"id"`
+    ProfileUseBackgroundImage      bool         `json:"profile_use_background_image"`
+    ListedCount                    int          `json:"listed_count"`
+    ProfileTextColor               string       `json:"profile_text_color"`
+    Lang                           string       `json:"lang"`
+    FollowersCount                 int          `json:"followers_count"`
+    Protected                      bool         `json:"protected"`
+    Notifications                  interface{}  `json:"notifications"`
+    ProfileBackgroundImageURLHTTPS string       `json:"profile_background_image_url_https"`
+    ProfileBackgroundColor         string       `json:"profile_background_color"`
+    Verified                       bool         `json:"verified"`
+    GeoEnabled                     bool         `json:"geo_enabled"`
+    TimeZone                       string       `json:"time_zone"`
+    Description                    string       `json:"description"`
+    DefaultProfileImage            bool         `json:"default_profile_image"`
+    ProfileBackgroundImageURL      string       `json:"profile_background_image_url"`
+    StatusesCount                  int          `json:"statuses_count"`
+    FriendsCount                   int          `json:"friends_count"`
+    Following                      interface{}  `json:"following"`
+    ShowAllInlineMedia             bool         `json:"show_all_inline_media"`
+    ScreenName                     string       `json:"screen_name"`
+}
+
+type Statuses struct {
+    Coordinates          interface{} `json:"coordinates"`
+    Favorited            bool        `json:"favorited"`
+    Truncated            bool        `json:"truncated"`
+    CreatedAt            string      `json:"created_at"`
+    IDStr                string      `json:"id_str"`
+    Entities             Entities    `json:"entities"`
+    InReplyToUserIDStr   interface{} `json:"in_reply_to_user_id_str"`
+    Contributors         interface{} `json:"contributors"`
+    Text                 string      `json:"text"`
+    Metadata             Metadata    `json:"metadata"`
+    RetweetCount         int         `json:"retweet_count"`
+    InReplyToStatusIDStr interface{} `json:"in_reply_to_status_id_str"`
+    ID                   int64       `json:"id"`
+    Geo                  interface{} `json:"geo"`
+    Retweeted            bool        `json:"retweeted"`
+    InReplyToUserID      interface{} `json:"in_reply_to_user_id"`
+    Place                interface{} `json:"place"`
+    User                 User        `json:"user"`
+    InReplyToScreenName  interface{} `json:"in_reply_to_screen_name"`
+    Source               string      `json:"source"`
+    InReplyToStatusID    interface{} `json:"in_reply_to_status_id"`
+}
+
+type SearchMetadata struct {
+    MaxID       int64   `json:"max_id"`
+    SinceID     int64   `json:"since_id"`
+    RefreshURL  string  `json:"refresh_url"`
+    NextResults string  `json:"next_results"`
+    Count       int     `json:"count"`
+    CompletedIn float64 `json:"completed_in"`
+    SinceIDStr  string  `json:"since_id_str"`
+    Query       string  `json:"query"`
+    MaxIDStr    string  `json:"max_id_str"`
+}