diff --git a/decoder/decoder_amd64.go b/decoder/decoder_amd64.go new file mode 100644 index 0000000..be2d36c --- /dev/null +++ b/decoder/decoder_amd64.go @@ -0,0 +1,50 @@ +// +build amd64,go1.15,!go1.21 + +/* +* Copyright 2023 ByteDance Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package decoder + +import ( + `github.com/bytedance/sonic/internal/decoder` +) + +type Decoder = decoder.Decoder + +type MismatchTypeError = decoder.MismatchTypeError + +type Options = decoder.Options + +const ( + OptionUseInt64 Options = decoder.OptionUseInt64 + OptionUseNumber Options = decoder.OptionUseNumber + OptionUseUnicodeErrors Options = decoder.OptionUseUnicodeErrors + OptionDisableUnknown Options = decoder.OptionDisableUnknown + OptionCopyString Options = decoder.OptionCopyString + OptionValidateString Options = decoder.OptionValidateString +) + +type StreamDecoder = decoder.StreamDecoder + +type SyntaxError = decoder.SyntaxError + +var Pretouch = decoder.Pretouch + +var Skip = decoder.Skip + +var NewDecoder = decoder.NewDecoder + +var NewStreamDecoder = decoder.NewStreamDecoder diff --git a/decoder/decoder_amd64_test.go b/decoder/decoder_amd64_test.go new file mode 100644 index 0000000..13fa0b4 --- /dev/null +++ b/decoder/decoder_amd64_test.go @@ -0,0 +1,276 @@ +// +build amd64,go1.15,!go1.21 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package decoder + +import ( + `encoding/json` + `strings` + `testing` + `reflect` + + `github.com/bytedance/sonic/internal/rt` + `github.com/stretchr/testify/assert` +) + +func TestSkipMismatchTypeAmd64Error(t *testing.T) { + t.Run("struct", func(t *testing.T) { + println("TestSkipError") + type skiptype struct { + A int `json:"a"` + B string `json:"b"` + + Pass *int `json:"pass"` + + C struct{ + + Pass4 interface{} `json:"pass4"` + + D struct{ + E float32 `json:"e"` + } `json:"d"` + + Pass2 int `json:"pass2"` + + } `json:"c"` + + E bool `json:"e"` + F []int `json:"f"` + G map[string]int `json:"g"` + H bool `json:"h,string"` + + Pass3 int `json:"pass2"` + + I json.Number `json:"i"` + } + var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)} + var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}` + d := NewDecoder(data) + err := d.Decode(obj) + err2 := json.Unmarshal([]byte(data), obj2) + println(err2.Error()) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + if te, ok := err.(*MismatchTypeError); ok { + assert.Equal(t, reflect.TypeOf(obj.I), te.Type) + assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos) + println(err.Error()) + } else { + t.Fatal("invalid error") + } + }) + t.Run("short array", func(t *testing.T) { + var obj, obj2 = &[]int{}, &[]int{} + var data = `[""]` + d := NewDecoder(data) + err := d.Decode(obj) + err2 := json.Unmarshal([]byte(data), obj2) + // println(err2.Error()) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + }) + + t.Run("int ", func(t *testing.T) { + var obj int = 123 + var obj2 int = 123 + var data = `[""]` + d := NewDecoder(data) + err := d.Decode(&obj) + err2 := json.Unmarshal([]byte(data), &obj2) + println(err.Error(), obj, obj2) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + }) + + t.Run("array", func(t *testing.T) { + var obj, obj2 = &[]int{}, &[]int{} + var data = `["",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]` + d := NewDecoder(data) + err := d.Decode(obj) + err2 := json.Unmarshal([]byte(data), obj2) + // println(err2.Error()) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + }) + + t.Run("map", func(t *testing.T) { + var obj, obj2 = &map[int]int{}, &map[int]int{} + var data = `{"true" : { },"1":1,"2" : true,"3":3}` + d := NewDecoder(data) + err := d.Decode(obj) + err2 := json.Unmarshal([]byte(data), obj2) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + assert.Equal(t, obj2, obj) + }) + t.Run("map error", func(t *testing.T) { + var obj, obj2 = &map[int]int{}, &map[int]int{} + var data = `{"true" : { ],"1":1,"2" : true,"3":3}` + d := NewDecoder(data) + err := d.Decode(obj) + err2 := json.Unmarshal([]byte(data), obj2) + println(err.Error()) + println(err2.Error()) + assert.Equal(t, err2 == nil, err == nil) + // assert.Equal(t, len(data), d.i) + // assert.Equal(t, obj2, obj) + }) +} + +func TestCopyString(t *testing.T) { + var data []byte + var dc *Decoder + var err error + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + dc.CopyString() + var obj struct{ + A string + B string + } + err = dc.Decode(&obj) + if err != nil { + t.Fatal(err) + } + data[6] = '1' + if obj.A != "0" { + t.Fatal(obj) + } + data[14] = '0' + if obj.B != "1" { + t.Fatal(obj) + } + + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + err = dc.Decode(&obj) + if err != nil { + t.Fatal(err) + } + data[6] = '1' + if obj.A != "1" { + t.Fatal(obj) + } + data[14] = '0' + if obj.B != "0" { + t.Fatal(obj) + } + + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + dc.CopyString() + m := map[string]interface{}{} + err = dc.Decode(&m) + if err != nil { + t.Fatal(err) + } + data[2] = 'C' + data[6] = '1' + if m["A"] != "0" { + t.Fatal(m) + } + data[10] = 'D' + data[14] = '0' + if m["B"] != "1" { + t.Fatal(m) + } + + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + m = map[string]interface{}{} + err = dc.Decode(&m) + if err != nil { + t.Fatal(err) + } + data[6] = '1' + if m["A"] != "1" { + t.Fatal(m) + } + data[14] = '0' + if m["B"] != "0" { + t.Fatal(m) + } + + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + dc.CopyString() + var x interface{} + err = dc.Decode(&x) + if err != nil { + t.Fatal(err) + } + data[2] = 'C' + data[6] = '1' + m = x.(map[string]interface{}) + if m["A"] != "0" { + t.Fatal(m) + } + data[10] = 'D' + data[14] = '0' + if m["B"] != "1" { + t.Fatal(m) + } + + data = []byte(`{"A":"0","B":"1"}`) + dc = NewDecoder(rt.Mem2Str(data)) + dc.UseNumber() + var y interface{} + err = dc.Decode(&y) + if err != nil { + t.Fatal(err) + } + m = y.(map[string]interface{}) + data[6] = '1' + if m["A"] != "1" { + t.Fatal(m) + } + data[14] = '0' + if m["B"] != "0" { + t.Fatal(m) + } +} + +func TestDecoder_SetOption(t *testing.T) { + var v interface{} + d := NewDecoder("123") + d.SetOptions(OptionUseInt64) + err := d.Decode(&v) + assert.NoError(t, err) + assert.Equal(t, v, int64(123)) +} + +func BenchmarkSkip_Sonic(b *testing.B) { + var data = rt.Str2Mem(TwitterJson) + if ret, _ := Skip(data); ret < 0 { + b.Fatal() + } + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i:=0; i"}} + ret, err := Encode(v, EscapeHTML) + require.NoError(t, err) + require.Equal(t, `{"\u0026\u0026":{"X":"\u003c\u003e"}}`, string(ret)) + ret, err = Encode(v, 0) + require.NoError(t, err) + require.Equal(t, `{"&&":{"X":"<>"}}`, string(ret)) + + // “ is \xe2\x80\x9c, and ” is \xe2\x80\x9d, + // similar as HTML escaped chars \u2028(\xe2\x80\xa8) and \u2029(\xe2\x80\xa9) + m := map[string]string{"test": "“123”"} + ret, err = Encode(m, EscapeHTML) + require.Equal(t, string(ret), `{"test":"“123”"}`) + require.NoError(t, err) + + m = map[string]string{"K": "\u2028\u2028\xe2"} + ret, err = Encode(m, EscapeHTML) + require.Equal(t, string(ret), "{\"K\":\"\\u2028\\u2028\xe2\"}") + require.NoError(t, err) +} diff --git a/encoder/encoder_compat.go b/encoder/encoder_compat.go new file mode 100644 index 0000000..afa80d5 --- /dev/null +++ b/encoder/encoder_compat.go @@ -0,0 +1,234 @@ +// +build !amd64 go1.21 + +/* +* Copyright 2023 ByteDance Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package encoder + +import ( + `io` + `bytes` + `encoding/json` + `reflect` + + `github.com/bytedance/sonic/option` +) + +// Options is a set of encoding options. +type Options uint64 + +const ( + bitSortMapKeys = iota + bitEscapeHTML + bitCompactMarshaler + bitNoQuoteTextMarshaler + bitNoNullSliceOrMap + bitValidateString + + // used for recursive compile + bitPointerValue = 63 +) + +const ( + // SortMapKeys indicates that the keys of a map needs to be sorted + // before serializing into JSON. + // WARNING: This hurts performance A LOT, USE WITH CARE. + SortMapKeys Options = 1 << bitSortMapKeys + + // EscapeHTML indicates encoder to escape all HTML characters + // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape). + // WARNING: This hurts performance A LOT, USE WITH CARE. + EscapeHTML Options = 1 << bitEscapeHTML + + // CompactMarshaler indicates that the output JSON from json.Marshaler + // is always compact and needs no validation + CompactMarshaler Options = 1 << bitCompactMarshaler + + // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler + // is always escaped string and needs no quoting + NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler + + // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}', + // instead of 'null' + NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap + + // ValidateString indicates that encoder should validate the input string + // before encoding it into JSON. + ValidateString Options = 1 << bitValidateString + + // CompatibleWithStd is used to be compatible with std encoder. + CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler +) + +// Encoder represents a specific set of encoder configurations. +type Encoder struct { + Opts Options + prefix string + indent string +} + +// Encode returns the JSON encoding of v. +func (self *Encoder) Encode(v interface{}) ([]byte, error) { + if self.indent != "" || self.prefix != "" { + return EncodeIndented(v, self.prefix, self.indent, self.Opts) + } + return Encode(v, self.Opts) +} + +// SortKeys enables the SortMapKeys option. +func (self *Encoder) SortKeys() *Encoder { + self.Opts |= SortMapKeys + return self +} + +// SetEscapeHTML specifies if option EscapeHTML opens +func (self *Encoder) SetEscapeHTML(f bool) { + if f { + self.Opts |= EscapeHTML + } else { + self.Opts &= ^EscapeHTML + } +} + +// SetValidateString specifies if option ValidateString opens +func (self *Encoder) SetValidateString(f bool) { + if f { + self.Opts |= ValidateString + } else { + self.Opts &= ^ValidateString + } +} + +// SetCompactMarshaler specifies if option CompactMarshaler opens +func (self *Encoder) SetCompactMarshaler(f bool) { + if f { + self.Opts |= CompactMarshaler + } else { + self.Opts &= ^CompactMarshaler + } +} + +// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens +func (self *Encoder) SetNoQuoteTextMarshaler(f bool) { + if f { + self.Opts |= NoQuoteTextMarshaler + } else { + self.Opts &= ^NoQuoteTextMarshaler + } +} + +// SetIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function EncodeIndent(). +// Calling SetIndent("", "") disables indentation. +func (enc *Encoder) SetIndent(prefix, indent string) { + enc.prefix = prefix + enc.indent = indent +} + +// Quote returns the JSON-quoted version of s. +func Quote(s string) string { + /* check for empty string */ + if s == "" { + return `""` + } + + out, _ := json.Marshal(s) + return string(out) +} + +// Encode returns the JSON encoding of val, encoded with opts. +func Encode(val interface{}, opts Options) ([]byte, error) { + return json.Marshal(val) +} + +// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating +// a new one. +func EncodeInto(buf *[]byte, val interface{}, opts Options) error { + if buf == nil { + panic("user-supplied buffer buf is nil") + } + w := bytes.NewBuffer(*buf) + enc := json.NewEncoder(w) + enc.SetEscapeHTML((opts & EscapeHTML) != 0) + err := enc.Encode(val) + *buf = w.Bytes() + return err +} + +// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 +// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 +// so that the JSON will be safe to embed inside HTML