diff --git a/README.md b/README.md index deb2491..21a7938 100644 --- a/README.md +++ b/README.md @@ -275,16 +275,16 @@ import ( "reflect" "github.com/bytedance/sonic" "github.com/bytedance/sonic/option" - ) +) - func init() { - var v HugeStruct +func init() { + var v HugeStruct // For most large types (nesting depth <= 5) - err := sonic.Pretouch(reflect.TypeOf(v)) + err := sonic.Pretouch(reflect.TypeOf(v)) // If the type is too deep nesting (nesting depth > 5), // you can set compile recursive depth in Pretouch for better stability in JIT. err := sonic.Pretouch(reflect.TypeOf(v), option.WithCompileRecursiveDepth(depth)) - } +} ``` ### Copy string diff --git a/api.go b/api.go index 5f31fe4..0c8aa79 100644 --- a/api.go +++ b/api.go @@ -19,9 +19,9 @@ package sonic import ( `io` ) - - // Config is a combination of sonic/encoder.Options and sonic/decoder.Options - type Config struct { + +// Config is a combination of sonic/encoder.Options and sonic/decoder.Options +type Config struct { // EscapeHTML indicates encoder to escape all HTML characters // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape). // WARNING: This hurts performance A LOT, USE WITH CARE. @@ -67,9 +67,9 @@ import ( // ValidateString indicates decoder to valid string values: decoder will return errors when // invalid UTF-8 chars or unescaped control chars(\u0000-\u001f) in the string value of JSON. ValidateString bool - } +} - var ( +var ( // ConfigDefault is the default config of APIs, aiming at efficiency and safty. ConfigDefault = Config{}.Froze() @@ -85,13 +85,13 @@ import ( ConfigFastest = Config{ NoQuoteTextMarshaler: true, }.Froze() - ) +) - // API is a binding of specific config. - // This interface is inspired by github.com/json-iterator/go, - // and has same behaviors under equavilent config. - type API interface { +// API is a binding of specific config. +// This interface is inspired by github.com/json-iterator/go, +// and has same behaviors under equavilent config. +type API interface { // MarshalToString returns the JSON encoding string of v MarshalToString(v interface{}) (string, error) // Marshal returns the JSON encoding bytes of v. @@ -108,10 +108,10 @@ import ( NewDecoder(reader io.Reader) Decoder // Valid validates the JSON-encoded bytes and reportes if it is valid Valid(data []byte) bool - } - - // Encoder encodes JSON into io.Writer - type Encoder interface { +} + +// Encoder encodes JSON into io.Writer +type Encoder interface { // Encode writes the JSON encoding of v to the stream, followed by a newline character. Encode(val interface{}) error // SetEscapeHTML specifies whether problematic HTML characters @@ -122,10 +122,10 @@ import ( // as if indented by the package-level function Indent(dst, src, prefix, indent). // Calling SetIndent("", "") disables indentation SetIndent(prefix, indent string) - } - - // Decoder decodes JSON from io.Read - type Decoder interface { +} + +// Decoder decodes JSON from io.Read +type Decoder interface { // Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v. Decode(val interface{}) error // Buffered returns a reader of the data remaining in the Decoder's buffer. @@ -138,9 +138,9 @@ import ( More() bool // UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64. UseNumber() - } +} - // Marshal returns the JSON encoding bytes of v. +// Marshal returns the JSON encoding bytes of v. func Marshal(val interface{}) ([]byte, error) { return ConfigDefault.Marshal(val) } diff --git a/encoder/debug_go117.go b/encoder/debug_go117.go index ed21932..19ec4b4 100644 --- a/encoder/debug_go117.go +++ b/encoder/debug_go117.go @@ -16,190 +16,190 @@ * limitations under the License. */ - package encoder +package encoder - import ( - `fmt` - `os` - `runtime` - `strings` - `unsafe` - - `github.com/bytedance/sonic/internal/jit` - `github.com/twitchyliquid64/golang-asm/obj` - ) - - const _FP_debug = 128 - - var ( - debugSyncGC = os.Getenv("SONIC_SYNC_GC") != "" - debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" - debugCheckPtr = os.Getenv("SONIC_CHECK_POINTER") != "" - ) - - var ( - _Instr_End _Instr = newInsOp(_OP_is_nil) - - _F_gc = jit.Func(gc) - _F_println = jit.Func(println_wrapper) - _F_print = jit.Func(print) - ) - - func (self *_Assembler) dsave(r ...obj.Addr) { - for i, v := range r { - if i > _FP_debug / 8 - 1 { - panic("too many registers to save") - } else { - self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8)) - } - } - } - - func (self *_Assembler) dload(r ...obj.Addr) { - for i, v := range r { - if i > _FP_debug / 8 - 1 { - panic("too many registers to load") - } else { - self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8), v) - } - } - } - - func println_wrapper(i int, op1 int, op2 int){ - println(i, " Intrs ", op1, _OpNames[op1], "next: ", op2, _OpNames[op2]) - } - - func print(i int){ - println(i) - } - - func gc() { - if !debugSyncGC { - return - } - runtime.GC() - // debug.FreeOSMemory() - } - - func (self *_Assembler) dcall(fn obj.Addr) { - self.Emit("MOVQ", fn, _R10) // MOVQ ${fn}, R10 - self.Rjmp("CALL", _R10) // CALL R10 - } - - func (self *_Assembler) debug_gc() { - if !debugSyncGC { - return - } - self.dsave(_REG_debug...) - self.dcall(_F_gc) - self.dload(_REG_debug...) - } - - func (self *_Assembler) debug_instr(i int, v *_Instr) { - if debugSyncGC { - if (i+1 == len(self.p)) { - self.print_gc(i, v, &_Instr_End) - } else { - next := &(self.p[i+1]) - self.print_gc(i, v, next) - name := _OpNames[next.op()] - if strings.Contains(name, "save") { - return - } - } - // self.debug_gc() - } - } - - //go:noescape - //go:linkname checkptrBase runtime.checkptrBase - func checkptrBase(p unsafe.Pointer) uintptr - - //go:noescape - //go:linkname findObject runtime.findObject - func findObject(p, refBase, refOff uintptr) (base uintptr, s unsafe.Pointer, objIndex uintptr) - - var ( - _F_checkptr = jit.Func(checkptr) - _F_printptr = jit.Func(printptr) - ) - - var ( - _R10 = jit.Reg("R10") - ) - var _REG_debug = []obj.Addr { - jit.Reg("AX"), - jit.Reg("BX"), - jit.Reg("CX"), - jit.Reg("DX"), - jit.Reg("DI"), - jit.Reg("SI"), - jit.Reg("BP"), - jit.Reg("SP"), - jit.Reg("R8"), - jit.Reg("R9"), - jit.Reg("R10"), - jit.Reg("R11"), - jit.Reg("R12"), - jit.Reg("R13"), - jit.Reg("R14"), - jit.Reg("R15"), - } - - func checkptr(ptr uintptr) { - if ptr == 0 { - return - } - fmt.Printf("pointer: %x\n", ptr) - f := checkptrBase(unsafe.Pointer(uintptr(ptr))) - if f == 0 { - fmt.Printf("! unknown-based pointer: %x\n", ptr) - } else if f == 1 { - fmt.Printf("! stack pointer: %x\n", ptr) - } else { - fmt.Printf("base: %x\n", f) - } - findobj(ptr) - } - - func findobj(ptr uintptr) { - base, s, objIndex := findObject(ptr, 0, 0) - if s != nil && base == 0 { - fmt.Printf("! invalid pointer: %x\n", ptr) - } - fmt.Printf("objIndex: %d\n", objIndex) - } - - func (self *_Assembler) check_ptr(ptr obj.Addr, lea bool) { - if !debugCheckPtr { - return - } - - self.dsave(_REG_debug...) - if lea { - self.Emit("LEAQ", ptr, _R10) - } else { - self.Emit("MOVQ", ptr, _R10) - } - self.Emit("MOVQ", _R10, jit.Ptr(_SP, 0)) - self.dcall(_F_checkptr) - self.dload(_REG_debug...) - } - - func printptr(i int, ptr uintptr) { - fmt.Printf("[%d] ptr: %x\n", i, ptr) - } - - func (self *_Assembler) print_ptr(i int, ptr obj.Addr, lea bool) { - self.dsave(_REG_debug...) - if lea { - self.Emit("LEAQ", ptr, _R10) - } else { - self.Emit("MOVQ", ptr, _R10) - } - - self.Emit("MOVQ", jit.Imm(int64(i)), _AX) - self.Emit("MOVQ", _R10, _BX) - self.dcall(_F_printptr) - self.dload(_REG_debug...) - } \ No newline at end of file +import ( + `fmt` + `os` + `runtime` + `strings` + `unsafe` + + `github.com/bytedance/sonic/internal/jit` + `github.com/twitchyliquid64/golang-asm/obj` +) + +const _FP_debug = 128 + +var ( + debugSyncGC = os.Getenv("SONIC_SYNC_GC") != "" + debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" + debugCheckPtr = os.Getenv("SONIC_CHECK_POINTER") != "" +) + +var ( + _Instr_End = newInsOp(_OP_is_nil) + + _F_gc = jit.Func(gc) + _F_println = jit.Func(println_wrapper) + _F_print = jit.Func(print) +) + +func (self *_Assembler) dsave(r ...obj.Addr) { + for i, v := range r { + if i > _FP_debug / 8 - 1 { + panic("too many registers to save") + } else { + self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8)) + } + } +} + +func (self *_Assembler) dload(r ...obj.Addr) { + for i, v := range r { + if i > _FP_debug / 8 - 1 { + panic("too many registers to load") + } else { + self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + _FP_saves + _FP_locals + int64(i) * 8), v) + } + } +} + +func println_wrapper(i int, op1 int, op2 int){ + println(i, " Intrs ", op1, _OpNames[op1], "next: ", op2, _OpNames[op2]) +} + +func print(i int){ + println(i) +} + +func gc() { + if !debugSyncGC { + return + } + runtime.GC() + // debug.FreeOSMemory() +} + +func (self *_Assembler) dcall(fn obj.Addr) { + self.Emit("MOVQ", fn, _R10) // MOVQ ${fn}, R10 + self.Rjmp("CALL", _R10) // CALL R10 +} + +func (self *_Assembler) debug_gc() { + if !debugSyncGC { + return + } + self.dsave(_REG_debug...) + self.dcall(_F_gc) + self.dload(_REG_debug...) +} + +func (self *_Assembler) debug_instr(i int, v *_Instr) { + if debugSyncGC { + if i+1 == len(self.p) { + self.print_gc(i, v, &_Instr_End) + } else { + next := &(self.p[i+1]) + self.print_gc(i, v, next) + name := _OpNames[next.op()] + if strings.Contains(name, "save") { + return + } + } + // self.debug_gc() + } +} + +//go:noescape +//go:linkname checkptrBase runtime.checkptrBase +func checkptrBase(p unsafe.Pointer) uintptr + +//go:noescape +//go:linkname findObject runtime.findObject +func findObject(p, refBase, refOff uintptr) (base uintptr, s unsafe.Pointer, objIndex uintptr) + +var ( + _F_checkptr = jit.Func(checkptr) + _F_printptr = jit.Func(printptr) +) + +var ( + _R10 = jit.Reg("R10") +) +var _REG_debug = []obj.Addr { + jit.Reg("AX"), + jit.Reg("BX"), + jit.Reg("CX"), + jit.Reg("DX"), + jit.Reg("DI"), + jit.Reg("SI"), + jit.Reg("BP"), + jit.Reg("SP"), + jit.Reg("R8"), + jit.Reg("R9"), + jit.Reg("R10"), + jit.Reg("R11"), + jit.Reg("R12"), + jit.Reg("R13"), + jit.Reg("R14"), + jit.Reg("R15"), +} + +func checkptr(ptr uintptr) { + if ptr == 0 { + return + } + fmt.Printf("pointer: %x\n", ptr) + f := checkptrBase(unsafe.Pointer(uintptr(ptr))) + if f == 0 { + fmt.Printf("! unknown-based pointer: %x\n", ptr) + } else if f == 1 { + fmt.Printf("! stack pointer: %x\n", ptr) + } else { + fmt.Printf("base: %x\n", f) + } + findobj(ptr) +} + +func findobj(ptr uintptr) { + base, s, objIndex := findObject(ptr, 0, 0) + if s != nil && base == 0 { + fmt.Printf("! invalid pointer: %x\n", ptr) + } + fmt.Printf("objIndex: %d\n", objIndex) +} + +func (self *_Assembler) check_ptr(ptr obj.Addr, lea bool) { + if !debugCheckPtr { + return + } + + self.dsave(_REG_debug...) + if lea { + self.Emit("LEAQ", ptr, _R10) + } else { + self.Emit("MOVQ", ptr, _R10) + } + self.Emit("MOVQ", _R10, jit.Ptr(_SP, 0)) + self.dcall(_F_checkptr) + self.dload(_REG_debug...) +} + +func printptr(i int, ptr uintptr) { + fmt.Printf("[%d] ptr: %x\n", i, ptr) +} + +func (self *_Assembler) print_ptr(i int, ptr obj.Addr, lea bool) { + self.dsave(_REG_debug...) + if lea { + self.Emit("LEAQ", ptr, _R10) + } else { + self.Emit("MOVQ", ptr, _R10) + } + + self.Emit("MOVQ", jit.Imm(int64(i)), _AX) + self.Emit("MOVQ", _R10, _BX) + self.dcall(_F_printptr) + self.dload(_REG_debug...) +} \ No newline at end of file diff --git a/encoder/stream_test.go b/encoder/stream_test.go index a1fd6c2..62d1c88 100644 --- a/encoder/stream_test.go +++ b/encoder/stream_test.go @@ -26,41 +26,41 @@ import ( `github.com/stretchr/testify/require` ) - func TestEncodeStream(t *testing.T) { - var o = map[string]interface{}{ - "a": "<>", - "b": json.RawMessage(" [ ] "), - } - var w1 = bytes.NewBuffer(nil) - var w2 = bytes.NewBuffer(nil) - var enc1 = json.NewEncoder(w1) - var enc2 = NewStreamEncoder(w2) - enc2.SetEscapeHTML(true) - enc2.SortKeys() - enc2.SetCompactMarshaler(true) +func TestEncodeStream(t *testing.T) { + var o = map[string]interface{}{ + "a": "<>", + "b": json.RawMessage(" [ ] "), + } + var w1 = bytes.NewBuffer(nil) + var w2 = bytes.NewBuffer(nil) + var enc1 = json.NewEncoder(w1) + var enc2 = NewStreamEncoder(w2) + enc2.SetEscapeHTML(true) + enc2.SortKeys() + enc2.SetCompactMarshaler(true) - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) - enc1.SetEscapeHTML(true) - enc2.SetEscapeHTML(true) - enc1.SetIndent("你好", "\b") - enc2.SetIndent("你好", "\b") - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) - - enc1.SetEscapeHTML(false) - enc2.SetEscapeHTML(false) - enc1.SetIndent("", "") - enc2.SetIndent("", "") - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) - } + enc1.SetEscapeHTML(true) + enc2.SetEscapeHTML(true) + enc1.SetIndent("你好", "\b") + enc2.SetIndent("你好", "\b") + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) - func BenchmarkEncodeStream_Sonic(b *testing.B) { + enc1.SetEscapeHTML(false) + enc2.SetEscapeHTML(false) + enc1.SetIndent("", "") + enc2.SetIndent("", "") + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) +} + +func BenchmarkEncodeStream_Sonic(b *testing.B) { var o = map[string]interface{}{ "a": `<`+strings.Repeat("1", 1024)+`>`, "b": json.RawMessage(` [ `+strings.Repeat(" ", 1024)+` ] `), @@ -99,9 +99,9 @@ import ( w.Reset() } }) - } +} - func BenchmarkEncodeStream_Std(b *testing.B) { +func BenchmarkEncodeStream_Std(b *testing.B) { var o = map[string]interface{}{ "a": `<`+strings.Repeat("1", 1024)+`>`, "b": json.RawMessage(` [ `+strings.Repeat(" ", 1024)+` ] `), @@ -138,9 +138,9 @@ import ( w.Reset() } }) - } +} - func BenchmarkEncodeStream_Jsoniter(b *testing.B) { +func BenchmarkEncodeStream_Jsoniter(b *testing.B) { var o = map[string]interface{}{ "a": `<`+strings.Repeat("1", 1024)+`>`, "b": json.RawMessage(` [ `+strings.Repeat(" ", 1024)+` ] `), @@ -187,4 +187,4 @@ import ( w.Reset() } }) - } \ No newline at end of file +} \ No newline at end of file diff --git a/issue_test/issue112_test.go b/issue_test/issue112_test.go index 7ee88ea..998d8eb 100644 --- a/issue_test/issue112_test.go +++ b/issue_test/issue112_test.go @@ -14,14 +14,14 @@ * limitations under the License. */ - package issue_test +package issue_test - import ( +import ( `testing` . `github.com/bytedance/sonic` `github.com/stretchr/testify/require` - ) +) func TestUnmarshalInvalidBase64EncodedString(t *testing.T) { var obj []byte diff --git a/issue_test/issue115_test.go b/issue_test/issue115_test.go index 94094fe..26a6aad 100644 --- a/issue_test/issue115_test.go +++ b/issue_test/issue115_test.go @@ -14,9 +14,9 @@ * limitations under the License. */ - package issue_test +package issue_test - import ( +import ( `encoding` `encoding/json` `strconv` diff --git a/tools/asm2asm b/tools/asm2asm index 09224ab..5e85f0d 160000 --- a/tools/asm2asm +++ b/tools/asm2asm @@ -1 +1 @@ -Subproject commit 09224ab8c109bdb8da13af04abd7c01cb6e38d87 +Subproject commit 5e85f0dbbd2eb4768d8413c326e5540612c86fae