mirror of
https://github.com/ii64/sonic.git
synced 2026-06-21 00:46:43 +08:00
feat: Pretouch recursively for large/deep struct (#137)
This commit is contained in:
parent
9a95e9de00
commit
49fc705341
10 changed files with 308 additions and 48 deletions
|
|
@ -670,7 +670,7 @@ type JsonStruct struct {
|
||||||
func TestAssembler_DecodeStruct(t *testing.T) {
|
func TestAssembler_DecodeStruct(t *testing.T) {
|
||||||
var v JsonStruct
|
var v JsonStruct
|
||||||
s := `{"A": 123, "B": "asdf", "C": {"qwer": 4567}, "D": [1, 2, 3, 4, 5]}`
|
s := `{"A": 123, "B": "asdf", "C": {"qwer": 4567}, "D": [1, 2, 3, 4, 5]}`
|
||||||
p, err := make(_Compiler).compile(reflect.TypeOf(v))
|
p, err := newCompiler().compile(reflect.TypeOf(v))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
k := new(_Stack)
|
k := new(_Stack)
|
||||||
a := newAssembler(p)
|
a := newAssembler(p)
|
||||||
|
|
@ -693,7 +693,7 @@ type Tx struct {
|
||||||
func TestAssembler_DecodeStruct_SinglePrivateField(t *testing.T) {
|
func TestAssembler_DecodeStruct_SinglePrivateField(t *testing.T) {
|
||||||
var v Tx
|
var v Tx
|
||||||
s := `{"x": 1}`
|
s := `{"x": 1}`
|
||||||
p, err := make(_Compiler).compile(reflect.TypeOf(v))
|
p, err := newCompiler().compile(reflect.TypeOf(v))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
k := new(_Stack)
|
k := new(_Stack)
|
||||||
a := newAssembler(p)
|
a := newAssembler(p)
|
||||||
|
|
@ -707,7 +707,7 @@ func TestAssembler_DecodeStruct_SinglePrivateField(t *testing.T) {
|
||||||
func TestAssembler_DecodeByteSlice_Bin(t *testing.T) {
|
func TestAssembler_DecodeByteSlice_Bin(t *testing.T) {
|
||||||
var v []byte
|
var v []byte
|
||||||
s := `"aGVsbG8sIHdvcmxk"`
|
s := `"aGVsbG8sIHdvcmxk"`
|
||||||
p, err := make(_Compiler).compile(reflect.TypeOf(v))
|
p, err := newCompiler().compile(reflect.TypeOf(v))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
k := new(_Stack)
|
k := new(_Stack)
|
||||||
a := newAssembler(p)
|
a := newAssembler(p)
|
||||||
|
|
@ -721,7 +721,7 @@ func TestAssembler_DecodeByteSlice_Bin(t *testing.T) {
|
||||||
func TestAssembler_DecodeByteSlice_List(t *testing.T) {
|
func TestAssembler_DecodeByteSlice_List(t *testing.T) {
|
||||||
var v []byte
|
var v []byte
|
||||||
s := `[104, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100]`
|
s := `[104, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100]`
|
||||||
p, err := make(_Compiler).compile(reflect.TypeOf(v))
|
p, err := newCompiler().compile(reflect.TypeOf(v))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
k := new(_Stack)
|
k := new(_Stack)
|
||||||
a := newAssembler(p)
|
a := newAssembler(p)
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ import (
|
||||||
`github.com/bytedance/sonic/internal/caching`
|
`github.com/bytedance/sonic/internal/caching`
|
||||||
`github.com/bytedance/sonic/internal/resolver`
|
`github.com/bytedance/sonic/internal/resolver`
|
||||||
`github.com/bytedance/sonic/internal/rt`
|
`github.com/bytedance/sonic/internal/rt`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
)
|
)
|
||||||
|
|
||||||
type _Op uint8
|
type _Op uint8
|
||||||
|
|
@ -389,7 +390,7 @@ func (self _Instr) formatStructFields() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
type (
|
type (
|
||||||
_Program []_Instr
|
_Program []_Instr
|
||||||
)
|
)
|
||||||
|
|
||||||
func (self _Program) pc() int {
|
func (self _Program) pc() int {
|
||||||
|
|
@ -474,11 +475,27 @@ func (self _Program) disassemble() string {
|
||||||
return strings.Join(append(ret, "\tend"), "\n")
|
return strings.Join(append(ret, "\tend"), "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
type (
|
type _Compiler struct {
|
||||||
_Compiler map[reflect.Type]bool
|
opts option.CompileOptions
|
||||||
)
|
tab map[reflect.Type]bool
|
||||||
|
rec map[reflect.Type]bool
|
||||||
|
}
|
||||||
|
|
||||||
func (self _Compiler) rescue(ep *error) {
|
func newCompiler() *_Compiler {
|
||||||
|
return &_Compiler {
|
||||||
|
tab: map[reflect.Type]bool{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (self *_Compiler) apply(opts option.CompileOptions) *_Compiler {
|
||||||
|
self.opts = opts
|
||||||
|
if self.opts.RecursiveDepth > 0 {
|
||||||
|
self.rec = map[reflect.Type]bool{}
|
||||||
|
}
|
||||||
|
return self
|
||||||
|
}
|
||||||
|
|
||||||
|
func (self *_Compiler) rescue(ep *error) {
|
||||||
if val := recover(); val != nil {
|
if val := recover(); val != nil {
|
||||||
if err, ok := val.(error); ok {
|
if err, ok := val.(error); ok {
|
||||||
*ep = err
|
*ep = err
|
||||||
|
|
@ -488,14 +505,14 @@ func (self _Compiler) rescue(ep *error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compile(vt reflect.Type) (ret _Program, err error) {
|
func (self *_Compiler) compile(vt reflect.Type) (ret _Program, err error) {
|
||||||
defer self.rescue(&err)
|
defer self.rescue(&err)
|
||||||
self.compileOne(&ret, 0, vt)
|
self.compileOne(&ret, 0, vt)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileOne(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) {
|
||||||
ok := self[vt]
|
ok := self.tab[vt]
|
||||||
pt := reflect.PtrTo(vt)
|
pt := reflect.PtrTo(vt)
|
||||||
|
|
||||||
/* check for recursive nesting */
|
/* check for recursive nesting */
|
||||||
|
|
@ -533,12 +550,12 @@ func (self _Compiler) compileOne(p *_Program, sp int, vt reflect.Type) {
|
||||||
|
|
||||||
/* enter the recursion */
|
/* enter the recursion */
|
||||||
p.add(_OP_lspace)
|
p.add(_OP_lspace)
|
||||||
self[vt] = true
|
self.tab[vt] = true
|
||||||
self.compileOps(p, sp, vt)
|
self.compileOps(p, sp, vt)
|
||||||
delete(self, vt)
|
delete(self.tab, vt)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
|
||||||
switch vt.Kind() {
|
switch vt.Kind() {
|
||||||
case reflect.Bool : self.compilePrimitive (p, _OP_bool)
|
case reflect.Bool : self.compilePrimitive (p, _OP_bool)
|
||||||
case reflect.Int : self.compilePrimitive (p, _OP_int())
|
case reflect.Int : self.compilePrimitive (p, _OP_int())
|
||||||
|
|
@ -565,7 +582,7 @@ func (self _Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
|
||||||
if reflect.PtrTo(vt.Key()).Implements(encodingTextUnmarshalerType) {
|
if reflect.PtrTo(vt.Key()).Implements(encodingTextUnmarshalerType) {
|
||||||
self.compileMapOp(p, sp, vt, _OP_map_key_utext_p)
|
self.compileMapOp(p, sp, vt, _OP_map_key_utext_p)
|
||||||
} else if vt.Key().Implements(encodingTextUnmarshalerType) {
|
} else if vt.Key().Implements(encodingTextUnmarshalerType) {
|
||||||
|
|
@ -575,7 +592,7 @@ func (self _Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileMapUt(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileMapUt(p *_Program, sp int, vt reflect.Type) {
|
||||||
switch vt.Key().Kind() {
|
switch vt.Key().Kind() {
|
||||||
case reflect.Int : self.compileMapOp(p, sp, vt, _OP_map_key_int())
|
case reflect.Int : self.compileMapOp(p, sp, vt, _OP_map_key_int())
|
||||||
case reflect.Int8 : self.compileMapOp(p, sp, vt, _OP_map_key_i8)
|
case reflect.Int8 : self.compileMapOp(p, sp, vt, _OP_map_key_i8)
|
||||||
|
|
@ -595,7 +612,7 @@ func (self _Compiler) compileMapUt(p *_Program, sp int, vt reflect.Type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op) {
|
func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.tag(sp + 1)
|
p.tag(sp + 1)
|
||||||
|
|
@ -649,7 +666,7 @@ func (self _Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op)
|
||||||
p.pin(x)
|
p.pin(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compilePtr(p *_Program, sp int, et reflect.Type) {
|
func (self *_Compiler) compilePtr(p *_Program, sp int, et reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
|
|
||||||
|
|
@ -668,7 +685,7 @@ func (self _Compiler) compilePtr(p *_Program, sp int, et reflect.Type) {
|
||||||
p.pin(j)
|
p.pin(j)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileArray(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type) {
|
||||||
x := p.pc()
|
x := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.tag(sp)
|
p.tag(sp)
|
||||||
|
|
@ -708,7 +725,7 @@ func (self _Compiler) compileArray(p *_Program, sp int, vt reflect.Type) {
|
||||||
p.pin(x)
|
p.pin(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileSlice(p *_Program, sp int, et reflect.Type) {
|
func (self *_Compiler) compileSlice(p *_Program, sp int, et reflect.Type) {
|
||||||
if et.Kind() == byteType.Kind() {
|
if et.Kind() == byteType.Kind() {
|
||||||
self.compileSliceBin(p, sp, et)
|
self.compileSliceBin(p, sp, et)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -716,7 +733,7 @@ func (self _Compiler) compileSlice(p *_Program, sp int, et reflect.Type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileSliceBin(p *_Program, sp int, et reflect.Type) {
|
func (self *_Compiler) compileSliceBin(p *_Program, sp int, et reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
j := p.pc()
|
j := p.pc()
|
||||||
|
|
@ -738,7 +755,7 @@ func (self _Compiler) compileSliceBin(p *_Program, sp int, et reflect.Type) {
|
||||||
p.pin(y)
|
p.pin(y)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileSliceList(p *_Program, sp int, et reflect.Type) {
|
func (self *_Compiler) compileSliceList(p *_Program, sp int, et reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.tag(sp)
|
p.tag(sp)
|
||||||
|
|
@ -751,7 +768,7 @@ func (self _Compiler) compileSliceList(p *_Program, sp int, et reflect.Type) {
|
||||||
p.pin(x)
|
p.pin(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) {
|
func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) {
|
||||||
p.rtt(_OP_slice_init, et)
|
p.rtt(_OP_slice_init, et)
|
||||||
p.add(_OP_save)
|
p.add(_OP_save)
|
||||||
p.add(_OP_lspace)
|
p.add(_OP_lspace)
|
||||||
|
|
@ -774,7 +791,7 @@ func (self _Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) {
|
||||||
p.add(_OP_drop)
|
p.add(_OP_drop)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileString(p *_Program, vt reflect.Type) {
|
func (self *_Compiler) compileString(p *_Program, vt reflect.Type) {
|
||||||
if vt == jsonNumberType {
|
if vt == jsonNumberType {
|
||||||
self.compilePrimitive(p, _OP_num)
|
self.compilePrimitive(p, _OP_num)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -782,7 +799,7 @@ func (self _Compiler) compileString(p *_Program, vt reflect.Type) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileStringBody(p *_Program) {
|
func (self *_Compiler) compileStringBody(p *_Program) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.chr(_OP_match_char, '"')
|
p.chr(_OP_match_char, '"')
|
||||||
|
|
@ -790,15 +807,18 @@ func (self _Compiler) compileStringBody(p *_Program) {
|
||||||
p.pin(i)
|
p.pin(i)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
|
||||||
if sp >= _MAX_STACK || p.pc() >= _MAX_ILBUF {
|
if sp >= _MAX_STACK || p.pc() >= _MAX_ILBUF {
|
||||||
p.rtt(_OP_recurse, vt)
|
p.rtt(_OP_recurse, vt)
|
||||||
|
if self.opts.RecursiveDepth > 0 {
|
||||||
|
self.rec[vt] = true
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
self.compileStructBody(p, sp, vt)
|
self.compileStructBody(p, sp, vt)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileStructBody(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileStructBody(p *_Program, sp int, vt reflect.Type) {
|
||||||
fv := resolver.ResolveStruct(vt)
|
fv := resolver.ResolveStruct(vt)
|
||||||
fm, sw := caching.CreateFieldMap(len(fv)), make([]int, len(fv))
|
fm, sw := caching.CreateFieldMap(len(fv)), make([]int, len(fv))
|
||||||
|
|
||||||
|
|
@ -870,7 +890,7 @@ end_of_object:
|
||||||
p.pin(n)
|
p.pin(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) {
|
||||||
n1 := -1
|
n1 := -1
|
||||||
ft := vt
|
ft := vt
|
||||||
sv := false
|
sv := false
|
||||||
|
|
@ -977,7 +997,7 @@ func (self _Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type
|
||||||
p.pin(pc)
|
p.pin(pc)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileInterface(p *_Program, vt reflect.Type) {
|
func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
|
|
||||||
|
|
@ -996,14 +1016,14 @@ func (self _Compiler) compileInterface(p *_Program, vt reflect.Type) {
|
||||||
p.pin(j)
|
p.pin(j)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compilePrimitive(p *_Program, op _Op) {
|
func (self *_Compiler) compilePrimitive(p *_Program, op _Op) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.add(op)
|
p.add(op)
|
||||||
p.pin(i)
|
p.pin(i)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) {
|
func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) {
|
||||||
j := p.pc()
|
j := p.pc()
|
||||||
k := vt.Kind()
|
k := vt.Kind()
|
||||||
|
|
||||||
|
|
@ -1020,7 +1040,7 @@ func (self _Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) {
|
||||||
p.pin(j)
|
p.pin(j)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileUnmarshalJson(p *_Program, vt reflect.Type) {
|
func (self *_Compiler) compileUnmarshalJson(p *_Program, vt reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
v := _OP_unmarshal
|
v := _OP_unmarshal
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
|
|
@ -1035,7 +1055,7 @@ func (self _Compiler) compileUnmarshalJson(p *_Program, vt reflect.Type) {
|
||||||
self.compileUnmarshalEnd(p, vt, i)
|
self.compileUnmarshalEnd(p, vt, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileUnmarshalText(p *_Program, vt reflect.Type) {
|
func (self *_Compiler) compileUnmarshalText(p *_Program, vt reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
v := _OP_unmarshal_text
|
v := _OP_unmarshal_text
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
|
|
@ -1052,7 +1072,7 @@ func (self _Compiler) compileUnmarshalText(p *_Program, vt reflect.Type) {
|
||||||
self.compileUnmarshalEnd(p, vt, i)
|
self.compileUnmarshalEnd(p, vt, i)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self _Compiler) compileUnmarshalTextPtr(p *_Program, vt reflect.Type) {
|
func (self *_Compiler) compileUnmarshalTextPtr(p *_Program, vt reflect.Type) {
|
||||||
i := p.pc()
|
i := p.pc()
|
||||||
p.add(_OP_is_null)
|
p.add(_OP_is_null)
|
||||||
p.chr(_OP_match_char, '"')
|
p.chr(_OP_match_char, '"')
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCompiler_Compile(t *testing.T) {
|
func TestCompiler_Compile(t *testing.T) {
|
||||||
prg, err := make(_Compiler).compile(reflect.TypeOf(TwitterStruct{}))
|
prg, err := newCompiler().compile(reflect.TypeOf(TwitterStruct{}))
|
||||||
assert.Nil(t, err)
|
assert.Nil(t, err)
|
||||||
prg.disassemble()
|
prg.disassemble()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ import (
|
||||||
`runtime`
|
`runtime`
|
||||||
|
|
||||||
`github.com/bytedance/sonic/internal/rt`
|
`github.com/bytedance/sonic/internal/rt`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -106,7 +107,56 @@ func (self *Decoder) DisallowUnknownFields() {
|
||||||
|
|
||||||
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
||||||
// order to reduce the first-hit latency.
|
// order to reduce the first-hit latency.
|
||||||
func Pretouch(vt reflect.Type) (err error) {
|
//
|
||||||
_, err = findOrCompile(rt.UnpackType(vt))
|
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
|
||||||
return
|
// a compile option to set the depth of recursive compile for the nested struct type.
|
||||||
|
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
|
||||||
|
cfg := option.DefaultCompileOptions()
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&cfg)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return pretouchRec(map[reflect.Type]bool{vt:true}, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Type]bool, error) {
|
||||||
|
/* compile function */
|
||||||
|
compiler := newCompiler().apply(opts)
|
||||||
|
|
||||||
|
/* compile function */
|
||||||
|
decoder := func(vt *rt.GoType) (interface{}, error) {
|
||||||
|
if pp, err := compiler.compile(_vt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else {
|
||||||
|
return newAssembler(pp).Load(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* find or compile */
|
||||||
|
vt := rt.UnpackType(_vt)
|
||||||
|
if val := programCache.Get(vt); val != nil {
|
||||||
|
return nil, nil
|
||||||
|
} else if _, err := programCache.Compute(vt, decoder); err == nil {
|
||||||
|
return compiler.rec, nil
|
||||||
|
} else {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pretouchRec(vtm map[reflect.Type]bool, opts option.CompileOptions) error {
|
||||||
|
if opts.RecursiveDepth < 0 || len(vtm) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
next := make(map[reflect.Type]bool)
|
||||||
|
for vt, _ := range(vtm) {
|
||||||
|
sub, err := pretouchType(vt, opts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for svt, _ := range(sub) {
|
||||||
|
next[svt] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts.RecursiveDepth -= 1
|
||||||
|
return pretouchRec(next, opts)
|
||||||
}
|
}
|
||||||
|
|
@ -115,7 +115,7 @@ func referenceFields(v *caching.FieldMap) int64 {
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeDecoder(vt *rt.GoType) (interface{}, error) {
|
func makeDecoder(vt *rt.GoType) (interface{}, error) {
|
||||||
if pp, err := make(_Compiler).compile(vt.Pack()); err != nil {
|
if pp, err := newCompiler().compile(vt.Pack()); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
} else {
|
} else {
|
||||||
return newAssembler(pp).Load(), nil
|
return newAssembler(pp).Load(), nil
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ import (
|
||||||
|
|
||||||
`github.com/bytedance/sonic/internal/resolver`
|
`github.com/bytedance/sonic/internal/resolver`
|
||||||
`github.com/bytedance/sonic/internal/rt`
|
`github.com/bytedance/sonic/internal/rt`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
)
|
)
|
||||||
|
|
||||||
type _Op uint8
|
type _Op uint8
|
||||||
|
|
@ -371,8 +372,10 @@ func (self _Program) disassemble() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
type _Compiler struct {
|
type _Compiler struct {
|
||||||
pv bool
|
opts option.CompileOptions
|
||||||
tab map[reflect.Type]bool
|
pv bool
|
||||||
|
tab map[reflect.Type]bool
|
||||||
|
rec map[reflect.Type]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func newCompiler() *_Compiler {
|
func newCompiler() *_Compiler {
|
||||||
|
|
@ -381,6 +384,14 @@ func newCompiler() *_Compiler {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (self *_Compiler) apply(opts option.CompileOptions) *_Compiler {
|
||||||
|
self.opts = opts
|
||||||
|
if self.opts.RecursiveDepth > 0 {
|
||||||
|
self.rec = map[reflect.Type]bool{}
|
||||||
|
}
|
||||||
|
return self
|
||||||
|
}
|
||||||
|
|
||||||
func (self *_Compiler) rescue(ep *error) {
|
func (self *_Compiler) rescue(ep *error) {
|
||||||
if val := recover(); val != nil {
|
if val := recover(); val != nil {
|
||||||
if err, ok := val.(error); ok {
|
if err, ok := val.(error); ok {
|
||||||
|
|
@ -645,6 +656,9 @@ func (self *_Compiler) compileString(p *_Program, vt reflect.Type) {
|
||||||
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
|
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
|
||||||
if sp >= _MAX_STACK || p.pc() >= _MAX_ILBUF {
|
if sp >= _MAX_STACK || p.pc() >= _MAX_ILBUF {
|
||||||
p.rtt(_OP_recurse, vt)
|
p.rtt(_OP_recurse, vt)
|
||||||
|
if self.opts.RecursiveDepth > 0 {
|
||||||
|
self.rec[vt] = true
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
self.compileStructBody(p, sp, vt)
|
self.compileStructBody(p, sp, vt)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ import (
|
||||||
`runtime`
|
`runtime`
|
||||||
|
|
||||||
`github.com/bytedance/sonic/internal/rt`
|
`github.com/bytedance/sonic/internal/rt`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
)
|
)
|
||||||
|
|
||||||
// Options is a set of encoding options.
|
// Options is a set of encoding options.
|
||||||
|
|
@ -150,7 +151,54 @@ func EncodeIndented(val interface{}, prefix string, indent string, opts Options)
|
||||||
|
|
||||||
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
||||||
// order to reduce the first-hit latency.
|
// order to reduce the first-hit latency.
|
||||||
func Pretouch(vt reflect.Type) (err error) {
|
//
|
||||||
_, err = findOrCompile(rt.UnpackType(vt))
|
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
|
||||||
return
|
// a compile option to set the depth of recursive compile for the nested struct type.
|
||||||
|
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
|
||||||
|
cfg := option.DefaultCompileOptions()
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&cfg)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return pretouchRec(map[reflect.Type]bool{vt:true}, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Type]bool, error) {
|
||||||
|
/* compile function */
|
||||||
|
compiler := newCompiler().apply(opts)
|
||||||
|
encoder := func(vt *rt.GoType) (interface{}, error) {
|
||||||
|
if pp, err := compiler.compile(_vt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else {
|
||||||
|
return newAssembler(pp).Load(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* find or compile */
|
||||||
|
vt := rt.UnpackType(_vt)
|
||||||
|
if val := programCache.Get(vt); val != nil {
|
||||||
|
return nil, nil
|
||||||
|
} else if _, err := programCache.Compute(vt, encoder); err == nil {
|
||||||
|
return compiler.rec, nil
|
||||||
|
} else {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pretouchRec(vtm map[reflect.Type]bool, opts option.CompileOptions) error {
|
||||||
|
if opts.RecursiveDepth < 0 || len(vtm) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
next := make(map[reflect.Type]bool)
|
||||||
|
for vt, _ := range(vtm) {
|
||||||
|
sub, err := pretouchType(vt, opts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for svt, _ := range(sub) {
|
||||||
|
next[svt] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts.RecursiveDepth -= 1
|
||||||
|
return pretouchRec(next, opts)
|
||||||
|
}
|
||||||
79
issue_test/issue138_test.go
Normal file
79
issue_test/issue138_test.go
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2021 ByteDance Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package issue_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
`fmt`
|
||||||
|
`time`
|
||||||
|
`testing`
|
||||||
|
`reflect`
|
||||||
|
|
||||||
|
`github.com/stretchr/testify/require`
|
||||||
|
`github.com/bytedance/sonic`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
|
)
|
||||||
|
|
||||||
|
type Issue138_DeepStruct struct {
|
||||||
|
L0 struct {
|
||||||
|
L1 struct {
|
||||||
|
L2 struct {
|
||||||
|
L3 struct {
|
||||||
|
L4 struct {
|
||||||
|
L5 struct {
|
||||||
|
L6 struct {
|
||||||
|
L7 struct {
|
||||||
|
L8 struct {
|
||||||
|
L9 struct {
|
||||||
|
L10 struct {
|
||||||
|
L11 struct {
|
||||||
|
L12 struct {
|
||||||
|
L13 struct {
|
||||||
|
L14 struct {
|
||||||
|
L15 struct {
|
||||||
|
L16 struct {
|
||||||
|
L17 struct {
|
||||||
|
L18 struct {
|
||||||
|
L19 struct {
|
||||||
|
L20 struct {
|
||||||
|
L21 struct {
|
||||||
|
L22 struct {
|
||||||
|
A int
|
||||||
|
B string
|
||||||
|
C []float64
|
||||||
|
E map[string]bool
|
||||||
|
F *Issue138_DeepStruct
|
||||||
|
}}}}}}}}}}}}}}}}}}}}}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPretouchTime(depth int) {
|
||||||
|
start := time.Now()
|
||||||
|
sonic.Pretouch(reflect.TypeOf(Issue138_DeepStruct{}), option.WithCompileRecursiveDepth(depth))
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
fmt.Printf("Pretouch with recursive depth %d, time is %s\n", depth, elapsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIssue138_PretouchTime(t *testing.T) {
|
||||||
|
testPretouchTime(4)
|
||||||
|
var obj Issue138_DeepStruct
|
||||||
|
start := time.Now()
|
||||||
|
data, err := sonic.Marshal(obj)
|
||||||
|
err = sonic.Unmarshal([]byte(data), &obj)
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
fmt.Printf("Marshal and unmarshal time is %s\n", elapsed)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
45
option/option.go
Normal file
45
option/option.go
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2021 ByteDance Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package option
|
||||||
|
|
||||||
|
// CompileOptions includes all options for encoder or decoder compiler.
|
||||||
|
type CompileOptions struct {
|
||||||
|
// the depth for recursive compile
|
||||||
|
RecursiveDepth int
|
||||||
|
}
|
||||||
|
|
||||||
|
func DefaultCompileOptions() CompileOptions {
|
||||||
|
return CompileOptions{
|
||||||
|
RecursiveDepth: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type CompileOption func(o *CompileOptions)
|
||||||
|
|
||||||
|
// WithCompileRecursiveDepth sets the depth of recursive compile
|
||||||
|
// in decoder or encoder.
|
||||||
|
//
|
||||||
|
// Default value(0) is suitable for basic types and small nested struct types.
|
||||||
|
//
|
||||||
|
// For large or deep nested struct, try to set larger depth to reduce compile
|
||||||
|
// time in the first Marshal or Unmarshal.
|
||||||
|
func WithCompileRecursiveDepth(depth int) CompileOption {
|
||||||
|
return func(o *CompileOptions) {
|
||||||
|
o.RecursiveDepth = depth
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
10
sonic.go
10
sonic.go
|
|
@ -23,6 +23,7 @@ import (
|
||||||
`github.com/bytedance/sonic/ast`
|
`github.com/bytedance/sonic/ast`
|
||||||
`github.com/bytedance/sonic/decoder`
|
`github.com/bytedance/sonic/decoder`
|
||||||
`github.com/bytedance/sonic/encoder`
|
`github.com/bytedance/sonic/encoder`
|
||||||
|
`github.com/bytedance/sonic/option`
|
||||||
`github.com/bytedance/sonic/internal/native/types`
|
`github.com/bytedance/sonic/internal/native/types`
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -74,10 +75,13 @@ func UnmarshalString(buf string, val interface{}) error {
|
||||||
|
|
||||||
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
|
||||||
// order to reduce the first-hit latency.
|
// order to reduce the first-hit latency.
|
||||||
func Pretouch(vt reflect.Type) error {
|
//
|
||||||
if err := encoder.Pretouch(vt); err != nil {
|
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
|
||||||
|
// a compile option to set the depth of recursive compile for the nested struct type.
|
||||||
|
func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
|
||||||
|
if err := encoder.Pretouch(vt, opts...); err != nil {
|
||||||
return err
|
return err
|
||||||
} else if err = decoder.Pretouch(vt); err != nil {
|
} else if err = decoder.Pretouch(vt, opts...); err != nil {
|
||||||
return err
|
return err
|
||||||
} else {
|
} else {
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue