XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-21 00:46:43 +08:00
Code Issues Projects Releases Wiki Activity

fix:(encoder) pass pv through compiler recursively (#336)

Browse source 
* pass pv

* test reflect indirect

* fix: pass `pv` throught compiler recursively
This commit is contained in:
Yi Duan 2022-12-13 14:26:58 +08:00 committed by GitHub
parent 01c0d36194
commit f421ee8530
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 191 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
decoder/decoder.go
View file

@ -179,7 +179,7 @@ func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Type]bool, error) {
/* compile function */
compiler := newCompiler().apply(opts)
decoder := func(vt *rt.GoType) (interface{}, error) {
decoder := func(vt *rt.GoType, _ ...interface{}) (interface{}, error) {
if pp, err := compiler.compile(_vt); err != nil {
return nil, err
} else {

2
decoder/pools.go
View file

@ -147,7 +147,7 @@ func referenceFields(v *caching.FieldMap) int64 {
return int64(uintptr(unsafe.Pointer(v)))
}
func makeDecoder(vt *rt.GoType) (interface{}, error) {
func makeDecoder(vt *rt.GoType, _ ...interface{}) (interface{}, error) {
if pp, err := newCompiler().compile(vt.Pack()); err != nil {
return nil, err
} else {

8
encoder/assembler_amd64_go116.go
View file

@ -1011,11 +1011,12 @@ func (self *_Assembler) _asm_OP_drop_2(_ *_Instr) {
func (self *_Assembler) _asm_OP_recurse(p *_Instr) {
self.prep_buffer() // MOVE {buf}, (SP)
self.Emit("MOVQ", jit.Type(p.vt()), _AX) // MOVQ $(type(p.vt())), AX
vt, pv := p.vp()
self.Emit("MOVQ", jit.Type(vt), _AX) // MOVQ $(type(p.vt())), AX
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP)
/* check for indirection */
if (p.vf() & rt.F_direct) != 0 {
if !rt.UnpackType(vt).Indirect() {
self.Emit("MOVQ", _SP_p, _AX) // MOVQ SP.p, AX
} else {
self.Emit("MOVQ", _SP_p, _VAR_vp) // MOVQ SP.p, 48(SP)
@ -1026,6 +1027,9 @@ func (self *_Assembler) _asm_OP_recurse(p *_Instr) {
self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP)
self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24)) // MOVQ ST, 24(SP)
self.Emit("MOVQ" , _ARG_fv, _AX) // MOVQ fv, AX
if pv {
self.Emit("BTCQ", jit.Imm(bitPointerValue), _AX) // BTCQ $1, AX
}
self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP)
self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer
self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET) // MOVQ 40(SP), ET

8
encoder/assembler_amd64_go117.go
View file

@ -1015,10 +1015,11 @@ func (self *_Assembler) _asm_OP_drop_2(_ *_Instr) {
func (self *_Assembler) _asm_OP_recurse(p *_Instr) {
self.prep_buffer_AX() // MOVE {buf}, (SP)
self.Emit("MOVQ", jit.Type(p.vt()), _BX) // MOVQ $(type(p.vt())), BX
vt, pv := p.vp()
self.Emit("MOVQ", jit.Type(vt), _BX) // MOVQ $(type(p.vt())), BX
/* check for indirection */
if (p.vf() & rt.F_direct) != 0 {
if !rt.UnpackType(vt).Indirect() {
self.Emit("MOVQ", _SP_p, _CX) // MOVQ SP.p, CX
} else {
self.Emit("MOVQ", _SP_p, _VAR_vp) // MOVQ SP.p, VAR.vp
@ -1028,6 +1029,9 @@ func (self *_Assembler) _asm_OP_recurse(p *_Instr) {
/* call the encoder */
self.Emit("MOVQ" , _ST, _DI) // MOVQ ST, DI
self.Emit("MOVQ" , _ARG_fv, _SI) // MOVQ $fv, SI
if pv {
self.Emit("BTCQ", jit.Imm(bitPointerValue), _SI) // BTCQ $1, SI
}
self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer
self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
self.Sjmp("JNZ" , _LB_error) // JNZ _error

4
encoder/assembler_test.go
View file

@ -52,7 +52,7 @@ func TestEncoderMemoryCorruption(t *testing.T) {
}
func TestAssembler_CompileAndLoad(t *testing.T) {
p, err := newCompiler().compile(reflect.TypeOf((*bool)(nil)))
p, err := newCompiler().compile(reflect.TypeOf((*bool)(nil)), true)
assert.Nil(t, err)
a := newAssembler(p)
f := a.Load()
@ -127,7 +127,7 @@ type RecursiveValue struct {
}
func mustCompile(t interface{}) _Program {
p, err := newCompiler().compile(reflect.TypeOf(t))
p, err := newCompiler().compile(reflect.TypeOf(t), !rt.UnpackEface(t).Type.Indirect())
if err != nil {
panic(err)
}

39
encoder/compiler.go
View file

@ -217,6 +217,17 @@ func newInsVt(op _Op, vt reflect.Type) _Instr {
}
}
func newInsVp(op _Op, vt reflect.Type, pv bool) _Instr {
i := 0
if pv {
i = 1
}
return _Instr {
u: packOp(op) | rt.PackInt(i),
p: unsafe.Pointer(rt.UnpackType(vt)),
}
}
func (self _Instr) op() _Op {
return _Op(self.u >> 56)
}
@ -243,6 +254,10 @@ func (self _Instr) vt() reflect.Type {
return (*rt.GoType)(self.p).Pack()
}
func (self _Instr) vp() (vt reflect.Type, pv bool) {
return (*rt.GoType)(self.p).Pack(), rt.UnpackInt(self.u) == 1
}
func (self _Instr) i64() int64 {
return int64(self.vi())
}
@ -345,6 +360,10 @@ func (self *_Program) rtt(op _Op, vt reflect.Type) {
*self = append(*self, newInsVt(op, vt))
}
func (self *_Program) vp(op _Op, vt reflect.Type, pv bool) {
*self = append(*self, newInsVp(op, vt, pv))
}
func (self _Program) disassemble() string {
nb := len(self)
tab := make([]bool, nb + 1)
@ -379,21 +398,21 @@ type _Compiler struct {
opts option.CompileOptions
pv bool
tab map[reflect.Type]bool
rec map[reflect.Type]bool
rec map[reflect.Type]uint8
}
func newCompiler() *_Compiler {
return &_Compiler {
opts: option.DefaultCompileOptions(),
tab: map[reflect.Type]bool{},
rec: map[reflect.Type]bool{},
rec: map[reflect.Type]uint8{},
}
}
func (self *_Compiler) apply(opts option.CompileOptions) *_Compiler {
self.opts = opts
if self.opts.RecursiveDepth > 0 {
self.rec = map[reflect.Type]bool{}
self.rec = map[reflect.Type]uint8{}
}
return self
}
@ -408,15 +427,15 @@ func (self *_Compiler) rescue(ep *error) {
}
}
func (self *_Compiler) compile(vt reflect.Type) (ret _Program, err error) {
func (self *_Compiler) compile(vt reflect.Type, pv bool) (ret _Program, err error) {
defer self.rescue(&err)
self.compileOne(&ret, 0, vt, false)
self.compileOne(&ret, 0, vt, pv)
return
}
func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type, pv bool) {
if self.tab[vt] {
p.rtt(_OP_recurse, vt)
p.vp(_OP_recurse, vt, pv)
} else {
self.compileRec(p, sp, vt, pv)
}
@ -661,9 +680,13 @@ func (self *_Compiler) compileString(p *_Program, vt reflect.Type) {
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
if sp >= self.opts.MaxInlineDepth || p.pc() >= _MAX_ILBUF || (sp > 0 && vt.NumField() >= _MAX_FIELDS) {
p.rtt(_OP_recurse, vt)
p.vp(_OP_recurse, vt, self.pv)
if self.opts.RecursiveDepth > 0 {
self.rec[vt] = true
if self.pv {
self.rec[vt] = 1
} else {
self.rec[vt] = 0
}
}
} else {
self.compileStructBody(p, sp, vt)

28
encoder/compiler_test.go
View file

@ -17,14 +17,34 @@
package encoder
import (
`reflect`
`testing`
"reflect"
"testing"
"unsafe"
`github.com/stretchr/testify/assert`
"github.com/bytedance/sonic/internal/rt"
"github.com/stretchr/testify/assert"
)
func TestCompiler_Compile(t *testing.T) {
p, err := newCompiler().compile(reflect.TypeOf(_BindingValue))
p, err := newCompiler().compile(reflect.TypeOf(_BindingValue), false)
assert.Nil(t, err)
p.disassemble()
}
func TestReflectDirect(t *testing.T) {
type A struct {
A int
B int
}
var a A
var b = &a
println("b:", unsafe.Pointer(b))
v := rt.UnpackEface(a)
vv := reflect.ValueOf(a)
_ = vv
println("v:", v.Type.KindFlags, v.Value)
pv := rt.UnpackEface(&a)
pvv := reflect.ValueOf(&a)
_ = pvv
println("pv:", pv.Type.KindFlags, pv.Value)
}

45
encoder/encoder.go
View file

@ -38,6 +38,9 @@ const (
bitCompactMarshaler
bitNoQuoteTextMarshaler
bitNoNullSliceOrMap
// used for recursive compile
bitPointerValue = 63
)
const (
@ -284,47 +287,7 @@ func Pretouch(vt reflect.Type, opts ...option.CompileOption) error {
opt(&cfg)
break
}
return pretouchRec(map[reflect.Type]bool{vt:true}, cfg)
}
func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Type]bool, error) {
/* compile function */
compiler := newCompiler().apply(opts)
encoder := func(vt *rt.GoType) (interface{}, error) {
if pp, err := compiler.compile(_vt); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
}
}
/* find or compile */
vt := rt.UnpackType(_vt)
if val := programCache.Get(vt); val != nil {
return nil, nil
} else if _, err := programCache.Compute(vt, encoder); err == nil {
return compiler.rec, nil
} else {
return nil, err
}
}
func pretouchRec(vtm map[reflect.Type]bool, opts option.CompileOptions) error {
if opts.RecursiveDepth < 0 || len(vtm) == 0 {
return nil
}
next := make(map[reflect.Type]bool)
for vt := range(vtm) {
sub, err := pretouchType(vt, opts)
if err != nil {
return err
}
for svt := range(sub) {
next[svt] = true
}
}
opts.RecursiveDepth -= 1
return pretouchRec(next, opts)
return pretouchRec(map[reflect.Type]uint8{vt: 0}, cfg)
}
// Valid validates json and returns first non-blank character position,

50
encoder/pools.go
View file

@ -21,8 +21,10 @@ import (
`sync`
`unsafe`
`errors`
`reflect`
`github.com/bytedance/sonic/internal/caching`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/internal/rt`
)
@ -129,20 +131,60 @@ func freeBuffer(p *bytes.Buffer) {
bufferPool.Put(p)
}
func makeEncoder(vt *rt.GoType) (interface{}, error) {
if pp, err := newCompiler().compile(vt.Pack()); err != nil {
func makeEncoder(vt *rt.GoType, ex ...interface{}) (interface{}, error) {
if pp, err := newCompiler().compile(vt.Pack(), ex[0].(bool)); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
}
}
func findOrCompile(vt *rt.GoType) (_Encoder, error) {
func findOrCompile(vt *rt.GoType, pv bool) (_Encoder, error) {
if val := programCache.Get(vt); val != nil {
return val.(_Encoder), nil
} else if ret, err := programCache.Compute(vt, makeEncoder); err == nil {
} else if ret, err := programCache.Compute(vt, makeEncoder, pv); err == nil {
return ret.(_Encoder), nil
} else {
return nil, err
}
}
func pretouchType(_vt reflect.Type, opts option.CompileOptions, v uint8) (map[reflect.Type]uint8, error) {
/* compile function */
compiler := newCompiler().apply(opts)
encoder := func(vt *rt.GoType, ex ...interface{}) (interface{}, error) {
if pp, err := compiler.compile(_vt, ex[0].(bool)); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
}
}
/* find or compile */
vt := rt.UnpackType(_vt)
if val := programCache.Get(vt); val != nil {
return nil, nil
} else if _, err := programCache.Compute(vt, encoder, v == 1); err == nil {
return compiler.rec, nil
} else {
return nil, err
}
}
func pretouchRec(vtm map[reflect.Type]uint8, opts option.CompileOptions) error {
if opts.RecursiveDepth < 0 || len(vtm) == 0 {
return nil
}
next := make(map[reflect.Type]uint8)
for vt, v := range vtm {
sub, err := pretouchType(vt, opts, v)
if err != nil {
return err
}
for svt, v := range sub {
next[svt] = v
}
}
opts.RecursiveDepth -= 1
return pretouchRec(next, opts)
}

4
encoder/primitives.go
View file

@ -68,9 +68,9 @@ func encodeString(buf *[]byte, val string) error {
func encodeTypedPointer(buf *[]byte, vt *rt.GoType, vp *unsafe.Pointer, sb *_Stack, fv uint64) error {
if vt == nil {
return encodeNil(buf)
} else if fn, err := findOrCompile(vt); err != nil {
} else if fn, err := findOrCompile(vt, (fv&(1<<bitPointerValue)) != 0); err != nil {
return err
} else if (vt.KindFlags & rt.F_direct) == 0 {
} else if vt.Indirect() {
rt.MoreStack(_FP_size + native.MaxFrameSize)
return fn(buf, *vp, sb, fv)
} else {

4
internal/caching/pcache.go
View file

@ -149,7 +149,7 @@ func (self *ProgramCache) Get(vt *rt.GoType) interface{} {
return (*_ProgramMap)(atomic.LoadPointer(&self.p)).get(vt)
}
func (self *ProgramCache) Compute(vt *rt.GoType, compute func(*rt.GoType) (interface{}, error)) (interface{}, error) {
func (self *ProgramCache) Compute(vt *rt.GoType, compute func(*rt.GoType, ... interface{}) (interface{}, error), ex ...interface{}) (interface{}, error) {
var err error
var val interface{}
@ -163,7 +163,7 @@ func (self *ProgramCache) Compute(vt *rt.GoType, compute func(*rt.GoType) (inter
}
/* compute the value */
if val, err = compute(vt); err != nil {
if val, err = compute(vt, ex...); err != nil {
return nil, err
}

2
internal/caching/pcache_test.go
View file

@ -36,7 +36,7 @@ func TestPcacheRace(t *testing.T) {
var k = map[string]interface{}{}
<- start
for i:=0; i<100; i++ {
_, _ = pc.Compute(rt.UnpackEface(k).Type, func(_ *rt.GoType) (interface{}, error) {
_, _ = pc.Compute(rt.UnpackEface(k).Type, func(*rt.GoType, ... interface{}) (interface{}, error) {
return map[string]interface{}{}, nil
})
}

4
internal/rt/fastvalue.go
View file

@ -58,6 +58,10 @@ func (self *GoType) String() string {
return self.Pack().String()
}
func (self *GoType) Indirect() bool {
return self.KindFlags & F_direct == 0
}
type GoMap struct {
Count int
Flags uint8

61
issue_test/issuex_test.go Normal file
View file

@ -0,0 +1,61 @@
package issue_test
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"testing"
"time"
"github.com/bytedance/sonic"
"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/require"
)
func TestPointerValueRecurseMarshal(t *testing.T) {
info := &TestStruct1{
StartTime: JSONTime(time.Now()),
}
infos := &[]*TestStruct1{info}
bytes, err1 := json.Marshal(infos)
fmt.Printf("%+v\n", string(bytes))
spew.Dump(bytes, err1)
jbytes, err2 := sonic.Marshal(infos)
fmt.Printf("%+v\n", string(jbytes))
spew.Dump(jbytes, err2)
require.Equal(t, bytes, jbytes)
}
func TestPointerValueRecursePretouch(t *testing.T) {
info := &TestStruct2{
StartTime: JSONTime(time.Now()),
}
infos := &[]*TestStruct2{info}
bytes, err1 := json.Marshal(infos)
fmt.Printf("%+v\n", string(bytes))
spew.Dump(bytes, err1)
sonic.Pretouch(reflect.TypeOf(infos))
jbytes, err2 := sonic.Marshal(infos)
fmt.Printf("%+v\n", string(jbytes))
spew.Dump(jbytes, err2)
require.Equal(t, bytes, jbytes)
}
type TestStruct1 struct {
StartTime JSONTime
}
type TestStruct2 struct {
StartTime JSONTime
}
type JSONTime time.Time
func (t *JSONTime) MarshalJSON() ([]byte, error) {
return []byte(strconv.FormatInt(time.Time(*t).Unix(), 10)), nil
}