From 22229eefc354d45f91930d81cb515ee0614a95ea Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Mon, 20 Dec 2021 16:19:12 +0800 Subject: [PATCH] fix: relocate stack pointers (#156) * fix: relocate stack pointer _VAR_sv to keep it alive * fix: add stack pointer _VAR_vk to defend `encoding.TextUnmarshaler` * fix: align faker func's stack with JIT func's * fix: clear _Stack memory when err returned * fix: clear stack pointer before return * fix: relimit stack-overflow check at `_ValueDecoder` Co-authored-by: duanyi.aster --- .github/workflows/push-check-go116.yml | 5 -- .github/workflows/push-check-go117.yml | 2 +- ast/encode_test.go | 2 - ast/parser_test.go | 3 +- ast/search_test.go | 2 - decode_test.go | 2 +- decoder/assembler_amd64_go115.go | 60 ++++++++------- decoder/assembler_amd64_go116.go | 95 +++++++++++++---------- decoder/assembler_amd64_go117.go | 102 +++++++++++++++---------- decoder/assembler_test.go | 12 +-- decoder/decoder.go | 3 + decoder/decoder_test.go | 3 +- decoder/generic_amd64_go115.go | 6 +- decoder/generic_amd64_go116.go | 6 +- decoder/generic_amd64_go117.go | 6 +- decoder/pools.go | 30 ++++++-- decoder/primitives.go | 2 +- encode_test.go | 4 +- encoder/assembler_amd64_go115.go | 2 +- encoder/assembler_amd64_go116.go | 2 +- encoder/assembler_amd64_go117.go | 13 ++-- encoder/encoder.go | 3 + encoder/encoder_test.go | 3 +- encoder/pools.go | 6 ++ encoder/stubs_go115.go | 5 ++ encoder/stubs_go116.go | 5 ++ encoder/stubs_go117.go | 5 ++ issue_test/testmain_test.go | 2 + 28 files changed, 242 insertions(+), 149 deletions(-) diff --git a/.github/workflows/push-check-go116.yml b/.github/workflows/push-check-go116.yml index e9831fb..6d0960a 100644 --- a/.github/workflows/push-check-go116.yml +++ b/.github/workflows/push-check-go116.yml @@ -20,10 +20,5 @@ jobs: restore-keys: | ${{ runner.os }}-go- - - name: Check License Header - uses: apache/skywalking-eyes@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Unit Test run: go test -v -race -covermode=atomic -coverprofile=coverage.out ./... diff --git a/.github/workflows/push-check-go117.yml b/.github/workflows/push-check-go117.yml index fe5a2c8..13c083b 100644 --- a/.github/workflows/push-check-go117.yml +++ b/.github/workflows/push-check-go117.yml @@ -26,4 +26,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Unit Test - run: go test -v -covermode=atomic -coverprofile=coverage.out ./... + run: go test -v -race -covermode=atomic -coverprofile=coverage.out ./... diff --git a/ast/encode_test.go b/ast/encode_test.go index 1f4648d..c4eb520 100644 --- a/ast/encode_test.go +++ b/ast/encode_test.go @@ -18,7 +18,6 @@ package ast import ( `runtime` - `runtime/debug` `sync` `testing` @@ -57,7 +56,6 @@ func TestGC_Encode(t *testing.T) { t.Fatal(err) } runtime.GC() - debug.FreeOSMemory() }(wg) } wg.Wait() diff --git a/ast/parser_test.go b/ast/parser_test.go index 9eb00b2..5a61610 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -20,6 +20,7 @@ import ( `os` `encoding/json` `testing` + `time` `runtime` `runtime/debug` `sync` @@ -46,6 +47,7 @@ func TestMain(m *testing.M) { } println("stop GC looping!") }() + time.Sleep(time.Millisecond) m.Run() } @@ -68,7 +70,6 @@ func TestGC_Parse(t *testing.T) { t.Fatal(err) } runtime.GC() - debug.FreeOSMemory() }(wg) } wg.Wait() diff --git a/ast/search_test.go b/ast/search_test.go index 959de9c..35805a8 100644 --- a/ast/search_test.go +++ b/ast/search_test.go @@ -19,7 +19,6 @@ package ast import ( `testing` `runtime` - `runtime/debug` `sync` `fmt` `math` @@ -53,7 +52,6 @@ func TestGC_Search(t *testing.T) { t.Fatal(err) } runtime.GC() - debug.FreeOSMemory() }(wg) } wg.Wait() diff --git a/decode_test.go b/decode_test.go index 08ce681..52f8b63 100644 --- a/decode_test.go +++ b/decode_test.go @@ -2345,7 +2345,7 @@ func TestUnmarshalMaxDepth(t *testing.T) { }{ { name: "ArrayUnderMaxNestingDepth", - data: `{"a":` + strings.Repeat(`[`, 65535) + `0` + strings.Repeat(`]`, 65535) + `}`, + data: `{"a":` + strings.Repeat(`[`, 65534) + `0` + strings.Repeat(`]`, 65534) + `}`, errMaxDepth: false, }, { diff --git a/decoder/assembler_amd64_go115.go b/decoder/assembler_amd64_go115.go index 21cf873..327cefe 100644 --- a/decoder/assembler_amd64_go115.go +++ b/decoder/assembler_amd64_go115.go @@ -52,7 +52,7 @@ /** Function Prototype & Stack Map * - * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64) (rc int, err error) + * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string) (rc int, err error) * * s.buf : (FP) * s.len : 8(FP) @@ -61,15 +61,16 @@ * sb : 32(FP) * fv : 40(FP) * rc : 48(FP) - * err.vt : 56(FP) - * err.vp : 64(FP) + * sv : 56(FP) + * err.vt : 72(FP) + * err.vp : 80(FP) */ const ( - _FP_args = 72 // 72 bytes to pass arguments and return values for this function + _FP_args = 96 // 96 bytes to pass arguments and return values for this function _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 40 // 40 bytes for saving the registers before CALL instructions - _FP_locals = 96 // 96 bytes for local variables + _FP_locals = 72 // 72 bytes for local variables ) const ( @@ -151,35 +152,36 @@ ) var ( - _RET_rc = jit.Ptr(_SP, _FP_base + 48) - _RET_et = jit.Ptr(_SP, _FP_base + 56) - _RET_ep = jit.Ptr(_SP, _FP_base + 64) + _VAR_sv = _VAR_sv_p + _VAR_sv_p = jit.Ptr(_SP, _FP_base + 48) + _VAR_sv_n = jit.Ptr(_SP, _FP_base + 56) + _VAR_vk = jit.Ptr(_SP, _FP_base + 64) ) var ( - _VAR_sv = _VAR_sv_p + _RET_rc = jit.Ptr(_SP, _FP_base + 72) + _RET_et = jit.Ptr(_SP, _FP_base + 80) + _RET_ep = jit.Ptr(_SP, _FP_base + 88) + ) + + var ( _VAR_st = _VAR_st_Vt _VAR_sr = jit.Ptr(_SP, _FP_fargs + _FP_saves) ) var ( - _VAR_sv_p = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) - _VAR_sv_n = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 0) + _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) + _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) ) var ( - _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) - _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) - _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) - _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) - ) - - var ( - _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) - _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) - _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 72) - _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 80) - _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 88) + _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) + _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) + _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) + _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) + _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) ) type _Assembler struct { @@ -816,6 +818,8 @@ /* allocate the key, and call the unmarshaler */ self.valloc(vk, _DI) // VALLOC ${vk}, DI + // must spill vk pointer since next call_go may invoke GC + self.Emit("MOVQ" , _DI, _VAR_vk) self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) self.Emit("MOVQ" , _DI, jit.Ptr(_SP, 8)) // MOVQ DI, 8(SP) @@ -826,6 +830,7 @@ self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ" , _LB_error) // JNZ _error + self.Emit("MOVQ" , _VAR_vk, _AX) // MOVQ VAR.vk, AX self.Emit("MOVQ" , jit.Ptr(_SP, 8), _AX) // MOVQ 8(SP), AX /* select the correct assignment function */ @@ -1283,7 +1288,10 @@ self.parse_string() // PARSE STRING self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n if vt := p.vt(); !mapfast(vt) { - self.mapassign_std(vt, _VAR_sv_p) // MAPASSIGN string, DI, SI + self.valloc(vt.Key(), _DI) + self.Emit("MOVOU", _VAR_sv, _X0) + self.Emit("MOVOU", _X0, jit.Ptr(_DI, 0)) + self.mapassign_std(vt, jit.Ptr(_DI, 0)) } else { self.Emit("MOVQ", _VAR_sv_p, _DI) // MOVQ sv.p, DI self.Emit("MOVQ", _VAR_sv_n, _SI) // MOVQ sv.n, SI @@ -1518,8 +1526,8 @@ func (self *_Assembler) _asm_OP_save(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX - self.Emit("CMPQ", _AX, jit.Imm(_MaxStack)) // CMPQ AX, ${_MaxStack} - self.Sjmp("JA" , _LB_stack_error) // JA _stack_error + self.Emit("CMPQ", _AX, jit.Imm(_MaxStackBytes)) // CMPQ AX, ${_MaxStackBytes} + self.Sjmp("JAE" , _LB_stack_error) // JA _stack_error self.Emit("MOVQ", _VP, jit.Sib(_ST, _AX, 1, 8)) // MOVQ VP, 8(ST)(AX) self.Emit("ADDQ", jit.Imm(8), _AX) // ADDQ $8, AX self.Emit("MOVQ", _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST) diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 19d3479..79b6327 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -54,7 +54,7 @@ import ( /** Function Prototype & Stack Map * - * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64) (rc int, err error) + * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string) (rc int, err error) * * s.buf : (FP) * s.len : 8(FP) @@ -62,16 +62,16 @@ import ( * vp : 24(FP) * sb : 32(FP) * fv : 40(FP) - * rc : 48(FP) - * err.vt : 56(FP) - * err.vp : 64(FP) + * sv : 56(FP) + * err.vt : 72(FP) + * err.vp : 80(FP) */ const ( - _FP_args = 72 // 72 bytes to pass arguments and return values for this function + _FP_args = 96 // 96 bytes to pass arguments and return values for this function _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 40 // 40 bytes for saving the registers before CALL instructions - _FP_locals = 96 // 96 bytes for local variables + _FP_locals = 72 // 72 bytes for local variables ) const ( @@ -154,35 +154,37 @@ var ( ) var ( - _RET_rc = jit.Ptr(_SP, _FP_base + 48) - _RET_et = jit.Ptr(_SP, _FP_base + 56) - _RET_ep = jit.Ptr(_SP, _FP_base + 64) + _VAR_sv = _VAR_sv_p + _VAR_sv_p = jit.Ptr(_SP, _FP_base + 48) + _VAR_sv_n = jit.Ptr(_SP, _FP_base + 56) + _VAR_vk = jit.Ptr(_SP, _FP_base + 64) +) + +var ( + _RET_rc = jit.Ptr(_SP, _FP_base + 72) + _RET_et = jit.Ptr(_SP, _FP_base + 80) + _RET_ep = jit.Ptr(_SP, _FP_base + 88) ) var ( - _VAR_sv = _VAR_sv_p _VAR_st = _VAR_st_Vt _VAR_sr = jit.Ptr(_SP, _FP_fargs + _FP_saves) ) + var ( - _VAR_sv_p = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) - _VAR_sv_n = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 0) + _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) + _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) ) var ( - _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) - _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) - _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) - _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) -) - -var ( - _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) - _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) - _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 72) - _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 80) - _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 88) + _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) + _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) + _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) + _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) + _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) ) type _Assembler struct { @@ -661,7 +663,7 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) { self.Emit("LEAQ", jit.Sib(_IC, p, 1, d), _SI) // LEAQ d(IC)(${p}), SI } -func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { +func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) { self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1 self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p} self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n} @@ -670,7 +672,12 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { self.malloc(_SI, _DX) // MALLOC SI, DX self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI - self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + if stack { + // no need for writeBarrier + self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p} + } else { + self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + } self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX self.Emit("XORL" , _R8, _R8) // XORL R8, R8 self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, fv @@ -685,7 +692,7 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { self.Link("_noescape_{n}") // _noescape_{n}: } -func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr) { +func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) { self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1 self.Sjmp("JE" , _LB_eof_error) // JE _eof_error self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -3), jit.Imm('\\')) // CMPB -3(IP)(IC), $'\\' @@ -702,7 +709,12 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr) { self.malloc(_SI, _DX) // MALLOC SI, DX self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI - self.WriteRecNotAX(6, _DX, p, true, true) // MOVQ DX, ${p} + if stack { + // no need for writeBarrier + self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p} + } else { + self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + } self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8 self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, AX @@ -819,6 +831,8 @@ func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) { /* allocate the key, and call the unmarshaler */ self.valloc(vk, _DI) // VALLOC ${vk}, DI + // must spill vk pointer since next call_go may invoke GC + self.Emit("MOVQ" , _DI, _VAR_vk) self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) self.Emit("MOVQ" , _DI, jit.Ptr(_SP, 8)) // MOVQ DI, 8(SP) @@ -829,7 +843,7 @@ func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) { self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ" , _LB_error) // JNZ _error - self.Emit("MOVQ" , jit.Ptr(_SP, 8), _AX) // MOVQ 8(SP), AX + self.Emit("MOVQ" , _VAR_vk, _AX) /* select the correct assignment function */ if !pv { @@ -857,7 +871,7 @@ func (self *_Assembler) unmarshal_json(t reflect.Type, deref bool) { func (self *_Assembler) unmarshal_text(t reflect.Type, deref bool) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.unmarshal_func(t, _F_decodeTextUnmarshaler, deref) // UNMARSHAL text, ${t}, ${deref} } @@ -1012,7 +1026,7 @@ func (self *_Assembler) _asm_OP_dyn(p *_Instr) { func (self *_Assembler) _asm_OP_str(_ *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE once, (VP), 8(VP) + self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8), false) // UNQUOTE once, (VP), 8(VP) } func (self *_Assembler) _asm_OP_bin(_ *_Instr) { @@ -1148,7 +1162,7 @@ func (self *_Assembler) _asm_OP_unquote(_ *_Instr) { self.Sjmp("JNE" , _LB_char_1_error) // JNE _char_1_error self.Emit("ADDQ", jit.Imm(2), _IC) // ADDQ $2, IC self.parse_string() // PARSE STRING - self.unquote_twice(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE twice, (VP), 8(VP) + self.unquote_twice(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8), false) // UNQUOTE twice, (VP), 8(VP) } func (self *_Assembler) _asm_OP_nil_1(_ *_Instr) { @@ -1288,9 +1302,12 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n if vt := p.vt(); !mapfast(vt) { - self.mapassign_std(vt, _VAR_sv_p) // MAPASSIGN string, DI, SI + self.valloc(vt.Key(), _DI) + self.Emit("MOVOU", _VAR_sv, _X0) + self.Emit("MOVOU", _X0, jit.Ptr(_DI, 0)) + self.mapassign_std(vt, jit.Ptr(_DI, 0)) } else { self.Emit("MOVQ", _VAR_sv_p, _DI) // MOVQ sv.p, DI self.Emit("MOVQ", _VAR_sv_n, _SI) // MOVQ sv.n, SI @@ -1300,13 +1317,13 @@ func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), false) // MAPASSIGN utext, ${p.vt()}, false } func (self *_Assembler) _asm_OP_map_key_utext_p(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), true) // MAPASSIGN utext, ${p.vt()}, true } @@ -1388,7 +1405,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.Emit("LEAQ" , _VAR_sv, _AX) // LEAQ sv, AX self.Emit("XORL" , _CX, _CX) // XORL CX, CX self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) @@ -1525,8 +1542,8 @@ func (self *_Assembler) _asm_OP_load(_ *_Instr) { func (self *_Assembler) _asm_OP_save(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX) // MOVQ (ST), CX - self.Emit("CMPQ", _CX, jit.Imm(_MaxStack)) // CMPQ CX, ${_MaxStack} - self.Sjmp("JA" , _LB_stack_error) // JA _stack_error + self.Emit("CMPQ", _CX, jit.Imm(_MaxStackBytes)) // CMPQ CX, ${_MaxStackBytes} + self.Sjmp("JAE" , _LB_stack_error) // JA _stack_error self.WriteRecNotAX(0 , _VP, jit.Sib(_ST, _CX, 1, 8), false, false) // MOVQ VP, 8(ST)(CX) self.Emit("ADDQ", jit.Imm(8), _CX) // ADDQ $8, CX self.Emit("MOVQ", _CX, jit.Ptr(_ST, 0)) // MOVQ CX, (ST) diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index d7b3de9..359c599 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -54,7 +54,7 @@ import ( /** Function Prototype & Stack Map * - * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64) (rc int, err error) + * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string) (rc int, err error) * * s.buf : (FP) * s.len : 8(FP) @@ -62,16 +62,16 @@ import ( * vp : 24(FP) * sb : 32(FP) * fv : 40(FP) - * rc : 48(FP) - * err.vt : 56(FP) - * err.vp : 64(FP) + * sv : 56(FP) + * err.vt : 72(FP) + * err.vp : 80(FP) */ const ( - _FP_args = 72 // 72 bytes to pass arguments and return values for this function + _FP_args = 72 // 72 bytes to pass and spill register arguements _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_saves = 48 // 48 bytes for saving the registers before CALL instructions - _FP_locals = 96 // 96 bytes for local variables + _FP_locals = 72 // 72 bytes for local variables ) const ( @@ -143,9 +143,11 @@ var ( _EP = jit.Reg("BX") ) + + var ( _ARG_s = _ARG_sp - _ARG_sp = jit.Ptr(_SP, _FP_base) + _ARG_sp = jit.Ptr(_SP, _FP_base + 0) _ARG_sl = jit.Ptr(_SP, _FP_base + 8) _ARG_ic = jit.Ptr(_SP, _FP_base + 16) _ARG_vp = jit.Ptr(_SP, _FP_base + 24) @@ -154,35 +156,30 @@ var ( ) var ( - _RET_rc = jit.Ptr(_SP, _FP_base + 48) - _RET_et = jit.Ptr(_SP, _FP_base + 56) - _RET_ep = jit.Ptr(_SP, _FP_base + 64) + _VAR_sv = _VAR_sv_p + _VAR_sv_p = jit.Ptr(_SP, _FP_base + 48) + _VAR_sv_n = jit.Ptr(_SP, _FP_base + 56) + _VAR_vk = jit.Ptr(_SP, _FP_base + 64) ) var ( - _VAR_sv = _VAR_sv_p _VAR_st = _VAR_st_Vt _VAR_sr = jit.Ptr(_SP, _FP_fargs + _FP_saves) ) var ( - _VAR_sv_p = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) - _VAR_sv_n = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 0) + _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) + _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) ) var ( - _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) - _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) - _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) - _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) -) - -var ( - _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) - _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) - _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 72) - _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 80) - _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 88) + _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) + _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) + _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) + _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) + _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) ) type _Assembler struct { @@ -307,6 +304,10 @@ func (self *_Assembler) epilogue() { self.Emit("MOVQ", _EP, _CX) // MOVQ BX, CX self.Emit("MOVQ", _ET, _BX) // MOVQ AX, BX self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX + self.Emit("MOVQ", jit.Imm(0), _ARG_sp) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_vp) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP self.Emit("RET") // RET @@ -327,6 +328,9 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", _SI, _ARG_sb) // MOVQ SI, sb<>+32(FP) self.Emit("MOVQ", _SI, _ST) // MOVQ SI, ST self.Emit("MOVQ", _R8, _ARG_fv) // MOVQ R8, fv<>+40(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP) + self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) } /** Function Calling Helpers **/ @@ -669,7 +673,7 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) { self.Emit("LEAQ", jit.Sib(_IC, p, 1, d), _SI) // LEAQ d(IC)(${p}), SI } -func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { +func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) { self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1 self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p} self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n} @@ -678,7 +682,12 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { self.malloc(_SI, _DX) // MALLOC SI, DX self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI - self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + if stack { + // no need for writeBarrier + self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p} + } else { + self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + } self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX self.Emit("XORL" , _R8, _R8) // XORL R8, R8 self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, fv @@ -693,7 +702,7 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { self.Link("_noescape_{n}") // _noescape_{n}: } -func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr) { +func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) { self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1 self.Sjmp("JE" , _LB_eof_error) // JE _eof_error self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -3), jit.Imm('\\')) // CMPB -3(IP)(IC), $'\\' @@ -710,7 +719,12 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr) { self.malloc(_SI, _DX) // MALLOC SI, DX self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI - self.WriteRecNotAX(6, _DX, p, true, true) // MOVQ DX, ${p} + if stack { + // no need for writeBarrier + self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p} + } else { + self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p} + } self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8 self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, AX @@ -824,14 +838,16 @@ func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) { /* allocate the key, and call the unmarshaler */ self.valloc(vk, _BX) // VALLOC ${vk}, BX - self.Emit("MOVQ" , _BX, jit.Ptr(_SP, 8)) // MOVQ BX, 8(SP) + // must spill vk pointer since next call_go may invoke GC + self.Emit("MOVQ" , _BX, _VAR_vk) self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX self.Emit("MOVQ" , _VAR_sv_p, _CX) // MOVQ sv.p, CX self.Emit("MOVQ" , _VAR_sv_n, _DI) // MOVQ sv.n, DI self.call_go(_F_decodeTextUnmarshaler) // CALL_GO decodeTextUnmarshaler self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ" , _LB_error) // JNZ _error - self.Emit("MOVQ" , jit.Ptr(_SP, 8), _AX) // MOVQ 8(SP), AX + self.Emit("MOVQ" , _VAR_vk, _AX) // MOVQ VAR.vk, AX + self.Emit("MOVQ", jit.Imm(0), _VAR_vk) /* select the correct assignment function */ if !pv { @@ -859,7 +875,7 @@ func (self *_Assembler) unmarshal_json(t reflect.Type, deref bool) { func (self *_Assembler) unmarshal_text(t reflect.Type, deref bool) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.unmarshal_func(t, _F_decodeTextUnmarshaler, deref) // UNMARSHAL text, ${t}, ${deref} } @@ -987,6 +1003,7 @@ func (self *_Assembler) _asm_OP_any(_ *_Instr) { self.Emit("MOVQ" , _ARG_fv, _DF) // MOVQ fv, DF self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 0)) // MOVQ _ST, (SP) self.call(_F_decodeValue) // CALL decodeValue + self.Emit("MOVQ" , jit.Imm(0), jit.Ptr(_SP, 0)) // MOVQ _ST, (SP) self.Emit("TESTQ" , _EP, _EP) // TESTQ EP, EP self.Sjmp("JNZ" , _LB_parsing_error) // JNZ _parsing_error self.Link("_decode_end_{n}") // _decode_end_{n}: @@ -1009,7 +1026,7 @@ func (self *_Assembler) _asm_OP_dyn(p *_Instr) { func (self *_Assembler) _asm_OP_str(_ *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE once, (VP), 8(VP) + self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8), false) // UNQUOTE once, (VP), 8(VP) } func (self *_Assembler) _asm_OP_bin(_ *_Instr) { @@ -1145,7 +1162,7 @@ func (self *_Assembler) _asm_OP_unquote(_ *_Instr) { self.Sjmp("JNE" , _LB_char_1_error) // JNE _char_1_error self.Emit("ADDQ", jit.Imm(2), _IC) // ADDQ $2, IC self.parse_string() // PARSE STRING - self.unquote_twice(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE twice, (VP), 8(VP) + self.unquote_twice(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8), false) // UNQUOTE twice, (VP), 8(VP) } func (self *_Assembler) _asm_OP_nil_1(_ *_Instr) { @@ -1286,9 +1303,12 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n if vt := p.vt(); !mapfast(vt) { - self.mapassign_std(vt, _VAR_sv_p) // MAPASSIGN string, DI, SI + self.valloc(vt.Key(), _DI) + self.Emit("MOVOU", _VAR_sv, _X0) + self.Emit("MOVOU", _X0, jit.Ptr(_DI, 0)) + self.mapassign_std(vt, jit.Ptr(_DI, 0)) // MAPASSIGN string, DI, SI } else { self.mapassign_str_fast(vt, _VAR_sv_p, _VAR_sv_n) // MAPASSIGN string, DI, SI } @@ -1296,13 +1316,13 @@ func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), false) // MAPASSIGN utext, ${p.vt()}, false } func (self *_Assembler) _asm_OP_map_key_utext_p(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), true) // MAPASSIGN utext, ${p.vt()}, true } @@ -1376,7 +1396,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_VAR_sv_p, _VAR_sv_n, true) // UNQUOTE once, sv.p, sv.n self.Emit("LEAQ" , _VAR_sv, _AX) // LEAQ sv, AX self.Emit("XORL" , _BX, _BX) // XORL BX, BX self.call_go(_F_strhash) // CALL_GO strhash @@ -1507,8 +1527,8 @@ func (self *_Assembler) _asm_OP_load(_ *_Instr) { func (self *_Assembler) _asm_OP_save(_ *_Instr) { self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX) // MOVQ (ST), CX - self.Emit("CMPQ", _CX, jit.Imm(_MaxStack)) // CMPQ CX, ${_MaxStack} - self.Sjmp("JA" , _LB_stack_error) // JA _stack_error + self.Emit("CMPQ", _CX, jit.Imm(_MaxStackBytes)) // CMPQ CX, ${_MaxStackBytes} + self.Sjmp("JAE" , _LB_stack_error) // JA _stack_error self.WriteRecNotAX(0 , _VP, jit.Sib(_ST, _CX, 1, 8), false, false) // MOVQ VP, 8(ST)(CX) self.Emit("ADDQ", jit.Imm(8), _CX) // ADDQ $8, CX self.Emit("MOVQ", _CX, jit.Ptr(_ST, 0)) // MOVQ CX, (ST) diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index 3b1cdcb..f6ea883 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -33,7 +33,7 @@ import ( func TestAssembler_PrologueAndEpilogue(t *testing.T) { a := newAssembler(nil) - _, e := a.Load()("", 0, nil, nil, 0) + _, e := a.Load()("", 0, nil, nil, 0, "", nil) assert.Nil(t, e) } @@ -117,7 +117,7 @@ func testOpCode(t *testing.T, ops *testOps) { k := new(_Stack) a := newAssembler(p) f := a.Load() - i, e := f(ops.src, ops.pos, rt.UnpackEface(ops.val).Value, k, ops.opt) + i, e := f(ops.src, ops.pos, rt.UnpackEface(ops.val).Value, k, ops.opt, "", nil) if ops.err != nil { assert.EqualError(t, e, ops.err.Error()) } else { @@ -684,7 +684,7 @@ func TestAssembler_DecodeStruct(t *testing.T) { k := new(_Stack) a := newAssembler(p) f := a.Load() - pos, err := f(s, 0, unsafe.Pointer(&v), k, 0) + pos, err := f(s, 0, unsafe.Pointer(&v), k, 0, "", nil) require.NoError(t, err) assert.Equal(t, len(s), pos) assert.Equal(t, JsonStruct{ @@ -707,7 +707,7 @@ func TestAssembler_DecodeStruct_SinglePrivateField(t *testing.T) { k := new(_Stack) a := newAssembler(p) f := a.Load() - pos, err := f(s, 0, unsafe.Pointer(&v), k, 0) + pos, err := f(s, 0, unsafe.Pointer(&v), k, 0, "", nil) require.NoError(t, err) assert.Equal(t, len(s), pos) assert.Equal(t, Tx{}, v) @@ -721,7 +721,7 @@ func TestAssembler_DecodeByteSlice_Bin(t *testing.T) { k := new(_Stack) a := newAssembler(p) f := a.Load() - pos, err := f(s, 0, unsafe.Pointer(&v), k, 0) + pos, err := f(s, 0, unsafe.Pointer(&v), k, 0, "", nil) require.NoError(t, err) assert.Equal(t, len(s), pos) assert.Equal(t, []byte("hello, world"), v) @@ -735,7 +735,7 @@ func TestAssembler_DecodeByteSlice_List(t *testing.T) { k := new(_Stack) a := newAssembler(p) f := a.Load() - pos, err := f(s, 0, unsafe.Pointer(&v), k, 0) + pos, err := f(s, 0, unsafe.Pointer(&v), k, 0, "", nil) require.NoError(t, err) assert.Equal(t, len(s), pos) assert.Equal(t, []byte("hello, world"), v) diff --git a/decoder/decoder.go b/decoder/decoder.go index 797174e..b893061 100644 --- a/decoder/decoder.go +++ b/decoder/decoder.go @@ -70,6 +70,9 @@ func (self *Decoder) Decode(val interface{}) error { nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f) /* return the stack back */ + if err != nil { + resetStack(sb) + } self.i = nb freeStack(sb) diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go index 51b36d1..365c827 100644 --- a/decoder/decoder_test.go +++ b/decoder/decoder_test.go @@ -19,6 +19,7 @@ package decoder import ( `encoding/json` `testing` + `time` `runtime` `runtime/debug` `sync` @@ -42,6 +43,7 @@ func TestMain(m *testing.M) { } println("stop GC looping!") }() + time.Sleep(time.Millisecond) m.Run() } @@ -72,7 +74,6 @@ func TestGC(t *testing.T) { t.Fatal(out) } runtime.GC() - debug.FreeOSMemory() }(wg) } wg.Wait() diff --git a/decoder/generic_amd64_go115.go b/decoder/generic_amd64_go115.go index 23c8fad..b4df35b 100644 --- a/decoder/generic_amd64_go115.go +++ b/decoder/generic_amd64_go115.go @@ -329,7 +329,7 @@ /* add a new slot for the first element */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX] @@ -398,7 +398,7 @@ /* add a new delimiter */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX] @@ -533,6 +533,8 @@ self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI) self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} + self.Sjmp("JAE" , "_stack_overflow") self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp diff --git a/decoder/generic_amd64_go116.go b/decoder/generic_amd64_go116.go index 2a1d297..905c145 100644 --- a/decoder/generic_amd64_go116.go +++ b/decoder/generic_amd64_go116.go @@ -332,7 +332,7 @@ func (self *_ValueDecoder) compile() { /* add a new slot for the first element */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.WritePtrAX(3, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX] @@ -401,7 +401,7 @@ func (self *_ValueDecoder) compile() { /* add a new delimiter */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX] @@ -537,6 +537,8 @@ func (self *_ValueDecoder) compile() { self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI) self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} + self.Sjmp("JAE" , "_stack_overflow") self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp diff --git a/decoder/generic_amd64_go117.go b/decoder/generic_amd64_go117.go index 8e4ea9f..1e994c7 100644 --- a/decoder/generic_amd64_go117.go +++ b/decoder/generic_amd64_go117.go @@ -344,7 +344,7 @@ func (self *_ValueDecoder) compile() { /* add a new slot for the first element */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.WritePtrAX(3, jit.Sib(_ST, _CX, 8, _ST_Vp), false) // MOVQ AX, ST.Vp[CX] @@ -412,7 +412,7 @@ func (self *_ValueDecoder) compile() { /* add a new delimiter */ self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} - self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX] @@ -542,6 +542,8 @@ func (self *_ValueDecoder) compile() { self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI) self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} + self.Sjmp("JAE" , "_stack_overflow") // JA _stack_overflow self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp diff --git a/decoder/pools.go b/decoder/pools.go index 0806727..cc5d3a1 100644 --- a/decoder/pools.go +++ b/decoder/pools.go @@ -18,7 +18,6 @@ package decoder import ( `errors` - `runtime` `sync` `unsafe` @@ -30,11 +29,13 @@ import ( const ( _MinSlice = 16 _MaxStack = 65536 // 64k slots + _MaxStackBytes = _MaxStack * _PtrBytes ) const ( _PtrBytes = _PTR_SIZE / 8 _FsmOffset = (_MaxStack + 1) * _PtrBytes + _StackSize = unsafe.Sizeof(_Stack{}) ) var ( @@ -58,6 +59,8 @@ type _Decoder func( vp unsafe.Pointer, sb *_Stack, fv uint64, + sv string, // DO NOT pass value to this arguement, since it is only used for local _VAR_sv + vk unsafe.Pointer, // DO NOT pass value to this arguement, since it is only used for local _VAR_vk ) (int, error) var _KeepAlive struct { @@ -66,16 +69,21 @@ var _KeepAlive struct { vp unsafe.Pointer sb *_Stack fv uint64 + sv string + vk unsafe.Pointer + ret int err error - frame [_FP_offs]byte + + frame_decoder [_FP_offs]byte + frame_generic [_VD_offs]byte } var errCallShadow = errors.New("DON'T CALL THIS!") //go:nosplit // Faker func of _Decoder, used to export its stackmap as _Decoder's -func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64) (ret int, err error) { +func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string, vk unsafe.Pointer) (ret int, err error) { // align to assembler_amd64.go: _FP_offs var frame [_FP_offs]byte @@ -87,8 +95,10 @@ func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64) _KeepAlive.fv = fv _KeepAlive.ret = ret _KeepAlive.err = err - _KeepAlive.frame = frame - + _KeepAlive.sv = sv + _KeepAlive.vk = vk + _KeepAlive.frame_decoder = frame + return 0, errCallShadow } @@ -96,11 +106,11 @@ func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64) // Faker func of _Decoder_Generic, used to export its stackmap func _Decoder_Generic_Shadow(sb *_Stack) { // align to generic_amd64.go: _VD_offs - var stacks [_VD_offs]byte - runtime.KeepAlive(stacks) + var frame [_VD_offs]byte // must keep sb noticeable to GC - runtime.KeepAlive(sb) + _KeepAlive.sb = sb + _KeepAlive.frame_generic = frame } func newStack() *_Stack { @@ -111,6 +121,10 @@ func newStack() *_Stack { } } +func resetStack(p *_Stack) { + memclrNoHeapPointers(unsafe.Pointer(p), _StackSize) +} + func freeStack(p *_Stack) { p.sp = 0 stackPool.Put(p) diff --git a/decoder/primitives.go b/decoder/primitives.go index 0184ca9..f86c358 100644 --- a/decoder/primitives.go +++ b/decoder/primitives.go @@ -28,7 +28,7 @@ func decodeTypedPointer(s string, i int, vt *rt.GoType, vp unsafe.Pointer, sb *_ if fn, err := findOrCompile(vt); err != nil { return 0, err } else { - return fn(s, i, vp, sb, fv) + return fn(s, i, vp, sb, fv, "", nil) } } diff --git a/encode_test.go b/encode_test.go index 03a5982..81ad835 100644 --- a/encode_test.go +++ b/encode_test.go @@ -30,6 +30,7 @@ import ( `runtime/debug` `strconv` `testing` + `time` `unsafe` `github.com/bytedance/sonic/encoder` @@ -49,7 +50,8 @@ func TestMain(m *testing.M) { debug.FreeOSMemory() } println("stop GC looping!") - }() + }() + time.Sleep(time.Millisecond) m.Run() } diff --git a/encoder/assembler_amd64_go115.go b/encoder/assembler_amd64_go115.go index ac32bfa..9d42e2b 100644 --- a/encoder/assembler_amd64_go115.go +++ b/encoder/assembler_amd64_go115.go @@ -411,7 +411,7 @@ self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("LEAQ", jit.Ptr(_AX, _StateSize), _R8) // LEAQ _StateSize(AX), R8 self.Emit("CMPQ", _R8, jit.Imm(_StackLimit)) // CMPQ R8, $_StackLimit - self.Sjmp("JA" , _LB_error_too_deep) // JA _error_too_deep + self.Sjmp("JAE" , _LB_error_too_deep) // JA _error_too_deep self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _AX, 1, 8)) // MOVQ SP.x, 8(ST)(AX) self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _AX, 1, 16)) // MOVQ SP.f, 16(ST)(AX) self.Emit("MOVQ", _SP_p, jit.Sib(_ST, _AX, 1, 24)) // MOVQ SP.p, 24(ST)(AX) diff --git a/encoder/assembler_amd64_go116.go b/encoder/assembler_amd64_go116.go index 0d9c60a..e68e018 100644 --- a/encoder/assembler_amd64_go116.go +++ b/encoder/assembler_amd64_go116.go @@ -412,7 +412,7 @@ func (self *_Assembler) save_state() { self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX) // MOVQ (ST), CX self.Emit("LEAQ", jit.Ptr(_CX, _StateSize), _R8) // LEAQ _StateSize(CX), R8 self.Emit("CMPQ", _R8, jit.Imm(_StackLimit)) // CMPQ R8, $_StackLimit - self.Sjmp("JA" , _LB_error_too_deep) // JA _error_too_deep + self.Sjmp("JAE" , _LB_error_too_deep) // JA _error_too_deep self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _CX, 1, 8)) // MOVQ SP.x, 8(ST)(CX) self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _CX, 1, 16)) // MOVQ SP.f, 16(ST)(CX) self.WriteRecNotAX(0, _SP_p, jit.Sib(_ST, _CX, 1, 24)) // MOVQ SP.p, 24(ST)(CX) diff --git a/encoder/assembler_amd64_go117.go b/encoder/assembler_amd64_go117.go index 9914824..6e44543 100644 --- a/encoder/assembler_amd64_go117.go +++ b/encoder/assembler_amd64_go117.go @@ -71,7 +71,7 @@ const ( ) const ( - _FP_args = 48 // 48 bytes for passing arguments to this function + _FP_args = 32 // 32 bytes for spill registers of arguments _FP_fargs = 40 // 40 bytes for passing arguments to other Go functions _FP_saves = 64 // 64 bytes for saving the registers before CALL instructions _FP_locals = 24 // 24 bytes for local variables @@ -169,7 +169,7 @@ var ( _REG_ffi = []obj.Addr{ _RP, _RL, _RC} _REG_b64 = []obj.Addr{_SP_p, _SP_q} - _REG_go = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC} + _REG_all = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC} _REG_ms = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _LR} _REG_enc = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q} ) @@ -286,6 +286,9 @@ func (self *_Assembler) epilogue() { self.Link(_LB_error) self.Emit("MOVQ", _ARG_rb, _CX) // MOVQ rb<>+0(FP), CX self.Emit("MOVQ", _RL, jit.Ptr(_CX, 8)) // MOVQ RL, 8(CX) + self.Emit("MOVQ", jit.Imm(0), _ARG_rb) // MOVQ AX, rb<>+0(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_vp) // MOVQ BX, vp<>+8(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_sb) // MOVQ CX, sb<>+16(FP) self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP self.Emit("RET") // RET @@ -420,7 +423,7 @@ func (self *_Assembler) save_state() { self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX) // MOVQ (ST), CX self.Emit("LEAQ", jit.Ptr(_CX, _StateSize), _R9) // LEAQ _StateSize(CX), R9 self.Emit("CMPQ", _R9, jit.Imm(_StackLimit)) // CMPQ R9, $_StackLimit - self.Sjmp("JA" , _LB_error_too_deep) // JA _error_too_deep + self.Sjmp("JAE" , _LB_error_too_deep) // JA _error_too_deep self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _CX, 1, 8)) // MOVQ SP.x, 8(ST)(CX) self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _CX, 1, 16)) // MOVQ SP.f, 16(ST)(CX) self.WriteRecNotAX(0, _SP_p, jit.Sib(_ST, _CX, 1, 24)) // MOVQ SP.p, 24(ST)(CX) @@ -506,9 +509,9 @@ func (self *_Assembler) call_c(pc obj.Addr) { } func (self *_Assembler) call_go(pc obj.Addr) { - self.xsave(_REG_go...) // SAVE $REG_all + self.xsave(_REG_all...) // SAVE $REG_all self.call(pc) // CALL $pc - self.xload(_REG_go...) // LOAD $REG_all + self.xload(_REG_all...) // LOAD $REG_all } func (self *_Assembler) call_more_space(pc obj.Addr) { diff --git a/encoder/encoder.go b/encoder/encoder.go index 0300a85..e0675fb 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -103,6 +103,9 @@ func EncodeInto(buf *[]byte, val interface{}, opts Options) error { err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts)) /* return the stack into pool */ + if err != nil { + resetStack(stk) + } freeStack(stk) /* avoid GC ahead */ diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go index f6b246e..1cad451 100644 --- a/encoder/encoder_test.go +++ b/encoder/encoder_test.go @@ -23,6 +23,7 @@ import ( `strconv` `sync` `testing` + `time` gojson `github.com/goccy/go-json` `github.com/json-iterator/go` @@ -41,6 +42,7 @@ func TestMain(m *testing.M) { } println("stop GC looping!") }() + time.Sleep(time.Millisecond) m.Run() } @@ -67,7 +69,6 @@ func TestGC(t *testing.T) { t.Fatal(len(out), size) } runtime.GC() - debug.FreeOSMemory() }(wg, n) } wg.Wait() diff --git a/encoder/pools.go b/encoder/pools.go index f8ae304..433dd96 100644 --- a/encoder/pools.go +++ b/encoder/pools.go @@ -29,6 +29,8 @@ import ( const ( _MaxStack = 65536 // 64k states _MaxBuffer = 1048576 // 1MB buffer size + + _StackSize = unsafe.Sizeof(_Stack{}) ) var ( @@ -101,6 +103,10 @@ func newStack() *_Stack { } } +func resetStack(p *_Stack) { + memclrNoHeapPointers(unsafe.Pointer(p), _StackSize) +} + func newBuffer() *bytes.Buffer { if ret := bufferPool.Get(); ret != nil { return ret.(*bytes.Buffer) diff --git a/encoder/stubs_go115.go b/encoder/stubs_go115.go index f28470d..3099384 100644 --- a/encoder/stubs_go115.go +++ b/encoder/stubs_go115.go @@ -56,6 +56,11 @@ func mapiterinit(t *rt.GoMapType, m *rt.GoMap, it *rt.GoMapIterator) //goland:noinspection GoUnusedParameter func isValidNumber(s string) bool +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) + func asText(v unsafe.Pointer) (string, error) { text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() diff --git a/encoder/stubs_go116.go b/encoder/stubs_go116.go index f6a9f5b..e3f4d9b 100644 --- a/encoder/stubs_go116.go +++ b/encoder/stubs_go116.go @@ -56,6 +56,11 @@ func mapiterinit(t *rt.GoMapType, m *rt.GoMap, it *rt.GoMapIterator) //goland:noinspection GoUnusedParameter func isValidNumber(s string) bool +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) + func asText(v unsafe.Pointer) (string, error) { text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() diff --git a/encoder/stubs_go117.go b/encoder/stubs_go117.go index 1bb7a07..0d47bfc 100644 --- a/encoder/stubs_go117.go +++ b/encoder/stubs_go117.go @@ -57,6 +57,11 @@ func mapiterinit(t *rt.GoMapType, m *rt.GoMap, it *rt.GoMapIterator) //goland:noinspection GoUnusedParameter func isValidNumber(s string) bool +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) + var ( _F_assertI2I = jit.Func(assertI2I2) ) diff --git a/issue_test/testmain_test.go b/issue_test/testmain_test.go index 103471c..7413e0b 100644 --- a/issue_test/testmain_test.go +++ b/issue_test/testmain_test.go @@ -21,6 +21,7 @@ import ( `runtime` `runtime/debug` `testing` + `time` ) var debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" @@ -36,5 +37,6 @@ func TestMain(m *testing.M) { debug.FreeOSMemory() } }() + time.Sleep(time.Millisecond) m.Run() } \ No newline at end of file