XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity

fix: premature GC on decoding string (#187)

Browse source 
* fix: cast `[]byte` to `string` for `Unmarshal()`

* fix: store `Decoder.s` to `_Stack` to avoid premature GC

* fix: add gcwritebarrir for `op_str`
This commit is contained in:
Yi Duan 2022-02-15 16:41:53 +08:00 committed by GitHub
parent 526c46883e
commit 5be8dafe41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 122 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
decoder/assembler_amd64_go116.go
View file

@ -671,8 +671,12 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) {
func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n}
if stack {
self.Emit("MOVQ", _DI, p) // MOVQ DI, ${p}
} else {
self.WriteRecNotAX(10, _DI, p, false, false)
}
self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1
self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n}
self.malloc(_SI, _DX) // MALLOC SI, DX
@ -682,7 +686,7 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
// no need for writeBarrier
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
} else {
self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p}
self.WriteRecNotAX(2, _DX, p, true, false) // MOVQ DX, ${p}
}
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
self.Emit("XORL" , _R8, _R8) // XORL R8, R8
@ -706,9 +710,13 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -2), jit.Imm('"')) // CMPB -2(IP)(IC), $'"'
self.Sjmp("JNE" , _LB_char_m2_error) // JNE _char_m2_error
self.slice_from(_VAR_st_Iv, -3) // SLICE st.Iv, $-3
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n}
self.Emit("MOVQ" , _SI, _AX) // MOVQ SI, AX
if stack {
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
} else {
self.WriteRecNotAX(9, _DI, p, false, false)
}
self.Emit("ADDQ" , _VAR_st_Iv, _AX) // ADDQ st.Iv, AX
self.Emit("CMPQ" , _VAR_st_Ep, _AX) // CMPQ st.Ep, AX
self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n}
@ -717,9 +725,9 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI
if stack {
// no need for writeBarrier
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
} else {
self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p}
self.WriteRecNotAX(2, _DX, p, true, false) // MOVQ DX, ${p}
}
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8

20
decoder/assembler_amd64_go117.go
View file

@ -681,18 +681,21 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) {
func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n}
if stack {
self.Emit("MOVQ", _DI, p) // MOVQ DI, ${p}
} else {
self.WriteRecNotAX(10, _DI, p, false, false)
}
self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1
self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n}
self.malloc(_SI, _DX) // MALLOC SI, DX
self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI
self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI
if stack {
// no need for writeBarrier
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
} else {
self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p}
self.WriteRecNotAX(2, _DX, p, true, false) // MOVQ DX, ${p}
}
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
self.Emit("XORL" , _R8, _R8) // XORL R8, R8
@ -716,9 +719,13 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -2), jit.Imm('"')) // CMPB -2(IP)(IC), $'"'
self.Sjmp("JNE" , _LB_char_m2_error) // JNE _char_m2_error
self.slice_from(_VAR_st_Iv, -3) // SLICE st.Iv, $-3
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n}
self.Emit("MOVQ" , _SI, _AX) // MOVQ SI, AX
if stack {
self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p}
} else {
self.WriteRecNotAX(9, _DI, p, false, false)
}
self.Emit("ADDQ" , _VAR_st_Iv, _AX) // ADDQ st.Iv, AX
self.Emit("CMPQ" , _VAR_st_Ep, _AX) // CMPQ st.Ep, AX
self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n}
@ -726,10 +733,9 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI
self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI
if stack {
// no need for writeBarrier
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
self.Emit("MOVQ", _DX, p) // MOVQ DX, ${p}
} else {
self.WriteRecNotAX(2, _DX, p, true, true) // MOVQ DX, ${p}
self.WriteRecNotAX(2, _DX, p, true, false) // MOVQ DX, ${p}
}
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8

94
issue_test/issue186_test.go Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package issue_test
import (
`math/rand`
`runtime`
`sync`
`testing`
`time`
`github.com/bytedance/sonic`
)
type GlobalConfig []Conf
type Conf struct {
A string `json:"A"`
B SubConf `json:"B"`
C []string `json:"C"`
}
type SubConf struct {
Slice []int64 `json:"Slice"`
Map map[int64]bool `json:"-"`
}
func IntSlide2Map(l []int64) map[int64]bool {
m := make(map[int64]bool)
for _, item := range l {
m[item] = true
}
return m
}
func Reload(t *testing.T, rawData string) (tmp GlobalConfig) {
buf := []byte(rawData)
runtime.GC()
// t.Logf("got bytes %x\n", unsafe.Pointer(&buf[0]))
// runtime.SetFinalizer(&buf[0], func(x *byte){
// t.Logf("&byte %x got free\n", x)
// })
err := sonic.Unmarshal(buf, &tmp) // better use sonic.UnmarshalString()!
if err != nil {
t.Fatalf("failed to unmarshal json, raw data: %v, err: %v", rawData, err)
}
runtime.GC()
// t.Log("unmarshal done")
for index, conf := range tmp {
tmp[index].B.Map = IntSlide2Map(conf.B.Slice)
}
runtime.GC()
// t.Log("calc done")
return
}
func TestIssue186(t *testing.T) {
t.Parallel()
var data = `[{"A":"xxx","B":{"Slice":[111]}},{"A":"yyy","B":{"Slice":[222]},"C":["extra"]},{"A":"zzz","B":{"Slice":[333]},"C":["extra"]},{"A":"zzz","B":{"Slice":[333]},"C":["extra"]},{"A":"zzz","B":{"Slice":[1111111111,2222222222,3333333333,44444444444,55555555555]},"C":["extra","aaaaaaaaaaaa","bbbbbbbbbbbbb","ccccccccccccc","ddddddddddddd"]}]`
// var obj interface{}
for k:=0; k<100; k++ {
wg := sync.WaitGroup{}
for i:=0; i<1000; i++ {
wg.Add(1)
go func(){
defer wg.Done()
time.Sleep(time.Duration(rand.Intn(100)+1000))
tmp := Reload(t, data)
runtime.GC()
_ = tmp[0].A
runtime.GC()
// obj = tmp
}()
}
runtime.GC()
// t.Log(obj)
wg.Wait()
}
}

3
sonic.go
View file

@ -25,6 +25,7 @@ import (
`github.com/bytedance/sonic/encoder`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
)
const (
@ -39,7 +40,7 @@ func Marshal(val interface{}) ([]byte, error) {
// Unmarshal parses the JSON-encoded data and stores the result in the value
// pointed to by v.
func Unmarshal(buf []byte, val interface{}) error {
return UnmarshalString(string(buf), val)
return UnmarshalString(rt.Mem2Str(buf), val)
}
// UnmarshalString is like Unmarshal, except buf is a string.