fix: premature GC on decoding string (#187)

* fix: cast `[]byte` to `string` for `Unmarshal()`

* fix: store `Decoder.s` to `_Stack` to avoid premature GC

* fix: add gcwritebarrir for `op_str`

decoder/assembler_amd64_go116.go

@@ -671,8 +671,12 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) {
 
 func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
     self.slice_from(_VAR_st_Iv, -1)                             // SLICE  st.Iv, $-1
-    self.Emit("MOVQ" , _DI, p)                                  // MOVQ   DI, ${p}
     self.Emit("MOVQ" , _SI, n)                                  // MOVQ   SI, ${n}
+    if stack {
+        self.Emit("MOVQ", _DI, p)                               // MOVQ   DI, ${p}
+    } else {
+        self.WriteRecNotAX(10, _DI, p, false, false)
+    }
     self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1))                 // CMPQ   st.Ep, $-1
     self.Sjmp("JE"   , "_noescape_{n}")                         // JE     _noescape_{n}
     self.malloc(_SI, _DX)                                       // MALLOC SI, DX
@@ -682,7 +686,7 @@ func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
         // no need for writeBarrier
         self.Emit("MOVQ", _DX, p)                               // MOVQ   DX, ${p}
     } else {
-        self.WriteRecNotAX(2, _DX, p, true, true)               // MOVQ   DX, ${p}
+        self.WriteRecNotAX(2, _DX, p, true, false)               // MOVQ   DX, ${p}
     }
     self.Emit("LEAQ" , _VAR_sr, _CX)                            // LEAQ   sr, CX
     self.Emit("XORL" , _R8, _R8)                                // XORL   R8, R8
@@ -706,9 +710,13 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
     self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -2), jit.Imm('"'))      // CMPB   -2(IP)(IC), $'"'
     self.Sjmp("JNE"  , _LB_char_m2_error)                           // JNE    _char_m2_error
     self.slice_from(_VAR_st_Iv, -3)                                 // SLICE  st.Iv, $-3
-    self.Emit("MOVQ" , _DI, p)                                      // MOVQ   DI, ${p}
     self.Emit("MOVQ" , _SI, n)                                      // MOVQ   SI, ${n}
     self.Emit("MOVQ" , _SI, _AX)                                    // MOVQ   SI, AX
+    if stack {
+        self.Emit("MOVQ" , _DI, p)                                  // MOVQ   DI, ${p}
+    } else {
+        self.WriteRecNotAX(9, _DI, p, false, false)
+    }
     self.Emit("ADDQ" , _VAR_st_Iv, _AX)                             // ADDQ   st.Iv, AX
     self.Emit("CMPQ" , _VAR_st_Ep, _AX)                             // CMPQ   st.Ep, AX
     self.Sjmp("JE"   , "_noescape_{n}")                             // JE     _noescape_{n}
@@ -717,9 +725,9 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
     self.Emit("MOVQ" , n, _SI)                                      // MOVQ   ${n}, SI
     if stack {
         // no need for writeBarrier
-        self.Emit("MOVQ", _DX, p)                               // MOVQ   DX, ${p}
+        self.Emit("MOVQ", _DX, p)                                   // MOVQ   DX, ${p}
     } else {
-        self.WriteRecNotAX(2, _DX, p, true, true)               // MOVQ   DX, ${p}
+        self.WriteRecNotAX(2, _DX, p, true, false)                  // MOVQ   DX, ${p}
     }
     self.Emit("LEAQ" , _VAR_sr, _CX)                                // LEAQ   sr, CX
     self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8)        // MOVL   ${types.F_DOUBLE_UNQUOTE}, R8

decoder/assembler_amd64_go117.go

@@ -681,18 +681,21 @@ func (self *_Assembler) slice_from_r(p obj.Addr, d int64) {
 
 func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr, stack bool) {
     self.slice_from(_VAR_st_Iv, -1)                             // SLICE  st.Iv, $-1
-    self.Emit("MOVQ" , _DI, p)                                  // MOVQ   DI, ${p}
     self.Emit("MOVQ" , _SI, n)                                  // MOVQ   SI, ${n}
+    if stack {
+        self.Emit("MOVQ", _DI, p)                               // MOVQ   DI, ${p}
+    } else {
+        self.WriteRecNotAX(10, _DI, p, false, false)
+    }
     self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1))                 // CMPQ   st.Ep, $-1
     self.Sjmp("JE"   , "_noescape_{n}")                         // JE     _noescape_{n}
     self.malloc(_SI, _DX)                                       // MALLOC SI, DX
     self.Emit("MOVQ" , p, _DI)                                  // MOVQ   ${p}, DI
     self.Emit("MOVQ" , n, _SI)                                  // MOVQ   ${n}, SI
     if stack {
-        // no need for writeBarrier
         self.Emit("MOVQ", _DX, p)                               // MOVQ   DX, ${p}
     } else {
-        self.WriteRecNotAX(2, _DX, p, true, true)               // MOVQ   DX, ${p}
+        self.WriteRecNotAX(2, _DX, p, true, false)              // MOVQ   DX, ${p}
     }
     self.Emit("LEAQ" , _VAR_sr, _CX)                            // LEAQ   sr, CX
     self.Emit("XORL" , _R8, _R8)                                // XORL   R8, R8
@@ -716,9 +719,13 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
     self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -2), jit.Imm('"'))      // CMPB   -2(IP)(IC), $'"'
     self.Sjmp("JNE"  , _LB_char_m2_error)                           // JNE    _char_m2_error
     self.slice_from(_VAR_st_Iv, -3)                                 // SLICE  st.Iv, $-3
-    self.Emit("MOVQ" , _DI, p)                                      // MOVQ   DI, ${p}
     self.Emit("MOVQ" , _SI, n)                                      // MOVQ   SI, ${n}
     self.Emit("MOVQ" , _SI, _AX)                                    // MOVQ   SI, AX
+    if stack {
+        self.Emit("MOVQ" , _DI, p)                                  // MOVQ   DI, ${p}
+    } else {
+        self.WriteRecNotAX(9, _DI, p, false, false)
+    }
     self.Emit("ADDQ" , _VAR_st_Iv, _AX)                             // ADDQ   st.Iv, AX
     self.Emit("CMPQ" , _VAR_st_Ep, _AX)                             // CMPQ   st.Ep, AX
     self.Sjmp("JE"   , "_noescape_{n}")                             // JE     _noescape_{n}
@@ -726,10 +733,9 @@ func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr, stack bool) {
     self.Emit("MOVQ" , p, _DI)                                      // MOVQ   ${p}, DI
     self.Emit("MOVQ" , n, _SI)                                      // MOVQ   ${n}, SI
     if stack {
-        // no need for writeBarrier
+        self.Emit("MOVQ", _DX, p)                                   // MOVQ   DX, ${p}
-        self.Emit("MOVQ", _DX, p)                               // MOVQ   DX, ${p}
     } else {
-        self.WriteRecNotAX(2, _DX, p, true, true)               // MOVQ   DX, ${p}
+        self.WriteRecNotAX(2, _DX, p, true, false)                  // MOVQ   DX, ${p}
     }
     self.Emit("LEAQ" , _VAR_sr, _CX)                                // LEAQ   sr, CX
     self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8)        // MOVL   ${types.F_DOUBLE_UNQUOTE}, R8

issue_test/issue186_test.go

@@ -0,0 +1,94 @@
+/*
+* Copyright 2021 ByteDance Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+
+package issue_test
+
+import (
+    `math/rand`
+    `runtime`
+    `sync`
+    `testing`
+    `time`
+
+    `github.com/bytedance/sonic`
+)
+
+
+type GlobalConfig []Conf
+
+type Conf struct {
+    A  string             `json:"A"`
+    B  SubConf            `json:"B"`
+    C []string            `json:"C"`
+}
+
+type SubConf struct {
+    Slice        []int64 `json:"Slice"`
+    Map          map[int64]bool `json:"-"`
+}
+
+func IntSlide2Map(l []int64) map[int64]bool {
+    m := make(map[int64]bool)
+    for _, item := range l {
+        m[item] = true
+    }
+    return m
+}
+
+func Reload(t *testing.T, rawData string) (tmp GlobalConfig) {
+    buf := []byte(rawData)
+    runtime.GC()
+    // t.Logf("got bytes %x\n", unsafe.Pointer(&buf[0]))
+    // runtime.SetFinalizer(&buf[0], func(x *byte){
+        // t.Logf("&byte %x got free\n", x)
+    // })
+    err := sonic.Unmarshal(buf, &tmp) // better use sonic.UnmarshalString()!
+    if err != nil {
+        t.Fatalf("failed to unmarshal json, raw data: %v, err: %v", rawData, err)
+    }
+    runtime.GC()
+    // t.Log("unmarshal done")
+    for index, conf := range tmp {
+        tmp[index].B.Map = IntSlide2Map(conf.B.Slice)
+    }
+    runtime.GC()
+    // t.Log("calc done")
+    return
+}
+
+func TestIssue186(t *testing.T) {
+    t.Parallel()
+    var data = `[{"A":"xxx","B":{"Slice":[111]}},{"A":"yyy","B":{"Slice":[222]},"C":["extra"]},{"A":"zzz","B":{"Slice":[333]},"C":["extra"]},{"A":"zzz","B":{"Slice":[333]},"C":["extra"]},{"A":"zzz","B":{"Slice":[1111111111,2222222222,3333333333,44444444444,55555555555]},"C":["extra","aaaaaaaaaaaa","bbbbbbbbbbbbb","ccccccccccccc","ddddddddddddd"]}]`
+    // var obj interface{}
+    for k:=0; k<100; k++ {
+        wg := sync.WaitGroup{}
+        for i:=0; i<1000; i++ {
+            wg.Add(1)
+            go func(){
+                defer wg.Done()
+                time.Sleep(time.Duration(rand.Intn(100)+1000))
+                tmp := Reload(t, data)
+                runtime.GC()
+                _ = tmp[0].A
+                runtime.GC()
+                // obj = tmp
+            }()
+        }
+        runtime.GC()
+        // t.Log(obj)
+        wg.Wait()
+    }
+}

sonic.go

@@ -25,6 +25,7 @@ import (
     `github.com/bytedance/sonic/encoder`
     `github.com/bytedance/sonic/option`
     `github.com/bytedance/sonic/internal/native/types`
+    `github.com/bytedance/sonic/internal/rt`
 )
 
 const (
@@ -39,7 +40,7 @@ func Marshal(val interface{}) ([]byte, error) {
 // Unmarshal parses the JSON-encoded data and stores the result in the value
 // pointed to by v.
 func Unmarshal(buf []byte, val interface{}) error {
-    return UnmarshalString(string(buf), val)
+    return UnmarshalString(rt.Mem2Str(buf), val)
 }
 
 // UnmarshalString is like Unmarshal, except buf is a string.