fix(#100): check type-size of map element to decide whether use mapassign_fastxx (#102)

Co-authored-by: duanyi.aster <duanyi.aster@bytedance.com>

decode_test.go

@@ -17,21 +17,21 @@
 package sonic
 
 import (
-	"bytes"
+	`bytes`
-	"encoding"
+	`encoding`
 	`encoding/json`
-	"errors"
+	`errors`
-	"fmt"
+	`fmt`
-	"image"
+	`image`
-	"math"
+	`math`
-	"math/big"
+	`math/big`
 	`math/rand`
-	"net"
+	`net`
-	"reflect"
+	`reflect`
-	"strconv"
+	`strconv`
-	"strings"
+	`strings`
-	"testing"
+	`testing`
-	"time"
+	`time`
 	`unsafe`
 
 	`github.com/bytedance/sonic/decoder`

decoder/assembler_amd64.go

@@ -768,7 +768,7 @@ func (self *_Assembler) mapassign_std(t reflect.Type, v obj.Addr) {
     self.mapassign_call(t, _F_mapassign)    // MAPASSIGN ${t}, mapassign
 }
 
-func (self *_Assembler) mapassign_str(t reflect.Type, p obj.Addr, n obj.Addr) {
+func (self *_Assembler) mapassign_str_fast(t reflect.Type, p obj.Addr, n obj.Addr) {
     self.Emit("MOVQ", jit.Type(t), _AX)         // MOVQ    ${t}, AX
     self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0))     // MOVQ    AX, (SP)
     self.Emit("MOVQ", _VP, jit.Ptr(_SP, 8))     // MOVQ    VP, 8(SP)
@@ -1201,13 +1201,21 @@ func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) {
 func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) {
     self.parse_signed()                                                     // PARSE     int32
     self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32)     // RANGE     int32
-    self.mapassign_fastx(p.vt(), _F_mapassign_fast32)                       // MAPASSIGN int32, mapassign_fast32
+    if vt := p.vt(); !mapfast(vt) {
+        self.mapassign_std(vt, _VAR_st_Iv)                                  // MAPASSIGN int32, mapassign, st.Iv
+    } else {
+        self.mapassign_fastx(vt, _F_mapassign_fast32)                       // MAPASSIGN int32, mapassign_fast32
+    }
 }
 
 func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) {
     self.parse_signed()                                 // PARSE     int64
-    self.Emit("MOVQ", _VAR_st_Iv, _AX)                  // MOVQ      st.Iv, AX
+    if vt := p.vt(); !mapfast(vt) {
-    self.mapassign_fastx(p.vt(), _F_mapassign_fast64)   // MAPASSIGN int64, mapassign_fast64
+        self.mapassign_std(vt, _VAR_st_Iv)              // MAPASSIGN int64, mapassign, st.Iv
+    } else {
+        self.Emit("MOVQ", _VAR_st_Iv, _AX)              // MOVQ      st.Iv, AX
+        self.mapassign_fastx(vt, _F_mapassign_fast64)   // MAPASSIGN int64, mapassign_fast64
+    }
 }
 
 func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) {
@@ -1225,13 +1233,21 @@ func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) {
 func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) {
     self.parse_unsigned()                                       // PARSE     uint32
     self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32)   // RANGE     uint32
-    self.mapassign_fastx(p.vt(), _F_mapassign_fast32)           // MAPASSIGN uint32, mapassign_fast32
+    if vt := p.vt(); !mapfast(vt) {
+        self.mapassign_std(vt, _VAR_st_Iv)                      // MAPASSIGN uint32, vt.Iv
+    } else {
+        self.mapassign_fastx(vt, _F_mapassign_fast32)           // MAPASSIGN uint32, mapassign_fast32
+    }
 }
 
 func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) {
-    self.parse_unsigned()                               // PARSE     uint64
+    self.parse_unsigned()                                       // PARSE     uint64
-    self.Emit("MOVQ", _VAR_st_Iv, _AX)                  // MOVQ      st.Iv, AX
+    if vt := p.vt(); !mapfast(vt) {
-    self.mapassign_fastx(p.vt(), _F_mapassign_fast64)   // MAPASSIGN uint64, mapassign_fast64
+        self.mapassign_std(vt, _VAR_st_Iv)                      // MAPASSIGN uint64, vt.Iv
+    } else {
+        self.Emit("MOVQ", _VAR_st_Iv, _AX)                      // MOVQ      st.Iv, AX
+        self.mapassign_fastx(vt, _F_mapassign_fast64)           // MAPASSIGN uint64, mapassign_fast64
+    }
 }
 
 func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) {
@@ -1247,11 +1263,15 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) {
 }
 
 func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) {
-    self.parse_string()                         // PARSE     STRING
+    self.parse_string()                          // PARSE     STRING
-    self.unquote_once(_VAR_sv_p, _VAR_sv_n)     // UNQUOTE   once, sv.p, sv.n
+    self.unquote_once(_VAR_sv_p, _VAR_sv_n)      // UNQUOTE   once, sv.p, sv.n
-    self.Emit("MOVQ", _VAR_sv_p, _DI)           // MOVQ      sv.p, DI
+    if vt := p.vt(); !mapfast(vt) {
-    self.Emit("MOVQ", _VAR_sv_n, _SI)           // MOVQ      sv.n, SI
+        self.mapassign_std(vt, _VAR_sv_p)        // MAPASSIGN string, DI, SI
-    self.mapassign_str(p.vt(), _DI, _SI)        // MAPASSIGN string, DI, SI
+    } else {
+        self.Emit("MOVQ", _VAR_sv_p, _DI)        // MOVQ      sv.p, DI
+        self.Emit("MOVQ", _VAR_sv_n, _SI)        // MOVQ      sv.n, SI
+        self.mapassign_str_fast(vt, _DI, _SI)    // MAPASSIGN string, DI, SI
+    }
 }
 
 func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) {

decoder/stubs.go

@@ -18,6 +18,7 @@ package decoder
 
 import (
     `unsafe`
+    `reflect`
 
     _ `github.com/chenzhuoyu/base64x`
 
@@ -27,6 +28,14 @@ import (
 //go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode
 var _subr__b64decode uintptr
 
+// runtime.maxElementSize
+const _max_map_element_size uintptr = 128
+
+func mapfast(vt reflect.Type) bool {
+    return vt.Elem().Size() <= _max_map_element_size
+}
+
+
 //go:nosplit
 //go:linkname throw runtime.throw
 //goland:noinspection GoUnusedParameter

encode_test.go

@@ -17,16 +17,16 @@
 package sonic
 
 import (
-	"bytes"
+	`bytes`
-	"encoding"
+	`encoding`
 	`encoding/json`
-	"fmt"
+	`fmt`
-	"log"
+	`log`
-	"math"
+	`math`
-	"reflect"
+	`reflect`
-	"regexp"
+	`regexp`
-	"strconv"
+	`strconv`
-	"testing"
+	`testing`
 	`unsafe`
 
 	`github.com/bytedance/sonic/encoder`

issue100_test.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sonic
+
+import (
+	`fmt`
+	`reflect`
+	_ `sync`
+	`testing`
+	`unsafe`
+
+	stdjson `encoding/json`
+)
+
+func TestLargeMapValue(t *testing.T) {
+	var jsonStr = `{
+		"1": {},
+		"2": {},
+		"3": {},
+		"4": {},
+		"5": {},
+		"6": {},
+		"7": {},
+		"8": {},
+		"9": {}
+	}`
+	type Case struct {
+		std interface{}
+		sonic interface{}
+	}
+	cases := []Case{
+		{&map[string]TestIssue100_LargeMapValue{},            &map[string]TestIssue100_LargeMapValue{}},
+		{&map[int32]TestIssue100_LargeMapValue{},             &map[int32]TestIssue100_LargeMapValue{}},
+		{&map[int64]TestIssue100_LargeMapValue{},             &map[int64]TestIssue100_LargeMapValue{}},
+		{&map[uint32]TestIssue100_LargeMapValue{},            &map[uint32]TestIssue100_LargeMapValue{}},
+		{&map[uint64]TestIssue100_LargeMapValue{},            &map[uint64]TestIssue100_LargeMapValue{}},
+		{&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{},    &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}},
+		{&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}},
+	}
+	for i, c := range cases {
+		var stdw, sonicw = c.std, c.sonic
+		if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil {
+			t.Fatal(i, err)
+		}
+		fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len())
+		if err := Unmarshal([]byte(jsonStr), sonicw); err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(stdw, sonicw) {
+			fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw)
+			t.Fatal(i)
+		}
+	}
+}
+
+type TestIssue100_textMarshalKey string 
+
+func(self TestIssue100_textMarshalKey) UnmarshalText(text []byte) error {
+	self = TestIssue100_textMarshalKey(text)
+	return nil
+}
+
+type TestIssue100_textMarshalKeyPtr string 
+
+func(self *TestIssue100_textMarshalKeyPtr) UnmarshalText(text []byte) error {
+	*self = TestIssue100_textMarshalKeyPtr(text)
+	return nil
+}
+
+type TestIssue100_LargeMapValue struct {
+	Id [129]byte 
+}