From c1749cfd1ff4b699364a6f8e143359d7072cd955 Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Thu, 16 Sep 2021 20:13:29 +0800 Subject: [PATCH] fix(#100): check type-size of map element to decide whether use mapassign_fastxx (#102) Co-authored-by: duanyi.aster --- decode_test.go | 26 ++++++------ decoder/assembler_amd64.go | 46 ++++++++++++++------ decoder/stubs.go | 9 ++++ encode_test.go | 18 ++++---- issue100_test.go | 86 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 35 deletions(-) create mode 100644 issue100_test.go diff --git a/decode_test.go b/decode_test.go index 204babf..08ce681 100644 --- a/decode_test.go +++ b/decode_test.go @@ -17,21 +17,21 @@ package sonic import ( - "bytes" - "encoding" + `bytes` + `encoding` `encoding/json` - "errors" - "fmt" - "image" - "math" - "math/big" + `errors` + `fmt` + `image` + `math` + `math/big` `math/rand` - "net" - "reflect" - "strconv" - "strings" - "testing" - "time" + `net` + `reflect` + `strconv` + `strings` + `testing` + `time` `unsafe` `github.com/bytedance/sonic/decoder` diff --git a/decoder/assembler_amd64.go b/decoder/assembler_amd64.go index 37363bd..edac21d 100644 --- a/decoder/assembler_amd64.go +++ b/decoder/assembler_amd64.go @@ -768,7 +768,7 @@ func (self *_Assembler) mapassign_std(t reflect.Type, v obj.Addr) { self.mapassign_call(t, _F_mapassign) // MAPASSIGN ${t}, mapassign } -func (self *_Assembler) mapassign_str(t reflect.Type, p obj.Addr, n obj.Addr) { +func (self *_Assembler) mapassign_str_fast(t reflect.Type, p obj.Addr, n obj.Addr) { self.Emit("MOVQ", jit.Type(t), _AX) // MOVQ ${t}, AX self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) self.Emit("MOVQ", _VP, jit.Ptr(_SP, 8)) // MOVQ VP, 8(SP) @@ -1201,13 +1201,21 @@ func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { self.parse_signed() // PARSE int32 self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32 - self.mapassign_fastx(p.vt(), _F_mapassign_fast32) // MAPASSIGN int32, mapassign_fast32 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv + } else { + self.mapassign_fastx(vt, _F_mapassign_fast32) // MAPASSIGN int32, mapassign_fast32 + } } func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { self.parse_signed() // PARSE int64 - self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX - self.mapassign_fastx(p.vt(), _F_mapassign_fast64) // MAPASSIGN int64, mapassign_fast64 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv + } else { + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.mapassign_fastx(vt, _F_mapassign_fast64) // MAPASSIGN int64, mapassign_fast64 + } } func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) { @@ -1225,13 +1233,21 @@ func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { self.parse_unsigned() // PARSE uint32 self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32 - self.mapassign_fastx(p.vt(), _F_mapassign_fast32) // MAPASSIGN uint32, mapassign_fast32 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv + } else { + self.mapassign_fastx(vt, _F_mapassign_fast32) // MAPASSIGN uint32, mapassign_fast32 + } } func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) { - self.parse_unsigned() // PARSE uint64 - self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX - self.mapassign_fastx(p.vt(), _F_mapassign_fast64) // MAPASSIGN uint64, mapassign_fast64 + self.parse_unsigned() // PARSE uint64 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv + } else { + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.mapassign_fastx(vt, _F_mapassign_fast64) // MAPASSIGN uint64, mapassign_fast64 + } } func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) { @@ -1247,11 +1263,15 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { } func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { - self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n - self.Emit("MOVQ", _VAR_sv_p, _DI) // MOVQ sv.p, DI - self.Emit("MOVQ", _VAR_sv_n, _SI) // MOVQ sv.n, SI - self.mapassign_str(p.vt(), _DI, _SI) // MAPASSIGN string, DI, SI + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_sv_p) // MAPASSIGN string, DI, SI + } else { + self.Emit("MOVQ", _VAR_sv_p, _DI) // MOVQ sv.p, DI + self.Emit("MOVQ", _VAR_sv_n, _SI) // MOVQ sv.n, SI + self.mapassign_str_fast(vt, _DI, _SI) // MAPASSIGN string, DI, SI + } } func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) { diff --git a/decoder/stubs.go b/decoder/stubs.go index 134c772..dcf1c83 100644 --- a/decoder/stubs.go +++ b/decoder/stubs.go @@ -18,6 +18,7 @@ package decoder import ( `unsafe` + `reflect` _ `github.com/chenzhuoyu/base64x` @@ -27,6 +28,14 @@ import ( //go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode var _subr__b64decode uintptr +// runtime.maxElementSize +const _max_map_element_size uintptr = 128 + +func mapfast(vt reflect.Type) bool { + return vt.Elem().Size() <= _max_map_element_size +} + + //go:nosplit //go:linkname throw runtime.throw //goland:noinspection GoUnusedParameter diff --git a/encode_test.go b/encode_test.go index c5586ca..efdca0b 100644 --- a/encode_test.go +++ b/encode_test.go @@ -17,16 +17,16 @@ package sonic import ( - "bytes" - "encoding" + `bytes` + `encoding` `encoding/json` - "fmt" - "log" - "math" - "reflect" - "regexp" - "strconv" - "testing" + `fmt` + `log` + `math` + `reflect` + `regexp` + `strconv` + `testing` `unsafe` `github.com/bytedance/sonic/encoder` diff --git a/issue100_test.go b/issue100_test.go new file mode 100644 index 0000000..d72b895 --- /dev/null +++ b/issue100_test.go @@ -0,0 +1,86 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sonic + +import ( + `fmt` + `reflect` + _ `sync` + `testing` + `unsafe` + + stdjson `encoding/json` +) + +func TestLargeMapValue(t *testing.T) { + var jsonStr = `{ + "1": {}, + "2": {}, + "3": {}, + "4": {}, + "5": {}, + "6": {}, + "7": {}, + "8": {}, + "9": {} + }` + type Case struct { + std interface{} + sonic interface{} + } + cases := []Case{ + {&map[string]TestIssue100_LargeMapValue{}, &map[string]TestIssue100_LargeMapValue{}}, + {&map[int32]TestIssue100_LargeMapValue{}, &map[int32]TestIssue100_LargeMapValue{}}, + {&map[int64]TestIssue100_LargeMapValue{}, &map[int64]TestIssue100_LargeMapValue{}}, + {&map[uint32]TestIssue100_LargeMapValue{}, &map[uint32]TestIssue100_LargeMapValue{}}, + {&map[uint64]TestIssue100_LargeMapValue{}, &map[uint64]TestIssue100_LargeMapValue{}}, + {&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}}, + {&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}}, + } + for i, c := range cases { + var stdw, sonicw = c.std, c.sonic + if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil { + t.Fatal(i, err) + } + fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len()) + if err := Unmarshal([]byte(jsonStr), sonicw); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(stdw, sonicw) { + fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw) + t.Fatal(i) + } + } +} + +type TestIssue100_textMarshalKey string + +func(self TestIssue100_textMarshalKey) UnmarshalText(text []byte) error { + self = TestIssue100_textMarshalKey(text) + return nil +} + +type TestIssue100_textMarshalKeyPtr string + +func(self *TestIssue100_textMarshalKeyPtr) UnmarshalText(text []byte) error { + *self = TestIssue100_textMarshalKeyPtr(text) + return nil +} + +type TestIssue100_LargeMapValue struct { + Id [129]byte +} \ No newline at end of file