From 8dbcce341ba0c0c9db5d5f68c6c20eaa0f1e677e Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Tue, 14 Mar 2023 13:49:27 +0800 Subject: [PATCH] fix: decimal map key may be truncated and populated unexpectedly (#382) * fix:(StreamDecoder) Integer map key may be truncated and populated unexpectedly * feat: expose option `DefaultBufferSize` * test adjust * test adjust * feat: add options to make default buffer size adjustable --- decoder/assembler_amd64_go116.go | 20 ++++++++++-- decoder/assembler_amd64_go117.go | 18 ++++++++-- decoder/assembler_test.go | 16 ++++----- decoder/compiler.go | 10 ------ decoder/stream.go | 11 +++---- decoder/stream_test.go | 56 +++++++++++++++++--------------- encoder/assembler_test.go | 3 +- encoder/pools.go | 5 ++- issue_test/issue381_test.go | 52 +++++++++++++++++++++++++++++ option/option.go | 8 +++++ 10 files changed, 139 insertions(+), 60 deletions(-) create mode 100644 issue_test/issue381_test.go diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 9ff1ad2..a839b16 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -1487,18 +1487,21 @@ func (self *_Assembler) _asm_OP_map_init(_ *_Instr) { func (self *_Assembler) _asm_OP_map_key_i8(p *_Instr) { self.parse_signed(int8Type, "", p.vi()) // PARSE int8 self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int8, mapassign, st.Iv } func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) { self.parse_signed(int16Type, "", p.vi()) // PARSE int16 self.range_signed(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int16, mapassign, st.Iv } func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { self.parse_signed(int32Type, "", p.vi()) // PARSE int32 self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv } else { @@ -1508,6 +1511,7 @@ func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { self.parse_signed(int64Type, "", p.vi()) // PARSE int64 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv } else { @@ -1519,18 +1523,21 @@ func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) { self.parse_unsigned(uint8Type, "", p.vi()) // PARSE uint8 self.range_unsigned(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv } func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) { self.parse_unsigned(uint16Type, "", p.vi()) // PARSE uint16 self.range_unsigned(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint16, vt.Iv } func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { self.parse_unsigned(uint32Type, "", p.vi()) // PARSE uint32 self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv } else { @@ -1540,6 +1547,7 @@ func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) { self.parse_unsigned(uint64Type, "", p.vi()) // PARSE uint64 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv } else { @@ -1552,11 +1560,13 @@ func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) { self.parse_number(float32Type, "", p.vi()) // PARSE NUMBER self.range_single() // RANGE float32 self.Emit("MOVSS", _X0, _VAR_st_Dv) // MOVSS X0, st.Dv + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv } func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { self.parse_number(float64Type, "", p.vi()) // PARSE NUMBER + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv } @@ -1787,10 +1797,14 @@ func (self *_Assembler) lspace(subfix string) { } func (self *_Assembler) _asm_OP_match_char(p *_Instr) { + self.match_char(p.vb()) +} + +func (self *_Assembler) match_char(char byte) { self.check_eof(1) - self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} - self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error - self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(char))) // CMPB (IP)(IC), ${p.vb()} + self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error + self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC } func (self *_Assembler) _asm_OP_check_char(p *_Instr) { diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index 8a70cff..5e852e5 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -1482,18 +1482,21 @@ func (self *_Assembler) _asm_OP_map_init(_ *_Instr) { func (self *_Assembler) _asm_OP_map_key_i8(p *_Instr) { self.parse_signed(int8Type, "", p.vi()) // PARSE int8 self.range_signed_CX(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int8, mapassign, st.Iv } func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) { self.parse_signed(int16Type, "", p.vi()) // PARSE int16 self.range_signed_CX(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int16, mapassign, st.Iv } func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { self.parse_signed(int32Type, "", p.vi()) // PARSE int32 self.range_signed_CX(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv } else { @@ -1504,6 +1507,7 @@ func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { self.parse_signed(int64Type, "", p.vi()) // PARSE int64 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv } else { @@ -1515,18 +1519,21 @@ func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) { self.parse_unsigned(uint8Type, "", p.vi()) // PARSE uint8 self.range_unsigned_CX(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8 - self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv + self.match_char('"') + self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv } func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) { self.parse_unsigned(uint16Type, "", p.vi()) // PARSE uint16 self.range_unsigned_CX(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16 + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint16, vt.Iv } func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { self.parse_unsigned(uint32Type, "", p.vi()) // PARSE uint32 self.range_unsigned_CX(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv } else { @@ -1537,6 +1544,7 @@ func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) { self.parse_unsigned(uint64Type, "", p.vi()) // PARSE uint64 + self.match_char('"') if vt := p.vt(); !mapfast(vt) { self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv } else { @@ -1549,11 +1557,13 @@ func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) { self.parse_number(float32Type, "", p.vi()) // PARSE NUMBER self.range_single_X0() // RANGE float32 self.Emit("MOVSS", _X0, _VAR_st_Dv) // MOVSS X0, st.Dv + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv } func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { self.parse_number(float64Type, "", p.vi()) // PARSE NUMBER + self.match_char('"') self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv } @@ -1767,8 +1777,12 @@ func (self *_Assembler) lspace(subfix string) { } func (self *_Assembler) _asm_OP_match_char(p *_Instr) { + self.match_char(p.vb()) +} + +func (self *_Assembler) match_char(char byte) { self.check_eof(1) - self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(char))) // CMPB (IP)(IC), ${p.vb()} self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC } diff --git a/decoder/assembler_test.go b/decoder/assembler_test.go index 3598b1a..ff37391 100644 --- a/decoder/assembler_test.go +++ b/decoder/assembler_test.go @@ -418,49 +418,49 @@ func TestAssembler_OpCode(t *testing.T) { }, { key: "_OP_map_key_i8", ins: []_Instr{newInsVt(_OP_map_key_i8, reflect.TypeOf(map[int8]int{}))}, - src: `123`, + src: `123"`, exp: map[int8]int{123: 0}, val: map[int8]int{}, }, { key: "_OP_map_key_i32", ins: []_Instr{newInsVt(_OP_map_key_i32, reflect.TypeOf(map[int32]int{}))}, - src: `123456789`, + src: `123456789"`, exp: map[int32]int{123456789: 0}, val: map[int32]int{}, }, { key: "_OP_map_key_i64", ins: []_Instr{newInsVt(_OP_map_key_i64, reflect.TypeOf(map[int64]int{}))}, - src: `123456789123456789`, + src: `123456789123456789"`, exp: map[int64]int{123456789123456789: 0}, val: map[int64]int{}, }, { key: "_OP_map_key_u8", ins: []_Instr{newInsVt(_OP_map_key_u8, reflect.TypeOf(map[uint8]int{}))}, - src: `123`, + src: `123"`, exp: map[uint8]int{123: 0}, val: map[uint8]int{}, }, { key: "_OP_map_key_u32", ins: []_Instr{newInsVt(_OP_map_key_u32, reflect.TypeOf(map[uint32]int{}))}, - src: `123456789`, + src: `123456789"`, exp: map[uint32]int{123456789: 0}, val: map[uint32]int{}, }, { key: "_OP_map_key_u64", ins: []_Instr{newInsVt(_OP_map_key_u64, reflect.TypeOf(map[uint64]int{}))}, - src: `123456789123456789`, + src: `123456789123456789"`, exp: map[uint64]int{123456789123456789: 0}, val: map[uint64]int{}, }, { key: "_OP_map_key_f32", ins: []_Instr{newInsVt(_OP_map_key_f32, reflect.TypeOf(map[float32]int{}))}, - src: `1.25`, + src: `1.25"`, exp: map[float32]int{1.25: 0}, val: map[float32]int{}, }, { key: "_OP_map_key_f64", ins: []_Instr{newInsVt(_OP_map_key_f64, reflect.TypeOf(map[float64]int{}))}, - src: `1.25`, + src: `1.25"`, exp: map[float64]int{1.25: 0}, val: map[float64]int{}, }, { diff --git a/decoder/compiler.go b/decoder/compiler.go index b4fc2fe..2779e7d 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -639,11 +639,6 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op skip2 := p.pc() p.rtt(op, vt) - /* match the closing quote if needed */ - if op != _OP_map_key_str && op != _OP_map_key_utext && op != _OP_map_key_utext_p { - p.chr(_OP_match_char, '"') - } - /* match the value separator */ p.add(_OP_lspace) p.chr(_OP_match_char, ':') @@ -660,11 +655,6 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op skip3 := p.pc() p.rtt(op, vt) - /* match the closing quote if needed */ - if op != _OP_map_key_str && op != _OP_map_key_utext && op != _OP_map_key_utext_p { - p.chr(_OP_match_char, '"') - } - /* match the value separator */ p.add(_OP_lspace) p.chr(_OP_match_char, ':') diff --git a/decoder/stream.go b/decoder/stream.go index 06dc818..f80284f 100644 --- a/decoder/stream.go +++ b/decoder/stream.go @@ -21,13 +21,12 @@ import ( `io` `sync` + `github.com/bytedance/sonic/option` `github.com/bytedance/sonic/internal/native/types` ) var ( - defaultBufferSize uint = 4096 - growSliceFactorShift uint = 1 - minLeftBufferShift uint = 2 + minLeftBufferShift uint = 1 ) type StreamDecoder struct { @@ -41,7 +40,7 @@ type StreamDecoder struct { var bufPool = sync.Pool{ New: func () interface{} { - return make([]byte, 0, defaultBufferSize) + return make([]byte, 0, option.DefaultDecoderBufferSize) }, } @@ -206,8 +205,8 @@ func realloc(buf *[]byte) { c := uint(cap(*buf)) if c - l <= c >> minLeftBufferShift { e := l+(l>>minLeftBufferShift) - if e < defaultBufferSize { - e = defaultBufferSize + if e < option.DefaultDecoderBufferSize { + e = option.DefaultDecoderBufferSize } tmp := make([]byte, l, e) copy(tmp, *buf) diff --git a/decoder/stream_test.go b/decoder/stream_test.go index b997ace..a94672e 100644 --- a/decoder/stream_test.go +++ b/decoder/stream_test.go @@ -24,45 +24,47 @@ import ( `strings` `testing` + `github.com/bytedance/sonic/option` `github.com/stretchr/testify/assert` `github.com/stretchr/testify/require` ) var ( - _Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} { ` - _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} {"11111":"` + strings.Repeat("2", int(defaultBufferSize)) + `"}` - _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"}{ } {"11111":"` + - strings.Repeat("2", int(defaultBufferSize))+`"} b {}` + DefaultBufferSize = option.DefaultDecoderBufferSize + _Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"} { ` + _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"} {"11111":"` + strings.Repeat("2", int(DefaultBufferSize)) + `"}` + _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}{ } {"11111":"` + + strings.Repeat("2", int(DefaultBufferSize))+`"} b {}` ) func TestStreamError(t *testing.T) { var qs = []string{ - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`""}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":]`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":1x]`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":t}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":true]`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":f}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":false]`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":n}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":null]`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a}`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"`, - `{`+strings.Repeat(" ", int(defaultBufferSize))+`"":"a"]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`""}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":true]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":false]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":null]`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a}`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"`, + `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"]`, } for i, q := range qs { - var qq = []byte(q[:int(defaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(defaultBufferSize):]) + var qq = []byte(q[:int(DefaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(DefaultBufferSize):]) var obj interface{} require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj)) } diff --git a/encoder/assembler_test.go b/encoder/assembler_test.go index 091d7f2..2cbb880 100644 --- a/encoder/assembler_test.go +++ b/encoder/assembler_test.go @@ -26,6 +26,7 @@ import ( `testing` `unsafe` + `github.com/bytedance/sonic/option` `github.com/bytedance/sonic/internal/rt` `github.com/davecgh/go-spew/spew` `github.com/stretchr/testify/assert` @@ -35,7 +36,7 @@ func TestEncoderMemoryCorruption(t *testing.T) { println("TestEncoderMemoryCorruption") var m = map[string]interface{}{ "1": map[string]interface{} { - `"`+strings.Repeat("a", _MaxBuffer - 38)+`"`: "b", + `"`+strings.Repeat("a", int(option.DefaultEncoderBufferSize) - 38)+`"`: "b", "1": map[string]int32{ "b": 1658219785, }, diff --git a/encoder/pools.go b/encoder/pools.go index 600605d..9892ba1 100644 --- a/encoder/pools.go +++ b/encoder/pools.go @@ -30,7 +30,6 @@ import ( const ( _MaxStack = 4096 // 4k states - _MaxBuffer = 1048576 // 1MB buffer size _StackSize = unsafe.Sizeof(_Stack{}) ) @@ -92,7 +91,7 @@ func newBytes() []byte { if ret := bytesPool.Get(); ret != nil { return ret.([]byte) } else { - return make([]byte, 0, _MaxBuffer) + return make([]byte, 0, option.DefaultEncoderBufferSize) } } @@ -112,7 +111,7 @@ func newBuffer() *bytes.Buffer { if ret := bufferPool.Get(); ret != nil { return ret.(*bytes.Buffer) } else { - return bytes.NewBuffer(make([]byte, 0, _MaxBuffer)) + return bytes.NewBuffer(make([]byte, 0, option.DefaultEncoderBufferSize)) } } diff --git a/issue_test/issue381_test.go b/issue_test/issue381_test.go new file mode 100644 index 0000000..96bfcc0 --- /dev/null +++ b/issue_test/issue381_test.go @@ -0,0 +1,52 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package issue_test + +import ( + `bytes` + `encoding/json` + `strings` + `testing` + + `github.com/bytedance/sonic` + `github.com/bytedance/sonic/option` + _ `github.com/davecgh/go-spew/spew` + `github.com/stretchr/testify/require` + + _ `github.com/stretchr/testify/assert` +) + +var decoderBufferSize = option.DefaultDecoderBufferSize + +func testSonicUnmarshal(t *testing.T) { + val := strings.Repeat(" ", int(decoderBufferSize-3)) + `{"123":{}}` + + res := make(map[int64]map[string]interface{}) + res2 := make(map[int64]map[string]interface{}) + + dec := sonic.ConfigDefault.NewDecoder(bytes.NewBufferString(val)) + err := dec.Decode(&res) + require.NoError(t, err) + + err2 := json.Unmarshal([]byte(val), &res2) + require.NoError(t, err2) + require.Equal(t, res2, res) +} + +func TestStreamxx(t *testing.T) { + testSonicUnmarshal(t) +} \ No newline at end of file diff --git a/option/option.go b/option/option.go index 359f992..71527cd 100644 --- a/option/option.go +++ b/option/option.go @@ -16,6 +16,14 @@ package option +var ( + // DefaultDecoderBufferSize is the initial buffer size of StreamDecoder + DefaultDecoderBufferSize uint = 128 * 1024 + + // DefaultEncoderBufferSize is the initial buffer size of Encoder + DefaultEncoderBufferSize uint = 128 * 1024 +) + // CompileOptions includes all options for encoder or decoder compiler. type CompileOptions struct { // the maximum depth for compilation inline