XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-22 09:26:44 +08:00
Code Issues Projects Releases Wiki Activity

fix: check character range before BTQ

Browse source
This commit is contained in:
chenzhuoyu 2021-09-17 18:44:37 +08:00 committed by Oxygen
parent c1749cfd1f
commit 0e4b0b8ee1
10 changed files with 3120 additions and 3479 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
decoder/assembler_amd64.go
View file

@ -1449,6 +1449,8 @@ func (self *_Assembler) _asm_OP_lspace(_ *_Instr) {
self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n}
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n} self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n}
@ -1458,6 +1460,8 @@ func (self *_Assembler) _asm_OP_lspace(_ *_Instr) {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n}
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n} self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n}
} }

41
decoder/generic_amd64.go
View file

@ -207,38 +207,25 @@ func (self *_ValueDecoder) compile() {
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 1-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 2-space case */ /* at least 1 to 3 spaces */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC for i := 0; i < 3; i++ {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
}
/* 3-space case */ /* at least 4 spaces */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 4-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX

3119
internal/native/avx/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 238 _subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 238
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0 _subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 4864 _subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 4864
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16761 _subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16320
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16796 _subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16355
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14439 _subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14527
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3644 _subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3644
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5885 _subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5885
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 10074 _subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 10010
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12633 _subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12721
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13889 _subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13977
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11680 _subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11768
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14166 _subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14254
) )
var ( var (

3248
internal/native/avx2/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx2/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 366 _subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 366
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0 _subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5212 _subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5212
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 19217 _subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18263
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 19252 _subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18298
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16417 _subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16517
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3892 _subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3892
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 7049 _subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 7049
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12063 _subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11871
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14611 _subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14711
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15867 _subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15967
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13774 _subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13874
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16144 _subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16244
) )
var ( var (

26
internal/native/types/types.go
View file

@ -25,19 +25,19 @@ type ParsingError uint
type SearchingError uint type SearchingError uint
const ( const (
V_EOF ValueType = 1 V_EOF ValueType = 1
V_NULL ValueType = 2 V_NULL ValueType = 2
V_TRUE ValueType = 3 V_TRUE ValueType = 3
V_FALSE ValueType = 4 V_FALSE ValueType = 4
V_ARRAY ValueType = 5 V_ARRAY ValueType = 5
V_OBJECT ValueType = 6 V_OBJECT ValueType = 6
V_STRING ValueType = 7 V_STRING ValueType = 7
V_DOUBLE ValueType = 8 V_DOUBLE ValueType = 8
V_INTEGER ValueType = 9 V_INTEGER ValueType = 9
V_KEY_SEP ValueType = 10 _ ValueType = 10 // V_KEY_SEP
V_ELEM_SEP ValueType = 11 _ ValueType = 11 // V_ELEM_SEP
V_ARRAY_END ValueType = 12 _ ValueType = 12 // V_ARRAY_END
V_OBJECT_END ValueType = 13 _ ValueType = 13 // V_OBJECT_END
V_MAX V_MAX
) )

83
issue100_test.go
View file

@ -17,17 +17,16 @@
package sonic package sonic
import ( import (
`fmt` stdjson `encoding/json`
`reflect` `fmt`
_ `sync` `reflect`
`testing` _ `sync`
`unsafe` `testing`
`unsafe`
stdjson `encoding/json`
) )
func TestLargeMapValue(t *testing.T) { func TestLargeMapValue(t *testing.T) {
var jsonStr = `{ var jsonStr = `{
"1": {}, "1": {},
"2": {}, "2": {},
"3": {}, "3": {},
@ -38,49 +37,49 @@ func TestLargeMapValue(t *testing.T) {
"8": {}, "8": {},
"9": {} "9": {}
}` }`
type Case struct { type Case struct {
std interface{} std interface{}
sonic interface{} sonic interface{}
} }
cases := []Case{ cases := []Case{
{&map[string]TestIssue100_LargeMapValue{}, &map[string]TestIssue100_LargeMapValue{}}, {&map[string]TestIssue100_LargeMapValue{} , &map[string]TestIssue100_LargeMapValue{}},
{&map[int32]TestIssue100_LargeMapValue{}, &map[int32]TestIssue100_LargeMapValue{}}, {&map[int32]TestIssue100_LargeMapValue{} , &map[int32]TestIssue100_LargeMapValue{}},
{&map[int64]TestIssue100_LargeMapValue{}, &map[int64]TestIssue100_LargeMapValue{}}, {&map[int64]TestIssue100_LargeMapValue{} , &map[int64]TestIssue100_LargeMapValue{}},
{&map[uint32]TestIssue100_LargeMapValue{}, &map[uint32]TestIssue100_LargeMapValue{}}, {&map[uint32]TestIssue100_LargeMapValue{} , &map[uint32]TestIssue100_LargeMapValue{}},
{&map[uint64]TestIssue100_LargeMapValue{}, &map[uint64]TestIssue100_LargeMapValue{}}, {&map[uint64]TestIssue100_LargeMapValue{} , &map[uint64]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}}, {&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{} , &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}}, {&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{} , &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}},
} }
for i, c := range cases { for i, c := range cases {
var stdw, sonicw = c.std, c.sonic var stdw, sonicw = c.std, c.sonic
if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil { if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil {
t.Fatal(i, err) t.Fatal(i, err)
} }
fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len()) fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len())
if err := Unmarshal([]byte(jsonStr), sonicw); err != nil { if err := Unmarshal([]byte(jsonStr), sonicw); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(stdw, sonicw) { if !reflect.DeepEqual(stdw, sonicw) {
fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw) fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw)
t.Fatal(i) t.Fatal(i)
} }
} }
} }
type TestIssue100_textMarshalKey string type TestIssue100_textMarshalKey string
func(self TestIssue100_textMarshalKey) UnmarshalText(text []byte) error { func(self TestIssue100_textMarshalKey) UnmarshalText(text []byte) error {
self = TestIssue100_textMarshalKey(text) _ = TestIssue100_textMarshalKey(text)
return nil return nil
} }
type TestIssue100_textMarshalKeyPtr string type TestIssue100_textMarshalKeyPtr string
func(self *TestIssue100_textMarshalKeyPtr) UnmarshalText(text []byte) error { func(self *TestIssue100_textMarshalKeyPtr) UnmarshalText(text []byte) error {
*self = TestIssue100_textMarshalKeyPtr(text) *self = TestIssue100_textMarshalKeyPtr(text)
return nil return nil
} }
type TestIssue100_LargeMapValue struct { type TestIssue100_LargeMapValue struct {
Id [129]byte Id [129]byte
} }

31
issue101_test.go Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`testing`
`github.com/davecgh/go-spew/spew`
`github.com/stretchr/testify/require`
)
func TestIssue101_UnmarshalMWithNumber(t *testing.T) {
var v interface{}
err := Unmarshal([]byte("M10"), &v) // MIJ`
spew.Dump(v)
require.Error(t, err)
}

15
native/scanning.c
View file

@ -63,10 +63,15 @@ static inline char advance_ns(const GoString *src, long *p) {
/* it's likely to run into non-spaces within a few /* it's likely to run into non-spaces within a few
* characters, so test up to 4 characters manually */ * characters, so test up to 4 characters manually */
for (int i = 0; i < 4 && vi < nb; i++, vi++) { if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
if (!isspace(sp[vi])) { if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
goto nospace; if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
} if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
/* check EOF */
if (vi >= nb) {
*p = vi;
return 0;
} }
/* too many spaces, use SIMD to search for characters */ /* too many spaces, use SIMD to search for characters */
@ -635,7 +640,7 @@ void vnumber(const GoString *src, long *p, JsonState *ret) {
if (ret->vt == V_INTEGER) { if (ret->vt == V_INTEGER) {
if (!is_overflow(man, sgn, exp10)) { if (!is_overflow(man, sgn, exp10)) {
ret->iv = (int64_t)man * sgn; ret->iv = (int64_t)man * sgn;
/* following lines equal to ret->dv = (double)(man) * sgn */ /* following lines equal to ret->dv = (double)(man) * sgn */
ret->dv = (double)(man); ret->dv = (double)(man);
*(uint64_t *)&ret->dv |= ((uint64_t)(sgn) >> 63 << 63); *(uint64_t *)&ret->dv |= ((uint64_t)(sgn) >> 63 << 63);