XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-21 00:46:43 +08:00
Code Issues Projects Releases Wiki Activity

fix: check character range before BTQ

Browse source
This commit is contained in:
chenzhuoyu 2021-09-17 18:44:37 +08:00 committed by Oxygen
parent c1749cfd1f
commit 0e4b0b8ee1
10 changed files with 3120 additions and 3479 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
decoder/assembler_amd64.go
View file

@ -1449,6 +1449,8 @@ func (self *_Assembler) _asm_OP_lspace(_ *_Instr) {
self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n}
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n}
@ -1458,6 +1460,8 @@ func (self *_Assembler) _asm_OP_lspace(_ *_Instr) {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n}
self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX
self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n}
}

41
decoder/generic_amd64.go
View file

@ -207,38 +207,25 @@ func (self *_ValueDecoder) compile() {
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 1-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 2-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* at least 1 to 3 spaces */
for i := 0; i < 3; i++ {
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' '
self.Sjmp("JA" , "_decode_fast") // JA _decode_fast
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
}
/* 3-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX
self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX
self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX
self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast
/* 4-space case */
self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC
/* at least 4 spaces */
self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL
self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF
self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX

3119
internal/native/avx/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 238
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 4864
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16761
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16796
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14439
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16320
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16355
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14527
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3644
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5885
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 10074
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12633
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13889
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11680
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14166
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 10010
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12721
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13977
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11768
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14254
)
var (

3248
internal/native/avx2/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx2/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 366
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5212
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 19217
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 19252
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16417
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18263
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18298
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16517
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3892
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 7049
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12063
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14611
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15867
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13774
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16144
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11871
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14711
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15967
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13874
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16244
)
var (

26
internal/native/types/types.go
View file

@ -25,19 +25,19 @@ type ParsingError uint
type SearchingError uint
const (
V_EOF ValueType = 1
V_NULL ValueType = 2
V_TRUE ValueType = 3
V_FALSE ValueType = 4
V_ARRAY ValueType = 5
V_OBJECT ValueType = 6
V_STRING ValueType = 7
V_DOUBLE ValueType = 8
V_INTEGER ValueType = 9
V_KEY_SEP ValueType = 10
V_ELEM_SEP ValueType = 11
V_ARRAY_END ValueType = 12
V_OBJECT_END ValueType = 13
V_EOF ValueType = 1
V_NULL ValueType = 2
V_TRUE ValueType = 3
V_FALSE ValueType = 4
V_ARRAY ValueType = 5
V_OBJECT ValueType = 6
V_STRING ValueType = 7
V_DOUBLE ValueType = 8
V_INTEGER ValueType = 9
_ ValueType = 10 // V_KEY_SEP
_ ValueType = 11 // V_ELEM_SEP
_ ValueType = 12 // V_ARRAY_END
_ ValueType = 13 // V_OBJECT_END
V_MAX
)

83
issue100_test.go
View file

@ -17,17 +17,16 @@
package sonic
import (
`fmt`
`reflect`
_ `sync`
`testing`
`unsafe`
stdjson `encoding/json`
stdjson `encoding/json`
`fmt`
`reflect`
_ `sync`
`testing`
`unsafe`
)
func TestLargeMapValue(t *testing.T) {
var jsonStr = `{
var jsonStr = `{
"1": {},
"2": {},
"3": {},
@ -38,49 +37,49 @@ func TestLargeMapValue(t *testing.T) {
"8": {},
"9": {}
}`
type Case struct {
std interface{}
sonic interface{}
}
cases := []Case{
{&map[string]TestIssue100_LargeMapValue{}, &map[string]TestIssue100_LargeMapValue{}},
{&map[int32]TestIssue100_LargeMapValue{}, &map[int32]TestIssue100_LargeMapValue{}},
{&map[int64]TestIssue100_LargeMapValue{}, &map[int64]TestIssue100_LargeMapValue{}},
{&map[uint32]TestIssue100_LargeMapValue{}, &map[uint32]TestIssue100_LargeMapValue{}},
{&map[uint64]TestIssue100_LargeMapValue{}, &map[uint64]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}, &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}},
}
for i, c := range cases {
var stdw, sonicw = c.std, c.sonic
if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil {
t.Fatal(i, err)
}
fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len())
if err := Unmarshal([]byte(jsonStr), sonicw); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(stdw, sonicw) {
fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw)
t.Fatal(i)
}
}
type Case struct {
std interface{}
sonic interface{}
}
cases := []Case{
{&map[string]TestIssue100_LargeMapValue{} , &map[string]TestIssue100_LargeMapValue{}},
{&map[int32]TestIssue100_LargeMapValue{} , &map[int32]TestIssue100_LargeMapValue{}},
{&map[int64]TestIssue100_LargeMapValue{} , &map[int64]TestIssue100_LargeMapValue{}},
{&map[uint32]TestIssue100_LargeMapValue{} , &map[uint32]TestIssue100_LargeMapValue{}},
{&map[uint64]TestIssue100_LargeMapValue{} , &map[uint64]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{} , &map[TestIssue100_textMarshalKey]TestIssue100_LargeMapValue{}},
{&map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{} , &map[TestIssue100_textMarshalKeyPtr]TestIssue100_LargeMapValue{}},
}
for i, c := range cases {
var stdw, sonicw = c.std, c.sonic
if err := stdjson.Unmarshal([]byte(jsonStr), stdw); err != nil {
t.Fatal(i, err)
}
fmt.Printf("[%d]struct size: %d\tmap length: %d\n", i, unsafe.Sizeof(TestIssue100_LargeMapValue{}), reflect.ValueOf(stdw).Elem().Len())
if err := Unmarshal([]byte(jsonStr), sonicw); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(stdw, sonicw) {
fmt.Printf("have:\n\t%#v\nwant:\n\t%#v\n", sonicw, stdw)
t.Fatal(i)
}
}
}
type TestIssue100_textMarshalKey string
type TestIssue100_textMarshalKey string
func(self TestIssue100_textMarshalKey) UnmarshalText(text []byte) error {
self = TestIssue100_textMarshalKey(text)
return nil
_ = TestIssue100_textMarshalKey(text)
return nil
}
type TestIssue100_textMarshalKeyPtr string
type TestIssue100_textMarshalKeyPtr string
func(self *TestIssue100_textMarshalKeyPtr) UnmarshalText(text []byte) error {
*self = TestIssue100_textMarshalKeyPtr(text)
return nil
*self = TestIssue100_textMarshalKeyPtr(text)
return nil
}
type TestIssue100_LargeMapValue struct {
Id [129]byte
Id [129]byte
}

31
issue101_test.go Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`testing`
`github.com/davecgh/go-spew/spew`
`github.com/stretchr/testify/require`
)
func TestIssue101_UnmarshalMWithNumber(t *testing.T) {
var v interface{}
err := Unmarshal([]byte("M10"), &v) // MIJ`
spew.Dump(v)
require.Error(t, err)
}

15
native/scanning.c
View file

@ -63,10 +63,15 @@ static inline char advance_ns(const GoString *src, long *p) {
/* it's likely to run into non-spaces within a few
* characters, so test up to 4 characters manually */
for (int i = 0; i < 4 && vi < nb; i++, vi++) {
if (!isspace(sp[vi])) {
goto nospace;
}
if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
if (vi < nb && !isspace(sp[vi])) goto nospace; else vi++;
/* check EOF */
if (vi >= nb) {
*p = vi;
return 0;
}
/* too many spaces, use SIMD to search for characters */
@ -635,7 +640,7 @@ void vnumber(const GoString *src, long *p, JsonState *ret) {
if (ret->vt == V_INTEGER) {
if (!is_overflow(man, sgn, exp10)) {
ret->iv = (int64_t)man * sgn;
/* following lines equal to ret->dv = (double)(man) * sgn */
ret->dv = (double)(man);
*(uint64_t *)&ret->dv |= ((uint64_t)(sgn) >> 63 << 63);