XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-21 00:46:43 +08:00
Code Issues Projects Releases Wiki Activity

fix(ub): marshal escape b f as encoding/json (#243)

Browse source 
* fix(ub): marshal b f as encoding/json

* test: fix old test for escape b

Co-authored-by: liuqiang <liuqiang.06@bytedance.com>
This commit is contained in:
liu 2022-06-10 11:21:35 +08:00 committed by GitHub
parent 410625ca9e
commit f8fb04a184
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 7534 additions and 7472 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ast/encode_test.go
View file

@ -90,7 +90,7 @@ func TestEncodeValue(t *testing.T) {
{NewArray([]Node{}), "[]", false},
{NewArray([]Node{NewBool(true), NewString("true"), NewString("\t")}), `[true,"true","\t"]`, false},
{NewObject([]Pair{Pair{"a", NewNull()}, Pair{"b", NewNumber("0")}}), `{"a":null,"b":0}`, false},
{NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}), `{"\ta":"\t","\bb":"\b","\nb":"\n","\ra":"\r"}`, false},
{NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}), `{"\ta":"\t","\u0008b":"\u0008","\nb":"\n","\ra":"\r"}`, false},
{NewObject([]Pair{}), `{}`, false},
{NewBytes([]byte("hello, world")), `"aGVsbG8sIHdvcmxk"`, false},
{NewAny(obj), string(buf), false},

6
encode_test.go
View file

@ -153,7 +153,7 @@ func TestRoundtripStringTag(t *testing.T) {
"BoolStr": "false",
"IntStr": "0",
"UintptrStr": "0",
"StrStr": "\"\\b\\f\\n\\r\\t\\\"\\\\\"",
"StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"",
"NumberStr": "0"
}`,
},
@ -779,11 +779,11 @@ var encodeStringTests = []struct {
{"\x05", `"\u0005"`},
{"\x06", `"\u0006"`},
{"\x07", `"\u0007"`},
{"\x08", `"\b"`},
{"\x08", `"\u0008"`},
{"\x09", `"\t"`},
{"\x0a", `"\n"`},
{"\x0b", `"\u000b"`},
{"\x0c", `"\f"`},
{"\x0c", `"\u000c"`},
{"\x0d", `"\r"`},
{"\x0e", `"\u000e"`},
{"\x0f", `"\u000f"`},

2
internal/native/avx/native_amd64.go
View file

@ -112,4 +112,4 @@ func __skip_number(s *string, p *int) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)

6641
internal/native/avx/native_amd64.s
View file

File diff suppressed because it is too large Load diff

14
internal/native/avx/native_amd64_test.go
View file

@ -63,8 +63,8 @@ func TestNative_Quote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 27, len(d))
assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
assert.Equal(t, 35, len(d))
assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
}
func TestNative_QuoteNoMem(t *testing.T) {
@ -73,9 +73,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
dp := (*rt.GoSlice)(unsafe.Pointer(&d))
sp := (*rt.GoString)(unsafe.Pointer(&s))
rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
assert.Equal(t, -8, rv)
assert.Equal(t, 9, len(d))
assert.Equal(t, `hello\b\f`, string(d))
assert.Equal(t, -6, rv)
assert.Equal(t, 5, len(d))
assert.Equal(t, `hello`, string(d))
}
func TestNative_DoubleQuote(t *testing.T) {
@ -88,8 +88,8 @@ func TestNative_DoubleQuote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 36, len(d))
assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
assert.Equal(t, 44, len(d))
assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
}
func TestNative_Unquote(t *testing.T) {

50
internal/native/avx/native_subr_amd64.go
View file

@ -10,44 +10,44 @@ func __native_entry__() uintptr
var (
_subr__f64toa = __native_entry__() + 570
_subr__html_escape = __native_entry__() + 9062
_subr__i64toa = __native_entry__() + 3205
_subr__html_escape = __native_entry__() + 8834
_subr__i64toa = __native_entry__() + 3653
_subr__lspace = __native_entry__() + 251
_subr__lzero = __native_entry__() + 13
_subr__quote = __native_entry__() + 4498
_subr__skip_array = __native_entry__() + 18295
_subr__skip_number = __native_entry__() + 21246
_subr__skip_object = __native_entry__() + 18332
_subr__skip_one = __native_entry__() + 16448
_subr__u64toa = __native_entry__() + 3300
_subr__unquote = __native_entry__() + 6037
_subr__validate_one = __native_entry__() + 21363
_subr__value = __native_entry__() + 11651
_subr__vnumber = __native_entry__() + 14561
_subr__vsigned = __native_entry__() + 15866
_subr__vstring = __native_entry__() + 13543
_subr__vunsigned = __native_entry__() + 16146
_subr__quote = __native_entry__() + 4970
_subr__skip_array = __native_entry__() + 18226
_subr__skip_number = __native_entry__() + 21165
_subr__skip_object = __native_entry__() + 18263
_subr__skip_one = __native_entry__() + 16378
_subr__u64toa = __native_entry__() + 3748
_subr__unquote = __native_entry__() + 6540
_subr__validate_one = __native_entry__() + 21282
_subr__value = __native_entry__() + 11437
_subr__vnumber = __native_entry__() + 14392
_subr__vsigned = __native_entry__() + 15822
_subr__vstring = __native_entry__() + 13345
_subr__vunsigned = __native_entry__() + 16102
)
const (
_stack__f64toa = 136
_stack__f64toa = 120
_stack__html_escape = 64
_stack__i64toa = 24
_stack__lspace = 8
_stack__lzero = 8
_stack__quote = 80
_stack__skip_array = 144
_stack__skip_number = 80
_stack__skip_object = 144
_stack__skip_one = 144
_stack__skip_array = 152
_stack__skip_number = 88
_stack__skip_object = 152
_stack__skip_one = 152
_stack__u64toa = 8
_stack__unquote = 72
_stack__validate_one = 144
_stack__value = 416
_stack__vnumber = 312
_stack__unquote = 80
_stack__validate_one = 152
_stack__value = 424
_stack__vnumber = 320
_stack__vsigned = 16
_stack__vstring = 120
_stack__vunsigned = 24
_stack__vunsigned = 16
)
var (

2
internal/native/avx2/native_amd64.go
View file

@ -112,4 +112,4 @@ func __skip_number(s *string, p *int) (ret int)
//go:nosplit
//go:noescape
//goland:noinspection GoUnusedParameter
func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)
func __validate_one(s *string, p *int, m *types.StateMachine) (ret int)

8166
internal/native/avx2/native_amd64.s
View file

File diff suppressed because it is too large Load diff

14
internal/native/avx2/native_amd64_test.go
View file

@ -63,8 +63,8 @@ func TestNative_Quote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 27, len(d))
assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
assert.Equal(t, 35, len(d))
assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
}
func TestNative_QuoteNoMem(t *testing.T) {
@ -73,9 +73,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
dp := (*rt.GoSlice)(unsafe.Pointer(&d))
sp := (*rt.GoString)(unsafe.Pointer(&s))
rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
assert.Equal(t, -8, rv)
assert.Equal(t, 9, len(d))
assert.Equal(t, `hello\b\f`, string(d))
assert.Equal(t, -6, rv)
assert.Equal(t, 5, len(d))
assert.Equal(t, `hello`, string(d))
}
func TestNative_DoubleQuote(t *testing.T) {
@ -88,8 +88,8 @@ func TestNative_DoubleQuote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 36, len(d))
assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
assert.Equal(t, 44, len(d))
assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
}
func TestNative_Unquote(t *testing.T) {

50
internal/native/avx2/native_subr_amd64.go
View file

@ -9,45 +9,45 @@ package avx2
func __native_entry__() uintptr
var (
_subr__f64toa = __native_entry__() + 814
_subr__html_escape = __native_entry__() + 10717
_subr__i64toa = __native_entry__() + 3449
_subr__f64toa = __native_entry__() + 825
_subr__html_escape = __native_entry__() + 10493
_subr__i64toa = __native_entry__() + 3908
_subr__lspace = __native_entry__() + 379
_subr__lzero = __native_entry__() + 13
_subr__quote = __native_entry__() + 4842
_subr__skip_array = __native_entry__() + 21271
_subr__skip_number = __native_entry__() + 24742
_subr__skip_object = __native_entry__() + 21308
_subr__skip_one = __native_entry__() + 19499
_subr__u64toa = __native_entry__() + 3544
_subr__unquote = __native_entry__() + 7467
_subr__validate_one = __native_entry__() + 24859
_subr__value = __native_entry__() + 14548
_subr__vnumber = __native_entry__() + 17612
_subr__vsigned = __native_entry__() + 18917
_subr__vstring = __native_entry__() + 16708
_subr__vunsigned = __native_entry__() + 19197
_subr__quote = __native_entry__() + 5325
_subr__skip_array = __native_entry__() + 21311
_subr__skip_number = __native_entry__() + 24831
_subr__skip_object = __native_entry__() + 21348
_subr__skip_one = __native_entry__() + 19468
_subr__u64toa = __native_entry__() + 4003
_subr__unquote = __native_entry__() + 7998
_subr__validate_one = __native_entry__() + 24948
_subr__value = __native_entry__() + 14390
_subr__vnumber = __native_entry__() + 17482
_subr__vsigned = __native_entry__() + 18912
_subr__vstring = __native_entry__() + 16575
_subr__vunsigned = __native_entry__() + 19192
)
const (
_stack__f64toa = 136
_stack__f64toa = 120
_stack__html_escape = 72
_stack__i64toa = 24
_stack__lspace = 8
_stack__lzero = 8
_stack__quote = 72
_stack__skip_array = 152
_stack__skip_number = 88
_stack__skip_object = 152
_stack__skip_one = 152
_stack__skip_array = 160
_stack__skip_number = 96
_stack__skip_object = 160
_stack__skip_one = 160
_stack__u64toa = 8
_stack__unquote = 72
_stack__validate_one = 152
_stack__value = 416
_stack__vnumber = 312
_stack__validate_one = 160
_stack__value = 424
_stack__vnumber = 320
_stack__vsigned = 16
_stack__vstring = 112
_stack__vunsigned = 24
_stack__vunsigned = 16
)
var (

14
internal/native/native_amd64_test.tmpl
View file

@ -61,8 +61,8 @@ func TestNative_Quote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 27, len(d))
assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
assert.Equal(t, 35, len(d))
assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
}
func TestNative_QuoteNoMem(t *testing.T) {
@ -71,9 +71,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
dp := (*rt.GoSlice)(unsafe.Pointer(&d))
sp := (*rt.GoString)(unsafe.Pointer(&s))
rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
assert.Equal(t, -8, rv)
assert.Equal(t, 9, len(d))
assert.Equal(t, `hello\b\f`, string(d))
assert.Equal(t, -6, rv)
assert.Equal(t, 5, len(d))
assert.Equal(t, `hello`, string(d))
}
func TestNative_DoubleQuote(t *testing.T) {
@ -86,8 +86,8 @@ func TestNative_DoubleQuote(t *testing.T) {
require.NoError(t, types.ParsingError(-rv))
}
assert.Equal(t, len(s), rv)
assert.Equal(t, 36, len(d))
assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
assert.Equal(t, 44, len(d))
assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
}
func TestNative_Unquote(t *testing.T) {

37
issue_test/issue242_test.go Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package issue_test
import (
`unicode/utf8`
`encoding/json`
`testing`
`github.com/stretchr/testify/require`
`github.com/bytedance/sonic`
)
func TestIssue242_MarshalControlChars(t *testing.T) {
for i := 0; i < utf8.RuneSelf; i++ {
input := string([]byte{byte(i)})
out1, err1 := sonic.ConfigStd.Marshal(input)
out2, err2 := json.Marshal(input)
require.NoError(t, err1)
require.NoError(t, err2)
require.Equal(t, out1, out2)
}
}

8
native/parsing.c
View file

@ -33,11 +33,11 @@ static const quoted_t _SingleQuoteTab[256] = {
['\x05'] = { .n = 6, .s = "\\u0005" },
['\x06'] = { .n = 6, .s = "\\u0006" },
['\x07'] = { .n = 6, .s = "\\u0007" },
['\b' ] = { .n = 2, .s = "\\b" },
['\b' ] = { .n = 6, .s = "\\u0008" },
['\t' ] = { .n = 2, .s = "\\t" },
['\n' ] = { .n = 2, .s = "\\n" },
['\x0b'] = { .n = 6, .s = "\\u000b" },
['\f' ] = { .n = 2, .s = "\\f" },
['\f' ] = { .n = 6, .s = "\\u000c" },
['\r' ] = { .n = 2, .s = "\\r" },
['\x0e'] = { .n = 6, .s = "\\u000e" },
['\x0f'] = { .n = 6, .s = "\\u000f" },
@ -70,11 +70,11 @@ static const quoted_t _DoubleQuoteTab[256] = {
['\x05'] = { .n = 7, .s = "\\\\u0005" },
['\x06'] = { .n = 7, .s = "\\\\u0006" },
['\x07'] = { .n = 7, .s = "\\\\u0007" },
['\b' ] = { .n = 3, .s = "\\\\b" },
['\b' ] = { .n = 7, .s = "\\\\u0008" },
['\t' ] = { .n = 3, .s = "\\\\t" },
['\n' ] = { .n = 3, .s = "\\\\n" },
['\x0b'] = { .n = 7, .s = "\\\\u000b" },
['\f' ] = { .n = 3, .s = "\\\\f" },
['\f' ] = { .n = 7, .s = "\\\\u000c" },
['\r' ] = { .n = 3, .s = "\\\\r" },
['\x0e'] = { .n = 7, .s = "\\\\u000e" },
['\x0f'] = { .n = 7, .s = "\\\\u000f" },