fix(ub): marshal escape b f as encoding/json (#243)

* fix(ub): marshal b f as encoding/json

* test: fix old test for escape b

Co-authored-by: liuqiang <liuqiang.06@bytedance.com>

ast/encode_test.go

@@ -90,7 +90,7 @@ func TestEncodeValue(t *testing.T) {
         {NewArray([]Node{}), "[]", false},
         {NewArray([]Node{NewBool(true), NewString("true"), NewString("\t")}), `[true,"true","\t"]`, false},
         {NewObject([]Pair{Pair{"a", NewNull()}, Pair{"b", NewNumber("0")}}), `{"a":null,"b":0}`, false},
-        {NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}), `{"\ta":"\t","\bb":"\b","\nb":"\n","\ra":"\r"}`, false},
+        {NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}), `{"\ta":"\t","\u0008b":"\u0008","\nb":"\n","\ra":"\r"}`, false},
         {NewObject([]Pair{}), `{}`, false},
         {NewBytes([]byte("hello, world")), `"aGVsbG8sIHdvcmxk"`, false},
         {NewAny(obj), string(buf), false},

encode_test.go

@@ -153,7 +153,7 @@ func TestRoundtripStringTag(t *testing.T) {
                 "BoolStr": "false",
                 "IntStr": "0",
                 "UintptrStr": "0",
-                "StrStr": "\"\\b\\f\\n\\r\\t\\\"\\\\\"",
+                "StrStr": "\"\\u0008\\u000c\\n\\r\\t\\\"\\\\\"",
                 "NumberStr": "0"
             }`,
         },
@@ -779,11 +779,11 @@ var encodeStringTests = []struct {
     {"\x05", `"\u0005"`},
     {"\x06", `"\u0006"`},
     {"\x07", `"\u0007"`},
-    {"\x08", `"\b"`},
+    {"\x08", `"\u0008"`},
     {"\x09", `"\t"`},
     {"\x0a", `"\n"`},
     {"\x0b", `"\u000b"`},
-    {"\x0c", `"\f"`},
+    {"\x0c", `"\u000c"`},
     {"\x0d", `"\r"`},
     {"\x0e", `"\u000e"`},
     {"\x0f", `"\u000f"`},

internal/native/avx/native_amd64_test.go

@@ -63,8 +63,8 @@ func TestNative_Quote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 27, len(d))
+    assert.Equal(t, 35, len(d))
-    assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
+    assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
 }
 
 func TestNative_QuoteNoMem(t *testing.T) {
@@ -73,9 +73,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
     dp := (*rt.GoSlice)(unsafe.Pointer(&d))
     sp := (*rt.GoString)(unsafe.Pointer(&s))
     rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
-    assert.Equal(t, -8, rv)
+    assert.Equal(t, -6, rv)
-    assert.Equal(t, 9, len(d))
+    assert.Equal(t, 5, len(d))
-    assert.Equal(t, `hello\b\f`, string(d))
+    assert.Equal(t, `hello`, string(d))
 }
 
 func TestNative_DoubleQuote(t *testing.T) {
@@ -88,8 +88,8 @@ func TestNative_DoubleQuote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 36, len(d))
+    assert.Equal(t, 44, len(d))
-    assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
+    assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
 }
 
 func TestNative_Unquote(t *testing.T) {

internal/native/avx/native_subr_amd64.go

@@ -10,44 +10,44 @@ func __native_entry__() uintptr
 
 var (
     _subr__f64toa       = __native_entry__() + 570
-    _subr__html_escape  = __native_entry__() + 9062
+    _subr__html_escape  = __native_entry__() + 8834
-    _subr__i64toa       = __native_entry__() + 3205
+    _subr__i64toa       = __native_entry__() + 3653
     _subr__lspace       = __native_entry__() + 251
     _subr__lzero        = __native_entry__() + 13
-    _subr__quote        = __native_entry__() + 4498
+    _subr__quote        = __native_entry__() + 4970
-    _subr__skip_array   = __native_entry__() + 18295
+    _subr__skip_array   = __native_entry__() + 18226
-    _subr__skip_number  = __native_entry__() + 21246
+    _subr__skip_number  = __native_entry__() + 21165
-    _subr__skip_object  = __native_entry__() + 18332
+    _subr__skip_object  = __native_entry__() + 18263
-    _subr__skip_one     = __native_entry__() + 16448
+    _subr__skip_one     = __native_entry__() + 16378
-    _subr__u64toa       = __native_entry__() + 3300
+    _subr__u64toa       = __native_entry__() + 3748
-    _subr__unquote      = __native_entry__() + 6037
+    _subr__unquote      = __native_entry__() + 6540
-    _subr__validate_one = __native_entry__() + 21363
+    _subr__validate_one = __native_entry__() + 21282
-    _subr__value        = __native_entry__() + 11651
+    _subr__value        = __native_entry__() + 11437
-    _subr__vnumber      = __native_entry__() + 14561
+    _subr__vnumber      = __native_entry__() + 14392
-    _subr__vsigned      = __native_entry__() + 15866
+    _subr__vsigned      = __native_entry__() + 15822
-    _subr__vstring      = __native_entry__() + 13543
+    _subr__vstring      = __native_entry__() + 13345
-    _subr__vunsigned    = __native_entry__() + 16146
+    _subr__vunsigned    = __native_entry__() + 16102
 )
 
 const (
-    _stack__f64toa = 136
+    _stack__f64toa = 120
     _stack__html_escape = 64
     _stack__i64toa = 24
     _stack__lspace = 8
     _stack__lzero = 8
     _stack__quote = 80
-    _stack__skip_array = 144
+    _stack__skip_array = 152
-    _stack__skip_number = 80
+    _stack__skip_number = 88
-    _stack__skip_object = 144
+    _stack__skip_object = 152
-    _stack__skip_one = 144
+    _stack__skip_one = 152
     _stack__u64toa = 8
-    _stack__unquote = 72
+    _stack__unquote = 80
-    _stack__validate_one = 144
+    _stack__validate_one = 152
-    _stack__value = 416
+    _stack__value = 424
-    _stack__vnumber = 312
+    _stack__vnumber = 320
     _stack__vsigned = 16
     _stack__vstring = 120
-    _stack__vunsigned = 24
+    _stack__vunsigned = 16
 )
 
 var (

internal/native/avx2/native_amd64_test.go

@@ -63,8 +63,8 @@ func TestNative_Quote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 27, len(d))
+    assert.Equal(t, 35, len(d))
-    assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
+    assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
 }
 
 func TestNative_QuoteNoMem(t *testing.T) {
@@ -73,9 +73,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
     dp := (*rt.GoSlice)(unsafe.Pointer(&d))
     sp := (*rt.GoString)(unsafe.Pointer(&s))
     rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
-    assert.Equal(t, -8, rv)
+    assert.Equal(t, -6, rv)
-    assert.Equal(t, 9, len(d))
+    assert.Equal(t, 5, len(d))
-    assert.Equal(t, `hello\b\f`, string(d))
+    assert.Equal(t, `hello`, string(d))
 }
 
 func TestNative_DoubleQuote(t *testing.T) {
@@ -88,8 +88,8 @@ func TestNative_DoubleQuote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 36, len(d))
+    assert.Equal(t, 44, len(d))
-    assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
+    assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
 }
 
 func TestNative_Unquote(t *testing.T) {

internal/native/avx2/native_subr_amd64.go

@@ -9,45 +9,45 @@ package avx2
 func __native_entry__() uintptr
 
 var (
-    _subr__f64toa       = __native_entry__() + 814
+    _subr__f64toa       = __native_entry__() + 825
-    _subr__html_escape  = __native_entry__() + 10717
+    _subr__html_escape  = __native_entry__() + 10493
-    _subr__i64toa       = __native_entry__() + 3449
+    _subr__i64toa       = __native_entry__() + 3908
     _subr__lspace       = __native_entry__() + 379
     _subr__lzero        = __native_entry__() + 13
-    _subr__quote        = __native_entry__() + 4842
+    _subr__quote        = __native_entry__() + 5325
-    _subr__skip_array   = __native_entry__() + 21271
+    _subr__skip_array   = __native_entry__() + 21311
-    _subr__skip_number  = __native_entry__() + 24742
+    _subr__skip_number  = __native_entry__() + 24831
-    _subr__skip_object  = __native_entry__() + 21308
+    _subr__skip_object  = __native_entry__() + 21348
-    _subr__skip_one     = __native_entry__() + 19499
+    _subr__skip_one     = __native_entry__() + 19468
-    _subr__u64toa       = __native_entry__() + 3544
+    _subr__u64toa       = __native_entry__() + 4003
-    _subr__unquote      = __native_entry__() + 7467
+    _subr__unquote      = __native_entry__() + 7998
-    _subr__validate_one = __native_entry__() + 24859
+    _subr__validate_one = __native_entry__() + 24948
-    _subr__value        = __native_entry__() + 14548
+    _subr__value        = __native_entry__() + 14390
-    _subr__vnumber      = __native_entry__() + 17612
+    _subr__vnumber      = __native_entry__() + 17482
-    _subr__vsigned      = __native_entry__() + 18917
+    _subr__vsigned      = __native_entry__() + 18912
-    _subr__vstring      = __native_entry__() + 16708
+    _subr__vstring      = __native_entry__() + 16575
-    _subr__vunsigned    = __native_entry__() + 19197
+    _subr__vunsigned    = __native_entry__() + 19192
 )
 
 const (
-    _stack__f64toa = 136
+    _stack__f64toa = 120
     _stack__html_escape = 72
     _stack__i64toa = 24
     _stack__lspace = 8
     _stack__lzero = 8
     _stack__quote = 72
-    _stack__skip_array = 152
+    _stack__skip_array = 160
-    _stack__skip_number = 88
+    _stack__skip_number = 96
-    _stack__skip_object = 152
+    _stack__skip_object = 160
-    _stack__skip_one = 152
+    _stack__skip_one = 160
     _stack__u64toa = 8
     _stack__unquote = 72
-    _stack__validate_one = 152
+    _stack__validate_one = 160
-    _stack__value = 416
+    _stack__value = 424
-    _stack__vnumber = 312
+    _stack__vnumber = 320
     _stack__vsigned = 16
     _stack__vstring = 112
-    _stack__vunsigned = 24
+    _stack__vunsigned = 16
 )
 
 var (

internal/native/native_amd64_test.tmpl

@@ -61,8 +61,8 @@ func TestNative_Quote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 27, len(d))
+    assert.Equal(t, 35, len(d))
-    assert.Equal(t, `hello\b\f\n\r\t\\\"景world`, string(d))
+    assert.Equal(t, `hello\u0008\u000c\n\r\t\\\"景world`, string(d))
 }
 
 func TestNative_QuoteNoMem(t *testing.T) {
@@ -71,9 +71,9 @@ func TestNative_QuoteNoMem(t *testing.T) {
     dp := (*rt.GoSlice)(unsafe.Pointer(&d))
     sp := (*rt.GoString)(unsafe.Pointer(&s))
     rv := __quote(sp.Ptr, sp.Len, dp.Ptr, &dp.Len, 0)
-    assert.Equal(t, -8, rv)
+    assert.Equal(t, -6, rv)
-    assert.Equal(t, 9, len(d))
+    assert.Equal(t, 5, len(d))
-    assert.Equal(t, `hello\b\f`, string(d))
+    assert.Equal(t, `hello`, string(d))
 }
 
 func TestNative_DoubleQuote(t *testing.T) {
@@ -86,8 +86,8 @@ func TestNative_DoubleQuote(t *testing.T) {
         require.NoError(t, types.ParsingError(-rv))
     }
     assert.Equal(t, len(s), rv)
-    assert.Equal(t, 36, len(d))
+    assert.Equal(t, 44, len(d))
-    assert.Equal(t, `hello\\b\\f\\n\\r\\t\\\\\\\"景world`, string(d))
+    assert.Equal(t, `hello\\u0008\\u000c\\n\\r\\t\\\\\\\"景world`, string(d))
 }
 
 func TestNative_Unquote(t *testing.T) {

issue_test/issue242_test.go

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package issue_test
+
+import (
+    `unicode/utf8`
+    `encoding/json`
+    `testing`
+
+    `github.com/stretchr/testify/require`
+    `github.com/bytedance/sonic`
+)
+
+func TestIssue242_MarshalControlChars(t *testing.T) {
+    for i := 0; i < utf8.RuneSelf; i++ {
+        input := string([]byte{byte(i)})
+        out1, err1 := sonic.ConfigStd.Marshal(input)
+        out2, err2 := json.Marshal(input)
+        require.NoError(t, err1)
+        require.NoError(t, err2)
+        require.Equal(t, out1, out2)
+    }
+}

native/parsing.c

@@ -33,11 +33,11 @@ static const quoted_t _SingleQuoteTab[256] = {
     ['\x05'] = { .n = 6, .s = "\\u0005" },
     ['\x06'] = { .n = 6, .s = "\\u0006" },
     ['\x07'] = { .n = 6, .s = "\\u0007" },
-    ['\b'  ] = { .n = 2, .s = "\\b"     },
+    ['\b'  ] = { .n = 6, .s = "\\u0008" },
     ['\t'  ] = { .n = 2, .s = "\\t"     },
     ['\n'  ] = { .n = 2, .s = "\\n"     },
     ['\x0b'] = { .n = 6, .s = "\\u000b" },
-    ['\f'  ] = { .n = 2, .s = "\\f"     },
+    ['\f'  ] = { .n = 6, .s = "\\u000c" },
     ['\r'  ] = { .n = 2, .s = "\\r"     },
     ['\x0e'] = { .n = 6, .s = "\\u000e" },
     ['\x0f'] = { .n = 6, .s = "\\u000f" },
@@ -70,11 +70,11 @@ static const quoted_t _DoubleQuoteTab[256] = {
     ['\x05'] = { .n = 7, .s = "\\\\u0005" },
     ['\x06'] = { .n = 7, .s = "\\\\u0006" },
     ['\x07'] = { .n = 7, .s = "\\\\u0007" },
-    ['\b'  ] = { .n = 3, .s = "\\\\b"     },
+    ['\b'  ] = { .n = 7, .s = "\\\\u0008" },
     ['\t'  ] = { .n = 3, .s = "\\\\t"     },
     ['\n'  ] = { .n = 3, .s = "\\\\n"     },
     ['\x0b'] = { .n = 7, .s = "\\\\u000b" },
-    ['\f'  ] = { .n = 3, .s = "\\\\f"     },
+    ['\f'  ] = { .n = 7, .s = "\\\\u000c" },
     ['\r'  ] = { .n = 3, .s = "\\\\r"     },
     ['\x0e'] = { .n = 7, .s = "\\\\u000e" },
     ['\x0f'] = { .n = 7, .s = "\\\\u000f" },