XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-21 00:46:43 +08:00
Code Issues Projects Releases Wiki Activity

fix: make it parse the UTF-16 surrogate pair after invalid unicode (#111)

Browse source
This commit is contained in:
liu 2021-09-24 16:33:02 +08:00 committed by GitHub
parent a7ace8cd03
commit 530d423243
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 1184 additions and 1321 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1131
internal/native/avx/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 238 _subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 238
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0 _subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 4864 _subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 4864
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16320 _subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16010
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16355 _subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16045
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14527 _subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14217
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3644 _subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3644
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5885 _subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5885
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 10010 _subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 9700
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12721 _subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 12411
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13977 _subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13667
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11768 _subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11458
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14254 _subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13944
) )
var ( var (

1306
internal/native/avx2/native_amd64.s
View file

File diff suppressed because it is too large Load diff

16
internal/native/avx2/native_subr_amd64.go
View file

@ -19,16 +19,16 @@ var (
_subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 366 _subr__lspace = **(**uintptr)(unsafe.Pointer(&_func__base)) + 366
_subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0 _subr__lzero = **(**uintptr)(unsafe.Pointer(&_func__base)) + 0
_subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5212 _subr__quote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 5212
_subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18263 _subr__skip_array = **(**uintptr)(unsafe.Pointer(&_func__base)) + 17881
_subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 18298 _subr__skip_object = **(**uintptr)(unsafe.Pointer(&_func__base)) + 17916
_subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16517 _subr__skip_one = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16135
_subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3892 _subr__u64toa = **(**uintptr)(unsafe.Pointer(&_func__base)) + 3892
_subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 7049 _subr__unquote = **(**uintptr)(unsafe.Pointer(&_func__base)) + 7049
_subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11871 _subr__value = **(**uintptr)(unsafe.Pointer(&_func__base)) + 11489
_subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14711 _subr__vnumber = **(**uintptr)(unsafe.Pointer(&_func__base)) + 14329
_subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15967 _subr__vsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15585
_subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13874 _subr__vstring = **(**uintptr)(unsafe.Pointer(&_func__base)) + 13492
_subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 16244 _subr__vunsigned = **(**uintptr)(unsafe.Pointer(&_func__base)) + 15862
) )
var ( var (

32
issue_test/issue107_test.go Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package issue_test
import (
`testing`
. `github.com/bytedance/sonic`
`github.com/stretchr/testify/require`
)
func TestIssue107_UnmarshalUTF16SurrogatePairAfterInvalidUnicode(t *testing.T) {
var obj string
err := Unmarshal([]byte(`"\uDA51\uD83D\uDE04"`), &obj)
require.NoError(t, err)
require.Equal(t, obj, "\ufffd" + `😄`)
}

4
native/parsing.c
View file

@ -588,10 +588,6 @@ ssize_t unquote(const char *sp, ssize_t nb, char *dp, ssize_t *ep, uint64_t flag
if (unlikely(!(flags & F_UNIREP))) { if (unlikely(!(flags & F_UNIREP))) {
*ep = sp - s - 4; *ep = sp - s - 4;
return -ERR_UNICODE; return -ERR_UNICODE;
} else if (likely(r1 >= 0xd800 && r1 <= 0xdfff)) {
unirep(&dp);
unirep(&dp);
continue;
} else { } else {
r0 = r1; r0 = r1;
unirep(&dp); unirep(&dp);