XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity

feat:(decoder) support skip mismatche-typed value (#325)

Browse source 
* feat:(decoder) support skip mismatche-typed value

* change test cases

* refactor: add type check down into `CompilePrimitive()` to avoid repeat `null` check

* opt call skip()

* bench: add  option `--repeat_times`

* test: omit check primitive

* opt: inline primitive check into its OP

* implement on Go1.15

* fix: support skip json.Numer

* fix: OP_go_skip

* update README.md
This commit is contained in:
Yi Duan 2022-11-04 14:27:21 +08:00 committed by GitHub
parent 2866800519
commit 02fe88266f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 496 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
README.md
View file

@ -181,16 +181,17 @@ ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e"
### Compact Format ### Compact Format
Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process.
### Print Syntax Error ### Print Error
If there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position
```go ```go
import "github.com/bytedance/sonic" import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder" import "github.com/bytedance/sonic/decoder"
var data interface{} var data interface{}
err := sonic.Unmarshal("[[[}]]", &data) err := sonic.UnmarshalString("[[[}]]", &data)
if err != nil { if err != nil {
/*one line by default*/ /*one line by default*/
println(e.Error())) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n" println(e.Error()) // "Syntax error at index 3: invalid char\n\n\t[[[}]]\n\t...^..\n"
/*pretty print*/ /*pretty print*/
if e, ok := err.(decoder.SyntaxError); ok { if e, ok := err.(decoder.SyntaxError); ok {
/*Syntax error at index 3: invalid char /*Syntax error at index 3: invalid char
@ -199,10 +200,24 @@ if err != nil {
...^.. ...^..
*/ */
print(e.Description()) print(e.Description())
}else if me, ok := err.(decoder.MismatchTypeError); ok {
print(e.Description()
} }
} }
``` ```
If there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON.
```go
import "github.com/bytedance/sonic"
import "github.com/bytedance/sonic/decoder"
var data = struct{
A int
B int
}{}
err := UnmarshalString(`{"A":"1","B":1}`, &data)
println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n"
fmt.Printf("%+v", data) // {A:0 B:1}
```
### Ast.Node ### Ast.Node
Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data. Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data.
#### Get/Index #### Get/Index

14
bench.py
View file

@ -19,8 +19,7 @@ import os
import subprocess import subprocess
import argparse import argparse
repeat_time = 100 gbench_prefix = "SONIC_NO_ASYNC_GC=1 go test -benchmem -run=none "
gbench_prefix = "SONIC_NO_ASYNC_GC=1 go test -benchmem -run=none -count=%d "%(repeat_time)
def run(cmd): def run(cmd):
print(cmd) print(cmd)
@ -87,8 +86,8 @@ def compare(args):
run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, args, main)) run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, args, main))
# diff the result # diff the result
benchstat = "go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat" # benchstat = "go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat"
run( "%s && benchstat -sort=delta %s %s"%(benchstat, main, target)) run( "benchstat -sort=delta %s %s"%(main, target))
run("git checkout -- .") run("git checkout -- .")
# restore branch # restore branch
@ -105,6 +104,8 @@ def main():
help='Compare with the main benchmarking') help='Compare with the main benchmarking')
argparser.add_argument('-t', '--times', dest='times', required=False, argparser.add_argument('-t', '--times', dest='times', required=False,
help='benchmark the times') help='benchmark the times')
argparser.add_argument('-r', '--repeat_times', dest='count', required=False,
help='benchmark the count')
args = argparser.parse_args() args = argparser.parse_args()
if args.filter: if args.filter:
@ -114,6 +115,11 @@ def main():
if args.times: if args.times:
gbench_args += " -benchtime=%s"%(args.times) gbench_args += " -benchtime=%s"%(args.times)
if args.count:
gbench_args += " -count=%s"%(args.count)
else:
gbench_args += " -count=10"
if args.compare: if args.compare:
target = compare(gbench_args) target = compare(gbench_args)

24
decode_test.go
View file

@ -1116,6 +1116,7 @@ func TestMarshalEmbeds(t *testing.T) {
func TestUnmarshal(t *testing.T) { func TestUnmarshal(t *testing.T) {
for i, tt := range unmarshalTests { for i, tt := range unmarshalTests {
t.Log(i, tt.in)
if !json.Valid([]byte(tt.in)) { if !json.Valid([]byte(tt.in)) {
continue continue
} }
@ -2005,7 +2006,6 @@ var decodeTypeErrorTests = []struct {
dest interface{} dest interface{}
src string src string
}{ }{
{new(string), `{"user": "name"}`}, // issue 4628.
{new(error), `{}`}, // issue 4222 {new(error), `{}`}, // issue 4222
{new(error), `[]`}, {new(error), `[]`},
{new(error), `""`}, {new(error), `""`},
@ -2025,6 +2025,28 @@ func TestUnmarshalTypeError(t *testing.T) {
} }
} }
var decodeMismatchErrorTests = []struct {
dest interface{}
src string
}{
{new(int), `{}`},
{new(string), `{}`},
{new(bool), `{}`},
{new([]byte), `{}`},
}
func TestMismatchTypeError(t *testing.T) {
for _, item := range decodeMismatchErrorTests {
err := Unmarshal([]byte(item.src), item.dest)
if _, ok := err.(*decoder.MismatchTypeError); !ok {
if _, ok = err.(decoder.SyntaxError); !ok {
t.Errorf("expected mismatch error for Unmarshal(%q, type %T): got %T",
item.src, item.dest, err)
}
}
}
}
var unmarshalSyntaxTests = []string{ var unmarshalSyntaxTests = []string{
"tru", "tru",
"fals", "fals",

152
decoder/assembler_amd64_go116.go
View file

@ -70,7 +70,7 @@ const (
_FP_args = 96 // 96 bytes to pass arguments and return values for this function _FP_args = 96 // 96 bytes to pass arguments and return values for this function
_FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions
_FP_saves = 40 // 40 bytes for saving the registers before CALL instructions _FP_saves = 40 // 40 bytes for saving the registers before CALL instructions
_FP_locals = 120 // 120 bytes for local variables _FP_locals = 144 // 144 bytes for local variables
) )
const ( const (
@ -101,6 +101,7 @@ const (
_LB_unquote_error = "_unquote_error" _LB_unquote_error = "_unquote_error"
_LB_parsing_error = "_parsing_error" _LB_parsing_error = "_parsing_error"
_LB_parsing_error_v = "_parsing_error_v" _LB_parsing_error_v = "_parsing_error_v"
_LB_mismatch_error = "_mismatch_error"
) )
const ( const (
@ -113,6 +114,8 @@ const (
_LB_char_m3_error = "_char_m3_error" _LB_char_m3_error = "_char_m3_error"
) )
const _LB_skip_one = "_skip_one"
var ( var (
_AX = jit.Reg("AX") _AX = jit.Reg("AX")
_CX = jit.Reg("CX") _CX = jit.Reg("CX")
@ -196,6 +199,12 @@ var (
var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112)
var (
_VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type
_VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save dismatched position
_VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save skip return pc
)
type _Assembler struct { type _Assembler struct {
jit.BaseAssembler jit.BaseAssembler
p _Program p _Program
@ -224,6 +233,8 @@ func (self *_Assembler) compile() {
self.copy_string() self.copy_string()
self.escape_string() self.escape_string()
self.escape_string_twice() self.escape_string_twice()
self.skip_one()
self.mismatch_error()
self.type_error() self.type_error()
self.field_error() self.field_error()
self.range_error() self.range_error()
@ -295,6 +306,10 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_recurse : (*_Assembler)._asm_OP_recurse,
_OP_goto : (*_Assembler)._asm_OP_goto, _OP_goto : (*_Assembler)._asm_OP_goto,
_OP_switch : (*_Assembler)._asm_OP_switch, _OP_switch : (*_Assembler)._asm_OP_switch,
_OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0,
_OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err,
_OP_go_skip : (*_Assembler)._asm_OP_go_skip,
_OP_add : (*_Assembler)._asm_OP_add,
} }
func (self *_Assembler) instr(v *_Instr) { func (self *_Assembler) instr(v *_Instr) {
@ -315,8 +330,10 @@ func (self *_Assembler) instrs() {
func (self *_Assembler) epilogue() { func (self *_Assembler) epilogue() {
self.Mark(len(self.p)) self.Mark(len(self.p))
self.Emit("XORL", _ET, _ET) // XORL ET, ET
self.Emit("XORL", _EP, _EP) // XORL EP, EP self.Emit("XORL", _EP, _EP) // XORL EP, EP
self.Emit("MOVQ", _VAR_et, _ET) // MOVQ VAR_et, ET
self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
self.Sjmp("JNZ", _LB_mismatch_error) // JNZ _LB_mismatch_error
self.Link(_LB_error) // _error: self.Link(_LB_error) // _error:
self.Emit("MOVQ", _IC, _RET_rc) // MOVQ IC, rc<>+40(FP) self.Emit("MOVQ", _IC, _RET_rc) // MOVQ IC, rc<>+40(FP)
self.Emit("MOVQ", _ET, _RET_et) // MOVQ ET, et<>+48(FP) self.Emit("MOVQ", _ET, _RET_et) // MOVQ ET, et<>+48(FP)
@ -339,6 +356,8 @@ func (self *_Assembler) prologue() {
self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap
self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX
self.Emit("MOVQ", _AX, _VAR_st_Db) // MOVQ AX, ss.Dbuf self.Emit("MOVQ", _AX, _VAR_st_Db) // MOVQ AX, ss.Dbuf
self.Emit("XORL", _AX, _AX) // XORL AX, AX
self.Emit("MOVQ", _AX, _VAR_et) // MOVQ AX, ss.Dp
} }
/** Function Calling Helpers **/ /** Function Calling Helpers **/
@ -404,11 +423,12 @@ func (self *_Assembler) call_vf(fn obj.Addr) {
/** Assembler Error Handlers **/ /** Assembler Error Handlers **/
var ( var (
_F_convT64 = jit.Func(convT64) _F_convT64 = jit.Func(convT64)
_F_error_wrap = jit.Func(error_wrap) _F_error_wrap = jit.Func(error_wrap)
_F_error_type = jit.Func(error_type) _F_error_type = jit.Func(error_type)
_F_error_field = jit.Func(error_field) _F_error_field = jit.Func(error_field)
_F_error_value = jit.Func(error_value) _F_error_value = jit.Func(error_value)
_F_error_mismatch = jit.Func(error_mismatch)
) )
var ( var (
@ -440,6 +460,46 @@ func (self *_Assembler) type_error() {
self.Sjmp("JMP" , _LB_error) // JMP _error self.Sjmp("JMP" , _LB_error) // JMP _error
} }
func (self *_Assembler) mismatch_error() {
self.Link(_LB_mismatch_error) // _type_error:
self.Emit("MOVQ", _ARG_sp, _AX)
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP)
self.Emit("MOVQ", _ARG_sl, _CX)
self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP)
self.Emit("MOVQ", _VAR_ic, _AX)
self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP)
self.Emit("MOVQ", _VAR_et, _CX)
self.Emit("MOVQ", _CX, jit.Ptr(_SP, 24)) // MOVQ CX, 24(SP)
self.call_go(_F_error_mismatch) // CALL_GO error_type
self.Emit("MOVQ", jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET
self.Emit("MOVQ", jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP
self.Sjmp("JMP" , _LB_error) // JMP _error
}
func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) {
self.Emit("MOVQ", _IC, _VAR_ic)
self.Emit("MOVQ", jit.Type(p.vt()), _ET)
self.Emit("MOVQ", _ET, _VAR_et)
}
func (self *_Assembler) _asm_OP_go_skip(p *_Instr) {
self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9
self.Xref(p.vi(), 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one) // JMP _skip_one
}
func (self *_Assembler) skip_one() {
self.Link(_LB_skip_one) // _skip:
self.Emit("MOVQ", _VAR_ic, _IC) // MOVQ _VAR_ic, IC
self.call_sf(_F_skip_one) // CALL_SF skip_one
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v
self.Emit("MOVQ" , _VAR_pc, _R9) // MOVQ pc, R9
self.Rjmp("JMP" , _R9) // JMP (R9)
}
func (self *_Assembler) field_error() { func (self *_Assembler) field_error() {
self.Link(_LB_field_error) // _field_error: self.Link(_LB_field_error) // _field_error:
self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0
@ -575,10 +635,23 @@ var (
_F_vunsigned = jit.Imm(int64(native.S_vunsigned)) _F_vunsigned = jit.Imm(int64(native.S_vunsigned))
) )
func (self *_Assembler) check_err() { func (self *_Assembler) check_err(vt reflect.Type) {
self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX
self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING} self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING}
self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v // try to skip the value
if vt != nil {
self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v
self.Emit("MOVQ", _BP, _VAR_ic)
self.Emit("MOVQ", jit.Type(vt), _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9
self.Sref("_check_err_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
self.Link("_check_err_{n}")
} else {
self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v
}
} }
func (self *_Assembler) check_eof(d int64) { func (self *_Assembler) check_eof(d int64) {
@ -595,22 +668,25 @@ func (self *_Assembler) check_eof(d int64) {
func (self *_Assembler) parse_string() { // parse_string has a validate flag params in the last func (self *_Assembler) parse_string() { // parse_string has a validate flag params in the last
self.Emit("MOVQ", _ARG_fv, _CX) self.Emit("MOVQ", _ARG_fv, _CX)
self.call_vf(_F_vstring) self.call_vf(_F_vstring)
self.check_err() self.check_err(nil)
} }
func (self *_Assembler) parse_number() { func (self *_Assembler) parse_number() {
self.Emit("MOVQ", _IC, _BP)
self.call_vf(_F_vnumber) // call vnumber self.call_vf(_F_vnumber) // call vnumber
self.check_err() self.check_err(floatType)
} }
func (self *_Assembler) parse_signed() { func (self *_Assembler) parse_signed() {
self.Emit("MOVQ", _IC, _BP)
self.call_vf(_F_vsigned) self.call_vf(_F_vsigned)
self.check_err() self.check_err(intType)
} }
func (self *_Assembler) parse_unsigned() { func (self *_Assembler) parse_unsigned() {
self.Emit("MOVQ", _IC, _BP)
self.call_vf(_F_vunsigned) self.call_vf(_F_vunsigned)
self.check_err() self.check_err(uintType)
} }
// Pointer: DI, Size: SI, Return: R9 // Pointer: DI, Size: SI, Return: R9
@ -1144,7 +1220,18 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Sjmp("JE" , "_false_{n}") // JE _false_{n} self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX
self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC)
self.Sjmp("JNE" , _LB_im_error) // JNE _im_error self.Sjmp("JE" , "_bool_true_{n}")
// try to skip the value
self.Emit("MOVQ", _IC, _VAR_ic)
self.Emit("MOVQ", _T_bool, _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9
self.Sref("_end_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
self.Link("_bool_true_{n}")
self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC
self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP) self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP)
self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n}
@ -1165,20 +1252,28 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
func (self *_Assembler) _asm_OP_num(_ *_Instr) { func (self *_Assembler) _asm_OP_num(_ *_Instr) {
self.Emit("MOVQ", jit.Imm(0), _VAR_fl) self.Emit("MOVQ", jit.Imm(0), _VAR_fl)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"'))
self.Emit("MOVQ", _IC, _BP)
self.Sjmp("JNE", "_skip_number_{n}") self.Sjmp("JNE", "_skip_number_{n}")
self.Emit("MOVQ", jit.Imm(1), _VAR_fl) self.Emit("MOVQ", jit.Imm(1), _VAR_fl)
self.Emit("ADDQ", jit.Imm(1), _IC) self.Emit("ADDQ", jit.Imm(1), _IC)
self.Link("_skip_number_{n}") self.Link("_skip_number_{n}")
/* call skip_number */ /* call skip_number */
self.Emit("LEAQ", _ARG_s, _DI) // LEAQ s<>+0(FP), DI self.call_sf(_F_skip_number) // CALL_SF skip_one
self.Emit("MOVQ", _IC, _ARG_ic) // MOVQ IC, ic<>+16(FP) self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Emit("LEAQ", _ARG_ic, _SI) // LEAQ ic<>+16(FP), SI self.Sjmp("JNS" , "_num_next_{n}")
self.call(_F_skip_number) // CALL _F_skip_number
self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v
/* call skip one */
self.Emit("MOVQ", _BP, _VAR_ic)
self.Emit("MOVQ", _T_number, _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c, 0x8d, 0x0d)
self.Sref("_num_end_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
/* assgin string */
self.Link("_num_next_{n}")
self.slice_from_r(_AX, 0) self.slice_from_r(_AX, 0)
self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv) self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv)
self.Sjmp("JNC", "_num_write_{n}") self.Sjmp("JNC", "_num_write_{n}")
@ -1187,7 +1282,9 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) {
self.Sjmp("JMP", "_copy_string") self.Sjmp("JMP", "_copy_string")
self.Link("_num_write_{n}") self.Link("_num_write_{n}")
self.Emit("MOVQ", _SI, jit.Ptr(_VP, 8)) // MOVQ SI, 8(VP) self.Emit("MOVQ", _SI, jit.Ptr(_VP, 8)) // MOVQ SI, 8(VP)
self.WriteRecNotAX(13, _DI, jit.Ptr(_VP, 0), false, false) self.WriteRecNotAX(13, _DI, jit.Ptr(_VP, 0), false, false)
/* check if quoted */
self.Emit("CMPQ", _VAR_fl, jit.Imm(1)) self.Emit("CMPQ", _VAR_fl, jit.Imm(1))
self.Sjmp("JNE", "_num_end_{n}") self.Sjmp("JNE", "_num_end_{n}")
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"'))
@ -1195,6 +1292,7 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) {
self.Emit("ADDQ", jit.Imm(1), _IC) self.Emit("ADDQ", jit.Imm(1), _IC)
self.Link("_num_end_{n}") self.Link("_num_end_{n}")
} }
func (self *_Assembler) _asm_OP_i8(_ *_Instr) { func (self *_Assembler) _asm_OP_i8(_ *_Instr) {
self.parse_signed() // PARSE int8 self.parse_signed() // PARSE int8
self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8
@ -1636,6 +1734,16 @@ func (self *_Assembler) _asm_OP_check_char(p *_Instr) {
self.Xjmp("JE" , p.vi()) // JE {p.vi()} self.Xjmp("JE" , p.vi()) // JE {p.vi()}
} }
func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) {
self.check_eof(1)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()}
self.Xjmp("JE" , p.vi()) // JE {p.vi()}
}
func (self *_Assembler) _asm_OP_add(p *_Instr) {
self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC
}
func (self *_Assembler) _asm_OP_load(_ *_Instr) { func (self *_Assembler) _asm_OP_load(_ *_Instr) {
self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX
self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP

138
decoder/assembler_amd64_go117.go
View file

@ -70,7 +70,7 @@ const (
_FP_args = 72 // 72 bytes to pass and spill register arguements _FP_args = 72 // 72 bytes to pass and spill register arguements
_FP_fargs = 80 // 80 bytes for passing arguments to other Go functions _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions
_FP_saves = 48 // 48 bytes for saving the registers before CALL instructions _FP_saves = 48 // 48 bytes for saving the registers before CALL instructions
_FP_locals = 120 // 112 bytes for local variables _FP_locals = 144 // 144 bytes for local variables
) )
const ( const (
@ -101,6 +101,7 @@ const (
_LB_unquote_error = "_unquote_error" _LB_unquote_error = "_unquote_error"
_LB_parsing_error = "_parsing_error" _LB_parsing_error = "_parsing_error"
_LB_parsing_error_v = "_parsing_error_v" _LB_parsing_error_v = "_parsing_error_v"
_LB_mismatch_error = "_mismatch_error"
) )
const ( const (
@ -113,6 +114,8 @@ const (
_LB_char_m3_error = "_char_m3_error" _LB_char_m3_error = "_char_m3_error"
) )
const _LB_skip_one = "_skip_one"
var ( var (
_AX = jit.Reg("AX") _AX = jit.Reg("AX")
_BX = jit.Reg("BX") _BX = jit.Reg("BX")
@ -191,6 +194,12 @@ var (
var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112) var _VAR_fl = jit.Ptr(_SP, _FP_fargs + _FP_saves + 112)
var (
_VAR_et = jit.Ptr(_SP, _FP_fargs + _FP_saves + 120) // save dismatched type
_VAR_pc = jit.Ptr(_SP, _FP_fargs + _FP_saves + 128) // save skip return pc
_VAR_ic = jit.Ptr(_SP, _FP_fargs + _FP_saves + 136) // save dismatched position
)
type _Assembler struct { type _Assembler struct {
jit.BaseAssembler jit.BaseAssembler
p _Program p _Program
@ -219,7 +228,9 @@ func (self *_Assembler) compile() {
self.copy_string() self.copy_string()
self.escape_string() self.escape_string()
self.escape_string_twice() self.escape_string_twice()
self.skip_one()
self.type_error() self.type_error()
self.mismatch_error()
self.field_error() self.field_error()
self.range_error() self.range_error()
self.stack_error() self.stack_error()
@ -290,6 +301,15 @@ var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_recurse : (*_Assembler)._asm_OP_recurse, _OP_recurse : (*_Assembler)._asm_OP_recurse,
_OP_goto : (*_Assembler)._asm_OP_goto, _OP_goto : (*_Assembler)._asm_OP_goto,
_OP_switch : (*_Assembler)._asm_OP_switch, _OP_switch : (*_Assembler)._asm_OP_switch,
_OP_check_char_0 : (*_Assembler)._asm_OP_check_char_0,
_OP_dismatch_err : (*_Assembler)._asm_OP_dismatch_err,
_OP_go_skip : (*_Assembler)._asm_OP_go_skip,
_OP_add : (*_Assembler)._asm_OP_add,
_OP_debug : (*_Assembler)._asm_OP_debug,
}
func (self *_Assembler) _asm_OP_debug(_ *_Instr) {
self.Byte(0xcc)
} }
func (self *_Assembler) instr(v *_Instr) { func (self *_Assembler) instr(v *_Instr) {
@ -310,8 +330,10 @@ func (self *_Assembler) instrs() {
func (self *_Assembler) epilogue() { func (self *_Assembler) epilogue() {
self.Mark(len(self.p)) self.Mark(len(self.p))
self.Emit("XORL", _ET, _ET) // XORL ET, ET
self.Emit("XORL", _EP, _EP) // XORL EP, EP self.Emit("XORL", _EP, _EP) // XORL EP, EP
self.Emit("MOVQ", _VAR_et, _ET) // MOVQ VAR_et, ET
self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
self.Sjmp("JNZ", _LB_mismatch_error) // JNZ _LB_mismatch_error
self.Link(_LB_error) // _error: self.Link(_LB_error) // _error:
self.Emit("MOVQ", _EP, _CX) // MOVQ BX, CX self.Emit("MOVQ", _EP, _CX) // MOVQ BX, CX
self.Emit("MOVQ", _ET, _BX) // MOVQ AX, BX self.Emit("MOVQ", _ET, _BX) // MOVQ AX, BX
@ -343,6 +365,7 @@ func (self *_Assembler) prologue() {
self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ $0, et<>+120(FP)
// initialize digital buffer first // initialize digital buffer first
self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap
self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX self.Emit("LEAQ", jit.Ptr(_ST, _DbufOffset), _AX) // LEAQ _DbufOffset(ST), AX
@ -421,11 +444,12 @@ func (self *_Assembler) call_vf(fn obj.Addr) {
/** Assembler Error Handlers **/ /** Assembler Error Handlers **/
var ( var (
_F_convT64 = jit.Func(convT64) _F_convT64 = jit.Func(convT64)
_F_error_wrap = jit.Func(error_wrap) _F_error_wrap = jit.Func(error_wrap)
_F_error_type = jit.Func(error_type) _F_error_type = jit.Func(error_type)
_F_error_field = jit.Func(error_field) _F_error_field = jit.Func(error_field)
_F_error_value = jit.Func(error_value) _F_error_value = jit.Func(error_value)
_F_error_mismatch = jit.Func(error_mismatch)
) )
var ( var (
@ -454,6 +478,16 @@ func (self *_Assembler) type_error() {
self.Sjmp("JMP" , _LB_error) // JMP _error self.Sjmp("JMP" , _LB_error) // JMP _error
} }
func (self *_Assembler) mismatch_error() {
self.Link(_LB_mismatch_error) // _type_error:
self.Emit("MOVQ", _ARG_sp, _AX)
self.Emit("MOVQ", _ARG_sl, _BX)
self.Emit("MOVQ", _VAR_ic, _CX)
self.Emit("MOVQ", _VAR_et, _DI)
self.call_go(_F_error_mismatch) // CALL_GO error_type
self.Sjmp("JMP" , _LB_error) // JMP _error
}
func (self *_Assembler) field_error() { func (self *_Assembler) field_error() {
self.Link(_LB_field_error) // _field_error: self.Link(_LB_field_error) // _field_error:
self.Emit("MOVQ", _VAR_sv_p, _AX) // MOVQ sv.p, AX self.Emit("MOVQ", _VAR_sv_p, _AX) // MOVQ sv.p, AX
@ -536,6 +570,31 @@ func (self *_Assembler) parsing_error() {
self.Sjmp("JMP" , _LB_error) // JMP _error self.Sjmp("JMP" , _LB_error) // JMP _error
} }
func (self *_Assembler) _asm_OP_dismatch_err(p *_Instr) {
self.Emit("MOVQ", _IC, _VAR_ic)
self.Emit("MOVQ", jit.Type(p.vt()), _ET)
self.Emit("MOVQ", _ET, _VAR_et)
}
func (self *_Assembler) _asm_OP_go_skip(p *_Instr) {
self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9
self.Xref(p.vi(), 4)
// self.Byte(0xcc)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one) // JMP _skip_one
}
func (self *_Assembler) skip_one() {
self.Link(_LB_skip_one) // _skip:
self.Emit("MOVQ", _VAR_ic, _IC) // MOVQ _VAR_ic, IC
self.call_sf(_F_skip_one) // CALL_SF skip_one
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v
self.Emit("MOVQ" , _VAR_pc, _R9) // MOVQ pc, R9
// self.Byte(0xcc)
self.Rjmp("JMP" , _R9) // JMP (R9)
}
/** Memory Management Routines **/ /** Memory Management Routines **/
var ( var (
@ -585,10 +644,23 @@ var (
_F_vunsigned = jit.Imm(int64(native.S_vunsigned)) _F_vunsigned = jit.Imm(int64(native.S_vunsigned))
) )
func (self *_Assembler) check_err() { func (self *_Assembler) check_err(vt reflect.Type) {
self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX
self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING} self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING}
self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v // try to skip the value
if vt != nil {
self.Sjmp("JNS" , "_check_err_{n}") // JNE _parsing_error_v
self.Emit("MOVQ", _BX, _VAR_ic)
self.Emit("MOVQ", jit.Type(vt), _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c , 0x8d, 0x0d) // LEAQ (PC), R9
self.Sref("_check_err_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
self.Link("_check_err_{n}")
} else {
self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v
}
} }
func (self *_Assembler) check_eof(d int64) { func (self *_Assembler) check_eof(d int64) {
@ -602,25 +674,29 @@ func (self *_Assembler) check_eof(d int64) {
} }
} }
func (self *_Assembler) parse_string() { func (self *_Assembler) parse_string() {
self.Emit("MOVQ", _ARG_fv, _CX) self.Emit("MOVQ", _ARG_fv, _CX)
self.call_vf(_F_vstring) self.call_vf(_F_vstring)
self.check_err() self.check_err(nil)
} }
func (self *_Assembler) parse_number() { func (self *_Assembler) parse_number() {
self.Emit("MOVQ", _IC, _BX) // save ic when call native func
self.call_vf(_F_vnumber) self.call_vf(_F_vnumber)
self.check_err() self.check_err(floatType)
} }
func (self *_Assembler) parse_signed() { func (self *_Assembler) parse_signed() {
self.Emit("MOVQ", _IC, _BX) // save ic when call native func
self.call_vf(_F_vsigned) self.call_vf(_F_vsigned)
self.check_err() self.check_err(intType)
} }
func (self *_Assembler) parse_unsigned() { func (self *_Assembler) parse_unsigned() {
self.Emit("MOVQ", _IC, _BX) // save ic when call native func
self.call_vf(_F_vunsigned) self.call_vf(_F_vunsigned)
self.check_err() self.check_err(uintType)
} }
// Pointer: DI, Size: SI, Return: R9 // Pointer: DI, Size: SI, Return: R9
@ -1138,7 +1214,17 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Sjmp("JE" , "_false_{n}") // JE _false_{n} self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX
self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC)
self.Sjmp("JNE" , _LB_im_error) // JNE _im_error self.Sjmp("JE" , "_bool_true_{n}")
// try to skip the value
self.Emit("MOVQ", _IC, _VAR_ic)
self.Emit("MOVQ", _T_bool, _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c, 0x8d, 0x0d) // LEAQ (PC), R9
self.Sref("_end_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
self.Link("_bool_true_{n}")
self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC
self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP) self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP)
self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n}
@ -1159,6 +1245,7 @@ func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
func (self *_Assembler) _asm_OP_num(_ *_Instr) { func (self *_Assembler) _asm_OP_num(_ *_Instr) {
self.Emit("MOVQ", jit.Imm(0), _VAR_fl) self.Emit("MOVQ", jit.Imm(0), _VAR_fl)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"')) self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('"'))
self.Emit("MOVQ", _IC, _BX)
self.Sjmp("JNE", "_skip_number_{n}") self.Sjmp("JNE", "_skip_number_{n}")
self.Emit("MOVQ", jit.Imm(1), _VAR_fl) self.Emit("MOVQ", jit.Imm(1), _VAR_fl)
self.Emit("ADDQ", jit.Imm(1), _IC) self.Emit("ADDQ", jit.Imm(1), _IC)
@ -1171,8 +1258,19 @@ func (self *_Assembler) _asm_OP_num(_ *_Instr) {
self.callc(_F_skip_number) // CALL _F_skip_number self.callc(_F_skip_number) // CALL _F_skip_number
self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v self.Sjmp("JNS" , "_num_next_{n}")
/* call skip one */
self.Emit("MOVQ", _BX, _VAR_ic)
self.Emit("MOVQ", _T_number, _ET)
self.Emit("MOVQ", _ET, _VAR_et)
self.Byte(0x4c, 0x8d, 0x0d)
self.Sref("_num_end_{n}", 4)
self.Emit("MOVQ", _R9, _VAR_pc)
self.Sjmp("JMP" , _LB_skip_one)
/* assgin string */
self.Link("_num_next_{n}")
self.slice_from_r(_AX, 0) self.slice_from_r(_AX, 0)
self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv) self.Emit("BTQ", jit.Imm(_F_copy_string), _ARG_fv)
self.Sjmp("JNC", "_num_write_{n}") self.Sjmp("JNC", "_num_write_{n}")
@ -1616,6 +1714,16 @@ func (self *_Assembler) _asm_OP_check_char(p *_Instr) {
self.Xjmp("JE" , p.vi()) // JE {p.vi()} self.Xjmp("JE" , p.vi()) // JE {p.vi()}
} }
func (self *_Assembler) _asm_OP_check_char_0(p *_Instr) {
self.check_eof(1)
self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()}
self.Xjmp("JE" , p.vi()) // JE {p.vi()}
}
func (self *_Assembler) _asm_OP_add(p *_Instr) {
self.Emit("ADDQ", jit.Imm(int64(p.vi())), _IC) // ADDQ ${p.vi()}, IC
}
func (self *_Assembler) _asm_OP_load(_ *_Instr) { func (self *_Assembler) _asm_OP_load(_ *_Instr) {
self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX
self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP

29
decoder/assembler_test.go
View file

@ -12,7 +12,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package decoder package decoder
@ -230,7 +230,16 @@ func TestAssembler_OpCode(t *testing.T) {
src: "true", src: "true",
exp: true, exp: true,
val: new(bool), val: new(bool),
}, { },
{
key: "_OP_bool/skip",
ins: []_Instr{newInsOp(_OP_bool)},
src: `"true"`,
exp: nil,
val: new(bool),
err: &MismatchTypeError{Src: `"true"`, Pos: 0, Type: reflect.TypeOf(true)},
},
{
key: "_OP_bool/false", key: "_OP_bool/false",
ins: []_Instr{newInsOp(_OP_bool)}, ins: []_Instr{newInsOp(_OP_bool)},
src: "false", src: "false",
@ -266,7 +275,8 @@ func TestAssembler_OpCode(t *testing.T) {
src: "falsx", src: "falsx",
err: SyntaxError{Src: `falsx`, Pos: 4, Code: types.ERR_INVALID_CHAR}, err: SyntaxError{Src: `falsx`, Pos: 4, Code: types.ERR_INVALID_CHAR},
val: new(bool), val: new(bool),
}, { },
{
key: "_OP_num/positive", key: "_OP_num/positive",
ins: []_Instr{newInsOp(_OP_num)}, ins: []_Instr{newInsOp(_OP_num)},
src: "1.234e5", src: "1.234e5",
@ -282,13 +292,13 @@ func TestAssembler_OpCode(t *testing.T) {
key: "_OP_num/error_eof", key: "_OP_num/error_eof",
ins: []_Instr{newInsOp(_OP_num)}, ins: []_Instr{newInsOp(_OP_num)},
src: "-", src: "-",
err: SyntaxError{Src: `-`, Pos: 1, Code: types.ERR_EOF}, err: SyntaxError{Src: `-`, Pos: 1, Code: types.ERR_INVALID_CHAR},
val: new(json.Number), val: new(json.Number),
}, { }, {
key: "_OP_num/error_invalid_char", key: "_OP_num/error_invalid_char",
ins: []_Instr{newInsOp(_OP_num)}, ins: []_Instr{newInsOp(_OP_num)},
src: "xxx", src: "xxx",
err: SyntaxError{Src: `xxx`, Pos: 0, Code: types.ERR_INVALID_CHAR}, err: SyntaxError{Src: `xxx`, Pos: 1, Code: types.ERR_INVALID_CHAR},
val: new(json.Number), val: new(json.Number),
}, { }, {
key: "_OP_i8", key: "_OP_i8",
@ -302,11 +312,12 @@ func TestAssembler_OpCode(t *testing.T) {
src: "1234", src: "1234",
err: error_value("1234", reflect.TypeOf(int8(0))), err: error_value("1234", reflect.TypeOf(int8(0))),
val: new(int8), val: new(int8),
}, { },
{
key: "_OP_i8/error_wrong_type", key: "_OP_i8/error_wrong_type",
ins: []_Instr{newInsOp(_OP_i8)}, ins: []_Instr{newInsOp(_OP_i8)},
src: "12.34", src: "12.34",
err: SyntaxError{Src: `12.34`, Pos: 2, Code: types.ERR_INVALID_NUMBER_FMT}, err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: intType},
val: new(int8), val: new(int8),
}, { }, {
key: "_OP_u8", key: "_OP_u8",
@ -324,13 +335,13 @@ func TestAssembler_OpCode(t *testing.T) {
key: "_OP_u8/error_underflow", key: "_OP_u8/error_underflow",
ins: []_Instr{newInsOp(_OP_u8)}, ins: []_Instr{newInsOp(_OP_u8)},
src: "-123", src: "-123",
err: SyntaxError{Src: `-123`, Pos: 0, Code: types.ERR_INVALID_NUMBER_FMT}, err: &MismatchTypeError{Src: `-123`, Pos: 0, Type: uintType},
val: new(uint8), val: new(uint8),
}, { }, {
key: "_OP_u8/error_wrong_type", key: "_OP_u8/error_wrong_type",
ins: []_Instr{newInsOp(_OP_u8)}, ins: []_Instr{newInsOp(_OP_u8)},
src: "12.34", src: "12.34",
err: SyntaxError{Src: `12.34`, Pos: 2, Code: types.ERR_INVALID_NUMBER_FMT}, err: &MismatchTypeError{Src: `12.34`, Pos: 0, Type: uintType},
val: new(uint8), val: new(uint8),
}, { }, {
key: "_OP_f32", key: "_OP_f32",

101
decoder/compiler.go
View file

@ -94,6 +94,11 @@ const (
_OP_recurse _OP_recurse
_OP_goto _OP_goto
_OP_switch _OP_switch
_OP_check_char_0
_OP_dismatch_err
_OP_go_skip
_OP_add
_OP_debug
) )
const ( const (
@ -165,6 +170,9 @@ var _OpNames = [256]string {
_OP_recurse : "recurse", _OP_recurse : "recurse",
_OP_goto : "goto", _OP_goto : "goto",
_OP_switch : "switch", _OP_switch : "switch",
_OP_check_char_0 : "check_char_0",
_OP_dismatch_err : "dismatch_err",
_OP_add : "add",
} }
func (self _Op) String() string { func (self _Op) String() string {
@ -560,26 +568,26 @@ func (self *_Compiler) compileOne(p *_Program, sp int, vt reflect.Type) {
func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) { func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
switch vt.Kind() { switch vt.Kind() {
case reflect.Bool : self.compilePrimitive (p, _OP_bool) case reflect.Bool : self.compilePrimitive (vt, p, _OP_bool)
case reflect.Int : self.compilePrimitive (p, _OP_int()) case reflect.Int : self.compilePrimitive (vt, p, _OP_int())
case reflect.Int8 : self.compilePrimitive (p, _OP_i8) case reflect.Int8 : self.compilePrimitive (vt, p, _OP_i8)
case reflect.Int16 : self.compilePrimitive (p, _OP_i16) case reflect.Int16 : self.compilePrimitive (vt, p, _OP_i16)
case reflect.Int32 : self.compilePrimitive (p, _OP_i32) case reflect.Int32 : self.compilePrimitive (vt, p, _OP_i32)
case reflect.Int64 : self.compilePrimitive (p, _OP_i64) case reflect.Int64 : self.compilePrimitive (vt, p, _OP_i64)
case reflect.Uint : self.compilePrimitive (p, _OP_uint()) case reflect.Uint : self.compilePrimitive (vt, p, _OP_uint())
case reflect.Uint8 : self.compilePrimitive (p, _OP_u8) case reflect.Uint8 : self.compilePrimitive (vt, p, _OP_u8)
case reflect.Uint16 : self.compilePrimitive (p, _OP_u16) case reflect.Uint16 : self.compilePrimitive (vt, p, _OP_u16)
case reflect.Uint32 : self.compilePrimitive (p, _OP_u32) case reflect.Uint32 : self.compilePrimitive (vt, p, _OP_u32)
case reflect.Uint64 : self.compilePrimitive (p, _OP_u64) case reflect.Uint64 : self.compilePrimitive (vt, p, _OP_u64)
case reflect.Uintptr : self.compilePrimitive (p, _OP_uintptr()) case reflect.Uintptr : self.compilePrimitive (vt, p, _OP_uintptr())
case reflect.Float32 : self.compilePrimitive (p, _OP_f32) case reflect.Float32 : self.compilePrimitive (vt, p, _OP_f32)
case reflect.Float64 : self.compilePrimitive (p, _OP_f64) case reflect.Float64 : self.compilePrimitive (vt, p, _OP_f64)
case reflect.String : self.compileString (p, vt) case reflect.String : self.compileString (p, vt)
case reflect.Array : self.compileArray (p, sp, vt) case reflect.Array : self.compileArray (p, sp, vt)
case reflect.Interface : self.compileInterface (p, vt) case reflect.Interface : self.compileInterface (p, vt)
case reflect.Map : self.compileMap (p, sp, vt) case reflect.Map : self.compileMap (p, sp, vt)
case reflect.Ptr : self.compilePtr (p, sp, vt) case reflect.Ptr : self.compilePtr (p, sp, vt)
case reflect.Slice : self.compileSlice (p, sp, vt.Elem()) case reflect.Slice : self.compileSlice (p, sp, vt)
case reflect.Struct : self.compileStruct (p, sp, vt) case reflect.Struct : self.compileStruct (p, sp, vt)
default : panic (&json.UnmarshalTypeError{Type: vt}) default : panic (&json.UnmarshalTypeError{Type: vt})
} }
@ -619,8 +627,8 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
i := p.pc() i := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.tag(sp + 1) p.tag(sp + 1)
skip := self.checkIfSkip(p, vt, '{')
p.add(_OP_save) p.add(_OP_save)
p.chr(_OP_match_char, '{')
p.add(_OP_map_init) p.add(_OP_map_init)
p.add(_OP_save) p.add(_OP_save)
p.add(_OP_lspace) p.add(_OP_lspace)
@ -666,6 +674,7 @@ func (self *_Compiler) compileMapOp(p *_Program, sp int, vt reflect.Type, op _Op
p.add(_OP_goto) p.add(_OP_goto)
p.pin(i) p.pin(i)
p.add(_OP_nil_1) p.add(_OP_nil_1)
p.pin(skip)
p.pin(x) p.pin(x)
} }
@ -692,7 +701,8 @@ func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type) {
x := p.pc() x := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.tag(sp) p.tag(sp)
p.chr(_OP_match_char, '[') skip := self.checkIfSkip(p, vt, '[')
p.add(_OP_save) p.add(_OP_save)
p.add(_OP_lspace) p.add(_OP_lspace)
v := []int{p.pc()} v := []int{p.pc()}
@ -725,50 +735,54 @@ func (self *_Compiler) compileArray(p *_Program, sp int, vt reflect.Type) {
/* restore the stack */ /* restore the stack */
p.pin(w) p.pin(w)
p.add(_OP_drop) p.add(_OP_drop)
p.pin(skip)
p.pin(x) p.pin(x)
} }
func (self *_Compiler) compileSlice(p *_Program, sp int, et reflect.Type) { func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
if et.Kind() == byteType.Kind() { if vt.Elem().Kind() == byteType.Kind() {
self.compileSliceBin(p, sp, et) self.compileSliceBin(p, sp, vt)
} else { } else {
self.compileSliceList(p, sp, et) self.compileSliceList(p, sp, vt)
} }
} }
func (self *_Compiler) compileSliceBin(p *_Program, sp int, et reflect.Type) { func (self *_Compiler) compileSliceBin(p *_Program, sp int, vt reflect.Type) {
i := p.pc() i := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
j := p.pc() j := p.pc()
p.chr(_OP_check_char, '[') p.chr(_OP_check_char, '[')
p.chr(_OP_match_char, '"') skip := self.checkIfSkip(p, vt, '"')
k := p.pc() k := p.pc()
p.chr(_OP_check_char, '"') p.chr(_OP_check_char, '"')
p.add(_OP_bin) p.add(_OP_bin)
x := p.pc() x := p.pc()
p.add(_OP_goto) p.add(_OP_goto)
p.pin(j) p.pin(j)
self.compileSliceBody(p, sp, et) self.compileSliceBody(p, sp, vt.Elem())
y := p.pc() y := p.pc()
p.add(_OP_goto) p.add(_OP_goto)
p.pin(i) p.pin(i)
p.pin(k) p.pin(k)
p.add(_OP_nil_3) p.add(_OP_nil_3)
p.pin(x) p.pin(x)
p.pin(skip)
p.pin(y) p.pin(y)
} }
func (self *_Compiler) compileSliceList(p *_Program, sp int, et reflect.Type) { func (self *_Compiler) compileSliceList(p *_Program, sp int, vt reflect.Type) {
i := p.pc() i := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.tag(sp) p.tag(sp)
p.chr(_OP_match_char, '[') skip := self.checkIfSkip(p, vt, '[')
self.compileSliceBody(p, sp, et) self.compileSliceBody(p, sp, vt.Elem())
x := p.pc() x := p.pc()
p.add(_OP_goto) p.add(_OP_goto)
p.pin(i) p.pin(i)
p.add(_OP_nil_3) p.add(_OP_nil_3)
p.pin(x) p.pin(x)
p.pin(skip)
} }
func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) { func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) {
@ -796,18 +810,19 @@ func (self *_Compiler) compileSliceBody(p *_Program, sp int, et reflect.Type) {
func (self *_Compiler) compileString(p *_Program, vt reflect.Type) { func (self *_Compiler) compileString(p *_Program, vt reflect.Type) {
if vt == jsonNumberType { if vt == jsonNumberType {
self.compilePrimitive(p, _OP_num) self.compilePrimitive(vt, p, _OP_num)
} else { } else {
self.compileStringBody(p) self.compileStringBody(vt, p)
} }
} }
func (self *_Compiler) compileStringBody(p *_Program) { func (self *_Compiler) compileStringBody(vt reflect.Type, p *_Program) {
i := p.pc() i := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.chr(_OP_match_char, '"') skip := self.checkIfSkip(p, vt, '"')
p.add(_OP_str) p.add(_OP_str)
p.pin(i) p.pin(i)
p.pin(skip)
} }
func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) { func (self *_Compiler) compileStruct(p *_Program, sp int, vt reflect.Type) {
@ -829,7 +844,9 @@ func (self *_Compiler) compileStructBody(p *_Program, sp int, vt reflect.Type) {
p.tag(sp) p.tag(sp)
n := p.pc() n := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.chr(_OP_match_char, '{')
skip := self.checkIfSkip(p, vt, '{')
p.add(_OP_save) p.add(_OP_save)
p.add(_OP_lspace) p.add(_OP_lspace)
x := p.pc() x := p.pc()
@ -891,6 +908,7 @@ end_of_object:
p.pin(y1) p.pin(y1)
p.add(_OP_drop) p.add(_OP_drop)
p.pin(n) p.pin(n)
p.pin(skip)
} }
func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) { func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Type) {
@ -933,7 +951,8 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ
p.add(_OP_lspace) p.add(_OP_lspace)
n0 := p.pc() n0 := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
p.chr(_OP_match_char, '"')
skip := self.checkIfSkip(p, stringType, '"')
/* also check for inner "null" */ /* also check for inner "null" */
n1 = p.pc() n1 = p.pc()
@ -997,6 +1016,7 @@ func (self *_Compiler) compileStructFieldStr(p *_Program, sp int, vt reflect.Typ
p.pin(n1) // `is_null_quote` jump location p.pin(n1) // `is_null_quote` jump location
p.add(_OP_nil_1) p.add(_OP_nil_1)
p.pin(pc) p.pin(pc)
p.pin(skip)
} }
func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) { func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) {
@ -1018,11 +1038,13 @@ func (self *_Compiler) compileInterface(p *_Program, vt reflect.Type) {
p.pin(j) p.pin(j)
} }
func (self *_Compiler) compilePrimitive(p *_Program, op _Op) { func (self *_Compiler) compilePrimitive(vt reflect.Type, p *_Program, op _Op) {
i := p.pc() i := p.pc()
p.add(_OP_is_null) p.add(_OP_is_null)
// skip := self.checkPrimitive(p, vt)
p.add(op) p.add(op)
p.pin(i) p.pin(i)
// p.pin(skip)
} }
func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) { func (self *_Compiler) compileUnmarshalEnd(p *_Program, vt reflect.Type, i int) {
@ -1081,3 +1103,14 @@ func (self *_Compiler) compileUnmarshalTextPtr(p *_Program, vt reflect.Type) {
p.rtt(_OP_unmarshal_text_p, vt) p.rtt(_OP_unmarshal_text_p, vt)
p.pin(i) p.pin(i)
} }
func (self *_Compiler) checkIfSkip(p *_Program, vt reflect.Type, c byte) int {
j := p.pc()
p.chr(_OP_check_char_0, c)
p.rtt(_OP_dismatch_err, vt)
s := p.pc()
p.add(_OP_go_skip)
p.pin(j)
p.int(_OP_add, 1)
return s
}

47
decoder/decoder_test.go
View file

@ -24,6 +24,7 @@ import (
`sync` `sync`
`testing` `testing`
`time` `time`
`reflect`
`github.com/bytedance/sonic/internal/rt` `github.com/bytedance/sonic/internal/rt`
`github.com/davecgh/go-spew/spew` `github.com/davecgh/go-spew/spew`
@ -85,6 +86,52 @@ func init() {
_ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
} }
func TestSkipError(t *testing.T) {
println("TestSkipError")
type skiptype struct {
A int `json:"a"`
B string `json:"b"`
Pass *int `json:"pass"`
C struct{
Pass4 interface{} `json:"pass4"`
D struct{
E float32 `json:"e"`
} `json:"d"`
Pass2 int `json:"pass2"`
} `json:"c"`
E bool `json:"e"`
F []int `json:"f"`
G map[string]int `json:"g"`
I json.Number `json:"i"`
Pass3 int `json:"pass2"`
}
var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}
var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"i":true,"pass3":1}`
d := NewDecoder(data)
err := d.Decode(obj)
// println("decoder out: ", err.Error())
err2 := json.Unmarshal([]byte(data), obj2)
assert.Equal(t, err2 == nil, err == nil)
// assert.Equal(t, len(data), d.i)
assert.Equal(t, obj2, obj)
if te, ok := err.(*MismatchTypeError); ok {
assert.Equal(t, reflect.TypeOf(obj.I), te.Type)
assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos)
println(err.Error())
} else {
t.Fatal("invalid error")
}
}
func TestDecodeCorrupt(t *testing.T) { func TestDecodeCorrupt(t *testing.T) {
var ds = []string{ var ds = []string{
`{,}`, `{,}`,

52
decoder/errors.go
View file

@ -40,6 +40,10 @@ func (self SyntaxError) Error() string {
} }
func (self SyntaxError) Description() string { func (self SyntaxError) Description() string {
return "Syntax error " + self.description()
}
func (self SyntaxError) description() string {
i := 16 i := 16
p := self.Pos - i p := self.Pos - i
q := self.Pos + i q := self.Pos + i
@ -72,7 +76,7 @@ func (self SyntaxError) Description() string {
/* compose the error description */ /* compose the error description */
return fmt.Sprintf( return fmt.Sprintf(
"Syntax error at index %d: %s\n\n\t%s\n\t%s^%s\n", "at index %d: %s\n\n\t%s\n\t%s^%s\n",
self.Pos, self.Pos,
self.Message(), self.Message(),
self.Src[p:q], self.Src[p:q],
@ -117,6 +121,52 @@ func error_type(vt *rt.GoType) error {
return &json.UnmarshalTypeError{Type: vt.Pack()} return &json.UnmarshalTypeError{Type: vt.Pack()}
} }
type MismatchTypeError struct {
Pos int
Src string
Type reflect.Type
}
func swithchJSONType (src string, pos int) string {
var val string
switch src[pos] {
case 'f': fallthrough
case 't': val = "bool"
case '"': val = "string"
case '{': val = "object"
case '[': val = "array"
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': val = "number"
}
return val
}
func (self MismatchTypeError) Error() string {
se := SyntaxError {
Pos : self.Pos,
Src : self.Src,
Code : types.ERR_MISMATCH,
}
return fmt.Sprintf("Mismatch type %s with value %s %q", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description())
}
func (self MismatchTypeError) Description() string {
se := SyntaxError {
Pos : self.Pos,
Src : self.Src,
Code : types.ERR_MISMATCH,
}
return fmt.Sprintf("Mismatch type %s with value %s %s", self.Type.String(), swithchJSONType(self.Src, self.Pos), se.description())
}
//go:nosplit
func error_mismatch(src string, pos int, vt *rt.GoType) error {
return &MismatchTypeError {
Pos : pos,
Src : src,
Type : vt.Pack(),
}
}
//go:nosplit //go:nosplit
func error_field(name string) error { func error_field(name string) error {
return errors.New("json: unknown field " + strconv.Quote(name)) return errors.New("json: unknown field " + strconv.Quote(name))

5
decoder/types.go
View file

@ -28,6 +28,11 @@ import (
var ( var (
byteType = reflect.TypeOf(byte(0)) byteType = reflect.TypeOf(byte(0))
intType = reflect.TypeOf(int(0))
uintType = reflect.TypeOf(uint(0))
floatType = reflect.TypeOf(float64(0))
stringType = reflect.TypeOf("")
bytesType = reflect.TypeOf([]byte(nil))
jsonNumberType = reflect.TypeOf(json.Number("")) jsonNumberType = reflect.TypeOf(json.Number(""))
base64CorruptInputError = reflect.TypeOf(base64.CorruptInputError(0)) base64CorruptInputError = reflect.TypeOf(base64.CorruptInputError(0))
) )

2
internal/native/types/types.go
View file

@ -74,6 +74,7 @@ const (
ERR_INVALID_NUMBER_FMT ParsingError = 6 ERR_INVALID_NUMBER_FMT ParsingError = 6
ERR_RECURSE_EXCEED_MAX ParsingError = 7 ERR_RECURSE_EXCEED_MAX ParsingError = 7
ERR_FLOAT_INFINITY ParsingError = 8 ERR_FLOAT_INFINITY ParsingError = 8
ERR_MISMATCH ParsingError = 9
) )
var _ParsingErrors = []string{ var _ParsingErrors = []string{
@ -86,6 +87,7 @@ var _ParsingErrors = []string{
ERR_INVALID_NUMBER_FMT : "invalid number format", ERR_INVALID_NUMBER_FMT : "invalid number format",
ERR_RECURSE_EXCEED_MAX : "recursion exceeded max depth", ERR_RECURSE_EXCEED_MAX : "recursion exceeded max depth",
ERR_FLOAT_INFINITY : "float number is infinity", ERR_FLOAT_INFINITY : "float number is infinity",
ERR_MISMATCH : "mismatched type with value",
} }
func (self ParsingError) Error() string { func (self ParsingError) Error() string {