XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-23 01:46:44 +08:00
Code Issues Projects Releases Wiki Activity

feat: add compatible API (#233)

Browse source 
* feat: add compatible API

* test: add api compatibility test

* fmt

* test: adjust CI

* fix: no trailing '\n' for std `Marshal()`

* doc: update README.md

* test: add windows CI

* doc: update README.md

* doc: update README.md

* fmt: add comments and refator
This commit is contained in:
Yi Duan 2022-05-25 14:16:56 +08:00 committed by GitHub
parent 8917a0a7d9
commit ced28302a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 3608 additions and 2970 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.github/workflows/benchmark.yml → .github/workflows/benchmark-linux-amd64.yml vendored
View file

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -55,7 +55,7 @@ jobs:
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v1 uses: github/codeql-action/init@v2
with: with:
languages: ${{ matrix.language }} languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file. # If you wish to specify custom queries, you can do so here or in a config file.

7
.github/workflows/push-check-linux.yml → .github/workflows/push-check-all.yml vendored
View file

@ -1,4 +1,4 @@
name: Push Check Linux name: Push Check All
on: push on: push
@ -7,7 +7,8 @@ jobs:
strategy: strategy:
matrix: matrix:
go-version: [1.15.x, 1.16.x, 1.17.x] go-version: [1.15.x, 1.16.x, 1.17.x]
runs-on: self-hosted os: [self-hosted]
runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -24,4 +25,4 @@ jobs:
${{ runner.os }}-go- ${{ runner.os }}-go-
- name: Unit Test - name: Unit Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./... run: go test -v -gcflags=-d=checkptr=0 ./...

27
.github/workflows/push-check-arm.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Push Check ARM
on: push
jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.18.x]
runs-on: [self-hosted, arm]
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Compatibility Test
run: go test -v -gcflags=-d=checkptr=0 .

2
.github/workflows/push-check-linux-go118.yml → .github/workflows/push-check-go118.yml vendored
View file

@ -24,4 +24,4 @@ jobs:
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./... run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
- name: Generic Test - name: Generic Test
run: go test -v -gcflags=-d=checkptr=0 -race ./generic_test run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test

43
.github/workflows/push-check-windows.yml vendored
View file

@ -1,26 +1,27 @@
# name: Push Check Go Windows name: Push Check Windows
# on: push on: push
# jobs: jobs:
# build: build:
# strategy: strategy:
# matrix: matrix:
# go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x] go-version: [1.15.x, 1.18.x]
# os: [windows-latest] runs-on: windows-latest
# runs-on: ${{ matrix.os }} steps:
# steps: - uses: actions/checkout@v2
# - uses: actions/checkout@v2
# - name: Set up Go - name: Set up Go
# uses: actions/setup-go@v3 uses: actions/setup-go@v2
# with: with:
# go-version: ${{ matrix.go-version }} go-version: ${{ matrix.go-version }}
# - name: Unit Test - uses: actions/cache@v2
# run: go test -v -gcflags -d=checkptr=0 -covermode atomic -coverprofile coverage.out ./... with:
# env: path: ~/go/pkg/mod
# GOMAXPROCS: 4 key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
# - name: Generic Test - name: Compatibility Test
# run: go test -v -gcflags -d=checkptr=0 -covermode atomic ./generic_test run: go test -v -gcflags=-d=checkptr=0 .

11
README.md
View file

@ -4,7 +4,7 @@ A blazingly fast JSON serializing & deserializing library, accelerated by JI
## Requirement ## Requirement
- Go 1.15/1.16/1.17/1.18 - Go 1.15/1.16/1.17/1.18
- Linux/darwin OS - Linux/MacOS/Windows
- Amd64 CPU with AVX instruction set - Amd64 CPU with AVX instruction set
## Features ## Features
@ -260,6 +260,12 @@ println(string(buf) == string(exp)) // true
- iteration: `Values()`, `Properties()`, `ForEach()`, `SortKeys()` - iteration: `Values()`, `Properties()`, `ForEach()`, `SortKeys()`
- modification: `Set()`, `SetByIndex()`, `Add()` - modification: `Set()`, `SetByIndex()`, `Add()`
## Compatibility
Sonic **DOSE NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. For developers who use sonic to build their applications in different environments (ex: developing on M1 Mac but running on linux server), or those who want to handle JSON strictly consistent with `encoding/json`, we provide some compatible APIs as `sonic.API`
- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run on sonic-supporting environment. It will fall back to `encoding/json` with corresponding config , and some options like `SortKeys=false` will be invalid.
- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) to run on sonic-supporting environment. It whill fall back to `encoding/json`.
- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic-supporting environment. It will fall back to `encoding/json` with corresponding config , and some options will be invalid.
## Tips ## Tips
### Pretouch ### Pretouch
@ -317,3 +323,6 @@ Why? Because `ast.Node` stores its children using `array`:
- Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**. - Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**.
**CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, your can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}` **CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, your can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}`
## Community
Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem.

127
api.go Normal file
View file

@ -0,0 +1,127 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`io`
)
// Config is a combination of sonic/encoder.Options and sonic/decoder.Options
type Config struct {
EscapeHTML bool
SortMapKeys bool
CompactMarshaler bool
NoQuoteTextMarshaler bool
UseInt64 bool
UseNumber bool
UseUnicodeErrors bool
DisallowUnknownFields bool
CopyString bool
}
var (
// ConfigDefault is the default config of APIs, aiming at efficiency and safty.
ConfigDefault = Config{}.Froze()
// ConfigStd is the standard config of APIs, aiming at being compatible with encoding/json.
ConfigStd = Config{
EscapeHTML : true,
SortMapKeys: true,
CompactMarshaler: true,
CopyString : true,
}.Froze()
// ConfigFastest is the fastest config of APIs, aiming at speed.
ConfigFastest = Config{
NoQuoteTextMarshaler: true,
}.Froze()
)
// API a binding of specific config.
// This interface is inspired by github.com/json-iterator/go,
// and has same behaviors under equavilent config.
type API interface {
// MarshalToString returns the JSON encoding string of v
MarshalToString(v interface{}) (string, error)
// Marshal returns the JSON encoding bytes of v.
Marshal(v interface{}) ([]byte, error)
// MarshalIndent returns the JSON encoding bytes with indent and prefix.
MarshalIndent(v interface{}, prefix, indent string) ([]byte, error)
// UnmarshalFromString parses the JSON-encoded bytes and stores the result in the value pointed to by v.
UnmarshalFromString(str string, v interface{}) error
// Unmarshal parses the JSON-encoded string and stores the result in the value pointed to by v.
Unmarshal(data []byte, v interface{}) error
// NewEncoder create a Encoder holding writer
NewEncoder(writer io.Writer) Encoder
// NewDecoder create a Decoder holding reader
NewDecoder(reader io.Reader) Decoder
// Valid validates the JSON-encoded bytes and reportes if it is valid
Valid(data []byte) bool
}
// Encoder encodes JSON into io.Writer
type Encoder interface {
// Encode writes the JSON encoding of v to the stream, followed by a newline character.
Encode(val interface{}) error
// SetEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior NOT ESCAPE
SetEscapeHTML(on bool)
// SetIndent instructs the encoder to format each subsequent encoded value
// as if indented by the package-level function Indent(dst, src, prefix, indent).
// Calling SetIndent("", "") disables indentation
SetIndent(prefix, indent string)
}
// Decoder decodes JSON from io.Read
type Decoder interface {
// Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v.
Decode(val interface{}) error
// Buffered returns a reader of the data remaining in the Decoder's buffer.
// The reader is valid until the next call to Decode.
Buffered() io.Reader
// DisallowUnknownFields causes the Decoder to return an error when the destination is a struct
// and the input contains object keys which do not match any non-ignored, exported fields in the destination.
DisallowUnknownFields()
// More reports whether there is another element in the current array or object being parsed.
More() bool
// UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
UseNumber()
}
// Marshal returns the JSON encoding bytes of v.
func Marshal(val interface{}) ([]byte, error) {
return ConfigDefault.Marshal(val)
}
// MarshalString returns the JSON encoding string of v.
func MarshalString(val interface{}) (string, error) {
return ConfigDefault.MarshalToString(val)
}
// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.
// NOTICE: This API copies given buffer by default,
// if you want to pass JSON more efficiently, use UnmarshalString instead.
func Unmarshal(buf []byte, val interface{}) error {
return ConfigDefault.Unmarshal(buf, val)
}
// UnmarshalString is like Unmarshal, except buf is a string.
func UnmarshalString(buf string, val interface{}) error {
return ConfigDefault.UnmarshalFromString(buf, val)
}

262
api_test.go Normal file
View file

@ -0,0 +1,262 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/json`
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
func TestCompatMarshalDefault(t *testing.T){
var obj = map[string]interface{}{
"c": json.RawMessage("[\"<&>\"]"),
}
sout, serr := ConfigDefault.Marshal(obj)
jout, jerr := jt.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
// obj = map[string]interface{}{
// "a": json.RawMessage(" [} "),
// }
// sout, serr = ConfigDefault.Marshal(obj)
// jout, jerr = json.Marshal(obj)
// require.NotNil(t, jerr)
// require.NotNil(t, serr)
// require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigDefault.MarshalIndent(obj, "", " ")
jout, jerr = jt.MarshalIndent(obj, "", " ")
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
}
func TestCompatMarshalStd(t *testing.T) {
t.Parallel()
var obj = map[string]interface{}{
"c": json.RawMessage(" [ \"<&>\" ] "),
"b": json.RawMessage(" [ ] "),
}
sout, serr := ConfigStd.Marshal(obj)
jout, jerr := json.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage(" [} "),
}
sout, serr = ConfigStd.Marshal(obj)
jout, jerr = json.Marshal(obj)
require.NotNil(t, jerr)
require.NotNil(t, serr)
require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ")
jout, jerr = json.MarshalIndent(obj, "xxxx", " ")
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
}
func TestCompatUnmarshalDefault(t *testing.T) {
var sobj = map[string]interface{}{}
var jobj = map[string]interface{}{}
var data = []byte(`{"a":-0}`)
var str = string(data)
serr := ConfigDefault.UnmarshalFromString(str, &sobj)
jerr := jt.UnmarshalFromString(str, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
serr = ConfigDefault.Unmarshal(data, &x)
jerr = jt.Unmarshal(data, &y)
require.Equal(t, jerr, serr)
require.Equal(t, y, x)
}
func TestCompatUnmarshalStd(t *testing.T) {
var sobj = map[string]interface{}{}
var jobj = map[string]interface{}{}
var data = []byte(`{"a":1.00000001E-10}`)
var str = string(data)
serr := ConfigStd.UnmarshalFromString(str, &sobj)
jerr := json.Unmarshal(data, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
data[2] = '0'
require.Equal(t, jobj, sobj)
sobj = map[string]interface{}{}
jobj = map[string]interface{}{}
data = []byte(`{"a":1}`)
cfg := Config{
UseNumber: true,
}.Froze()
serr = cfg.Unmarshal(data, &sobj)
dec := json.NewDecoder(bytes.NewBuffer(data))
dec.UseNumber()
jerr = dec.Decode(&jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
cfg = Config{
DisallowUnknownFields: true,
}.Froze()
serr = cfg.Unmarshal(data, &x)
dec = json.NewDecoder(bytes.NewBuffer(data))
dec.UseNumber()
dec.DisallowUnknownFields()
jerr = dec.Decode(&y)
require.Equal(t, jerr, serr)
require.Equal(t, y, x)
}
func TestCompatEncoderDefault(t *testing.T) {
var o = map[string]interface{}{
"a": "<>",
// "b": json.RawMessage(" [ ] "),
}
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = jt.NewEncoder(w1)
var enc2 = ConfigDefault.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(true)
enc2.SetEscapeHTML(true)
enc1.SetIndent("", " ")
enc2.SetIndent("", " ")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
enc2.SetIndent("", "")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatEncoderStd(t *testing.T) {
var o = map[string]interface{}{
"a": "<>",
"b": json.RawMessage(" [ ] "),
}
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = json.NewEncoder(w1)
var enc2 = ConfigStd.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(true)
enc2.SetEscapeHTML(true)
enc1.SetIndent("\n", " ")
enc2.SetIndent("\n", " ")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
enc2.SetIndent("", "")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatDecoderStd(t *testing.T) {
var o1 = map[string]interface{}{}
var o2 = map[string]interface{}{}
var s = `{"a":"b"} {"1":"2"} a {}`
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = json.NewDecoder(w1)
var enc2 = ConfigStd.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatDecoderDefault(t *testing.T) {
var o1 = map[string]interface{}{}
var o2 = map[string]interface{}{}
var s = `{"a":"b"} {"1":"2"} a {}`
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = jt.NewDecoder(w1)
var enc2 = ConfigDefault.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
// require.Equal(t, enc1.More(), enc2.More())
// require.NotNil(t, enc1.Decode(&o1))
// require.NotNil(t, enc2.Decode(&o2))
// require.Equal(t, w1.String(), w2.String())
}

117
compat.go Normal file
View file

@ -0,0 +1,117 @@
// +build !amd64
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/json`
`io`
)
type frozenConfig struct {
Config
}
// Froze convert the Config to API
func (cfg Config) Froze() API {
api := &frozenConfig{Config: cfg}
return api
}
func (cfg *frozenConfig) marshalOptions(val interface{}, prefix, indent string) ([]byte, error) {
w := bytes.NewBuffer([]byte{})
enc := json.NewEncoder(w)
enc.SetEscapeHTML(cfg.EscapeHTML)
enc.SetIndent(prefix, indent)
err := enc.Encode(val)
out := w.Bytes()
// json.Encoder always appends '\n' after encoding,
// which is not same with json.Marshal()
if len(out) > 0 && out[len(out)-1] == '\n' {
out = out[:len(out)-1]
}
return out, err
}
// Marshal is implemented by sonic
func (cfg *frozenConfig) Marshal(val interface{}) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, "", "")
}
return json.Marshal(val)
}
// MarshalToString is implemented by sonic
func (cfg *frozenConfig) MarshalToString(val interface{}) (string, error) {
out, err := cfg.Marshal(val)
return string(out), err
}
// MarshalIndent is implemented by sonic
func (cfg *frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, prefix, indent)
}
return json.MarshalIndent(val, prefix, indent)
}
// UnmarshalFromString is implemented by sonic
func (cfg *frozenConfig) UnmarshalFromString(buf string, val interface{}) error {
r := bytes.NewBufferString(buf)
dec := json.NewDecoder(r)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec.Decode(val)
}
// Unmarshal is implemented by sonic
func (cfg *frozenConfig) Unmarshal(buf []byte, val interface{}) error {
return cfg.UnmarshalFromString(string(buf), val)
}
// NewEncoder is implemented by sonic
func (cfg *frozenConfig) NewEncoder(writer io.Writer) Encoder {
enc := json.NewEncoder(writer)
if !cfg.EscapeHTML {
enc.SetEscapeHTML(cfg.EscapeHTML)
}
return enc
}
// NewDecoder is implemented by sonic
func (cfg *frozenConfig) NewDecoder(reader io.Reader) Decoder {
dec := json.NewDecoder(reader)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec
}
// Valid is implemented by sonic
func (cfg *frozenConfig) Valid(data []byte) bool {
return json.Valid(data)
}

234
decode_float_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/* /*
* Copyright 2021 ByteDance Inc. * Copyright 2021 ByteDance Inc.
* *
@ -17,146 +19,146 @@
package sonic package sonic
import ( import (
"encoding/json" `encoding/json`
"reflect" `reflect`
"strings" `strings`
"testing" `testing`
"github.com/bytedance/sonic/decoder" `github.com/bytedance/sonic/decoder`
) )
type atofTest struct { type atofTest struct {
in string in string
out string out string
err error err error
} }
// Tests from Go strconv package, https://github.com/golang/go/blob/master/src/strconv/atof_test.go // Tests from Go strconv package, https://github.com/golang/go/blob/master/src/strconv/atof_test.go
// All tests are passed in Go encoding/json. // All tests are passed in Go encoding/json.
var atoftests = []atofTest{ var atoftests = []atofTest{
{"1.234e", "", nil}, // error {"1.234e", "", nil}, // error
{"1i", "1", nil}, // pass {"1i", "1", nil}, // pass
{"1", "1", nil}, {"1", "1", nil},
{"1e23", "1e+23", nil}, {"1e23", "1e+23", nil},
{"1E23", "1e+23", nil}, {"1E23", "1e+23", nil},
{"100000000000000000000000", "1e+23", nil}, {"100000000000000000000000", "1e+23", nil},
{"1e-100", "1e-100", nil}, {"1e-100", "1e-100", nil},
{"123456700", "1.234567e+08", nil}, {"123456700", "1.234567e+08", nil},
{"99999999999999974834176", "9.999999999999997e+22", nil}, {"99999999999999974834176", "9.999999999999997e+22", nil},
{"100000000000000000000001", "1.0000000000000001e+23", nil}, {"100000000000000000000001", "1.0000000000000001e+23", nil},
{"100000000000000008388608", "1.0000000000000001e+23", nil}, {"100000000000000008388608", "1.0000000000000001e+23", nil},
{"100000000000000016777215", "1.0000000000000001e+23", nil}, {"100000000000000016777215", "1.0000000000000001e+23", nil},
{"100000000000000016777216", "1.0000000000000003e+23", nil}, {"100000000000000016777216", "1.0000000000000003e+23", nil},
{"-1", "-1", nil}, {"-1", "-1", nil},
{"-0.1", "-0.1", nil}, {"-0.1", "-0.1", nil},
{"-0", "-0", nil}, {"-0", "-0", nil},
{"1e-20", "1e-20", nil}, {"1e-20", "1e-20", nil},
{"625e-3", "0.625", nil}, {"625e-3", "0.625", nil},
// zeros // zeros
{"0", "0", nil}, {"0", "0", nil},
{"0e0", "0", nil}, {"0e0", "0", nil},
{"-0e0", "-0", nil}, {"-0e0", "-0", nil},
{"0e-0", "0", nil}, {"0e-0", "0", nil},
{"-0e-0", "-0", nil}, {"-0e-0", "-0", nil},
{"0e+0", "0", nil}, {"0e+0", "0", nil},
{"-0e+0", "-0", nil}, {"-0e+0", "-0", nil},
{"0e+01234567890123456789", "0", nil}, {"0e+01234567890123456789", "0", nil},
{"0.00e-01234567890123456789", "0", nil}, {"0.00e-01234567890123456789", "0", nil},
{"-0e+01234567890123456789", "-0", nil}, {"-0e+01234567890123456789", "-0", nil},
{"-0.00e-01234567890123456789", "-0", nil}, {"-0.00e-01234567890123456789", "-0", nil},
{"0e291", "0", nil}, // issue 15364 {"0e291", "0", nil}, // issue 15364
{"0e292", "0", nil}, // issue 15364 {"0e292", "0", nil}, // issue 15364
{"0e347", "0", nil}, // issue 15364 {"0e347", "0", nil}, // issue 15364
{"0e348", "0", nil}, // issue 15364 {"0e348", "0", nil}, // issue 15364
{"-0e291", "-0", nil}, {"-0e291", "-0", nil},
{"-0e292", "-0", nil}, {"-0e292", "-0", nil},
{"-0e347", "-0", nil}, {"-0e347", "-0", nil},
{"-0e348", "-0", nil}, {"-0e348", "-0", nil},
// largest float64 // largest float64
{"1.7976931348623157e308", "1.7976931348623157e+308", nil}, {"1.7976931348623157e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623157e308", "-1.7976931348623157e+308", nil}, {"-1.7976931348623157e308", "-1.7976931348623157e+308", nil},
// the border is ...158079 // the border is ...158079
// borderline - okay // borderline - okay
{"1.7976931348623158e308", "1.7976931348623157e+308", nil}, {"1.7976931348623158e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623158e308", "-1.7976931348623157e+308", nil}, {"-1.7976931348623158e308", "-1.7976931348623157e+308", nil},
// a little too large // a little too large
{"1e308", "1e+308", nil}, {"1e308", "1e+308", nil},
// denormalized // denormalized
{"1e-305", "1e-305", nil}, {"1e-305", "1e-305", nil},
{"1e-306", "1e-306", nil}, {"1e-306", "1e-306", nil},
{"1e-307", "1e-307", nil}, {"1e-307", "1e-307", nil},
{"1e-308", "1e-308", nil}, {"1e-308", "1e-308", nil},
{"1e-309", "1e-309", nil}, {"1e-309", "1e-309", nil},
{"1e-310", "1e-310", nil}, {"1e-310", "1e-310", nil},
{"1e-322", "1e-322", nil}, {"1e-322", "1e-322", nil},
// smallest denormal // smallest denormal
{"5e-324", "5e-324", nil}, {"5e-324", "5e-324", nil},
{"4e-324", "5e-324", nil}, {"4e-324", "5e-324", nil},
{"3e-324", "5e-324", nil}, {"3e-324", "5e-324", nil},
// too small // too small
{"2e-324", "0", nil}, {"2e-324", "0", nil},
// way too small // way too small
{"1e-350", "0", nil}, {"1e-350", "0", nil},
{"1e-400000", "0", nil}, {"1e-400000", "0", nil},
// try to overflow exponent // try to overflow exponent
{"1e-4294967296", "0", nil}, {"1e-4294967296", "0", nil},
{"1e-18446744073709551616", "0", nil}, {"1e-18446744073709551616", "0", nil},
// https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ // https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
{"2.2250738585072012e-308", "2.2250738585072014e-308", nil}, {"2.2250738585072012e-308", "2.2250738585072014e-308", nil},
// https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/ // https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
{"2.2250738585072011e-308", "2.225073858507201e-308", nil}, {"2.2250738585072011e-308", "2.225073858507201e-308", nil},
// A very large number (initially wrongly parsed by the fast algorithm). // A very large number (initially wrongly parsed by the fast algorithm).
{"4.630813248087435e+307", "4.630813248087435e+307", nil}, {"4.630813248087435e+307", "4.630813248087435e+307", nil},
// A different kind of very large number. // A different kind of very large number.
{"22.222222222222222", "22.22222222222222", nil}, {"22.222222222222222", "22.22222222222222", nil},
{"2." + strings.Repeat("2", 800) + "e+1", "22.22222222222222", nil}, {"2." + strings.Repeat("2", 800) + "e+1", "22.22222222222222", nil},
// Exactly halfway between 1 and math.Nextafter(1, 2). // Exactly halfway between 1 and math.Nextafter(1, 2).
// Round to even (down). // Round to even (down).
{"1.00000000000000011102230246251565404236316680908203125", "1", nil}, {"1.00000000000000011102230246251565404236316680908203125", "1", nil},
// Slightly lower; still round down. // Slightly lower; still round down.
{"1.00000000000000011102230246251565404236316680908203124", "1", nil}, {"1.00000000000000011102230246251565404236316680908203124", "1", nil},
// Slightly higher; round up. // Slightly higher; round up.
{"1.00000000000000011102230246251565404236316680908203126", "1.0000000000000002", nil}, {"1.00000000000000011102230246251565404236316680908203126", "1.0000000000000002", nil},
// Slightly higher, but you have to read all the way to the end. // Slightly higher, but you have to read all the way to the end.
{"1.00000000000000011102230246251565404236316680908203125" + strings.Repeat("0", 10000) + "1", "1.0000000000000002", nil}, {"1.00000000000000011102230246251565404236316680908203125" + strings.Repeat("0", 10000) + "1", "1.0000000000000002", nil},
// Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2) // Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2)
// Round to even (up). // Round to even (up).
{"1.00000000000000033306690738754696212708950042724609375", "1.0000000000000004", nil}, {"1.00000000000000033306690738754696212708950042724609375", "1.0000000000000004", nil},
// Halfway between 1090544144181609278303144771584 and 1090544144181609419040633126912 // Halfway between 1090544144181609278303144771584 and 1090544144181609419040633126912
// (15497564393479157p+46, should round to even 15497564393479156p+46, issue 36657) // (15497564393479157p+46, should round to even 15497564393479156p+46, issue 36657)
{"1090544144181609348671888949248", "1.0905441441816093e+30", nil}, {"1090544144181609348671888949248", "1.0905441441816093e+30", nil},
// Corner case between int64 and float64 for the input // Corner case between int64 and float64 for the input
{"9223372036854775807", "9223372036854775807", nil}, // max int64: (1 << 63) - 1 {"9223372036854775807", "9223372036854775807", nil}, // max int64: (1 << 63) - 1
{"9223372036854775808", "9223372036854775808", nil}, {"9223372036854775808", "9223372036854775808", nil},
{"-9223372036854775808", "-9223372036854775808", nil}, // min int64: 1 << 63 {"-9223372036854775808", "-9223372036854775808", nil}, // min int64: 1 << 63
{"-9223372036854775809", "-9223372036854775809", nil}, {"-9223372036854775809", "-9223372036854775809", nil},
} }
func TestDecodeFloat(t *testing.T) { func TestDecodeFloat(t *testing.T) {
for i, tt := range atoftests { for i, tt := range atoftests {
// default float64 // default float64
var sonicout, stdout float64 var sonicout, stdout float64
sonicerr := decoder.NewDecoder(tt.in).Decode(&sonicout) sonicerr := decoder.NewDecoder(tt.in).Decode(&sonicout)
stderr := json.NewDecoder(strings.NewReader(tt.in)).Decode(&stdout) stderr := json.NewDecoder(strings.NewReader(tt.in)).Decode(&stdout)
if !reflect.DeepEqual(sonicout, stdout) { if !reflect.DeepEqual(sonicout, stdout) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicout, stdout) t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicout, stdout)
} }
if !reflect.DeepEqual(sonicerr == nil, stderr == nil) { if !reflect.DeepEqual(sonicerr == nil, stderr == nil) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicerr, stderr) t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicerr, stderr)
} }
} }
} }

3768
decode_test.go
View file

File diff suppressed because it is too large Load diff

18
decoder/decoder.go
View file

@ -37,6 +37,24 @@ const (
_F_allow_control = 31 _F_allow_control = 31
) )
type Options uint64
const (
OptionUseInt64 Options = 1 << _F_use_int64
OptionUseNumber Options = 1 << _F_use_number
OptionUseUnicodeErrors Options = 1 << _F_disable_urc
OptionDisableUnknown Options = 1 << _F_disable_unknown
OptionCopyString Options = 1 << _F_copy_string
)
func (self *Decoder) SetOptions(opts Options) {
if (opts & 1<<_F_use_number != 0) && (opts & 1<<_F_use_int64 != 0) {
panic("can't set OptionUseInt64 and OptionUseNumber both!")
}
self.f = uint64(opts)
}
// Decoder is the decoder context object // Decoder is the decoder context object
type Decoder struct { type Decoder struct {
i int i int

1682
encode_test.go
View file

File diff suppressed because it is too large Load diff

2
search_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/* /*
* Copyright 2021 ByteDance Inc. * Copyright 2021 ByteDance Inc.
* *

128
sonic.go
View file

@ -1,3 +1,5 @@
// +build amd64
/* /*
* Copyright 2021 ByteDance Inc. * Copyright 2021 ByteDance Inc.
* *
@ -18,6 +20,7 @@
package sonic package sonic
import ( import (
`io`
`reflect` `reflect`
`github.com/bytedance/sonic/ast` `github.com/bytedance/sonic/ast`
@ -28,36 +31,7 @@ import (
`github.com/bytedance/sonic/internal/rt` `github.com/bytedance/sonic/internal/rt`
) )
// Marshal returns the JSON encoding bytes of v. func checkTrailings(buf string, pos int) error {
func Marshal(val interface{}) ([]byte, error) {
return encoder.Encode(val, 0)
}
// MarshalString returns the JSON encoding string of v.
func MarshalString(val interface{}) (string, error) {
buf, err := encoder.Encode(val, 0)
return rt.Mem2Str(buf), err
}
// Unmarshal parses the JSON-encoded data and stores the result in the value
// pointed to by v.
// NOTICE: This API copies given buffer by default,
// if you want to pass JSON more efficiently, use UnmarshalString instead.
func Unmarshal(buf []byte, val interface{}) error {
return UnmarshalString(string(buf), val)
}
// UnmarshalString is like Unmarshal, except buf is a string.
func UnmarshalString(buf string, val interface{}) error {
dec := decoder.NewDecoder(buf)
err := dec.Decode(val)
pos := dec.Pos()
/* check for errors */
if err != nil {
return err
}
/* skip all the trailing spaces */ /* skip all the trailing spaces */
if pos != len(buf) { if pos != len(buf) {
for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 { for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 {
@ -73,11 +47,103 @@ func UnmarshalString(buf string, val interface{}) error {
/* junk after JSON value */ /* junk after JSON value */
return decoder.SyntaxError { return decoder.SyntaxError {
Src : buf, Src : buf,
Pos : dec.Pos(), Pos : pos,
Code : types.ERR_INVALID_CHAR, Code : types.ERR_INVALID_CHAR,
} }
} }
type frozenConfig struct {
Config
encoderOpts encoder.Options
decoderOpts decoder.Options
}
// Froze convert the Config to API
func (cfg Config) Froze() API {
api := &frozenConfig{Config: cfg}
if cfg.EscapeHTML {
api.encoderOpts |= encoder.EscapeHTML
}
if cfg.SortMapKeys {
api.encoderOpts |= encoder.SortMapKeys
}
if cfg.CompactMarshaler {
api.encoderOpts |= encoder.CompactMarshaler
}
if cfg.NoQuoteTextMarshaler {
api.encoderOpts |= encoder.NoQuoteTextMarshaler
}
if cfg.UseInt64 {
api.decoderOpts |= decoder.OptionUseInt64
}
if cfg.UseNumber {
api.decoderOpts |= decoder.OptionUseNumber
}
if cfg.DisallowUnknownFields {
api.decoderOpts |= decoder.OptionDisableUnknown
}
if cfg.CopyString {
api.decoderOpts |= decoder.OptionCopyString
}
return api
}
// Marshal is implemented by sonic
func (cfg *frozenConfig) Marshal(val interface{}) ([]byte, error) {
return encoder.Encode(val, cfg.encoderOpts)
}
// MarshalToString is implemented by sonic
func (cfg *frozenConfig) MarshalToString(val interface{}) (string, error) {
buf, err := encoder.Encode(val, cfg.encoderOpts)
return rt.Mem2Str(buf), err
}
// MarshalIndent is implemented by sonic
func (cfg *frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {
return encoder.EncodeIndented(val, prefix, indent, cfg.encoderOpts)
}
// UnmarshalFromString is implemented by sonic
func (cfg *frozenConfig) UnmarshalFromString(buf string, val interface{}) error {
dec := decoder.NewDecoder(buf)
dec.SetOptions(cfg.decoderOpts)
err := dec.Decode(val)
pos := dec.Pos()
/* check for errors */
if err != nil {
return err
}
return checkTrailings(buf, pos)
}
// Unmarshal is implemented by sonic
func (cfg *frozenConfig) Unmarshal(buf []byte, val interface{}) error {
return cfg.UnmarshalFromString(string(buf), val)
}
// NewEncoder is implemented by sonic
func (cfg *frozenConfig) NewEncoder(writer io.Writer) Encoder {
enc := encoder.NewStreamEncoder(writer)
enc.Opts = cfg.encoderOpts
return enc
}
// NewDecoder is implemented by sonic
func (cfg *frozenConfig) NewDecoder(reader io.Reader) Decoder {
dec := decoder.NewStreamDecoder(reader)
dec.SetOptions(cfg.decoderOpts)
return dec
}
// Valid is implemented by sonic
func (cfg *frozenConfig) Valid(data []byte) bool {
ok, _ := encoder.Valid(data)
return ok
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in // Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency. // order to reduce the first-hit latency.
// //

146
use_number_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/* /*
* Copyright 2021 ByteDance Inc. * Copyright 2021 ByteDance Inc.
* *
@ -17,98 +19,98 @@
package sonic package sonic
import ( import (
"encoding/json" `encoding/json`
"reflect" `reflect`
"testing" `testing`
"github.com/bytedance/sonic/decoder" `github.com/bytedance/sonic/decoder`
) )
type useInt64Test struct { type useInt64Test struct {
in string in string
out int64 out int64
} }
type useFloatTest struct { type useFloatTest struct {
in string in string
out float64 out float64
} }
var useinttest = []useInt64Test{ var useinttest = []useInt64Test{
// int64 // int64
{"0", 0}, {"0", 0},
{"1", 1}, {"1", 1},
{"-1", -1}, {"-1", -1},
{"100", 100}, {"100", 100},
{"-9223372036854775807", -9223372036854775807}, {"-9223372036854775807", -9223372036854775807},
{"-9223372036854775808", -9223372036854775808}, //min int64 {"-9223372036854775808", -9223372036854775808}, //min int64
{"9223372036854775807", 9223372036854775807}, //max int64 {"9223372036854775807", 9223372036854775807}, //max int64
{"9223372036854775806", 9223372036854775806}, {"9223372036854775806", 9223372036854775806},
} }
var usefloattest = []useFloatTest{ var usefloattest = []useFloatTest{
// float64 // float64
{"-9223372036854775809", -9223372036854775809}, // int64 overflow {"-9223372036854775809", -9223372036854775809}, // int64 overflow
{"9223372036854775808", 9223372036854775808}, // int64 overflow {"9223372036854775808", 9223372036854775808}, // int64 overflow
{"1e2", 1e2}, {"1e2", 1e2},
{"1e-20", 1e-20}, {"1e-20", 1e-20},
{"1.0", 1}, {"1.0", 1},
} }
func TestUseInt64(t *testing.T) { func TestUseInt64(t *testing.T) {
for i, tt := range useinttest { for i, tt := range useinttest {
var sout interface{} var sout interface{}
dc := decoder.NewDecoder(tt.in) dc := decoder.NewDecoder(tt.in)
dc.UseInt64() dc.UseInt64()
serr := dc.Decode(&sout) serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) { if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
} }
if serr != nil { if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
} }
} }
for i, tt := range usefloattest { for i, tt := range usefloattest {
var sout interface{} var sout interface{}
dc := decoder.NewDecoder(tt.in) dc := decoder.NewDecoder(tt.in)
dc.UseInt64() dc.UseInt64()
//the input string is not int64, still return float64 //the input string is not int64, still return float64
serr := dc.Decode(&sout) serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) { if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
} }
if serr != nil { if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
} }
} }
} }
func TestUseNumber(t *testing.T) { func TestUseNumber(t *testing.T) {
for i, tt := range useinttest { for i, tt := range useinttest {
var sout interface{} var sout interface{}
dc := decoder.NewDecoder(tt.in) dc := decoder.NewDecoder(tt.in)
dc.UseNumber() dc.UseNumber()
serr := dc.Decode(&sout) serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) { if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
} }
if serr != nil { if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
} }
} }
for i, tt := range usefloattest { for i, tt := range usefloattest {
var sout interface{} var sout interface{}
dc := decoder.NewDecoder(tt.in) dc := decoder.NewDecoder(tt.in)
dc.UseNumber() dc.UseNumber()
serr := dc.Decode(&sout) serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) { if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
} }
if serr != nil { if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr) t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
} }
} }
} }