XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity

feat: add compatible API (#233)

Browse source 
* feat: add compatible API

* test: add api compatibility test

* fmt

* test: adjust CI

* fix: no trailing '\n' for std `Marshal()`

* doc: update README.md

* test: add windows CI

* doc: update README.md

* doc: update README.md

* fmt: add comments and refator
This commit is contained in:
Yi Duan 2022-05-25 14:16:56 +08:00 committed by GitHub
parent 8917a0a7d9
commit ced28302a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 3608 additions and 2970 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.github/workflows/benchmark.yml → .github/workflows/benchmark-linux-amd64.yml vendored
View file

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -55,7 +55,7 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.

7
.github/workflows/push-check-linux.yml → .github/workflows/push-check-all.yml vendored
View file

@ -1,4 +1,4 @@
name: Push Check Linux
name: Push Check All
on: push
@ -7,7 +7,8 @@ jobs:
strategy:
matrix:
go-version: [1.15.x, 1.16.x, 1.17.x]
runs-on: self-hosted
os: [self-hosted]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
@ -24,4 +25,4 @@ jobs:
${{ runner.os }}-go-
- name: Unit Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./...
run: go test -v -gcflags=-d=checkptr=0 ./...

27
.github/workflows/push-check-arm.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Push Check ARM
on: push
jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.18.x]
runs-on: [self-hosted, arm]
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Compatibility Test
run: go test -v -gcflags=-d=checkptr=0 .

2
.github/workflows/push-check-linux-go118.yml → .github/workflows/push-check-go118.yml vendored
View file

@ -24,4 +24,4 @@ jobs:
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
- name: Generic Test
run: go test -v -gcflags=-d=checkptr=0 -race ./generic_test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test

43
.github/workflows/push-check-windows.yml vendored
View file

@ -1,26 +1,27 @@
# name: Push Check Go Windows
name: Push Check Windows
# on: push
on: push
# jobs:
# build:
# strategy:
# matrix:
# go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x]
# os: [windows-latest]
# runs-on: ${{ matrix.os }}
# steps:
# - uses: actions/checkout@v2
jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.18.x]
runs-on: windows-latest
steps:
- uses: actions/checkout@v2
# - name: Set up Go
# uses: actions/setup-go@v3
# with:
# go-version: ${{ matrix.go-version }}
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
# - name: Unit Test
# run: go test -v -gcflags -d=checkptr=0 -covermode atomic -coverprofile coverage.out ./...
# env:
# GOMAXPROCS: 4
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
# - name: Generic Test
# run: go test -v -gcflags -d=checkptr=0 -covermode atomic ./generic_test
- name: Compatibility Test
run: go test -v -gcflags=-d=checkptr=0 .

13
README.md
View file

@ -4,7 +4,7 @@ A blazingly fast JSON serializing & deserializing library, accelerated by JI
## Requirement
- Go 1.15/1.16/1.17/1.18
- Linux/darwin OS
- Linux/MacOS/Windows
- Amd64 CPU with AVX instruction set
## Features
@ -260,6 +260,12 @@ println(string(buf) == string(exp)) // true
- iteration: `Values()`, `Properties()`, `ForEach()`, `SortKeys()`
- modification: `Set()`, `SetByIndex()`, `Add()`
## Compatibility
Sonic **DOSE NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. For developers who use sonic to build their applications in different environments (ex: developing on M1 Mac but running on linux server), or those who want to handle JSON strictly consistent with `encoding/json`, we provide some compatible APIs as `sonic.API`
- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run on sonic-supporting environment. It will fall back to `encoding/json` with corresponding config , and some options like `SortKeys=false` will be invalid.
- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) to run on sonic-supporting environment. It whill fall back to `encoding/json`.
- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic-supporting environment. It will fall back to `encoding/json` with corresponding config , and some options will be invalid.
## Tips
### Pretouch
@ -316,4 +322,7 @@ Why? Because `ast.Node` stores its children using `array`:
- **Hashing** (`map[x]`) is not as efficient as **Indexing** (`array[x]`), which `ast.Node` can conduct on **both array and object**;
- Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**.
**CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, your can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}`
**CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, your can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}`
## Community
Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem.

127
api.go Normal file
View file

@ -0,0 +1,127 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`io`
)
// Config is a combination of sonic/encoder.Options and sonic/decoder.Options
type Config struct {
EscapeHTML bool
SortMapKeys bool
CompactMarshaler bool
NoQuoteTextMarshaler bool
UseInt64 bool
UseNumber bool
UseUnicodeErrors bool
DisallowUnknownFields bool
CopyString bool
}
var (
// ConfigDefault is the default config of APIs, aiming at efficiency and safty.
ConfigDefault = Config{}.Froze()
// ConfigStd is the standard config of APIs, aiming at being compatible with encoding/json.
ConfigStd = Config{
EscapeHTML : true,
SortMapKeys: true,
CompactMarshaler: true,
CopyString : true,
}.Froze()
// ConfigFastest is the fastest config of APIs, aiming at speed.
ConfigFastest = Config{
NoQuoteTextMarshaler: true,
}.Froze()
)
// API a binding of specific config.
// This interface is inspired by github.com/json-iterator/go,
// and has same behaviors under equavilent config.
type API interface {
// MarshalToString returns the JSON encoding string of v
MarshalToString(v interface{}) (string, error)
// Marshal returns the JSON encoding bytes of v.
Marshal(v interface{}) ([]byte, error)
// MarshalIndent returns the JSON encoding bytes with indent and prefix.
MarshalIndent(v interface{}, prefix, indent string) ([]byte, error)
// UnmarshalFromString parses the JSON-encoded bytes and stores the result in the value pointed to by v.
UnmarshalFromString(str string, v interface{}) error
// Unmarshal parses the JSON-encoded string and stores the result in the value pointed to by v.
Unmarshal(data []byte, v interface{}) error
// NewEncoder create a Encoder holding writer
NewEncoder(writer io.Writer) Encoder
// NewDecoder create a Decoder holding reader
NewDecoder(reader io.Reader) Decoder
// Valid validates the JSON-encoded bytes and reportes if it is valid
Valid(data []byte) bool
}
// Encoder encodes JSON into io.Writer
type Encoder interface {
// Encode writes the JSON encoding of v to the stream, followed by a newline character.
Encode(val interface{}) error
// SetEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior NOT ESCAPE
SetEscapeHTML(on bool)
// SetIndent instructs the encoder to format each subsequent encoded value
// as if indented by the package-level function Indent(dst, src, prefix, indent).
// Calling SetIndent("", "") disables indentation
SetIndent(prefix, indent string)
}
// Decoder decodes JSON from io.Read
type Decoder interface {
// Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v.
Decode(val interface{}) error
// Buffered returns a reader of the data remaining in the Decoder's buffer.
// The reader is valid until the next call to Decode.
Buffered() io.Reader
// DisallowUnknownFields causes the Decoder to return an error when the destination is a struct
// and the input contains object keys which do not match any non-ignored, exported fields in the destination.
DisallowUnknownFields()
// More reports whether there is another element in the current array or object being parsed.
More() bool
// UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
UseNumber()
}
// Marshal returns the JSON encoding bytes of v.
func Marshal(val interface{}) ([]byte, error) {
return ConfigDefault.Marshal(val)
}
// MarshalString returns the JSON encoding string of v.
func MarshalString(val interface{}) (string, error) {
return ConfigDefault.MarshalToString(val)
}
// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.
// NOTICE: This API copies given buffer by default,
// if you want to pass JSON more efficiently, use UnmarshalString instead.
func Unmarshal(buf []byte, val interface{}) error {
return ConfigDefault.Unmarshal(buf, val)
}
// UnmarshalString is like Unmarshal, except buf is a string.
func UnmarshalString(buf string, val interface{}) error {
return ConfigDefault.UnmarshalFromString(buf, val)
}

262
api_test.go Normal file
View file

@ -0,0 +1,262 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/json`
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
func TestCompatMarshalDefault(t *testing.T){
var obj = map[string]interface{}{
"c": json.RawMessage("[\"<&>\"]"),
}
sout, serr := ConfigDefault.Marshal(obj)
jout, jerr := jt.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
// obj = map[string]interface{}{
// "a": json.RawMessage(" [} "),
// }
// sout, serr = ConfigDefault.Marshal(obj)
// jout, jerr = json.Marshal(obj)
// require.NotNil(t, jerr)
// require.NotNil(t, serr)
// require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigDefault.MarshalIndent(obj, "", " ")
jout, jerr = jt.MarshalIndent(obj, "", " ")
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
}
func TestCompatMarshalStd(t *testing.T) {
t.Parallel()
var obj = map[string]interface{}{
"c": json.RawMessage(" [ \"<&>\" ] "),
"b": json.RawMessage(" [ ] "),
}
sout, serr := ConfigStd.Marshal(obj)
jout, jerr := json.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage(" [} "),
}
sout, serr = ConfigStd.Marshal(obj)
jout, jerr = json.Marshal(obj)
require.NotNil(t, jerr)
require.NotNil(t, serr)
require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ")
jout, jerr = json.MarshalIndent(obj, "xxxx", " ")
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
}
func TestCompatUnmarshalDefault(t *testing.T) {
var sobj = map[string]interface{}{}
var jobj = map[string]interface{}{}
var data = []byte(`{"a":-0}`)
var str = string(data)
serr := ConfigDefault.UnmarshalFromString(str, &sobj)
jerr := jt.UnmarshalFromString(str, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
serr = ConfigDefault.Unmarshal(data, &x)
jerr = jt.Unmarshal(data, &y)
require.Equal(t, jerr, serr)
require.Equal(t, y, x)
}
func TestCompatUnmarshalStd(t *testing.T) {
var sobj = map[string]interface{}{}
var jobj = map[string]interface{}{}
var data = []byte(`{"a":1.00000001E-10}`)
var str = string(data)
serr := ConfigStd.UnmarshalFromString(str, &sobj)
jerr := json.Unmarshal(data, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
data[2] = '0'
require.Equal(t, jobj, sobj)
sobj = map[string]interface{}{}
jobj = map[string]interface{}{}
data = []byte(`{"a":1}`)
cfg := Config{
UseNumber: true,
}.Froze()
serr = cfg.Unmarshal(data, &sobj)
dec := json.NewDecoder(bytes.NewBuffer(data))
dec.UseNumber()
jerr = dec.Decode(&jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
cfg = Config{
DisallowUnknownFields: true,
}.Froze()
serr = cfg.Unmarshal(data, &x)
dec = json.NewDecoder(bytes.NewBuffer(data))
dec.UseNumber()
dec.DisallowUnknownFields()
jerr = dec.Decode(&y)
require.Equal(t, jerr, serr)
require.Equal(t, y, x)
}
func TestCompatEncoderDefault(t *testing.T) {
var o = map[string]interface{}{
"a": "<>",
// "b": json.RawMessage(" [ ] "),
}
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = jt.NewEncoder(w1)
var enc2 = ConfigDefault.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(true)
enc2.SetEscapeHTML(true)
enc1.SetIndent("", " ")
enc2.SetIndent("", " ")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
enc2.SetIndent("", "")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatEncoderStd(t *testing.T) {
var o = map[string]interface{}{
"a": "<>",
"b": json.RawMessage(" [ ] "),
}
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = json.NewEncoder(w1)
var enc2 = ConfigStd.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(true)
enc2.SetEscapeHTML(true)
enc1.SetIndent("\n", " ")
enc2.SetIndent("\n", " ")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
enc2.SetIndent("", "")
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatDecoderStd(t *testing.T) {
var o1 = map[string]interface{}{}
var o2 = map[string]interface{}{}
var s = `{"a":"b"} {"1":"2"} a {}`
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = json.NewDecoder(w1)
var enc2 = ConfigStd.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
}
func TestCompatDecoderDefault(t *testing.T) {
var o1 = map[string]interface{}{}
var o2 = map[string]interface{}{}
var s = `{"a":"b"} {"1":"2"} a {}`
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = jt.NewDecoder(w1)
var enc2 = ConfigDefault.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
// require.Equal(t, enc1.More(), enc2.More())
// require.NotNil(t, enc1.Decode(&o1))
// require.NotNil(t, enc2.Decode(&o2))
// require.Equal(t, w1.String(), w2.String())
}

117
compat.go Normal file
View file

@ -0,0 +1,117 @@
// +build !amd64
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sonic
import (
`bytes`
`encoding/json`
`io`
)
type frozenConfig struct {
Config
}
// Froze convert the Config to API
func (cfg Config) Froze() API {
api := &frozenConfig{Config: cfg}
return api
}
func (cfg *frozenConfig) marshalOptions(val interface{}, prefix, indent string) ([]byte, error) {
w := bytes.NewBuffer([]byte{})
enc := json.NewEncoder(w)
enc.SetEscapeHTML(cfg.EscapeHTML)
enc.SetIndent(prefix, indent)
err := enc.Encode(val)
out := w.Bytes()
// json.Encoder always appends '\n' after encoding,
// which is not same with json.Marshal()
if len(out) > 0 && out[len(out)-1] == '\n' {
out = out[:len(out)-1]
}
return out, err
}
// Marshal is implemented by sonic
func (cfg *frozenConfig) Marshal(val interface{}) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, "", "")
}
return json.Marshal(val)
}
// MarshalToString is implemented by sonic
func (cfg *frozenConfig) MarshalToString(val interface{}) (string, error) {
out, err := cfg.Marshal(val)
return string(out), err
}
// MarshalIndent is implemented by sonic
func (cfg *frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {
if !cfg.EscapeHTML {
return cfg.marshalOptions(val, prefix, indent)
}
return json.MarshalIndent(val, prefix, indent)
}
// UnmarshalFromString is implemented by sonic
func (cfg *frozenConfig) UnmarshalFromString(buf string, val interface{}) error {
r := bytes.NewBufferString(buf)
dec := json.NewDecoder(r)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec.Decode(val)
}
// Unmarshal is implemented by sonic
func (cfg *frozenConfig) Unmarshal(buf []byte, val interface{}) error {
return cfg.UnmarshalFromString(string(buf), val)
}
// NewEncoder is implemented by sonic
func (cfg *frozenConfig) NewEncoder(writer io.Writer) Encoder {
enc := json.NewEncoder(writer)
if !cfg.EscapeHTML {
enc.SetEscapeHTML(cfg.EscapeHTML)
}
return enc
}
// NewDecoder is implemented by sonic
func (cfg *frozenConfig) NewDecoder(reader io.Reader) Decoder {
dec := json.NewDecoder(reader)
if cfg.UseNumber {
dec.UseNumber()
}
if cfg.DisallowUnknownFields {
dec.DisallowUnknownFields()
}
return dec
}
// Valid is implemented by sonic
func (cfg *frozenConfig) Valid(data []byte) bool {
return json.Valid(data)
}

234
decode_float_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/*
* Copyright 2021 ByteDance Inc.
*
@ -17,146 +19,146 @@
package sonic
import (
"encoding/json"
"reflect"
"strings"
"testing"
`encoding/json`
`reflect`
`strings`
`testing`
"github.com/bytedance/sonic/decoder"
`github.com/bytedance/sonic/decoder`
)
type atofTest struct {
in string
out string
err error
in string
out string
err error
}
// Tests from Go strconv package, https://github.com/golang/go/blob/master/src/strconv/atof_test.go
// All tests are passed in Go encoding/json.
var atoftests = []atofTest{
{"1.234e", "", nil}, // error
{"1i", "1", nil}, // pass
{"1", "1", nil},
{"1e23", "1e+23", nil},
{"1E23", "1e+23", nil},
{"100000000000000000000000", "1e+23", nil},
{"1e-100", "1e-100", nil},
{"123456700", "1.234567e+08", nil},
{"99999999999999974834176", "9.999999999999997e+22", nil},
{"100000000000000000000001", "1.0000000000000001e+23", nil},
{"100000000000000008388608", "1.0000000000000001e+23", nil},
{"100000000000000016777215", "1.0000000000000001e+23", nil},
{"100000000000000016777216", "1.0000000000000003e+23", nil},
{"-1", "-1", nil},
{"-0.1", "-0.1", nil},
{"-0", "-0", nil},
{"1e-20", "1e-20", nil},
{"625e-3", "0.625", nil},
{"1.234e", "", nil}, // error
{"1i", "1", nil}, // pass
{"1", "1", nil},
{"1e23", "1e+23", nil},
{"1E23", "1e+23", nil},
{"100000000000000000000000", "1e+23", nil},
{"1e-100", "1e-100", nil},
{"123456700", "1.234567e+08", nil},
{"99999999999999974834176", "9.999999999999997e+22", nil},
{"100000000000000000000001", "1.0000000000000001e+23", nil},
{"100000000000000008388608", "1.0000000000000001e+23", nil},
{"100000000000000016777215", "1.0000000000000001e+23", nil},
{"100000000000000016777216", "1.0000000000000003e+23", nil},
{"-1", "-1", nil},
{"-0.1", "-0.1", nil},
{"-0", "-0", nil},
{"1e-20", "1e-20", nil},
{"625e-3", "0.625", nil},
// zeros
{"0", "0", nil},
{"0e0", "0", nil},
{"-0e0", "-0", nil},
{"0e-0", "0", nil},
{"-0e-0", "-0", nil},
{"0e+0", "0", nil},
{"-0e+0", "-0", nil},
{"0e+01234567890123456789", "0", nil},
{"0.00e-01234567890123456789", "0", nil},
{"-0e+01234567890123456789", "-0", nil},
{"-0.00e-01234567890123456789", "-0", nil},
// zeros
{"0", "0", nil},
{"0e0", "0", nil},
{"-0e0", "-0", nil},
{"0e-0", "0", nil},
{"-0e-0", "-0", nil},
{"0e+0", "0", nil},
{"-0e+0", "-0", nil},
{"0e+01234567890123456789", "0", nil},
{"0.00e-01234567890123456789", "0", nil},
{"-0e+01234567890123456789", "-0", nil},
{"-0.00e-01234567890123456789", "-0", nil},
{"0e291", "0", nil}, // issue 15364
{"0e292", "0", nil}, // issue 15364
{"0e347", "0", nil}, // issue 15364
{"0e348", "0", nil}, // issue 15364
{"-0e291", "-0", nil},
{"-0e292", "-0", nil},
{"-0e347", "-0", nil},
{"-0e348", "-0", nil},
{"0e291", "0", nil}, // issue 15364
{"0e292", "0", nil}, // issue 15364
{"0e347", "0", nil}, // issue 15364
{"0e348", "0", nil}, // issue 15364
{"-0e291", "-0", nil},
{"-0e292", "-0", nil},
{"-0e347", "-0", nil},
{"-0e348", "-0", nil},
// largest float64
{"1.7976931348623157e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623157e308", "-1.7976931348623157e+308", nil},
// largest float64
{"1.7976931348623157e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623157e308", "-1.7976931348623157e+308", nil},
// the border is ...158079
// borderline - okay
{"1.7976931348623158e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623158e308", "-1.7976931348623157e+308", nil},
// the border is ...158079
// borderline - okay
{"1.7976931348623158e308", "1.7976931348623157e+308", nil},
{"-1.7976931348623158e308", "-1.7976931348623157e+308", nil},
// a little too large
{"1e308", "1e+308", nil},
// a little too large
{"1e308", "1e+308", nil},
// denormalized
{"1e-305", "1e-305", nil},
{"1e-306", "1e-306", nil},
{"1e-307", "1e-307", nil},
{"1e-308", "1e-308", nil},
{"1e-309", "1e-309", nil},
{"1e-310", "1e-310", nil},
{"1e-322", "1e-322", nil},
// smallest denormal
{"5e-324", "5e-324", nil},
{"4e-324", "5e-324", nil},
{"3e-324", "5e-324", nil},
// too small
{"2e-324", "0", nil},
// way too small
{"1e-350", "0", nil},
{"1e-400000", "0", nil},
// denormalized
{"1e-305", "1e-305", nil},
{"1e-306", "1e-306", nil},
{"1e-307", "1e-307", nil},
{"1e-308", "1e-308", nil},
{"1e-309", "1e-309", nil},
{"1e-310", "1e-310", nil},
{"1e-322", "1e-322", nil},
// smallest denormal
{"5e-324", "5e-324", nil},
{"4e-324", "5e-324", nil},
{"3e-324", "5e-324", nil},
// too small
{"2e-324", "0", nil},
// way too small
{"1e-350", "0", nil},
{"1e-400000", "0", nil},
// try to overflow exponent
{"1e-4294967296", "0", nil},
{"1e-18446744073709551616", "0", nil},
// try to overflow exponent
{"1e-4294967296", "0", nil},
{"1e-18446744073709551616", "0", nil},
// https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
{"2.2250738585072012e-308", "2.2250738585072014e-308", nil},
// https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
{"2.2250738585072011e-308", "2.225073858507201e-308", nil},
// https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
{"2.2250738585072012e-308", "2.2250738585072014e-308", nil},
// https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
{"2.2250738585072011e-308", "2.225073858507201e-308", nil},
// A very large number (initially wrongly parsed by the fast algorithm).
{"4.630813248087435e+307", "4.630813248087435e+307", nil},
// A very large number (initially wrongly parsed by the fast algorithm).
{"4.630813248087435e+307", "4.630813248087435e+307", nil},
// A different kind of very large number.
{"22.222222222222222", "22.22222222222222", nil},
{"2." + strings.Repeat("2", 800) + "e+1", "22.22222222222222", nil},
// A different kind of very large number.
{"22.222222222222222", "22.22222222222222", nil},
{"2." + strings.Repeat("2", 800) + "e+1", "22.22222222222222", nil},
// Exactly halfway between 1 and math.Nextafter(1, 2).
// Round to even (down).
{"1.00000000000000011102230246251565404236316680908203125", "1", nil},
// Slightly lower; still round down.
{"1.00000000000000011102230246251565404236316680908203124", "1", nil},
// Slightly higher; round up.
{"1.00000000000000011102230246251565404236316680908203126", "1.0000000000000002", nil},
// Slightly higher, but you have to read all the way to the end.
{"1.00000000000000011102230246251565404236316680908203125" + strings.Repeat("0", 10000) + "1", "1.0000000000000002", nil},
// Exactly halfway between 1 and math.Nextafter(1, 2).
// Round to even (down).
{"1.00000000000000011102230246251565404236316680908203125", "1", nil},
// Slightly lower; still round down.
{"1.00000000000000011102230246251565404236316680908203124", "1", nil},
// Slightly higher; round up.
{"1.00000000000000011102230246251565404236316680908203126", "1.0000000000000002", nil},
// Slightly higher, but you have to read all the way to the end.
{"1.00000000000000011102230246251565404236316680908203125" + strings.Repeat("0", 10000) + "1", "1.0000000000000002", nil},
// Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2)
// Round to even (up).
{"1.00000000000000033306690738754696212708950042724609375", "1.0000000000000004", nil},
// Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2)
// Round to even (up).
{"1.00000000000000033306690738754696212708950042724609375", "1.0000000000000004", nil},
// Halfway between 1090544144181609278303144771584 and 1090544144181609419040633126912
// (15497564393479157p+46, should round to even 15497564393479156p+46, issue 36657)
{"1090544144181609348671888949248", "1.0905441441816093e+30", nil},
// Halfway between 1090544144181609278303144771584 and 1090544144181609419040633126912
// (15497564393479157p+46, should round to even 15497564393479156p+46, issue 36657)
{"1090544144181609348671888949248", "1.0905441441816093e+30", nil},
// Corner case between int64 and float64 for the input
{"9223372036854775807", "9223372036854775807", nil}, // max int64: (1 << 63) - 1
{"9223372036854775808", "9223372036854775808", nil},
{"-9223372036854775808", "-9223372036854775808", nil}, // min int64: 1 << 63
{"-9223372036854775809", "-9223372036854775809", nil},
// Corner case between int64 and float64 for the input
{"9223372036854775807", "9223372036854775807", nil}, // max int64: (1 << 63) - 1
{"9223372036854775808", "9223372036854775808", nil},
{"-9223372036854775808", "-9223372036854775808", nil}, // min int64: 1 << 63
{"-9223372036854775809", "-9223372036854775809", nil},
}
func TestDecodeFloat(t *testing.T) {
for i, tt := range atoftests {
// default float64
var sonicout, stdout float64
sonicerr := decoder.NewDecoder(tt.in).Decode(&sonicout)
stderr := json.NewDecoder(strings.NewReader(tt.in)).Decode(&stdout)
if !reflect.DeepEqual(sonicout, stdout) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicout, stdout)
}
if !reflect.DeepEqual(sonicerr == nil, stderr == nil) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicerr, stderr)
}
}
for i, tt := range atoftests {
// default float64
var sonicout, stdout float64
sonicerr := decoder.NewDecoder(tt.in).Decode(&sonicout)
stderr := json.NewDecoder(strings.NewReader(tt.in)).Decode(&stdout)
if !reflect.DeepEqual(sonicout, stdout) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicout, stdout)
}
if !reflect.DeepEqual(sonicerr == nil, stderr == nil) {
t.Fatalf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sonicerr, stderr)
}
}
}

3768
decode_test.go
View file

File diff suppressed because it is too large Load diff

18
decoder/decoder.go
View file

@ -37,6 +37,24 @@ const (
_F_allow_control = 31
)
type Options uint64
const (
OptionUseInt64 Options = 1 << _F_use_int64
OptionUseNumber Options = 1 << _F_use_number
OptionUseUnicodeErrors Options = 1 << _F_disable_urc
OptionDisableUnknown Options = 1 << _F_disable_unknown
OptionCopyString Options = 1 << _F_copy_string
)
func (self *Decoder) SetOptions(opts Options) {
if (opts & 1<<_F_use_number != 0) && (opts & 1<<_F_use_int64 != 0) {
panic("can't set OptionUseInt64 and OptionUseNumber both!")
}
self.f = uint64(opts)
}
// Decoder is the decoder context object
type Decoder struct {
i int

1682
encode_test.go
View file

File diff suppressed because it is too large Load diff

2
search_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/*
* Copyright 2021 ByteDance Inc.
*

128
sonic.go
View file

@ -1,3 +1,5 @@
// +build amd64
/*
* Copyright 2021 ByteDance Inc.
*
@ -18,6 +20,7 @@
package sonic
import (
`io`
`reflect`
`github.com/bytedance/sonic/ast`
@ -28,36 +31,7 @@ import (
`github.com/bytedance/sonic/internal/rt`
)
// Marshal returns the JSON encoding bytes of v.
func Marshal(val interface{}) ([]byte, error) {
return encoder.Encode(val, 0)
}
// MarshalString returns the JSON encoding string of v.
func MarshalString(val interface{}) (string, error) {
buf, err := encoder.Encode(val, 0)
return rt.Mem2Str(buf), err
}
// Unmarshal parses the JSON-encoded data and stores the result in the value
// pointed to by v.
// NOTICE: This API copies given buffer by default,
// if you want to pass JSON more efficiently, use UnmarshalString instead.
func Unmarshal(buf []byte, val interface{}) error {
return UnmarshalString(string(buf), val)
}
// UnmarshalString is like Unmarshal, except buf is a string.
func UnmarshalString(buf string, val interface{}) error {
dec := decoder.NewDecoder(buf)
err := dec.Decode(val)
pos := dec.Pos()
/* check for errors */
if err != nil {
return err
}
func checkTrailings(buf string, pos int) error {
/* skip all the trailing spaces */
if pos != len(buf) {
for pos < len(buf) && (types.SPACE_MASK & (1 << buf[pos])) != 0 {
@ -73,11 +47,103 @@ func UnmarshalString(buf string, val interface{}) error {
/* junk after JSON value */
return decoder.SyntaxError {
Src : buf,
Pos : dec.Pos(),
Pos : pos,
Code : types.ERR_INVALID_CHAR,
}
}
type frozenConfig struct {
Config
encoderOpts encoder.Options
decoderOpts decoder.Options
}
// Froze convert the Config to API
func (cfg Config) Froze() API {
api := &frozenConfig{Config: cfg}
if cfg.EscapeHTML {
api.encoderOpts |= encoder.EscapeHTML
}
if cfg.SortMapKeys {
api.encoderOpts |= encoder.SortMapKeys
}
if cfg.CompactMarshaler {
api.encoderOpts |= encoder.CompactMarshaler
}
if cfg.NoQuoteTextMarshaler {
api.encoderOpts |= encoder.NoQuoteTextMarshaler
}
if cfg.UseInt64 {
api.decoderOpts |= decoder.OptionUseInt64
}
if cfg.UseNumber {
api.decoderOpts |= decoder.OptionUseNumber
}
if cfg.DisallowUnknownFields {
api.decoderOpts |= decoder.OptionDisableUnknown
}
if cfg.CopyString {
api.decoderOpts |= decoder.OptionCopyString
}
return api
}
// Marshal is implemented by sonic
func (cfg *frozenConfig) Marshal(val interface{}) ([]byte, error) {
return encoder.Encode(val, cfg.encoderOpts)
}
// MarshalToString is implemented by sonic
func (cfg *frozenConfig) MarshalToString(val interface{}) (string, error) {
buf, err := encoder.Encode(val, cfg.encoderOpts)
return rt.Mem2Str(buf), err
}
// MarshalIndent is implemented by sonic
func (cfg *frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {
return encoder.EncodeIndented(val, prefix, indent, cfg.encoderOpts)
}
// UnmarshalFromString is implemented by sonic
func (cfg *frozenConfig) UnmarshalFromString(buf string, val interface{}) error {
dec := decoder.NewDecoder(buf)
dec.SetOptions(cfg.decoderOpts)
err := dec.Decode(val)
pos := dec.Pos()
/* check for errors */
if err != nil {
return err
}
return checkTrailings(buf, pos)
}
// Unmarshal is implemented by sonic
func (cfg *frozenConfig) Unmarshal(buf []byte, val interface{}) error {
return cfg.UnmarshalFromString(string(buf), val)
}
// NewEncoder is implemented by sonic
func (cfg *frozenConfig) NewEncoder(writer io.Writer) Encoder {
enc := encoder.NewStreamEncoder(writer)
enc.Opts = cfg.encoderOpts
return enc
}
// NewDecoder is implemented by sonic
func (cfg *frozenConfig) NewDecoder(reader io.Reader) Decoder {
dec := decoder.NewStreamDecoder(reader)
dec.SetOptions(cfg.decoderOpts)
return dec
}
// Valid is implemented by sonic
func (cfg *frozenConfig) Valid(data []byte) bool {
ok, _ := encoder.Valid(data)
return ok
}
// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//

146
use_number_test.go
View file

@ -1,3 +1,5 @@
// +build amd64
/*
* Copyright 2021 ByteDance Inc.
*
@ -17,98 +19,98 @@
package sonic
import (
"encoding/json"
"reflect"
"testing"
`encoding/json`
`reflect`
`testing`
"github.com/bytedance/sonic/decoder"
`github.com/bytedance/sonic/decoder`
)
type useInt64Test struct {
in string
out int64
in string
out int64
}
type useFloatTest struct {
in string
out float64
in string
out float64
}
var useinttest = []useInt64Test{
// int64
{"0", 0},
{"1", 1},
{"-1", -1},
{"100", 100},
// int64
{"0", 0},
{"1", 1},
{"-1", -1},
{"100", 100},
{"-9223372036854775807", -9223372036854775807},
{"-9223372036854775808", -9223372036854775808}, //min int64
{"9223372036854775807", 9223372036854775807}, //max int64
{"9223372036854775806", 9223372036854775806},
{"-9223372036854775807", -9223372036854775807},
{"-9223372036854775808", -9223372036854775808}, //min int64
{"9223372036854775807", 9223372036854775807}, //max int64
{"9223372036854775806", 9223372036854775806},
}
var usefloattest = []useFloatTest{
// float64
{"-9223372036854775809", -9223372036854775809}, // int64 overflow
{"9223372036854775808", 9223372036854775808}, // int64 overflow
{"1e2", 1e2},
{"1e-20", 1e-20},
{"1.0", 1},
// float64
{"-9223372036854775809", -9223372036854775809}, // int64 overflow
{"9223372036854775808", 9223372036854775808}, // int64 overflow
{"1e2", 1e2},
{"1e-20", 1e-20},
{"1.0", 1},
}
func TestUseInt64(t *testing.T) {
for i, tt := range useinttest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseInt64()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range useinttest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseInt64()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range usefloattest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseInt64()
//the input string is not int64, still return float64
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range usefloattest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseInt64()
//the input string is not int64, still return float64
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, tt.out) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.in)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
}
func TestUseNumber(t *testing.T) {
for i, tt := range useinttest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseNumber()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range useinttest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseNumber()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range usefloattest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseNumber()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
for i, tt := range usefloattest {
var sout interface{}
dc := decoder.NewDecoder(tt.in)
dc.UseNumber()
serr := dc.Decode(&sout)
if !reflect.DeepEqual(sout, json.Number(tt.in)) {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n %#v\n", i, tt.in, sout, tt.out)
}
if serr != nil {
t.Errorf("Test %d, %#v\ngot:\n %#v\nexp:\n nil\n", i, tt, serr)
}
}
}