XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity

fmt: move thrid-party tests into another module (#299)

Browse source 
* fix:move benchmarkTests and unitTests related to external json library

* remove extra spaces

* merge go.mod files

* refactor: add external_jsonlib_test into go.work and CI

Co-authored-by: Yi Duan <duanyi.aster@bytedance.com>
This commit is contained in:
Chen Yannan 2022-09-20 12:02:41 +08:00 committed by GitHub
parent ccc0f3f1e3
commit 5e54c02172
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 1786 additions and 830 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/benchmark-linux-amd64.yml vendored
View file

@ -23,8 +23,11 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Benchmark
- name: Benchmark sonic
run: sh bench.sh
# - name: Benchmark third-party
# run: go test -benchmem -run=^$ -bench . -v ./generic_test
# - name: Diff
# run: ./bench.py -b '"^Benchmark.*Sonic"' -c

9
.github/workflows/push-check-go118.yml vendored
View file

@ -21,10 +21,9 @@ jobs:
${{ runner.os }}-go-
- name: Unit Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
run: |
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./external_jsonlib_test/...
- name: Generic Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test
- name: Benchmark
run: go test -benchmem -run=^$ -bench . -v ./generic_test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test

5
.github/workflows/push-check-linux-amd64.yml vendored
View file

@ -24,4 +24,7 @@ jobs:
${{ runner.os }}-go-
- name: Unit Test
run: go test -v -gcflags=-d=checkptr=0 ./...
run: |
go test -v -gcflags=-d=checkptr=0 ./...
cd ./external_jsonlib_test
go test -v -gcflags=-d=checkptr=0 ./...

226
ast/parser_test.go
View file

@ -17,17 +17,15 @@
package ast
import (
`os`
`encoding/json`
`testing`
`time`
`os`
`runtime`
`runtime/debug`
`sync`
`testing`
`time`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/assert`
`github.com/tidwall/gjson`
)
var (
@ -43,7 +41,7 @@ func TestMain(m *testing.M) {
println("Begin GC looping...")
for {
runtime.GC()
debug.FreeOSMemory()
debug.FreeOSMemory()
}
println("stop GC looping!")
}()
@ -108,7 +106,7 @@ func runDecoderTestUseNumber(t *testing.T, src string, expect interface{}) {
}
}
func n2f64(i json.Number) float64{
func n2f64(i json.Number) float64{
x, err := i.Float64()
if err != nil {
panic(err)
@ -257,39 +255,6 @@ func BenchmarkParser_Sonic(b *testing.B) {
}
}
func BenchmarkParser_Gjson(b *testing.B) {
gjson.Parse(_TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
gjson.Parse(_TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
}
}
func BenchmarkParser_Jsoniter(b *testing.B) {
v := jsoniter.Get([]byte(_TwitterJson)).GetInterface()
if v == nil {
b.Fatal(v)
}
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = jsoniter.Get([]byte(_TwitterJson)).GetInterface()
}
}
func BenchmarkParser_Parallel_Sonic(b *testing.B) {
r, _ := NewParser(_TwitterJson).Parse()
if err := r.LoadAll(); err != nil {
@ -305,40 +270,6 @@ func BenchmarkParser_Parallel_Sonic(b *testing.B) {
})
}
func BenchmarkParser_Parallel_Gjson(b *testing.B) {
gjson.Parse(_TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
return true
})
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
gjson.Parse(_TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
}
})
}
func BenchmarkParser_Parallel_Jsoniter(b *testing.B) {
var bv = []byte(_TwitterJson)
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var out interface{}
_ = jsoniter.Unmarshal(bv, &out)
}
})
}
func BenchmarkParseOne_Sonic(b *testing.B) {
ast, _ := NewParser(_TwitterJson).Parse()
node, _ := ast.Get("statuses").Index(2).Get("id").Int64()
@ -353,37 +284,6 @@ func BenchmarkParseOne_Sonic(b *testing.B) {
}
}
func BenchmarkParseOne_Gjson(b *testing.B) {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.2.id")
v := node.Int()
if v != 249289491129438208 {
b.Fatal(node)
}
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.2.id")
_ = node.Int()
}
}
func BenchmarkParseOne_Jsoniter(b *testing.B) {
data := []byte(_TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
node := ast.ToInt()
if node != 249289491129438208 {
b.Fail()
}
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data, "statuses", 2, "id")
_ = ast.ToInt()
}
}
func BenchmarkParseOne_Parallel_Sonic(b *testing.B) {
ast, _ := NewParser(_TwitterJson).Parse()
node, _ := ast.Get("statuses").Index(2).Get("id").Int64()
@ -400,42 +300,6 @@ func BenchmarkParseOne_Parallel_Sonic(b *testing.B) {
})
}
func BenchmarkParseOne_Parallel_Gjson(b *testing.B) {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.2.id")
v := node.Int()
if v != 249289491129438208 {
b.Fatal(node)
}
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.2.id")
_ = node.Int()
}
})
}
func BenchmarkParseOne_Parallel_Jsoniter(b *testing.B) {
data := []byte(_TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
node := ast.ToInt()
if node != 249289491129438208 {
b.Fail()
}
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
data := []byte(_TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
_ = ast.ToInt()
}
})
}
func BenchmarkParseSeven_Sonic(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
@ -454,43 +318,6 @@ func BenchmarkParseSeven_Sonic(b *testing.B) {
}
}
func BenchmarkParseSeven_Gjson(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.3.id")
node = ast.Get( "statuses.3.user.entities.description")
node = ast.Get( "statuses.3.user.entities.url.urls")
node = ast.Get( "statuses.3.user.entities.url")
node = ast.Get( "statuses.3.user.created_at")
node = ast.Get( "statuses.3.user.name")
node = ast.Get( "statuses.3.text")
if node.Value() == nil {
b.Fail()
}
}
}
func BenchmarkParseSeven_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
data := []byte(_TwitterJson)
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data)
node := ast.Get("statuses", 3, "id")
node = ast.Get("statuses", 3, "user", "entities","description")
node = ast.Get("statuses", 3, "user", "entities","url","urls")
node = ast.Get("statuses", 3, "user", "entities","url")
node = ast.Get("statuses", 3, "user", "created_at")
node = ast.Get("statuses", 3, "user", "name")
node = ast.Get("statuses", 3, "text")
if node.LastError() != nil {
b.Fail()
}
}
}
func BenchmarkParseSeven_Parallel_Sonic(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
@ -509,45 +336,4 @@ func BenchmarkParseSeven_Parallel_Sonic(b *testing.B) {
}
}
})
}
func BenchmarkParseSeven_Parallel_Gjson(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Parse(_TwitterJson)
node := ast.Get("statuses.3.id")
node = ast.Get( "statuses.3.user.entities.description")
node = ast.Get( "statuses.3.user.entities.url.urls")
node = ast.Get( "statuses.3.user.entities.url")
node = ast.Get( "statuses.3.user.created_at")
node = ast.Get( "statuses.3.user.name")
node = ast.Get( "statuses.3.text")
if node.Value() == nil {
b.Fail()
}
}
})
}
func BenchmarkParseSeven_Parallel_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
data := []byte(_TwitterJson)
ast := jsoniter.Get(data)
node := ast.Get("statuses", 3, "id")
node = ast.Get("statuses", 3, "user", "entities","description")
node = ast.Get("statuses", 3, "user", "entities","url","urls")
node = ast.Get("statuses", 3, "user", "entities","url")
node = ast.Get("statuses", 3, "user", "created_at")
node = ast.Get("statuses", 3, "user", "name")
node = ast.Get("statuses", 3, "text")
if node.LastError() != nil {
b.Fail()
}
}
})
}
}

124
ast/search_test.go
View file

@ -17,17 +17,13 @@
package ast
import (
`testing`
`runtime`
`sync`
`fmt`
`math`
`runtime`
`strconv`
`sync`
`testing`
`github.com/tidwall/sjson`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/assert`
`github.com/tidwall/gjson`
)
@ -192,29 +188,6 @@ func TestSearchNotExist(t *testing.T) {
}
}
func BenchmarkGetOne_Gjson(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
for i := 0; i < b.N; i++ {
ast := gjson.Get(_TwitterJson, "statuses.3.id")
node := ast.Int()
if node != 249279667666817024 {
b.Fail()
}
}
}
func BenchmarkGetOne_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
data := []byte(_TwitterJson)
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data, "statuses", 3, "id")
node := ast.ToInt()
if node != 249279667666817024 {
b.Fail()
}
}
}
func BenchmarkGetOne_Sonic(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
ast := NewSearcher(_TwitterJson)
@ -230,33 +203,6 @@ func BenchmarkGetOne_Sonic(b *testing.B) {
}
}
func BenchmarkGetOne_Parallel_Gjson(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Get(_TwitterJson, "statuses.3.id")
node := ast.Int()
if node != 249279667666817024 {
b.Fail()
}
}
})
}
func BenchmarkGetOne_Parallel_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
data := []byte(_TwitterJson)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := jsoniter.Get(data, "statuses", 3, "id")
node := ast.ToInt()
if node != 249279667666817024 {
b.Fail()
}
}
})
}
func BenchmarkGetOne_Parallel_Sonic(b *testing.B) {
b.SetBytes(int64(len(_TwitterJson)))
b.RunParallel(func(pb *testing.PB) {
@ -293,36 +239,6 @@ func BenchmarkSetOne_Sonic(b *testing.B) {
}
}
func BenchmarkSetOne_Sjson(b *testing.B) {
path := fmt.Sprintf("%s.%d.%s", "statuses", 3, "id")
_, err := sjson.Set(_TwitterJson, path, math.MaxInt32)
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(_TwitterJson)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
sjson.Set(_TwitterJson, path, math.MaxInt32)
}
}
func BenchmarkSetOne_Jsoniter(b *testing.B) {
data := []byte(_TwitterJson)
node, ok := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
if !ok {
b.Fatal(node)
}
b.SetBytes(int64(len(data)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
node, _ := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
node["id"] = math.MaxInt32
}
}
func BenchmarkSetOne_Parallel_Sonic(b *testing.B) {
node, err := NewSearcher(_TwitterJson).GetByPath("statuses", 3)
if err != nil {
@ -342,38 +258,4 @@ func BenchmarkSetOne_Parallel_Sonic(b *testing.B) {
_, _ = node.Set("id", n)
}
})
}
func BenchmarkSetOne_Parallel_Sjson(b *testing.B) {
path := fmt.Sprintf("%s.%d.%s", "statuses", 3, "id")
_, err := sjson.Set(_TwitterJson, path, math.MaxInt32)
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(_TwitterJson)))
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
sjson.Set(_TwitterJson, path, math.MaxInt32)
}
})
}
func BenchmarkSetOne_Parallel_Jsoniter(b *testing.B) {
data := []byte(_TwitterJson)
node, ok := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
if !ok {
b.Fatal(node)
}
b.SetBytes(int64(len(data)))
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
node, _ := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
node["id"] = math.MaxInt32
}
})
}

7
bench.sh
View file

@ -16,5 +16,12 @@ go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*|Benchma
go test -benchmem -run=^$ -benchtime=10000000x -bench "^(BenchmarkNodeGetByPath|BenchmarkStructGetByPath|BenchmarkNodeIndex|BenchmarkStructIndex|BenchmarkSliceIndex|BenchmarkMapIndex|BenchmarkNodeGet|BenchmarkSliceGet|BenchmarkMapGet|BenchmarkNodeSet|BenchmarkMapSet|BenchmarkNodeSetByIndex|BenchmarkSliceSetByIndex|BenchmarkStructSetByIndex|BenchmarkNodeUnset|BenchmarkMapUnset|BenchmarkNodUnsetByIndex|BenchmarkSliceUnsetByIndex|BenchmarkNodeAdd|BenchmarkSliceAdd|BenchmarkMapAdd)$"
cd $pwd/external_jsonlib_test/benchmark_test
go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkEncoder_.*|BenchmarkDecoder_.*)$"
go test -benchmem -run=^$ -benchtime=1000000x -bench "^(BenchmarkGet.*|BenchmarkSet.*)$"
go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*)$"
unset SONIC_NO_ASYNC_GC
cd $pwd

106
decoder/decoder_test.go
View file

@ -27,8 +27,6 @@ import (
`github.com/bytedance/sonic/internal/rt`
`github.com/davecgh/go-spew/spew`
gojson `github.com/goccy/go-json`
`github.com/json-iterator/go`
`github.com/stretchr/testify/assert`
`github.com/stretchr/testify/require`
)
@ -328,30 +326,6 @@ func BenchmarkDecoder_Generic_StdLib(b *testing.B) {
}
}
func BenchmarkDecoder_Generic_JsonIter(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
_ = jsoniter.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Generic_GoJson(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
_ = gojson.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Binding_Sonic(b *testing.B) {
var w TwitterStruct
_, _ = decode(TwitterJson, &w, true)
@ -386,30 +360,6 @@ func BenchmarkDecoder_Binding_StdLib(b *testing.B) {
}
}
func BenchmarkDecoder_Binding_JsonIter(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v TwitterStruct
_ = jsoniter.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Binding_GoJson(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v TwitterStruct
_ = gojson.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Parallel_Generic_Sonic(b *testing.B) {
var w interface{}
_, _ = decode(TwitterJson, &w, true)
@ -450,34 +400,6 @@ func BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) {
})
}
func BenchmarkDecoder_Parallel_Generic_JsonIter(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v interface{}
_ = jsoniter.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Generic_GoJson(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v interface{}
_ = gojson.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Binding_Sonic(b *testing.B) {
var w TwitterStruct
_, _ = decode(TwitterJson, &w, true)
@ -518,34 +440,6 @@ func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {
})
}
func BenchmarkDecoder_Parallel_Binding_JsonIter(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v TwitterStruct
_ = jsoniter.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Binding_GoJson(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v TwitterStruct
_ = gojson.Unmarshal(m, &v)
}
})
}
func BenchmarkSkip_Sonic(b *testing.B) {
var data = rt.Str2Mem(TwitterJson)
if ret, _ := Skip(data); ret < 0 {

185
decoder/stream_test.go
View file

@ -24,7 +24,6 @@ import (
`strings`
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/assert`
`github.com/stretchr/testify/require`
)
@ -83,76 +82,6 @@ func TestDecodeEmpty(t *testing.T) {
assert.Equal(t, v1, v2)
}
func TestDecodeSingle(t *testing.T) {
var str = _Single_JSON
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var r2 = strings.NewReader(str)
var v2 map[string]interface{}
var d2 = NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
es1 := d1.Decode(&v1)
ee1 := d2.Decode(&v2)
assert.Equal(t, es1, ee1)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
}
func TestDecodeMulti(t *testing.T) {
var str = _Triple_JSON
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var r2 = strings.NewReader(str)
var v2 map[string]interface{}
var d2 = NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
es1 := d1.Decode(&v1)
ee1 := d2.Decode(&v2)
assert.Equal(t, es1, ee1)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
// fmt.Printf("v:%#v\n", v1)
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
}
type HaltReader struct {
halts map[int]bool
buf string
@ -199,49 +128,6 @@ var testHalts = func () map[int]bool {
20: true}
}
func TestDecodeHalt(t *testing.T) {
var str = _Triple_JSON
var r1 = NewHaltReader(str, testHalts())
var r2 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
var v2 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var d2 = NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
err1 := d1.Decode(&v1)
err2 := d2.Decode(&v2)
assert.Equal(t, err1, err2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
}
func TestBuffered(t *testing.T) {
var str = _Triple_JSON
var r1 = NewHaltReader(str, testHalts())
@ -274,43 +160,6 @@ func TestBuffered(t *testing.T) {
assert.Equal(t, d1.InputOffset(), d2.InputOffset())
}
func TestMore(t *testing.T) {
var str = _Triple_JSON
var r2 = NewHaltReader(str, testHalts())
var v2 map[string]interface{}
var d2 = NewStreamDecoder(r2)
var r1 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
require.Nil(t, d1.Decode(&v1))
require.Nil(t, d2.Decode(&v2))
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
require.Equal(t, d1.More(), d2.More())
}
func BenchmarkDecodeStream_Std(b *testing.B) {
b.Run("single", func (b *testing.B) {
var str = _Single_JSON
@ -345,40 +194,6 @@ func BenchmarkDecodeStream_Std(b *testing.B) {
})
}
func BenchmarkDecodeStream_Jsoniter(b *testing.B) {
b.Run("single", func (b *testing.B) {
var str = _Single_JSON
for i:=0; i<b.N; i++ {
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
_ = dc.Decode(&v1)
}
})
b.Run("double", func (b *testing.B) {
var str = _Double_JSON
for i:=0; i<b.N; i++ {
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
_ = dc.Decode(&v1)
}
})
b.Run("halt", func (b *testing.B) {
var str = _Double_JSON
for i:=0; i<b.N; i++ {
var r1 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
}
})
}
// func BenchmarkDecodeError_Sonic(b *testing.B) {
// var str = `\b测试1234`
// for i:=0; i<b.N; i++ {

92
encoder/encoder_test.go
View file

@ -27,20 +27,18 @@ import (
`time`
`github.com/bytedance/sonic/internal/rt`
gojson `github.com/goccy/go-json`
`github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
func TestMain(m *testing.M) {
go func () {
if !debugAsyncGC {
return
return
}
println("Begin GC looping...")
for {
runtime.GC()
debug.FreeOSMemory()
debug.FreeOSMemory()
}
println("stop GC looping!")
}()
@ -50,7 +48,7 @@ func TestMain(m *testing.M) {
func TestGC(t *testing.T) {
if debugSyncGC {
return
return
}
out, err := Encode(_GenericValue, 0)
if err != nil {
@ -92,7 +90,7 @@ func TestOptionSliceOrMapNoNull(t *testing.T) {
t.Fatal(err)
}
require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
obj2 := sample{}
out, err = Encode(obj2, 0)
if err != nil {
@ -113,7 +111,7 @@ func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
_, _ = Encode(obj, NoNullSliceOrMap)
}
})
b.Run("false", func (b *testing.B) {
obj2 := sample{}
_, err := Encode(obj2, 0)
@ -356,24 +354,6 @@ func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) {
}
}
func BenchmarkEncoder_Generic_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = jsoniter.Marshal(_GenericValue)
}
}
func BenchmarkEncoder_Generic_GoJson(b *testing.B) {
_, _ = gojson.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = gojson.Marshal(_GenericValue)
}
}
func BenchmarkEncoder_Generic_StdLib(b *testing.B) {
_, _ = json.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
@ -401,24 +381,6 @@ func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) {
}
}
func BenchmarkEncoder_Binding_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = jsoniter.Marshal(&_BindingValue)
}
}
func BenchmarkEncoder_Binding_GoJson(b *testing.B) {
_, _ = gojson.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = gojson.Marshal(&_BindingValue)
}
}
func BenchmarkEncoder_Binding_StdLib(b *testing.B) {
_, _ = json.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
@ -450,28 +412,6 @@ func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
})
}
func BenchmarkEncoder_Parallel_Generic_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = jsoniter.Marshal(_GenericValue)
}
})
}
func BenchmarkEncoder_Parallel_Generic_GoJson(b *testing.B) {
_, _ = gojson.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = gojson.Marshal(_GenericValue)
}
})
}
func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) {
_, _ = json.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
@ -505,28 +445,6 @@ func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
})
}
func BenchmarkEncoder_Parallel_Binding_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = jsoniter.Marshal(&_BindingValue)
}
})
}
func BenchmarkEncoder_Parallel_Binding_GoJson(b *testing.B) {
_, _ = gojson.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = gojson.Marshal(&_BindingValue)
}
})
}
func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) {
_, _ = json.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))

50
encoder/stream_test.go
View file

@ -22,7 +22,6 @@ import (
`strings`
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
@ -138,53 +137,4 @@ func BenchmarkEncodeStream_Std(b *testing.B) {
w.Reset()
}
})
}
func BenchmarkEncodeStream_Jsoniter(b *testing.B) {
var o = map[string]interface{}{
"a": `<`+strings.Repeat("1", 1024)+`>`,
"b": json.RawMessage(` [ `+strings.Repeat(" ", 1024)+` ] `),
}
b.Run("single", func(b *testing.B){
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i:=0; i<b.N; i++ {
_ = enc.Encode(o)
w.Reset()
}
})
b.Run("double", func(b *testing.B){
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i:=0; i<b.N; i++ {
_ = enc.Encode(o)
_ = enc.Encode(o)
w.Reset()
}
})
b.Run("compatible", func(b *testing.B){
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
EscapeHTML: true,
SortMapKeys: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i:=0; i<b.N; i++ {
_ = enc.Encode(o)
w.Reset()
}
})
}

111
external_jsonlib_test/benchmark_test/decoder_stream_test.go Normal file
View file

@ -0,0 +1,111 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
"io"
`strings`
`testing`
jsoniter `github.com/json-iterator/go`
)
var (
defaultBufferSize uint = 4096
_Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} { `
_Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} {"11111":"` + strings.Repeat("2", int(defaultBufferSize)) + `"}`
)
type HaltReader struct {
halts map[int]bool
buf string
p int
}
func NewHaltReader(buf string, halts map[int]bool) *HaltReader {
return &HaltReader{
halts: halts,
buf: buf,
p: 0,
}
}
func (self *HaltReader) Read(p []byte) (int, error) {
t := 0
for ; t < len(p); {
if self.p >= len(self.buf) {
return t, io.EOF
}
if b, ok := self.halts[self.p]; b {
self.halts[self.p] = false
return t, nil
} else if ok {
delete(self.halts, self.p)
return 0, nil
}
p[t] = self.buf[self.p]
self.p++
t++
}
return t, nil
}
func (self *HaltReader) Reset(buf string) {
self.p = 0
self.buf = buf
}
var testHalts = func() map[int]bool {
return map[int]bool{
1: true,
10: true,
20: true}
}
func BenchmarkDecodeStream_Jsoniter(b *testing.B) {
b.Run("single", func(b *testing.B) {
var str = _Single_JSON
for i := 0; i < b.N; i++ {
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
_ = dc.Decode(&v1)
}
})
b.Run("double", func(b *testing.B) {
var str = _Double_JSON
for i := 0; i < b.N; i++ {
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
_ = dc.Decode(&v1)
}
})
b.Run("halt", func(b *testing.B) {
var str = _Double_JSON
for i := 0; i < b.N; i++ {
var r1 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
dc := jsoniter.NewDecoder(r1)
_ = dc.Decode(&v1)
}
})
}

185
external_jsonlib_test/benchmark_test/decoder_test.go Normal file
View file

@ -0,0 +1,185 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
`encoding/json`
`testing`
gojson `github.com/goccy/go-json`
jsoniter "github.com/json-iterator/go"
)
func init() {
_ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
}
func BenchmarkDecoder_Generic_StdLib(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = json.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
_ = json.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Generic_JsonIter(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
_ = jsoniter.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Generic_GoJson(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v interface{}
_ = gojson.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Binding_StdLib(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = json.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v TwitterStruct
_ = json.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Binding_JsonIter(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v TwitterStruct
_ = jsoniter.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Binding_GoJson(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
var v TwitterStruct
_ = gojson.Unmarshal(m, &v)
}
}
func BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = json.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v interface{}
_ = json.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Generic_JsonIter(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v interface{}
_ = jsoniter.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Generic_GoJson(b *testing.B) {
var w interface{}
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v interface{}
_ = gojson.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = json.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v TwitterStruct
_ = json.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Binding_JsonIter(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = jsoniter.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v TwitterStruct
_ = jsoniter.Unmarshal(m, &v)
}
})
}
func BenchmarkDecoder_Parallel_Binding_GoJson(b *testing.B) {
var w TwitterStruct
m := []byte(TwitterJson)
_ = gojson.Unmarshal(m, &w)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var v TwitterStruct
_ = gojson.Unmarshal(m, &v)
}
})
}

75
external_jsonlib_test/benchmark_test/encoder_stream_test.go Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
`bytes`
`encoding/json`
`strings`
`testing`
jsoniter `github.com/json-iterator/go`
)
func BenchmarkEncodeStream_Jsoniter(b *testing.B) {
var o = map[string]interface{}{
"a": `<` + strings.Repeat("1", 1024) + `>`,
"b": json.RawMessage(` [ ` + strings.Repeat(" ", 1024) + ` ] `),
}
b.Run("single", func(b *testing.B) {
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(o)
w.Reset()
}
})
b.Run("double", func(b *testing.B) {
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(o)
_ = enc.Encode(o)
w.Reset()
}
})
b.Run("compatible", func(b *testing.B) {
var w = bytes.NewBuffer(nil)
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
EscapeHTML: true,
SortMapKeys: true,
}.Froze()
var enc = jt.NewEncoder(w)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(o)
w.Reset()
}
})
}

113
external_jsonlib_test/benchmark_test/encoder_test.go Normal file
View file

@ -0,0 +1,113 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
`encoding/json`
`testing`
gojson `github.com/goccy/go-json`
jsoniter `github.com/json-iterator/go`
)
var _GenericValue interface{}
var _BindingValue TwitterStruct
func init() {
_ = json.Unmarshal([]byte(TwitterJson), &_GenericValue)
_ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
}
func BenchmarkEncoder_Generic_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = jsoniter.Marshal(_GenericValue)
}
}
func BenchmarkEncoder_Generic_GoJson(b *testing.B) {
_, _ = gojson.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = gojson.Marshal(_GenericValue)
}
}
func BenchmarkEncoder_Binding_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = jsoniter.Marshal(&_BindingValue)
}
}
func BenchmarkEncoder_Binding_GoJson(b *testing.B) {
_, _ = gojson.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = gojson.Marshal(&_BindingValue)
}
}
func BenchmarkEncoder_Parallel_Generic_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = jsoniter.Marshal(_GenericValue)
}
})
}
func BenchmarkEncoder_Parallel_Generic_GoJson(b *testing.B) {
_, _ = gojson.Marshal(_GenericValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = gojson.Marshal(_GenericValue)
}
})
}
func BenchmarkEncoder_Parallel_Binding_JsonIter(b *testing.B) {
_, _ = jsoniter.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = jsoniter.Marshal(&_BindingValue)
}
})
}
func BenchmarkEncoder_Parallel_Binding_GoJson(b *testing.B) {
_, _ = gojson.Marshal(&_BindingValue)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = gojson.Marshal(&_BindingValue)
}
})
}

236
external_jsonlib_test/benchmark_test/parser_test.go Normal file
View file

@ -0,0 +1,236 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/tidwall/gjson`
)
func BenchmarkParser_Gjson(b *testing.B) {
gjson.Parse(TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
gjson.Parse(TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
}
}
func BenchmarkParser_Jsoniter(b *testing.B) {
v := jsoniter.Get([]byte(TwitterJson)).GetInterface()
if v == nil {
b.Fatal(v)
}
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = jsoniter.Get([]byte(TwitterJson)).GetInterface()
}
}
func BenchmarkParser_Parallel_Gjson(b *testing.B) {
gjson.Parse(TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
return true
})
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
gjson.Parse(TwitterJson).ForEach(func(key, value gjson.Result) bool {
if !value.Exists() {
b.Fatal(value.Index)
}
_ = value.Value()
return true
})
}
})
}
func BenchmarkParser_Parallel_Jsoniter(b *testing.B) {
var bv = []byte(TwitterJson)
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var out interface{}
_ = jsoniter.Unmarshal(bv, &out)
}
})
}
func BenchmarkParseOne_Gjson(b *testing.B) {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.2.id")
v := node.Int()
if v != 249289491129438208 {
b.Fatal(node)
}
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.2.id")
_ = node.Int()
}
}
func BenchmarkParseOne_Jsoniter(b *testing.B) {
data := []byte(TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
node := ast.ToInt()
if node != 249289491129438208 {
b.Fail()
}
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data, "statuses", 2, "id")
_ = ast.ToInt()
}
}
func BenchmarkParseOne_Parallel_Gjson(b *testing.B) {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.2.id")
v := node.Int()
if v != 249289491129438208 {
b.Fatal(node)
}
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.2.id")
_ = node.Int()
}
})
}
func BenchmarkParseOne_Parallel_Jsoniter(b *testing.B) {
data := []byte(TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
node := ast.ToInt()
if node != 249289491129438208 {
b.Fail()
}
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
data := []byte(TwitterJson)
ast := jsoniter.Get(data, "statuses", 2, "id")
_ = ast.ToInt()
}
})
}
func BenchmarkParseSeven_Gjson(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.3.id")
node = ast.Get("statuses.3.user.entities.description")
node = ast.Get("statuses.3.user.entities.url.urls")
node = ast.Get("statuses.3.user.entities.url")
node = ast.Get("statuses.3.user.created_at")
node = ast.Get("statuses.3.user.name")
node = ast.Get("statuses.3.text")
if node.Value() == nil {
b.Fail()
}
}
}
func BenchmarkParseSeven_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
data := []byte(TwitterJson)
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data)
node := ast.Get("statuses", 3, "id")
node = ast.Get("statuses", 3, "user", "entities", "description")
node = ast.Get("statuses", 3, "user", "entities", "url", "urls")
node = ast.Get("statuses", 3, "user", "entities", "url")
node = ast.Get("statuses", 3, "user", "created_at")
node = ast.Get("statuses", 3, "user", "name")
node = ast.Get("statuses", 3, "text")
if node.LastError() != nil {
b.Fail()
}
}
}
func BenchmarkParseSeven_Parallel_Gjson(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Parse(TwitterJson)
node := ast.Get("statuses.3.id")
node = ast.Get("statuses.3.user.entities.description")
node = ast.Get("statuses.3.user.entities.url.urls")
node = ast.Get("statuses.3.user.entities.url")
node = ast.Get("statuses.3.user.created_at")
node = ast.Get("statuses.3.user.name")
node = ast.Get("statuses.3.text")
if node.Value() == nil {
b.Fail()
}
}
})
}
func BenchmarkParseSeven_Parallel_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
data := []byte(TwitterJson)
ast := jsoniter.Get(data)
node := ast.Get("statuses", 3, "id")
node = ast.Get("statuses", 3, "user", "entities", "description")
node = ast.Get("statuses", 3, "user", "entities", "url", "urls")
node = ast.Get("statuses", 3, "user", "entities", "url")
node = ast.Get("statuses", 3, "user", "created_at")
node = ast.Get("statuses", 3, "user", "name")
node = ast.Get("statuses", 3, "text")
if node.LastError() != nil {
b.Fail()
}
}
})
}

141
external_jsonlib_test/benchmark_test/search_test.go Normal file
View file

@ -0,0 +1,141 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
import (
`fmt`
`math`
`testing`
jsoniter `github.com/json-iterator/go`
`github.com/tidwall/gjson`
`github.com/tidwall/sjson`
)
func BenchmarkGetOne_Gjson(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
for i := 0; i < b.N; i++ {
ast := gjson.Get(TwitterJson, "statuses.3.id")
node := ast.Int()
if node != 249279667666817024 {
b.Fail()
}
}
}
func BenchmarkGetOne_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
data := []byte(TwitterJson)
for i := 0; i < b.N; i++ {
ast := jsoniter.Get(data, "statuses", 3, "id")
node := ast.ToInt()
if node != 249279667666817024 {
b.Fail()
}
}
}
func BenchmarkGetOne_Parallel_Gjson(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := gjson.Get(TwitterJson, "statuses.3.id")
node := ast.Int()
if node != 249279667666817024 {
b.Fail()
}
}
})
}
func BenchmarkGetOne_Parallel_Jsoniter(b *testing.B) {
b.SetBytes(int64(len(TwitterJson)))
data := []byte(TwitterJson)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ast := jsoniter.Get(data, "statuses", 3, "id")
node := ast.ToInt()
if node != 249279667666817024 {
b.Fail()
}
}
})
}
func BenchmarkSetOne_Sjson(b *testing.B) {
path := fmt.Sprintf("%s.%d.%s", "statuses", 3, "id")
_, err := sjson.Set(TwitterJson, path, math.MaxInt32)
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(TwitterJson)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
sjson.Set(TwitterJson, path, math.MaxInt32)
}
}
func BenchmarkSetOne_Jsoniter(b *testing.B) {
data := []byte(TwitterJson)
node, ok := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
if !ok {
b.Fatal(node)
}
b.SetBytes(int64(len(data)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
node, _ := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
node["id"] = math.MaxInt32
}
}
func BenchmarkSetOne_Parallel_Sjson(b *testing.B) {
path := fmt.Sprintf("%s.%d.%s", "statuses", 3, "id")
_, err := sjson.Set(TwitterJson, path, math.MaxInt32)
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(TwitterJson)))
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
sjson.Set(TwitterJson, path, math.MaxInt32)
}
})
}
func BenchmarkSetOne_Parallel_Jsoniter(b *testing.B) {
data := []byte(TwitterJson)
node, ok := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
if !ok {
b.Fatal(node)
}
b.SetBytes(int64(len(data)))
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
node, _ := jsoniter.Get(data, "statuses", 3).GetInterface().(map[string]interface{})
node["id"] = math.MaxInt32
}
})
}

551
external_jsonlib_test/benchmark_test/testdata_test.go Normal file
View file

@ -0,0 +1,551 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package benchmark_test
const TwitterJson = `{
"statuses": [
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Mon Sep 24 03:35:21 +0000 2012",
"id_str": "250075927172759552",
"entities": {
"urls": [
],
"hashtags": [
{
"text": "freebandnames",
"indices": [
20,
34
]
}
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "Aggressive Ponytail #freebandnames",
"metadata": {
"iso_language_code": "en",
"result_type": "recent"
},
"retweet_count": 0,
"in_reply_to_status_id_str": null,
"id": 250075927172759552,
"geo": null,
"retweeted": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "DDEEF6",
"profile_sidebar_border_color": "C0DEED",
"profile_background_tile": false,
"name": "Sean Cummings",
"profile_image_url": "https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
"created_at": "Mon Apr 26 06:01:55 +0000 2010",
"location": "LA, CA",
"follow_request_sent": null,
"profile_link_color": "0084B4",
"is_translator": false,
"id_str": "137238150",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "",
"indices": [
0,
0
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": true,
"contributors_enabled": false,
"favourites_count": 0,
"url": null,
"profile_image_url_https": "https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg",
"utc_offset": -28800,
"id": 137238150,
"profile_use_background_image": true,
"listed_count": 2,
"profile_text_color": "333333",
"lang": "en",
"followers_count": 70,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png",
"profile_background_color": "C0DEED",
"verified": false,
"geo_enabled": true,
"time_zone": "Pacific Time (US & Canada)",
"description": "Born 330 Live 310",
"default_profile_image": false,
"profile_background_image_url": "https://a0.twimg.com/images/themes/theme1/bg.png",
"statuses_count": 579,
"friends_count": 110,
"following": null,
"show_all_inline_media": false,
"screen_name": "sean_cummings"
},
"in_reply_to_screen_name": null,
"source": "<a href=\"//itunes.apple.com/us/app/twitter/id409789998?mt=12%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for Mac</a>",
"in_reply_to_status_id": null
},
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Fri Sep 21 23:40:54 +0000 2012",
"id_str": "249292149810667520",
"entities": {
"urls": [
],
"hashtags": [
{
"text": "FreeBandNames",
"indices": [
20,
34
]
}
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "Thee Namaste Nerdz. #FreeBandNames",
"metadata": {
"iso_language_code": "pl",
"result_type": "recent"
},
"retweet_count": 0,
"in_reply_to_status_id_str": null,
"id": 249292149810667520,
"geo": null,
"retweeted": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "DDFFCC",
"profile_sidebar_border_color": "BDDCAD",
"profile_background_tile": true,
"name": "Chaz Martenstein",
"profile_image_url": "https://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
"created_at": "Tue Apr 07 19:05:07 +0000 2009",
"location": "Durham, NC",
"follow_request_sent": null,
"profile_link_color": "0084B4",
"is_translator": false,
"id_str": "29516238",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "https://bullcityrecords.com/wnng/",
"indices": [
0,
32
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": false,
"contributors_enabled": false,
"favourites_count": 8,
"url": "https://bullcityrecords.com/wnng/",
"profile_image_url_https": "https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg",
"utc_offset": -18000,
"id": 29516238,
"profile_use_background_image": true,
"listed_count": 118,
"profile_text_color": "333333",
"lang": "en",
"followers_count": 2052,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp",
"profile_background_color": "9AE4E8",
"verified": false,
"geo_enabled": false,
"time_zone": "Eastern Time (US & Canada)",
"description": "You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.",
"default_profile_image": false,
"profile_background_image_url": "https://a0.twimg.com/profile_background_images/9423277/background_tile.bmp",
"statuses_count": 7579,
"friends_count": 348,
"following": null,
"show_all_inline_media": true,
"screen_name": "bullcityrecords"
},
"in_reply_to_screen_name": null,
"source": "web",
"in_reply_to_status_id": null
},
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Fri Sep 21 23:30:20 +0000 2012",
"id_str": "249289491129438208",
"entities": {
"urls": [
],
"hashtags": [
{
"text": "freebandnames",
"indices": [
29,
43
]
}
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "Mexican Heaven, Mexican Hell #freebandnames",
"metadata": {
"iso_language_code": "en",
"result_type": "recent"
},
"retweet_count": 0,
"in_reply_to_status_id_str": null,
"id": 249289491129438208,
"geo": null,
"retweeted": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "99CC33",
"profile_sidebar_border_color": "829D5E",
"profile_background_tile": false,
"name": "Thomas John Wakeman",
"profile_image_url": "https://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
"created_at": "Tue Sep 01 21:21:35 +0000 2009",
"location": "Kingston New York",
"follow_request_sent": null,
"profile_link_color": "D02B55",
"is_translator": false,
"id_str": "70789458",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "",
"indices": [
0,
0
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": false,
"contributors_enabled": false,
"favourites_count": 19,
"url": null,
"profile_image_url_https": "https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png",
"utc_offset": -18000,
"id": 70789458,
"profile_use_background_image": true,
"listed_count": 1,
"profile_text_color": "3E4415",
"lang": "en",
"followers_count": 63,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif",
"profile_background_color": "352726",
"verified": false,
"geo_enabled": false,
"time_zone": "Eastern Time (US & Canada)",
"description": "Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.",
"default_profile_image": false,
"profile_background_image_url": "https://a0.twimg.com/images/themes/theme5/bg.gif",
"statuses_count": 1048,
"friends_count": 63,
"following": null,
"show_all_inline_media": false,
"screen_name": "MonkiesFist"
},
"in_reply_to_screen_name": null,
"source": "web",
"in_reply_to_status_id": null
},
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Fri Sep 21 22:51:18 +0000 2012",
"id_str": "249279667666817024",
"entities": {
"urls": [
],
"hashtags": [
{
"text": "freebandnames",
"indices": [
20,
34
]
}
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "The Foolish Mortals #freebandnames",
"metadata": {
"iso_language_code": "en",
"result_type": "recent"
},
"retweet_count": 0,
"in_reply_to_status_id_str": null,
"id": 249279667666817024,
"geo": null,
"retweeted": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "BFAC83",
"profile_sidebar_border_color": "615A44",
"profile_background_tile": true,
"name": "Marty Elmer",
"profile_image_url": "https://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
"created_at": "Mon May 04 00:05:00 +0000 2009",
"location": "Wisconsin, USA",
"follow_request_sent": null,
"profile_link_color": "3B2A26",
"is_translator": false,
"id_str": "37539828",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "https://www.omnitarian.me",
"indices": [
0,
24
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": false,
"contributors_enabled": false,
"favourites_count": 647,
"url": "https://www.omnitarian.me",
"profile_image_url_https": "https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png",
"utc_offset": -21600,
"id": 37539828,
"profile_use_background_image": true,
"listed_count": 52,
"profile_text_color": "000000",
"lang": "en",
"followers_count": 608,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png",
"profile_background_color": "EEE3C4",
"verified": false,
"geo_enabled": false,
"time_zone": "Central Time (US & Canada)",
"description": "Cartoonist, Illustrator, and T-Shirt connoisseur",
"default_profile_image": false,
"profile_background_image_url": "https://a0.twimg.com/profile_background_images/106455659/rect6056-9.png",
"statuses_count": 3575,
"friends_count": 249,
"following": null,
"show_all_inline_media": true,
"screen_name": "Omnitarian"
},
"in_reply_to_screen_name": null,
"source": "<a href=\"//twitter.com/download/iphone%5C%22\" rel=\"\\\"nofollow\\\"\">Twitter for iPhone</a>",
"in_reply_to_status_id": null
}
],
"search_metadata": {
"max_id": 250126199840518145,
"since_id": 24012619984051000,
"refresh_url": "?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1",
"next_results": "?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed",
"count": 4,
"completed_in": 0.035,
"since_id_str": "24012619984051000",
"query": "%23freebandnames",
"max_id_str": "250126199840518145"
}
}`
type TwitterStruct struct {
Statuses []Statuses `json:"statuses"`
SearchMetadata SearchMetadata `json:"search_metadata"`
}
type Hashtags struct {
Text string `json:"text"`
Indices []int `json:"indices"`
}
type Entities struct {
Urls []interface{} `json:"urls"`
Hashtags []Hashtags `json:"hashtags"`
UserMentions []interface{} `json:"user_mentions"`
}
type Metadata struct {
IsoLanguageCode string `json:"iso_language_code"`
ResultType string `json:"result_type"`
}
type Urls struct {
ExpandedURL interface{} `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
}
type URL struct {
Urls []Urls `json:"urls"`
}
type Description struct {
Urls []interface{} `json:"urls"`
}
type UserEntities struct {
URL URL `json:"url"`
Description Description `json:"description"`
}
type User struct {
ProfileSidebarFillColor string `json:"profile_sidebar_fill_color"`
ProfileSidebarBorderColor string `json:"profile_sidebar_border_color"`
ProfileBackgroundTile bool `json:"profile_background_tile"`
Name string `json:"name"`
ProfileImageURL string `json:"profile_image_url"`
CreatedAt string `json:"created_at"`
Location string `json:"location"`
FollowRequestSent interface{} `json:"follow_request_sent"`
ProfileLinkColor string `json:"profile_link_color"`
IsTranslator bool `json:"is_translator"`
IDStr string `json:"id_str"`
Entities UserEntities `json:"entities"`
DefaultProfile bool `json:"default_profile"`
ContributorsEnabled bool `json:"contributors_enabled"`
FavouritesCount int `json:"favourites_count"`
URL interface{} `json:"url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
UtcOffset int `json:"utc_offset"`
ID int `json:"id"`
ProfileUseBackgroundImage bool `json:"profile_use_background_image"`
ListedCount int `json:"listed_count"`
ProfileTextColor string `json:"profile_text_color"`
Lang string `json:"lang"`
FollowersCount int `json:"followers_count"`
Protected bool `json:"protected"`
Notifications interface{} `json:"notifications"`
ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https"`
ProfileBackgroundColor string `json:"profile_background_color"`
Verified bool `json:"verified"`
GeoEnabled bool `json:"geo_enabled"`
TimeZone string `json:"time_zone"`
Description string `json:"description"`
DefaultProfileImage bool `json:"default_profile_image"`
ProfileBackgroundImageURL string `json:"profile_background_image_url"`
StatusesCount int `json:"statuses_count"`
FriendsCount int `json:"friends_count"`
Following interface{} `json:"following"`
ShowAllInlineMedia bool `json:"show_all_inline_media"`
ScreenName string `json:"screen_name"`
}
type Statuses struct {
Coordinates interface{} `json:"coordinates"`
Favorited bool `json:"favorited"`
Truncated bool `json:"truncated"`
CreatedAt string `json:"created_at"`
IDStr string `json:"id_str"`
Entities Entities `json:"entities"`
InReplyToUserIDStr interface{} `json:"in_reply_to_user_id_str"`
Contributors interface{} `json:"contributors"`
Text string `json:"text"`
Metadata Metadata `json:"metadata"`
RetweetCount int `json:"retweet_count"`
InReplyToStatusIDStr interface{} `json:"in_reply_to_status_id_str"`
ID int64 `json:"id"`
Geo interface{} `json:"geo"`
Retweeted bool `json:"retweeted"`
InReplyToUserID interface{} `json:"in_reply_to_user_id"`
Place interface{} `json:"place"`
User User `json:"user"`
InReplyToScreenName interface{} `json:"in_reply_to_screen_name"`
Source string `json:"source"`
InReplyToStatusID interface{} `json:"in_reply_to_status_id"`
}
type SearchMetadata struct {
MaxID int64 `json:"max_id"`
SinceID int64 `json:"since_id"`
RefreshURL string `json:"refresh_url"`
NextResults string `json:"next_results"`
Count int `json:"count"`
CompletedIn float64 `json:"completed_in"`
SinceIDStr string `json:"since_id_str"`
Query string `json:"query"`
MaxIDStr string `json:"max_id_str"`
}

26
external_jsonlib_test/go.mod Normal file
View file

@ -0,0 +1,26 @@
module github.com/bytedance/sonic/external_jsonlib_test
go 1.18
require (
github.com/bytedance/sonic v1.4.0
github.com/goccy/go-json v0.9.11
github.com/json-iterator/go v1.1.12
github.com/stretchr/testify v1.8.0
github.com/tidwall/gjson v1.14.3
github.com/tidwall/sjson v1.2.5
)
require (
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

50
external_jsonlib_test/go.sum Normal file
View file

@ -0,0 +1,50 @@
github.com/bytedance/sonic v1.4.0 h1:d6vgPhwgHfpmEiz/9Fzea9fGzWY7RO1TQEySBiRwDLY=
github.com/bytedance/sonic v1.4.0/go.mod h1:V973WhNhGmvHxW6nQmsHEfHaoU9F3zTF+93rH03hcUQ=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 h1:1sDoSuDPWzhkdzNVxCxtIaKiAe96ESVPv8coGwc1gZ4=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

51
api_test.go → external_jsonlib_test/unit_test/api_test.go
View file

@ -14,13 +14,14 @@
* limitations under the License.
*/
package sonic
package unit_test
import (
`bytes`
`encoding/json`
`testing`
`github.com/bytedance/sonic`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
@ -33,7 +34,7 @@ func TestCompatMarshalDefault(t *testing.T){
var obj = map[string]interface{}{
"c": json.RawMessage("[\"<&>\"]"),
}
sout, serr := ConfigDefault.Marshal(obj)
sout, serr := sonic.ConfigDefault.Marshal(obj)
jout, jerr := jt.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
@ -41,18 +42,18 @@ func TestCompatMarshalDefault(t *testing.T){
// obj = map[string]interface{}{
// "a": json.RawMessage(" [} "),
// }
// sout, serr = ConfigDefault.Marshal(obj)
// sout, serr = sonic.ConfigDefault.Marshal(obj)
// jout, jerr = json.Marshal(obj)
// require.NotNil(t, jerr)
// require.NotNil(t, serr)
// require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
// require.Equal(t, string(jout), string(sout))
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigDefault.MarshalIndent(obj, "", " ")
sout, serr = sonic.ConfigDefault.MarshalIndent(obj, "", " ")
jout, jerr = jt.MarshalIndent(obj, "", " ")
require.Equal(t, jerr, serr)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
}
@ -62,7 +63,7 @@ func TestCompatMarshalStd(t *testing.T) {
"c": json.RawMessage(" [ \"<&>\" ] "),
"b": json.RawMessage(" [ ] "),
}
sout, serr := ConfigStd.Marshal(obj)
sout, serr := sonic.ConfigStd.Marshal(obj)
jout, jerr := json.Marshal(obj)
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
@ -70,7 +71,7 @@ func TestCompatMarshalStd(t *testing.T) {
obj = map[string]interface{}{
"a": json.RawMessage(" [} "),
}
sout, serr = ConfigStd.Marshal(obj)
sout, serr = sonic.ConfigStd.Marshal(obj)
jout, jerr = json.Marshal(obj)
require.NotNil(t, jerr)
require.NotNil(t, serr)
@ -79,7 +80,7 @@ func TestCompatMarshalStd(t *testing.T) {
obj = map[string]interface{}{
"a": json.RawMessage("1"),
}
sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ")
sout, serr = sonic.ConfigStd.MarshalIndent(obj, "xxxx", " ")
jout, jerr = json.MarshalIndent(obj, "xxxx", " ")
require.Equal(t, jerr, serr)
require.Equal(t, string(jout), string(sout))
@ -90,15 +91,15 @@ func TestCompatUnmarshalDefault(t *testing.T) {
var jobj = map[string]interface{}{}
var data = []byte(`{"a":-0}`)
var str = string(data)
serr := ConfigDefault.UnmarshalFromString(str, &sobj)
serr := sonic.ConfigDefault.UnmarshalFromString(str, &sobj)
jerr := jt.UnmarshalFromString(str, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
require.Equal(t, jobj, sobj)
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
serr = ConfigDefault.Unmarshal(data, &x)
serr = sonic.ConfigDefault.Unmarshal(data, &x)
jerr = jt.Unmarshal(data, &y)
require.Equal(t, jerr, serr)
require.Equal(t, y, x)
@ -109,7 +110,7 @@ func TestCompatUnmarshalStd(t *testing.T) {
var jobj = map[string]interface{}{}
var data = []byte(`{"a":1.00000001E-10}`)
var str = string(data)
serr := ConfigStd.UnmarshalFromString(str, &sobj)
serr := sonic.ConfigStd.UnmarshalFromString(str, &sobj)
jerr := json.Unmarshal(data, &jobj)
require.Equal(t, jerr, serr)
require.Equal(t, jobj, sobj)
@ -119,7 +120,7 @@ func TestCompatUnmarshalStd(t *testing.T) {
sobj = map[string]interface{}{}
jobj = map[string]interface{}{}
data = []byte(`{"a":1}`)
cfg := Config{
cfg := sonic.Config{
UseNumber: true,
}.Froze()
serr = cfg.Unmarshal(data, &sobj)
@ -132,7 +133,7 @@ func TestCompatUnmarshalStd(t *testing.T) {
x := struct{A json.Number}{}
y := struct{A json.Number}{}
data = []byte(`{"A":"1", "B":-1}`)
cfg = Config{
cfg = sonic.Config{
DisallowUnknownFields: true,
}.Froze()
serr = cfg.Unmarshal(data, &x)
@ -152,7 +153,7 @@ func TestCompatEncoderDefault(t *testing.T) {
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = jt.NewEncoder(w1)
var enc2 = ConfigDefault.NewEncoder(w2)
var enc2 = sonic.ConfigDefault.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
@ -165,7 +166,7 @@ func TestCompatEncoderDefault(t *testing.T) {
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
@ -183,7 +184,7 @@ func TestCompatEncoderStd(t *testing.T) {
var w1 = bytes.NewBuffer(nil)
var w2 = bytes.NewBuffer(nil)
var enc1 = json.NewEncoder(w1)
var enc2 = ConfigStd.NewEncoder(w2)
var enc2 = sonic.ConfigStd.NewEncoder(w2)
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
@ -196,7 +197,7 @@ func TestCompatEncoderStd(t *testing.T) {
require.Nil(t, enc1.Encode(o))
require.Nil(t, enc2.Encode(o))
require.Equal(t, w1.String(), w2.String())
enc1.SetEscapeHTML(false)
enc2.SetEscapeHTML(false)
enc1.SetIndent("", "")
@ -213,7 +214,7 @@ func TestCompatDecoderStd(t *testing.T) {
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = json.NewDecoder(w1)
var enc2 = ConfigStd.NewDecoder(w2)
var enc2 = sonic.ConfigStd.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
@ -224,7 +225,7 @@ func TestCompatDecoderStd(t *testing.T) {
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))
@ -238,7 +239,7 @@ func TestCompatDecoderDefault(t *testing.T) {
var w1 = bytes.NewBuffer([]byte(s))
var w2 = bytes.NewBuffer([]byte(s))
var enc1 = jt.NewDecoder(w1)
var enc2 = ConfigDefault.NewDecoder(w2)
var enc2 = sonic.ConfigDefault.NewDecoder(w2)
require.Equal(t, enc1.More(), enc2.More())
require.Nil(t, enc1.Decode(&o1))
@ -249,7 +250,7 @@ func TestCompatDecoderDefault(t *testing.T) {
require.Nil(t, enc1.Decode(&o1))
require.Nil(t, enc2.Decode(&o2))
require.Equal(t, w1.String(), w2.String())
require.Equal(t, enc1.More(), enc2.More())
require.NotNil(t, enc1.Decode(&o1))
require.NotNil(t, enc2.Decode(&o2))

13
ast/compat_test.go → external_jsonlib_test/unit_test/ast_compat_test.go
View file

@ -14,11 +14,12 @@
* limitations under the License.
*/
package ast
package unit_test
import (
`testing`
`github.com/bytedance/sonic/ast`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
`github.com/tidwall/gjson`
@ -35,8 +36,8 @@ func TestNotFoud(t *testing.T) {
require.True(t, ga.Type == gjson.Null)
require.Equal(t, false, ga.Bool())
sa, err := NewSearcher(data).GetByPath("b")
require.True(t, sa.Type() == V_NONE)
sa, err := ast.NewSearcher(data).GetByPath("b")
require.True(t, sa.Type() == ast.V_NONE)
require.Error(t, err)
sv, err := sa.Bool()
require.Error(t, err)
@ -54,10 +55,10 @@ func TestNull(t *testing.T) {
require.True(t, ga.Type == gjson.Null)
require.Equal(t, false, ga.Bool())
sa, err := NewSearcher(data).GetByPath("b")
require.True(t, sa.Type() == V_NULL)
sa, err := ast.NewSearcher(data).GetByPath("b")
require.True(t, sa.Type() == ast.V_NULL)
require.NoError(t, err)
sv, err := sa.Bool()
require.NoError(t, err)
require.Equal(t, false, sv)
}
}

231
external_jsonlib_test/unit_test/decoder_stream_test.go Normal file
View file

@ -0,0 +1,231 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package unit_test
import (
`io`
`strings`
`testing`
`github.com/bytedance/sonic/decoder`
jsoniter "github.com/json-iterator/go"
`github.com/stretchr/testify/assert`
`github.com/stretchr/testify/require`
)
var (
defaultBufferSize uint = 4096
_Single_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"} { `
_Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(defaultBufferSize)) + `"}{ } {"11111":"` +
strings.Repeat("2", int(defaultBufferSize)) + `"} b {}`
)
func TestDecodeSingle(t *testing.T) {
var str = _Single_JSON
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var r2 = strings.NewReader(str)
var v2 map[string]interface{}
var d2 = decoder.NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
es1 := d1.Decode(&v1)
ee1 := d2.Decode(&v2)
assert.Equal(t, es1, ee1)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
}
func TestDecodeMulti(t *testing.T) {
var str = _Triple_JSON
var r1 = strings.NewReader(str)
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var r2 = strings.NewReader(str)
var v2 map[string]interface{}
var d2 = decoder.NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
es1 := d1.Decode(&v1)
ee1 := d2.Decode(&v2)
assert.Equal(t, es1, ee1)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
// fmt.Printf("v:%#v\n", v1)
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
}
type HaltReader struct {
halts map[int]bool
buf string
p int
}
func NewHaltReader(buf string, halts map[int]bool) *HaltReader {
return &HaltReader{
halts: halts,
buf: buf,
p: 0,
}
}
func (self *HaltReader) Read(p []byte) (int, error) {
t := 0
for ; t < len(p); {
if self.p >= len(self.buf) {
return t, io.EOF
}
if b, ok := self.halts[self.p]; b {
self.halts[self.p] = false
return t, nil
} else if ok {
delete(self.halts, self.p)
return 0, nil
}
p[t] = self.buf[self.p]
self.p++
t++
}
return t, nil
}
func (self *HaltReader) Reset(buf string) {
self.p = 0
self.buf = buf
}
var testHalts = func() map[int]bool {
return map[int]bool{
1: true,
10: true,
20: true}
}
func TestDecodeHalt(t *testing.T) {
var str = _Triple_JSON
var r1 = NewHaltReader(str, testHalts())
var r2 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
var v2 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
var d2 = decoder.NewStreamDecoder(r2)
require.Equal(t, d1.More(), d2.More())
err1 := d1.Decode(&v1)
err2 := d2.Decode(&v2)
assert.Equal(t, err1, err2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
// assert.Equal(t, d1.InputOffset(), d2.InputOffset())
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
}
func TestMore(t *testing.T) {
var str = _Triple_JSON
var r2 = NewHaltReader(str, testHalts())
var v2 map[string]interface{}
var d2 = decoder.NewStreamDecoder(r2)
var r1 = NewHaltReader(str, testHalts())
var v1 map[string]interface{}
var d1 = jsoniter.NewDecoder(r1)
require.Nil(t, d1.Decode(&v1))
require.Nil(t, d2.Decode(&v2))
require.Equal(t, d1.More(), d2.More())
es4 := d1.Decode(&v1)
ee4 := d2.Decode(&v2)
assert.Equal(t, es4, ee4)
assert.Equal(t, v1, v2)
require.Equal(t, d1.More(), d2.More())
es2 := d1.Decode(&v1)
ee2 := d2.Decode(&v2)
assert.Equal(t, es2, ee2)
assert.Equal(t, v1, v2)
require.Equal(t, d1.More(), d2.More())
es3 := d1.Decode(&v1)
assert.NotNil(t, es3)
ee3 := d2.Decode(&v2)
assert.NotNil(t, ee3)
require.Equal(t, d1.More(), d2.More())
es5 := d1.Decode(&v1)
assert.NotNil(t, es5)
ee5 := d2.Decode(&v2)
assert.NotNil(t, ee5)
require.Equal(t, d1.More(), d2.More())
}

4
go.mod
View file

@ -5,12 +5,8 @@ go 1.15
require (
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06
github.com/davecgh/go-spew v1.1.1
github.com/goccy/go-json v0.9.4
github.com/json-iterator/go v1.1.12
github.com/klauspost/cpuid/v2 v2.0.9
github.com/stretchr/testify v1.7.0
github.com/tidwall/gjson v1.13.0
github.com/tidwall/sjson v1.2.4
github.com/twitchyliquid64/golang-asm v0.15.1
golang.org/x/arch v0.0.0-20210923205945-b76863e36670
)

19
go.sum
View file

@ -3,32 +3,13 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/goccy/go-json v0.9.4 h1:L8MLKG2mvVXiQu07qB6hmfqeSYQdOnqPot2GhsIwIaI=
github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=

1
go.work
View file

@ -4,4 +4,5 @@ use (
.
./generic_test
./fuzz
./external_jsonlib_test
)