diff --git a/.github/workflows/push-check-arm.yml b/.github/workflows/push-check-arm.yml index 544953d..541085f 100644 --- a/.github/workflows/push-check-arm.yml +++ b/.github/workflows/push-check-arm.yml @@ -7,7 +7,8 @@ jobs: strategy: matrix: go-version: [1.15.x, 1.18.x] - runs-on: [self-hosted, arm] + os: [arm, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/push-check-go118.yml b/.github/workflows/push-check-go118.yml index 3a4a7dc..7d0fcd8 100644 --- a/.github/workflows/push-check-go118.yml +++ b/.github/workflows/push-check-go118.yml @@ -4,7 +4,7 @@ on: push jobs: build: - runs-on: self-hosted + runs-on: [self-hosted, X64] steps: - uses: actions/checkout@v2 @@ -25,3 +25,6 @@ jobs: - name: Generic Test run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test + + - name: Benchmark + run: go test -benchmem -run=^$ -bench . -v ./generic_test \ No newline at end of file diff --git a/.github/workflows/push-check-all.yml b/.github/workflows/push-check-linux-amd64.yml similarity index 90% rename from .github/workflows/push-check-all.yml rename to .github/workflows/push-check-linux-amd64.yml index fcb9f93..6c7db0d 100644 --- a/.github/workflows/push-check-all.yml +++ b/.github/workflows/push-check-linux-amd64.yml @@ -7,8 +7,7 @@ jobs: strategy: matrix: go-version: [1.15.x, 1.16.x, 1.17.x] - os: [self-hosted] - runs-on: ${{ matrix.os }} + runs-on: [self-hosted, X64] steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 1094bac..f38964f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ junit.xml *.out ast/test.out ast/bench.sh + +!testdata/*.json.gz diff --git a/bench.py b/bench.py index 47a24dc..9e6c8ca 100755 --- a/bench.py +++ b/bench.py @@ -56,11 +56,10 @@ def compare(args): (fd, diff) = tempfile.mkstemp() run("git diff > %s"%diff) - # early return if currrent is main branch and not diff found. - # notice that, the new file won't be counted in. + # early return if currrent is main branch. print ("Current branch: %s"%(current_branch)) - if current_branch == "main" and os.stat(diff).st_size == 0: - print ("No change found.") + if current_branch == "main": + print ("Cannot compare at the main branch.Please build a new branch") return None # benchmark current branch diff --git a/encoder/encoder.go b/encoder/encoder.go index deb2e18..28b79b8 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -57,6 +57,8 @@ const ( // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler // is always escaped string and needs no quoting NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler + + CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler ) // Encoder represents a specific set of encoder configurations. diff --git a/generic_test/benchmark_test.go b/generic_test/benchmark_test.go new file mode 100644 index 0000000..377aef0 --- /dev/null +++ b/generic_test/benchmark_test.go @@ -0,0 +1,268 @@ +// +build go1.18 + +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package generic_test + +import ( + `os` + `testing` + `reflect` + `fmt` + `github.com/bytedance/sonic` + `encoding/json` + gojson `github.com/goccy/go-json` + jsoniter `github.com/json-iterator/go` + jsonv2 `github.com/go-json-experiment/json` +) + +var ( + validFlag = os.Getenv("SONIC_VALID_GENERIC_BENCH") != "" + pretouchFlag = os.Getenv("SONIC_NO_PRETOUCH_BENCH") == "" +) + +type jsonLibEntry struct { + name string + marshal func(any) ([]byte, error) + unmarshal func([]byte, any) error +} + +var jsonLibs = []jsonLibEntry { + {"Std", json.Marshal, json.Unmarshal}, + {"StdV2", jsonv2.Marshal, jsonv2.Unmarshal}, + {"Sonic", sonic.Marshal, sonic.Unmarshal}, + {"SonicStd", sonic.ConfigStd.Marshal, sonic.ConfigStd.Unmarshal}, + {"GoJson", gojson.Marshal, gojson.Unmarshal}, + {"JsonIter", jsoniter.Marshal, jsoniter.Unmarshal}, + {"JsonIterStd", jsoniter.ConfigCompatibleWithStandardLibrary.Marshal, jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal}, +} + +func BenchmarkUnmarshalConcrete(b *testing.B) { + runUnmarshalC(b) +} + +func BenchmarkUnmarshalInterface(b *testing.B) { + runUnmarshalI(b) +} + +func BenchmarkMarshalConcrete(b *testing.B) { + runMarshalC(b) +} + +func BenchmarkMarshalInterface(b *testing.B) { + runMarshalI(b) +} + +func runUnmarshalC(b *testing.B) { + for _, tt := range jsonTestdata() { + for _, lib := range jsonLibs { + var val any = tt.new() + pretouch := func() { + _ = lib.unmarshal(tt.data, val) + } + + run := func(b *testing.B) { + val = tt.new() + if err := lib.unmarshal(tt.data, val); err != nil { + b.Fatalf("%s Unmarshal error: %v", lib.name, err) + } + } + + valid := func(b *testing.B) { + val1, val2 := tt.new(), tt.new() + if err := json.Unmarshal(tt.data, val1); err != nil { + panic(err) + } + if err := lib.unmarshal(tt.data, val2); err != nil { + panic(err) + } + if !reflect.DeepEqual(val1, val2) { + b.Fatalf("%s Unmarshal output mismatch:\ngot %v\nwant %v", lib.name, val2, val1) + } + } + + name := fmt.Sprintf("%s_%s", tt.name, lib.name) + b.Run(name, func(b *testing.B) { + if pretouchFlag { + pretouch() + } + valid(b) + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(int64(len(tt.data))) + for i := 0; i < b.N; i++ { + run(b) + } + }) + } + } +} + +func runUnmarshalI(b *testing.B) { + for _, tt := range jsonTestdata() { + for _, lib := range jsonLibs { + var val any = tt.newI() + pretouch := func() { + _ = lib.unmarshal(tt.data, val) + } + + run := func(b *testing.B) { + val = tt.newI() + if err := lib.unmarshal(tt.data, val); err != nil { + b.Fatalf("%s Unmarshal error: %v", lib.name, err) + } + } + + valid := func(b *testing.B) { + val1, val2 := tt.newI(), tt.newI() + if err := json.Unmarshal(tt.data, val1); err != nil { + panic(err) + } + if err := lib.unmarshal(tt.data, val2); err != nil { + panic(err) + } + if !reflect.DeepEqual(val1, val2) { + b.Fatalf("%s Unmarshal output mismatch:\ngot %v\nwant %v", lib.name, val2, val1) + } + } + + name := fmt.Sprintf("%s_%s", tt.name, lib.name) + b.Run(name, func(b *testing.B) { + if pretouchFlag { + pretouch() + } + valid(b) + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(int64(len(tt.data))) + for i := 0; i < b.N; i++ { + run(b) + } + }) + } + } +} + +func runMarshalC(b *testing.B) { + for _, tt := range jsonTestdata() { + for _, lib := range jsonLibs { + pretouch := func() { + _, _ = lib.marshal(tt.val) + } + + run := func(b *testing.B) { + if _, err := lib.marshal(tt.val); err != nil { + b.Fatalf("%s Marshal error: %v", lib.name, err) + } + } + + valid := func(b *testing.B) { + // some details are different with encoding/json, so we compare the unmarshal results from marshaled buffer + var buf1, buf2 []byte + buf1, err := json.Marshal(tt.val) + if err != nil { + b.Fatalf("encoding/json Marshal error: %v", err) + } + buf2, err = lib.marshal(tt.val); + if err != nil { + b.Fatalf("%s Marshal error: %v", lib.name, err) + } + var val1, val2 = tt.new(), tt.new() + if err := json.Unmarshal(buf1, val1); err != nil { + b.Fatalf("encoding/json Unmarshal again error: %v", err) + } + if err := lib.unmarshal(buf2, val2); err != nil { + b.Fatalf("%s Unmarshal again error: %v", lib.name, err) + } + if !reflect.DeepEqual(val1, val2) { + b.Fatalf("Unmarshal again output mismatch\n") + } + } + + name := fmt.Sprintf("%s_%s", tt.name, lib.name) + b.Run(name, func(b *testing.B) { + if pretouchFlag { + pretouch() + } + if (validFlag) { + valid(b) + } + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(int64(len(tt.data))) + for i := 0; i < b.N; i++ { + run(b) + } + }) + } + } +} + +func runMarshalI(b *testing.B) { + for _, tt := range jsonTestdata() { + for _, lib := range jsonLibs { + pretouch := func() { + _, _ = lib.marshal(tt.valI) + } + + run := func(b *testing.B) { + if _, err := lib.marshal(tt.valI); err != nil { + b.Fatalf("%s Marshal error: %v", lib.name, err) + } + } + + valid := func(b *testing.B) { + // some details are different with encoding/json, so we compare the unmarshal results from marshaled buffer + var buf1, buf2 []byte + buf1, err := json.Marshal(tt.valI) + if err != nil { + b.Fatalf("encoding/json Marshal error: %v", err) + } + buf2, err = lib.marshal(tt.valI); + if err != nil { + b.Fatalf("%s Marshal error: %v", lib.name, err) + } + var val1, val2 = tt.new(), tt.new() + if err := json.Unmarshal(buf1, val1); err != nil { + b.Fatalf("encoding/json Unmarshal again error: %v", err) + } + if err := lib.unmarshal(buf2, val2); err != nil { + b.Fatalf("%s Unmarshal again error: %v", lib.name, err) + } + if !reflect.DeepEqual(val1, val2) { + b.Fatalf("Unmarshal again output mismatch\n") + } + } + + name := fmt.Sprintf("%s_%s", tt.name, lib.name) + b.Run(name, func(b *testing.B) { + if pretouchFlag { + pretouch() + } + if (validFlag) { + valid(b) + } + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(int64(len(tt.data))) + for i := 0; i < b.N; i++ { + run(b) + } + }) + } + } +} \ No newline at end of file diff --git a/generic_test/go.mod b/generic_test/go.mod index c8e8f50..3eb8e60 100644 --- a/generic_test/go.mod +++ b/generic_test/go.mod @@ -4,8 +4,16 @@ go 1.18 require ( github.com/bytedance/sonic v1.0.0 + github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539 + github.com/goccy/go-json v0.9.4 + github.com/json-iterator/go v1.1.12 +) + +require ( github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect ) diff --git a/generic_test/go.sum b/generic_test/go.sum index 9085190..de638d9 100644 --- a/generic_test/go.sum +++ b/generic_test/go.sum @@ -3,6 +3,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539 h1:mhqU04ze8mDNsTU2WFQVw1tbjYalOdIXx91JlBse4tk= +github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539/go.mod h1:jbpkervfdK2HCcB2YEFmwYeaq057KFiaaKTNTHV4OOQ= github.com/goccy/go-json v0.9.4 h1:L8MLKG2mvVXiQu07qB6hmfqeSYQdOnqPot2GhsIwIaI= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/generic_test/sonic_test.go b/generic_test/sonic_test.go index 3ba5284..e87a453 100644 --- a/generic_test/sonic_test.go +++ b/generic_test/sonic_test.go @@ -19,11 +19,11 @@ package generic_test import ( - `testing` + `testing` `reflect` `github.com/bytedance/sonic` - `github.com/bytedance/sonic/option` + `github.com/bytedance/sonic/option` `github.com/bytedance/sonic/ast` ) @@ -56,8 +56,8 @@ func getAny[S Str, B Bytes, T SliceAny](src S, path T) (ast.Node, error) { } func pretouchAny[T Any](v T, opts ...option.CompileOption) error { - rt := reflect.TypeOf(v) - return sonic.Pretouch(rt, opts...) + rt := reflect.TypeOf(v) + return sonic.Pretouch(rt, opts...) } type Basic interface { @@ -73,8 +73,8 @@ func marshalBasic[B Bytes, T Basic](val T) (B, error) { } func pretouchBasic[T Basic](v T, opts ...option.CompileOption) error { - rt := reflect.TypeOf(v) - return sonic.Pretouch(rt, opts...) + rt := reflect.TypeOf(v) + return sonic.Pretouch(rt, opts...) } type Float64 float64 @@ -88,22 +88,22 @@ func TestGenericAPI(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf("%s", out) - - var x0 = struct{ - A []Any `json:"a"` - }{} - if err := pretouchAny(x0); err != nil { - t.Fatal(err) - } - if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x0); err != nil { + t.Logf("%s", out) + + var x0 = struct{ + A []Any `json:"a"` + }{} + if err := pretouchAny(x0); err != nil { + t.Fatal(err) + } + if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x0); err != nil { t.Fatal(t) } out0, err := marshalAny(x0) if err != nil { t.Fatal(err) } - t.Logf("%s", out0) + t.Logf("%s", out0) var x1 int if err := unmarshalBasic(`1`, &x1); err != nil { @@ -115,8 +115,8 @@ func TestGenericAPI(t *testing.T) { } t.Logf("%s", out1) - var x2 Float64 = 1 - // if err := unmarshalBasic(`1`, &x2); err != nil { + var x2 Float64 = 1 + // if err := unmarshalBasic(`1`, &x2); err != nil { // t.Fatal(t) // } out2, err := marshalBasic(x2) @@ -133,20 +133,20 @@ func TestGenericAPI(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf("%s", out3) + t.Logf("%s", out3) - var x4 Float64 = 1 - if err := pretouchBasic(x4); err != nil { - t.Fatal(err) - } - // if err := unmarshalBasic(`0.5`, &x4); err != nil { + var x4 Float64 = 1 + if err := pretouchBasic(x4); err != nil { + t.Fatal(err) + } + // if err := unmarshalBasic(`0.5`, &x4); err != nil { // t.Fatal(t) // } out4, err := marshalBasic(x4) if err != nil { t.Fatal(err) } - t.Logf("%s", out4) + t.Logf("%s", out4) root, err := getAny(`{"a":[true,1,"hello world"]}`, []interface{}{"a", 1}) if err != nil { diff --git a/generic_test/testdata_test.go b/generic_test/testdata_test.go new file mode 100644 index 0000000..4d5e6ac --- /dev/null +++ b/generic_test/testdata_test.go @@ -0,0 +1,620 @@ +// +build go1.18 + +/* + * Copyright 2020 The Go Authors. All rights reserved. + * Modifications Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package generic_test + +import ( + `bytes` + `compress/gzip` + `io/ioutil` + `path/filepath` + `sort` + `strings` + `sync` + `time` + `encoding/json` +) + +type jsonTestdataEntry struct { + name string + data []byte + val any + valI any + new func() any // nil if there is no concrete type for this + newI func() any +} + +var ( + jsonTestdataOnce sync.Once + jsonTestdataLazy []jsonTestdataEntry +) + +func jsonTestdata() []jsonTestdataEntry { + jsonTestdataOnce.Do(func() { + fis, err := ioutil.ReadDir("../testdata") + if err != nil { + panic(err) + } + sort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() }) + for _, fi := range fis { + if !strings.HasSuffix(fi.Name(), ".json.gz") { + continue + } + + // Convert snake_case file name to CamelCase. + words := strings.Split(strings.TrimSuffix(fi.Name(), ".json.gz"), "_") + for i := range words { + words[i] = strings.Title(words[i]) + } + name := strings.Join(words, "") + + // Read and decompress the test data. + b, err := ioutil.ReadFile(filepath.Join("../testdata", fi.Name())) + if err != nil { + panic(err) + } + zr, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + panic(err) + } + data, err := ioutil.ReadAll(zr) + if err != nil { + panic(err) + } + + // Check whether there is a concrete type for this data. + var newFn func() any + switch name { + case "CanadaGeometry": + newFn = func() any { return new(canadaRoot) } + case "CitmCatalog": + newFn = func() any { return new(citmRoot) } + case "GolangSource": + newFn = func() any { return new(golangRoot) } + case "StringEscaped": + newFn = func() any { return new(stringRoot) } + case "StringUnicode": + newFn = func() any { return new(stringRoot) } + case "SyntheaFhir": + newFn = func() any { return new(syntheaRoot) } + case "TwitterStatus": + newFn = func() any { return new(twitterRoot) } + } + + var newFnI = func() any { return new(any) } + + // Fill into value + var val any = newFn() + err = json.Unmarshal(data, val) + if err != nil { + panic(err) + } + var valI any = newFnI() + err = json.Unmarshal(data, valI) + if err != nil { + panic(err) + } + + jsonTestdataLazy = append(jsonTestdataLazy, jsonTestdataEntry{name, data, val, valI, newFn, newFnI}) + } + }) + return jsonTestdataLazy +} + +type ( + canadaRoot struct { + Type string `json:"type"` + Features []struct { + Type string `json:"type"` + Properties struct { + Name string `json:"name"` + } `json:"properties"` + Geometry struct { + Type string `json:"type"` + Coordinates [][][2]float64 `json:"coordinates"` + } `json:"geometry"` + } `json:"features"` + } +) + +type ( + citmRoot struct { + AreaNames map[int64]string `json:"areaNames"` + AudienceSubCategoryNames map[int64]string `json:"audienceSubCategoryNames"` + BlockNames map[int64]string `json:"blockNames"` + Events map[int64]struct { + Description string `json:"description"` + ID int `json:"id"` + Logo string `json:"logo"` + Name string `json:"name"` + SubTopicIds []int `json:"subTopicIds"` + SubjectCode any `json:"subjectCode"` + Subtitle any `json:"subtitle"` + TopicIds []int `json:"topicIds"` + } `json:"events"` + Performances []struct { + EventID int `json:"eventId"` + ID int `json:"id"` + Logo any `json:"logo"` + Name any `json:"name"` + Prices []struct { + Amount int `json:"amount"` + AudienceSubCategoryID int64 `json:"audienceSubCategoryId"` + SeatCategoryID int64 `json:"seatCategoryId"` + } `json:"prices"` + SeatCategories []struct { + Areas []struct { + AreaID int `json:"areaId"` + BlockIds []any `json:"blockIds"` + } `json:"areas"` + SeatCategoryID int `json:"seatCategoryId"` + } `json:"seatCategories"` + SeatMapImage any `json:"seatMapImage"` + Start int64 `json:"start"` + VenueCode string `json:"venueCode"` + } `json:"performances"` + SeatCategoryNames map[uint64]string `json:"seatCategoryNames"` + SubTopicNames map[uint64]string `json:"subTopicNames"` + SubjectNames map[uint64]string `json:"subjectNames"` + TopicNames map[uint64]string `json:"topicNames"` + TopicSubTopics map[uint64][]uint64 `json:"topicSubTopics"` + VenueNames map[string]string `json:"venueNames"` + } +) + +type ( + golangRoot struct { + Tree *golangNode `json:"tree"` + Username string `json:"username"` + } + golangNode struct { + Name string `json:"name"` + Kids []golangNode `json:"kids"` + CLWeight float64 `json:"cl_weight"` + Touches int `json:"touches"` + MinT uint64 `json:"min_t"` + MaxT uint64 `json:"max_t"` + MeanT uint64 `json:"mean_t"` + } +) + +type ( + stringRoot struct { + Arabic string `json:"Arabic"` + ArabicPresentationFormsA string `json:"Arabic Presentation Forms-A"` + ArabicPresentationFormsB string `json:"Arabic Presentation Forms-B"` + Armenian string `json:"Armenian"` + Arrows string `json:"Arrows"` + Bengali string `json:"Bengali"` + Bopomofo string `json:"Bopomofo"` + BoxDrawing string `json:"Box Drawing"` + CJKCompatibility string `json:"CJK Compatibility"` + CJKCompatibilityForms string `json:"CJK Compatibility Forms"` + CJKCompatibilityIdeographs string `json:"CJK Compatibility Ideographs"` + CJKSymbolsAndPunctuation string `json:"CJK Symbols and Punctuation"` + CJKUnifiedIdeographs string `json:"CJK Unified Ideographs"` + CJKUnifiedIdeographsExtensionA string `json:"CJK Unified Ideographs Extension A"` + CJKUnifiedIdeographsExtensionB string `json:"CJK Unified Ideographs Extension B"` + Cherokee string `json:"Cherokee"` + CurrencySymbols string `json:"Currency Symbols"` + Cyrillic string `json:"Cyrillic"` + CyrillicSupplementary string `json:"Cyrillic Supplementary"` + Devanagari string `json:"Devanagari"` + EnclosedAlphanumerics string `json:"Enclosed Alphanumerics"` + EnclosedCJKLettersAndMonths string `json:"Enclosed CJK Letters and Months"` + Ethiopic string `json:"Ethiopic"` + GeometricShapes string `json:"Geometric Shapes"` + Georgian string `json:"Georgian"` + GreekAndCoptic string `json:"Greek and Coptic"` + Gujarati string `json:"Gujarati"` + Gurmukhi string `json:"Gurmukhi"` + HangulCompatibilityJamo string `json:"Hangul Compatibility Jamo"` + HangulJamo string `json:"Hangul Jamo"` + HangulSyllables string `json:"Hangul Syllables"` + Hebrew string `json:"Hebrew"` + Hiragana string `json:"Hiragana"` + IPAExtentions string `json:"IPA Extentions"` + KangxiRadicals string `json:"Kangxi Radicals"` + Katakana string `json:"Katakana"` + Khmer string `json:"Khmer"` + KhmerSymbols string `json:"Khmer Symbols"` + Latin string `json:"Latin"` + LatinExtendedAdditional string `json:"Latin Extended Additional"` + Latin1Supplement string `json:"Latin-1 Supplement"` + LatinExtendedA string `json:"Latin-Extended A"` + LatinExtendedB string `json:"Latin-Extended B"` + LetterlikeSymbols string `json:"Letterlike Symbols"` + Malayalam string `json:"Malayalam"` + MathematicalAlphanumericSymbols string `json:"Mathematical Alphanumeric Symbols"` + MathematicalOperators string `json:"Mathematical Operators"` + MiscellaneousSymbols string `json:"Miscellaneous Symbols"` + Mongolian string `json:"Mongolian"` + NumberForms string `json:"Number Forms"` + Oriya string `json:"Oriya"` + PhoneticExtensions string `json:"Phonetic Extensions"` + SupplementalArrowsB string `json:"Supplemental Arrows-B"` + Syriac string `json:"Syriac"` + Tamil string `json:"Tamil"` + Thaana string `json:"Thaana"` + Thai string `json:"Thai"` + UnifiedCanadianAboriginalSyllabics string `json:"Unified Canadian Aboriginal Syllabics"` + YiRadicals string `json:"Yi Radicals"` + YiSyllables string `json:"Yi Syllables"` + } +) + +type ( + syntheaRoot struct { + Entry []struct { + FullURL string `json:"fullUrl"` + Request *struct { + Method string `json:"method"` + URL string `json:"url"` + } `json:"request"` + Resource *struct { + AbatementDateTime time.Time `json:"abatementDateTime"` + AchievementStatus syntheaCode `json:"achievementStatus"` + Active bool `json:"active"` + Activity []struct { + Detail *struct { + Code syntheaCode `json:"code"` + Location syntheaReference `json:"location"` + Status string `json:"status"` + } `json:"detail"` + } `json:"activity"` + Address []syntheaAddress `json:"address"` + Addresses []syntheaReference `json:"addresses"` + AuthoredOn time.Time `json:"authoredOn"` + BillablePeriod syntheaRange `json:"billablePeriod"` + BirthDate string `json:"birthDate"` + CareTeam []struct { + Provider syntheaReference `json:"provider"` + Reference string `json:"reference"` + Role syntheaCode `json:"role"` + Sequence int64 `json:"sequence"` + } `json:"careTeam"` + Category []syntheaCode `json:"category"` + Claim syntheaReference `json:"claim"` + Class syntheaCoding `json:"class"` + ClinicalStatus syntheaCode `json:"clinicalStatus"` + Code syntheaCode `json:"code"` + Communication []struct { + Language syntheaCode `json:"language"` + } `json:"communication"` + Component []struct { + Code syntheaCode `json:"code"` + ValueQuantity syntheaCoding `json:"valueQuantity"` + } `json:"component"` + Contained []struct { + Beneficiary syntheaReference `json:"beneficiary"` + ID string `json:"id"` + Intent string `json:"intent"` + Payor []syntheaReference `json:"payor"` + Performer []syntheaReference `json:"performer"` + Requester syntheaReference `json:"requester"` + ResourceType string `json:"resourceType"` + Status string `json:"status"` + Subject syntheaReference `json:"subject"` + Type syntheaCode `json:"type"` + } `json:"contained"` + Created time.Time `json:"created"` + DeceasedDateTime time.Time `json:"deceasedDateTime"` + Description syntheaCode `json:"description"` + Diagnosis []struct { + DiagnosisReference syntheaReference `json:"diagnosisReference"` + Sequence int64 `json:"sequence"` + Type []syntheaCode `json:"type"` + } `json:"diagnosis"` + DosageInstruction []struct { + AsNeededBoolean bool `json:"asNeededBoolean"` + DoseAndRate []struct { + DoseQuantity *struct { + Value float64 `json:"value"` + } `json:"doseQuantity"` + Type syntheaCode `json:"type"` + } `json:"doseAndRate"` + Sequence int64 `json:"sequence"` + Timing *struct { + Repeat *struct { + Frequency int64 `json:"frequency"` + Period float64 `json:"period"` + PeriodUnit string `json:"periodUnit"` + } `json:"repeat"` + } `json:"timing"` + } `json:"dosageInstruction"` + EffectiveDateTime time.Time `json:"effectiveDateTime"` + Encounter syntheaReference `json:"encounter"` + Extension []syntheaExtension `json:"extension"` + Gender string `json:"gender"` + Goal []syntheaReference `json:"goal"` + ID string `json:"id"` + Identifier []struct { + System string `json:"system"` + Type syntheaCode `json:"type"` + Use string `json:"use"` + Value string `json:"value"` + } `json:"identifier"` + Insurance []struct { + Coverage syntheaReference `json:"coverage"` + Focal bool `json:"focal"` + Sequence int64 `json:"sequence"` + } `json:"insurance"` + Insurer syntheaReference `json:"insurer"` + Intent string `json:"intent"` + Issued time.Time `json:"issued"` + Item []struct { + Adjudication []struct { + Amount syntheaCurrency `json:"amount"` + Category syntheaCode `json:"category"` + } `json:"adjudication"` + Category syntheaCode `json:"category"` + DiagnosisSequence []int64 `json:"diagnosisSequence"` + Encounter []syntheaReference `json:"encounter"` + InformationSequence []int64 `json:"informationSequence"` + LocationCodeableConcept syntheaCode `json:"locationCodeableConcept"` + Net syntheaCurrency `json:"net"` + ProcedureSequence []int64 `json:"procedureSequence"` + ProductOrService syntheaCode `json:"productOrService"` + Sequence int64 `json:"sequence"` + ServicedPeriod syntheaRange `json:"servicedPeriod"` + } `json:"item"` + LifecycleStatus string `json:"lifecycleStatus"` + ManagingOrganization []syntheaReference `json:"managingOrganization"` + MaritalStatus syntheaCode `json:"maritalStatus"` + MedicationCodeableConcept syntheaCode `json:"medicationCodeableConcept"` + MultipleBirthBoolean bool `json:"multipleBirthBoolean"` + Name json.RawMessage `json:"name"` + NumberOfInstances int64 `json:"numberOfInstances"` + NumberOfSeries int64 `json:"numberOfSeries"` + OccurrenceDateTime time.Time `json:"occurrenceDateTime"` + OnsetDateTime time.Time `json:"onsetDateTime"` + Outcome string `json:"outcome"` + Participant []struct { + Individual syntheaReference `json:"individual"` + Member syntheaReference `json:"member"` + Role []syntheaCode `json:"role"` + } `json:"participant"` + Patient syntheaReference `json:"patient"` + Payment *struct { + Amount syntheaCurrency `json:"amount"` + } `json:"payment"` + PerformedPeriod syntheaRange `json:"performedPeriod"` + Period syntheaRange `json:"period"` + Prescription syntheaReference `json:"prescription"` + PrimarySource bool `json:"primarySource"` + Priority syntheaCode `json:"priority"` + Procedure []struct { + ProcedureReference syntheaReference `json:"procedureReference"` + Sequence int64 `json:"sequence"` + } `json:"procedure"` + Provider syntheaReference `json:"provider"` + ReasonCode []syntheaCode `json:"reasonCode"` + ReasonReference []syntheaReference `json:"reasonReference"` + RecordedDate time.Time `json:"recordedDate"` + Referral syntheaReference `json:"referral"` + Requester syntheaReference `json:"requester"` + ResourceType string `json:"resourceType"` + Result []syntheaReference `json:"result"` + Series []struct { + BodySite syntheaCoding `json:"bodySite"` + Instance []struct { + Number int64 `json:"number"` + SopClass syntheaCoding `json:"sopClass"` + Title string `json:"title"` + UID string `json:"uid"` + } `json:"instance"` + Modality syntheaCoding `json:"modality"` + Number int64 `json:"number"` + NumberOfInstances int64 `json:"numberOfInstances"` + Started string `json:"started"` + UID string `json:"uid"` + } `json:"series"` + ServiceProvider syntheaReference `json:"serviceProvider"` + Started time.Time `json:"started"` + Status string `json:"status"` + Subject syntheaReference `json:"subject"` + SupportingInfo []struct { + Category syntheaCode `json:"category"` + Sequence int64 `json:"sequence"` + ValueReference syntheaReference `json:"valueReference"` + } `json:"supportingInfo"` + Telecom []map[string]string `json:"telecom"` + Text map[string]string `json:"text"` + Total json.RawMessage `json:"total"` + Type json.RawMessage `json:"type"` + Use string `json:"use"` + VaccineCode syntheaCode `json:"vaccineCode"` + ValueCodeableConcept syntheaCode `json:"valueCodeableConcept"` + ValueQuantity syntheaCoding `json:"valueQuantity"` + VerificationStatus syntheaCode `json:"verificationStatus"` + } `json:"resource"` + } `json:"entry"` + ResourceType string `json:"resourceType"` + Type string `json:"type"` + } + syntheaCode struct { + Coding []syntheaCoding `json:"coding"` + Text string `json:"text"` + } + syntheaCoding struct { + Code string `json:"code"` + Display string `json:"display"` + System string `json:"system"` + Unit string `json:"unit"` + Value float64 `json:"value"` + } + syntheaReference struct { + Display string `json:"display"` + Reference string `json:"reference"` + } + syntheaAddress struct { + City string `json:"city"` + Country string `json:"country"` + Extension []syntheaExtension `json:"extension"` + Line []string `json:"line"` + PostalCode string `json:"postalCode"` + State string `json:"state"` + } + syntheaExtension struct { + URL string `json:"url"` + ValueAddress syntheaAddress `json:"valueAddress"` + ValueCode string `json:"valueCode"` + ValueDecimal float64 `json:"valueDecimal"` + ValueString string `json:"valueString"` + Extension []syntheaExtension `json:"extension"` + } + syntheaRange struct { + End time.Time `json:"end"` + Start time.Time `json:"start"` + } + syntheaCurrency struct { + Currency string `json:"currency"` + Value float64 `json:"value"` + } +) + +type ( + twitterRoot struct { + Statuses []twitterStatus `json:"statuses"` + SearchMetadata struct { + CompletedIn float64 `json:"completed_in"` + MaxID int64 `json:"max_id"` + MaxIDStr int64 `json:"max_id_str,string"` + NextResults string `json:"next_results"` + Query string `json:"query"` + RefreshURL string `json:"refresh_url"` + Count int `json:"count"` + SinceID int `json:"since_id"` + SinceIDStr int `json:"since_id_str,string"` + } `json:"search_metadata"` + } + twitterStatus struct { + Metadata struct { + ResultType string `json:"result_type"` + IsoLanguageCode string `json:"iso_language_code"` + } `json:"metadata"` + CreatedAt string `json:"created_at"` + ID int64 `json:"id"` + IDStr int64 `json:"id_str,string"` + Text string `json:"text"` + Source string `json:"source"` + Truncated bool `json:"truncated"` + InReplyToStatusID int64 `json:"in_reply_to_status_id"` + InReplyToStatusIDStr int64 `json:"in_reply_to_status_id_str,string"` + InReplyToUserID int64 `json:"in_reply_to_user_id"` + InReplyToUserIDStr int64 `json:"in_reply_to_user_id_str,string"` + InReplyToScreenName string `json:"in_reply_to_screen_name"` + User twitterUser `json:"user,omitempty"` + Geo any `json:"geo"` + Coordinates any `json:"coordinates"` + Place any `json:"place"` + Contributors any `json:"contributors"` + RetweeetedStatus *twitterStatus `json:"retweeted_status"` + RetweetCount int `json:"retweet_count"` + FavoriteCount int `json:"favorite_count"` + Entities twitterEntities `json:"entities,omitempty"` + Favorited bool `json:"favorited"` + Retweeted bool `json:"retweeted"` + PossiblySensitive bool `json:"possibly_sensitive"` + Lang string `json:"lang"` + } + twitterUser struct { + ID int64 `json:"id"` + IDStr string `json:"id_str"` + Name string `json:"name"` + ScreenName string `json:"screen_name"` + Location string `json:"location"` + Description string `json:"description"` + URL any `json:"url"` + Entities twitterEntities `json:"entities"` + Protected bool `json:"protected"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + ListedCount int `json:"listed_count"` + CreatedAt string `json:"created_at"` + FavouritesCount int `json:"favourites_count"` + UtcOffset int `json:"utc_offset"` + TimeZone string `json:"time_zone"` + GeoEnabled bool `json:"geo_enabled"` + Verified bool `json:"verified"` + StatusesCount int `json:"statuses_count"` + Lang string `json:"lang"` + ContributorsEnabled bool `json:"contributors_enabled"` + IsTranslator bool `json:"is_translator"` + IsTranslationEnabled bool `json:"is_translation_enabled"` + ProfileBackgroundColor string `json:"profile_background_color"` + ProfileBackgroundImageURL string `json:"profile_background_image_url"` + ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https"` + ProfileBackgroundTile bool `json:"profile_background_tile"` + ProfileImageURL string `json:"profile_image_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileLinkColor string `json:"profile_link_color"` + ProfileSidebarBorderColor string `json:"profile_sidebar_border_color"` + ProfileSidebarFillColor string `json:"profile_sidebar_fill_color"` + ProfileTextColor string `json:"profile_text_color"` + ProfileUseBackgroundImage bool `json:"profile_use_background_image"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + Following bool `json:"following"` + FollowRequestSent bool `json:"follow_request_sent"` + Notifications bool `json:"notifications"` + } + twitterEntities struct { + Hashtags []any `json:"hashtags"` + Symbols []any `json:"symbols"` + URL *twitterURL `json:"url"` + URLs []twitterURL `json:"urls"` + UserMentions []struct { + ScreenName string `json:"screen_name"` + Name string `json:"name"` + ID int64 `json:"id"` + IDStr int64 `json:"id_str,string"` + Indices []int `json:"indices"` + } `json:"user_mentions"` + Description struct { + URLs []twitterURL `json:"urls"` + } `json:"description"` + Media []struct { + ID int64 `json:"id"` + IDStr string `json:"id_str"` + Indices []int `json:"indices"` + MediaURL string `json:"media_url"` + MediaURLHTTPS string `json:"media_url_https"` + URL string `json:"url"` + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + Type string `json:"type"` + Sizes map[string]struct { + W int `json:"w"` + H int `json:"h"` + Resize string `json:"resize"` + } `json:"sizes"` + SourceStatusID int64 `json:"source_status_id"` + SourceStatusIDStr int64 `json:"source_status_id_str,string"` + } `json:"media"` + } + twitterURL struct { + URL string `json:"url"` + URLs []twitterURL `json:"urls"` + ExpandedURL string `json:"expanded_url"` + DisplayURL string `json:"display_url"` + Indices []int `json:"indices"` + } +) \ No newline at end of file diff --git a/testdata/canada_geometry.json.gz b/testdata/canada_geometry.json.gz new file mode 100644 index 0000000..2c602f0 Binary files /dev/null and b/testdata/canada_geometry.json.gz differ diff --git a/testdata/citm_catalog.json.gz b/testdata/citm_catalog.json.gz new file mode 100644 index 0000000..e270495 Binary files /dev/null and b/testdata/citm_catalog.json.gz differ diff --git a/testdata/golang_source.json.gz b/testdata/golang_source.json.gz new file mode 100644 index 0000000..3ee1e66 Binary files /dev/null and b/testdata/golang_source.json.gz differ diff --git a/testdata/string_escaped.json.gz b/testdata/string_escaped.json.gz new file mode 100644 index 0000000..fae4d77 Binary files /dev/null and b/testdata/string_escaped.json.gz differ diff --git a/testdata/string_unicode.json.gz b/testdata/string_unicode.json.gz new file mode 100644 index 0000000..5a5a00d Binary files /dev/null and b/testdata/string_unicode.json.gz differ diff --git a/testdata/synthea_fhir.json.gz b/testdata/synthea_fhir.json.gz new file mode 100644 index 0000000..9bb0810 Binary files /dev/null and b/testdata/synthea_fhir.json.gz differ diff --git a/testdata/twitter_status.json.gz b/testdata/twitter_status.json.gz new file mode 100644 index 0000000..ff6760d Binary files /dev/null and b/testdata/twitter_status.json.gz differ