test: add generic benchmark (#239)

* test: add generic benchmark

* test: add ci

* chore: adjust generic benchmark variable name

* ci: fix compare bug in bench.py

* build: adjust CI yaml

* test: use sonic.Config

* chore: generic test

* test: add interface type bench

Co-authored-by: liuqiang <liuqiang.06@bytedance.com>
Co-authored-by: duanyi.aster <duanyi.aster@bytedance.com>

.github/workflows/push-check-arm.yml

@@ -7,7 +7,8 @@ jobs:
     strategy:
       matrix:
         go-version: [1.15.x, 1.18.x]
-    runs-on: [self-hosted, arm]
+        os: [arm, macos-latest]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/push-check-go118.yml

@@ -4,7 +4,7 @@ on: push
 
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: [self-hosted, X64]
     steps:
       - uses: actions/checkout@v2
 
@@ -25,3 +25,6 @@ jobs:
       
       - name: Generic Test
         run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test
+
+      - name: Benchmark
+        run: go test -benchmem -run=^$ -bench . -v ./generic_test

.github/workflows/push-check-linux-amd64.yml

@@ -7,8 +7,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.15.x, 1.16.x, 1.17.x]
-        os: [self-hosted]
+    runs-on: [self-hosted, X64]
-    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
 

.gitignore

@@ -46,3 +46,5 @@ junit.xml
 *.out
 ast/test.out
 ast/bench.sh
+
+!testdata/*.json.gz

bench.py

@@ -56,11 +56,10 @@ def compare(args):
     (fd, diff) = tempfile.mkstemp()
     run("git diff > %s"%diff)
 
-    # early return if currrent is main branch and not diff found.
+    # early return if currrent is main branch.
-    # notice that, the new file won't be counted in.
     print ("Current branch: %s"%(current_branch))
-    if current_branch == "main" and os.stat(diff).st_size == 0:
+    if current_branch == "main":
-        print ("No change found.")
+        print ("Cannot compare at the main branch.Please build a new branch")
         return None
 
     # benchmark current branch    

encoder/encoder.go

@@ -57,6 +57,8 @@ const (
     // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler 
     // is always escaped string and needs no quoting
     NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
+
+    CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
 )
 
 // Encoder represents a specific set of encoder configurations.

generic_test/benchmark_test.go

@@ -0,0 +1,268 @@
+// +build go1.18
+
+/*
+ * Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package generic_test
+
+import (
+    `os`
+    `testing`
+    `reflect`
+    `fmt`
+    `github.com/bytedance/sonic`
+    `encoding/json`
+    gojson `github.com/goccy/go-json`
+    jsoniter `github.com/json-iterator/go`
+    jsonv2 `github.com/go-json-experiment/json`
+)
+
+var (
+    validFlag = os.Getenv("SONIC_VALID_GENERIC_BENCH")  != ""
+    pretouchFlag = os.Getenv("SONIC_NO_PRETOUCH_BENCH") == ""
+)
+
+type jsonLibEntry struct {
+    name      string
+    marshal   func(any) ([]byte, error)
+    unmarshal func([]byte, any) error
+}
+
+var jsonLibs = []jsonLibEntry {
+    {"Std", json.Marshal, json.Unmarshal},
+    {"StdV2", jsonv2.Marshal, jsonv2.Unmarshal},
+    {"Sonic", sonic.Marshal, sonic.Unmarshal},
+    {"SonicStd", sonic.ConfigStd.Marshal, sonic.ConfigStd.Unmarshal},
+    {"GoJson", gojson.Marshal, gojson.Unmarshal},
+    {"JsonIter", jsoniter.Marshal, jsoniter.Unmarshal},
+    {"JsonIterStd", jsoniter.ConfigCompatibleWithStandardLibrary.Marshal, jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal},
+}
+
+func BenchmarkUnmarshalConcrete(b *testing.B)     { 
+    runUnmarshalC(b)
+}
+
+func BenchmarkUnmarshalInterface(b *testing.B)     { 
+    runUnmarshalI(b)
+}
+
+func BenchmarkMarshalConcrete(b *testing.B)     { 
+    runMarshalC(b) 
+}
+
+func BenchmarkMarshalInterface(b *testing.B)     { 
+    runMarshalI(b) 
+}
+
+func runUnmarshalC(b *testing.B) {
+    for _, tt := range jsonTestdata() {
+        for _, lib := range jsonLibs {
+            var val any = tt.new()
+            pretouch := func() {
+                _ = lib.unmarshal(tt.data, val)
+            }
+    
+            run := func(b *testing.B) {
+                val = tt.new()
+                if err := lib.unmarshal(tt.data, val); err != nil {
+                    b.Fatalf("%s Unmarshal error: %v", lib.name, err)
+                }
+            }
+
+            valid := func(b *testing.B) {
+                val1, val2 := tt.new(), tt.new()
+                if err := json.Unmarshal(tt.data, val1); err != nil {
+                    panic(err)
+                }
+                if err := lib.unmarshal(tt.data, val2); err != nil {
+                    panic(err)
+                }
+                if !reflect.DeepEqual(val1, val2) {
+                    b.Fatalf("%s Unmarshal output mismatch:\ngot  %v\nwant %v", lib.name, val2, val1)
+                }
+            }
+
+            name := fmt.Sprintf("%s_%s", tt.name, lib.name)
+            b.Run(name, func(b *testing.B) {
+                if pretouchFlag {
+                    pretouch()
+                }
+                valid(b)
+                b.ResetTimer()
+                b.ReportAllocs()
+                b.SetBytes(int64(len(tt.data)))
+                for i := 0; i < b.N; i++ {
+                    run(b)
+                }
+            })
+        }
+    }
+}
+
+func runUnmarshalI(b *testing.B) {
+    for _, tt := range jsonTestdata() {
+        for _, lib := range jsonLibs {
+            var val any = tt.newI()
+            pretouch := func() {
+                _ = lib.unmarshal(tt.data, val)
+            }
+    
+            run := func(b *testing.B) {
+                val = tt.newI()
+                if err := lib.unmarshal(tt.data, val); err != nil {
+                    b.Fatalf("%s Unmarshal error: %v", lib.name, err)
+                }
+            }
+
+            valid := func(b *testing.B) {
+                val1, val2 := tt.newI(), tt.newI()
+                if err := json.Unmarshal(tt.data, val1); err != nil {
+                    panic(err)
+                }
+                if err := lib.unmarshal(tt.data, val2); err != nil {
+                    panic(err)
+                }
+                if !reflect.DeepEqual(val1, val2) {
+                    b.Fatalf("%s Unmarshal output mismatch:\ngot  %v\nwant %v", lib.name, val2, val1)
+                }
+            }
+
+            name := fmt.Sprintf("%s_%s", tt.name, lib.name)
+            b.Run(name, func(b *testing.B) {
+                if pretouchFlag {
+                    pretouch()
+                }
+                valid(b)
+                b.ResetTimer()
+                b.ReportAllocs()
+                b.SetBytes(int64(len(tt.data)))
+                for i := 0; i < b.N; i++ {
+                    run(b)
+                }
+            })
+        }
+    }
+}
+
+func runMarshalC(b *testing.B) {
+    for _, tt := range jsonTestdata() {
+        for _, lib := range jsonLibs {
+            pretouch := func() {
+                _, _ = lib.marshal(tt.val)
+            }
+
+            run := func(b *testing.B) {
+                if _, err := lib.marshal(tt.val); err != nil {
+                    b.Fatalf("%s Marshal error: %v", lib.name, err)
+                }
+            }
+
+            valid := func(b *testing.B) {
+                // some details are different with encoding/json, so we compare the unmarshal results from marshaled buffer
+                var buf1, buf2 []byte
+                buf1, err := json.Marshal(tt.val)
+                if err != nil {
+                    b.Fatalf("encoding/json Marshal error: %v", err)
+                }
+                buf2, err = lib.marshal(tt.val); 
+                if err != nil {
+                    b.Fatalf("%s Marshal error: %v", lib.name, err)
+                }
+                var val1, val2 = tt.new(), tt.new()
+                if err := json.Unmarshal(buf1, val1); err != nil {
+                    b.Fatalf("encoding/json Unmarshal again error: %v", err)
+                }
+                if err := lib.unmarshal(buf2, val2); err != nil {
+                    b.Fatalf("%s Unmarshal again error: %v", lib.name, err)
+                }
+                if !reflect.DeepEqual(val1, val2) {
+                    b.Fatalf("Unmarshal again output mismatch\n")
+                }
+            }
+
+            name := fmt.Sprintf("%s_%s", tt.name, lib.name)
+            b.Run(name, func(b *testing.B) {
+                if pretouchFlag {
+                    pretouch()
+                }
+                if (validFlag) {
+                    valid(b)
+                }
+                b.ResetTimer()
+                b.ReportAllocs()
+                b.SetBytes(int64(len(tt.data)))
+                for i := 0; i < b.N; i++ {
+                    run(b)
+                }
+            })
+        }
+    }
+}
+
+func runMarshalI(b *testing.B) {
+    for _, tt := range jsonTestdata() {
+        for _, lib := range jsonLibs {
+            pretouch := func() {
+                _, _ = lib.marshal(tt.valI)
+            }
+
+            run := func(b *testing.B) {
+                if _, err := lib.marshal(tt.valI); err != nil {
+                    b.Fatalf("%s Marshal error: %v", lib.name, err)
+                }
+            }
+
+            valid := func(b *testing.B) {
+                // some details are different with encoding/json, so we compare the unmarshal results from marshaled buffer
+                var buf1, buf2 []byte
+                buf1, err := json.Marshal(tt.valI)
+                if err != nil {
+                    b.Fatalf("encoding/json Marshal error: %v", err)
+                }
+                buf2, err = lib.marshal(tt.valI); 
+                if err != nil {
+                    b.Fatalf("%s Marshal error: %v", lib.name, err)
+                }
+                var val1, val2 = tt.new(), tt.new()
+                if err := json.Unmarshal(buf1, val1); err != nil {
+                    b.Fatalf("encoding/json Unmarshal again error: %v", err)
+                }
+                if err := lib.unmarshal(buf2, val2); err != nil {
+                    b.Fatalf("%s Unmarshal again error: %v", lib.name, err)
+                }
+                if !reflect.DeepEqual(val1, val2) {
+                    b.Fatalf("Unmarshal again output mismatch\n")
+                }
+            }
+
+            name := fmt.Sprintf("%s_%s", tt.name, lib.name)
+            b.Run(name, func(b *testing.B) {
+                if pretouchFlag {
+                    pretouch()
+                }
+                if (validFlag) {
+                    valid(b)
+                }
+                b.ResetTimer()
+                b.ReportAllocs()
+                b.SetBytes(int64(len(tt.data)))
+                for i := 0; i < b.N; i++ {
+                    run(b)
+                }
+            })
+        }
+    }
+}

generic_test/go.mod

@@ -4,8 +4,16 @@ go 1.18
 
 require (
 	github.com/bytedance/sonic v1.0.0
+	github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539
+	github.com/goccy/go-json v0.9.4
+	github.com/json-iterator/go v1.1.12
+)
+
+require (
 	github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 )

generic_test/go.sum

@@ -3,6 +3,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539 h1:mhqU04ze8mDNsTU2WFQVw1tbjYalOdIXx91JlBse4tk=
+github.com/go-json-experiment/json v0.0.0-20220603215908-554802c1e539/go.mod h1:jbpkervfdK2HCcB2YEFmwYeaq057KFiaaKTNTHV4OOQ=
 github.com/goccy/go-json v0.9.4 h1:L8MLKG2mvVXiQu07qB6hmfqeSYQdOnqPot2GhsIwIaI=
 github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

generic_test/sonic_test.go

@@ -19,11 +19,11 @@
 package generic_test
 
 import (
-	`testing`
+    `testing`
     `reflect`
     
     `github.com/bytedance/sonic`
-	`github.com/bytedance/sonic/option`
+    `github.com/bytedance/sonic/option`
     `github.com/bytedance/sonic/ast`
 )
 
@@ -56,8 +56,8 @@ func getAny[S Str, B Bytes, T SliceAny](src S, path T) (ast.Node, error) {
 }
 
 func pretouchAny[T Any](v T, opts ...option.CompileOption) error {
-	rt := reflect.TypeOf(v)
+    rt := reflect.TypeOf(v)
-	return sonic.Pretouch(rt, opts...)
+    return sonic.Pretouch(rt, opts...)
 }
 
 type Basic interface {
@@ -73,8 +73,8 @@ func marshalBasic[B Bytes, T Basic](val T) (B, error) {
 }
 
 func pretouchBasic[T Basic](v T, opts ...option.CompileOption) error {
-	rt := reflect.TypeOf(v)
+    rt := reflect.TypeOf(v)
-	return sonic.Pretouch(rt, opts...)
+    return sonic.Pretouch(rt, opts...)
 }
 
 type Float64 float64
@@ -88,22 +88,22 @@ func TestGenericAPI(t *testing.T) {
     if err != nil {
         t.Fatal(err)
     }
-	t.Logf("%s", out)
+    t.Logf("%s", out)
     
-	var x0 = struct{
+    var x0 = struct{
-		A []Any `json:"a"`
+        A []Any `json:"a"`
-	}{}
+    }{}
-	if err := pretouchAny(x0); err != nil {
+    if err := pretouchAny(x0); err != nil {
-		t.Fatal(err)
+        t.Fatal(err)
-	}
+    }
-	if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x0); err != nil {
+    if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x0); err != nil {
         t.Fatal(t)
     }
     out0, err := marshalAny(x0)
     if err != nil {
         t.Fatal(err)
     }
-	t.Logf("%s", out0)
+    t.Logf("%s", out0)
 
     var x1 int
     if err := unmarshalBasic(`1`, &x1); err != nil {
@@ -115,8 +115,8 @@ func TestGenericAPI(t *testing.T) {
     }
     t.Logf("%s", out1)
 
-	var x2 Float64 = 1
+    var x2 Float64 = 1
-	// if err := unmarshalBasic(`1`, &x2); err != nil {
+    // if err := unmarshalBasic(`1`, &x2); err != nil {
     //     t.Fatal(t)
     // }
     out2, err := marshalBasic(x2)
@@ -133,20 +133,20 @@ func TestGenericAPI(t *testing.T) {
     if err != nil {
         t.Fatal(err)
     }
-	t.Logf("%s", out3)
+    t.Logf("%s", out3)
 
-	var x4 Float64 = 1
+    var x4 Float64 = 1
-	if err := pretouchBasic(x4); err != nil {
+    if err := pretouchBasic(x4); err != nil {
-		t.Fatal(err)
+        t.Fatal(err)
-	}
+    }
-	// if err := unmarshalBasic(`0.5`, &x4); err != nil {
+    // if err := unmarshalBasic(`0.5`, &x4); err != nil {
     //     t.Fatal(t)
     // }
     out4, err := marshalBasic(x4)
     if err != nil {
         t.Fatal(err)
     }
-	t.Logf("%s", out4)
+    t.Logf("%s", out4)
 
     root, err := getAny(`{"a":[true,1,"hello world"]}`, []interface{}{"a", 1})
     if err != nil {

generic_test/testdata_test.go

@@ -0,0 +1,620 @@
+// +build go1.18
+
+/*
+ * Copyright 2020 The Go Authors. All rights reserved.
+ * Modifications Copyright 2022 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package generic_test
+
+import (
+    `bytes`
+    `compress/gzip`
+    `io/ioutil`
+    `path/filepath`
+    `sort`
+    `strings`
+    `sync`
+    `time`
+    `encoding/json`
+)
+
+type jsonTestdataEntry struct {
+    name string
+    data []byte
+    val  any
+    valI any
+    new  func() any // nil if there is no concrete type for this
+    newI func() any
+}
+
+var (
+    jsonTestdataOnce sync.Once
+    jsonTestdataLazy []jsonTestdataEntry
+)
+
+func jsonTestdata() []jsonTestdataEntry {
+    jsonTestdataOnce.Do(func() {
+        fis, err := ioutil.ReadDir("../testdata")
+        if err != nil {
+            panic(err)
+        }
+        sort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() })
+        for _, fi := range fis {
+            if !strings.HasSuffix(fi.Name(), ".json.gz") {
+                continue
+            }
+
+            // Convert snake_case file name to CamelCase.
+            words := strings.Split(strings.TrimSuffix(fi.Name(), ".json.gz"), "_")
+            for i := range words {
+                words[i] = strings.Title(words[i])
+            }
+            name := strings.Join(words, "")
+
+            // Read and decompress the test data.
+            b, err := ioutil.ReadFile(filepath.Join("../testdata", fi.Name()))
+            if err != nil {
+                panic(err)
+            }
+            zr, err := gzip.NewReader(bytes.NewReader(b))
+            if err != nil {
+                panic(err)
+            }
+            data, err := ioutil.ReadAll(zr)
+            if err != nil {
+                panic(err)
+            }
+
+            // Check whether there is a concrete type for this data.
+            var newFn func() any
+            switch name {
+            case "CanadaGeometry":
+                newFn = func() any { return new(canadaRoot) }
+            case "CitmCatalog":
+                newFn = func() any { return new(citmRoot) }
+            case "GolangSource":
+                newFn = func() any { return new(golangRoot) }
+            case "StringEscaped":
+                newFn = func() any { return new(stringRoot) }
+            case "StringUnicode":
+                newFn = func() any { return new(stringRoot) }
+            case "SyntheaFhir":
+                newFn = func() any { return new(syntheaRoot) }
+            case "TwitterStatus":
+                newFn = func() any { return new(twitterRoot) }
+            }
+
+            var newFnI = func() any { return new(any) }
+
+            // Fill into value
+            var val any = newFn()
+            err = json.Unmarshal(data, val)
+            if err != nil {
+                panic(err)
+            }
+            var valI any = newFnI()
+            err = json.Unmarshal(data, valI)
+            if err != nil {
+                panic(err)
+            }
+                        
+            jsonTestdataLazy = append(jsonTestdataLazy, jsonTestdataEntry{name, data, val, valI, newFn, newFnI})
+        }
+    })
+    return jsonTestdataLazy
+}
+
+type (
+    canadaRoot struct {
+        Type     string `json:"type"`
+        Features []struct {
+            Type       string `json:"type"`
+            Properties struct {
+                Name string `json:"name"`
+            } `json:"properties"`
+            Geometry struct {
+                Type        string         `json:"type"`
+                Coordinates [][][2]float64 `json:"coordinates"`
+            } `json:"geometry"`
+        } `json:"features"`
+    }
+)
+
+type (
+    citmRoot struct {
+        AreaNames                map[int64]string `json:"areaNames"`
+        AudienceSubCategoryNames map[int64]string `json:"audienceSubCategoryNames"`
+        BlockNames               map[int64]string `json:"blockNames"`
+        Events                   map[int64]struct {
+            Description string `json:"description"`
+            ID          int    `json:"id"`
+            Logo        string `json:"logo"`
+            Name        string `json:"name"`
+            SubTopicIds []int  `json:"subTopicIds"`
+            SubjectCode any    `json:"subjectCode"`
+            Subtitle    any    `json:"subtitle"`
+            TopicIds    []int  `json:"topicIds"`
+        } `json:"events"`
+        Performances []struct {
+            EventID int `json:"eventId"`
+            ID      int `json:"id"`
+            Logo    any `json:"logo"`
+            Name    any `json:"name"`
+            Prices  []struct {
+                Amount                int   `json:"amount"`
+                AudienceSubCategoryID int64 `json:"audienceSubCategoryId"`
+                SeatCategoryID        int64 `json:"seatCategoryId"`
+            } `json:"prices"`
+            SeatCategories []struct {
+                Areas []struct {
+                    AreaID   int   `json:"areaId"`
+                    BlockIds []any `json:"blockIds"`
+                } `json:"areas"`
+                SeatCategoryID int `json:"seatCategoryId"`
+            } `json:"seatCategories"`
+            SeatMapImage any    `json:"seatMapImage"`
+            Start        int64  `json:"start"`
+            VenueCode    string `json:"venueCode"`
+        } `json:"performances"`
+        SeatCategoryNames map[uint64]string   `json:"seatCategoryNames"`
+        SubTopicNames     map[uint64]string   `json:"subTopicNames"`
+        SubjectNames      map[uint64]string   `json:"subjectNames"`
+        TopicNames        map[uint64]string   `json:"topicNames"`
+        TopicSubTopics    map[uint64][]uint64 `json:"topicSubTopics"`
+        VenueNames        map[string]string   `json:"venueNames"`
+    }
+)
+
+type (
+    golangRoot struct {
+        Tree     *golangNode `json:"tree"`
+        Username string      `json:"username"`
+    }
+    golangNode struct {
+        Name     string       `json:"name"`
+        Kids     []golangNode `json:"kids"`
+        CLWeight float64      `json:"cl_weight"`
+        Touches  int          `json:"touches"`
+        MinT     uint64       `json:"min_t"`
+        MaxT     uint64       `json:"max_t"`
+        MeanT    uint64       `json:"mean_t"`
+    }
+)
+
+type (
+    stringRoot struct {
+        Arabic                             string `json:"Arabic"`
+        ArabicPresentationFormsA           string `json:"Arabic Presentation Forms-A"`
+        ArabicPresentationFormsB           string `json:"Arabic Presentation Forms-B"`
+        Armenian                           string `json:"Armenian"`
+        Arrows                             string `json:"Arrows"`
+        Bengali                            string `json:"Bengali"`
+        Bopomofo                           string `json:"Bopomofo"`
+        BoxDrawing                         string `json:"Box Drawing"`
+        CJKCompatibility                   string `json:"CJK Compatibility"`
+        CJKCompatibilityForms              string `json:"CJK Compatibility Forms"`
+        CJKCompatibilityIdeographs         string `json:"CJK Compatibility Ideographs"`
+        CJKSymbolsAndPunctuation           string `json:"CJK Symbols and Punctuation"`
+        CJKUnifiedIdeographs               string `json:"CJK Unified Ideographs"`
+        CJKUnifiedIdeographsExtensionA     string `json:"CJK Unified Ideographs Extension A"`
+        CJKUnifiedIdeographsExtensionB     string `json:"CJK Unified Ideographs Extension B"`
+        Cherokee                           string `json:"Cherokee"`
+        CurrencySymbols                    string `json:"Currency Symbols"`
+        Cyrillic                           string `json:"Cyrillic"`
+        CyrillicSupplementary              string `json:"Cyrillic Supplementary"`
+        Devanagari                         string `json:"Devanagari"`
+        EnclosedAlphanumerics              string `json:"Enclosed Alphanumerics"`
+        EnclosedCJKLettersAndMonths        string `json:"Enclosed CJK Letters and Months"`
+        Ethiopic                           string `json:"Ethiopic"`
+        GeometricShapes                    string `json:"Geometric Shapes"`
+        Georgian                           string `json:"Georgian"`
+        GreekAndCoptic                     string `json:"Greek and Coptic"`
+        Gujarati                           string `json:"Gujarati"`
+        Gurmukhi                           string `json:"Gurmukhi"`
+        HangulCompatibilityJamo            string `json:"Hangul Compatibility Jamo"`
+        HangulJamo                         string `json:"Hangul Jamo"`
+        HangulSyllables                    string `json:"Hangul Syllables"`
+        Hebrew                             string `json:"Hebrew"`
+        Hiragana                           string `json:"Hiragana"`
+        IPAExtentions                      string `json:"IPA Extentions"`
+        KangxiRadicals                     string `json:"Kangxi Radicals"`
+        Katakana                           string `json:"Katakana"`
+        Khmer                              string `json:"Khmer"`
+        KhmerSymbols                       string `json:"Khmer Symbols"`
+        Latin                              string `json:"Latin"`
+        LatinExtendedAdditional            string `json:"Latin Extended Additional"`
+        Latin1Supplement                   string `json:"Latin-1 Supplement"`
+        LatinExtendedA                     string `json:"Latin-Extended A"`
+        LatinExtendedB                     string `json:"Latin-Extended B"`
+        LetterlikeSymbols                  string `json:"Letterlike Symbols"`
+        Malayalam                          string `json:"Malayalam"`
+        MathematicalAlphanumericSymbols    string `json:"Mathematical Alphanumeric Symbols"`
+        MathematicalOperators              string `json:"Mathematical Operators"`
+        MiscellaneousSymbols               string `json:"Miscellaneous Symbols"`
+        Mongolian                          string `json:"Mongolian"`
+        NumberForms                        string `json:"Number Forms"`
+        Oriya                              string `json:"Oriya"`
+        PhoneticExtensions                 string `json:"Phonetic Extensions"`
+        SupplementalArrowsB                string `json:"Supplemental Arrows-B"`
+        Syriac                             string `json:"Syriac"`
+        Tamil                              string `json:"Tamil"`
+        Thaana                             string `json:"Thaana"`
+        Thai                               string `json:"Thai"`
+        UnifiedCanadianAboriginalSyllabics string `json:"Unified Canadian Aboriginal Syllabics"`
+        YiRadicals                         string `json:"Yi Radicals"`
+        YiSyllables                        string `json:"Yi Syllables"`
+    }
+)
+
+type (
+    syntheaRoot struct {
+        Entry []struct {
+            FullURL string `json:"fullUrl"`
+            Request *struct {
+                Method string `json:"method"`
+                URL    string `json:"url"`
+            } `json:"request"`
+            Resource *struct {
+                AbatementDateTime time.Time   `json:"abatementDateTime"`
+                AchievementStatus syntheaCode `json:"achievementStatus"`
+                Active            bool        `json:"active"`
+                Activity          []struct {
+                    Detail *struct {
+                        Code     syntheaCode      `json:"code"`
+                        Location syntheaReference `json:"location"`
+                        Status   string           `json:"status"`
+                    } `json:"detail"`
+                } `json:"activity"`
+                Address        []syntheaAddress   `json:"address"`
+                Addresses      []syntheaReference `json:"addresses"`
+                AuthoredOn     time.Time          `json:"authoredOn"`
+                BillablePeriod syntheaRange       `json:"billablePeriod"`
+                BirthDate      string             `json:"birthDate"`
+                CareTeam       []struct {
+                    Provider  syntheaReference `json:"provider"`
+                    Reference string           `json:"reference"`
+                    Role      syntheaCode      `json:"role"`
+                    Sequence  int64            `json:"sequence"`
+                } `json:"careTeam"`
+                Category       []syntheaCode    `json:"category"`
+                Claim          syntheaReference `json:"claim"`
+                Class          syntheaCoding    `json:"class"`
+                ClinicalStatus syntheaCode      `json:"clinicalStatus"`
+                Code           syntheaCode      `json:"code"`
+                Communication  []struct {
+                    Language syntheaCode `json:"language"`
+                } `json:"communication"`
+                Component []struct {
+                    Code          syntheaCode   `json:"code"`
+                    ValueQuantity syntheaCoding `json:"valueQuantity"`
+                } `json:"component"`
+                Contained []struct {
+                    Beneficiary  syntheaReference   `json:"beneficiary"`
+                    ID           string             `json:"id"`
+                    Intent       string             `json:"intent"`
+                    Payor        []syntheaReference `json:"payor"`
+                    Performer    []syntheaReference `json:"performer"`
+                    Requester    syntheaReference   `json:"requester"`
+                    ResourceType string             `json:"resourceType"`
+                    Status       string             `json:"status"`
+                    Subject      syntheaReference   `json:"subject"`
+                    Type         syntheaCode        `json:"type"`
+                } `json:"contained"`
+                Created          time.Time   `json:"created"`
+                DeceasedDateTime time.Time   `json:"deceasedDateTime"`
+                Description      syntheaCode `json:"description"`
+                Diagnosis        []struct {
+                    DiagnosisReference syntheaReference `json:"diagnosisReference"`
+                    Sequence           int64            `json:"sequence"`
+                    Type               []syntheaCode    `json:"type"`
+                } `json:"diagnosis"`
+                DosageInstruction []struct {
+                    AsNeededBoolean bool `json:"asNeededBoolean"`
+                    DoseAndRate     []struct {
+                        DoseQuantity *struct {
+                            Value float64 `json:"value"`
+                        } `json:"doseQuantity"`
+                        Type syntheaCode `json:"type"`
+                    } `json:"doseAndRate"`
+                    Sequence int64 `json:"sequence"`
+                    Timing   *struct {
+                        Repeat *struct {
+                            Frequency  int64   `json:"frequency"`
+                            Period     float64 `json:"period"`
+                            PeriodUnit string  `json:"periodUnit"`
+                        } `json:"repeat"`
+                    } `json:"timing"`
+                } `json:"dosageInstruction"`
+                EffectiveDateTime time.Time          `json:"effectiveDateTime"`
+                Encounter         syntheaReference   `json:"encounter"`
+                Extension         []syntheaExtension `json:"extension"`
+                Gender            string             `json:"gender"`
+                Goal              []syntheaReference `json:"goal"`
+                ID                string             `json:"id"`
+                Identifier        []struct {
+                    System string      `json:"system"`
+                    Type   syntheaCode `json:"type"`
+                    Use    string      `json:"use"`
+                    Value  string      `json:"value"`
+                } `json:"identifier"`
+                Insurance []struct {
+                    Coverage syntheaReference `json:"coverage"`
+                    Focal    bool             `json:"focal"`
+                    Sequence int64            `json:"sequence"`
+                } `json:"insurance"`
+                Insurer syntheaReference `json:"insurer"`
+                Intent  string           `json:"intent"`
+                Issued  time.Time        `json:"issued"`
+                Item    []struct {
+                    Adjudication []struct {
+                        Amount   syntheaCurrency `json:"amount"`
+                        Category syntheaCode     `json:"category"`
+                    } `json:"adjudication"`
+                    Category                syntheaCode        `json:"category"`
+                    DiagnosisSequence       []int64            `json:"diagnosisSequence"`
+                    Encounter               []syntheaReference `json:"encounter"`
+                    InformationSequence     []int64            `json:"informationSequence"`
+                    LocationCodeableConcept syntheaCode        `json:"locationCodeableConcept"`
+                    Net                     syntheaCurrency    `json:"net"`
+                    ProcedureSequence       []int64            `json:"procedureSequence"`
+                    ProductOrService        syntheaCode        `json:"productOrService"`
+                    Sequence                int64              `json:"sequence"`
+                    ServicedPeriod          syntheaRange       `json:"servicedPeriod"`
+                } `json:"item"`
+                LifecycleStatus           string             `json:"lifecycleStatus"`
+                ManagingOrganization      []syntheaReference `json:"managingOrganization"`
+                MaritalStatus             syntheaCode        `json:"maritalStatus"`
+                MedicationCodeableConcept syntheaCode        `json:"medicationCodeableConcept"`
+                MultipleBirthBoolean      bool               `json:"multipleBirthBoolean"`
+                Name                      json.RawMessage           `json:"name"`
+                NumberOfInstances         int64              `json:"numberOfInstances"`
+                NumberOfSeries            int64              `json:"numberOfSeries"`
+                OccurrenceDateTime        time.Time          `json:"occurrenceDateTime"`
+                OnsetDateTime             time.Time          `json:"onsetDateTime"`
+                Outcome                   string             `json:"outcome"`
+                Participant               []struct {
+                    Individual syntheaReference `json:"individual"`
+                    Member     syntheaReference `json:"member"`
+                    Role       []syntheaCode    `json:"role"`
+                } `json:"participant"`
+                Patient syntheaReference `json:"patient"`
+                Payment *struct {
+                    Amount syntheaCurrency `json:"amount"`
+                } `json:"payment"`
+                PerformedPeriod syntheaRange     `json:"performedPeriod"`
+                Period          syntheaRange     `json:"period"`
+                Prescription    syntheaReference `json:"prescription"`
+                PrimarySource   bool             `json:"primarySource"`
+                Priority        syntheaCode      `json:"priority"`
+                Procedure       []struct {
+                    ProcedureReference syntheaReference `json:"procedureReference"`
+                    Sequence           int64            `json:"sequence"`
+                } `json:"procedure"`
+                Provider        syntheaReference   `json:"provider"`
+                ReasonCode      []syntheaCode      `json:"reasonCode"`
+                ReasonReference []syntheaReference `json:"reasonReference"`
+                RecordedDate    time.Time          `json:"recordedDate"`
+                Referral        syntheaReference   `json:"referral"`
+                Requester       syntheaReference   `json:"requester"`
+                ResourceType    string             `json:"resourceType"`
+                Result          []syntheaReference `json:"result"`
+                Series          []struct {
+                    BodySite syntheaCoding `json:"bodySite"`
+                    Instance []struct {
+                        Number   int64         `json:"number"`
+                        SopClass syntheaCoding `json:"sopClass"`
+                        Title    string        `json:"title"`
+                        UID      string        `json:"uid"`
+                    } `json:"instance"`
+                    Modality          syntheaCoding `json:"modality"`
+                    Number            int64         `json:"number"`
+                    NumberOfInstances int64         `json:"numberOfInstances"`
+                    Started           string        `json:"started"`
+                    UID               string        `json:"uid"`
+                } `json:"series"`
+                ServiceProvider syntheaReference `json:"serviceProvider"`
+                Started         time.Time        `json:"started"`
+                Status          string           `json:"status"`
+                Subject         syntheaReference `json:"subject"`
+                SupportingInfo  []struct {
+                    Category       syntheaCode      `json:"category"`
+                    Sequence       int64            `json:"sequence"`
+                    ValueReference syntheaReference `json:"valueReference"`
+                } `json:"supportingInfo"`
+                Telecom              []map[string]string `json:"telecom"`
+                Text                 map[string]string   `json:"text"`
+                Total                json.RawMessage            `json:"total"`
+                Type                 json.RawMessage            `json:"type"`
+                Use                  string              `json:"use"`
+                VaccineCode          syntheaCode         `json:"vaccineCode"`
+                ValueCodeableConcept syntheaCode         `json:"valueCodeableConcept"`
+                ValueQuantity        syntheaCoding       `json:"valueQuantity"`
+                VerificationStatus   syntheaCode         `json:"verificationStatus"`
+            } `json:"resource"`
+        } `json:"entry"`
+        ResourceType string `json:"resourceType"`
+        Type         string `json:"type"`
+    }
+    syntheaCode struct {
+        Coding []syntheaCoding `json:"coding"`
+        Text   string          `json:"text"`
+    }
+    syntheaCoding struct {
+        Code    string  `json:"code"`
+        Display string  `json:"display"`
+        System  string  `json:"system"`
+        Unit    string  `json:"unit"`
+        Value   float64 `json:"value"`
+    }
+    syntheaReference struct {
+        Display   string `json:"display"`
+        Reference string `json:"reference"`
+    }
+    syntheaAddress struct {
+        City       string             `json:"city"`
+        Country    string             `json:"country"`
+        Extension  []syntheaExtension `json:"extension"`
+        Line       []string           `json:"line"`
+        PostalCode string             `json:"postalCode"`
+        State      string             `json:"state"`
+    }
+    syntheaExtension struct {
+        URL          string             `json:"url"`
+        ValueAddress syntheaAddress     `json:"valueAddress"`
+        ValueCode    string             `json:"valueCode"`
+        ValueDecimal float64            `json:"valueDecimal"`
+        ValueString  string             `json:"valueString"`
+        Extension    []syntheaExtension `json:"extension"`
+    }
+    syntheaRange struct {
+        End   time.Time `json:"end"`
+        Start time.Time `json:"start"`
+    }
+    syntheaCurrency struct {
+        Currency string  `json:"currency"`
+        Value    float64 `json:"value"`
+    }
+)
+
+type (
+    twitterRoot struct {
+        Statuses       []twitterStatus `json:"statuses"`
+        SearchMetadata struct {
+            CompletedIn float64 `json:"completed_in"`
+            MaxID       int64   `json:"max_id"`
+            MaxIDStr    int64   `json:"max_id_str,string"`
+            NextResults string  `json:"next_results"`
+            Query       string  `json:"query"`
+            RefreshURL  string  `json:"refresh_url"`
+            Count       int     `json:"count"`
+            SinceID     int     `json:"since_id"`
+            SinceIDStr  int     `json:"since_id_str,string"`
+        } `json:"search_metadata"`
+    }
+    twitterStatus struct {
+        Metadata struct {
+            ResultType      string `json:"result_type"`
+            IsoLanguageCode string `json:"iso_language_code"`
+        } `json:"metadata"`
+        CreatedAt            string          `json:"created_at"`
+        ID                   int64           `json:"id"`
+        IDStr                int64           `json:"id_str,string"`
+        Text                 string          `json:"text"`
+        Source               string          `json:"source"`
+        Truncated            bool            `json:"truncated"`
+        InReplyToStatusID    int64           `json:"in_reply_to_status_id"`
+        InReplyToStatusIDStr int64           `json:"in_reply_to_status_id_str,string"`
+        InReplyToUserID      int64           `json:"in_reply_to_user_id"`
+        InReplyToUserIDStr   int64           `json:"in_reply_to_user_id_str,string"`
+        InReplyToScreenName  string          `json:"in_reply_to_screen_name"`
+        User                 twitterUser     `json:"user,omitempty"`
+        Geo                  any             `json:"geo"`
+        Coordinates          any             `json:"coordinates"`
+        Place                any             `json:"place"`
+        Contributors         any             `json:"contributors"`
+        RetweeetedStatus     *twitterStatus  `json:"retweeted_status"`
+        RetweetCount         int             `json:"retweet_count"`
+        FavoriteCount        int             `json:"favorite_count"`
+        Entities             twitterEntities `json:"entities,omitempty"`
+        Favorited            bool            `json:"favorited"`
+        Retweeted            bool            `json:"retweeted"`
+        PossiblySensitive    bool            `json:"possibly_sensitive"`
+        Lang                 string          `json:"lang"`
+    }
+    twitterUser struct {
+        ID                             int64           `json:"id"`
+        IDStr                          string          `json:"id_str"`
+        Name                           string          `json:"name"`
+        ScreenName                     string          `json:"screen_name"`
+        Location                       string          `json:"location"`
+        Description                    string          `json:"description"`
+        URL                            any             `json:"url"`
+        Entities                       twitterEntities `json:"entities"`
+        Protected                      bool            `json:"protected"`
+        FollowersCount                 int             `json:"followers_count"`
+        FriendsCount                   int             `json:"friends_count"`
+        ListedCount                    int             `json:"listed_count"`
+        CreatedAt                      string          `json:"created_at"`
+        FavouritesCount                int             `json:"favourites_count"`
+        UtcOffset                      int             `json:"utc_offset"`
+        TimeZone                       string          `json:"time_zone"`
+        GeoEnabled                     bool            `json:"geo_enabled"`
+        Verified                       bool            `json:"verified"`
+        StatusesCount                  int             `json:"statuses_count"`
+        Lang                           string          `json:"lang"`
+        ContributorsEnabled            bool            `json:"contributors_enabled"`
+        IsTranslator                   bool            `json:"is_translator"`
+        IsTranslationEnabled           bool            `json:"is_translation_enabled"`
+        ProfileBackgroundColor         string          `json:"profile_background_color"`
+        ProfileBackgroundImageURL      string          `json:"profile_background_image_url"`
+        ProfileBackgroundImageURLHTTPS string          `json:"profile_background_image_url_https"`
+        ProfileBackgroundTile          bool            `json:"profile_background_tile"`
+        ProfileImageURL                string          `json:"profile_image_url"`
+        ProfileImageURLHTTPS           string          `json:"profile_image_url_https"`
+        ProfileBannerURL               string          `json:"profile_banner_url"`
+        ProfileLinkColor               string          `json:"profile_link_color"`
+        ProfileSidebarBorderColor      string          `json:"profile_sidebar_border_color"`
+        ProfileSidebarFillColor        string          `json:"profile_sidebar_fill_color"`
+        ProfileTextColor               string          `json:"profile_text_color"`
+        ProfileUseBackgroundImage      bool            `json:"profile_use_background_image"`
+        DefaultProfile                 bool            `json:"default_profile"`
+        DefaultProfileImage            bool            `json:"default_profile_image"`
+        Following                      bool            `json:"following"`
+        FollowRequestSent              bool            `json:"follow_request_sent"`
+        Notifications                  bool            `json:"notifications"`
+    }
+    twitterEntities struct {
+        Hashtags     []any        `json:"hashtags"`
+        Symbols      []any        `json:"symbols"`
+        URL          *twitterURL  `json:"url"`
+        URLs         []twitterURL `json:"urls"`
+        UserMentions []struct {
+            ScreenName string `json:"screen_name"`
+            Name       string `json:"name"`
+            ID         int64  `json:"id"`
+            IDStr      int64  `json:"id_str,string"`
+            Indices    []int  `json:"indices"`
+        } `json:"user_mentions"`
+        Description struct {
+            URLs []twitterURL `json:"urls"`
+        } `json:"description"`
+        Media []struct {
+            ID            int64  `json:"id"`
+            IDStr         string `json:"id_str"`
+            Indices       []int  `json:"indices"`
+            MediaURL      string `json:"media_url"`
+            MediaURLHTTPS string `json:"media_url_https"`
+            URL           string `json:"url"`
+            DisplayURL    string `json:"display_url"`
+            ExpandedURL   string `json:"expanded_url"`
+            Type          string `json:"type"`
+            Sizes         map[string]struct {
+                W      int    `json:"w"`
+                H      int    `json:"h"`
+                Resize string `json:"resize"`
+            } `json:"sizes"`
+            SourceStatusID    int64 `json:"source_status_id"`
+            SourceStatusIDStr int64 `json:"source_status_id_str,string"`
+        } `json:"media"`
+    }
+    twitterURL struct {
+        URL         string       `json:"url"`
+        URLs        []twitterURL `json:"urls"`
+        ExpandedURL string       `json:"expanded_url"`
+        DisplayURL  string       `json:"display_url"`
+        Indices     []int        `json:"indices"`
+    }
+)