XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-21 00:46:43 +08:00
Code Issues Projects Releases Wiki Activity

feat: support Go1.18 (#205)

Browse source 
* feat: support Go1.18

* fix: re-encoding `moduledata.pclntab`

* test: add generic func wrap test
This commit is contained in:
Yi Duan 2022-03-28 20:43:43 +08:00 committed by GitHub
parent 94232a7b2e
commit cb1702dc8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 469 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.github/workflows/push-check-go118.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Push Check Go1.18
on: push
jobs:
build:
runs-on: self-hosted
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Unit Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race -covermode=atomic -coverprofile=coverage.out ./...
- name: Generic Test
run: go test -v -gcflags=-d=checkptr=0 -race -covermode=atomic ./generic_test

2
decoder/assembler_amd64_go117.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
/*
* Copyright 2021 ByteDance Inc.

36
decoder/assembler_test.go
View file

@ -1,18 +1,18 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
@ -72,9 +72,9 @@ func (self *UjsonStruct) UnmarshalJSON(v []byte) error {
}
const (
_OP_dbg_get_sr _Op = 253
_OP_dbg_set_sr _Op = 254
_OP_dbg_break _Op = 255
_OP_dbg_get_sr _Op = 253
_OP_dbg_set_sr _Op = 254
_OP_dbg_break _Op = 255
)
func (self *_Assembler) _asm_OP_dbg_get_sr(_ *_Instr) {
@ -739,4 +739,4 @@ func TestAssembler_DecodeByteSlice_List(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, len(s), pos)
assert.Equal(t, []byte("hello, world"), v)
}
}

2
decoder/generic_amd64_go117.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
/*
* Copyright 2021 ByteDance Inc.

2
decoder/generic_amd64_go117_test.s
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
//
// Copyright 2021 ByteDance Inc.

2
encoder/assembler_amd64_go117.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
/*
* Copyright 2021 ByteDance Inc.

2
encoder/stubs_go117.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
/*
* Copyright 2021 ByteDance Inc.

13
generic_test/go.mod Normal file
View file

@ -0,0 +1,13 @@
module github.com/bytedance/sonic/generic_test
go 1.18
require (
github.com/bytedance/sonic v1.0.0
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
)
replace github.com/bytedance/sonic => ../.

39
generic_test/go.sum Normal file
View file

@ -0,0 +1,39 @@
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06 h1:1sDoSuDPWzhkdzNVxCxtIaKiAe96ESVPv8coGwc1gZ4=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/goccy/go-json v0.9.4 h1:L8MLKG2mvVXiQu07qB6hmfqeSYQdOnqPot2GhsIwIaI=
github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

160
generic_test/sonic_test.go Normal file
View file

@ -0,0 +1,160 @@
// +build go1.18
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package generic_test
import (
`testing`
`reflect`
`github.com/bytedance/sonic`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/ast`
)
type Str interface {
string
}
type Bytes interface {
[]byte
}
type Any interface {
any
}
type SliceAny interface {
[]any
}
func unmarshalAny[S Str, B Bytes, T Any](data S, val T) error {
return sonic.Unmarshal(B(data), val)
}
func marshalAny[B Bytes, T Any](val T) (B, error) {
return sonic.Marshal(val)
}
func getAny[S Str, B Bytes, T SliceAny](src S, path T) (ast.Node, error) {
return sonic.Get(B(src), path...)
}
func pretouchAny[T Any](v T, opts ...option.CompileOption) error {
rt := reflect.TypeOf(v)
return sonic.Pretouch(rt, opts...)
}
type Basic interface {
~*int|~*float64|~float64|~*string
}
func unmarshalBasic[S Str, B Bytes, T Basic](data S, val T) error {
return sonic.Unmarshal(B(data), val)
}
func marshalBasic[B Bytes, T Basic](val T) (B, error) {
return sonic.Marshal(val)
}
func pretouchBasic[T Basic](v T, opts ...option.CompileOption) error {
rt := reflect.TypeOf(v)
return sonic.Pretouch(rt, opts...)
}
type Float64 float64
func TestGenericAPI(t *testing.T) {
var x interface{}
if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x); err != nil {
t.Fatal(t)
}
out, err := marshalAny(x)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out)
var x0 = struct{
A []Any `json:"a"`
}{}
if err := pretouchAny(x0); err != nil {
t.Fatal(err)
}
if err := unmarshalAny(`{"a":[true,0.5,"hello world"]}`, &x0); err != nil {
t.Fatal(t)
}
out0, err := marshalAny(x0)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out0)
var x1 int
if err := unmarshalBasic(`1`, &x1); err != nil {
t.Fatal(t)
}
out1, err := marshalBasic(&x1)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out1)
var x2 Float64 = 1
// if err := unmarshalBasic(`1`, &x2); err != nil {
// t.Fatal(t)
// }
out2, err := marshalBasic(x2)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out2)
var x3 string
if err := unmarshalBasic(`"1"`, &x3); err != nil {
t.Fatal(t)
}
out3, err := marshalBasic(&x3)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out3)
var x4 Float64 = 1
if err := pretouchBasic(x4); err != nil {
t.Fatal(err)
}
// if err := unmarshalBasic(`0.5`, &x4); err != nil {
// t.Fatal(t)
// }
out4, err := marshalBasic(x4)
if err != nil {
t.Fatal(err)
}
t.Logf("%s", out4)
root, err := getAny(`{"a":[true,1,"hello world"]}`, []interface{}{"a", 1})
if err != nil {
t.Fatal(err)
}
f, err := root.Float64()
if err != nil {
t.Fatal(err)
}
t.Log(f)
}

6
go.work Normal file
View file

@ -0,0 +1,6 @@
go 1.18
use (
.
./generic_test
)

2
internal/loader/funcdata_go115.go
View file

@ -98,7 +98,7 @@ var findFuncTab = &_FindFuncBucket {
idx: 1,
}
func registerFunction(name string, pc uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
minpc := pc
maxpc := pc + size

2
internal/loader/funcdata_go116.go
View file

@ -126,7 +126,7 @@ func makePCtab(fp int) []byte {
return append([]byte{0}, encodeVariant((fp + 1) << 1)...)
}
func registerFunction(name string, pc uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
minpc := pc
maxpc := pc + size

195
internal/loader/funcdata_go118.go Normal file
View file

@ -0,0 +1,195 @@
// +build go1.18,!go1.19
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
// A FuncFlag holds bits about a function.
// This list must match the list in cmd/internal/objabi/funcid.go.
type funcFlag uint8
type _Func struct {
entryOff uint32 // start pc
nameoff int32 // function name
args int32 // in/out args size
deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
pcsp uint32
pcfile uint32
pcln uint32
npcdata uint32
cuOffset uint32 // runtime.cutab offset of this function's CU
funcID uint8 // set for certain special runtime functions
flag funcFlag
_ [1]byte // pad
nfuncdata uint8 // must be last
argptrs uint32
localptrs uint32
}
type _FuncTab struct {
entry uint32
funcoff uint32
}
type _PCHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type _BitVector struct {
n int32 // # of bits
bytedata *uint8
}
type _PtabEntry struct {
name int32
typ int32
}
type _TextSection struct {
vaddr uintptr // prelinked section vaddr
length uintptr // section length
baseaddr uintptr // relocated section address
}
type _ModuleData struct {
pcHeader *_PCHeader
funcnametab []byte
cutab []uint32
filetab []byte
pctab []byte
pclntable []byte
ftab []_FuncTab
findfunctab *_FindFuncBucket
minpc, maxpc uintptr
text, etext uintptr
noptrdata, enoptrdata uintptr
data, edata uintptr
bss, ebss uintptr
noptrbss, enoptrbss uintptr
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
gofunc uintptr
textsectmap []_TextSection
typelinks []int32
itablinks []unsafe.Pointer
ptab []_PtabEntry
pluginpath string
pkghashes []struct{}
modulename string
modulehashes []struct{}
hasmain uint8
gcdatamask, gcbssmask _BitVector
typemap map[int32]unsafe.Pointer
bad bool
next *_ModuleData
}
type _FindFuncBucket struct {
idx uint32
subbuckets [16]byte
}
func makePCtab(fp int) []byte {
return append([]byte{0}, encodeVariant((fp + 1) << 1)...)
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
minpc := pc
maxpc := pc + size
findFuncTab := make([]_FindFuncBucket, textSize/4096 + 1)
modHeader := &_PCHeader {
magic : 0xfffffff0,
minLC : 1,
nfunc : 1,
ptrSize : 4 << (^uintptr(0) >> 63),
textStart: minpc,
}
base := argptrs
if argptrs > localptrs {
base = localptrs
}
/* function entry */
lnt := []_Func {{
entryOff : 0,
nameoff : 1,
args : int32(args),
pcsp : 1,
nfuncdata : 2,
argptrs: uint32(argptrs - base),
localptrs: uint32(localptrs - base),
}}
nlnt := len(lnt)*int(unsafe.Sizeof(_Func{}))
plnt := unsafe.Pointer(&lnt[0])
/* function table */
ftab := []_FuncTab {
{entry : 0, funcoff : 16},
{entry : uint32(size)},
}
nftab := len(ftab)*int(unsafe.Sizeof(_FuncTab{}))
pftab := unsafe.Pointer(&ftab[0])
pclntab := make([]byte, 0, nftab + nlnt)
pclntab = append(pclntab, rt.BytesFrom(pftab, nftab, nftab)...)
pclntab = append(pclntab, rt.BytesFrom(plnt, nlnt, nlnt)...)
/* module data */
mod := &_ModuleData {
pcHeader : modHeader,
funcnametab : append(append([]byte{0}, name...), 0),
pctab : append(makePCtab(fp), encodeVariant(int(size))...),
pclntable : pclntab,
ftab : ftab,
text : minpc,
etext : pc + textSize,
findfunctab : &findFuncTab[0],
minpc : minpc,
maxpc : maxpc,
modulename : name,
gofunc: base,
}
/* verify and register the new module */
moduledataverify1(mod)
registerModule(mod)
}

4
internal/loader/funcdata_invalid.go
View file

@ -1,4 +1,4 @@
// +build !go1.15 go1.18
// +build !go1.15 go1.19
/*
* Copyright 2021 ByteDance Inc.
@ -20,7 +20,7 @@ package loader
// triggers a compilation error
const (
_ = panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17")
_ = panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17, 1.18")
)
func registerFunction(_ string, _ uintptr, _ int, _ int, _ uintptr) {

2
internal/loader/loader.go
View file

@ -41,7 +41,7 @@ func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{})
m := mmap(n)
v := fmt.Sprintf("runtime.__%s_%x", fn, m)
argsptr, localsptr := stackMap(faker)
registerFunction(v, m, fp, args, uintptr(len(self)), argsptr, localsptr)
registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr)
/* reference as a slice */
s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader {

2
internal/loader/loader_go117_test.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.18
// +build go1.17,!go1.19
/*
* Copyright 2021 ByteDance Inc.