XeLabs/sonic
2
0
Fork 0
mirror of https://github.com/ii64/sonic.git synced 2026-06-20 16:45:22 +08:00
Code Issues Projects Releases Wiki Activity

feat: more complete function loader (#354)

Browse source 
* follow complete implementation of go symtab
* support go1.20
This commit is contained in:
Yi Duan 2023-02-08 14:35:00 +08:00 committed by GitHub
parent cfa4fe1736
commit e7ac2f25fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
58 changed files with 3236 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/push-check-go118.yml vendored
View file

@ -25,8 +25,8 @@ jobs:
- name: Unit Test
run: |
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./external_jsonlib_test/...
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./...
GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./external_jsonlib_test/...
- name: Generic Test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test
run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./generic_test

2
.github/workflows/push-check-linux-arm64.yml vendored
View file

@ -6,7 +6,7 @@ jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.19.x]
go-version: [1.15.x, 1.20.x]
os: [arm]
runs-on: ${{ matrix.os }}
steps:

2
.github/workflows/push-check-linux-x64.yml vendored
View file

@ -6,7 +6,7 @@ jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x]
go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x, 1.20.x]
runs-on: [self-hosted, X64]
steps:
- name: Clear repository

2
.github/workflows/push-check-qemu.yml vendored
View file

@ -6,7 +6,7 @@ jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.19.x]
go-version: [1.15.x, 1.20.x]
os: [arm]
runs-on: ${{ matrix.os }}
steps:

2
.github/workflows/push-check-windows.yml vendored
View file

@ -6,7 +6,7 @@ jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.19.x]
go-version: [1.15.x, 1.20.x]
runs-on: windows-latest
steps:
- uses: actions/checkout@v2

3
ast/api_amd64_test.go
View file

@ -10,7 +10,8 @@ import (
`github.com/stretchr/testify/assert`
)
func TestSortNodeTwitter(t *testing.T) {root, err := NewSearcher(_TwitterJson).GetByPath()
func TestSortNodeTwitter(t *testing.T) {
root, err := NewSearcher(_TwitterJson).GetByPath()
if err != nil {
t.Fatal(err)
}

6
ast/encode.go
View file

@ -19,6 +19,8 @@ package ast
import (
`sync`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
const (
@ -163,7 +165,7 @@ func (self *Node) encodeFalse(buf *[]byte) error {
}
func (self *Node) encodeNumber(buf *[]byte) error {
str := addr2str(self.p, self.v)
str := rt.StrFrom(self.p, self.v)
*buf = append(*buf, str...)
return nil
}
@ -174,7 +176,7 @@ func (self *Node) encodeString(buf *[]byte) error {
return nil
}
quote(buf, addr2str(self.p, self.v))
quote(buf, rt.StrFrom(self.p, self.v))
return nil
}

31
ast/node.go
View file

@ -21,6 +21,7 @@ import (
`fmt`
`strconv`
`unsafe`
`reflect`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
@ -60,6 +61,10 @@ const (
V_ANY = int(_V_ANY)
)
var (
byteType = rt.UnpackType(reflect.TypeOf(byte(0)))
)
type Node struct {
v int64
t types.ValueType
@ -151,7 +156,7 @@ func (self *Node) Raw() (string, error) {
buf, err := self.MarshalJSON()
return rt.Mem2Str(buf), err
}
return addr2str(self.p, self.v), nil
return rt.StrFrom(self.p, self.v), nil
}
func (self *Node) checkRaw() error {
@ -183,7 +188,7 @@ func (self *Node) Bool() (bool, error) {
} else {
return false, err
}
case types.V_STRING: return strconv.ParseBool(addr2str(self.p, self.v))
case types.V_STRING: return strconv.ParseBool(rt.StrFrom(self.p, self.v))
case _V_ANY :
any := self.packAny()
switch v := any.(type) {
@ -389,7 +394,7 @@ func (self *Node) String() (string, error) {
case types.V_NULL : return "" , nil
case types.V_TRUE : return "true" , nil
case types.V_FALSE : return "false", nil
case types.V_STRING, _V_NUMBER : return addr2str(self.p, self.v), nil
case types.V_STRING, _V_NUMBER : return rt.StrFrom(self.p, self.v), nil
case _V_ANY :
any := self.packAny()
switch v := any.(type) {
@ -421,7 +426,7 @@ func (self *Node) StrictString() (string, error) {
return "", err
}
switch self.t {
case types.V_STRING : return addr2str(self.p, self.v), nil
case types.V_STRING : return rt.StrFrom(self.p, self.v), nil
case _V_ANY :
if v, ok := self.packAny().(string); ok {
return v, nil
@ -836,7 +841,7 @@ func (self *Node) UnsafeMap() ([]Pair, error) {
if err := self.skipAllKey(); err != nil {
return nil, err
}
s := ptr2slice(self.p, int(self.len()), self.cap())
s := rt.Ptr2SlicePtr(self.p, int(self.len()), self.cap())
return *(*[]Pair)(s), nil
}
@ -935,7 +940,7 @@ func (self *Node) UnsafeArray() ([]Node, error) {
if err := self.skipAllIndex(); err != nil {
return nil, err
}
s := ptr2slice(self.p, self.len(), self.cap())
s := rt.Ptr2SlicePtr(self.p, self.len(), self.cap())
return *(*[]Node)(s), nil
}
@ -953,7 +958,7 @@ func (self *Node) Interface() (interface{}, error) {
case types.V_FALSE : return false, nil
case types.V_ARRAY : return self.toGenericArray()
case types.V_OBJECT : return self.toGenericObject()
case types.V_STRING : return addr2str(self.p, self.v), nil
case types.V_STRING : return rt.StrFrom(self.p, self.v), nil
case _V_NUMBER :
v, err := numberToFloat64(self)
if err != nil {
@ -997,7 +1002,7 @@ func (self *Node) InterfaceUseNumber() (interface{}, error) {
case types.V_FALSE : return false, nil
case types.V_ARRAY : return self.toGenericArrayUseNumber()
case types.V_OBJECT : return self.toGenericObjectUseNumber()
case types.V_STRING : return addr2str(self.p, self.v), nil
case types.V_STRING : return rt.StrFrom(self.p, self.v), nil
case _V_NUMBER : return toNumber(self), nil
case _V_ARRAY_LAZY :
if err := self.loadAllIndex(); err != nil {
@ -1597,13 +1602,13 @@ func NewBool(v bool) Node {
func NewNumber(v string) Node {
return Node{
v: int64(len(v) & _LEN_MASK),
p: str2ptr(v),
p: rt.StrPtr(v),
t: _V_NUMBER,
}
}
func toNumber(node *Node) json.Number {
return json.Number(addr2str(node.p, node.v))
return json.Number(rt.StrFrom(node.p, node.v))
}
func numberToFloat64(node *Node) (float64, error) {
@ -1637,7 +1642,7 @@ func newBytes(v []byte) Node {
func NewString(v string) Node {
return Node{
t: types.V_STRING,
p: str2ptr(v),
p: rt.StrPtr(v),
v: int64(len(v) & _LEN_MASK),
}
}
@ -1727,13 +1732,13 @@ func (self *Node) setLazyObject(p *Parser, v []Pair) {
func newRawNode(str string, typ types.ValueType) Node {
return Node{
t: _V_RAW | typ,
p: str2ptr(str),
p: rt.StrPtr(str),
v: int64(len(str) & _LEN_MASK),
}
}
func (self *Node) parseRaw(full bool) {
raw := addr2str(self.p, self.v)
raw := rt.StrFrom(self.p, self.v)
parser := NewParser(raw)
if full {
parser.noLazy = true

30
ast/stubs.go → ast/stubs_go115.go
View file

@ -1,3 +1,5 @@
// +build go1.15,!go1.20
/*
* Copyright 2021 ByteDance Inc.
*
@ -18,16 +20,11 @@ package ast
import (
`unsafe`
`reflect`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
var (
byteType = rt.UnpackType(reflect.TypeOf(byte(0)))
)
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
@ -46,29 +43,6 @@ func mem2ptr(s []byte) unsafe.Pointer {
return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
}
//go:nosplit
func ptr2slice(s unsafe.Pointer, l int, c int) unsafe.Pointer {
slice := &rt.GoSlice{
Ptr: s,
Len: l,
Cap: c,
}
return unsafe.Pointer(slice)
}
//go:nosplit
func str2ptr(s string) unsafe.Pointer {
return (*rt.GoString)(unsafe.Pointer(&s)).Ptr
}
//go:nosplit
func addr2str(p unsafe.Pointer, n int64) (s string) {
(*rt.GoString)(unsafe.Pointer(&s)).Ptr = p
(*rt.GoString)(unsafe.Pointer(&s)).Len = int(n)
return
}
var (
//go:linkname safeSet encoding/json.safeSet
safeSet [utf8.RuneSelf]bool

55
ast/stubs_go120.go Normal file
View file

@ -0,0 +1,55 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ast
import (
`unsafe`
`unicode/utf8`
`github.com/bytedance/sonic/internal/rt`
)
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname unsafe_NewArray reflect.unsafe_NewArray
//goland:noinspection GoUnusedParameter
func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer
//go:linkname growslice reflect.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:nosplit
func mem2ptr(s []byte) unsafe.Pointer {
return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr
}
var (
//go:linkname safeSet encoding/json.safeSet
safeSet [utf8.RuneSelf]bool
//go:linkname hex encoding/json.hex
hex string
)
//go:linkname unquoteBytes encoding/json.unquoteBytes
func unquoteBytes(s []byte) (t []byte, ok bool)

3
decoder/assembler_amd64_go116.go
View file

@ -215,6 +215,7 @@ var (
type _Assembler struct {
jit.BaseAssembler
p _Program
name string
}
func newAssembler(p _Program) *_Assembler {
@ -224,7 +225,7 @@ func newAssembler(p _Program) *_Assembler {
/** Assembler Interface **/
func (self *_Assembler) Load() _Decoder {
return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow))
return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
}
func (self *_Assembler) Init(p _Program) *_Assembler {

84
decoder/assembler_amd64_go117.go
View file

@ -1,4 +1,5 @@
// +build go1.17,!go1.20
//go:build go1.17 && !go1.21
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.
@ -25,7 +26,7 @@ import (
`reflect`
`strconv`
`unsafe`
`github.com/bytedance/sonic/internal/caching`
`github.com/bytedance/sonic/internal/jit`
`github.com/bytedance/sonic/internal/native`
@ -165,10 +166,10 @@ var (
)
var (
_VAR_sv = _VAR_sv_p
_VAR_sv_p = jit.Ptr(_SP, _FP_base + 48)
_VAR_sv_n = jit.Ptr(_SP, _FP_base + 56)
_VAR_vk = jit.Ptr(_SP, _FP_base + 64)
_ARG_sv = _ARG_sv_p
_ARG_sv_p = jit.Ptr(_SP, _FP_base + 48)
_ARG_sv_n = jit.Ptr(_SP, _FP_base + 56)
_ARG_vk = jit.Ptr(_SP, _FP_base + 64)
)
var (
@ -210,6 +211,7 @@ var (
type _Assembler struct {
jit.BaseAssembler
p _Program
name string
}
func newAssembler(p _Program) *_Assembler {
@ -219,7 +221,7 @@ func newAssembler(p _Program) *_Assembler {
/** Assembler Interface **/
func (self *_Assembler) Load() _Decoder {
return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow))
return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
}
func (self *_Assembler) Init(p _Program) *_Assembler {
@ -348,8 +350,8 @@ func (self *_Assembler) epilogue() {
self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX
self.Emit("MOVQ", jit.Imm(0), _ARG_sp) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_vp) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_sv_p) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_vk) // MOVQ $0, vk<>+64(FP)
self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP
self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP
self.Emit("RET") // RET
@ -370,9 +372,9 @@ func (self *_Assembler) prologue() {
self.Emit("MOVQ", _SI, _ARG_sb) // MOVQ SI, sb<>+32(FP)
self.Emit("MOVQ", _SI, _ST) // MOVQ SI, ST
self.Emit("MOVQ", _R8, _ARG_fv) // MOVQ R8, fv<>+40(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_sv_p) // MOVQ $0, sv.p<>+48(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_sv_n) // MOVQ $0, sv.n<>+56(FP)
self.Emit("MOVQ", jit.Imm(0), _ARG_vk) // MOVQ $0, vk<>+64(FP)
self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ $0, et<>+120(FP)
// initialize digital buffer first
self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap
@ -498,8 +500,8 @@ func (self *_Assembler) mismatch_error() {
func (self *_Assembler) field_error() {
self.Link(_LB_field_error) // _field_error:
self.Emit("MOVQ", _VAR_sv_p, _AX) // MOVQ sv.p, AX
self.Emit("MOVQ", _VAR_sv_n, _BX) // MOVQ sv.n, BX
self.Emit("MOVQ", _ARG_sv_p, _AX) // MOVQ sv.p, AX
self.Emit("MOVQ", _ARG_sv_n, _BX) // MOVQ sv.n, BX
self.call_go(_F_error_field) // CALL_GO error_field
self.Sjmp("JMP" , _LB_error) // JMP _error
}
@ -745,11 +747,11 @@ func (self *_Assembler) copy_string() {
self.Emit("MOVQ", _DI, _VAR_bs_p)
self.Emit("MOVQ", _SI, _VAR_bs_n)
self.Emit("MOVQ", _R9, _VAR_bs_LR)
self.malloc_AX(_SI, _VAR_sv_p)
self.malloc_AX(_SI, _ARG_sv_p)
self.Emit("MOVQ", _VAR_bs_p, _BX)
self.Emit("MOVQ", _VAR_bs_n, _CX)
self.call_go(_F_memmove)
self.Emit("MOVQ", _VAR_sv_p, _DI)
self.Emit("MOVQ", _ARG_sv_p, _DI)
self.Emit("MOVQ", _VAR_bs_n, _SI)
self.Emit("MOVQ", _VAR_bs_LR, _R9)
self.Rjmp("JMP", _R9)
@ -762,7 +764,7 @@ func (self *_Assembler) escape_string() {
self.Emit("MOVQ" , _SI, _VAR_bs_n)
self.Emit("MOVQ" , _R9, _VAR_bs_LR)
self.malloc_AX(_SI, _DX) // MALLOC SI, DX
self.Emit("MOVQ" , _DX, _VAR_sv_p)
self.Emit("MOVQ" , _DX, _ARG_sv_p)
self.Emit("MOVQ" , _VAR_bs_p, _DI)
self.Emit("MOVQ" , _VAR_bs_n, _SI)
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
@ -776,7 +778,7 @@ func (self *_Assembler) escape_string() {
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error
self.Emit("MOVQ" , _AX, _SI)
self.Emit("MOVQ" , _VAR_sv_p, _DI)
self.Emit("MOVQ" , _ARG_sv_p, _DI)
self.Emit("MOVQ" , _VAR_bs_LR, _R9)
self.Rjmp("JMP", _R9)
}
@ -787,7 +789,7 @@ func (self *_Assembler) escape_string_twice() {
self.Emit("MOVQ" , _SI, _VAR_bs_n)
self.Emit("MOVQ" , _R9, _VAR_bs_LR)
self.malloc_AX(_SI, _DX) // MALLOC SI, DX
self.Emit("MOVQ" , _DX, _VAR_sv_p)
self.Emit("MOVQ" , _DX, _ARG_sv_p)
self.Emit("MOVQ" , _VAR_bs_p, _DI)
self.Emit("MOVQ" , _VAR_bs_n, _SI)
self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX
@ -803,7 +805,7 @@ func (self *_Assembler) escape_string_twice() {
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error
self.Emit("MOVQ" , _AX, _SI)
self.Emit("MOVQ" , _VAR_sv_p, _DI)
self.Emit("MOVQ" , _ARG_sv_p, _DI)
self.Emit("MOVQ" , _VAR_bs_LR, _R9)
self.Rjmp("JMP", _R9)
}
@ -1029,15 +1031,15 @@ func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) {
/* allocate the key, and call the unmarshaler */
self.valloc(vk, _BX) // VALLOC ${vk}, BX
// must spill vk pointer since next call_go may invoke GC
self.Emit("MOVQ" , _BX, _VAR_vk)
self.Emit("MOVQ" , _BX, _ARG_vk)
self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX
self.Emit("MOVQ" , _VAR_sv_p, _CX) // MOVQ sv.p, CX
self.Emit("MOVQ" , _VAR_sv_n, _DI) // MOVQ sv.n, DI
self.Emit("MOVQ" , _ARG_sv_p, _CX) // MOVQ sv.p, CX
self.Emit("MOVQ" , _ARG_sv_n, _DI) // MOVQ sv.n, DI
self.call_go(_F_decodeTextUnmarshaler) // CALL_GO decodeTextUnmarshaler
self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
self.Sjmp("JNZ" , _LB_error) // JNZ _error
self.Emit("MOVQ" , _VAR_vk, _AX) // MOVQ VAR.vk, AX
self.Emit("MOVQ", jit.Imm(0), _VAR_vk)
self.Emit("MOVQ" , _ARG_vk, _AX) // MOVQ VAR.vk, AX
self.Emit("MOVQ", jit.Imm(0), _ARG_vk)
/* select the correct assignment function */
if !pv {
@ -1061,14 +1063,14 @@ func (self *_Assembler) unmarshal_json(t reflect.Type, deref bool) {
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v
self.slice_from_r(_AX, 0) // SLICE_R AX, $0
self.Emit("MOVQ" , _DI, _VAR_sv_p) // MOVQ DI, sv.p
self.Emit("MOVQ" , _SI, _VAR_sv_n) // MOVQ SI, sv.n
self.Emit("MOVQ" , _DI, _ARG_sv_p) // MOVQ DI, sv.p
self.Emit("MOVQ" , _SI, _ARG_sv_n) // MOVQ SI, sv.n
self.unmarshal_func(t, _F_decodeJsonUnmarshaler, deref) // UNMARSHAL json, ${t}, ${deref}
}
func (self *_Assembler) unmarshal_text(t reflect.Type, deref bool) {
self.parse_string() // PARSE STRING
self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.unmarshal_func(t, _F_decodeTextUnmarshaler, deref) // UNMARSHAL text, ${t}, ${deref}
}
@ -1094,8 +1096,8 @@ func (self *_Assembler) unmarshal_func(t reflect.Type, fn obj.Addr, deref bool)
self.Emit("MOVQ", jit.Type(pt), _AX) // MOVQ ${pt}, AX
/* set the source string and call the unmarshaler */
self.Emit("MOVQ" , _VAR_sv_p, _CX) // MOVQ sv.p, CX
self.Emit("MOVQ" , _VAR_sv_n, _DI) // MOVQ sv.n, DI
self.Emit("MOVQ" , _ARG_sv_p, _CX) // MOVQ sv.p, CX
self.Emit("MOVQ" , _ARG_sv_n, _DI) // MOVQ sv.n, DI
self.call_go(fn) // CALL_GO ${fn}
self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
self.Sjmp("JNZ" , _LB_error) // JNZ _error
@ -1557,26 +1559,26 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) {
func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) {
self.parse_string() // PARSE STRING
self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
if vt := p.vt(); !mapfast(vt) {
self.valloc(vt.Key(), _DI)
self.Emit("MOVOU", _VAR_sv, _X0)
self.Emit("MOVOU", _ARG_sv, _X0)
self.Emit("MOVOU", _X0, jit.Ptr(_DI, 0))
self.mapassign_std(vt, jit.Ptr(_DI, 0)) // MAPASSIGN string, DI, SI
} else {
self.mapassign_str_fast(vt, _VAR_sv_p, _VAR_sv_n) // MAPASSIGN string, DI, SI
self.mapassign_str_fast(vt, _ARG_sv_p, _ARG_sv_n) // MAPASSIGN string, DI, SI
}
}
func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) {
self.parse_string() // PARSE STRING
self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.mapassign_utext(p.vt(), false) // MAPASSIGN utext, ${p.vt()}, false
}
func (self *_Assembler) _asm_OP_map_key_utext_p(p *_Instr) {
self.parse_string() // PARSE STRING
self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n
self.mapassign_utext(p.vt(), true) // MAPASSIGN utext, ${p.vt()}, true
}
@ -1650,8 +1652,8 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) {
self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX
self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr
self.parse_string() // PARSE STRING
self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, false) // UNQUOTE once, sv.p, sv.n
self.Emit("LEAQ" , _VAR_sv, _AX) // LEAQ sv, AX
self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, false) // UNQUOTE once, sv.p, sv.n
self.Emit("LEAQ" , _ARG_sv, _AX) // LEAQ sv, AX
self.Emit("XORL" , _BX, _BX) // XORL BX, BX
self.call_go(_F_strhash) // CALL_GO strhash
self.Emit("MOVQ" , _AX, _R9) // MOVQ AX, R9
@ -1672,7 +1674,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) {
self.Emit("CMPQ" , _R8, _R9) // CMPQ R8, R9
self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n}
self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name + 8), _DX) // MOVQ FieldEntry.Name+8(DI), DX
self.Emit("CMPQ" , _DX, _VAR_sv_n) // CMPQ DX, sv.n
self.Emit("CMPQ" , _DX, _ARG_sv_n) // CMPQ DX, sv.n
self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n}
self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_ID), _R8) // MOVQ FieldEntry.ID(DI), R8
self.Emit("MOVQ" , _AX, _VAR_ss_AX) // MOVQ AX, ss.AX
@ -1680,7 +1682,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) {
self.Emit("MOVQ" , _SI, _VAR_ss_SI) // MOVQ SI, ss.SI
self.Emit("MOVQ" , _R8, _VAR_ss_R8) // MOVQ R8, ss.R8
self.Emit("MOVQ" , _R9, _VAR_ss_R9) // MOVQ R9, ss.R9
self.Emit("MOVQ" , _VAR_sv_p, _AX) // MOVQ _VAR_sv_p, AX
self.Emit("MOVQ" , _ARG_sv_p, _AX) // MOVQ _VAR_sv_p, AX
self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name), _CX) // MOVQ FieldEntry.Name(DI), CX
self.Emit("MOVQ" , _CX, _BX) // MOVQ CX, 8(SP)
self.Emit("MOVQ" , _DX, _CX) // MOVQ DX, 16(SP)
@ -1697,8 +1699,8 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) {
self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n}
self.Link("_try_lowercase_{n}") // _try_lowercase_{n}:
self.Emit("MOVQ" , jit.Imm(referenceFields(p.vf())), _AX) // MOVQ ${p.vf()}, AX
self.Emit("MOVQ", _VAR_sv_p, _BX) // MOVQ sv, BX
self.Emit("MOVQ", _VAR_sv_n, _CX) // MOVQ sv, CX
self.Emit("MOVQ", _ARG_sv_p, _BX) // MOVQ sv, BX
self.Emit("MOVQ", _ARG_sv_n, _CX) // MOVQ sv, CX
self.call_go(_F_FieldMap_GetCaseInsensitive) // CALL_GO FieldMap::GetCaseInsensitive
self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, _VAR_sr
self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX

1
decoder/compiler.go
View file

@ -104,6 +104,7 @@ const (
const (
_INT_SIZE = 32 << (^uint(0) >> 63)
_PTR_SIZE = 32 << (^uintptr(0) >> 63)
_PTR_BYTE = unsafe.Sizeof(uintptr(0))
)
const (

4
decoder/decoder.go
View file

@ -168,7 +168,9 @@ func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Typ
if pp, err := compiler.compile(_vt); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
as := newAssembler(pp)
as.name = _vt.String()
return as.Load(), nil
}
}

2
decoder/generic_amd64_go116.go
View file

@ -81,7 +81,7 @@ type _ValueDecoder struct {
func (self *_ValueDecoder) build() uintptr {
self.Init(self.compile)
return *(*uintptr)(self.LoadWithFaker("decode_value", _VD_size, _VD_args, _Decoder_Generic_Shadow))
return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic))
}
/** Function Calling Helpers **/

5
decoder/generic_amd64_go117.go
View file

@ -1,4 +1,5 @@
// +build go1.17,!go1.20
//go:build go1.17 && !go1.21
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.
@ -84,7 +85,7 @@ var (
func (self *_ValueDecoder) build() uintptr {
self.Init(self.compile)
return *(*uintptr)(self.LoadWithFaker("decode_value", _VD_size, _VD_args, _Decoder_Generic_Shadow))
return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic))
}
/** Function Calling Helpers **/

2
decoder/generic_amd64_go117_test.s
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.20
// +build go1.17,!go1.21
//
// Copyright 2021 ByteDance Inc.

14
decoder/norace_test.go
View file

@ -1,3 +1,4 @@
//go:build !race
// +build !race
/*
@ -19,11 +20,12 @@
package decoder
import (
`testing`
`unsafe`
`runtime`
`runtime`
`testing`
`time`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/internal/rt`
)
var referred = false
@ -34,7 +36,7 @@ func TestStringReferring(t *testing.T) {
println("malloc *byte ", sp)
runtime.SetFinalizer(sp, func(sp *byte){
referred = false
println("*byte ", sp, " got free")
println("*byte ", sp, " got free 1")
})
runtime.GC()
println("first GC")
@ -50,6 +52,7 @@ func TestStringReferring(t *testing.T) {
}
runtime.GC()
println("second GC")
time.Sleep(time.Millisecond)
if referred {
t.Fatal("*byte is being referred")
}
@ -73,6 +76,7 @@ func TestStringReferring(t *testing.T) {
}
runtime.GC()
println("second GC")
time.Sleep(time.Millisecond)
if referred {
t.Fatal("*byte is being referred")
}

39
decoder/pools.go
View file

@ -17,7 +17,6 @@
package decoder
import (
`errors`
`sync`
`unsafe`
@ -82,37 +81,15 @@ var _KeepAlive struct {
frame_generic [_VD_offs]byte
}
var errCallShadow = errors.New("DON'T CALL THIS!")
var (
argPtrs = []bool{true, false, false, true, true, false, true, false, true}
localPtrs = []bool{}
)
// Faker func of _Decoder, used to export its stackmap as _Decoder's
func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string, vk unsafe.Pointer) (ret int, err error) {
// align to assembler_amd64.go: _FP_offs
var frame [_FP_offs]byte
// keep all args and stacks alive
_KeepAlive.s = s
_KeepAlive.i = i
_KeepAlive.vp = vp
_KeepAlive.sb = sb
_KeepAlive.fv = fv
_KeepAlive.ret = ret
_KeepAlive.err = err
_KeepAlive.sv = sv
_KeepAlive.vk = vk
_KeepAlive.frame_decoder = frame
return 0, errCallShadow
}
// Faker func of _Decoder_Generic, used to export its stackmap
func _Decoder_Generic_Shadow(sb *_Stack) {
// align to generic_amd64.go: _VD_offs
var frame [_VD_offs]byte
// must keep sb noticeable to GC
_KeepAlive.sb = sb
_KeepAlive.frame_generic = frame
}
var (
argPtrs_generic = []bool{true}
localPtrs_generic = []bool{}
)
func newStack() *_Stack {
if ret := stackPool.Get(); ret == nil {

1
decoder/stubs.go → decoder/stubs_go115.go
View file

@ -1,3 +1,4 @@
// +build go1.15,!go1.20
/*
* Copyright 2021 ByteDance Inc.

111
decoder/stubs_go120.go Normal file
View file

@ -0,0 +1,111 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package decoder
import (
`unsafe`
`reflect`
_ `github.com/chenzhuoyu/base64x`
`github.com/bytedance/sonic/internal/rt`
)
//go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode
var _subr__b64decode uintptr
// runtime.maxElementSize
const _max_map_element_size uintptr = 128
func mapfast(vt reflect.Type) bool {
return vt.Elem().Size() <= _max_map_element_size
}
//go:nosplit
//go:linkname throw runtime.throw
//goland:noinspection GoUnusedParameter
func throw(s string)
//go:linkname convT64 runtime.convT64
//goland:noinspection GoUnusedParameter
func convT64(v uint64) unsafe.Pointer
//go:linkname convTslice runtime.convTslice
//goland:noinspection GoUnusedParameter
func convTslice(v []byte) unsafe.Pointer
//go:linkname convTstring runtime.convTstring
//goland:noinspection GoUnusedParameter
func convTstring(v string) unsafe.Pointer
//go:noescape
//go:linkname memequal runtime.memequal
//goland:noinspection GoUnusedParameter
func memequal(a unsafe.Pointer, b unsafe.Pointer, size uintptr) bool
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname mallocgc runtime.mallocgc
//goland:noinspection GoUnusedParameter
func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer
//go:linkname makeslice runtime.makeslice
//goland:noinspection GoUnusedParameter
func makeslice(et *rt.GoType, len int, cap int) unsafe.Pointer
//go:noescape
//go:linkname growslice reflect.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:linkname makemap_small runtime.makemap_small
func makemap_small() unsafe.Pointer
//go:linkname mapassign runtime.mapassign
//goland:noinspection GoUnusedParameter
func mapassign(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_fast32 runtime.mapassign_fast32
//goland:noinspection GoUnusedParameter
func mapassign_fast32(t *rt.GoType, h unsafe.Pointer, k uint32) unsafe.Pointer
//go:linkname mapassign_fast64 runtime.mapassign_fast64
//goland:noinspection GoUnusedParameter
func mapassign_fast64(t *rt.GoType, h unsafe.Pointer, k uint64) unsafe.Pointer
//go:linkname mapassign_fast64ptr runtime.mapassign_fast64ptr
//goland:noinspection GoUnusedParameter
func mapassign_fast64ptr(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer
//go:linkname mapassign_faststr runtime.mapassign_faststr
//goland:noinspection GoUnusedParameter
func mapassign_faststr(t *rt.GoType, h unsafe.Pointer, s string) unsafe.Pointer
//go:nosplit
//go:linkname memclrHasPointers runtime.memclrHasPointers
//goland:noinspection GoUnusedParameter
func memclrHasPointers(ptr unsafe.Pointer, n uintptr)
//go:noescape
//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)

2
decoder/utils.go
View file

@ -19,7 +19,7 @@ package decoder
import (
`unsafe`
`github.com/bytedance/sonic/internal/loader`
`github.com/bytedance/sonic/loader`
)
//go:nosplit

8
encoder/assembler_amd64_go116.go
View file

@ -177,6 +177,7 @@ type _Assembler struct {
jit.BaseAssembler
p _Program
x int
name string
}
func newAssembler(p _Program) *_Assembler {
@ -184,9 +185,8 @@ func newAssembler(p _Program) *_Assembler {
}
/** Assembler Interface **/
func (self *_Assembler) Load() _Encoder {
return ptoenc(self.BaseAssembler.LoadWithFaker("json_encoder", _FP_size, _FP_args, _Encoder_Shadow))
return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
}
func (self *_Assembler) Init(p _Program) *_Assembler {
@ -483,10 +483,6 @@ func (self *_Assembler) load_buffer() {
/** Function Interface Helpers **/
var (
_F_assertI2I = jit.Func(assertI2I)
)
func (self *_Assembler) call(pc obj.Addr) {
self.Emit("MOVQ", pc, _AX) // MOVQ $pc, AX
self.Rjmp("CALL", _AX) // CALL AX

6
encoder/assembler_amd64_go117.go
View file

@ -1,4 +1,5 @@
// +build go1.17,!go1.20
//go:build go1.17 && !go1.21
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.
@ -182,6 +183,7 @@ type _Assembler struct {
jit.BaseAssembler
p _Program
x int
name string
}
func newAssembler(p _Program) *_Assembler {
@ -191,7 +193,7 @@ func newAssembler(p _Program) *_Assembler {
/** Assembler Interface **/
func (self *_Assembler) Load() _Encoder {
return ptoenc(self.BaseAssembler.LoadWithFaker("json_encoder", _FP_size, _FP_args, _Encoder_Shadow))
return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
}
func (self *_Assembler) Init(p _Program) *_Assembler {

1
encoder/compiler.go
View file

@ -86,6 +86,7 @@ const (
const (
_INT_SIZE = 32 << (^uint(0) >> 63)
_PTR_SIZE = 32 << (^uintptr(0) >> 63)
_PTR_BYTE = unsafe.Sizeof(uintptr(0))
)
const (

2
encoder/debug_go117.go
View file

@ -1,4 +1,4 @@
// +build go1.17,!go1.20
// +build go1.17,!go1.21
/*
* Copyright 2021 ByteDance Inc.

8
encoder/pools.go
View file

@ -135,7 +135,9 @@ func makeEncoder(vt *rt.GoType, ex ...interface{}) (interface{}, error) {
if pp, err := newCompiler().compile(vt.Pack(), ex[0].(bool)); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
as := newAssembler(pp)
as.name = vt.String()
return as.Load(), nil
}
}
@ -156,7 +158,9 @@ func pretouchType(_vt reflect.Type, opts option.CompileOptions, v uint8) (map[re
if pp, err := compiler.compile(_vt, ex[0].(bool)); err != nil {
return nil, err
} else {
return newAssembler(pp).Load(), nil
as := newAssembler(pp)
as.name = vt.String()
return as.Load(), nil
}
}

22
encoder/primitives.go
View file

@ -21,6 +21,7 @@ import (
`encoding/json`
`unsafe`
`github.com/bytedance/sonic/internal/jit`
`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/rt`
)
@ -110,4 +111,25 @@ func encodeTextMarshaler(buf *[]byte, val encoding.TextMarshaler, opt Options) e
}
return encodeString(buf, rt.Mem2Str(ret) )
}
}
var (
argPtrs = []bool { true, true, true, false }
localPtrs = []bool{}
)
var (
_F_assertI2I = jit.Func(assertI2I)
)
func asText(v unsafe.Pointer) (string, error) {
text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v))
r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText()
return rt.Mem2Str(r), e
}
func asJson(v unsafe.Pointer) (string, error) {
text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v))
r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON()
return rt.Mem2Str(r), e
}

14
encoder/stubs_go116.go
View file

@ -20,8 +20,6 @@ package encoder
import (
`unsafe`
`encoding`
`encoding/json`
_ `github.com/chenzhuoyu/base64x`
@ -61,18 +59,6 @@ func isValidNumber(s string) bool
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
func asText(v unsafe.Pointer) (string, error) {
text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v))
r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText()
return rt.Mem2Str(r), e
}
func asJson(v unsafe.Pointer) (string, error) {
text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v))
r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON()
return rt.Mem2Str(r), e
}
var _runtime_writeBarrier uintptr = rt.GcwbAddr()
//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier

23
encoder/stubs_go117.go
View file

@ -19,13 +19,10 @@
package encoder
import (
`encoding`
`encoding/json`
`unsafe`
_ `github.com/chenzhuoyu/base64x`
`github.com/bytedance/sonic/internal/jit`
`github.com/bytedance/sonic/internal/rt`
)
@ -41,9 +38,9 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:linkname assertI2I2 runtime.assertI2I2
//go:linkname assertI2I runtime.assertI2I2
//goland:noinspection GoUnusedParameter
func assertI2I2(inter *rt.GoType, i rt.GoIface) rt.GoIface
func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface
//go:linkname mapiternext runtime.mapiternext
//goland:noinspection GoUnusedParameter
@ -62,22 +59,6 @@ func isValidNumber(s string) bool
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
var (
_F_assertI2I = jit.Func(assertI2I2)
)
func asText(v unsafe.Pointer) (string, error) {
text := assertI2I2(_T_encoding_TextMarshaler, *(*rt.GoIface)(v))
r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText()
return rt.Mem2Str(r), e
}
func asJson(v unsafe.Pointer) (string, error) {
text := assertI2I2(_T_json_Marshaler, *(*rt.GoIface)(v))
r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON()
return rt.Mem2Str(r), e
}
//go:linkname _runtime_writeBarrier runtime.writeBarrier
var _runtime_writeBarrier uintptr

66
encoder/stubs_go120.go Normal file
View file

@ -0,0 +1,66 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package encoder
import (
`unsafe`
_ `github.com/chenzhuoyu/base64x`
`github.com/bytedance/sonic/internal/rt`
)
//go:linkname _subr__b64encode github.com/chenzhuoyu/base64x._subr__b64encode
var _subr__b64encode uintptr
//go:noescape
//go:linkname memmove runtime.memmove
//goland:noinspection GoUnusedParameter
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)
//go:linkname growslice reflect.growslice
//goland:noinspection GoUnusedParameter
func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice
//go:linkname assertI2I runtime.assertI2I2
//goland:noinspection GoUnusedParameter
func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface
//go:linkname mapiternext runtime.mapiternext
//goland:noinspection GoUnusedParameter
func mapiternext(it *rt.GoMapIterator)
//go:linkname mapiterinit runtime.mapiterinit
//goland:noinspection GoUnusedParameter
func mapiterinit(t *rt.GoMapType, m *rt.GoMap, it *rt.GoMapIterator)
//go:linkname isValidNumber encoding/json.isValidNumber
//goland:noinspection GoUnusedParameter
func isValidNumber(s string) bool
//go:noescape
//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers
//goland:noinspection GoUnusedParameter
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
//go:linkname _runtime_writeBarrier runtime.writeBarrier
var _runtime_writeBarrier uintptr
//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier
func gcWriteBarrierAX()

2
encoder/utils.go
View file

@ -20,7 +20,7 @@ import (
`encoding/json`
`unsafe`
`github.com/bytedance/sonic/internal/loader`
`github.com/bytedance/sonic/loader`
)
//go:nosplit

25
external_jsonlib_test/benchmark_test/encoder_test.go
View file

@ -18,12 +18,37 @@ package benchmark_test
import (
`encoding/json`
`os`
`runtime`
`runtime/debug`
`testing`
`time`
gojson `github.com/goccy/go-json`
jsoniter `github.com/json-iterator/go`
)
var (
debugSyncGC = os.Getenv("SONIC_SYNC_GC") != ""
debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
)
func TestMain(m *testing.M) {
go func () {
if !debugAsyncGC {
return
}
println("Begin GC looping...")
for {
runtime.GC()
debug.FreeOSMemory()
}
println("stop GC looping!")
}()
time.Sleep(time.Millisecond)
m.Run()
}
var _GenericValue interface{}
var _BindingValue TwitterStruct

25
external_jsonlib_test/unit_test/api_test.go
View file

@ -19,13 +19,38 @@ package unit_test
import (
`bytes`
`encoding/json`
`os`
`runtime`
`runtime/debug`
`testing`
`time`
`github.com/bytedance/sonic`
jsoniter `github.com/json-iterator/go`
`github.com/stretchr/testify/require`
)
var (
debugSyncGC = os.Getenv("SONIC_SYNC_GC") != ""
debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == ""
)
func TestMain(m *testing.M) {
go func () {
if !debugAsyncGC {
return
}
println("Begin GC looping...")
for {
runtime.GC()
debug.FreeOSMemory()
}
println("stop GC looping!")
}()
time.Sleep(time.Millisecond)
m.Run()
}
var jt = jsoniter.Config{
ValidateJsonRawMessage: true,
}.Froze()

15
internal/jit/assembler_amd64.go
View file

@ -22,7 +22,7 @@ import (
`strings`
`sync`
`github.com/bytedance/sonic/internal/loader`
`github.com/bytedance/sonic/loader`
`github.com/bytedance/sonic/internal/rt`
`github.com/twitchyliquid64/golang-asm/obj`
`github.com/twitchyliquid64/golang-asm/obj/x86`
@ -202,14 +202,17 @@ func (self *BaseAssembler) Init(f func()) {
self.o = sync.Once{}
}
func (self *BaseAssembler) Load(fn string, fp int, args int) loader.Function {
self.build()
return loader.Loader(self.c).Load(fn, fp, args)
var jitLoader = loader.Loader{
Name: "sonic.jit.",
File: "github.com/bytedance/sonic/jit.go",
Options: loader.Options{
NoPreempt: true,
},
}
func (self *BaseAssembler) LoadWithFaker(fn string, fp int, args int, faker interface{}) loader.Function {
func (self *BaseAssembler) Load(name string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) loader.Function {
self.build()
return loader.Loader(self.c).LoadWithFaker(fn, fp, args, faker)
return jitLoader.LoadOne(self.c, name, frameSize, argSize, argStackmap, localStackmap)
}
/** Assembler Stages **/

32
internal/loader/funcdata.go
View file

@ -17,9 +17,11 @@
package loader
import (
`reflect`
`sync`
`unsafe`
`reflect`
`github.com/bytedance/sonic/internal/rt`
)
//go:linkname lastmoduledatap runtime.lastmoduledatap
@ -92,3 +94,31 @@ func stackMap(f interface{}) (args uintptr, locals uintptr) {
fi := findfunc(fv.Pointer())
return uintptr(funcdata(fi, uint8(_FUNCDATA_ArgsPointerMaps))), uintptr(funcdata(fi, uint8(_FUNCDATA_LocalsPointerMaps)))
}
var moduleCache = struct{
m map[*_ModuleData][]byte
l sync.Mutex
}{
m : make(map[*_ModuleData][]byte),
}
func cacheStackmap(argPtrs []bool, localPtrs []bool, mod *_ModuleData) (argptrs uintptr, localptrs uintptr) {
as := rt.StackMapBuilder{}
for _, b := range argPtrs {
as.AddField(b)
}
ab, _ := as.Build().MarshalBinary()
ls := rt.StackMapBuilder{}
for _, b := range localPtrs {
ls.AddField(b)
}
lb, _ := ls.Build().MarshalBinary()
cache := make([]byte, len(ab) + len(lb))
copy(cache, ab)
copy(cache[len(ab):], lb)
moduleCache.l.Lock()
moduleCache.m[mod] = cache
moduleCache.l.Unlock()
return uintptr(rt.IndexByte(cache, 0)), uintptr(rt.IndexByte(cache, len(ab)))
}

12
internal/loader/funcdata_go115.go
View file

@ -98,7 +98,8 @@ var findFuncTab = &_FindFuncBucket {
idx: 1,
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) {
mod := new(_ModuleData)
minpc := pc
maxpc := pc + size
@ -111,6 +112,9 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
4 << (^uintptr(0) >> 63), // ptrSize : 4 << (^uintptr(0) >> 63)
}
// cache arg and local stackmap
argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod)
/* add the function name */
noff := len(pclnt)
pclnt = append(append(pclnt, name...), 0)
@ -127,8 +131,8 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
args : int32(args),
pcsp : int32(pcsp),
nfuncdata : 2,
argptrs : argptrs,
localptrs : localptrs,
argptrs : uintptr(argptrs),
localptrs : uintptr(localptrs),
}
/* align the func to 8 bytes */
@ -148,7 +152,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
}
/* module data */
mod := &_ModuleData {
*mod = _ModuleData {
pclntable : pclnt,
ftab : tab,
findfunctab : findFuncTab,

14
internal/loader/funcdata_go116.go
View file

@ -1,3 +1,4 @@
//go:build go1.16 && !go1.18
// +build go1.16,!go1.18
/*
@ -126,10 +127,15 @@ func makePCtab(fp int) []byte {
return append([]byte{0}, encodeVariant((fp + 1) << 1)...)
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) {
mod := new(_ModuleData)
minpc := pc
maxpc := pc + size
// cache arg and local stackmap
argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod)
/* function entry */
lnt := []_Func {{
entry : pc,
@ -137,8 +143,8 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
args : int32(args),
pcsp : 1,
nfuncdata : 2,
argptrs : argptrs,
localptrs : localptrs,
argptrs : uintptr(argptrs),
localptrs : uintptr(localptrs),
}}
/* function table */
@ -149,7 +155,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
}
/* module data */
mod := &_ModuleData {
*mod = _ModuleData {
pcHeader : modHeader,
funcnametab : append(append([]byte{0}, name...), 0),
pctab : append(makePCtab(fp), encodeVariant(int(size))...),

9
internal/loader/funcdata_go118.go
View file

@ -128,7 +128,9 @@ func makePCtab(fp int) []byte {
return append([]byte{0}, encodeVariant((fp + 1) << 1)...)
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) {
mod := new(_ModuleData)
minpc := pc
maxpc := pc + size
@ -142,6 +144,9 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
textStart: minpc,
}
// cache arg and local stackmap
argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod)
base := argptrs
if argptrs > localptrs {
base = localptrs
@ -173,7 +178,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in
pclntab = append(pclntab, rt.BytesFrom(plnt, nlnt, nlnt)...)
/* module data */
mod := &_ModuleData {
*mod = _ModuleData {
pcHeader : modHeader,
funcnametab : append(append([]byte{0}, name...), 0),
pctab : append(makePCtab(fp), encodeVariant(int(size))...),

201
internal/loader/funcdata_go120.go Normal file
View file

@ -0,0 +1,201 @@
// +build go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
// A FuncFlag holds bits about a function.
// This list must match the list in cmd/internal/objabi/funcid.go.
type funcFlag uint8
type _Func struct {
entryOff uint32 // start pc
nameoff int32 // function name
args int32 // in/out args size
deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
pcsp uint32
pcfile uint32
pcln uint32
npcdata uint32
cuOffset uint32 // runtime.cutab offset of this function's CU
funcID uint8 // set for certain special runtime functions
flag funcFlag
_ [1]byte // pad
nfuncdata uint8 // must be last
argptrs uint32
localptrs uint32
}
type _FuncTab struct {
entry uint32
funcoff uint32
}
type _PCHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type _BitVector struct {
n int32 // # of bits
bytedata *uint8
}
type _PtabEntry struct {
name int32
typ int32
}
type _TextSection struct {
vaddr uintptr // prelinked section vaddr
length uintptr // section length
baseaddr uintptr // relocated section address
}
type _ModuleData struct {
pcHeader *_PCHeader
funcnametab []byte
cutab []uint32
filetab []byte
pctab []byte
pclntable []byte
ftab []_FuncTab
findfunctab *_FindFuncBucket
minpc, maxpc uintptr
text, etext uintptr
noptrdata, enoptrdata uintptr
data, edata uintptr
bss, ebss uintptr
noptrbss, enoptrbss uintptr
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
gofunc uintptr
textsectmap []_TextSection
typelinks []int32
itablinks []unsafe.Pointer
ptab []_PtabEntry
pluginpath string
pkghashes []struct{}
modulename string
modulehashes []struct{}
hasmain uint8
gcdatamask, gcbssmask _BitVector
typemap map[int32]unsafe.Pointer
bad bool
next *_ModuleData
}
type _FindFuncBucket struct {
idx uint32
subbuckets [16]byte
}
func makePCtab(fp int) []byte {
return append([]byte{0}, encodeVariant((fp + 1) << 1)...)
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) {
mod := new(_ModuleData)
minpc := pc
maxpc := pc + size
findFuncTab := make([]_FindFuncBucket, textSize/4096 + 1)
modHeader := &_PCHeader {
magic : 0xfffffff0,
minLC : 1,
nfunc : 1,
ptrSize : 4 << (^uintptr(0) >> 63),
textStart: minpc,
}
// cache arg and local stackmap
argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod)
base := argptrs
if argptrs > localptrs {
base = localptrs
}
/* function entry */
lnt := []_Func {{
entryOff : 0,
nameoff : 1,
args : int32(args),
pcsp : 1,
nfuncdata : 2,
argptrs: uint32(argptrs - base),
localptrs: uint32(localptrs - base),
}}
nlnt := len(lnt)*int(unsafe.Sizeof(_Func{}))
plnt := unsafe.Pointer(&lnt[0])
/* function table */
ftab := []_FuncTab {
{entry : 0, funcoff : 16},
{entry : uint32(size)},
}
nftab := len(ftab)*int(unsafe.Sizeof(_FuncTab{}))
pftab := unsafe.Pointer(&ftab[0])
pclntab := make([]byte, 0, nftab + nlnt)
pclntab = append(pclntab, rt.BytesFrom(pftab, nftab, nftab)...)
pclntab = append(pclntab, rt.BytesFrom(plnt, nlnt, nlnt)...)
/* module data */
*mod = _ModuleData {
pcHeader : modHeader,
funcnametab : append(append([]byte{0}, name...), 0),
pctab : append(makePCtab(fp), encodeVariant(int(size))...),
pclntable : pclntab,
ftab : ftab,
text : minpc,
etext : pc + textSize,
findfunctab : &findFuncTab[0],
minpc : minpc,
maxpc : maxpc,
modulename : name,
gcdata: uintptr(unsafe.Pointer(&emptyByte)),
gcbss: uintptr(unsafe.Pointer(&emptyByte)),
gofunc: base,
}
/* verify and register the new module */
moduledataverify1(mod)
registerModule(mod)
}

14
internal/loader/loader.go
View file

@ -1,5 +1,5 @@
//go:build linux || darwin
// +build linux darwin
//go:build darwin || linux
// +build darwin linux
/*
* Copyright 2021 ByteDance Inc.
@ -36,15 +36,15 @@ const (
type Loader []byte
type Function unsafe.Pointer
func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) {
func (self Loader) Load(fn string, fp int, args int, argPtrs []bool, localPtrs []bool) (f Function) {
p := os.Getpagesize()
n := (((len(self) - 1) / p) + 1) * p
/* register the function */
m := mmap(n)
v := fmt.Sprintf("runtime.__%s_%x", fn, m)
argsptr, localsptr := stackMap(faker)
registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr)
registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argPtrs, localPtrs)
/* reference as a slice */
s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader {
@ -59,10 +59,6 @@ func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{})
return Function(&m)
}
func (self Loader) Load(fn string, fp int, args int) (f Function) {
return self.LoadWithFaker(fn, fp, args, func(){})
}
func mmap(nb int) uintptr {
if m, _, e := syscall.RawSyscall6(syscall.SYS_MMAP, 0, uintptr(nb), _RW, _AP, 0, 0); e != 0 {
panic(e)

23
internal/loader/loader_go117_test.go
View file

@ -23,7 +23,6 @@ import (
`fmt`
`reflect`
`runtime`
`runtime/debug`
`testing`
`unsafe`
@ -36,7 +35,7 @@ func TestLoader_Load(t *testing.T) {
0xc3, // RET
}
v0 := 0
fn := Loader(bc).Load("test", 0, 8)
fn := Loader(bc).Load("test", 0, 8, nil, nil)
(*(*func(*int))(unsafe.Pointer(&fn)))(&v0)
assert.Equal(t, 1234, v0)
println(runtime.FuncForPC(*(*uintptr)(fn)).Name())
@ -90,23 +89,3 @@ func funcWrap(f func(i *int)) int {
ret = x
return ret
}
func TestLoadWithStackMap(t *testing.T) {
var f = func(i *int) {
*i = 1234
}
v1 := funcWrap(f)
bc := []byte {
0x48, 0xc7, 0x00, 0xd2, 0x04, 0x00, 0x00, // MOVQ $1234, (%rax)
0xc3, // RET
}
fn := Loader(bc).LoadWithFaker("test", 0, 8, f)
f2 := (*(*func(*int))(unsafe.Pointer(&fn)))
v2 := funcWrap(f2)
runtime.GC()
debug.FreeOSMemory()
println(v1, v2)
assert.Equal(t, v1, v2)
}

24
internal/loader/loader_test.go
View file

@ -23,7 +23,6 @@ import (
`fmt`
`reflect`
`runtime`
`runtime/debug`
`testing`
`unsafe`
@ -37,7 +36,7 @@ func TestLoader_Load(t *testing.T) {
0xc3, // RET
}
v0 := 0
fn := Loader(bc).Load("test", 0, 8)
fn := Loader(bc).Load("test", 0, 8, nil, nil)
(*(*func(*int))(unsafe.Pointer(&fn)))(&v0)
assert.Equal(t, 1234, v0)
println(runtime.FuncForPC(*(*uintptr)(fn)).Name())
@ -91,24 +90,3 @@ func funcWrap(f func(i *int)) int {
ret = x
return ret
}
func TestLoadWithStackMap(t *testing.T) {
var f = func(i *int) {
*i = 1234
}
v1 := funcWrap(f)
bc := []byte {
0x48, 0x8b, 0x44, 0x24, 0x08, // MOVQ 8(%rsp), %rax
0x48, 0xc7, 0x00, 0xd2, 0x04, 0x00, 0x00, // MOVQ $1234, (%rax)
0xc3, // RET
}
fn := Loader(bc).LoadWithFaker("test", 0, 8, f)
f2 := (*(*func(*int))(unsafe.Pointer(&fn)))
v2 := funcWrap(f2)
runtime.GC()
debug.FreeOSMemory()
println(v1, v2)
assert.Equal(t, v1, v2)
}

13
internal/loader/loader_windows.go
View file

@ -1,3 +1,6 @@
//go:build windows
// +build windows
/*
* Copyright 2021 ByteDance Inc.
*
@ -38,15 +41,15 @@ var (
type Loader []byte
type Function unsafe.Pointer
func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) {
func (self Loader) Load(fn string, fp int, args int, argPtrs []bool, localPtrs []bool) (f Function) {
p := os.Getpagesize()
n := (((len(self) - 1) / p) + 1) * p
/* register the function */
m := mmap(n)
v := fmt.Sprintf("runtime.__%s_%x", fn, m)
argsptr, localsptr := stackMap(faker)
registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr)
registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argPtrs, localPtrs)
/* reference as a slice */
s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader {
@ -61,10 +64,6 @@ func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{})
return Function(&m)
}
func (self Loader) Load(fn string, fp int, args int) (f Function) {
return self.LoadWithFaker(fn, fp, args, func(){})
}
func mmap(nb int) uintptr {
addr, err := winapi_VirtualAlloc(0, nb, MEM_COMMIT|MEM_RESERVE, syscall.PAGE_READWRITE)
if err != nil {

23
internal/rt/fastmem.go
View file

@ -74,6 +74,7 @@ func IndexByte(ptr []byte, index int) unsafe.Pointer {
return unsafe.Pointer(uintptr((*GoSlice)(unsafe.Pointer(&ptr)).Ptr) + uintptr(index))
}
//go:nosplit
func GuardSlice(buf *[]byte, n int) {
c := cap(*buf)
l := len(*buf)
@ -88,3 +89,25 @@ func GuardSlice(buf *[]byte, n int) {
}
return
}
//go:nosplit
func Ptr2SlicePtr(s unsafe.Pointer, l int, c int) unsafe.Pointer {
slice := &GoSlice{
Ptr: s,
Len: l,
Cap: c,
}
return unsafe.Pointer(slice)
}
//go:nosplit
func StrPtr(s string) unsafe.Pointer {
return (*GoString)(unsafe.Pointer(&s)).Ptr
}
//go:nosplit
func StrFrom(p unsafe.Pointer, n int64) (s string) {
(*GoString)(unsafe.Pointer(&s)).Ptr = p
(*GoString)(unsafe.Pointer(&s)).Len = int(n)
return
}

180
internal/rt/stackmap.go Normal file
View file

@ -0,0 +1,180 @@
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package rt
import (
`fmt`
`strings`
`unsafe`
)
type Bitmap struct {
N int
B []byte
}
func (self *Bitmap) grow() {
if self.N >= len(self.B) * 8 {
self.B = append(self.B, 0)
}
}
func (self *Bitmap) mark(i int, bv int) {
if bv != 0 {
self.B[i / 8] |= 1 << (i % 8)
} else {
self.B[i / 8] &^= 1 << (i % 8)
}
}
func (self *Bitmap) Set(i int, bv int) {
if i >= self.N {
panic("bitmap: invalid bit position")
} else {
self.mark(i, bv)
}
}
func (self *Bitmap) Append(bv int) {
self.grow()
self.mark(self.N, bv)
self.N++
}
func (self *Bitmap) AppendMany(n int, bv int) {
for i := 0; i < n; i++ {
self.Append(bv)
}
}
// var (
// _stackMapLock = sync.Mutex{}
// _stackMapCache = make(map[*StackMap]struct{})
// )
type BitVec struct {
N uintptr
B unsafe.Pointer
}
func (self BitVec) Bit(i uintptr) byte {
return (*(*byte)(unsafe.Pointer(uintptr(self.B) + i / 8)) >> (i % 8)) & 1
}
func (self BitVec) String() string {
var i uintptr
var v []string
/* add each bit */
for i = 0; i < self.N; i++ {
v = append(v, fmt.Sprintf("%d", self.Bit(i)))
}
/* join them together */
return fmt.Sprintf(
"BitVec { %s }",
strings.Join(v, ", "),
)
}
type StackMap struct {
N int32
L int32
B [1]byte
}
// func (self *StackMap) add() {
// _stackMapLock.Lock()
// _stackMapCache[self] = struct{}{}
// _stackMapLock.Unlock()
// }
func (self *StackMap) Pin() uintptr {
// self.add()
return uintptr(unsafe.Pointer(self))
}
func (self *StackMap) Get(i int32) BitVec {
return BitVec {
N: uintptr(self.L),
B: unsafe.Pointer(uintptr(unsafe.Pointer(&self.B)) + uintptr(i * ((self.L + 7) >> 3))),
}
}
func (self *StackMap) String() string {
sb := strings.Builder{}
sb.WriteString("StackMap {")
/* dump every stack map */
for i := int32(0); i < self.N; i++ {
sb.WriteRune('\n')
sb.WriteString(" " + self.Get(i).String())
}
/* close the stackmap */
sb.WriteString("\n}")
return sb.String()
}
func (self *StackMap) MarshalBinary() ([]byte, error) {
size := int(self.N) * int(self.L) + int(unsafe.Sizeof(self.L)) + int(unsafe.Sizeof(self.N))
return BytesFrom(unsafe.Pointer(self), size, size), nil
}
var (
byteType = UnpackEface(byte(0)).Type
)
const (
_StackMapSize = unsafe.Sizeof(StackMap{})
)
//go:linkname mallocgc runtime.mallocgc
//goland:noinspection GoUnusedParameter
func mallocgc(nb uintptr, vt *GoType, zero bool) unsafe.Pointer
type StackMapBuilder struct {
b Bitmap
}
func (self *StackMapBuilder) Build() (p *StackMap) {
nb := len(self.b.B)
bm := mallocgc(_StackMapSize + uintptr(nb) - 1, byteType, false)
/* initialize as 1 bitmap of N bits */
p = (*StackMap)(bm)
p.N, p.L = 1, int32(self.b.N)
copy(BytesFrom(unsafe.Pointer(&p.B), nb, nb), self.b.B)
return
}
func (self *StackMapBuilder) AddField(ptr bool) {
if ptr {
self.b.Append(1)
} else {
self.b.Append(0)
}
}
func (self *StackMapBuilder) AddFields(n int, ptr bool) {
if ptr {
self.b.AppendMany(n, 1)
} else {
self.b.AppendMany(n, 0)
}
}

144
loader/funcdata.go Normal file
View file

@ -0,0 +1,144 @@
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`encoding`
`encoding/binary`
`fmt`
`reflect`
`strings`
`sync`
`unsafe`
)
const (
_MinLC uint8 = 1
_PtrSize uint8 = 8
)
const (
_N_FUNCDATA = 8
_INVALID_FUNCDATA_OFFSET = ^uint32(0)
_FUNC_SIZE = unsafe.Sizeof(_func{})
_MINFUNC = 16 // minimum size for a function
_BUCKETSIZE = 256 * _MINFUNC
_SUBBUCKETS = 16
_SUB_BUCKETSIZE = _BUCKETSIZE / _SUBBUCKETS
)
// PCDATA and FUNCDATA table indexes.
//
// See funcdata.h and $GROOT/src/cmd/internal/objabi/funcdata.go.
const (
_FUNCDATA_ArgsPointerMaps = 0
_FUNCDATA_LocalsPointerMaps = 1
_FUNCDATA_StackObjects = 2
_FUNCDATA_InlTree = 3
_FUNCDATA_OpenCodedDeferInfo = 4
_FUNCDATA_ArgInfo = 5
_FUNCDATA_ArgLiveInfo = 6
_FUNCDATA_WrapInfo = 7
// ArgsSizeUnknown is set in Func.argsize to mark all functions
// whose argument size is unknown (C vararg functions, and
// assembly code without an explicit specification).
// This value is generated by the compiler, assembler, or linker.
ArgsSizeUnknown = -0x80000000
)
// moduledata used to cache the funcdata and findfuncbucket of one module
var moduleCache = struct {
m map[*moduledata][]byte
sync.Mutex
}{
m: make(map[*moduledata][]byte),
}
// Func contains information about a function.
type Func struct {
ID uint8 // see runtime/symtab.go
Flag uint8 // see runtime/symtab.go
ArgsSize int32 // args byte size
EntryOff uint32 // start pc, offset to moduledata.text
TextSize uint32 // size of func text
DeferReturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
FileIndex uint32 // index into filetab
Name string // name of function
// PC data
Pcsp *Pcdata // PC -> SP delta
Pcfile *Pcdata // PC -> file index
Pcline *Pcdata // PC -> line number
PcUnsafePoint *Pcdata // PC -> unsafe point, must be PCDATA_UnsafePointSafe or PCDATA_UnsafePointUnsafe
PcStackMapIndex *Pcdata // PC -> stack map index, relative to ArgsPointerMaps and LocalsPointerMaps
PcInlTreeIndex *Pcdata // PC -> inlining tree index, relative to InlTree
PcArgLiveIndex *Pcdata // PC -> arg live index, relative to ArgLiveInfo
// Func data, must implement encoding.BinaryMarshaler
ArgsPointerMaps encoding.BinaryMarshaler // concrete type: *StackMap
LocalsPointerMaps encoding.BinaryMarshaler // concrete type: *StackMap
StackObjects encoding.BinaryMarshaler
InlTree encoding.BinaryMarshaler
OpenCodedDeferInfo encoding.BinaryMarshaler
ArgInfo encoding.BinaryMarshaler
ArgLiveInfo encoding.BinaryMarshaler
WrapInfo encoding.BinaryMarshaler
}
func getOffsetOf(data interface{}, field string) uintptr {
t := reflect.TypeOf(data)
fv, ok := t.FieldByName(field)
if !ok {
panic(fmt.Sprintf("field %s not found in struct %s", field, t.Name()))
}
return fv.Offset
}
func rnd(v int64, r int64) int64 {
if r <= 0 {
return v
}
v += r - 1
c := v % r
if c < 0 {
c += r
}
v -= c
return v
}
var (
byteOrder binary.ByteOrder = binary.LittleEndian
)
func funcNameParts(name string) (string, string, string) {
i := strings.IndexByte(name, '[')
if i < 0 {
return name, "", ""
}
// TODO: use LastIndexByte once the bootstrap compiler is >= Go 1.5.
j := len(name) - 1
for j > i && name[j] != ']' {
j--
}
if j <= i {
return name, "", ""
}
return name[:i], "[...]", name[j+1:]
}

541
loader/funcdata_go115.go Normal file
View file

@ -0,0 +1,541 @@
// go:build go1.15 && !go1.18
// +build go1.15,!go1.18
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`encoding`
`os`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
const (
_Magic uint32 = 0xfffffff0
)
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type moduledata struct {
pcHeader *pcHeader
funcnametab []byte
cutab []uint32
filetab []byte
pctab []byte
pclntable []byte
ftab []funcTab
findfunctab uintptr
minpc, maxpc uintptr // first func address, last func address + last func size
text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC
noptrdata, enoptrdata uintptr
data, edata uintptr
bss, ebss uintptr
noptrbss, enoptrbss uintptr
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
gofunc uintptr // go.func.* is actual funcinfo object in image
textsectmap []textSection // see runtime/symtab.go: textAddr()
typelinks []int32 // offsets from types
itablinks []*rt.GoItab
ptab []ptabEntry
pluginpath string
pkghashes []modulehash
modulename string
modulehashes []modulehash
hasmain uint8 // 1 if module contains the main function, 0 otherwise
gcdatamask, gcbssmask bitVector
typemap map[int32]*rt.GoType // offset to *_rtype in previous module
bad bool // module failed to load and should be ignored
next *moduledata
}
type _func struct {
entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart
nameOff int32 // function name, as index into moduledata.funcnametab.
args int32 // in/out args size
deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
pcsp uint32
pcfile uint32
pcln uint32
npcdata uint32
cuOffset uint32 // runtime.cutab offset of this function's CU
funcID uint8 // set for certain special runtime functions
flag uint8
_ [1]byte // pad
nfuncdata uint8 //
// The end of the struct is followed immediately by two variable-length
// arrays that reference the pcdata and funcdata locations for this
// function.
// pcdata contains the offset into moduledata.pctab for the start of
// that index's table. e.g.,
// &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
// the unsafe point table.
//
// An offset of 0 indicates that there is no table.
//
// pcdata [npcdata]uint32
// funcdata contains the offset past moduledata.gofunc which contains a
// pointer to that index's funcdata. e.g.,
// *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
// the argument pointer map.
//
// An offset of ^uint32(0) indicates that there is no entry.
//
// funcdata [nfuncdata]uint32
}
type funcTab struct {
entry uint32
funcoff uint32
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
type compilationUnit struct {
fileNames []string
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4)
}
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize))
// write func info table
for i, f := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uint32((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab := makeFtab(_funcs, lastFuncsize)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// make pclnt table
pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

541
loader/funcdata_go118.go Normal file
View file

@ -0,0 +1,541 @@
// go:build go1.18 && !go1.20
// +build go1.18,!go1.20
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`encoding`
`os`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
const (
_Magic uint32 = 0xfffffff0
)
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type moduledata struct {
pcHeader *pcHeader
funcnametab []byte
cutab []uint32
filetab []byte
pctab []byte
pclntable []byte
ftab []funcTab
findfunctab uintptr
minpc, maxpc uintptr // first func address, last func address + last func size
text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC
noptrdata, enoptrdata uintptr
data, edata uintptr
bss, ebss uintptr
noptrbss, enoptrbss uintptr
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
gofunc uintptr // go.func.* is actual funcinfo object in image
textsectmap []textSection // see runtime/symtab.go: textAddr()
typelinks []int32 // offsets from types
itablinks []*rt.GoItab
ptab []ptabEntry
pluginpath string
pkghashes []modulehash
modulename string
modulehashes []modulehash
hasmain uint8 // 1 if module contains the main function, 0 otherwise
gcdatamask, gcbssmask bitVector
typemap map[int32]*rt.GoType // offset to *_rtype in previous module
bad bool // module failed to load and should be ignored
next *moduledata
}
type _func struct {
entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart
nameOff int32 // function name, as index into moduledata.funcnametab.
args int32 // in/out args size
deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
pcsp uint32
pcfile uint32
pcln uint32
npcdata uint32
cuOffset uint32 // runtime.cutab offset of this function's CU
funcID uint8 // set for certain special runtime functions
flag uint8
_ [1]byte // pad
nfuncdata uint8 //
// The end of the struct is followed immediately by two variable-length
// arrays that reference the pcdata and funcdata locations for this
// function.
// pcdata contains the offset into moduledata.pctab for the start of
// that index's table. e.g.,
// &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
// the unsafe point table.
//
// An offset of 0 indicates that there is no table.
//
// pcdata [npcdata]uint32
// funcdata contains the offset past moduledata.gofunc which contains a
// pointer to that index's funcdata. e.g.,
// *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
// the argument pointer map.
//
// An offset of ^uint32(0) indicates that there is no entry.
//
// funcdata [nfuncdata]uint32
}
type funcTab struct {
entry uint32
funcoff uint32
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
type compilationUnit struct {
fileNames []string
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4)
}
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize))
// write func info table
for i, f := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uint32((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab := makeFtab(_funcs, lastFuncsize)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// make pclnt table
pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

545
loader/funcdata_go120.go Normal file
View file

@ -0,0 +1,545 @@
//go:build go1.20 && !go1.21
// +build go1.20,!go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`encoding`
`os`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
)
const (
_Magic uint32 = 0xFFFFFFF1
)
type moduledata struct {
pcHeader *pcHeader
funcnametab []byte
cutab []uint32
filetab []byte
pctab []byte
pclntable []byte
ftab []funcTab
findfunctab uintptr
minpc, maxpc uintptr // first func address, last func address + last func size
text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC
noptrdata, enoptrdata uintptr
data, edata uintptr
bss, ebss uintptr
noptrbss, enoptrbss uintptr
covctrs, ecovctrs uintptr
end, gcdata, gcbss uintptr
types, etypes uintptr
rodata uintptr
// TODO: generate funcinfo object to memory
gofunc uintptr // go.func.* is actual funcinfo object in image
textsectmap []textSection // see runtime/symtab.go: textAddr()
typelinks []int32 // offsets from types
itablinks []*rt.GoItab
ptab []ptabEntry
pluginpath string
pkghashes []modulehash
modulename string
modulehashes []modulehash
hasmain uint8 // 1 if module contains the main function, 0 otherwise
gcdatamask, gcbssmask bitVector
typemap map[int32]*rt.GoType // offset to *_rtype in previous module
bad bool // module failed to load and should be ignored
next *moduledata
}
type _func struct {
entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart
nameOff int32 // function name, as index into moduledata.funcnametab.
args int32 // in/out args size
deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
pcsp uint32
pcfile uint32
pcln uint32
npcdata uint32
cuOffset uint32 // runtime.cutab offset of this function's CU
startLine int32 // line number of start of function (func keyword/TEXT directive)
funcID uint8 // set for certain special runtime functions
flag uint8
_ [1]byte // pad
nfuncdata uint8 //
// The end of the struct is followed immediately by two variable-length
// arrays that reference the pcdata and funcdata locations for this
// function.
// pcdata contains the offset into moduledata.pctab for the start of
// that index's table. e.g.,
// &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
// the unsafe point table.
//
// An offset of 0 indicates that there is no table.
//
// pcdata [npcdata]uint32
// funcdata contains the offset past moduledata.gofunc which contains a
// pointer to that index's funcdata. e.g.,
// *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
// the argument pointer map.
//
// An offset of ^uint32(0) indicates that there is no entry.
//
// funcdata [nfuncdata]uint32
}
type funcTab struct {
entry uint32
funcoff uint32
}
type pcHeader struct {
magic uint32 // 0xFFFFFFF0
pad1, pad2 uint8 // 0,0
minLC uint8 // min instruction size
ptrSize uint8 // size of a ptr in bytes
nfunc int // number of functions in the module
nfiles uint // number of entries in the file tab
textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text
funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
cuOffset uintptr // offset to the cutab variable from pcHeader
filetabOffset uintptr // offset to the filetab variable from pcHeader
pctabOffset uintptr // offset to the pctab variable from pcHeader
pclnOffset uintptr // offset to the pclntab variable from pcHeader
}
type bitVector struct {
n int32 // # of bits
bytedata *uint8
}
type ptabEntry struct {
name int32
typ int32
}
type textSection struct {
vaddr uintptr // prelinked section vaddr
end uintptr // vaddr + section length
baseaddr uintptr // relocated section address
}
type modulehash struct {
modulename string
linktimehash string
runtimehash *string
}
// findfuncbucket is an array of these structures.
// Each bucket represents 4096 bytes of the text segment.
// Each subbucket represents 256 bytes of the text segment.
// To find a function given a pc, locate the bucket and subbucket for
// that pc. Add together the idx and subbucket value to obtain a
// function index. Then scan the functab array starting at that
// index to find the target function.
// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
type findfuncbucket struct {
idx uint32
_SUBBUCKETS [16]byte
}
// func name table format:
// nameOff[0] -> namePartA namePartB namePartC \x00
// nameOff[1] -> namePartA namePartB namePartC \x00
// ...
func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) {
offs = make([]int32, len(funcs))
offset := 0
for i, f := range funcs {
offs[i] = int32(offset)
a, b, c := funcNameParts(f.Name)
tab = append(tab, a...)
tab = append(tab, b...)
tab = append(tab, c...)
tab = append(tab, 0)
offset += len(a) + len(b) + len(c) + 1
}
return
}
type compilationUnit struct {
fileNames []string
}
// CU table format:
// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1]
// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1]
// ...
//
// file name table format:
// filetabOffset[0] -> CUs[0].fileNames[0] \x00
// ...
// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00
// ...
// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00
func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) {
cuOffsets = make([]uint32, len(cus))
cuOffset := 0
fileOffset := 0
for i, cu := range cus {
cuOffsets[i] = uint32(cuOffset)
for _, name := range cu.fileNames {
cutab = append(cutab, uint32(fileOffset))
fileOffset += len(name) + 1
filetab = append(filetab, name...)
filetab = append(filetab, 0)
}
cuOffset += len(cu.fileNames)
}
return
}
func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) {
fstart = len(*out)
*out = append(*out, byte(0))
offs := uint32(1)
funcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
var writer = func(fd encoding.BinaryMarshaler) {
var ab []byte
var err error
if fd != nil {
ab, err = fd.MarshalBinary()
if err != nil {
panic(err)
}
funcdataOffs[i] = append(funcdataOffs[i], offs)
} else {
ab = []byte{0}
funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET)
}
*out = append(*out, ab...)
offs += uint32(len(ab))
}
writer(f.ArgsPointerMaps)
writer(f.LocalsPointerMaps)
writer(f.StackObjects)
writer(f.InlTree)
writer(f.OpenCodedDeferInfo)
writer(f.ArgInfo)
writer(f.ArgLiveInfo)
writer(f.WrapInfo)
}
return
}
func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i, f := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4)
}
ftab = make([]funcTab, 0, len(funcs)+1)
// write a map of pc->func info offsets
for i, f := range funcs {
ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])})
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0})
return
}
// Pcln table format: [...]funcTab + [...]_Func
func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) {
// Allocate space for the pc->func table. This structure consists of a pc offset
// and an offset to the func structure. After that, we have a single pc
// value that marks the end of the last function in the binary.
var size int64 = int64(len(funcs)*2*4 + 4)
var startLocations = make([]uint32, len(funcs))
for i := range funcs {
size = rnd(size, int64(_PtrSize))
//writePCToFunc
startLocations[i] = uint32(size)
size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4)
}
pclntab = make([]byte, size, size)
// write a map of pc->func info offsets
offs := 0
for i, f := range funcs {
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff))
byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i]))
offs += 8
}
// Final entry of table is just end pc offset.
lastFunc := funcs[len(funcs)-1]
byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize))
// write func info table
for i, f := range funcs {
off := startLocations[i]
// write _func structure to pclntab
fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE))
copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb)
off += uint32(_FUNC_SIZE)
// NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
for j := 3; j < len(pcdataOffs[i]); j++ {
byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
off += 4
}
// funcdata refs as offsets from gofunc
for _, funcdata := range funcdataOffs[i] {
byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata))
off += 4
}
}
return
}
// findfunc table used to map pc to belonging func,
// returns the index in the func table.
//
// All text section are divided into buckets sized _BUCKETSIZE(4K):
// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
// and it has a base idx to plus the offset stored in jth subbucket.
// see findfunc() in runtime/symtab.go
func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
start = len(*out)
max := ftab[len(ftab)-1].entry
min := ftab[0].entry
nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
tab := make([]findfuncbucket, 0, nbuckets)
var s, e = 0, 0
for i := 0; i<int(nbuckets); i++ {
var pc = min + uint32((i+1)*_BUCKETSIZE)
// find the end func of the bucket
for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
// store the start func of the bucket
var fb = findfuncbucket{idx: uint32(s)}
for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE)
var ss = s
// find the end func of the subbucket
for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {}
// store the start func of the subbucket
fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
s = ss
}
s = e
tab = append(tab, fb)
}
// write findfuncbucket
if len(tab) > 0 {
size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
*out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
}
return
}
func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) {
mod = new(moduledata)
mod.modulename = name
// make filename table
cu := make([]string, 0, len(filenames))
for _, f := range filenames {
cu = append(cu, f)
}
cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
mod.cutab = cutab
mod.filetab = filetab
// make funcname table
funcnametab, nameOffs := makeFuncnameTab(funcs)
mod.funcnametab = funcnametab
// make pcdata table
// NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs)
mod.pctab = pctab
// write func data
// NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
// TODO: estimate accurate capacity
cache := make([]byte, 0, len(funcs)*int(_PtrSize))
fstart, funcdataOffs := writeFuncdata(&cache, funcs)
// make pc->func (binary search) func table
lastFuncsize := funcs[len(funcs)-1].TextSize
ftab := makeFtab(_funcs, lastFuncsize)
mod.ftab = ftab
// write pc->func (modmap) findfunc table
ffstart := writeFindfunctab(&cache, ftab)
// make pclnt table
pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs)
mod.pclntable = pclntab
// mmap() text and funcdata segements
p := os.Getpagesize()
size := int(rnd(int64(len(text)), int64(p)))
addr := mmap(size)
// copy the machine code
s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
copy(s, text)
// make it executable
mprotect(addr, size)
// assign addresses
mod.text = addr
mod.etext = addr + uintptr(size)
mod.minpc = addr
mod.maxpc = addr + uintptr(len(text))
// cache funcdata and findfuncbucket
moduleCache.Lock()
moduleCache.m[mod] = cache
moduleCache.Unlock()
mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart]))
mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart]))
// make pc header
mod.pcHeader = &pcHeader {
magic : _Magic,
minLC : _MinLC,
ptrSize : _PtrSize,
nfunc : len(funcs),
nfiles: uint(len(cu)),
textStart: mod.text,
funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
cuOffset: getOffsetOf(moduledata{}, "cutab"),
filetabOffset: getOffsetOf(moduledata{}, "filetab"),
pctabOffset: getOffsetOf(moduledata{}, "pctab"),
pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
}
// sepecial case: gcdata and gcbss must by non-empty
mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
return
}
// makePctab generates pcdelta->valuedelta tables for functions,
// and returns the table and the entry offset of every kind pcdata in the table.
func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
_funcs = make([]_func, len(funcs))
// Pctab offsets of 0 are considered invalid in the runtime. We respect
// that by just padding a single byte at the beginning of runtime.pctab,
// that way no real offsets can be zero.
pctab = make([]byte, 1, 12*len(funcs)+1)
pcdataOffs = make([][]uint32, len(funcs))
for i, f := range funcs {
_f := &_funcs[i]
var writer = func(pc *Pcdata) {
var ab []byte
var err error
if pc != nil {
ab, err = pc.MarshalBinary()
if err != nil {
panic(err)
}
pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
} else {
ab = []byte{0}
pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
}
pctab = append(pctab, ab...)
}
if f.Pcsp != nil {
_f.pcsp = uint32(len(pctab))
}
writer(f.Pcsp)
if f.Pcfile != nil {
_f.pcfile = uint32(len(pctab))
}
writer(f.Pcfile)
if f.Pcline != nil {
_f.pcln = uint32(len(pctab))
}
writer(f.Pcline)
writer(f.PcUnsafePoint)
writer(f.PcStackMapIndex)
writer(f.PcInlTreeIndex)
writer(f.PcArgLiveIndex)
_f.entryOff = f.EntryOff
_f.nameOff = nameOffset[i]
_f.args = f.ArgsSize
_f.deferreturn = f.DeferReturn
// NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
_f.npcdata = uint32(_N_PCDATA)
_f.cuOffset = cuOffset[i]
_f.funcID = f.ID
_f.flag = f.Flag
_f.nfuncdata = uint8(_N_FUNCDATA)
}
return
}
func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}

37
loader/loader.go Normal file
View file

@ -0,0 +1,37 @@
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`unsafe`
)
// Function is a function pointer
type Function unsafe.Pointer
// Options used to load a module
type Options struct {
// NoPreempt is used to disable async preemption for this module
NoPreempt bool
}
// Loader is a helper used to load a module simply
type Loader struct {
Name string // module name
File string // file name
Options
}

33
loader/loader_go115.go Normal file
View file

@ -0,0 +1,33 @@
//go:build go1.15 && !go1.18
// +build go1.15,!go1.18
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`github.com/bytedance/sonic/internal/loader`
)
func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function {
return Function(loader.Loader(text).Load(funcName, frameSize, argSize, argStackmap, localStackmap))
}
func Load(modulename string, filenames []string, funcs []Func, text []byte) (out []Function) {
panic("not implemented")
}

104
loader/loader_go118.go Normal file
View file

@ -0,0 +1,104 @@
//go:build go1.18 && !go1.21
// +build go1.18,!go1.21
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`github.com/bytedance/sonic/internal/rt`
)
// LoadFuncs loads only one function as module, and returns the function pointer
// - text: machine code
// - funcName: function name
// - frameSize: stack frame size.
// - argSize: argument total size (in bytes)
// - argPtrs: indicates if a slot (8 Bytes) of arguments memory stores pointer, from low to high
// - localPtrs: indicates if a slot (8 Bytes) of local variants memory stores pointer, from low to high
//
// WARN:
// - the function MUST has fixed SP offset equaling to this, otherwise it go.gentraceback will fail
// - the function MUST has only one stack map for all arguments and local variants
func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function {
size := uint32(len(text))
fn := Func{
Name: funcName,
TextSize: size,
ArgsSize: int32(argSize),
}
// NOTICE: suppose the function has fixed SP offset equaling to frameSize, thus make only one pcsp pair
fn.Pcsp = &Pcdata{
{PC: size, Val: int32(frameSize)},
}
if self.NoPreempt {
fn.PcUnsafePoint = &Pcdata{
{PC: size, Val: PCDATA_UnsafePointUnsafe},
}
} else {
fn.PcUnsafePoint = &Pcdata{
{PC: size, Val: PCDATA_UnsafePointSafe},
}
}
// NOTICE: suppose the function has only one stack map at index 0
fn.PcStackMapIndex = &Pcdata{
{PC: size, Val: 0},
}
if argPtrs != nil {
args := rt.StackMapBuilder{}
for _, b := range argPtrs {
args.AddField(b)
}
fn.ArgsPointerMaps = args.Build()
}
if localPtrs != nil {
locals := rt .StackMapBuilder{}
for _, b := range localPtrs {
locals.AddField(b)
}
fn.LocalsPointerMaps = locals.Build()
}
out := Load(text, []Func{fn}, self.Name + funcName, []string{self.File})
return out[0]
}
// Load loads given machine codes and corresponding function information into go moduledata
// and returns runnable function pointer
// WARN: this API is experimental, use it carefully
func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) {
// generate module data and allocate memory address
mod := makeModuledata(modulename, filenames, funcs, text)
// verify and register the new module
moduledataverify1(mod)
registerModule(mod)
// encapsulate function address
out = make([]Function, len(funcs))
for i, f := range funcs {
m := uintptr(mod.text + uintptr(f.EntryOff))
out[i] = Function(&m)
}
return
}

134
loader/loader_test.go Normal file
View file

@ -0,0 +1,134 @@
//go:build go1.18
// +build go1.18
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`runtime`
`runtime/debug`
`strconv`
`testing`
`unsafe`
`github.com/bytedance/sonic/internal/rt`
`github.com/stretchr/testify/require`
)
func TestLoad(t *testing.T) {
// defer func() {
// if r := recover(); r != nil {
// runtime.GC()
// if r != "hook1" {
// t.Fatal("not right panic:" + r.(string))
// }
// } else {
// t.Fatal("not panic")
// }
// }()
var hstr string
type TestFunc func(i *int, hook func(i *int)) int
var hook = func(i *int) {
runtime.GC()
debug.FreeOSMemory()
hstr = ("hook" + strconv.Itoa(*i))
runtime.GC()
debug.FreeOSMemory()
}
// var f TestFunc = func(i *int, hook func(i *int)) int {
// var t = *i
// hook(i)
// return t + *i
// }
bc := []byte {
0x48, 0x83, 0xec, 0x18, // (0x00) subq $24, %rsp
0x48, 0x89, 0x6c, 0x24, 0x10, // (0x04) movq %rbp, 16(%rsp)
0x48, 0x8d, 0x6c, 0x24, 0x10, // (0x09) leaq 16(%rsp), %rbp
0x48, 0x89, 0x44, 0x24, 0x20, // (0x0e) movq %rax, 32(%rsp)
0x48, 0x8b, 0x08, // (0x13) movq (%rax), %rcx
0x48, 0x89, 0x4c, 0x24, 0x08, // (0x16) movq %rcx, 8(%rsp)
0x48, 0x8b, 0x33, // (0x1b) movq (%rbx), %rsi
0x48, 0x89, 0xda, // (0x1e) movq %rbx, %rdx
0xff, 0xd6, // (0x21) callq %rsi
0x48, 0x8b, 0x44, 0x24, 0x08, // (0x23) movq 8(%rsp), %rax
0x48, 0x8b, 0x4c, 0x24, 0x20, // (0x28) movq 32(%rsp), %rcx
0x48, 0x03, 0x01, // (0x2d) addq (%rcx), %rax
0x48, 0x8b, 0x6c, 0x24, 0x10, // (0x30) movq 16(%rsp), %rbp
0x48, 0x83, 0xc4, 0x18, // (0x35) addq $24, %rsp
0xc3, // (0x39) ret
}
size := uint32(len(bc))
fn := Func{
ID: 0,
Flag: 0,
ArgsSize: 16,
EntryOff: 0,
TextSize: size,
DeferReturn: 0,
FileIndex: 0,
Name: "dummy",
}
fn.Pcsp = &Pcdata{
{PC: 0x04, Val: 0},
{PC: size, Val: 24},
}
fn.Pcline = &Pcdata{
{PC: 0x00, Val: 0},
{PC: 0x0e, Val: 1},
{PC: 0x1d, Val: 2},
{PC: size, Val: 3},
}
fn.Pcfile = &Pcdata{
{PC: size, Val: 0},
}
fn.PcUnsafePoint = &Pcdata{
{PC: size, Val: PCDATA_UnsafePointUnsafe},
}
fn.PcStackMapIndex = &Pcdata{
{PC: size, Val: 0},
}
args := rt.StackMapBuilder{}
args.AddField(true)
args.AddField(true)
fn.ArgsPointerMaps = args.Build()
locals := rt.StackMapBuilder{}
locals.AddField(false)
locals.AddField(false)
fn.LocalsPointerMaps = locals.Build()
rets := Load(bc, []Func{fn}, "dummy_module", []string{"github.com/bytedance/sonic/dummy.go"})
println("func address ", *(*unsafe.Pointer)(rets[0]))
// for k, _ := range moduleCache.m {
// spew.Dump(k)
// }
f := *(*TestFunc)(unsafe.Pointer(&rets[0]))
i := 1
j := f(&i, hook)
require.Equal(t, 2, j)
require.Equal(t, "hook1", hstr)
}

45
loader/mmap_unix.go Normal file
View file

@ -0,0 +1,45 @@
//go:build darwin || linux
// +build darwin linux
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`syscall`
)
const (
_AP = syscall.MAP_ANON | syscall.MAP_PRIVATE
_RX = syscall.PROT_READ | syscall.PROT_EXEC
_RW = syscall.PROT_READ | syscall.PROT_WRITE
)
func mmap(nb int) uintptr {
if m, _, e := syscall.RawSyscall6(syscall.SYS_MMAP, 0, uintptr(nb), _RW, _AP, 0, 0); e != 0 {
panic(e)
} else {
return m
}
}
func mprotect(p uintptr, nb int) {
if _, _, err := syscall.RawSyscall(syscall.SYS_MPROTECT, p, uintptr(nb), _RX); err != 0 {
panic(err)
}
}

84
loader/mmap_windows.go Normal file
View file

@ -0,0 +1,84 @@
//go:build windows
// +build windows
// build
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
import (
`syscall`
`unsafe`
)
const (
MEM_COMMIT = 0x00001000
MEM_RESERVE = 0x00002000
)
var (
libKernel32 = syscall.NewLazyDLL("KERNEL32.DLL")
libKernel32_VirtualAlloc = libKernel32.NewProc("VirtualAlloc")
libKernel32_VirtualProtect = libKernel32.NewProc("VirtualProtect")
)
func mmap(nb int) uintptr {
addr, err := winapi_VirtualAlloc(0, nb, MEM_COMMIT|MEM_RESERVE, syscall.PAGE_READWRITE)
if err != nil {
panic(err)
}
return addr
}
func mprotect(p uintptr, nb int) (oldProtect int) {
err := winapi_VirtualProtect(p, nb, syscall.PAGE_EXECUTE_READ, &oldProtect)
if err != nil {
panic(err)
}
return
}
// winapi_VirtualAlloc allocate memory
// Doc: https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualalloc
func winapi_VirtualAlloc(lpAddr uintptr, dwSize int, flAllocationType int, flProtect int) (uintptr, error) {
r1, _, err := libKernel32_VirtualAlloc.Call(
lpAddr,
uintptr(dwSize),
uintptr(flAllocationType),
uintptr(flProtect),
)
if r1 == 0 {
return 0, err
}
return r1, nil
}
// winapi_VirtualProtect change memory protection
// Doc: https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualprotect
func winapi_VirtualProtect(lpAddr uintptr, dwSize int, flNewProtect int, lpflOldProtect *int) error {
r1, _, err := libKernel32_VirtualProtect.Call(
lpAddr,
uintptr(dwSize),
uintptr(flNewProtect),
uintptr(unsafe.Pointer(lpflOldProtect)),
)
if r1 == 0 {
return err
}
return nil
}

100
loader/pcdata.go Normal file
View file

@ -0,0 +1,100 @@
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package loader
const (
_N_PCDATA = 4
_PCDATA_UnsafePoint = 0
_PCDATA_StackMapIndex = 1
_PCDATA_InlTreeIndex = 2
_PCDATA_ArgLiveIndex = 3
_PCDATA_INVALID_OFFSET = 0
)
const (
// PCDATA_UnsafePoint values.
PCDATA_UnsafePointSafe = -1 // Safe for async preemption
PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption
// PCDATA_Restart1(2) apply on a sequence of instructions, within
// which if an async preemption happens, we should back off the PC
// to the start of the sequence when resume.
// We need two so we can distinguish the start/end of the sequence
// in case that two sequences are next to each other.
PCDATA_Restart1 = -3
PCDATA_Restart2 = -4
// Like PCDATA_RestartAtEntry, but back to function entry if async
// preempted.
PCDATA_RestartAtEntry = -5
_PCDATA_START_VAL = -1
)
var emptyByte byte
func encodeValue(v int) []byte {
return encodeVariant(toZigzag(v))
}
func toZigzag(v int) int {
return (v << 1) ^ (v >> 31)
}
func encodeVariant(v int) []byte {
var u int
var r []byte
/* split every 7 bits */
for v > 127 {
u = v & 0x7f
v = v >> 7
r = append(r, byte(u) | 0x80)
}
/* check for last one */
if v == 0 {
return r
}
/* add the last one */
r = append(r, byte(v))
return r
}
type Pcvalue struct {
PC uint32 // PC offset from func entry
Val int32
}
type Pcdata []Pcvalue
// see https://docs.google.com/document/d/1lyPIbmsYbXnpNj57a261hgOYVpNRcgydurVQIyZOz_o/pub
func (self Pcdata) MarshalBinary() (data []byte, err error) {
// delta value always starts from -1
sv := int32(_PCDATA_START_VAL)
sp := uint32(0)
for _, v := range self {
data = append(data, encodeVariant(toZigzag(int(v.Val - sv)))...)
data = append(data, encodeVariant(int(v.PC - sp))...)
sp = v.PC
sv = v.Val
}
return
}

31
internal/loader/funcdata_invalid.go → loader/stubs.go
View file

@ -1,14 +1,12 @@
// +build !go1.15 go1.20
/*
* Copyright 2021 ByteDance Inc.
*
/**
* Copyright 2023 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,11 +16,20 @@
package loader
// triggers a compilation error
const (
_ = panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17, 1.18, 1.19")
import (
_ `unsafe`
)
func registerFunction(_ string, _ uintptr, _ int, _ int, _ uintptr) {
panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17, 1.18, 1.19")
//go:linkname lastmoduledatap runtime.lastmoduledatap
//goland:noinspection GoUnusedGlobalVariable
var lastmoduledatap *moduledata
func registerModule(mod *moduledata) {
lastmoduledatap.next = mod
lastmoduledatap = mod
}
//go:linkname moduledataverify1 runtime.moduledataverify1
func moduledataverify1(_ *moduledata)