From e7ac2f25fccf9abf94fc7d407261c32a5b1250a6 Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Wed, 8 Feb 2023 14:35:00 +0800 Subject: [PATCH] feat: more complete function loader (#354) * follow complete implementation of go symtab * support go1.20 --- .github/workflows/push-check-go118.yml | 6 +- .github/workflows/push-check-linux-arm64.yml | 2 +- .github/workflows/push-check-linux-x64.yml | 2 +- .github/workflows/push-check-qemu.yml | 2 +- .github/workflows/push-check-windows.yml | 2 +- ast/api_amd64_test.go | 3 +- ast/encode.go | 6 +- ast/node.go | 31 +- ast/{stubs.go => stubs_go115.go} | 30 +- ast/stubs_go120.go | 55 ++ decoder/assembler_amd64_go116.go | 3 +- decoder/assembler_amd64_go117.go | 84 +-- decoder/compiler.go | 1 + decoder/decoder.go | 4 +- decoder/generic_amd64_go116.go | 2 +- decoder/generic_amd64_go117.go | 5 +- decoder/generic_amd64_go117_test.s | 2 +- decoder/norace_test.go | 14 +- decoder/pools.go | 39 +- decoder/{stubs.go => stubs_go115.go} | 1 + decoder/stubs_go120.go | 111 ++++ decoder/utils.go | 2 +- encoder/assembler_amd64_go116.go | 8 +- encoder/assembler_amd64_go117.go | 6 +- encoder/compiler.go | 1 + encoder/debug_go117.go | 2 +- encoder/pools.go | 8 +- encoder/primitives.go | 22 + encoder/stubs_go116.go | 14 - encoder/stubs_go117.go | 23 +- encoder/stubs_go120.go | 66 +++ encoder/utils.go | 2 +- .../benchmark_test/encoder_test.go | 25 + external_jsonlib_test/unit_test/api_test.go | 25 + internal/jit/assembler_amd64.go | 15 +- internal/loader/funcdata.go | 32 +- internal/loader/funcdata_go115.go | 12 +- internal/loader/funcdata_go116.go | 14 +- internal/loader/funcdata_go118.go | 9 +- internal/loader/funcdata_go120.go | 201 +++++++ internal/loader/loader.go | 14 +- internal/loader/loader_go117_test.go | 23 +- internal/loader/loader_test.go | 24 +- internal/loader/loader_windows.go | 13 +- internal/rt/fastmem.go | 23 + internal/rt/stackmap.go | 180 ++++++ loader/funcdata.go | 144 +++++ loader/funcdata_go115.go | 541 +++++++++++++++++ loader/funcdata_go118.go | 541 +++++++++++++++++ loader/funcdata_go120.go | 545 ++++++++++++++++++ loader/loader.go | 37 ++ loader/loader_go115.go | 33 ++ loader/loader_go118.go | 104 ++++ loader/loader_test.go | 134 +++++ loader/mmap_unix.go | 45 ++ loader/mmap_windows.go | 84 +++ loader/pcdata.go | 100 ++++ .../funcdata_invalid.go => loader/stubs.go | 31 +- 58 files changed, 3236 insertions(+), 272 deletions(-) rename ast/{stubs.go => stubs_go115.go} (73%) create mode 100644 ast/stubs_go120.go rename decoder/{stubs.go => stubs_go115.go} (99%) create mode 100644 decoder/stubs_go120.go create mode 100644 encoder/stubs_go120.go create mode 100644 internal/loader/funcdata_go120.go create mode 100644 internal/rt/stackmap.go create mode 100644 loader/funcdata.go create mode 100644 loader/funcdata_go115.go create mode 100644 loader/funcdata_go118.go create mode 100644 loader/funcdata_go120.go create mode 100644 loader/loader.go create mode 100644 loader/loader_go115.go create mode 100644 loader/loader_go118.go create mode 100644 loader/loader_test.go create mode 100644 loader/mmap_unix.go create mode 100644 loader/mmap_windows.go create mode 100644 loader/pcdata.go rename internal/loader/funcdata_invalid.go => loader/stubs.go (59%) diff --git a/.github/workflows/push-check-go118.yml b/.github/workflows/push-check-go118.yml index a9c2306..795e140 100644 --- a/.github/workflows/push-check-go118.yml +++ b/.github/workflows/push-check-go118.yml @@ -25,8 +25,8 @@ jobs: - name: Unit Test run: | - GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./... - GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./external_jsonlib_test/... + GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./... + GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./external_jsonlib_test/... - name: Generic Test - run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./generic_test \ No newline at end of file + run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./generic_test \ No newline at end of file diff --git a/.github/workflows/push-check-linux-arm64.yml b/.github/workflows/push-check-linux-arm64.yml index 19f421c..9ee5f80 100644 --- a/.github/workflows/push-check-linux-arm64.yml +++ b/.github/workflows/push-check-linux-arm64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.19.x] + go-version: [1.15.x, 1.20.x] os: [arm] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/push-check-linux-x64.yml b/.github/workflows/push-check-linux-x64.yml index a52c56e..650dd8a 100644 --- a/.github/workflows/push-check-linux-x64.yml +++ b/.github/workflows/push-check-linux-x64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x] + go-version: [1.15.x, 1.16.x, 1.17.x, 1.19.x, 1.20.x] runs-on: [self-hosted, X64] steps: - name: Clear repository diff --git a/.github/workflows/push-check-qemu.yml b/.github/workflows/push-check-qemu.yml index f40c8bc..ffbdf48 100644 --- a/.github/workflows/push-check-qemu.yml +++ b/.github/workflows/push-check-qemu.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.19.x] + go-version: [1.15.x, 1.20.x] os: [arm] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/push-check-windows.yml b/.github/workflows/push-check-windows.yml index 453ec9a..77fc6f0 100644 --- a/.github/workflows/push-check-windows.yml +++ b/.github/workflows/push-check-windows.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.19.x] + go-version: [1.15.x, 1.20.x] runs-on: windows-latest steps: - uses: actions/checkout@v2 diff --git a/ast/api_amd64_test.go b/ast/api_amd64_test.go index 391baaf..aacaf75 100644 --- a/ast/api_amd64_test.go +++ b/ast/api_amd64_test.go @@ -10,7 +10,8 @@ import ( `github.com/stretchr/testify/assert` ) -func TestSortNodeTwitter(t *testing.T) {root, err := NewSearcher(_TwitterJson).GetByPath() +func TestSortNodeTwitter(t *testing.T) { + root, err := NewSearcher(_TwitterJson).GetByPath() if err != nil { t.Fatal(err) } diff --git a/ast/encode.go b/ast/encode.go index b9bcaea..1187e30 100644 --- a/ast/encode.go +++ b/ast/encode.go @@ -19,6 +19,8 @@ package ast import ( `sync` `unicode/utf8` + + `github.com/bytedance/sonic/internal/rt` ) const ( @@ -163,7 +165,7 @@ func (self *Node) encodeFalse(buf *[]byte) error { } func (self *Node) encodeNumber(buf *[]byte) error { - str := addr2str(self.p, self.v) + str := rt.StrFrom(self.p, self.v) *buf = append(*buf, str...) return nil } @@ -174,7 +176,7 @@ func (self *Node) encodeString(buf *[]byte) error { return nil } - quote(buf, addr2str(self.p, self.v)) + quote(buf, rt.StrFrom(self.p, self.v)) return nil } diff --git a/ast/node.go b/ast/node.go index d9f9c0d..c8f148f 100644 --- a/ast/node.go +++ b/ast/node.go @@ -21,6 +21,7 @@ import ( `fmt` `strconv` `unsafe` + `reflect` `github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/rt` @@ -60,6 +61,10 @@ const ( V_ANY = int(_V_ANY) ) +var ( + byteType = rt.UnpackType(reflect.TypeOf(byte(0))) +) + type Node struct { v int64 t types.ValueType @@ -151,7 +156,7 @@ func (self *Node) Raw() (string, error) { buf, err := self.MarshalJSON() return rt.Mem2Str(buf), err } - return addr2str(self.p, self.v), nil + return rt.StrFrom(self.p, self.v), nil } func (self *Node) checkRaw() error { @@ -183,7 +188,7 @@ func (self *Node) Bool() (bool, error) { } else { return false, err } - case types.V_STRING: return strconv.ParseBool(addr2str(self.p, self.v)) + case types.V_STRING: return strconv.ParseBool(rt.StrFrom(self.p, self.v)) case _V_ANY : any := self.packAny() switch v := any.(type) { @@ -389,7 +394,7 @@ func (self *Node) String() (string, error) { case types.V_NULL : return "" , nil case types.V_TRUE : return "true" , nil case types.V_FALSE : return "false", nil - case types.V_STRING, _V_NUMBER : return addr2str(self.p, self.v), nil + case types.V_STRING, _V_NUMBER : return rt.StrFrom(self.p, self.v), nil case _V_ANY : any := self.packAny() switch v := any.(type) { @@ -421,7 +426,7 @@ func (self *Node) StrictString() (string, error) { return "", err } switch self.t { - case types.V_STRING : return addr2str(self.p, self.v), nil + case types.V_STRING : return rt.StrFrom(self.p, self.v), nil case _V_ANY : if v, ok := self.packAny().(string); ok { return v, nil @@ -836,7 +841,7 @@ func (self *Node) UnsafeMap() ([]Pair, error) { if err := self.skipAllKey(); err != nil { return nil, err } - s := ptr2slice(self.p, int(self.len()), self.cap()) + s := rt.Ptr2SlicePtr(self.p, int(self.len()), self.cap()) return *(*[]Pair)(s), nil } @@ -935,7 +940,7 @@ func (self *Node) UnsafeArray() ([]Node, error) { if err := self.skipAllIndex(); err != nil { return nil, err } - s := ptr2slice(self.p, self.len(), self.cap()) + s := rt.Ptr2SlicePtr(self.p, self.len(), self.cap()) return *(*[]Node)(s), nil } @@ -953,7 +958,7 @@ func (self *Node) Interface() (interface{}, error) { case types.V_FALSE : return false, nil case types.V_ARRAY : return self.toGenericArray() case types.V_OBJECT : return self.toGenericObject() - case types.V_STRING : return addr2str(self.p, self.v), nil + case types.V_STRING : return rt.StrFrom(self.p, self.v), nil case _V_NUMBER : v, err := numberToFloat64(self) if err != nil { @@ -997,7 +1002,7 @@ func (self *Node) InterfaceUseNumber() (interface{}, error) { case types.V_FALSE : return false, nil case types.V_ARRAY : return self.toGenericArrayUseNumber() case types.V_OBJECT : return self.toGenericObjectUseNumber() - case types.V_STRING : return addr2str(self.p, self.v), nil + case types.V_STRING : return rt.StrFrom(self.p, self.v), nil case _V_NUMBER : return toNumber(self), nil case _V_ARRAY_LAZY : if err := self.loadAllIndex(); err != nil { @@ -1597,13 +1602,13 @@ func NewBool(v bool) Node { func NewNumber(v string) Node { return Node{ v: int64(len(v) & _LEN_MASK), - p: str2ptr(v), + p: rt.StrPtr(v), t: _V_NUMBER, } } func toNumber(node *Node) json.Number { - return json.Number(addr2str(node.p, node.v)) + return json.Number(rt.StrFrom(node.p, node.v)) } func numberToFloat64(node *Node) (float64, error) { @@ -1637,7 +1642,7 @@ func newBytes(v []byte) Node { func NewString(v string) Node { return Node{ t: types.V_STRING, - p: str2ptr(v), + p: rt.StrPtr(v), v: int64(len(v) & _LEN_MASK), } } @@ -1727,13 +1732,13 @@ func (self *Node) setLazyObject(p *Parser, v []Pair) { func newRawNode(str string, typ types.ValueType) Node { return Node{ t: _V_RAW | typ, - p: str2ptr(str), + p: rt.StrPtr(str), v: int64(len(str) & _LEN_MASK), } } func (self *Node) parseRaw(full bool) { - raw := addr2str(self.p, self.v) + raw := rt.StrFrom(self.p, self.v) parser := NewParser(raw) if full { parser.noLazy = true diff --git a/ast/stubs.go b/ast/stubs_go115.go similarity index 73% rename from ast/stubs.go rename to ast/stubs_go115.go index 002b920..5796524 100644 --- a/ast/stubs.go +++ b/ast/stubs_go115.go @@ -1,3 +1,5 @@ +// +build go1.15,!go1.20 + /* * Copyright 2021 ByteDance Inc. * @@ -18,16 +20,11 @@ package ast import ( `unsafe` - `reflect` `unicode/utf8` `github.com/bytedance/sonic/internal/rt` ) -var ( - byteType = rt.UnpackType(reflect.TypeOf(byte(0))) -) - //go:noescape //go:linkname memmove runtime.memmove //goland:noinspection GoUnusedParameter @@ -46,29 +43,6 @@ func mem2ptr(s []byte) unsafe.Pointer { return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr } -//go:nosplit -func ptr2slice(s unsafe.Pointer, l int, c int) unsafe.Pointer { - slice := &rt.GoSlice{ - Ptr: s, - Len: l, - Cap: c, - } - return unsafe.Pointer(slice) -} - -//go:nosplit -func str2ptr(s string) unsafe.Pointer { - return (*rt.GoString)(unsafe.Pointer(&s)).Ptr -} - -//go:nosplit -func addr2str(p unsafe.Pointer, n int64) (s string) { - (*rt.GoString)(unsafe.Pointer(&s)).Ptr = p - (*rt.GoString)(unsafe.Pointer(&s)).Len = int(n) - return -} - - var ( //go:linkname safeSet encoding/json.safeSet safeSet [utf8.RuneSelf]bool diff --git a/ast/stubs_go120.go b/ast/stubs_go120.go new file mode 100644 index 0000000..bd6fff6 --- /dev/null +++ b/ast/stubs_go120.go @@ -0,0 +1,55 @@ +// +build go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ast + +import ( + `unsafe` + `unicode/utf8` + + `github.com/bytedance/sonic/internal/rt` +) + +//go:noescape +//go:linkname memmove runtime.memmove +//goland:noinspection GoUnusedParameter +func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) + +//go:linkname unsafe_NewArray reflect.unsafe_NewArray +//goland:noinspection GoUnusedParameter +func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer + +//go:linkname growslice reflect.growslice +//goland:noinspection GoUnusedParameter +func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice + +//go:nosplit +func mem2ptr(s []byte) unsafe.Pointer { + return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr +} + +var ( + //go:linkname safeSet encoding/json.safeSet + safeSet [utf8.RuneSelf]bool + + //go:linkname hex encoding/json.hex + hex string +) + +//go:linkname unquoteBytes encoding/json.unquoteBytes +func unquoteBytes(s []byte) (t []byte, ok bool) \ No newline at end of file diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 8069b37..9ff1ad2 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -215,6 +215,7 @@ var ( type _Assembler struct { jit.BaseAssembler p _Program + name string } func newAssembler(p _Program) *_Assembler { @@ -224,7 +225,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Decoder { - return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow)) + return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index 08ee10c..8a70cff 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -1,4 +1,5 @@ -// +build go1.17,!go1.20 +//go:build go1.17 && !go1.21 +// +build go1.17,!go1.21 /* * Copyright 2021 ByteDance Inc. @@ -25,7 +26,7 @@ import ( `reflect` `strconv` `unsafe` - + `github.com/bytedance/sonic/internal/caching` `github.com/bytedance/sonic/internal/jit` `github.com/bytedance/sonic/internal/native` @@ -165,10 +166,10 @@ var ( ) var ( - _VAR_sv = _VAR_sv_p - _VAR_sv_p = jit.Ptr(_SP, _FP_base + 48) - _VAR_sv_n = jit.Ptr(_SP, _FP_base + 56) - _VAR_vk = jit.Ptr(_SP, _FP_base + 64) + _ARG_sv = _ARG_sv_p + _ARG_sv_p = jit.Ptr(_SP, _FP_base + 48) + _ARG_sv_n = jit.Ptr(_SP, _FP_base + 56) + _ARG_vk = jit.Ptr(_SP, _FP_base + 64) ) var ( @@ -210,6 +211,7 @@ var ( type _Assembler struct { jit.BaseAssembler p _Program + name string } func newAssembler(p _Program) *_Assembler { @@ -219,7 +221,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Decoder { - return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow)) + return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs)) } func (self *_Assembler) Init(p _Program) *_Assembler { @@ -348,8 +350,8 @@ func (self *_Assembler) epilogue() { self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX self.Emit("MOVQ", jit.Imm(0), _ARG_sp) // MOVQ $0, sv.p<>+48(FP) self.Emit("MOVQ", jit.Imm(0), _ARG_vp) // MOVQ $0, sv.p<>+48(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_sv_p) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_vk) // MOVQ $0, vk<>+64(FP) self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP self.Emit("RET") // RET @@ -370,9 +372,9 @@ func (self *_Assembler) prologue() { self.Emit("MOVQ", _SI, _ARG_sb) // MOVQ SI, sb<>+32(FP) self.Emit("MOVQ", _SI, _ST) // MOVQ SI, ST self.Emit("MOVQ", _R8, _ARG_fv) // MOVQ R8, fv<>+40(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_sv_p) // MOVQ $0, sv.p<>+48(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_sv_n) // MOVQ $0, sv.n<>+56(FP) - self.Emit("MOVQ", jit.Imm(0), _VAR_vk) // MOVQ $0, vk<>+64(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_sv_p) // MOVQ $0, sv.p<>+48(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_sv_n) // MOVQ $0, sv.n<>+56(FP) + self.Emit("MOVQ", jit.Imm(0), _ARG_vk) // MOVQ $0, vk<>+64(FP) self.Emit("MOVQ", jit.Imm(0), _VAR_et) // MOVQ $0, et<>+120(FP) // initialize digital buffer first self.Emit("MOVQ", jit.Imm(_MaxDigitNums), _VAR_st_Dc) // MOVQ $_MaxDigitNums, ss.Dcap @@ -498,8 +500,8 @@ func (self *_Assembler) mismatch_error() { func (self *_Assembler) field_error() { self.Link(_LB_field_error) // _field_error: - self.Emit("MOVQ", _VAR_sv_p, _AX) // MOVQ sv.p, AX - self.Emit("MOVQ", _VAR_sv_n, _BX) // MOVQ sv.n, BX + self.Emit("MOVQ", _ARG_sv_p, _AX) // MOVQ sv.p, AX + self.Emit("MOVQ", _ARG_sv_n, _BX) // MOVQ sv.n, BX self.call_go(_F_error_field) // CALL_GO error_field self.Sjmp("JMP" , _LB_error) // JMP _error } @@ -745,11 +747,11 @@ func (self *_Assembler) copy_string() { self.Emit("MOVQ", _DI, _VAR_bs_p) self.Emit("MOVQ", _SI, _VAR_bs_n) self.Emit("MOVQ", _R9, _VAR_bs_LR) - self.malloc_AX(_SI, _VAR_sv_p) + self.malloc_AX(_SI, _ARG_sv_p) self.Emit("MOVQ", _VAR_bs_p, _BX) self.Emit("MOVQ", _VAR_bs_n, _CX) self.call_go(_F_memmove) - self.Emit("MOVQ", _VAR_sv_p, _DI) + self.Emit("MOVQ", _ARG_sv_p, _DI) self.Emit("MOVQ", _VAR_bs_n, _SI) self.Emit("MOVQ", _VAR_bs_LR, _R9) self.Rjmp("JMP", _R9) @@ -762,7 +764,7 @@ func (self *_Assembler) escape_string() { self.Emit("MOVQ" , _SI, _VAR_bs_n) self.Emit("MOVQ" , _R9, _VAR_bs_LR) self.malloc_AX(_SI, _DX) // MALLOC SI, DX - self.Emit("MOVQ" , _DX, _VAR_sv_p) + self.Emit("MOVQ" , _DX, _ARG_sv_p) self.Emit("MOVQ" , _VAR_bs_p, _DI) self.Emit("MOVQ" , _VAR_bs_n, _SI) self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX @@ -776,7 +778,7 @@ func (self *_Assembler) escape_string() { self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error self.Emit("MOVQ" , _AX, _SI) - self.Emit("MOVQ" , _VAR_sv_p, _DI) + self.Emit("MOVQ" , _ARG_sv_p, _DI) self.Emit("MOVQ" , _VAR_bs_LR, _R9) self.Rjmp("JMP", _R9) } @@ -787,7 +789,7 @@ func (self *_Assembler) escape_string_twice() { self.Emit("MOVQ" , _SI, _VAR_bs_n) self.Emit("MOVQ" , _R9, _VAR_bs_LR) self.malloc_AX(_SI, _DX) // MALLOC SI, DX - self.Emit("MOVQ" , _DX, _VAR_sv_p) + self.Emit("MOVQ" , _DX, _ARG_sv_p) self.Emit("MOVQ" , _VAR_bs_p, _DI) self.Emit("MOVQ" , _VAR_bs_n, _SI) self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX @@ -803,7 +805,7 @@ func (self *_Assembler) escape_string_twice() { self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error self.Emit("MOVQ" , _AX, _SI) - self.Emit("MOVQ" , _VAR_sv_p, _DI) + self.Emit("MOVQ" , _ARG_sv_p, _DI) self.Emit("MOVQ" , _VAR_bs_LR, _R9) self.Rjmp("JMP", _R9) } @@ -1029,15 +1031,15 @@ func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) { /* allocate the key, and call the unmarshaler */ self.valloc(vk, _BX) // VALLOC ${vk}, BX // must spill vk pointer since next call_go may invoke GC - self.Emit("MOVQ" , _BX, _VAR_vk) + self.Emit("MOVQ" , _BX, _ARG_vk) self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX - self.Emit("MOVQ" , _VAR_sv_p, _CX) // MOVQ sv.p, CX - self.Emit("MOVQ" , _VAR_sv_n, _DI) // MOVQ sv.n, DI + self.Emit("MOVQ" , _ARG_sv_p, _CX) // MOVQ sv.p, CX + self.Emit("MOVQ" , _ARG_sv_n, _DI) // MOVQ sv.n, DI self.call_go(_F_decodeTextUnmarshaler) // CALL_GO decodeTextUnmarshaler self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ" , _LB_error) // JNZ _error - self.Emit("MOVQ" , _VAR_vk, _AX) // MOVQ VAR.vk, AX - self.Emit("MOVQ", jit.Imm(0), _VAR_vk) + self.Emit("MOVQ" , _ARG_vk, _AX) // MOVQ VAR.vk, AX + self.Emit("MOVQ", jit.Imm(0), _ARG_vk) /* select the correct assignment function */ if !pv { @@ -1061,14 +1063,14 @@ func (self *_Assembler) unmarshal_json(t reflect.Type, deref bool) { self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v self.slice_from_r(_AX, 0) // SLICE_R AX, $0 - self.Emit("MOVQ" , _DI, _VAR_sv_p) // MOVQ DI, sv.p - self.Emit("MOVQ" , _SI, _VAR_sv_n) // MOVQ SI, sv.n + self.Emit("MOVQ" , _DI, _ARG_sv_p) // MOVQ DI, sv.p + self.Emit("MOVQ" , _SI, _ARG_sv_n) // MOVQ SI, sv.n self.unmarshal_func(t, _F_decodeJsonUnmarshaler, deref) // UNMARSHAL json, ${t}, ${deref} } func (self *_Assembler) unmarshal_text(t reflect.Type, deref bool) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n self.unmarshal_func(t, _F_decodeTextUnmarshaler, deref) // UNMARSHAL text, ${t}, ${deref} } @@ -1094,8 +1096,8 @@ func (self *_Assembler) unmarshal_func(t reflect.Type, fn obj.Addr, deref bool) self.Emit("MOVQ", jit.Type(pt), _AX) // MOVQ ${pt}, AX /* set the source string and call the unmarshaler */ - self.Emit("MOVQ" , _VAR_sv_p, _CX) // MOVQ sv.p, CX - self.Emit("MOVQ" , _VAR_sv_n, _DI) // MOVQ sv.n, DI + self.Emit("MOVQ" , _ARG_sv_p, _CX) // MOVQ sv.p, CX + self.Emit("MOVQ" , _ARG_sv_n, _DI) // MOVQ sv.n, DI self.call_go(fn) // CALL_GO ${fn} self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET self.Sjmp("JNZ" , _LB_error) // JNZ _error @@ -1557,26 +1559,26 @@ func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n if vt := p.vt(); !mapfast(vt) { self.valloc(vt.Key(), _DI) - self.Emit("MOVOU", _VAR_sv, _X0) + self.Emit("MOVOU", _ARG_sv, _X0) self.Emit("MOVOU", _X0, jit.Ptr(_DI, 0)) self.mapassign_std(vt, jit.Ptr(_DI, 0)) // MAPASSIGN string, DI, SI } else { - self.mapassign_str_fast(vt, _VAR_sv_p, _VAR_sv_n) // MAPASSIGN string, DI, SI + self.mapassign_str_fast(vt, _ARG_sv_p, _ARG_sv_n) // MAPASSIGN string, DI, SI } } func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), false) // MAPASSIGN utext, ${p.vt()}, false } func (self *_Assembler) _asm_OP_map_key_utext_p(p *_Instr) { self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, true) // UNQUOTE once, sv.p, sv.n + self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, true) // UNQUOTE once, sv.p, sv.n self.mapassign_utext(p.vt(), true) // MAPASSIGN utext, ${p.vt()}, true } @@ -1650,8 +1652,8 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr self.parse_string() // PARSE STRING - self.unquote_once(_VAR_sv_p, _VAR_sv_n, true, false) // UNQUOTE once, sv.p, sv.n - self.Emit("LEAQ" , _VAR_sv, _AX) // LEAQ sv, AX + self.unquote_once(_ARG_sv_p, _ARG_sv_n, true, false) // UNQUOTE once, sv.p, sv.n + self.Emit("LEAQ" , _ARG_sv, _AX) // LEAQ sv, AX self.Emit("XORL" , _BX, _BX) // XORL BX, BX self.call_go(_F_strhash) // CALL_GO strhash self.Emit("MOVQ" , _AX, _R9) // MOVQ AX, R9 @@ -1672,7 +1674,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("CMPQ" , _R8, _R9) // CMPQ R8, R9 self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n} self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name + 8), _DX) // MOVQ FieldEntry.Name+8(DI), DX - self.Emit("CMPQ" , _DX, _VAR_sv_n) // CMPQ DX, sv.n + self.Emit("CMPQ" , _DX, _ARG_sv_n) // CMPQ DX, sv.n self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n} self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_ID), _R8) // MOVQ FieldEntry.ID(DI), R8 self.Emit("MOVQ" , _AX, _VAR_ss_AX) // MOVQ AX, ss.AX @@ -1680,7 +1682,7 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Emit("MOVQ" , _SI, _VAR_ss_SI) // MOVQ SI, ss.SI self.Emit("MOVQ" , _R8, _VAR_ss_R8) // MOVQ R8, ss.R8 self.Emit("MOVQ" , _R9, _VAR_ss_R9) // MOVQ R9, ss.R9 - self.Emit("MOVQ" , _VAR_sv_p, _AX) // MOVQ _VAR_sv_p, AX + self.Emit("MOVQ" , _ARG_sv_p, _AX) // MOVQ _VAR_sv_p, AX self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name), _CX) // MOVQ FieldEntry.Name(DI), CX self.Emit("MOVQ" , _CX, _BX) // MOVQ CX, 8(SP) self.Emit("MOVQ" , _DX, _CX) // MOVQ DX, 16(SP) @@ -1697,8 +1699,8 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} self.Link("_try_lowercase_{n}") // _try_lowercase_{n}: self.Emit("MOVQ" , jit.Imm(referenceFields(p.vf())), _AX) // MOVQ ${p.vf()}, AX - self.Emit("MOVQ", _VAR_sv_p, _BX) // MOVQ sv, BX - self.Emit("MOVQ", _VAR_sv_n, _CX) // MOVQ sv, CX + self.Emit("MOVQ", _ARG_sv_p, _BX) // MOVQ sv, BX + self.Emit("MOVQ", _ARG_sv_n, _CX) // MOVQ sv, CX self.call_go(_F_FieldMap_GetCaseInsensitive) // CALL_GO FieldMap::GetCaseInsensitive self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, _VAR_sr self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX diff --git a/decoder/compiler.go b/decoder/compiler.go index e2c04c8..b4fc2fe 100644 --- a/decoder/compiler.go +++ b/decoder/compiler.go @@ -104,6 +104,7 @@ const ( const ( _INT_SIZE = 32 << (^uint(0) >> 63) _PTR_SIZE = 32 << (^uintptr(0) >> 63) + _PTR_BYTE = unsafe.Sizeof(uintptr(0)) ) const ( diff --git a/decoder/decoder.go b/decoder/decoder.go index fde6611..190e318 100644 --- a/decoder/decoder.go +++ b/decoder/decoder.go @@ -168,7 +168,9 @@ func pretouchType(_vt reflect.Type, opts option.CompileOptions) (map[reflect.Typ if pp, err := compiler.compile(_vt); err != nil { return nil, err } else { - return newAssembler(pp).Load(), nil + as := newAssembler(pp) + as.name = _vt.String() + return as.Load(), nil } } diff --git a/decoder/generic_amd64_go116.go b/decoder/generic_amd64_go116.go index d6da6a3..b597043 100644 --- a/decoder/generic_amd64_go116.go +++ b/decoder/generic_amd64_go116.go @@ -81,7 +81,7 @@ type _ValueDecoder struct { func (self *_ValueDecoder) build() uintptr { self.Init(self.compile) - return *(*uintptr)(self.LoadWithFaker("decode_value", _VD_size, _VD_args, _Decoder_Generic_Shadow)) + return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic)) } /** Function Calling Helpers **/ diff --git a/decoder/generic_amd64_go117.go b/decoder/generic_amd64_go117.go index f08ede2..df1cd9f 100644 --- a/decoder/generic_amd64_go117.go +++ b/decoder/generic_amd64_go117.go @@ -1,4 +1,5 @@ -// +build go1.17,!go1.20 +//go:build go1.17 && !go1.21 +// +build go1.17,!go1.21 /* * Copyright 2021 ByteDance Inc. @@ -84,7 +85,7 @@ var ( func (self *_ValueDecoder) build() uintptr { self.Init(self.compile) - return *(*uintptr)(self.LoadWithFaker("decode_value", _VD_size, _VD_args, _Decoder_Generic_Shadow)) + return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs_generic, localPtrs_generic)) } /** Function Calling Helpers **/ diff --git a/decoder/generic_amd64_go117_test.s b/decoder/generic_amd64_go117_test.s index 7b9fed2..6c2686d 100644 --- a/decoder/generic_amd64_go117_test.s +++ b/decoder/generic_amd64_go117_test.s @@ -1,4 +1,4 @@ -// +build go1.17,!go1.20 +// +build go1.17,!go1.21 // // Copyright 2021 ByteDance Inc. diff --git a/decoder/norace_test.go b/decoder/norace_test.go index f72a481..9a06363 100644 --- a/decoder/norace_test.go +++ b/decoder/norace_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race /* @@ -19,11 +20,12 @@ package decoder import ( - `testing` - `unsafe` - `runtime` + `runtime` + `testing` + `time` + `unsafe` - `github.com/bytedance/sonic/internal/rt` + `github.com/bytedance/sonic/internal/rt` ) var referred = false @@ -34,7 +36,7 @@ func TestStringReferring(t *testing.T) { println("malloc *byte ", sp) runtime.SetFinalizer(sp, func(sp *byte){ referred = false - println("*byte ", sp, " got free") + println("*byte ", sp, " got free 1") }) runtime.GC() println("first GC") @@ -50,6 +52,7 @@ func TestStringReferring(t *testing.T) { } runtime.GC() println("second GC") + time.Sleep(time.Millisecond) if referred { t.Fatal("*byte is being referred") } @@ -73,6 +76,7 @@ func TestStringReferring(t *testing.T) { } runtime.GC() println("second GC") + time.Sleep(time.Millisecond) if referred { t.Fatal("*byte is being referred") } diff --git a/decoder/pools.go b/decoder/pools.go index 781efac..ab1e5f2 100644 --- a/decoder/pools.go +++ b/decoder/pools.go @@ -17,7 +17,6 @@ package decoder import ( - `errors` `sync` `unsafe` @@ -82,37 +81,15 @@ var _KeepAlive struct { frame_generic [_VD_offs]byte } -var errCallShadow = errors.New("DON'T CALL THIS!") +var ( + argPtrs = []bool{true, false, false, true, true, false, true, false, true} + localPtrs = []bool{} +) -// Faker func of _Decoder, used to export its stackmap as _Decoder's -func _Decoder_Shadow(s string, i int, vp unsafe.Pointer, sb *_Stack, fv uint64, sv string, vk unsafe.Pointer) (ret int, err error) { - // align to assembler_amd64.go: _FP_offs - var frame [_FP_offs]byte - - // keep all args and stacks alive - _KeepAlive.s = s - _KeepAlive.i = i - _KeepAlive.vp = vp - _KeepAlive.sb = sb - _KeepAlive.fv = fv - _KeepAlive.ret = ret - _KeepAlive.err = err - _KeepAlive.sv = sv - _KeepAlive.vk = vk - _KeepAlive.frame_decoder = frame - - return 0, errCallShadow -} - -// Faker func of _Decoder_Generic, used to export its stackmap -func _Decoder_Generic_Shadow(sb *_Stack) { - // align to generic_amd64.go: _VD_offs - var frame [_VD_offs]byte - - // must keep sb noticeable to GC - _KeepAlive.sb = sb - _KeepAlive.frame_generic = frame -} +var ( + argPtrs_generic = []bool{true} + localPtrs_generic = []bool{} +) func newStack() *_Stack { if ret := stackPool.Get(); ret == nil { diff --git a/decoder/stubs.go b/decoder/stubs_go115.go similarity index 99% rename from decoder/stubs.go rename to decoder/stubs_go115.go index e91486b..1a0917c 100644 --- a/decoder/stubs.go +++ b/decoder/stubs_go115.go @@ -1,3 +1,4 @@ +// +build go1.15,!go1.20 /* * Copyright 2021 ByteDance Inc. diff --git a/decoder/stubs_go120.go b/decoder/stubs_go120.go new file mode 100644 index 0000000..cde6a19 --- /dev/null +++ b/decoder/stubs_go120.go @@ -0,0 +1,111 @@ +// +build go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package decoder + +import ( + `unsafe` + `reflect` + + _ `github.com/chenzhuoyu/base64x` + + `github.com/bytedance/sonic/internal/rt` +) + +//go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode +var _subr__b64decode uintptr + +// runtime.maxElementSize +const _max_map_element_size uintptr = 128 + +func mapfast(vt reflect.Type) bool { + return vt.Elem().Size() <= _max_map_element_size +} + +//go:nosplit +//go:linkname throw runtime.throw +//goland:noinspection GoUnusedParameter +func throw(s string) + +//go:linkname convT64 runtime.convT64 +//goland:noinspection GoUnusedParameter +func convT64(v uint64) unsafe.Pointer + +//go:linkname convTslice runtime.convTslice +//goland:noinspection GoUnusedParameter +func convTslice(v []byte) unsafe.Pointer + +//go:linkname convTstring runtime.convTstring +//goland:noinspection GoUnusedParameter +func convTstring(v string) unsafe.Pointer + +//go:noescape +//go:linkname memequal runtime.memequal +//goland:noinspection GoUnusedParameter +func memequal(a unsafe.Pointer, b unsafe.Pointer, size uintptr) bool + +//go:noescape +//go:linkname memmove runtime.memmove +//goland:noinspection GoUnusedParameter +func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) + +//go:linkname mallocgc runtime.mallocgc +//goland:noinspection GoUnusedParameter +func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer + +//go:linkname makeslice runtime.makeslice +//goland:noinspection GoUnusedParameter +func makeslice(et *rt.GoType, len int, cap int) unsafe.Pointer + +//go:noescape +//go:linkname growslice reflect.growslice +//goland:noinspection GoUnusedParameter +func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice + +//go:linkname makemap_small runtime.makemap_small +func makemap_small() unsafe.Pointer + +//go:linkname mapassign runtime.mapassign +//goland:noinspection GoUnusedParameter +func mapassign(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer + +//go:linkname mapassign_fast32 runtime.mapassign_fast32 +//goland:noinspection GoUnusedParameter +func mapassign_fast32(t *rt.GoType, h unsafe.Pointer, k uint32) unsafe.Pointer + +//go:linkname mapassign_fast64 runtime.mapassign_fast64 +//goland:noinspection GoUnusedParameter +func mapassign_fast64(t *rt.GoType, h unsafe.Pointer, k uint64) unsafe.Pointer + +//go:linkname mapassign_fast64ptr runtime.mapassign_fast64ptr +//goland:noinspection GoUnusedParameter +func mapassign_fast64ptr(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer + +//go:linkname mapassign_faststr runtime.mapassign_faststr +//goland:noinspection GoUnusedParameter +func mapassign_faststr(t *rt.GoType, h unsafe.Pointer, s string) unsafe.Pointer + +//go:nosplit +//go:linkname memclrHasPointers runtime.memclrHasPointers +//goland:noinspection GoUnusedParameter +func memclrHasPointers(ptr unsafe.Pointer, n uintptr) + +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) \ No newline at end of file diff --git a/decoder/utils.go b/decoder/utils.go index 69587cb..23ee5d5 100644 --- a/decoder/utils.go +++ b/decoder/utils.go @@ -19,7 +19,7 @@ package decoder import ( `unsafe` - `github.com/bytedance/sonic/internal/loader` + `github.com/bytedance/sonic/loader` ) //go:nosplit diff --git a/encoder/assembler_amd64_go116.go b/encoder/assembler_amd64_go116.go index fbfa470..9b59784 100644 --- a/encoder/assembler_amd64_go116.go +++ b/encoder/assembler_amd64_go116.go @@ -177,6 +177,7 @@ type _Assembler struct { jit.BaseAssembler p _Program x int + name string } func newAssembler(p _Program) *_Assembler { @@ -184,9 +185,8 @@ func newAssembler(p _Program) *_Assembler { } /** Assembler Interface **/ - func (self *_Assembler) Load() _Encoder { - return ptoenc(self.BaseAssembler.LoadWithFaker("json_encoder", _FP_size, _FP_args, _Encoder_Shadow)) + return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs)) } func (self *_Assembler) Init(p _Program) *_Assembler { @@ -483,10 +483,6 @@ func (self *_Assembler) load_buffer() { /** Function Interface Helpers **/ -var ( - _F_assertI2I = jit.Func(assertI2I) -) - func (self *_Assembler) call(pc obj.Addr) { self.Emit("MOVQ", pc, _AX) // MOVQ $pc, AX self.Rjmp("CALL", _AX) // CALL AX diff --git a/encoder/assembler_amd64_go117.go b/encoder/assembler_amd64_go117.go index 217a5d2..8cd83e8 100644 --- a/encoder/assembler_amd64_go117.go +++ b/encoder/assembler_amd64_go117.go @@ -1,4 +1,5 @@ -// +build go1.17,!go1.20 +//go:build go1.17 && !go1.21 +// +build go1.17,!go1.21 /* * Copyright 2021 ByteDance Inc. @@ -182,6 +183,7 @@ type _Assembler struct { jit.BaseAssembler p _Program x int + name string } func newAssembler(p _Program) *_Assembler { @@ -191,7 +193,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Encoder { - return ptoenc(self.BaseAssembler.LoadWithFaker("json_encoder", _FP_size, _FP_args, _Encoder_Shadow)) + return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/encoder/compiler.go b/encoder/compiler.go index ef2dc3c..a949c90 100644 --- a/encoder/compiler.go +++ b/encoder/compiler.go @@ -86,6 +86,7 @@ const ( const ( _INT_SIZE = 32 << (^uint(0) >> 63) _PTR_SIZE = 32 << (^uintptr(0) >> 63) + _PTR_BYTE = unsafe.Sizeof(uintptr(0)) ) const ( diff --git a/encoder/debug_go117.go b/encoder/debug_go117.go index 19ec4b4..e1016de 100644 --- a/encoder/debug_go117.go +++ b/encoder/debug_go117.go @@ -1,4 +1,4 @@ -// +build go1.17,!go1.20 +// +build go1.17,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/encoder/pools.go b/encoder/pools.go index 3f1fc22..600605d 100644 --- a/encoder/pools.go +++ b/encoder/pools.go @@ -135,7 +135,9 @@ func makeEncoder(vt *rt.GoType, ex ...interface{}) (interface{}, error) { if pp, err := newCompiler().compile(vt.Pack(), ex[0].(bool)); err != nil { return nil, err } else { - return newAssembler(pp).Load(), nil + as := newAssembler(pp) + as.name = vt.String() + return as.Load(), nil } } @@ -156,7 +158,9 @@ func pretouchType(_vt reflect.Type, opts option.CompileOptions, v uint8) (map[re if pp, err := compiler.compile(_vt, ex[0].(bool)); err != nil { return nil, err } else { - return newAssembler(pp).Load(), nil + as := newAssembler(pp) + as.name = vt.String() + return as.Load(), nil } } diff --git a/encoder/primitives.go b/encoder/primitives.go index b0aebfb..46faece 100644 --- a/encoder/primitives.go +++ b/encoder/primitives.go @@ -21,6 +21,7 @@ import ( `encoding/json` `unsafe` + `github.com/bytedance/sonic/internal/jit` `github.com/bytedance/sonic/internal/native` `github.com/bytedance/sonic/internal/rt` ) @@ -110,4 +111,25 @@ func encodeTextMarshaler(buf *[]byte, val encoding.TextMarshaler, opt Options) e } return encodeString(buf, rt.Mem2Str(ret) ) } +} + +var ( + argPtrs = []bool { true, true, true, false } + localPtrs = []bool{} +) + +var ( + _F_assertI2I = jit.Func(assertI2I) +) + +func asText(v unsafe.Pointer) (string, error) { + text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) + r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() + return rt.Mem2Str(r), e +} + +func asJson(v unsafe.Pointer) (string, error) { + text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v)) + r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON() + return rt.Mem2Str(r), e } \ No newline at end of file diff --git a/encoder/stubs_go116.go b/encoder/stubs_go116.go index 082bb79..40d06f4 100644 --- a/encoder/stubs_go116.go +++ b/encoder/stubs_go116.go @@ -20,8 +20,6 @@ package encoder import ( `unsafe` - `encoding` - `encoding/json` _ `github.com/chenzhuoyu/base64x` @@ -61,18 +59,6 @@ func isValidNumber(s string) bool //goland:noinspection GoUnusedParameter func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) -func asText(v unsafe.Pointer) (string, error) { - text := assertI2I(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) - r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() - return rt.Mem2Str(r), e -} - -func asJson(v unsafe.Pointer) (string, error) { - text := assertI2I(_T_json_Marshaler, *(*rt.GoIface)(v)) - r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON() - return rt.Mem2Str(r), e -} - var _runtime_writeBarrier uintptr = rt.GcwbAddr() //go:linkname gcWriteBarrierAX runtime.gcWriteBarrier diff --git a/encoder/stubs_go117.go b/encoder/stubs_go117.go index 89ce553..6c8c6ec 100644 --- a/encoder/stubs_go117.go +++ b/encoder/stubs_go117.go @@ -19,13 +19,10 @@ package encoder import ( - `encoding` - `encoding/json` `unsafe` _ `github.com/chenzhuoyu/base64x` - `github.com/bytedance/sonic/internal/jit` `github.com/bytedance/sonic/internal/rt` ) @@ -41,9 +38,9 @@ func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) //goland:noinspection GoUnusedParameter func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice -//go:linkname assertI2I2 runtime.assertI2I2 +//go:linkname assertI2I runtime.assertI2I2 //goland:noinspection GoUnusedParameter -func assertI2I2(inter *rt.GoType, i rt.GoIface) rt.GoIface +func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface //go:linkname mapiternext runtime.mapiternext //goland:noinspection GoUnusedParameter @@ -62,22 +59,6 @@ func isValidNumber(s string) bool //goland:noinspection GoUnusedParameter func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) -var ( - _F_assertI2I = jit.Func(assertI2I2) -) - -func asText(v unsafe.Pointer) (string, error) { - text := assertI2I2(_T_encoding_TextMarshaler, *(*rt.GoIface)(v)) - r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText() - return rt.Mem2Str(r), e -} - -func asJson(v unsafe.Pointer) (string, error) { - text := assertI2I2(_T_json_Marshaler, *(*rt.GoIface)(v)) - r, e := (*(*json.Marshaler)(unsafe.Pointer(&text))).MarshalJSON() - return rt.Mem2Str(r), e -} - //go:linkname _runtime_writeBarrier runtime.writeBarrier var _runtime_writeBarrier uintptr diff --git a/encoder/stubs_go120.go b/encoder/stubs_go120.go new file mode 100644 index 0000000..f1a7d10 --- /dev/null +++ b/encoder/stubs_go120.go @@ -0,0 +1,66 @@ +// +build go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package encoder + +import ( + `unsafe` + + _ `github.com/chenzhuoyu/base64x` + + `github.com/bytedance/sonic/internal/rt` +) + +//go:linkname _subr__b64encode github.com/chenzhuoyu/base64x._subr__b64encode +var _subr__b64encode uintptr + +//go:noescape +//go:linkname memmove runtime.memmove +//goland:noinspection GoUnusedParameter +func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) + +//go:linkname growslice reflect.growslice +//goland:noinspection GoUnusedParameter +func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice + +//go:linkname assertI2I runtime.assertI2I2 +//goland:noinspection GoUnusedParameter +func assertI2I(inter *rt.GoType, i rt.GoIface) rt.GoIface + +//go:linkname mapiternext runtime.mapiternext +//goland:noinspection GoUnusedParameter +func mapiternext(it *rt.GoMapIterator) + +//go:linkname mapiterinit runtime.mapiterinit +//goland:noinspection GoUnusedParameter +func mapiterinit(t *rt.GoMapType, m *rt.GoMap, it *rt.GoMapIterator) + +//go:linkname isValidNumber encoding/json.isValidNumber +//goland:noinspection GoUnusedParameter +func isValidNumber(s string) bool + +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) + +//go:linkname _runtime_writeBarrier runtime.writeBarrier +var _runtime_writeBarrier uintptr + +//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier +func gcWriteBarrierAX() \ No newline at end of file diff --git a/encoder/utils.go b/encoder/utils.go index 4186637..510596f 100644 --- a/encoder/utils.go +++ b/encoder/utils.go @@ -20,7 +20,7 @@ import ( `encoding/json` `unsafe` - `github.com/bytedance/sonic/internal/loader` + `github.com/bytedance/sonic/loader` ) //go:nosplit diff --git a/external_jsonlib_test/benchmark_test/encoder_test.go b/external_jsonlib_test/benchmark_test/encoder_test.go index f71dc03..59fce93 100644 --- a/external_jsonlib_test/benchmark_test/encoder_test.go +++ b/external_jsonlib_test/benchmark_test/encoder_test.go @@ -18,12 +18,37 @@ package benchmark_test import ( `encoding/json` + `os` + `runtime` + `runtime/debug` `testing` + `time` gojson `github.com/goccy/go-json` jsoniter `github.com/json-iterator/go` ) +var ( + debugSyncGC = os.Getenv("SONIC_SYNC_GC") != "" + debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" +) + +func TestMain(m *testing.M) { + go func () { + if !debugAsyncGC { + return + } + println("Begin GC looping...") + for { + runtime.GC() + debug.FreeOSMemory() + } + println("stop GC looping!") + }() + time.Sleep(time.Millisecond) + m.Run() +} + var _GenericValue interface{} var _BindingValue TwitterStruct diff --git a/external_jsonlib_test/unit_test/api_test.go b/external_jsonlib_test/unit_test/api_test.go index 7ddb2c2..7e4c808 100644 --- a/external_jsonlib_test/unit_test/api_test.go +++ b/external_jsonlib_test/unit_test/api_test.go @@ -19,13 +19,38 @@ package unit_test import ( `bytes` `encoding/json` + `os` + `runtime` + `runtime/debug` `testing` + `time` `github.com/bytedance/sonic` jsoniter `github.com/json-iterator/go` `github.com/stretchr/testify/require` ) +var ( + debugSyncGC = os.Getenv("SONIC_SYNC_GC") != "" + debugAsyncGC = os.Getenv("SONIC_NO_ASYNC_GC") == "" +) + +func TestMain(m *testing.M) { + go func () { + if !debugAsyncGC { + return + } + println("Begin GC looping...") + for { + runtime.GC() + debug.FreeOSMemory() + } + println("stop GC looping!") + }() + time.Sleep(time.Millisecond) + m.Run() +} + var jt = jsoniter.Config{ ValidateJsonRawMessage: true, }.Froze() diff --git a/internal/jit/assembler_amd64.go b/internal/jit/assembler_amd64.go index fff1c77..00e6009 100644 --- a/internal/jit/assembler_amd64.go +++ b/internal/jit/assembler_amd64.go @@ -22,7 +22,7 @@ import ( `strings` `sync` - `github.com/bytedance/sonic/internal/loader` + `github.com/bytedance/sonic/loader` `github.com/bytedance/sonic/internal/rt` `github.com/twitchyliquid64/golang-asm/obj` `github.com/twitchyliquid64/golang-asm/obj/x86` @@ -202,14 +202,17 @@ func (self *BaseAssembler) Init(f func()) { self.o = sync.Once{} } -func (self *BaseAssembler) Load(fn string, fp int, args int) loader.Function { - self.build() - return loader.Loader(self.c).Load(fn, fp, args) +var jitLoader = loader.Loader{ + Name: "sonic.jit.", + File: "github.com/bytedance/sonic/jit.go", + Options: loader.Options{ + NoPreempt: true, + }, } -func (self *BaseAssembler) LoadWithFaker(fn string, fp int, args int, faker interface{}) loader.Function { +func (self *BaseAssembler) Load(name string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) loader.Function { self.build() - return loader.Loader(self.c).LoadWithFaker(fn, fp, args, faker) + return jitLoader.LoadOne(self.c, name, frameSize, argSize, argStackmap, localStackmap) } /** Assembler Stages **/ diff --git a/internal/loader/funcdata.go b/internal/loader/funcdata.go index 3e916f6..59a3cb3 100644 --- a/internal/loader/funcdata.go +++ b/internal/loader/funcdata.go @@ -17,9 +17,11 @@ package loader import ( + `reflect` `sync` `unsafe` - `reflect` + + `github.com/bytedance/sonic/internal/rt` ) //go:linkname lastmoduledatap runtime.lastmoduledatap @@ -92,3 +94,31 @@ func stackMap(f interface{}) (args uintptr, locals uintptr) { fi := findfunc(fv.Pointer()) return uintptr(funcdata(fi, uint8(_FUNCDATA_ArgsPointerMaps))), uintptr(funcdata(fi, uint8(_FUNCDATA_LocalsPointerMaps))) } + +var moduleCache = struct{ + m map[*_ModuleData][]byte + l sync.Mutex +}{ + m : make(map[*_ModuleData][]byte), +} + +func cacheStackmap(argPtrs []bool, localPtrs []bool, mod *_ModuleData) (argptrs uintptr, localptrs uintptr) { + as := rt.StackMapBuilder{} + for _, b := range argPtrs { + as.AddField(b) + } + ab, _ := as.Build().MarshalBinary() + ls := rt.StackMapBuilder{} + for _, b := range localPtrs { + ls.AddField(b) + } + lb, _ := ls.Build().MarshalBinary() + cache := make([]byte, len(ab) + len(lb)) + copy(cache, ab) + copy(cache[len(ab):], lb) + moduleCache.l.Lock() + moduleCache.m[mod] = cache + moduleCache.l.Unlock() + return uintptr(rt.IndexByte(cache, 0)), uintptr(rt.IndexByte(cache, len(ab))) + +} \ No newline at end of file diff --git a/internal/loader/funcdata_go115.go b/internal/loader/funcdata_go115.go index 3673da8..b0d2d6c 100644 --- a/internal/loader/funcdata_go115.go +++ b/internal/loader/funcdata_go115.go @@ -98,7 +98,8 @@ var findFuncTab = &_FindFuncBucket { idx: 1, } -func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) { +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) { + mod := new(_ModuleData) minpc := pc maxpc := pc + size @@ -111,6 +112,9 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in 4 << (^uintptr(0) >> 63), // ptrSize : 4 << (^uintptr(0) >> 63) } + // cache arg and local stackmap + argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod) + /* add the function name */ noff := len(pclnt) pclnt = append(append(pclnt, name...), 0) @@ -127,8 +131,8 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in args : int32(args), pcsp : int32(pcsp), nfuncdata : 2, - argptrs : argptrs, - localptrs : localptrs, + argptrs : uintptr(argptrs), + localptrs : uintptr(localptrs), } /* align the func to 8 bytes */ @@ -148,7 +152,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in } /* module data */ - mod := &_ModuleData { + *mod = _ModuleData { pclntable : pclnt, ftab : tab, findfunctab : findFuncTab, diff --git a/internal/loader/funcdata_go116.go b/internal/loader/funcdata_go116.go index 20d7df1..f01747f 100644 --- a/internal/loader/funcdata_go116.go +++ b/internal/loader/funcdata_go116.go @@ -1,3 +1,4 @@ +//go:build go1.16 && !go1.18 // +build go1.16,!go1.18 /* @@ -126,10 +127,15 @@ func makePCtab(fp int) []byte { return append([]byte{0}, encodeVariant((fp + 1) << 1)...) } -func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) { +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) { + mod := new(_ModuleData) + minpc := pc maxpc := pc + size + // cache arg and local stackmap + argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod) + /* function entry */ lnt := []_Func {{ entry : pc, @@ -137,8 +143,8 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in args : int32(args), pcsp : 1, nfuncdata : 2, - argptrs : argptrs, - localptrs : localptrs, + argptrs : uintptr(argptrs), + localptrs : uintptr(localptrs), }} /* function table */ @@ -149,7 +155,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in } /* module data */ - mod := &_ModuleData { + *mod = _ModuleData { pcHeader : modHeader, funcnametab : append(append([]byte{0}, name...), 0), pctab : append(makePCtab(fp), encodeVariant(int(size))...), diff --git a/internal/loader/funcdata_go118.go b/internal/loader/funcdata_go118.go index f49c52d..f1d585d 100644 --- a/internal/loader/funcdata_go118.go +++ b/internal/loader/funcdata_go118.go @@ -128,7 +128,9 @@ func makePCtab(fp int) []byte { return append([]byte{0}, encodeVariant((fp + 1) << 1)...) } -func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) { +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) { + mod := new(_ModuleData) + minpc := pc maxpc := pc + size @@ -142,6 +144,9 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in textStart: minpc, } + // cache arg and local stackmap + argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod) + base := argptrs if argptrs > localptrs { base = localptrs @@ -173,7 +178,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in pclntab = append(pclntab, rt.BytesFrom(plnt, nlnt, nlnt)...) /* module data */ - mod := &_ModuleData { + *mod = _ModuleData { pcHeader : modHeader, funcnametab : append(append([]byte{0}, name...), 0), pctab : append(makePCtab(fp), encodeVariant(int(size))...), diff --git a/internal/loader/funcdata_go120.go b/internal/loader/funcdata_go120.go new file mode 100644 index 0000000..c12f8a7 --- /dev/null +++ b/internal/loader/funcdata_go120.go @@ -0,0 +1,201 @@ +// +build go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `unsafe` + + `github.com/bytedance/sonic/internal/rt` +) + +// A FuncFlag holds bits about a function. +// This list must match the list in cmd/internal/objabi/funcid.go. +type funcFlag uint8 + +type _Func struct { + entryOff uint32 // start pc + nameoff int32 // function name + args int32 // in/out args size + deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 // runtime.cutab offset of this function's CU + funcID uint8 // set for certain special runtime functions + flag funcFlag + _ [1]byte // pad + nfuncdata uint8 // must be last + argptrs uint32 + localptrs uint32 +} + +type _FuncTab struct { + entry uint32 + funcoff uint32 +} + +type _PCHeader struct { + magic uint32 // 0xFFFFFFF0 + pad1, pad2 uint8 // 0,0 + minLC uint8 // min instruction size + ptrSize uint8 // size of a ptr in bytes + nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab + textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text + funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab variable from pcHeader + pclnOffset uintptr // offset to the pclntab variable from pcHeader +} + +type _BitVector struct { + n int32 // # of bits + bytedata *uint8 +} + +type _PtabEntry struct { + name int32 + typ int32 +} + +type _TextSection struct { + vaddr uintptr // prelinked section vaddr + length uintptr // section length + baseaddr uintptr // relocated section address +} + +type _ModuleData struct { + pcHeader *_PCHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []byte + ftab []_FuncTab + findfunctab *_FindFuncBucket + minpc, maxpc uintptr + text, etext uintptr + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + rodata uintptr + gofunc uintptr + textsectmap []_TextSection + typelinks []int32 + itablinks []unsafe.Pointer + ptab []_PtabEntry + pluginpath string + pkghashes []struct{} + modulename string + modulehashes []struct{} + hasmain uint8 + gcdatamask, gcbssmask _BitVector + typemap map[int32]unsafe.Pointer + bad bool + next *_ModuleData +} + + +type _FindFuncBucket struct { + idx uint32 + subbuckets [16]byte +} + + + +func makePCtab(fp int) []byte { + return append([]byte{0}, encodeVariant((fp + 1) << 1)...) +} + +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argPtrs []bool, localPtrs []bool) { + mod := new(_ModuleData) + + minpc := pc + maxpc := pc + size + + findFuncTab := make([]_FindFuncBucket, textSize/4096 + 1) + + modHeader := &_PCHeader { + magic : 0xfffffff0, + minLC : 1, + nfunc : 1, + ptrSize : 4 << (^uintptr(0) >> 63), + textStart: minpc, + } + + // cache arg and local stackmap + argptrs, localptrs := cacheStackmap(argPtrs, localPtrs, mod) + + base := argptrs + if argptrs > localptrs { + base = localptrs + } + + /* function entry */ + lnt := []_Func {{ + entryOff : 0, + nameoff : 1, + args : int32(args), + pcsp : 1, + nfuncdata : 2, + argptrs: uint32(argptrs - base), + localptrs: uint32(localptrs - base), + }} + nlnt := len(lnt)*int(unsafe.Sizeof(_Func{})) + plnt := unsafe.Pointer(&lnt[0]) + + /* function table */ + ftab := []_FuncTab { + {entry : 0, funcoff : 16}, + {entry : uint32(size)}, + } + nftab := len(ftab)*int(unsafe.Sizeof(_FuncTab{})) + pftab := unsafe.Pointer(&ftab[0]) + + pclntab := make([]byte, 0, nftab + nlnt) + pclntab = append(pclntab, rt.BytesFrom(pftab, nftab, nftab)...) + pclntab = append(pclntab, rt.BytesFrom(plnt, nlnt, nlnt)...) + + /* module data */ + *mod = _ModuleData { + pcHeader : modHeader, + funcnametab : append(append([]byte{0}, name...), 0), + pctab : append(makePCtab(fp), encodeVariant(int(size))...), + pclntable : pclntab, + ftab : ftab, + text : minpc, + etext : pc + textSize, + findfunctab : &findFuncTab[0], + minpc : minpc, + maxpc : maxpc, + modulename : name, + gcdata: uintptr(unsafe.Pointer(&emptyByte)), + gcbss: uintptr(unsafe.Pointer(&emptyByte)), + gofunc: base, + } + + /* verify and register the new module */ + moduledataverify1(mod) + registerModule(mod) +} \ No newline at end of file diff --git a/internal/loader/loader.go b/internal/loader/loader.go index 5fa8eb2..6446a5f 100644 --- a/internal/loader/loader.go +++ b/internal/loader/loader.go @@ -1,5 +1,5 @@ -//go:build linux || darwin -// +build linux darwin +//go:build darwin || linux +// +build darwin linux /* * Copyright 2021 ByteDance Inc. @@ -36,15 +36,15 @@ const ( type Loader []byte type Function unsafe.Pointer -func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) { +func (self Loader) Load(fn string, fp int, args int, argPtrs []bool, localPtrs []bool) (f Function) { p := os.Getpagesize() n := (((len(self) - 1) / p) + 1) * p /* register the function */ m := mmap(n) v := fmt.Sprintf("runtime.__%s_%x", fn, m) - argsptr, localsptr := stackMap(faker) - registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr) + + registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argPtrs, localPtrs) /* reference as a slice */ s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader { @@ -59,10 +59,6 @@ func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) return Function(&m) } -func (self Loader) Load(fn string, fp int, args int) (f Function) { - return self.LoadWithFaker(fn, fp, args, func(){}) -} - func mmap(nb int) uintptr { if m, _, e := syscall.RawSyscall6(syscall.SYS_MMAP, 0, uintptr(nb), _RW, _AP, 0, 0); e != 0 { panic(e) diff --git a/internal/loader/loader_go117_test.go b/internal/loader/loader_go117_test.go index 2b22fbf..b521fb1 100644 --- a/internal/loader/loader_go117_test.go +++ b/internal/loader/loader_go117_test.go @@ -23,7 +23,6 @@ import ( `fmt` `reflect` `runtime` - `runtime/debug` `testing` `unsafe` @@ -36,7 +35,7 @@ func TestLoader_Load(t *testing.T) { 0xc3, // RET } v0 := 0 - fn := Loader(bc).Load("test", 0, 8) + fn := Loader(bc).Load("test", 0, 8, nil, nil) (*(*func(*int))(unsafe.Pointer(&fn)))(&v0) assert.Equal(t, 1234, v0) println(runtime.FuncForPC(*(*uintptr)(fn)).Name()) @@ -90,23 +89,3 @@ func funcWrap(f func(i *int)) int { ret = x return ret } - -func TestLoadWithStackMap(t *testing.T) { - var f = func(i *int) { - *i = 1234 - } - v1 := funcWrap(f) - - bc := []byte { - 0x48, 0xc7, 0x00, 0xd2, 0x04, 0x00, 0x00, // MOVQ $1234, (%rax) - 0xc3, // RET - } - fn := Loader(bc).LoadWithFaker("test", 0, 8, f) - f2 := (*(*func(*int))(unsafe.Pointer(&fn))) - v2 := funcWrap(f2) - - runtime.GC() - debug.FreeOSMemory() - println(v1, v2) - assert.Equal(t, v1, v2) -} diff --git a/internal/loader/loader_test.go b/internal/loader/loader_test.go index c09fafa..c707114 100644 --- a/internal/loader/loader_test.go +++ b/internal/loader/loader_test.go @@ -23,7 +23,6 @@ import ( `fmt` `reflect` `runtime` - `runtime/debug` `testing` `unsafe` @@ -37,7 +36,7 @@ func TestLoader_Load(t *testing.T) { 0xc3, // RET } v0 := 0 - fn := Loader(bc).Load("test", 0, 8) + fn := Loader(bc).Load("test", 0, 8, nil, nil) (*(*func(*int))(unsafe.Pointer(&fn)))(&v0) assert.Equal(t, 1234, v0) println(runtime.FuncForPC(*(*uintptr)(fn)).Name()) @@ -91,24 +90,3 @@ func funcWrap(f func(i *int)) int { ret = x return ret } - -func TestLoadWithStackMap(t *testing.T) { - var f = func(i *int) { - *i = 1234 - } - v1 := funcWrap(f) - - bc := []byte { - 0x48, 0x8b, 0x44, 0x24, 0x08, // MOVQ 8(%rsp), %rax - 0x48, 0xc7, 0x00, 0xd2, 0x04, 0x00, 0x00, // MOVQ $1234, (%rax) - 0xc3, // RET - } - fn := Loader(bc).LoadWithFaker("test", 0, 8, f) - f2 := (*(*func(*int))(unsafe.Pointer(&fn))) - v2 := funcWrap(f2) - - runtime.GC() - debug.FreeOSMemory() - println(v1, v2) - assert.Equal(t, v1, v2) -} diff --git a/internal/loader/loader_windows.go b/internal/loader/loader_windows.go index 455f019..4053ee9 100644 --- a/internal/loader/loader_windows.go +++ b/internal/loader/loader_windows.go @@ -1,3 +1,6 @@ +//go:build windows +// +build windows + /* * Copyright 2021 ByteDance Inc. * @@ -38,15 +41,15 @@ var ( type Loader []byte type Function unsafe.Pointer -func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) { +func (self Loader) Load(fn string, fp int, args int, argPtrs []bool, localPtrs []bool) (f Function) { p := os.Getpagesize() n := (((len(self) - 1) / p) + 1) * p /* register the function */ m := mmap(n) v := fmt.Sprintf("runtime.__%s_%x", fn, m) - argsptr, localsptr := stackMap(faker) - registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr) + + registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argPtrs, localPtrs) /* reference as a slice */ s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader { @@ -61,10 +64,6 @@ func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) return Function(&m) } -func (self Loader) Load(fn string, fp int, args int) (f Function) { - return self.LoadWithFaker(fn, fp, args, func(){}) -} - func mmap(nb int) uintptr { addr, err := winapi_VirtualAlloc(0, nb, MEM_COMMIT|MEM_RESERVE, syscall.PAGE_READWRITE) if err != nil { diff --git a/internal/rt/fastmem.go b/internal/rt/fastmem.go index c6daab5..358ce80 100644 --- a/internal/rt/fastmem.go +++ b/internal/rt/fastmem.go @@ -74,6 +74,7 @@ func IndexByte(ptr []byte, index int) unsafe.Pointer { return unsafe.Pointer(uintptr((*GoSlice)(unsafe.Pointer(&ptr)).Ptr) + uintptr(index)) } +//go:nosplit func GuardSlice(buf *[]byte, n int) { c := cap(*buf) l := len(*buf) @@ -88,3 +89,25 @@ func GuardSlice(buf *[]byte, n int) { } return } + +//go:nosplit +func Ptr2SlicePtr(s unsafe.Pointer, l int, c int) unsafe.Pointer { + slice := &GoSlice{ + Ptr: s, + Len: l, + Cap: c, + } + return unsafe.Pointer(slice) +} + +//go:nosplit +func StrPtr(s string) unsafe.Pointer { + return (*GoString)(unsafe.Pointer(&s)).Ptr +} + +//go:nosplit +func StrFrom(p unsafe.Pointer, n int64) (s string) { + (*GoString)(unsafe.Pointer(&s)).Ptr = p + (*GoString)(unsafe.Pointer(&s)).Len = int(n) + return +} \ No newline at end of file diff --git a/internal/rt/stackmap.go b/internal/rt/stackmap.go new file mode 100644 index 0000000..e2c28c5 --- /dev/null +++ b/internal/rt/stackmap.go @@ -0,0 +1,180 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rt + +import ( + `fmt` + `strings` + `unsafe` + +) + +type Bitmap struct { + N int + B []byte +} + +func (self *Bitmap) grow() { + if self.N >= len(self.B) * 8 { + self.B = append(self.B, 0) + } +} + +func (self *Bitmap) mark(i int, bv int) { + if bv != 0 { + self.B[i / 8] |= 1 << (i % 8) + } else { + self.B[i / 8] &^= 1 << (i % 8) + } +} + +func (self *Bitmap) Set(i int, bv int) { + if i >= self.N { + panic("bitmap: invalid bit position") + } else { + self.mark(i, bv) + } +} + +func (self *Bitmap) Append(bv int) { + self.grow() + self.mark(self.N, bv) + self.N++ +} + +func (self *Bitmap) AppendMany(n int, bv int) { + for i := 0; i < n; i++ { + self.Append(bv) + } +} + +// var ( +// _stackMapLock = sync.Mutex{} +// _stackMapCache = make(map[*StackMap]struct{}) +// ) + +type BitVec struct { + N uintptr + B unsafe.Pointer +} + +func (self BitVec) Bit(i uintptr) byte { + return (*(*byte)(unsafe.Pointer(uintptr(self.B) + i / 8)) >> (i % 8)) & 1 +} + +func (self BitVec) String() string { + var i uintptr + var v []string + + /* add each bit */ + for i = 0; i < self.N; i++ { + v = append(v, fmt.Sprintf("%d", self.Bit(i))) + } + + /* join them together */ + return fmt.Sprintf( + "BitVec { %s }", + strings.Join(v, ", "), + ) +} + +type StackMap struct { + N int32 + L int32 + B [1]byte +} + +// func (self *StackMap) add() { +// _stackMapLock.Lock() +// _stackMapCache[self] = struct{}{} +// _stackMapLock.Unlock() +// } + +func (self *StackMap) Pin() uintptr { + // self.add() + return uintptr(unsafe.Pointer(self)) +} + +func (self *StackMap) Get(i int32) BitVec { + return BitVec { + N: uintptr(self.L), + B: unsafe.Pointer(uintptr(unsafe.Pointer(&self.B)) + uintptr(i * ((self.L + 7) >> 3))), + } +} + +func (self *StackMap) String() string { + sb := strings.Builder{} + sb.WriteString("StackMap {") + + /* dump every stack map */ + for i := int32(0); i < self.N; i++ { + sb.WriteRune('\n') + sb.WriteString(" " + self.Get(i).String()) + } + + /* close the stackmap */ + sb.WriteString("\n}") + return sb.String() +} + +func (self *StackMap) MarshalBinary() ([]byte, error) { + size := int(self.N) * int(self.L) + int(unsafe.Sizeof(self.L)) + int(unsafe.Sizeof(self.N)) + return BytesFrom(unsafe.Pointer(self), size, size), nil +} + +var ( + byteType = UnpackEface(byte(0)).Type +) + +const ( + _StackMapSize = unsafe.Sizeof(StackMap{}) +) + +//go:linkname mallocgc runtime.mallocgc +//goland:noinspection GoUnusedParameter +func mallocgc(nb uintptr, vt *GoType, zero bool) unsafe.Pointer + +type StackMapBuilder struct { + b Bitmap +} + +func (self *StackMapBuilder) Build() (p *StackMap) { + nb := len(self.b.B) + bm := mallocgc(_StackMapSize + uintptr(nb) - 1, byteType, false) + + /* initialize as 1 bitmap of N bits */ + p = (*StackMap)(bm) + p.N, p.L = 1, int32(self.b.N) + copy(BytesFrom(unsafe.Pointer(&p.B), nb, nb), self.b.B) + return +} + +func (self *StackMapBuilder) AddField(ptr bool) { + if ptr { + self.b.Append(1) + } else { + self.b.Append(0) + } +} + +func (self *StackMapBuilder) AddFields(n int, ptr bool) { + if ptr { + self.b.AppendMany(n, 1) + } else { + self.b.AppendMany(n, 0) + } +} \ No newline at end of file diff --git a/loader/funcdata.go b/loader/funcdata.go new file mode 100644 index 0000000..9b760f6 --- /dev/null +++ b/loader/funcdata.go @@ -0,0 +1,144 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `encoding` + `encoding/binary` + `fmt` + `reflect` + `strings` + `sync` + `unsafe` +) + +const ( + _MinLC uint8 = 1 + _PtrSize uint8 = 8 +) + +const ( + _N_FUNCDATA = 8 + _INVALID_FUNCDATA_OFFSET = ^uint32(0) + _FUNC_SIZE = unsafe.Sizeof(_func{}) + + _MINFUNC = 16 // minimum size for a function + _BUCKETSIZE = 256 * _MINFUNC + _SUBBUCKETS = 16 + _SUB_BUCKETSIZE = _BUCKETSIZE / _SUBBUCKETS +) + +// PCDATA and FUNCDATA table indexes. +// +// See funcdata.h and $GROOT/src/cmd/internal/objabi/funcdata.go. +const ( + _FUNCDATA_ArgsPointerMaps = 0 + _FUNCDATA_LocalsPointerMaps = 1 + _FUNCDATA_StackObjects = 2 + _FUNCDATA_InlTree = 3 + _FUNCDATA_OpenCodedDeferInfo = 4 + _FUNCDATA_ArgInfo = 5 + _FUNCDATA_ArgLiveInfo = 6 + _FUNCDATA_WrapInfo = 7 + + // ArgsSizeUnknown is set in Func.argsize to mark all functions + // whose argument size is unknown (C vararg functions, and + // assembly code without an explicit specification). + // This value is generated by the compiler, assembler, or linker. + ArgsSizeUnknown = -0x80000000 +) + +// moduledata used to cache the funcdata and findfuncbucket of one module +var moduleCache = struct { + m map[*moduledata][]byte + sync.Mutex +}{ + m: make(map[*moduledata][]byte), +} + +// Func contains information about a function. +type Func struct { + ID uint8 // see runtime/symtab.go + Flag uint8 // see runtime/symtab.go + ArgsSize int32 // args byte size + EntryOff uint32 // start pc, offset to moduledata.text + TextSize uint32 // size of func text + DeferReturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + FileIndex uint32 // index into filetab + Name string // name of function + + // PC data + Pcsp *Pcdata // PC -> SP delta + Pcfile *Pcdata // PC -> file index + Pcline *Pcdata // PC -> line number + PcUnsafePoint *Pcdata // PC -> unsafe point, must be PCDATA_UnsafePointSafe or PCDATA_UnsafePointUnsafe + PcStackMapIndex *Pcdata // PC -> stack map index, relative to ArgsPointerMaps and LocalsPointerMaps + PcInlTreeIndex *Pcdata // PC -> inlining tree index, relative to InlTree + PcArgLiveIndex *Pcdata // PC -> arg live index, relative to ArgLiveInfo + + // Func data, must implement encoding.BinaryMarshaler + ArgsPointerMaps encoding.BinaryMarshaler // concrete type: *StackMap + LocalsPointerMaps encoding.BinaryMarshaler // concrete type: *StackMap + StackObjects encoding.BinaryMarshaler + InlTree encoding.BinaryMarshaler + OpenCodedDeferInfo encoding.BinaryMarshaler + ArgInfo encoding.BinaryMarshaler + ArgLiveInfo encoding.BinaryMarshaler + WrapInfo encoding.BinaryMarshaler +} + +func getOffsetOf(data interface{}, field string) uintptr { + t := reflect.TypeOf(data) + fv, ok := t.FieldByName(field) + if !ok { + panic(fmt.Sprintf("field %s not found in struct %s", field, t.Name())) + } + return fv.Offset +} + +func rnd(v int64, r int64) int64 { + if r <= 0 { + return v + } + v += r - 1 + c := v % r + if c < 0 { + c += r + } + v -= c + return v +} + +var ( + byteOrder binary.ByteOrder = binary.LittleEndian +) + +func funcNameParts(name string) (string, string, string) { + i := strings.IndexByte(name, '[') + if i < 0 { + return name, "", "" + } + // TODO: use LastIndexByte once the bootstrap compiler is >= Go 1.5. + j := len(name) - 1 + for j > i && name[j] != ']' { + j-- + } + if j <= i { + return name, "", "" + } + return name[:i], "[...]", name[j+1:] +} \ No newline at end of file diff --git a/loader/funcdata_go115.go b/loader/funcdata_go115.go new file mode 100644 index 0000000..f35f03d --- /dev/null +++ b/loader/funcdata_go115.go @@ -0,0 +1,541 @@ +// go:build go1.15 && !go1.18 +// +build go1.15,!go1.18 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `encoding` + `os` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` +) + +const ( + _Magic uint32 = 0xfffffff0 +) + +type pcHeader struct { + magic uint32 // 0xFFFFFFF0 + pad1, pad2 uint8 // 0,0 + minLC uint8 // min instruction size + ptrSize uint8 // size of a ptr in bytes + nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab + textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text + funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab variable from pcHeader + pclnOffset uintptr // offset to the pclntab variable from pcHeader +} + +type moduledata struct { + pcHeader *pcHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []byte + ftab []funcTab + findfunctab uintptr + minpc, maxpc uintptr // first func address, last func address + last func size + + text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + rodata uintptr + gofunc uintptr // go.func.* is actual funcinfo object in image + + textsectmap []textSection // see runtime/symtab.go: textAddr() + typelinks []int32 // offsets from types + itablinks []*rt.GoItab + + ptab []ptabEntry + + pluginpath string + pkghashes []modulehash + + modulename string + modulehashes []modulehash + + hasmain uint8 // 1 if module contains the main function, 0 otherwise + + gcdatamask, gcbssmask bitVector + + typemap map[int32]*rt.GoType // offset to *_rtype in previous module + + bad bool // module failed to load and should be ignored + + next *moduledata +} + +type _func struct { + entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart + nameOff int32 // function name, as index into moduledata.funcnametab. + + args int32 // in/out args size + deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 // runtime.cutab offset of this function's CU + funcID uint8 // set for certain special runtime functions + flag uint8 + _ [1]byte // pad + nfuncdata uint8 // + + // The end of the struct is followed immediately by two variable-length + // arrays that reference the pcdata and funcdata locations for this + // function. + + // pcdata contains the offset into moduledata.pctab for the start of + // that index's table. e.g., + // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of + // the unsafe point table. + // + // An offset of 0 indicates that there is no table. + // + // pcdata [npcdata]uint32 + + // funcdata contains the offset past moduledata.gofunc which contains a + // pointer to that index's funcdata. e.g., + // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is + // the argument pointer map. + // + // An offset of ^uint32(0) indicates that there is no entry. + // + // funcdata [nfuncdata]uint32 +} + +type funcTab struct { + entry uint32 + funcoff uint32 +} + +type bitVector struct { + n int32 // # of bits + bytedata *uint8 +} + +type ptabEntry struct { + name int32 + typ int32 +} + +type textSection struct { + vaddr uintptr // prelinked section vaddr + end uintptr // vaddr + section length + baseaddr uintptr // relocated section address +} + +type modulehash struct { + modulename string + linktimehash string + runtimehash *string +} + +// findfuncbucket is an array of these structures. +// Each bucket represents 4096 bytes of the text segment. +// Each subbucket represents 256 bytes of the text segment. +// To find a function given a pc, locate the bucket and subbucket for +// that pc. Add together the idx and subbucket value to obtain a +// function index. Then scan the functab array starting at that +// index to find the target function. +// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. +type findfuncbucket struct { + idx uint32 + _SUBBUCKETS [16]byte +} + +// func name table format: +// nameOff[0] -> namePartA namePartB namePartC \x00 +// nameOff[1] -> namePartA namePartB namePartC \x00 +// ... +func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { + offs = make([]int32, len(funcs)) + offset := 0 + + for i, f := range funcs { + offs[i] = int32(offset) + + a, b, c := funcNameParts(f.Name) + tab = append(tab, a...) + tab = append(tab, b...) + tab = append(tab, c...) + tab = append(tab, 0) + offset += len(a) + len(b) + len(c) + 1 + } + + return +} + +type compilationUnit struct { + fileNames []string +} + +// CU table format: +// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] +// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] +// ... +// +// file name table format: +// filetabOffset[0] -> CUs[0].fileNames[0] \x00 +// ... +// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 +// ... +// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 +func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { + cuOffsets = make([]uint32, len(cus)) + cuOffset := 0 + fileOffset := 0 + + for i, cu := range cus { + cuOffsets[i] = uint32(cuOffset) + + for _, name := range cu.fileNames { + cutab = append(cutab, uint32(fileOffset)) + + fileOffset += len(name) + 1 + filetab = append(filetab, name...) + filetab = append(filetab, 0) + } + + cuOffset += len(cu.fileNames) + } + + return +} + +func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { + fstart = len(*out) + *out = append(*out, byte(0)) + offs := uint32(1) + + funcdataOffs = make([][]uint32, len(funcs)) + for i, f := range funcs { + + var writer = func(fd encoding.BinaryMarshaler) { + var ab []byte + var err error + if fd != nil { + ab, err = fd.MarshalBinary() + if err != nil { + panic(err) + } + funcdataOffs[i] = append(funcdataOffs[i], offs) + } else { + ab = []byte{0} + funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) + } + *out = append(*out, ab...) + offs += uint32(len(ab)) + } + + writer(f.ArgsPointerMaps) + writer(f.LocalsPointerMaps) + writer(f.StackObjects) + writer(f.InlTree) + writer(f.OpenCodedDeferInfo) + writer(f.ArgInfo) + writer(f.ArgLiveInfo) + writer(f.WrapInfo) + } + return +} + +func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i, f := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4) + } + + ftab = make([]funcTab, 0, len(funcs)+1) + + // write a map of pc->func info offsets + for i, f := range funcs { + ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])}) + } + + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0}) + + return +} + +// Pcln table format: [...]funcTab + [...]_Func +func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4) + } + + pclntab = make([]byte, size, size) + + // write a map of pc->func info offsets + offs := 0 + for i, f := range funcs { + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff)) + byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i])) + offs += 8 + } + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize)) + + // write func info table + for i, f := range funcs { + off := startLocations[i] + + // write _func structure to pclntab + fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE)) + copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb) + off += uint32(_FUNC_SIZE) + + // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 + for j := 3; j < len(pcdataOffs[i]); j++ { + byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) + off += 4 + } + + // funcdata refs as offsets from gofunc + for _, funcdata := range funcdataOffs[i] { + byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata)) + off += 4 + } + + } + + return +} + +// findfunc table used to map pc to belonging func, +// returns the index in the func table. +// +// All text section are divided into buckets sized _BUCKETSIZE(4K): +// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), +// and it has a base idx to plus the offset stored in jth subbucket. +// see findfunc() in runtime/symtab.go +func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { + start = len(*out) + + max := ftab[len(ftab)-1].entry + min := ftab[0].entry + nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE + n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE + + tab := make([]findfuncbucket, 0, nbuckets) + var s, e = 0, 0 + for i := 0; i 0 { + size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) + *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) + } + return +} + +func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { + mod = new(moduledata) + mod.modulename = name + + // make filename table + cu := make([]string, 0, len(filenames)) + for _, f := range filenames { + cu = append(cu, f) + } + cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) + mod.cutab = cutab + mod.filetab = filetab + + // make funcname table + funcnametab, nameOffs := makeFuncnameTab(funcs) + mod.funcnametab = funcnametab + + // make pcdata table + // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata + pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs) + mod.pctab = pctab + + // write func data + // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata + // TODO: estimate accurate capacity + cache := make([]byte, 0, len(funcs)*int(_PtrSize)) + fstart, funcdataOffs := writeFuncdata(&cache, funcs) + + // make pc->func (binary search) func table + lastFuncsize := funcs[len(funcs)-1].TextSize + ftab := makeFtab(_funcs, lastFuncsize) + mod.ftab = ftab + + // write pc->func (modmap) findfunc table + ffstart := writeFindfunctab(&cache, ftab) + + // make pclnt table + pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs) + mod.pclntable = pclntab + + // mmap() text and funcdata segements + p := os.Getpagesize() + size := int(rnd(int64(len(text)), int64(p))) + addr := mmap(size) + // copy the machine code + s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) + copy(s, text) + // make it executable + mprotect(addr, size) + + // assign addresses + mod.text = addr + mod.etext = addr + uintptr(size) + mod.minpc = addr + mod.maxpc = addr + uintptr(len(text)) + + // cache funcdata and findfuncbucket + moduleCache.Lock() + moduleCache.m[mod] = cache + moduleCache.Unlock() + mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart])) + mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart])) + + // make pc header + mod.pcHeader = &pcHeader { + magic : _Magic, + minLC : _MinLC, + ptrSize : _PtrSize, + nfunc : len(funcs), + nfiles: uint(len(cu)), + textStart: mod.text, + funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), + cuOffset: getOffsetOf(moduledata{}, "cutab"), + filetabOffset: getOffsetOf(moduledata{}, "filetab"), + pctabOffset: getOffsetOf(moduledata{}, "pctab"), + pclnOffset: getOffsetOf(moduledata{}, "pclntable"), + } + + // sepecial case: gcdata and gcbss must by non-empty + mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) + mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) + + return +} + +// makePctab generates pcdelta->valuedelta tables for functions, +// and returns the table and the entry offset of every kind pcdata in the table. +func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { + _funcs = make([]_func, len(funcs)) + + // Pctab offsets of 0 are considered invalid in the runtime. We respect + // that by just padding a single byte at the beginning of runtime.pctab, + // that way no real offsets can be zero. + pctab = make([]byte, 1, 12*len(funcs)+1) + pcdataOffs = make([][]uint32, len(funcs)) + + for i, f := range funcs { + _f := &_funcs[i] + + var writer = func(pc *Pcdata) { + var ab []byte + var err error + if pc != nil { + ab, err = pc.MarshalBinary() + if err != nil { + panic(err) + } + pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) + } else { + ab = []byte{0} + pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) + } + pctab = append(pctab, ab...) + } + + if f.Pcsp != nil { + _f.pcsp = uint32(len(pctab)) + } + writer(f.Pcsp) + if f.Pcfile != nil { + _f.pcfile = uint32(len(pctab)) + } + writer(f.Pcfile) + if f.Pcline != nil { + _f.pcln = uint32(len(pctab)) + } + writer(f.Pcline) + writer(f.PcUnsafePoint) + writer(f.PcStackMapIndex) + writer(f.PcInlTreeIndex) + writer(f.PcArgLiveIndex) + + _f.entryOff = f.EntryOff + _f.nameOff = nameOffset[i] + _f.args = f.ArgsSize + _f.deferreturn = f.DeferReturn + // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] + _f.npcdata = uint32(_N_PCDATA) + _f.cuOffset = cuOffset[i] + _f.funcID = f.ID + _f.flag = f.Flag + _f.nfuncdata = uint8(_N_FUNCDATA) + } + + return +} + +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {} \ No newline at end of file diff --git a/loader/funcdata_go118.go b/loader/funcdata_go118.go new file mode 100644 index 0000000..a2bac85 --- /dev/null +++ b/loader/funcdata_go118.go @@ -0,0 +1,541 @@ +// go:build go1.18 && !go1.20 +// +build go1.18,!go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `encoding` + `os` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` +) + +const ( + _Magic uint32 = 0xfffffff0 +) + +type pcHeader struct { + magic uint32 // 0xFFFFFFF0 + pad1, pad2 uint8 // 0,0 + minLC uint8 // min instruction size + ptrSize uint8 // size of a ptr in bytes + nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab + textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text + funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab variable from pcHeader + pclnOffset uintptr // offset to the pclntab variable from pcHeader +} + +type moduledata struct { + pcHeader *pcHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []byte + ftab []funcTab + findfunctab uintptr + minpc, maxpc uintptr // first func address, last func address + last func size + + text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + rodata uintptr + gofunc uintptr // go.func.* is actual funcinfo object in image + + textsectmap []textSection // see runtime/symtab.go: textAddr() + typelinks []int32 // offsets from types + itablinks []*rt.GoItab + + ptab []ptabEntry + + pluginpath string + pkghashes []modulehash + + modulename string + modulehashes []modulehash + + hasmain uint8 // 1 if module contains the main function, 0 otherwise + + gcdatamask, gcbssmask bitVector + + typemap map[int32]*rt.GoType // offset to *_rtype in previous module + + bad bool // module failed to load and should be ignored + + next *moduledata +} + +type _func struct { + entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart + nameOff int32 // function name, as index into moduledata.funcnametab. + + args int32 // in/out args size + deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 // runtime.cutab offset of this function's CU + funcID uint8 // set for certain special runtime functions + flag uint8 + _ [1]byte // pad + nfuncdata uint8 // + + // The end of the struct is followed immediately by two variable-length + // arrays that reference the pcdata and funcdata locations for this + // function. + + // pcdata contains the offset into moduledata.pctab for the start of + // that index's table. e.g., + // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of + // the unsafe point table. + // + // An offset of 0 indicates that there is no table. + // + // pcdata [npcdata]uint32 + + // funcdata contains the offset past moduledata.gofunc which contains a + // pointer to that index's funcdata. e.g., + // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is + // the argument pointer map. + // + // An offset of ^uint32(0) indicates that there is no entry. + // + // funcdata [nfuncdata]uint32 +} + +type funcTab struct { + entry uint32 + funcoff uint32 +} + +type bitVector struct { + n int32 // # of bits + bytedata *uint8 +} + +type ptabEntry struct { + name int32 + typ int32 +} + +type textSection struct { + vaddr uintptr // prelinked section vaddr + end uintptr // vaddr + section length + baseaddr uintptr // relocated section address +} + +type modulehash struct { + modulename string + linktimehash string + runtimehash *string +} + +// findfuncbucket is an array of these structures. +// Each bucket represents 4096 bytes of the text segment. +// Each subbucket represents 256 bytes of the text segment. +// To find a function given a pc, locate the bucket and subbucket for +// that pc. Add together the idx and subbucket value to obtain a +// function index. Then scan the functab array starting at that +// index to find the target function. +// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. +type findfuncbucket struct { + idx uint32 + _SUBBUCKETS [16]byte +} + +// func name table format: +// nameOff[0] -> namePartA namePartB namePartC \x00 +// nameOff[1] -> namePartA namePartB namePartC \x00 +// ... +func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { + offs = make([]int32, len(funcs)) + offset := 0 + + for i, f := range funcs { + offs[i] = int32(offset) + + a, b, c := funcNameParts(f.Name) + tab = append(tab, a...) + tab = append(tab, b...) + tab = append(tab, c...) + tab = append(tab, 0) + offset += len(a) + len(b) + len(c) + 1 + } + + return +} + +type compilationUnit struct { + fileNames []string +} + +// CU table format: +// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] +// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] +// ... +// +// file name table format: +// filetabOffset[0] -> CUs[0].fileNames[0] \x00 +// ... +// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 +// ... +// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 +func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { + cuOffsets = make([]uint32, len(cus)) + cuOffset := 0 + fileOffset := 0 + + for i, cu := range cus { + cuOffsets[i] = uint32(cuOffset) + + for _, name := range cu.fileNames { + cutab = append(cutab, uint32(fileOffset)) + + fileOffset += len(name) + 1 + filetab = append(filetab, name...) + filetab = append(filetab, 0) + } + + cuOffset += len(cu.fileNames) + } + + return +} + +func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { + fstart = len(*out) + *out = append(*out, byte(0)) + offs := uint32(1) + + funcdataOffs = make([][]uint32, len(funcs)) + for i, f := range funcs { + + var writer = func(fd encoding.BinaryMarshaler) { + var ab []byte + var err error + if fd != nil { + ab, err = fd.MarshalBinary() + if err != nil { + panic(err) + } + funcdataOffs[i] = append(funcdataOffs[i], offs) + } else { + ab = []byte{0} + funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) + } + *out = append(*out, ab...) + offs += uint32(len(ab)) + } + + writer(f.ArgsPointerMaps) + writer(f.LocalsPointerMaps) + writer(f.StackObjects) + writer(f.InlTree) + writer(f.OpenCodedDeferInfo) + writer(f.ArgInfo) + writer(f.ArgLiveInfo) + writer(f.WrapInfo) + } + return +} + +func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i, f := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4) + } + + ftab = make([]funcTab, 0, len(funcs)+1) + + // write a map of pc->func info offsets + for i, f := range funcs { + ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])}) + } + + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0}) + + return +} + +// Pcln table format: [...]funcTab + [...]_Func +func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4) + } + + pclntab = make([]byte, size, size) + + // write a map of pc->func info offsets + offs := 0 + for i, f := range funcs { + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff)) + byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i])) + offs += 8 + } + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize)) + + // write func info table + for i, f := range funcs { + off := startLocations[i] + + // write _func structure to pclntab + fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE)) + copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb) + off += uint32(_FUNC_SIZE) + + // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 + for j := 3; j < len(pcdataOffs[i]); j++ { + byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) + off += 4 + } + + // funcdata refs as offsets from gofunc + for _, funcdata := range funcdataOffs[i] { + byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata)) + off += 4 + } + + } + + return +} + +// findfunc table used to map pc to belonging func, +// returns the index in the func table. +// +// All text section are divided into buckets sized _BUCKETSIZE(4K): +// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), +// and it has a base idx to plus the offset stored in jth subbucket. +// see findfunc() in runtime/symtab.go +func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { + start = len(*out) + + max := ftab[len(ftab)-1].entry + min := ftab[0].entry + nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE + n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE + + tab := make([]findfuncbucket, 0, nbuckets) + var s, e = 0, 0 + for i := 0; i 0 { + size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) + *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) + } + return +} + +func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { + mod = new(moduledata) + mod.modulename = name + + // make filename table + cu := make([]string, 0, len(filenames)) + for _, f := range filenames { + cu = append(cu, f) + } + cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) + mod.cutab = cutab + mod.filetab = filetab + + // make funcname table + funcnametab, nameOffs := makeFuncnameTab(funcs) + mod.funcnametab = funcnametab + + // make pcdata table + // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata + pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs) + mod.pctab = pctab + + // write func data + // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata + // TODO: estimate accurate capacity + cache := make([]byte, 0, len(funcs)*int(_PtrSize)) + fstart, funcdataOffs := writeFuncdata(&cache, funcs) + + // make pc->func (binary search) func table + lastFuncsize := funcs[len(funcs)-1].TextSize + ftab := makeFtab(_funcs, lastFuncsize) + mod.ftab = ftab + + // write pc->func (modmap) findfunc table + ffstart := writeFindfunctab(&cache, ftab) + + // make pclnt table + pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs) + mod.pclntable = pclntab + + // mmap() text and funcdata segements + p := os.Getpagesize() + size := int(rnd(int64(len(text)), int64(p))) + addr := mmap(size) + // copy the machine code + s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) + copy(s, text) + // make it executable + mprotect(addr, size) + + // assign addresses + mod.text = addr + mod.etext = addr + uintptr(size) + mod.minpc = addr + mod.maxpc = addr + uintptr(len(text)) + + // cache funcdata and findfuncbucket + moduleCache.Lock() + moduleCache.m[mod] = cache + moduleCache.Unlock() + mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart])) + mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart])) + + // make pc header + mod.pcHeader = &pcHeader { + magic : _Magic, + minLC : _MinLC, + ptrSize : _PtrSize, + nfunc : len(funcs), + nfiles: uint(len(cu)), + textStart: mod.text, + funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), + cuOffset: getOffsetOf(moduledata{}, "cutab"), + filetabOffset: getOffsetOf(moduledata{}, "filetab"), + pctabOffset: getOffsetOf(moduledata{}, "pctab"), + pclnOffset: getOffsetOf(moduledata{}, "pclntable"), + } + + // sepecial case: gcdata and gcbss must by non-empty + mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) + mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) + + return +} + +// makePctab generates pcdelta->valuedelta tables for functions, +// and returns the table and the entry offset of every kind pcdata in the table. +func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { + _funcs = make([]_func, len(funcs)) + + // Pctab offsets of 0 are considered invalid in the runtime. We respect + // that by just padding a single byte at the beginning of runtime.pctab, + // that way no real offsets can be zero. + pctab = make([]byte, 1, 12*len(funcs)+1) + pcdataOffs = make([][]uint32, len(funcs)) + + for i, f := range funcs { + _f := &_funcs[i] + + var writer = func(pc *Pcdata) { + var ab []byte + var err error + if pc != nil { + ab, err = pc.MarshalBinary() + if err != nil { + panic(err) + } + pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) + } else { + ab = []byte{0} + pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) + } + pctab = append(pctab, ab...) + } + + if f.Pcsp != nil { + _f.pcsp = uint32(len(pctab)) + } + writer(f.Pcsp) + if f.Pcfile != nil { + _f.pcfile = uint32(len(pctab)) + } + writer(f.Pcfile) + if f.Pcline != nil { + _f.pcln = uint32(len(pctab)) + } + writer(f.Pcline) + writer(f.PcUnsafePoint) + writer(f.PcStackMapIndex) + writer(f.PcInlTreeIndex) + writer(f.PcArgLiveIndex) + + _f.entryOff = f.EntryOff + _f.nameOff = nameOffset[i] + _f.args = f.ArgsSize + _f.deferreturn = f.DeferReturn + // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] + _f.npcdata = uint32(_N_PCDATA) + _f.cuOffset = cuOffset[i] + _f.funcID = f.ID + _f.flag = f.Flag + _f.nfuncdata = uint8(_N_FUNCDATA) + } + + return +} + +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {} \ No newline at end of file diff --git a/loader/funcdata_go120.go b/loader/funcdata_go120.go new file mode 100644 index 0000000..906fe37 --- /dev/null +++ b/loader/funcdata_go120.go @@ -0,0 +1,545 @@ +//go:build go1.20 && !go1.21 +// +build go1.20,!go1.21 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `encoding` + `os` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` +) + +const ( + _Magic uint32 = 0xFFFFFFF1 +) + +type moduledata struct { + pcHeader *pcHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []byte + ftab []funcTab + findfunctab uintptr + minpc, maxpc uintptr // first func address, last func address + last func size + + text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + covctrs, ecovctrs uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + rodata uintptr + + // TODO: generate funcinfo object to memory + gofunc uintptr // go.func.* is actual funcinfo object in image + + textsectmap []textSection // see runtime/symtab.go: textAddr() + typelinks []int32 // offsets from types + itablinks []*rt.GoItab + + ptab []ptabEntry + + pluginpath string + pkghashes []modulehash + + modulename string + modulehashes []modulehash + + hasmain uint8 // 1 if module contains the main function, 0 otherwise + + gcdatamask, gcbssmask bitVector + + typemap map[int32]*rt.GoType // offset to *_rtype in previous module + + bad bool // module failed to load and should be ignored + + next *moduledata +} + +type _func struct { + entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart + nameOff int32 // function name, as index into moduledata.funcnametab. + + args int32 // in/out args size + deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. + + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 // runtime.cutab offset of this function's CU + startLine int32 // line number of start of function (func keyword/TEXT directive) + funcID uint8 // set for certain special runtime functions + flag uint8 + _ [1]byte // pad + nfuncdata uint8 // + + // The end of the struct is followed immediately by two variable-length + // arrays that reference the pcdata and funcdata locations for this + // function. + + // pcdata contains the offset into moduledata.pctab for the start of + // that index's table. e.g., + // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of + // the unsafe point table. + // + // An offset of 0 indicates that there is no table. + // + // pcdata [npcdata]uint32 + + // funcdata contains the offset past moduledata.gofunc which contains a + // pointer to that index's funcdata. e.g., + // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is + // the argument pointer map. + // + // An offset of ^uint32(0) indicates that there is no entry. + // + // funcdata [nfuncdata]uint32 +} + +type funcTab struct { + entry uint32 + funcoff uint32 +} + +type pcHeader struct { + magic uint32 // 0xFFFFFFF0 + pad1, pad2 uint8 // 0,0 + minLC uint8 // min instruction size + ptrSize uint8 // size of a ptr in bytes + nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab + textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text + funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab variable from pcHeader + pclnOffset uintptr // offset to the pclntab variable from pcHeader +} + +type bitVector struct { + n int32 // # of bits + bytedata *uint8 +} + +type ptabEntry struct { + name int32 + typ int32 +} + +type textSection struct { + vaddr uintptr // prelinked section vaddr + end uintptr // vaddr + section length + baseaddr uintptr // relocated section address +} + +type modulehash struct { + modulename string + linktimehash string + runtimehash *string +} + +// findfuncbucket is an array of these structures. +// Each bucket represents 4096 bytes of the text segment. +// Each subbucket represents 256 bytes of the text segment. +// To find a function given a pc, locate the bucket and subbucket for +// that pc. Add together the idx and subbucket value to obtain a +// function index. Then scan the functab array starting at that +// index to find the target function. +// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. +type findfuncbucket struct { + idx uint32 + _SUBBUCKETS [16]byte +} + +// func name table format: +// nameOff[0] -> namePartA namePartB namePartC \x00 +// nameOff[1] -> namePartA namePartB namePartC \x00 +// ... +func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { + offs = make([]int32, len(funcs)) + offset := 0 + + for i, f := range funcs { + offs[i] = int32(offset) + + a, b, c := funcNameParts(f.Name) + tab = append(tab, a...) + tab = append(tab, b...) + tab = append(tab, c...) + tab = append(tab, 0) + offset += len(a) + len(b) + len(c) + 1 + } + + return +} + +type compilationUnit struct { + fileNames []string +} + +// CU table format: +// cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] +// cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] +// ... +// +// file name table format: +// filetabOffset[0] -> CUs[0].fileNames[0] \x00 +// ... +// filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 +// ... +// filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 +func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { + cuOffsets = make([]uint32, len(cus)) + cuOffset := 0 + fileOffset := 0 + + for i, cu := range cus { + cuOffsets[i] = uint32(cuOffset) + + for _, name := range cu.fileNames { + cutab = append(cutab, uint32(fileOffset)) + + fileOffset += len(name) + 1 + filetab = append(filetab, name...) + filetab = append(filetab, 0) + } + + cuOffset += len(cu.fileNames) + } + + return +} + +func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { + fstart = len(*out) + *out = append(*out, byte(0)) + offs := uint32(1) + + funcdataOffs = make([][]uint32, len(funcs)) + for i, f := range funcs { + + var writer = func(fd encoding.BinaryMarshaler) { + var ab []byte + var err error + if fd != nil { + ab, err = fd.MarshalBinary() + if err != nil { + panic(err) + } + funcdataOffs[i] = append(funcdataOffs[i], offs) + } else { + ab = []byte{0} + funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) + } + *out = append(*out, ab...) + offs += uint32(len(ab)) + } + + writer(f.ArgsPointerMaps) + writer(f.LocalsPointerMaps) + writer(f.StackObjects) + writer(f.InlTree) + writer(f.OpenCodedDeferInfo) + writer(f.ArgInfo) + writer(f.ArgLiveInfo) + writer(f.WrapInfo) + } + return +} + +func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i, f := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4) + } + + ftab = make([]funcTab, 0, len(funcs)+1) + + // write a map of pc->func info offsets + for i, f := range funcs { + ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])}) + } + + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0}) + + return +} + +// Pcln table format: [...]funcTab + [...]_Func +func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) { + // Allocate space for the pc->func table. This structure consists of a pc offset + // and an offset to the func structure. After that, we have a single pc + // value that marks the end of the last function in the binary. + var size int64 = int64(len(funcs)*2*4 + 4) + var startLocations = make([]uint32, len(funcs)) + for i := range funcs { + size = rnd(size, int64(_PtrSize)) + //writePCToFunc + startLocations[i] = uint32(size) + size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4) + } + + pclntab = make([]byte, size, size) + + // write a map of pc->func info offsets + offs := 0 + for i, f := range funcs { + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff)) + byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i])) + offs += 8 + } + // Final entry of table is just end pc offset. + lastFunc := funcs[len(funcs)-1] + byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize)) + + // write func info table + for i, f := range funcs { + off := startLocations[i] + + // write _func structure to pclntab + fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE)) + copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb) + off += uint32(_FUNC_SIZE) + + // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 + for j := 3; j < len(pcdataOffs[i]); j++ { + byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) + off += 4 + } + + // funcdata refs as offsets from gofunc + for _, funcdata := range funcdataOffs[i] { + byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata)) + off += 4 + } + + } + + return +} + +// findfunc table used to map pc to belonging func, +// returns the index in the func table. +// +// All text section are divided into buckets sized _BUCKETSIZE(4K): +// every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), +// and it has a base idx to plus the offset stored in jth subbucket. +// see findfunc() in runtime/symtab.go +func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { + start = len(*out) + + max := ftab[len(ftab)-1].entry + min := ftab[0].entry + nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE + n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE + + tab := make([]findfuncbucket, 0, nbuckets) + var s, e = 0, 0 + for i := 0; i 0 { + size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) + *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) + } + return +} + +func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { + mod = new(moduledata) + mod.modulename = name + + // make filename table + cu := make([]string, 0, len(filenames)) + for _, f := range filenames { + cu = append(cu, f) + } + cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) + mod.cutab = cutab + mod.filetab = filetab + + // make funcname table + funcnametab, nameOffs := makeFuncnameTab(funcs) + mod.funcnametab = funcnametab + + // make pcdata table + // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata + pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs) + mod.pctab = pctab + + // write func data + // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata + // TODO: estimate accurate capacity + cache := make([]byte, 0, len(funcs)*int(_PtrSize)) + fstart, funcdataOffs := writeFuncdata(&cache, funcs) + + // make pc->func (binary search) func table + lastFuncsize := funcs[len(funcs)-1].TextSize + ftab := makeFtab(_funcs, lastFuncsize) + mod.ftab = ftab + + // write pc->func (modmap) findfunc table + ffstart := writeFindfunctab(&cache, ftab) + + // make pclnt table + pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs) + mod.pclntable = pclntab + + // mmap() text and funcdata segements + p := os.Getpagesize() + size := int(rnd(int64(len(text)), int64(p))) + addr := mmap(size) + // copy the machine code + s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) + copy(s, text) + // make it executable + mprotect(addr, size) + + // assign addresses + mod.text = addr + mod.etext = addr + uintptr(size) + mod.minpc = addr + mod.maxpc = addr + uintptr(len(text)) + + // cache funcdata and findfuncbucket + moduleCache.Lock() + moduleCache.m[mod] = cache + moduleCache.Unlock() + mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart])) + mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart])) + + // make pc header + mod.pcHeader = &pcHeader { + magic : _Magic, + minLC : _MinLC, + ptrSize : _PtrSize, + nfunc : len(funcs), + nfiles: uint(len(cu)), + textStart: mod.text, + funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), + cuOffset: getOffsetOf(moduledata{}, "cutab"), + filetabOffset: getOffsetOf(moduledata{}, "filetab"), + pctabOffset: getOffsetOf(moduledata{}, "pctab"), + pclnOffset: getOffsetOf(moduledata{}, "pclntable"), + } + + // sepecial case: gcdata and gcbss must by non-empty + mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) + mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) + + return +} + +// makePctab generates pcdelta->valuedelta tables for functions, +// and returns the table and the entry offset of every kind pcdata in the table. +func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { + _funcs = make([]_func, len(funcs)) + + // Pctab offsets of 0 are considered invalid in the runtime. We respect + // that by just padding a single byte at the beginning of runtime.pctab, + // that way no real offsets can be zero. + pctab = make([]byte, 1, 12*len(funcs)+1) + pcdataOffs = make([][]uint32, len(funcs)) + + for i, f := range funcs { + _f := &_funcs[i] + + var writer = func(pc *Pcdata) { + var ab []byte + var err error + if pc != nil { + ab, err = pc.MarshalBinary() + if err != nil { + panic(err) + } + pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) + } else { + ab = []byte{0} + pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) + } + pctab = append(pctab, ab...) + } + + if f.Pcsp != nil { + _f.pcsp = uint32(len(pctab)) + } + writer(f.Pcsp) + if f.Pcfile != nil { + _f.pcfile = uint32(len(pctab)) + } + writer(f.Pcfile) + if f.Pcline != nil { + _f.pcln = uint32(len(pctab)) + } + writer(f.Pcline) + writer(f.PcUnsafePoint) + writer(f.PcStackMapIndex) + writer(f.PcInlTreeIndex) + writer(f.PcArgLiveIndex) + + _f.entryOff = f.EntryOff + _f.nameOff = nameOffset[i] + _f.args = f.ArgsSize + _f.deferreturn = f.DeferReturn + // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] + _f.npcdata = uint32(_N_PCDATA) + _f.cuOffset = cuOffset[i] + _f.funcID = f.ID + _f.flag = f.Flag + _f.nfuncdata = uint8(_N_FUNCDATA) + } + + return +} + +func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {} \ No newline at end of file diff --git a/loader/loader.go b/loader/loader.go new file mode 100644 index 0000000..929d8c2 --- /dev/null +++ b/loader/loader.go @@ -0,0 +1,37 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `unsafe` +) + +// Function is a function pointer +type Function unsafe.Pointer + +// Options used to load a module +type Options struct { + // NoPreempt is used to disable async preemption for this module + NoPreempt bool +} + +// Loader is a helper used to load a module simply +type Loader struct { + Name string // module name + File string // file name + Options +} \ No newline at end of file diff --git a/loader/loader_go115.go b/loader/loader_go115.go new file mode 100644 index 0000000..1e44b36 --- /dev/null +++ b/loader/loader_go115.go @@ -0,0 +1,33 @@ +//go:build go1.15 && !go1.18 +// +build go1.15,!go1.18 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + + `github.com/bytedance/sonic/internal/loader` +) + +func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function { + return Function(loader.Loader(text).Load(funcName, frameSize, argSize, argStackmap, localStackmap)) +} + +func Load(modulename string, filenames []string, funcs []Func, text []byte) (out []Function) { + panic("not implemented") +} \ No newline at end of file diff --git a/loader/loader_go118.go b/loader/loader_go118.go new file mode 100644 index 0000000..9a0fe48 --- /dev/null +++ b/loader/loader_go118.go @@ -0,0 +1,104 @@ +//go:build go1.18 && !go1.21 +// +build go1.18,!go1.21 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `github.com/bytedance/sonic/internal/rt` +) + +// LoadFuncs loads only one function as module, and returns the function pointer +// - text: machine code +// - funcName: function name +// - frameSize: stack frame size. +// - argSize: argument total size (in bytes) +// - argPtrs: indicates if a slot (8 Bytes) of arguments memory stores pointer, from low to high +// - localPtrs: indicates if a slot (8 Bytes) of local variants memory stores pointer, from low to high +// +// WARN: +// - the function MUST has fixed SP offset equaling to this, otherwise it go.gentraceback will fail +// - the function MUST has only one stack map for all arguments and local variants +func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function { + size := uint32(len(text)) + + fn := Func{ + Name: funcName, + TextSize: size, + ArgsSize: int32(argSize), + } + + // NOTICE: suppose the function has fixed SP offset equaling to frameSize, thus make only one pcsp pair + fn.Pcsp = &Pcdata{ + {PC: size, Val: int32(frameSize)}, + } + + if self.NoPreempt { + fn.PcUnsafePoint = &Pcdata{ + {PC: size, Val: PCDATA_UnsafePointUnsafe}, + } + } else { + fn.PcUnsafePoint = &Pcdata{ + {PC: size, Val: PCDATA_UnsafePointSafe}, + } + } + + // NOTICE: suppose the function has only one stack map at index 0 + fn.PcStackMapIndex = &Pcdata{ + {PC: size, Val: 0}, + } + + if argPtrs != nil { + args := rt.StackMapBuilder{} + for _, b := range argPtrs { + args.AddField(b) + } + fn.ArgsPointerMaps = args.Build() + } + + if localPtrs != nil { + locals := rt .StackMapBuilder{} + for _, b := range localPtrs { + locals.AddField(b) + } + fn.LocalsPointerMaps = locals.Build() + } + + out := Load(text, []Func{fn}, self.Name + funcName, []string{self.File}) + return out[0] +} + +// Load loads given machine codes and corresponding function information into go moduledata +// and returns runnable function pointer +// WARN: this API is experimental, use it carefully +func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) { + // generate module data and allocate memory address + mod := makeModuledata(modulename, filenames, funcs, text) + + // verify and register the new module + moduledataverify1(mod) + registerModule(mod) + + // encapsulate function address + out = make([]Function, len(funcs)) + for i, f := range funcs { + m := uintptr(mod.text + uintptr(f.EntryOff)) + out[i] = Function(&m) + } + return +} \ No newline at end of file diff --git a/loader/loader_test.go b/loader/loader_test.go new file mode 100644 index 0000000..f186406 --- /dev/null +++ b/loader/loader_test.go @@ -0,0 +1,134 @@ +//go:build go1.18 +// +build go1.18 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `runtime` + `runtime/debug` + `strconv` + `testing` + `unsafe` + + `github.com/bytedance/sonic/internal/rt` + `github.com/stretchr/testify/require` +) + +func TestLoad(t *testing.T) { + // defer func() { + // if r := recover(); r != nil { + // runtime.GC() + // if r != "hook1" { + // t.Fatal("not right panic:" + r.(string)) + // } + // } else { + // t.Fatal("not panic") + // } + // }() + + var hstr string + + type TestFunc func(i *int, hook func(i *int)) int + var hook = func(i *int) { + runtime.GC() + debug.FreeOSMemory() + hstr = ("hook" + strconv.Itoa(*i)) + runtime.GC() + debug.FreeOSMemory() + } + // var f TestFunc = func(i *int, hook func(i *int)) int { + // var t = *i + // hook(i) + // return t + *i + // } + bc := []byte { + 0x48, 0x83, 0xec, 0x18, // (0x00) subq $24, %rsp + 0x48, 0x89, 0x6c, 0x24, 0x10, // (0x04) movq %rbp, 16(%rsp) + 0x48, 0x8d, 0x6c, 0x24, 0x10, // (0x09) leaq 16(%rsp), %rbp + 0x48, 0x89, 0x44, 0x24, 0x20, // (0x0e) movq %rax, 32(%rsp) + 0x48, 0x8b, 0x08, // (0x13) movq (%rax), %rcx + 0x48, 0x89, 0x4c, 0x24, 0x08, // (0x16) movq %rcx, 8(%rsp) + 0x48, 0x8b, 0x33, // (0x1b) movq (%rbx), %rsi + 0x48, 0x89, 0xda, // (0x1e) movq %rbx, %rdx + 0xff, 0xd6, // (0x21) callq %rsi + 0x48, 0x8b, 0x44, 0x24, 0x08, // (0x23) movq 8(%rsp), %rax + 0x48, 0x8b, 0x4c, 0x24, 0x20, // (0x28) movq 32(%rsp), %rcx + 0x48, 0x03, 0x01, // (0x2d) addq (%rcx), %rax + 0x48, 0x8b, 0x6c, 0x24, 0x10, // (0x30) movq 16(%rsp), %rbp + 0x48, 0x83, 0xc4, 0x18, // (0x35) addq $24, %rsp + 0xc3, // (0x39) ret + } + size := uint32(len(bc)) + fn := Func{ + ID: 0, + Flag: 0, + ArgsSize: 16, + EntryOff: 0, + TextSize: size, + DeferReturn: 0, + FileIndex: 0, + Name: "dummy", + } + + fn.Pcsp = &Pcdata{ + {PC: 0x04, Val: 0}, + {PC: size, Val: 24}, + } + + fn.Pcline = &Pcdata{ + {PC: 0x00, Val: 0}, + {PC: 0x0e, Val: 1}, + {PC: 0x1d, Val: 2}, + {PC: size, Val: 3}, + } + + fn.Pcfile = &Pcdata{ + {PC: size, Val: 0}, + } + + fn.PcUnsafePoint = &Pcdata{ + {PC: size, Val: PCDATA_UnsafePointUnsafe}, + } + + fn.PcStackMapIndex = &Pcdata{ + {PC: size, Val: 0}, + } + + args := rt.StackMapBuilder{} + args.AddField(true) + args.AddField(true) + fn.ArgsPointerMaps = args.Build() + + locals := rt.StackMapBuilder{} + locals.AddField(false) + locals.AddField(false) + fn.LocalsPointerMaps = locals.Build() + + rets := Load(bc, []Func{fn}, "dummy_module", []string{"github.com/bytedance/sonic/dummy.go"}) + println("func address ", *(*unsafe.Pointer)(rets[0])) + // for k, _ := range moduleCache.m { + // spew.Dump(k) + // } + + f := *(*TestFunc)(unsafe.Pointer(&rets[0])) + i := 1 + j := f(&i, hook) + require.Equal(t, 2, j) + require.Equal(t, "hook1", hstr) +} \ No newline at end of file diff --git a/loader/mmap_unix.go b/loader/mmap_unix.go new file mode 100644 index 0000000..50b80bf --- /dev/null +++ b/loader/mmap_unix.go @@ -0,0 +1,45 @@ +//go:build darwin || linux +// +build darwin linux + +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `syscall` +) + +const ( + _AP = syscall.MAP_ANON | syscall.MAP_PRIVATE + _RX = syscall.PROT_READ | syscall.PROT_EXEC + _RW = syscall.PROT_READ | syscall.PROT_WRITE +) + + +func mmap(nb int) uintptr { + if m, _, e := syscall.RawSyscall6(syscall.SYS_MMAP, 0, uintptr(nb), _RW, _AP, 0, 0); e != 0 { + panic(e) + } else { + return m + } +} + +func mprotect(p uintptr, nb int) { + if _, _, err := syscall.RawSyscall(syscall.SYS_MPROTECT, p, uintptr(nb), _RX); err != 0 { + panic(err) + } +} \ No newline at end of file diff --git a/loader/mmap_windows.go b/loader/mmap_windows.go new file mode 100644 index 0000000..1760a71 --- /dev/null +++ b/loader/mmap_windows.go @@ -0,0 +1,84 @@ +//go:build windows +// +build windows + +// build + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +import ( + `syscall` + `unsafe` +) + +const ( + MEM_COMMIT = 0x00001000 + MEM_RESERVE = 0x00002000 +) + +var ( + libKernel32 = syscall.NewLazyDLL("KERNEL32.DLL") + libKernel32_VirtualAlloc = libKernel32.NewProc("VirtualAlloc") + libKernel32_VirtualProtect = libKernel32.NewProc("VirtualProtect") +) + +func mmap(nb int) uintptr { + addr, err := winapi_VirtualAlloc(0, nb, MEM_COMMIT|MEM_RESERVE, syscall.PAGE_READWRITE) + if err != nil { + panic(err) + } + return addr +} + +func mprotect(p uintptr, nb int) (oldProtect int) { + err := winapi_VirtualProtect(p, nb, syscall.PAGE_EXECUTE_READ, &oldProtect) + if err != nil { + panic(err) + } + return +} + +// winapi_VirtualAlloc allocate memory +// Doc: https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualalloc +func winapi_VirtualAlloc(lpAddr uintptr, dwSize int, flAllocationType int, flProtect int) (uintptr, error) { + r1, _, err := libKernel32_VirtualAlloc.Call( + lpAddr, + uintptr(dwSize), + uintptr(flAllocationType), + uintptr(flProtect), + ) + if r1 == 0 { + return 0, err + } + return r1, nil +} + +// winapi_VirtualProtect change memory protection +// Doc: https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualprotect +func winapi_VirtualProtect(lpAddr uintptr, dwSize int, flNewProtect int, lpflOldProtect *int) error { + r1, _, err := libKernel32_VirtualProtect.Call( + lpAddr, + uintptr(dwSize), + uintptr(flNewProtect), + uintptr(unsafe.Pointer(lpflOldProtect)), + ) + if r1 == 0 { + return err + } + return nil +} diff --git a/loader/pcdata.go b/loader/pcdata.go new file mode 100644 index 0000000..b5c62d1 --- /dev/null +++ b/loader/pcdata.go @@ -0,0 +1,100 @@ +/** + * Copyright 2023 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + +const ( + _N_PCDATA = 4 + + _PCDATA_UnsafePoint = 0 + _PCDATA_StackMapIndex = 1 + _PCDATA_InlTreeIndex = 2 + _PCDATA_ArgLiveIndex = 3 + + _PCDATA_INVALID_OFFSET = 0 +) + +const ( + // PCDATA_UnsafePoint values. + PCDATA_UnsafePointSafe = -1 // Safe for async preemption + PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption + + // PCDATA_Restart1(2) apply on a sequence of instructions, within + // which if an async preemption happens, we should back off the PC + // to the start of the sequence when resume. + // We need two so we can distinguish the start/end of the sequence + // in case that two sequences are next to each other. + PCDATA_Restart1 = -3 + PCDATA_Restart2 = -4 + + // Like PCDATA_RestartAtEntry, but back to function entry if async + // preempted. + PCDATA_RestartAtEntry = -5 + + _PCDATA_START_VAL = -1 +) + +var emptyByte byte + +func encodeValue(v int) []byte { + return encodeVariant(toZigzag(v)) +} + +func toZigzag(v int) int { + return (v << 1) ^ (v >> 31) +} + +func encodeVariant(v int) []byte { + var u int + var r []byte + + /* split every 7 bits */ + for v > 127 { + u = v & 0x7f + v = v >> 7 + r = append(r, byte(u) | 0x80) + } + + /* check for last one */ + if v == 0 { + return r + } + + /* add the last one */ + r = append(r, byte(v)) + return r +} + +type Pcvalue struct { + PC uint32 // PC offset from func entry + Val int32 +} + +type Pcdata []Pcvalue + +// see https://docs.google.com/document/d/1lyPIbmsYbXnpNj57a261hgOYVpNRcgydurVQIyZOz_o/pub +func (self Pcdata) MarshalBinary() (data []byte, err error) { + // delta value always starts from -1 + sv := int32(_PCDATA_START_VAL) + sp := uint32(0) + for _, v := range self { + data = append(data, encodeVariant(toZigzag(int(v.Val - sv)))...) + data = append(data, encodeVariant(int(v.PC - sp))...) + sp = v.PC + sv = v.Val + } + return +} \ No newline at end of file diff --git a/internal/loader/funcdata_invalid.go b/loader/stubs.go similarity index 59% rename from internal/loader/funcdata_invalid.go rename to loader/stubs.go index 65e3fdc..71439c4 100644 --- a/internal/loader/funcdata_invalid.go +++ b/loader/stubs.go @@ -1,14 +1,12 @@ -// +build !go1.15 go1.20 - -/* - * Copyright 2021 ByteDance Inc. - * +/** + * Copyright 2023 ByteDance Inc. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +16,20 @@ package loader -// triggers a compilation error -const ( - _ = panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17, 1.18, 1.19") +import ( + _ `unsafe` ) -func registerFunction(_ string, _ uintptr, _ int, _ int, _ uintptr) { - panic("Unsupported Go version. Supported versions are: 1.15, 1.16, 1.17, 1.18, 1.19") +//go:linkname lastmoduledatap runtime.lastmoduledatap +//goland:noinspection GoUnusedGlobalVariable +var lastmoduledatap *moduledata + +func registerModule(mod *moduledata) { + lastmoduledatap.next = mod + lastmoduledatap = mod } + +//go:linkname moduledataverify1 runtime.moduledataverify1 +func moduledataverify1(_ *moduledata) + +