From 0f53ef4f0d937a4fc8225c8de4c1a075b3ddba05 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Mon, 9 Sep 2024 19:13:04 +0800 Subject: [PATCH 1/6] build:(loader) make loader buildable under unsupported version --- loader/funcdata_compat.go | 4 +- .../{funcdata_go116.go => funcdata_go117.go} | 0 loader/loader_compat.go | 42 +++++ loader/loader_go116_test.go | 146 ------------------ 4 files changed, 44 insertions(+), 148 deletions(-) rename loader/{funcdata_go116.go => funcdata_go117.go} (100%) create mode 100644 loader/loader_compat.go delete mode 100644 loader/loader_go116_test.go diff --git a/loader/funcdata_compat.go b/loader/funcdata_compat.go index ca2f735ea..17c840613 100644 --- a/loader/funcdata_compat.go +++ b/loader/funcdata_compat.go @@ -1,5 +1,5 @@ -//go:build !go1.17 -// +build !go1.17 +//go:build !go1.17 || go1.24 +// +build !go1.17 go1.24 /* * Copyright 2021 ByteDance Inc. diff --git a/loader/funcdata_go116.go b/loader/funcdata_go117.go similarity index 100% rename from loader/funcdata_go116.go rename to loader/funcdata_go117.go diff --git a/loader/loader_compat.go b/loader/loader_compat.go new file mode 100644 index 000000000..9fe69e120 --- /dev/null +++ b/loader/loader_compat.go @@ -0,0 +1,42 @@ +// +build !go1.17 go1.24 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loader + + +// LoadFuncs loads only one function as module, and returns the function pointer +// - text: machine code +// - funcName: function name +// - frameSize: stack frame size. +// - argSize: argument total size (in bytes) +// - argPtrs: indicates if a slot (8 Bytes) of arguments memory stores pointer, from low to high +// - localPtrs: indicates if a slot (8 Bytes) of local variants memory stores pointer, from low to high +// +// WARN: +// - the function MUST has fixed SP offset equaling to this, otherwise it go.gentraceback will fail +// - the function MUST has only one stack map for all arguments and local variants +func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function { + panic("not implemented") +} + +// Load loads given machine codes and corresponding function information into go moduledata +// and returns runnable function pointer +// WARN: this API is experimental, use it carefully +func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) { + panic("not implemented") +} diff --git a/loader/loader_go116_test.go b/loader/loader_go116_test.go deleted file mode 100644 index cd12f4c49..000000000 --- a/loader/loader_go116_test.go +++ /dev/null @@ -1,146 +0,0 @@ -//go:build go1.17 && !go1.17 -// +build go1.17,!go1.17 - -/* - * Copyright 2021 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package loader - -import ( - `fmt` - `runtime` - `runtime/debug` - `strconv` - `testing` - `unsafe` - - `github.com/bytedance/sonic/loader/internal/rt` - `github.com/stretchr/testify/require` -) - -func TestLoad(t *testing.T) { - // defer func() { - // if r := recover(); r != nil { - // runtime.GC() - // if r != "hook1" { - // t.Fatal("not right panic:" + r.(string)) - // } - // } else { - // t.Fatal("not panic") - // } - // }() - - var hstr string - - type TestFunc func(i *int, hook func(i *int)) int - var hook = func(i *int) { - runtime.GC() - debug.FreeOSMemory() - hstr = ("hook" + strconv.Itoa(*i)) - runtime.GC() - debug.FreeOSMemory() - } - // var f TestFunc = func(i *int, hook func(i *int)) int { - // var t = *i - // hook(i) - // return t + *i - // } - bc := []byte { - 0x48, 0x83, 0xec, 0x18, // (0x00) subq $24, %rsp - 0x48, 0x89, 0x6c, 0x24, 0x10, // (0x04) movq %rbp, 16(%rsp) - 0x48, 0x8d, 0x6c, 0x24, 0x10, // (0x09) leaq 16(%rsp), %rbp - 0x48, 0x8b, 0x44, 0x24, 0x20, // (0x0e) movq 32(%rsp), %rax - 0x48, 0x8b, 0x08, // (0x13) movq (%rax), %rcx - 0x48, 0x89, 0x4c, 0x24, 0x08, // (0x16) movq %rcx, 8(%rsp) - 0x48, 0x8b, 0x54, 0x24, 0x28, // (0x1b) movq 40(%rsp), %rdx - 0x48, 0x8b, 0x1a, // (0x20) movq (%rdx), %rbx - 0x48, 0x89, 0x04, 0x24, // (0x23) movq %rax, (%rsp) - 0xff, 0xd3, // (0x27) callq %rbx - 0x48, 0x8b, 0x44, 0x24, 0x08, // (0x29) movq 8(%rsp), %rax - 0x48, 0x8b, 0x4c, 0x24, 0x20, // (0x2e) movq 32(%rsp), %rcx - 0x48, 0x03, 0x01, // (0x33) addq (%rcx), %rax - 0x48, 0x89, 0x44, 0x24, 0x30, // (0x36) movq %rax, 48(%rsp) - 0x48, 0x8b, 0x6c, 0x24, 0x10, // (0x3b) movq 16(%rsp), %rbp - 0x48, 0x83, 0xc4, 0x18, // (0x40) addq $24, %rsp - 0xc3, // (0x44) ret - } - - size := uint32(len(bc)) - fn := Func{ - ID: 0, - Flag: 0, - ArgsSize: 16, - EntryOff: 0, - TextSize: size, - DeferReturn: 0, - FileIndex: 0, - Name: "dummy", - } - - fn.Pcsp = &Pcdata{ - {PC: size, Val: 24}, - } - - fn.Pcline = &Pcdata{ - {PC: 0x13, Val: 0}, - {PC: 0x1b, Val: 1}, - {PC: 0x23, Val: 2}, - {PC: size, Val: 3}, - } - - fn.Pcfile = &Pcdata{ - {PC: size, Val: 0}, - } - - fn.PcUnsafePoint = &Pcdata{ - {PC: size, Val: PCDATA_UnsafePointUnsafe}, - } - - fn.PcStackMapIndex = &Pcdata{ - {PC: size, Val: 0}, - } - - args := rt.StackMapBuilder{} - args.AddField(true) - args.AddField(true) - fn.ArgsPointerMaps = args.Build() - ab, _ := fn.ArgsPointerMaps.MarshalBinary() - fmt.Printf("args: %+v\n", ab) - - locals := rt.StackMapBuilder{} - locals.AddField(false) - locals.AddField(false) - fn.LocalsPointerMaps = locals.Build() - lb, _ := fn.LocalsPointerMaps.MarshalBinary() - fmt.Printf("locals: %+v\n", lb) - - rets := Load(bc, []Func{fn}, "dummy_module", []string{"github.com/bytedance/sonic/dummy.go"}) - println("func address ", *(*unsafe.Pointer)(rets[0])) - // for k, _ := range moduleCache.m { - // spew.Dump(k) - // } - f := *(*TestFunc)(unsafe.Pointer(&rets[0])) - i := 1 - j := f(&i, hook) - require.Equal(t, 2, j) - require.Equal(t, "hook1", hstr) - - fi := runtime.FuncForPC(*(*uintptr)(rets[0])) - require.Equal(t, "dummy", fi.Name()) - file, line := fi.FileLine(0) - require.Equal(t, "github.com/bytedance/sonic/dummy.go", file) - require.Equal(t, 0, line) -} From dcdd2dda21cdda2d03f27155b497051cef5f404b Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 10 Sep 2024 17:05:53 +0800 Subject: [PATCH 2/6] fix: base64x --- internal/base64/b64_amd64.go | 2 +- internal/base64/b64_compat.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/base64/b64_amd64.go b/internal/base64/b64_amd64.go index 01f99f931..43db80d8a 100644 --- a/internal/base64/b64_amd64.go +++ b/internal/base64/b64_amd64.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16 +// +build amd64,go1.17,!go1.24 /** * Copyright 2023 ByteDance Inc. diff --git a/internal/base64/b64_compat.go b/internal/base64/b64_compat.go index ba8f8b562..6688faa20 100644 --- a/internal/base64/b64_compat.go +++ b/internal/base64/b64_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 +// +build !amd64 !go1.17 go1.24 /* * Copyright 2022 ByteDance Inc. From 09f25ea40c8184d1c65101c675d3856a04796fac Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 10 Sep 2024 17:41:55 +0800 Subject: [PATCH 3/6] tmp --- loader/loader_compat.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/loader/loader_compat.go b/loader/loader_compat.go index 9fe69e120..50a4e089a 100644 --- a/loader/loader_compat.go +++ b/loader/loader_compat.go @@ -19,24 +19,12 @@ package loader -// LoadFuncs loads only one function as module, and returns the function pointer -// - text: machine code -// - funcName: function name -// - frameSize: stack frame size. -// - argSize: argument total size (in bytes) -// - argPtrs: indicates if a slot (8 Bytes) of arguments memory stores pointer, from low to high -// - localPtrs: indicates if a slot (8 Bytes) of local variants memory stores pointer, from low to high -// -// WARN: -// - the function MUST has fixed SP offset equaling to this, otherwise it go.gentraceback will fail -// - the function MUST has only one stack map for all arguments and local variants +// Deprecated: not use it func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function { - panic("not implemented") + println("sonic/loader for high go version is not implemented") } -// Load loads given machine codes and corresponding function information into go moduledata -// and returns runnable function pointer -// WARN: this API is experimental, use it carefully +// Deprecated: not use it func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) { - panic("not implemented") + println("sonic/loader for high go version is not implemented") } From 83842ae59d230758425bb907efebdc71d628a046 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 10 Sep 2024 18:24:51 +0800 Subject: [PATCH 4/6] fix --- loader/loader_compat.go | 30 ------------------------------ loader/loader_latest.go | 1 - 2 files changed, 31 deletions(-) delete mode 100644 loader/loader_compat.go diff --git a/loader/loader_compat.go b/loader/loader_compat.go deleted file mode 100644 index 50a4e089a..000000000 --- a/loader/loader_compat.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build !go1.17 go1.24 - -/* - * Copyright 2021 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package loader - - -// Deprecated: not use it -func (self Loader) LoadOne(text []byte, funcName string, frameSize int, argSize int, argPtrs []bool, localPtrs []bool) Function { - println("sonic/loader for high go version is not implemented") -} - -// Deprecated: not use it -func Load(text []byte, funcs []Func, modulename string, filenames []string) (out []Function) { - println("sonic/loader for high go version is not implemented") -} diff --git a/loader/loader_latest.go b/loader/loader_latest.go index 71431227a..f2c78989d 100644 --- a/loader/loader_latest.go +++ b/loader/loader_latest.go @@ -1,4 +1,3 @@ -// +build go1.17,!go1.24 /* * Copyright 2021 ByteDance Inc. From 04197e764ff74a1a0aba57e50c9be3b2d39a1c9d Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 15 Oct 2024 11:03:39 +0800 Subject: [PATCH 5/6] compatible for other cpu --- internal/rt/asm_amd64.s | 2 +- internal/rt/{asm_arm64.s => asm_compat.s} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename internal/rt/{asm_arm64.s => asm_compat.s} (80%) diff --git a/internal/rt/asm_amd64.s b/internal/rt/asm_amd64.s index 86d264230..48a564bee 100644 --- a/internal/rt/asm_amd64.s +++ b/internal/rt/asm_amd64.s @@ -1,4 +1,4 @@ -// +build !noasm !appengine +// +build !noasm,amd64 !appengine,amd64 // Code generated by asm2asm, DO NOT EDIT· #include "go_asm.h" diff --git a/internal/rt/asm_arm64.s b/internal/rt/asm_compat.s similarity index 80% rename from internal/rt/asm_arm64.s rename to internal/rt/asm_compat.s index a168a8266..c807d8343 100644 --- a/internal/rt/asm_arm64.s +++ b/internal/rt/asm_compat.s @@ -1,4 +1,4 @@ -// +build !noasm !appengine +// +build !noasm,!amd64 !appengine,!amd64 // Code generated by asm2asm, DO NOT EDIT. #include "go_asm.h" From e55bd38eb3053b15da8603994eb20ade154d5092 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 15 Oct 2024 11:19:12 +0800 Subject: [PATCH 6/6] build: fixed golang.org/x/tools/cmd/file2fuzz to v0.10.0 --- fuzz/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/Makefile b/fuzz/Makefile index b85b82718..eba60be83 100644 --- a/fuzz/Makefile +++ b/fuzz/Makefile @@ -5,7 +5,7 @@ fuzz: mkdir -p ${corpusdir} rm -rf ./go-fuzz-corpus git clone https://github.com/dvyukov/go-fuzz-corpus.git ./go-fuzz-corpus/ - go install golang.org/x/tools/cmd/file2fuzz@latest + go install golang.org/x/tools/cmd/file2fuzz@v0.10.0 file2fuzz -o ${corpusdir} ./go-fuzz-corpus/json/corpus/* ./corpus/* run: