Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support aarch64, used for tests #646

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ on: pull_request

jobs:
build:
runs-on: [self-hosted, X64]
strategy:
matrix:
os: [X64, arm]
runs-on: ${{ matrix.os }}
steps:
- name: Clear repository
run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
Expand Down
12 changes: 9 additions & 3 deletions .github/workflows/compatibility_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ jobs:
build:
strategy:
matrix:
go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
os: [arm, X64]
runs-on: ${{ matrix.os }}
steps:
Expand All @@ -28,10 +28,16 @@ jobs:
${{ runner.os }}-go-

- name: main
run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic
run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic

- name: decoder
run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/decoder

- name: encoder
run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/encoder

- name: ast
run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic/ast
run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/ast

- name: qemu
run: sh scripts/qemu.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
name: Benchmark Linux-ARM
name: Fuzz Test Linux-X64

on: pull_request

jobs:
build:
runs-on: [arm]
strategy:
matrix:
os: [arm, X64]
runs-on: ${{ matrix.os }}
steps:
- name: Clear repository
run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
Expand All @@ -14,7 +17,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.22
go-version: 1.20.x

- uses: actions/cache@v2
with:
Expand All @@ -23,5 +26,8 @@ jobs:
restore-keys: |
${{ runner.os }}-go-

- name: Benchmark sonic
run: sh scripts/bench-arm.sh
- name: Fuzz sonic
run: |
cd ./fuzz
make fuzz
make runopt
7 changes: 5 additions & 2 deletions .github/workflows/fuzzing-linux-x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ on: pull_request

jobs:
build:
runs-on: [self-hosted, X64]
strategy:
matrix:
os: [arm, X64]
runs-on: ${{ matrix.os }}
steps:
- name: Clear repository
run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
Expand All @@ -14,7 +17,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
go-version: 1.20.x

- uses: actions/cache@v2
with:
Expand Down
42 changes: 42 additions & 0 deletions .github/workflows/unit_test-linux-arm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Unit Test Linux arm

on: push

jobs:
build:
strategy:
matrix:
# TODO: 1.17.x, 1.18.x, 1.19.x not supported because golang asm bug
go-version: [1.20.x, 1.21.x, 1.22.x]
runs-on: [arm]
steps:
- name: Clear repository
run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE

- uses: actions/checkout@v2
with:
fetch-depth: 0

- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}

- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-

- name: Unit Test
run: |
go test -race -covermode=atomic -coverprofile=coverage.txt $(go list ./... | grep -v -E 'loader|jit|avx|x86|sse')

- name: external
run: |
cd ./external_jsonlib_test
GOMAXPROCS=4 go test -v -race ./...

- name: Codecov
run: bash <(curl -s https://codecov.io/bash)
46 changes: 46 additions & 0 deletions .github/workflows/unit_test-linux-opt-x64.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Unit Test Linux X64

on: push

jobs:
build:
strategy:
matrix:
go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
runs-on: [self-hosted, X64]
steps:
- name: Clear repository
run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE

- uses: actions/checkout@v2
with:
fetch-depth: 0

- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}

- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-

- name: Unit Test
run: |
GOMAXPROCS=4 SONIC_USE_OPTDEC=1 go test -race -covermode=atomic -coverprofile=coverage.txt ./...

- name: external
run: |
cd ./external_jsonlib_test
SONIC_USE_OPTDEC=1 go test -v -race ./...

- name: external
run: |
cd ./loader
SONIC_USE_OPTDEC=1 go test -v -race ./...

- name: Codecov
run: bash <(curl -s https://codecov.io/bash)
10 changes: 5 additions & 5 deletions .github/workflows/unit_test-linux-x64.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
name: Unit Test Linux-X64
name: Unit Test Linux X64

on: push

jobs:
build:
strategy:
matrix:
go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
runs-on: [self-hosted, X64]
steps:
- name: Clear repository
Expand All @@ -30,17 +30,17 @@ jobs:

- name: Unit Test
run: |
go test -race -covermode=atomic -coverprofile=coverage.txt ./...
GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage.txt ./...

- name: external
run: |
cd ./external_jsonlib_test
GOMAXPROCS=4 go test -v -race ./...
go test -v -race ./...

- name: external
run: |
cd ./loader
GOMAXPROCS=4 go test -v -race ./...
go test -v -race ./...

- name: Codecov
run: bash <(curl -s https://codecov.io/bash)
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,6 @@ ast/bench.sh

!testdata/*.json.gz
fuzz/testdata
*__debug_bin
*__debug_bin*
*pprof
*coverage.txt
4 changes: 2 additions & 2 deletions ast/api.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23

/*
* Copyright 2022 ByteDance Inc.
Expand Down
4 changes: 2 additions & 2 deletions ast/api_compat.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build !amd64,!arm64 go1.23 !go1.16 arm64,!go1.20
// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20

/*
* Copyright 2022 ByteDance Inc.
Expand Down Expand Up @@ -27,7 +27,7 @@ import (
)

func init() {
println("WARNING:(ast) sonic only supports Go1.16~1.22, but your environment is not suitable")
println("WARNING:(ast) sonic only supports go1.17~1.22, but your environment is not suitable")
}

func quote(buf *[]byte, val string) {
Expand Down
4 changes: 2 additions & 2 deletions ast/api_native_test.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23

/*
* Copyright 2022 ByteDance Inc.
Expand Down
2 changes: 1 addition & 1 deletion ast/b64_amd64.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build amd64,go1.16
// +build amd64,go1.17

/**
* Copyright 2023 ByteDance Inc.
Expand Down
2 changes: 1 addition & 1 deletion ast/b64_compat.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build !amd64 !go1.16
// +build !amd64 !go1.17

/*
* Copyright 2022 ByteDance Inc.
Expand Down
2 changes: 1 addition & 1 deletion compat.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build !amd64 !go1.16 go1.23
// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20

/*
* Copyright 2021 ByteDance Inc.
Expand Down
6 changes: 4 additions & 2 deletions decode_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// +build amd64,go1.16,!go1.23
//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23

/*
* Copyright 2021 ByteDance Inc.
Expand Down Expand Up @@ -458,6 +459,8 @@ var unmarshalTests = []unmarshalTest{
{in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true},
{in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64},
{in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true},
{in: `{"":""}`, ptr: new(struct{}), out: struct{}{}},
{in: `{"x":""}`, ptr: new(struct{ X json.Number }), err: errors.New("empty string into json number")},

// raw values with whitespace
{in: "\n true ", ptr: new(bool), out: true},
Expand Down Expand Up @@ -1116,7 +1119,6 @@ func TestMarshalEmbeds(t *testing.T) {

func TestUnmarshal(t *testing.T) {
for i, tt := range unmarshalTests {
t.Log(i, tt.in)
if !json.Valid([]byte(tt.in)) {
continue
}
Expand Down
4 changes: 2 additions & 2 deletions decoder/decoder_compat.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build !amd64 !go1.16 go1.23
// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20

/*
* Copyright 2023 ByteDance Inc.
Expand Down Expand Up @@ -30,7 +30,7 @@ import (
)

func init() {
println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable")
println("WARNING: sonic/decoder only supports (Go1.17~1.22 && CPU amd64) or (go1.20~1.22 && CPU arm64), but your environment is not suitable")
}

const (
Expand Down
36 changes: 19 additions & 17 deletions decoder/decoder_amd64.go → decoder/decoder_native.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
// +build amd64,go1.16,!go1.23
//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23


/*
* Copyright 2023 ByteDance Inc.
Expand All @@ -19,50 +21,50 @@
package decoder

import (
`github.com/bytedance/sonic/internal/decoder`
`github.com/bytedance/sonic/internal/decoder/api`
)

// Decoder is the decoder context object
type Decoder = decoder.Decoder
type Decoder = api.Decoder

// SyntaxError represents json syntax error
type SyntaxError = decoder.SyntaxError
type SyntaxError = api.SyntaxError

// MismatchTypeError represents dismatching between json and object
type MismatchTypeError = decoder.MismatchTypeError
type MismatchTypeError = api.MismatchTypeError

// Options for decode.
type Options = decoder.Options
type Options = api.Options

const (
OptionUseInt64 Options = decoder.OptionUseInt64
OptionUseNumber Options = decoder.OptionUseNumber
OptionUseUnicodeErrors Options = decoder.OptionUseUnicodeErrors
OptionDisableUnknown Options = decoder.OptionDisableUnknown
OptionCopyString Options = decoder.OptionCopyString
OptionValidateString Options = decoder.OptionValidateString
OptionUseInt64 Options = api.OptionUseInt64
OptionUseNumber Options = api.OptionUseNumber
OptionUseUnicodeErrors Options = api.OptionUseUnicodeErrors
OptionDisableUnknown Options = api.OptionDisableUnknown
OptionCopyString Options = api.OptionCopyString
OptionValidateString Options = api.OptionValidateString
)

// StreamDecoder is the decoder context object for streaming input.
type StreamDecoder = decoder.StreamDecoder
type StreamDecoder = api.StreamDecoder

var (
// NewDecoder creates a new decoder instance.
NewDecoder = decoder.NewDecoder
NewDecoder = api.NewDecoder

// NewStreamDecoder adapts to encoding/json.NewDecoder API.
//
// NewStreamDecoder returns a new decoder that reads from r.
NewStreamDecoder = decoder.NewStreamDecoder
NewStreamDecoder = api.NewStreamDecoder

// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in
// order to reduce the first-hit latency.
//
// Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is
// a compile option to set the depth of recursive compile for the nested struct type.
Pretouch = decoder.Pretouch
Pretouch = api.Pretouch

// Skip skips only one json value, and returns first non-blank character position and its ending position if it is valid.
// Otherwise, returns negative error code using start and invalid character position using end
Skip = decoder.Skip
Skip = api.Skip
)
Loading
Loading