indexer: optimize idx.BlockList performance #7823
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| - stage | |
| - main | |
| - release** | |
| pull_request: | |
| jobs: | |
| job_go_checks: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Print github env vars | |
| run: | | |
| echo github.event_name: ${{ github.event_name }} | |
| echo github.ref: ${{ github.ref }} | |
| echo github.ref_name: ${{ github.ref_name }} | |
| echo github.head_ref: ${{ github.head_ref }} | |
| echo github.base_ref: ${{ github.base_ref }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - name: Tidy go module | |
| run: | | |
| go mod tidy | |
| if [[ $(git status --porcelain) ]]; then | |
| git diff | |
| echo | |
| echo "go mod tidy made these changes, please run 'go mod tidy' and include those changes in a commit" | |
| exit 1 | |
| fi | |
| - name: Run gofumpt | |
| # Run gofumpt first, as it's quick and issues are common. | |
| run: diff -u <(echo -n) <(go run mvdan.cc/[email protected] -d .) | |
| - name: Run go vet | |
| run: go vet ./... | |
| - name: Run go generate | |
| run: | | |
| go generate ./... | |
| if [[ $(git status --porcelain) ]]; then | |
| git diff | |
| echo | |
| echo "go generate made these changes, please run 'go generate ./...' and include those changes in a commit" | |
| exit 1 | |
| fi | |
| - name: Run staticcheck | |
| run: | | |
| go install honnef.co/go/tools/cmd/[email protected] | |
| staticcheck -debug.version | |
| staticcheck ./... 2> staticcheck-stderr | |
| - name: Check staticcheck stderr (this step isn't needed because we are using actions/setup-go@v5 on GitHub hosted runner) | |
| run: | | |
| if cat staticcheck-stderr | grep "matched no packages" ; then | |
| echo "staticcheck step did nothing, due to https://github.com/vocdoni/vocdoni-node/issues/444" | |
| echo "Please re-run job." | |
| # seize the opportunity to fix the underlying problem: a permissions error in ~/.cache | |
| epoch=$(date +%s) | |
| # if any file is reported by find, grep returns true and the mv is done | |
| if [ -d ~/.cache ] && find ~/.cache -not -user `id --user` -print0 | grep -qz . ; then | |
| echo "~/.cache had broken permissions, moving it away... (cache will be rebuilt with usage)" | |
| mv -v ~/.cache ~/.cache-broken-by-root-$epoch | |
| fi | |
| exit 2 | |
| fi | |
| job_go_test: | |
| runs-on: ubuntu-latest | |
| env: | |
| LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: benjlevesque/[email protected] # sets env.SHA to the first 7 chars of github.sha | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - run: mkdir -p "$PWD/gocoverage-unit/" | |
| - name: Run Go test -race | |
| id: go-test-race | |
| # note that -race can easily make the crypto stuff 10x slower | |
| # this is further limited to selected branches at the beginning of this file | |
| if: runner.debug || | |
| github.event_name == 'push' && | |
| github.ref != 'refs/heads/dev' | |
| env: | |
| GORACE: atexit_sleep_ms=10 # the default of 1000 makes every Go package test sleep for 1s; see https://go.dev/issues/20364 | |
| run: go test ./... | |
| -race -timeout=15m -vet=off | |
| -cover -coverpkg=./... -covermode=atomic -args -test.gocoverdir="$PWD/gocoverage-unit/" | |
| - name: Run Go test | |
| if: steps.go-test-race.outcome == 'skipped' | |
| # quicker, non-race test in case it's a PR or push to dev | |
| run: go test ./... | |
| -cover -coverpkg=./... -covermode=count -args -test.gocoverdir="$PWD/gocoverage-unit/" | |
| - name: Store code coverage artifact (unit) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gocoverage-unit@${{ env.SHA }} | |
| path: gocoverage-unit/ | |
| job_compose_test: | |
| runs-on: [self-hosted] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: benjlevesque/[email protected] # sets env.SHA to the first 7 chars of github.sha | |
| - name: Run compose script | |
| env: | |
| TESTSUITE_BUILD_TAG: ${{ github.sha }} | |
| COMPOSE_PROJECT_NAME: testsuite_${{ github.run_id }} # unique name for docker compose (needed for concurrent job runs) | |
| COMPOSE_DVOTE_PORT_MAPPING: "9090" # this binds gateway0 to a random available port on docker host (needed for concurrent job runs) | |
| COMPOSE_HOST_PATH: ${{ github.workspace }}/dockerfiles/testsuite | |
| LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch) | |
| GOCOVERDIR: "./gocoverage-integration/" # collect code coverage when running binaries | |
| CONCURRENT: 1 # run all the start_test.sh tests concurrently | |
| BUILDARGS: "-race" # this makes the integration test only slightly slower (around +10%) unlike the abismal effect in unit test (10x) | |
| run: | | |
| cd dockerfiles/testsuite && ./start_test.sh | |
| - name: Store code coverage artifact (integration) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gocoverage-integration@${{ env.SHA }} | |
| path: dockerfiles/testsuite/gocoverage-integration/ | |
| call-docker-release: | |
| name: Docker | |
| needs: [job_go_checks, job_go_test, job_compose_test] | |
| # docker releases are triggered only on push to the selected branches at the beginning of this file | |
| if: github.event_name == 'push' && | |
| github.ref != 'refs/heads/dev' # but pushes to dev trigger a release via a quicker workflow (deploy-dev.yml) which skips the tests | |
| uses: vocdoni/vocdoni-node/.github/workflows/docker-release.yml@main | |
| secrets: inherit | |
| with: | |
| image-tag: ${{ github.ref_name }} | |
| job_gocoverage_textfmt: | |
| name: Convert coverage (bin->txt) | |
| continue-on-error: true # never mark the whole CI as failed because of this job | |
| needs: [job_go_test, job_compose_test] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: benjlevesque/[email protected] # sets env.SHA to the first 7 chars of github.sha | |
| - uses: actions/download-artifact@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| cache: false | |
| - name: Convert gocoverage format | |
| run: | | |
| go tool covdata textfmt -i=gocoverage-unit@${{ env.SHA }}/ \ | |
| -o gocoverage-unit@${{ env.SHA }}.txt | |
| go tool covdata textfmt -i=gocoverage-integration@${{ env.SHA }}/ \ | |
| -o gocoverage-integration@${{ env.SHA }}.txt | |
| - name: Merge both files | |
| run: | | |
| go install github.com/wadey/gocovmerge@latest | |
| # dirty hack since integration is mode atomic and unit mode count, which are perfectly mergeable | |
| # but gocovmerge doesn't recognize this: "cannot merge profiles with different modes" | |
| sed 's/mode: count/mode: atomic/' gocoverage-unit@${{ env.SHA }}.txt \ | |
| > gocoverage-unit@${{ env.SHA }}.tmp | |
| gocovmerge gocoverage-unit@${{ env.SHA }}.tmp \ | |
| gocoverage-integration@${{ env.SHA }}.txt \ | |
| > gocoverage-merged@${{ env.SHA }}.txt | |
| rm -f gocoverage-unit@${{ env.SHA }}.tmp | |
| - name: Store code coverage artifact (all, textfmt) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gocoverage-all-textfmt@${{ env.SHA }} | |
| path: gocoverage-*.txt | |
| job_gocoverage_coveralls: | |
| name: Publish coverage (Coveralls) | |
| runs-on: ubuntu-latest | |
| needs: [job_gocoverage_textfmt] | |
| continue-on-error: true # never mark the whole CI as failed because of this job | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: benjlevesque/[email protected] # sets env.SHA to the first 7 chars of github.sha | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: gocoverage-all-textfmt@${{ env.SHA }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| cache: false | |
| - name: Send coverage to coveralls.io (unit) | |
| if: ${{ always() }} | |
| uses: shogo82148/actions-goveralls@v1 | |
| with: | |
| path-to-profile: gocoverage-unit@${{ env.SHA }}.txt | |
| flag-name: unit | |
| parallel: true | |
| - name: Send coverage to coveralls.io (integration) | |
| if: ${{ always() }} | |
| uses: shogo82148/actions-goveralls@v1 | |
| with: | |
| path-to-profile: gocoverage-integration@${{ env.SHA }}.txt | |
| flag-name: integration | |
| parallel: true | |
| - name: Tell coveralls.io we're done | |
| if: ${{ always() }} | |
| uses: shogo82148/actions-goveralls@v1 | |
| with: | |
| parallel-finished: true |