From c4cf71a4af77344d5e258a9a868fad424c859ba4 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 28 Sep 2022 11:02:51 -0400 Subject: [PATCH] Added GitHub Actions to build `thegarii` --- .github/ISSUE_TEMPLATE.md | 7 ++ .github/generate_change_log.sh | 36 ++++++ .github/pull_request_template.md | 35 ------ .github/workflows/audit.yml | 13 +++ .github/workflows/check-license.yml | 28 ----- .github/workflows/push.yml | 79 +++++++++++++ .github/workflows/release.yml | 173 ++++++++++++++++++++++++++++ CHANGELOG.md | 7 ++ Dockerfile | 34 +++--- rust-toolchain.toml | 3 + 10 files changed, 331 insertions(+), 84 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100755 .github/generate_change_log.sh create mode 100644 .github/workflows/audit.yml delete mode 100644 .github/workflows/check-license.yml create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 rust-toolchain.toml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..cb67232 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,7 @@ +**Do you want to request a *feature* or report a *bug*?** + +**What is the current behavior?** + +**If the current behavior is a bug, please provide the steps to reproduce and if possible a minimal demo of the problem.** + +**What is the expected behavior?** diff --git a/.github/generate_change_log.sh b/.github/generate_change_log.sh new file mode 100755 index 0000000..c17477a --- /dev/null +++ b/.github/generate_change_log.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +checksum() { + echo $(sha256sum $@ | awk '{print $1}') +} + +change_log_file="./CHANGELOG.md" +version="## $@" +version_prefix="## [0-9]{1,2}\." +start=0 +CHANGE_LOG="" +while read line; do + if [[ $line == *"$version"* ]]; then + start=1 + continue + fi + if [[ $line =~ $version_prefix ]] && [ $start == 1 ]; then + break; + fi + if [ $start == 1 ]; then + CHANGE_LOG+="$line\n" + fi +done < ${change_log_file} + +LINUX_X86_64_BIN_SUM="$(checksum ./linux-x86_64-unknown-linux-gnu)" + +OUTPUT=$(cat <<-END +## Changelog\n +${CHANGE_LOG}\n +## Checksums\n +| Assets | Sha256 Checksum |\n +| :-----------: |------------|\n +| thegarii-linux-x86-64 | ${LINUX_X86_64_BIN_SUM} |\n +END +) + +echo -e ${OUTPUT} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7eb6a04..e69de29 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,35 +0,0 @@ -## Changes - - - -- -- -- - -## Tests - - - -``` - -``` - -## Issues - - - -- diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..e6f3174 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,13 @@ +# See https://github.com/actions-rs/audit-check +name: Security audit +on: + schedule: + - cron: '0 0 */7 * *' +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check-license.yml b/.github/workflows/check-license.yml deleted file mode 100644 index 60bb899..0000000 --- a/.github/workflows/check-license.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check Licence - -on: - pull_request: - branches: - - main - push: - branches: - - main - -jobs: - check-licence: - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v2 - - name: Set up Golang - uses: actions/setup-go@v2 - with: - go-version: '^1.16' - - name: Install addlicense - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go install github.com/google/addlicense@latest - - name: Check license - run: | - export PATH=${PATH}:`go env GOPATH`/bin - addlicense -check -c "ChainSafe Systems" -f ./res/header.txt -y 2021 $(find $PWD -type f -name '*.rs') diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..946da36 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,79 @@ +name: Continuous Integration + +on: + push: + branches: [master, develop] + pull_request: + types: [opened, synchronize, reopened] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + +jobs: + rustfmt: + name: Check rustfmt style + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Rust Toolchain(s) + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: rustfmt + override: true + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: rustfmt-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Check formating + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: fmt + args: --all -- --check + + release-check: + name: Build in release mode + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Rust Toolchain(s) + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: release-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Cargo check (release) + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: check + args: --release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9b28861 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,173 @@ +name: Release + +on: + push: + tags: + - "*" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + name: Build Release + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + rust: [stable] + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: rustfmt + override: true + + - uses: actions/cache@v2 + name: Cache cargo registry + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: release-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Cache LLVM and Clang + uses: actions/cache@v2 + id: cache-llvm + with: + path: | + ./llvm + key: llvm-10 + + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + version: "10" + cached: ${{ steps.cache-llvm.outputs.cache-hit }} + + - name: Build target + uses: actions-rs/cargo@v1 + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '1' + CARGO_PROFILE_RELEASE_LTO: 'fat' + with: + # We cannot use `cross` tool right now. The reason is that we require some + # local libraries, `libclang.so` specifically. The `cross` tool runs a Docker + # container which does not have the library in question. We will need to wait to + # have support of https://github.com/cross-rs/cross/pull/635 to be able to cross + # compile properly. + # use-cross: true + command: build + args: --release + + - name: Upload Build + uses: actions/upload-artifact@v2 + with: + name: linux-x86_64-unknown-linux-gnu + path: ./target/release/thegarii + + release: + name: Release + needs: [build] + runs-on: ubuntu-latest + + permissions: + contents: write + packages: write + + steps: + - name: Set Env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Generate Change Log + id: changelog + if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + chmod 755 ./.github/generate_change_log.sh + CHANGELOG=$(./.github/generate_change_log.sh ${{ env.RELEASE_VERSION }}) + + echo "CHANGELOG<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Download All Artifacts + id: download-artifacts + uses: actions/download-artifact@v2 + with: + path: ./binaries + + - name: Sanitize Downloaded Files + run: | + # We downloaded all the artifacts previously uploaded and we put them in + # the 'binaries' folder. In this folder, the layout is: + # + # binaries + # ├── linux-arm64-unknown-linux-gnu + # │ └── + # └── linux-x86_64-unknown-linux-gnu + # └── + # + # The sub-folder name comes from the 'name' field of the 'actions/upload-artifact@v2' + # step. The '' file name is the filename of the uploaded 'path' field, + # we used './target/release/' in the upload step so the file name here + # is ''. + + download_path="${{steps.download-artifacts.outputs.download-path}}" + chmod +x "${download_path}/linux-x86_64-unknown-linux-gnu/thegarii" + mv "$download_path/linux-x86_64-unknown-linux-gnu/thegarii" "$download_path/thegarii-x86_64-unknown-linux-gnu" + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate docker tags/labels from github build context + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=tag + type=sha,prefix=,enable=true + flavor: | + latest=${{ startsWith(github.ref, 'refs/tags/') }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ${{steps.download-artifacts.outputs.download-path}} + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Create Release + uses: softprops/action-gh-release@v1 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + name: ${{ env.RELEASE_VERSION }} + tag_name: ${{ env.RELEASE_VERSION }} + draft: false + prerelease: false + body: ${{ env.CHANGELOG }} + token: ${{ secrets.GITHUB_TOKEN }} + fail_on_unmatched_files: true + generate_release_notes: true + files: | + ${{steps.download-artifacts.outputs.download-path}}/* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1b329a5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +## 0.2.0 + +- Re-branded under Firehose (remove all usage of `deep mind`, `DMLOG` and other similar names). + + **Important** https://github.com/streamingfast/firehose-arweave/commit/5f34975b59911eefdf0f5c3862b2f01f4d34738a + or older is required as the Firehose logs exchanged format has changed and requires a recent enough version + of `firehose-arweave`. diff --git a/Dockerfile b/Dockerfile index 589f37c..67c30b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,14 @@ -FROM debian:bullseye -RUN apt-get update && apt-get install -y git curl build-essential cmake pkg-config libssl-dev libsqlite3-dev libgmp-dev ncurses-bin libncurses-dev net-tools ufw -WORKDIR /usr/src -RUN git clone --depth=1 -b maint-24 https://github.com/erlang/otp erlang-otp -WORKDIR /usr/src/erlang-otp -RUN ./configure && make -RUN make install -WORKDIR /usr/src/ -RUN git clone -b N.2.5.1.0 --recursive https://github.com/ArweaveTeam/arweave.git -WORKDIR /usr/src/arweave -RUN ./rebar3 as prod tar -WORKDIR /opt/arweave -RUN cp /usr/src/arweave/_build/prod/rel/arweave/arweave-2.5.1.0.tar.gz /opt/arweave -RUN tar -xzvf arweave-2.5.1.0.tar.gz -WORKDIR /opt/ +FROM ubuntu:20.04 -COPY scripts/run_docker.sh /opt/arweave/run.sh -RUN echo -n "fs.file-max=100000000" >> /etc/sysctl.conf -RUN echo -n "DefaultLimitNOFILE=10000000" >> /etc/systemd/user.conf -RUN echo -n "DefaultLimitNOFILE=10000000" >> /etc/systemd/system.conf -EXPOSE 1984 -CMD ["/opt/arweave/run.sh"] +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install -y \ + ca-certificates libssl1.1 vim htop iotop sysstat wget \ + dstat strace lsof curl jq tzdata && \ + rm -rf /var/cache/apt /var/lib/apt/lists/* + +RUN rm /etc/localtime && ln -snf /usr/share/zoneinfo/America/Montreal /etc/localtime && dpkg-reconfigure -f noninteractive tzdata + +COPY thegarii-x86_64-unknown-linux-gnu /app/thegarii +RUN chmod +x /app/thegarii + +ENV PATH "$PATH:/app" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..45d559f --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.64.0" +components = [ "rustfmt" ] \ No newline at end of file