diff --git a/.cargo/config.toml b/.cargo/config.toml index cb6e6d707..2bae7a236 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,8 @@ +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" + +[target.riscv64gc-unknown-linux-gnu] +linker = "riscv64-linux-gnu-gcc" [alias] ci-fmt = "fmt --all -- --check" @@ -6,4 +11,4 @@ ci-clippy = "lints clippy --all-targets --all-features" ci-test-compile = "test --no-run --workspace --all-features --no-default-features" ci-test = "nextest run --all-features --release --workspace --exclude integration_tests --profile ci" ci-cucumber = "test --all-features --release --package integration_tests" -ci-check = "check --workspace --release --all-features --all-targets --locked" \ No newline at end of file +ci-check = "check --workspace --release --all-features --all-targets --locked" diff --git a/.github/workflows/build_binaries.json b/.github/workflows/build_binaries.json index 40ef4ca74..493ca1404 100644 --- a/.github/workflows/build_binaries.json +++ b/.github/workflows/build_binaries.json @@ -13,8 +13,9 @@ "rust": "1.77", "target": "aarch64-unknown-linux-gnu", "cross": true, - "features": "", - "build_enabled": false + "flags": "--workspace --exclude tari_integration_tests", + "build_enabled": true, + "best_effort": true }, { "name": "linux-riscv64", @@ -22,14 +23,14 @@ "rust": "stable", "target": "riscv64gc-unknown-linux-gnu", "cross": true, - "flags": "--workspace --exclude --exclude tari_integration_tests", + "flags": "--workspace --exclude tari_integration_tests", "build_enabled": true, "best_effort": true }, { "name": "macos-x86_64", - "runs-on": "macos-11", + "runs-on": "macos-12", "rust": "1.77", "target": "x86_64-apple-darwin", "cross": false, @@ -61,6 +62,7 @@ "cross": false, "target_bins": "tari_dan_wallet_cli, tari_dan_wallet_daemon, tari_indexer, tari_validator_node, tari_signaling_server", "features": "", - "build_enabled": false + "build_enabled": true, + "best_effort": true } ] diff --git a/.github/workflows/build_binaries.yml b/.github/workflows/build_binaries.yml index 7f8001122..0be6eb900 100644 --- a/.github/workflows/build_binaries.yml +++ b/.github/workflows/build_binaries.yml @@ -221,6 +221,7 @@ jobs: # Should already be installed # choco upgrade strawberryperl -y choco upgrade protoc -y + rustup target add ${{ matrix.builds.target }} - name: Set environment variables - Nix if: ${{ ! startsWith(runner.os,'Windows') }} @@ -287,7 +288,8 @@ jobs: if: ${{ matrix.builds.cross }} shell: bash run: | - cargo install cross + #cargo install cross + cargo install cross --git https://github.com/cross-rs/cross echo "CARGO=cross" >> $GITHUB_ENV - name: Install and setup cargo-auditable @@ -307,6 +309,7 @@ jobs: echo "cross flag: ${{ matrix.builds.cross }}" - name: Build release binaries + timeout-minutes: 30 shell: bash run: | ${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \ @@ -531,6 +534,203 @@ jobs: name: ${{ env.TS_FILENAME }}_archive-${{ matrix.builds.name }} path: "${{ github.workspace }}${{ env.TS_DIST }}/${{ env.BINFILE }}.zip*" + macOS-universal-assemble: + name: macOS universal assemble + needs: builds + + env: + TARI_VERSION: ${{ needs.builds.outputs.TARI_VERSION }} + VSHA_SHORT: ${{ needs.builds.outputs.VSHA_SHORT }} + SHARUN: "shasum --algorithm 256" + + continue-on-error: true + + runs-on: macos-14 + + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Download macOS binaries + uses: actions/download-artifact@v4 + with: + path: osxuni + # macos - x86_64 / arm64 + pattern: ${{ env.TS_FILENAME }}_archive-macos-* + merge-multiple: true + + - name: Set environment variables for macOS universal + shell: bash + run: | + BINFN="${TS_FILENAME}-${TARI_VERSION}-${VSHA_SHORT}" + echo "BINFN=${BINFN}" >> $GITHUB_ENV + + - name: Install macOS dependencies + shell: bash + run: | + brew install coreutils + + - name: Verify checksums and extract + shell: bash + working-directory: osxuni + run: | + ls -alhtR + ${SHARUN} --ignore-missing --check \ + "${{ env.BINFN }}-macos-x86_64.zip.sha256" + ${SHARUN} --ignore-missing --check \ + "${{ env.BINFN }}-macos-arm64.zip.sha256" + ls -alhtR + mkdir macos-universal macos-x86_64 macos-arm64 + cd macos-x86_64 + 7z e "../${{ env.BINFN }}-macos-x86_64.zip" + cd ../macos-arm64 + 7z e "../${{ env.BINFN }}-macos-arm64.zip" + + - name: Assemble macOS universal binaries + shell: bash + working-directory: osxuni + run: | + ls -alhtR + ARRAY_FILES=( $(echo ${TS_FILES} | jq --raw-output '.[]' | awk '{ print $1 }') ) + for FILE in "${ARRAY_FILES[@]}"; do + echo "processing binary file - ${FILE}" + lipo -create -output macos-universal/${FILE} \ + macos-x86_64/${FILE} \ + macos-arm64/${FILE} + done + ARRAY_LIBS=( $(echo ${TS_LIBRARIES} | tr ', ' '\n') ) + for FILE in "${ARRAY_LIBS[@]}"; do + echo "processing library file - lib${FILE}.dylib" + lipo -create -output macos-universal/lib${FILE}.dylib \ + macos-x86_64/lib${FILE}.dylib \ + macos-arm64/lib${FILE}.dylib + done + ls -alhtR macos-universal + + - name: Build the macOS universal pkg + # Disabled + if: ${{ false }} + continue-on-error: true + env: + MACOS_KEYCHAIN_PASS: ${{ secrets.MACOS_KEYCHAIN_PASS }} + MACOS_APPLICATION_ID: ${{ secrets.MACOS_APPLICATION_ID }} + MACOS_APPLICATION_CERT: ${{ secrets.MACOS_APPLICATION_CERT }} + MACOS_APPLICATION_PASS: ${{ secrets.MACOS_APPLICATION_PASS }} + MACOS_INSTALLER_ID: ${{ secrets.MACOS_INSTALLER_ID }} + MACOS_INSTALLER_CERT: ${{ secrets.MACOS_INSTALLER_CERT }} + MACOS_INSTALLER_PASS: ${{ secrets.MACOS_INSTALLER_PASS }} + MACOS_NOTARIZE_USERNAME: ${{ secrets.MACOS_NOTARIZE_USERNAME }} + MACOS_NOTARIZE_PASSWORD: ${{ secrets.MACOS_NOTARIZE_PASSWORD }} + MACOS_ASC_PROVIDER: ${{ secrets.MACOS_ASC_PROVIDER }} + run: | + echo $MACOS_APPLICATION_CERT | base64 --decode > application.p12 + echo $MACOS_INSTALLER_CERT | base64 --decode > installer.p12 + security create-keychain -p $MACOS_KEYCHAIN_PASS build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p $MACOS_KEYCHAIN_PASS build.keychain + security import application.p12 -k build.keychain -P $MACOS_APPLICATION_PASS -T /usr/bin/codesign + security import installer.p12 -k build.keychain -P $MACOS_INSTALLER_PASS -T /usr/bin/pkgbuild + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PASS build.keychain + OSX_CODESIGN_EXTRAS="--entitlements ${GITHUB_WORKSPACE}/applications/minotari_node/osx-pkg/entitlements.xml" + cd buildtools + # export target_release="target/${{ matrix.builds.target }}/release" + # matrix.builds.target=macos-universal + # matrix.builds.name=macos-universal + export target_release="osxuni/macos-universal" + mkdir -p "${{ runner.temp }}/osxpkg" + export tarball_parent="${{ runner.temp }}/osxpkg" + export tarball_source="${{ env.TARI_NETWORK_DIR }}" + ./create_osx_install_zip.sh unused nozip + ARRAY_FILES=( $(echo ${TS_FILES} | jq --raw-output '.[]' | awk '{ print $1 }') ) + for FILE in "${ARRAY_FILES[@]}"; do + codesign --options runtime --force --verify --verbose --timestamp ${OSX_CODESIGN_EXTRAS} \ + --prefix "${{ env.TS_BUNDLE_ID_BASE }}.${{ env.TS_FILENAME }}." \ + --sign "Developer ID Application: $MACOS_APPLICATION_ID" \ + "${{ runner.temp }}/osxpkg/${{ env.TARI_NETWORK_DIR }}/runtime/$FILE" + codesign --verify --deep --display --verbose=4 \ + "${{ runner.temp }}/osxpkg/${{ env.TARI_NETWORK_DIR }}/runtime/$FILE" + cp -vf "${{ runner.temp }}/osxpkg/${{ env.TARI_NETWORK_DIR }}/runtime/$FILE" \ + "${{ github.workspace }}/osxuni/macos-universal/" + done + distDirPKG=$(mktemp -d -t ${{ env.TS_FILENAME }}) + echo "${distDirPKG}" + echo "distDirPKG=${distDirPKG}" >> $GITHUB_ENV + TS_Temp=${{ env.TS_FILENAME }} + TS_BUNDLE_ID_VALID_NAME=$(echo "${TS_Temp//_/-}") + TS_ARCH=universal + pkgbuild --root "${{ runner.temp }}/osxpkg/${{ env.TARI_NETWORK_DIR }}" \ + --identifier "${{ env.TS_BUNDLE_ID_BASE }}.pkg.${TS_BUNDLE_ID_VALID_NAME}" \ + --version "${TARI_VERSION}" \ + --install-location "/tmp/tari" \ + --scripts "${{ runner.temp }}/osxpkg/${{ env.TARI_NETWORK_DIR }}/scripts" \ + --sign "Developer ID Installer: ${MACOS_INSTALLER_ID}" \ + "${distDirPKG}/${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg" + echo -e "Submitting to Apple...\n\n" + xcrun notarytool submit \ + "${distDirPKG}/${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg" \ + --apple-id "${MACOS_NOTARIZE_USERNAME}" \ + --password ${MACOS_NOTARIZE_PASSWORD} \ + --team-id ${MACOS_ASC_PROVIDER} \ + --verbose --wait 2>&1 | tee -a notarisation.result + # Maybe use line from with "Processing complete"? + requestUUID=$(tail -n5 notarisation.result | grep "id:" | cut -d" " -f 4) + requestSTATUS=$(tail -n5 notarisation.result | grep "\ \ status:" | cut -d" " -f 4) + if [[ ${requestUUID} == "" ]] || [[ ${requestSTATUS} != "Accepted" ]]; then + echo "## status: ${requestSTATUS} - could not notarize - ${requestUUID} - ${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg" + exit 1 + else + echo "Notarization RequestUUID: ${requestUUID}" + echo -e "\nStapling package...\ + ${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg\n" + xcrun stapler staple -v \ + "${distDirPKG}/${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg" + fi + cd ${distDirPKG} + echo "Compute pkg shasum" + ${SHARUN} "${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg" \ + >> "${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg.sha256" + cat "${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg.sha256" + echo "Checksum verification for pkg is " + ${SHARUN} --check "${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg.sha256" + + - name: Artifact upload for macOS universal pkg + # Disabled + if: ${{ false }} + # if: startsWith(runner.os,'macOS') + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}.pkg + path: "${{ env.distDirPKG }}/${{ env.TS_FILENAME }}-macos-universal-${{ env.TARI_VERSION }}*.pkg*" + + - name: Archive and Checksum macOS universal Binaries + shell: bash + working-directory: osxuni/macos-universal + run: | + # set -xo pipefail + BINFILE="${BINFN}-macos-universal" + echo "BINFILE=${BINFILE}" >> $GITHUB_ENV + echo "Archive ${BINFILE} into ${BINFILE}.zip" + echo "Compute files shasum into ${BINFILE}.sha256" + ${SHARUN} * >> "${BINFILE}.sha256" + echo "Show the shasum" + cat "${BINFILE}.sha256" + echo "Checksum verification for files is " + ${SHARUN} --check "${BINFILE}.sha256" + 7z a "${BINFILE}.zip" * + echo "Compute archive shasum into ${BINFILE}.zip.sha256" + ${SHARUN} "${BINFILE}.zip" >> "${BINFILE}.zip.sha256" + echo "Show the shasum from ${BINFILE}.zip.sha256" + cat "${BINFILE}.zip.sha256" + echo "Checksum verification archive is " + ${SHARUN} --check "${BINFILE}.zip.sha256" + + - name: Artifact upload for Archive + uses: actions/upload-artifact@v4 + with: + name: ${{ env.TS_FILENAME }}_archive-macos-universal + path: "${{ github.workspace }}/osxuni/macos-universal/${{ env.BINFILE }}.zip*" + create-release: if: ${{ startsWith(github.ref, 'refs/tags/v') }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0819e3f8c..265b8f105 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ --- name: CI -"on": +'on': workflow_dispatch: push: branches: @@ -15,6 +15,11 @@ name: CI - synchronize merge_group: +concurrency: + # https://docs.github.com/en/actions/examples/using-concurrency-expressions-and-a-test-matrix + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') || github.ref != 'refs/heads/development' }} + env: nightly_toolchain: nightly-2023-06-12 stable_toolchain: 1.77 @@ -50,6 +55,7 @@ jobs: - name: cargo format run: cargo +${{ env.nightly_toolchain }} fmt --all -- --check + prettier: name: prettier runs-on: [ self-hosted, ubuntu-high-cpu ] @@ -99,7 +105,7 @@ jobs: machete: # Checks for unused dependencies. name: machete - runs-on: ubuntu-latest + runs-on: [ ubuntu-latest ] steps: - name: checkout @@ -181,16 +187,20 @@ jobs: licenses: name: file licenses - runs-on: ubuntu-latest + runs-on: [ ubuntu-latest ] steps: - name: checkout uses: actions/checkout@v4 - name: install ripgrep + # https://github.com/BurntSushi/ripgrep/releases + env: + ripgrep_version: 14.1.0 run: | - wget https://github.com/BurntSushi/ripgrep/releases/download/13.0.0/ripgrep_13.0.0_amd64.deb - sudo dpkg -i ripgrep_13.0.0_amd64.deb + #wget https://github.com/BurntSushi/ripgrep/releases/download/14.1.0/ripgrep_14.1.0-1_amd64.deb.sha256 + wget https://github.com/BurntSushi/ripgrep/releases/download/${{ env.ripgrep_version }}/ripgrep_${{ env.ripgrep_version }}-1_amd64.deb + sudo dpkg -i ripgrep_${{ env.ripgrep_version }}-1_amd64.deb rg --version || exit 1 - name: run the license check @@ -254,7 +264,7 @@ jobs: # needed for test results event_file: - runs-on: ubuntu-latest + runs-on: [ ubuntu-latest ] steps: - name: Upload diff --git a/.license.ignore b/.license.ignore index a2cc470eb..49cd4a25d 100644 --- a/.license.ignore +++ b/.license.ignore @@ -1,9 +1,5 @@ -./CODEOWNERS -./LICENSE ./applications/tari_indexer/src/substate_storage_sqlite/schema.rs -./buildtools/vagrant/Vagrantfile ./networking/sqlite_message_logger/src/schema.rs ./dan_layer/storage_sqlite/src/global/schema.rs ./dan_layer/state_store_sqlite/src/schema.rs ./dan_layer/wallet/storage_sqlite/src/schema.rs -./scripts/env_sample \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index ed8f8520e..b11a70246 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,6 @@ -@stringhandler @CjS77 @sdbondi \ No newline at end of file +# CI/CD-related files require a review by the devops team +.github/**/* @tari-project/devops +scripts/**/* @tari-project/devops +CODEOWNERS @tari-project/devops + +@stringhandler @CjS77 @sdbondi diff --git a/Cross.toml b/Cross.toml index 41c338c77..9eca0b53b 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,56 +1,54 @@ -[target.x86_64-linux-android.env] -passthrough = ["CFLAGS"] - -[target.aarch64-linux-android.env] -passthrough = ["CFLAGS"] - -[target.arm-linux-androideabi.env] -passthrough = ["CFLAGS"] - -[target.aarch64-unknown-linux-gnu] -image = "ubuntu:18.04" -# Mergered all scripts/install_ubuntu_dependencies*.sh scripts -pre-build = [ """ -apt-get update && \ -apt-get -y install \ - openssl \ - libssl-dev \ - pkg-config \ - libsqlite3-dev \ - clang-10 \ - git \ - cmake \ - dh-autoreconf \ - libc++-dev \ - libc++abi-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libncurses5-dev \ - libncursesw5-dev \ - zip \ - curl \ - pkg-config-aarch64-linux-gnu \ - gcc-aarch64-linux-gnu \ - g++-aarch64-linux-gnu && \ -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ -export PATH="$HOME/.cargo/bin:$PATH" && \ -rustup target add aarch64-unknown-linux-gnu && \ -rustup toolchain install stable-aarch64-unknown-linux-gnu -""" ] - -[target.aarch64-unknown-linux-gnu.env] +[build.env] passthrough = [ "CFLAGS", - "RUST_DEBUG", - "RUSTFLAGS", - "ARCH", - "FEATURES", - "ROARING_ARCH", "TARGET", "RUST_TARGET", "BUILD_TARGET", "CARGO_BUILD_TARGET", - "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc", - "BINDGEN_EXTRA_CLANG_ARGS=--sysroot /usr/aarch64-linux-gnu/include/", + "TARGET_CFLAGS", + "CC_aarch64_unknown_linux_gnu", + "PKG_CONFIG_SYSROOT_DIR", + "PKG_CONFIG_ALLOW_CROSS", + "RUSTFLAGS", + "RUST_BACKTRACE", + "RUST_DEBUG", + "RUST_LOG", + "ARCH", + "FEATURES", + "ROARING_ARCH", "TARI_NETWORK", -] \ No newline at end of file + "TARI_TARGET_NETWORK", + "TARI_NETWORK_DIR", +] + +# Currently needs cross-rs from git +# ```cargo install cross --git https://github.com/cross-rs/cross``` +[target.aarch64-unknown-linux-gnu] +image.name = "ubuntu:18.04" +# targetting is needed for apple silicon +image.toolchain = ["linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"] +pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh" + +[target.aarch64-unknown-linux-gnu.env] +passthrough = [ + "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc", + "PKG_CONFIG_SYSROOT_DIR=/usr/lib/aarch64-linux-gnu/", + "PKG_CONFIG_ALLOW_CROSS=true", +] + +[target.x86_64-unknown-linux-gnu] +image = "ubuntu:18.04" +pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh" + +[target.riscv64gc-unknown-linux-gnu] +#image.name = "riscv64/ubuntu:20.04" +image.name = "ubuntu:20.04" +image.toolchain = ["linux/riscv64=riscv64gc-unknown-linux-gnu", "linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"] +pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh" + +[target.riscv64gc-unknown-linux-gnu.env] +passthrough = [ + "RISCV64GC_UNKNOWN_LINUX_GNU_OPENSSL_NO_VENDOR=true", + "PKG_CONFIG_SYSROOT_DIR=/usr/lib/riscv64-linux-gnu/", + "PKG_CONFIG_ALLOW_CROSS=true", +] diff --git a/scripts/cross_compile_ubuntu_18-pre-build.sh b/scripts/cross_compile_ubuntu_18-pre-build.sh new file mode 100755 index 000000000..c6c7d969c --- /dev/null +++ b/scripts/cross_compile_ubuntu_18-pre-build.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# +# Single script for Ubuntu 18.04 package setup, mostly used for cross-compiling +# + +set -e + +USAGE="Usage: $0 {target build} + where target build is one of the following: + x86_64-unknown-linux-gnu or + aarch64-unknown-linux-gnu or + riscv64gc-unknown-linux-gnu +" + +if [ "$#" == "0" ]; then + echo -e "${USAGE}" + exit 1 +fi + +if [ -z "${CROSS_DEB_ARCH}" ]; then + echo -e "Should be run from cross, which sets the env 'CROSS_DEB_ARCH' + amd64 + arm64 + riscv64 +" + exit 1 +fi + +DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive} +# Hack of Note! +TimeZone=${TimeZone:-"Etc/GMT"} +ln -snf /usr/share/zoneinfo/${TimeZone} /etc/localtime +echo ${TimeZone} > /etc/timezone + +targetBuild="${1}" +nativeRunTime=$(uname -m) +echo "Native RunTime is ${nativeRunTime}" + +if [ "${nativeRunTime}" == "x86_64" ]; then + nativeArch=amd64 +elif [ "${nativeRunTime}" == "aarch64" ]; then + nativeArch=arm64 +elif [ "${nativeRunTime}" == "riscv64" ]; then + nativeArch=riscv64 + echo "ToDo!" +else + echo "!!Unsupport platform!!" + exit 1 +fi + +if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then + targetArch=arm64 + targetPlatform=aarch64 +elif [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then + targetArch=amd64 + targetPlatform=x86-64 +elif [ "${targetBuild}" == "riscv64gc-unknown-linux-gnu" ]; then + targetArch=riscv64 + targetPlatform=riscv64 + echo "ToDo!" +else + echo "!!Unsupport platform!!" + exit 1 +fi + +crossArch=${CROSS_DEB_ARCH} +apt-get update + +# Base install packages +# scripts/install_ubuntu_dependencies.sh +apt-get install --no-install-recommends --assume-yes \ + apt-transport-https \ + ca-certificates \ + curl \ + gpg \ + bash \ + less \ + openssl \ + libssl-dev \ + pkg-config \ + libsqlite3-dev \ + libsqlite3-0 \ + libreadline-dev \ + git \ + cmake \ + dh-autoreconf \ + clang \ + g++ \ + libc++-dev \ + libc++abi-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libncurses5-dev \ + libncursesw5-dev \ + libudev-dev \ + libhidapi-dev \ + zip + +echo "Installing rust ..." +mkdir -p "$HOME/.cargo/bin/" +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +export PATH="$HOME/.cargo/bin:$PATH" +. "$HOME/.cargo/env" + +# Cross-CPU compile setup +if [ "${CROSS_DEB_ARCH}" != "${nativeArch}" ]; then + echo "Setup Cross CPU Compile ..." + sed -i.save -e "s/^deb\ http/deb [arch="${nativeArch}"] http/g" /etc/apt/sources.list + + . /etc/lsb-release + ubuntu_tag=${DISTRIB_CODENAME} + + if [[ "${crossArch}" =~ ^(arm|riscv)64$ ]]; then + cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-updates main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-backports main restricted universe multiverse + +deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse +# deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag}-security main restricted universe multiverse + +deb [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +# deb-src [arch=${crossArch}] http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +EoF + fi + + if [ "${crossArch}" == "amd64" ]; then + cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} main restricted + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates main restricted + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} universe +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates universe + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag} multiverse +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-updates multiverse + +deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse +# deb-src http://archive.ubuntu.com/ubuntu/ ${ubuntu_tag}-backports main restricted universe multiverse + +# deb http://archive.canonical.com/ubuntu ${ubuntu_tag} partner +# deb-src http://archive.canonical.com/ubuntu ${ubuntu_tag} partner + +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security main restricted +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security universe +deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse +# deb-src http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multiverse +EoF + fi + + dpkg --add-architecture ${CROSS_DEB_ARCH} + apt-get update + + # scripts/install_ubuntu_dependencies-cross_compile.sh x86-64 + apt-get --assume-yes install \ + pkg-config-${targetPlatform}-linux-gnu \ + gcc-${targetPlatform}-linux-gnu \ + g++-${targetPlatform}-linux-gnu + + # packages needed for Ledger and hidapi + apt-get --assume-yes install \ + libudev-dev:${CROSS_DEB_ARCH} \ + libhidapi-dev:${CROSS_DEB_ARCH} + +fi + +rustup target add ${targetBuild} +rustup toolchain install stable-${targetBuild} --force-non-host + +rustup target list --installed +rustup toolchain list diff --git a/scripts/file_license_check.sh b/scripts/file_license_check.sh index adb40901e..6f60ebd86 100755 --- a/scripts/file_license_check.sh +++ b/scripts/file_license_check.sh @@ -1,20 +1,75 @@ -#!/bin/bash +#!/usr/bin/env bash +# # Must be run from the repo root +# +#set -xo pipefail + +check_for() { + if prog_location=$(which ${1}) ; then + if result=$(${1} --version 2>/dev/null); then + result="${1}: ${result} INSTALLED ✓" + else + result="${1}: INSTALLED ✓" + fi + else + result="${1}: MISSING ⨯" + fi +} + +check_requirements() { + echo "List of requirements and possible test:" + + req_progs=( + mktemp + rg + diff + ) + for RProg in "${req_progs[@]}"; do + check_for ${RProg} + echo "${result}" + if [[ "${result}" == "${RProg}: MISSING ⨯" ]]; then + echo "!! Install ${RProg} and try again !!" + exit -1 + fi + done + + if [ ! -f .license.ignore ]; then + echo "!! No .license.ignore file !!" + exit -1 + fi +} + +check_requirements + +# Exclude files without extensions as well as those with extensions that are not in the list +rgTemp=$(mktemp) rg -i "Copyright.*The Tari Project" --files-without-match \ - -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue,liquid,otf,d.ts,mjs}' . \ - -g '!bindings/src/types/*' | sort > /tmp/rgtemp + -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue,liquid,otf,d.ts,mjs}' . \ + -g '!bindings/src/types/*' \ + | while IFS= read -r file; do + if [[ -n $(basename "${file}" | grep -E '\.') ]]; then + echo "${file}" + fi + done | sort > ${rgTemp} + +# Sort the .license.ignore file as sorting seems to behave differently on different platforms +licenseIgnoreTemp=$(mktemp) +cat .license.ignore | sort > ${licenseIgnoreTemp} -# sort the .license.ignore file as sorting seems to behave differently on different platforms -cat .license.ignore | sort > /tmp/.license.ignore +DIFFS=$(diff -u --strip-trailing-cr ${licenseIgnoreTemp} ${rgTemp}) -DIFFS=$(diff -u /tmp/.license.ignore /tmp/rgtemp) +# clean up +rm -vf ${rgTemp} +rm -vf ${licenseIgnoreTemp} -if [ -n "$DIFFS" ]; then - echo "New files detected that either need copyright/license identifiers added, or they need to be added to .license.ignore" - echo "NB: The ignore file must be sorted alphabetically!" +if [ -n "${DIFFS}" ]; then + echo "New files detected that either need copyright/license identifiers added, or they need to be added to .license.ignore" + echo "NB: The ignore file must be sorted alphabetically!" - echo "Diff:" - echo "$DIFFS" - exit 1 + echo "Diff:" + echo "${DIFFS}" + exit 1 +else + exit 0 fi