ZIR-303: Optimize bootstrap, add CI for generated files (#138)

* Copy keccak zkrs directly instead of zipping and unzipping them

* Convert predicates bootstrap to use rules

* Run bootstrap copies/checks in parallel for great speed

* Check fib and calculator generated files in CI

* Update calculator files, add calculator generated files to CI

* reduce dependencies

.github/actions/bazelisk/action.yml

@@ -1,8 +1,28 @@
 name: bazelisk installer
 description: Install bazelisk
+inputs:
+  cache-key:
+    required: true
 runs:
   using: composite
   steps:
+    # Cache bazel build on linux.  (MacOS uses the local cache on the runner.)
+    - name: Get Date
+      id: get-date
+      run: |
+           echo "date=$(/bin/date -u "+%Y-%m-%d")" >> $GITHUB_OUTPUT
+      shell: bash
+    - if: matrix.os == 'Linux'
+      uses: actions/cache@v4
+      env:
+        cache-name: bazel-build
+      with:
+        path: "~/.cache/bazel"
+        # Generate a new build cache once a day, reusing the previous day's if available
+        key: "bazel-${{ matrix.os }}-${{ inputs.cache-key }}-${{ steps.get-date.outputs.date }}"
+        restore-keys: |
+          bazel-${{ matrix.os }}-${{ inputs.cache-key }}-
+          bazel-${{ matrix.os }}-
     - if: runner.os == 'Linux'
       run: |
         echo "BAZELISK_OS=linux" >> $GITHUB_ENV

.github/workflows/main.yml

@@ -42,6 +42,31 @@ jobs:
           python-version: "3.10"
       - run: python license-check.py
 
+  check-generated-zirgen:
+    runs-on: [self-hosted, prod, Linux, cpu]
+    strategy:
+        fail-fast: true
+        matrix:
+          os: [Linux]
+    env:
+      RUST_BACKTRACE: full
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/bazelisk
+        with:
+          cache-key: bootstrap
+      - uses: risc0/risc0/.github/actions/rustup@4ab1f0bba1b0515819221bebc59f95aad0a6a3a9
+      - uses: risc0/risc0/.github/actions/sccache@1a373c71585766e4f6985b96c48389daaf69d528
+        with:
+          key: ${{ matrix.os }}-bootstrap
+      # TODO: check generated fib files once they generate properly
+      #      - name: fib generated files up to date
+      #        run: cargo bootstrap -- --check fib
+      #        continue-on-error: true
+      - name: calculator generated files up to date
+        run: cargo bootstrap -- --check calculator
+        continue-on-error: true
+
   bazel:
     runs-on: [self-hosted, prod, cpu, "${{ matrix.os }}"]
     strategy:
@@ -52,23 +77,10 @@ jobs:
           - os: Linux
             config: --config=ci
     steps:
-      - name: Get Date
-        id: get-date
-        run: |
-             echo "date=$(/bin/date -u "+%Y-%m-%d")" >> $GITHUB_OUTPUT
-        shell: bash
       - uses: actions/checkout@v3
-      # Cache bazel build on linux.  (MacOS uses the local cache on the runner.)
-      - if: matrix.os == 'Linux'
-        uses: actions/cache@v4
-        env:
-          cache-name: bazel-build
-        with:
-          path: "~/.cache/bazel"
-          # Generate a new build cache once a day, reusing the previous day's if available
-          key: "bazel-${{ matrix.os }}-${{ steps.get-date.outputs.date }}"
-          restore-keys: "bazel-${{ matrix.os }}-"
       - uses: ./.github/actions/bazelisk
+        with:
+          cache-key: build
       - name: Build & test
         env:
           CONFIG: ${{ matrix.config }}
@@ -139,6 +151,7 @@ jobs:
       - bazel
       - test
       - doc
+      - check-generated-zirgen
     runs-on: ubuntu-latest
     steps:
       - name: Check all job status

zirgen/Main/BUILD.bazel

@@ -22,9 +22,8 @@ cc_library(
         "//zirgen/Dialect/Zll/Transforms:passes",
         "//zirgen/dsl",
         "//zirgen/dsl/passes",
-        "@llvm-project//mlir:AllTranslations",
+        "@llvm-project//mlir:Debug",
         "@llvm-project//mlir:FuncExtensions",
-        "@llvm-project//mlir:MlirOptLib",
     ],
 )