Added metadata-crawler dockerfile. Refactor container workflows

.github/workflows/celo-monorepo.yml

@@ -130,7 +130,7 @@ jobs:
  code-${{ github.sha }}
  - name: Detect files changed in PR (or commit), and expose as output
  id: changed-files
- uses: tj-actions/changed-files@v37
+ uses: tj-actions/changed-files@v40
  with:
  # Using comma as separator to be able to easily match full paths (using ,<path>)
  separator: ','

.github/workflows/containers.yaml

@@ -0,0 +1,157 @@
+---
+name: Build containers
+
+on:
+ push:
+ paths:
+ - 'dockerfiles/**'
+ branches:
+ - master
+ pull_request:
+ paths:
+ - 'dockerfiles/**'
+ workflow_dispatch:
+
+jobs:
+ changed-files:
+ runs-on: ubuntu-latest
+ outputs:
+ # Propagate more outputs if you need https://github.com/tj-actions/changed-files#outputs
+ # Adding a initial comma so ',<path>' matches also for the first file
+ all_modified_files: ',${{ steps.changed-files.outputs.all_modified_files }}'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Detect files changed in PR (or commit), and expose as output
+ id: changed-files
+ uses: tj-actions/changed-files@v40
+ with:
+ # Using comma as separator to be able to easily match full paths (using ,<path>)
+ separator: ','
+
+ # Celotool images
+ celotool-build-dev:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ name: Build us-west1-docker.pkg.dev/devopsre/dev-images/celotool:${{ github.sha }}
+ needs: changed-files
+ if: |
+ github.ref != 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/celotool/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/celotool
+ tags: ${{ github.sha }}
+ context: .
+ file: dockerfiles/celotool/Dockerfile
+ trivy: true
+ celotool-build:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ name: Build us-west1-docker.pkg.dev/devopsre/celo-monorepo/celotool:${{ github.sha }}
+ needs: changed-files
+ if: |
+ github.ref == 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/celotool/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo-master/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/celo-monorepo/celotool
+ tags: ${{ github.sha }}
+ context: .
+ file: dockerfiles/celotool/Dockerfile
+ trivy: true
+
+ # All monorepo
+ celomonorepo-build-dev:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ name: Build us-west1-docker.pkg.dev/devopsre/dev-images/monorepo:${{ github.sha }}
+ needs: changed-files
+ if: |
+ github.ref != 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/all-monorepo/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/monorepo
+ tags: ${{ github.sha }}
+ context: .
+ file: dockerfiles/all-monorepo/Dockerfile
+ trivy: true
+ celomonorepo-build:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ name: Build us-west1-docker.pkg.dev/devopsre/celo-monorepo/monorepo:${{ github.sha }}
+ needs: changed-files
+ if: |
+ github.ref == 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/all-monorepo/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo-master/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/celo-monorepo/monorepo
+ tags: ${{ github.sha }}
+ context: .
+ file: dockerfiles/all-monorepo/Dockerfile
+ trivy: true
+
+ # CeloCli images
+ celocli-build-dev:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ needs: changed-files
+ name: Build us-west1-docker.pkg.dev/devopsre/dev-images/celocli:testing
+ if: |
+ github.ref != 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/cli-standalone/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/celocli
+ tags: testing
+ context: .
+ file: dockerfiles/cli-standalone/Dockerfile
+ trivy: true
+ celocli-build:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ needs: changed-files
+ name: Build us-west1-docker.pkg.dev/devopsre/celo-monorepo/celocli:latest
+ if: |
+ github.ref == 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/cli-standalone/Dockerfile')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo-master/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/celo-monorepo/celocli
+ tags: latest
+ context: .
+ file: dockerfiles/cli-standalone/Dockerfile
+ trivy: true
+
+ # Blockscout Metadata crawler images
+ metadata-crawler-build-dev:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ needs: changed-files
+ name: Build us-west1-docker.pkg.dev/devopsre/dev-images/blockscoute-metadata-crawler:testing
+ if: |
+ github.ref != 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/metadata-crawler')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/blockscoute-metadata-crawler
+ tags: testing
+ context: .
+ file: dockerfiles/metadata-crawler/Dockerfile
+ trivy: true
+ metadata-crawler-build:
+ uses: celo-org/reusable-workflows/.github/workflows/[email protected]
+ needs: changed-files
+ name: Build us-west1-docker.pkg.dev/devopsre/dev-images/blockscoute-metadata-crawler:latest
+ if: |
+ github.ref == 'refs/heads/master' &&
+ contains(needs.changed-files.outputs.all_modified_files, ',dockerfiles/metadata-crawler')
+ with:
+ workload-id-provider: projects/1094498259535/locations/global/workloadIdentityPools/gh-celo-monorepo/providers/github-by-repos
+ service-account: '[email protected]'
+ artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/blockscoute-metadata-crawler
+ tags: latest
+ context: .
+ file: dockerfiles/metadata-crawler/Dockerfile
+ trivy: true

dockerfiles/metadata-crawler/Dockerfile

@@ -0,0 +1,28 @@
+FROM node:18
+LABEL org.opencontainers.image.authors="[email protected]"
+
+WORKDIR /celo-monorepo
+
+# Monorepo dependencies
+RUN apt-get update && \
+ apt-get upgrade -y && \
+ apt-get install -y lsb-release && \
+ apt-get install -y curl build-essential git python3 && \
+ alias python=python3 && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g typescript npm
+
+COPY lerna.json package.json yarn.lock ./
+COPY scripts scripts/
+COPY packages/typescript packages/typescript/
+COPY packages/sdk packages/sdk/
+COPY packages/dev-utils packages/dev-utils/
+COPY packages/metadata-crawler packages/metadata-crawler/
+
+RUN yarn install --network-timeout 100000 --frozen-lockfile && \
+ yarn cache clean && \
+ yarn build
+
+WORKDIR /celo-monorepo/packages/metadata-crawler
+CMD ["node", "lib/crawler.js"]