diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9616feca1..bd3964125 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,6 @@ on: [push, pull_request] env: GCS_BUCKET: autobuilds.grr-response.com GCS_BUCKET_OPENAPI: autobuilds-grr-openapi - GCS_LATEST_PATH: _latest_server_deb DOCKER_REPOSITORY: ghcr.io/google/grr jobs: test-devenv: @@ -28,6 +27,7 @@ jobs: sleep 5 done [[ $OK = true ]] + test-ubuntu: runs-on: ubuntu-22.04 env: @@ -54,6 +54,7 @@ jobs: pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/ # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463 if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi + build-openapi: runs-on: ubuntu-22.04 steps: @@ -183,10 +184,11 @@ jobs: path: gcs_upload_dir/ retention-days: 1 - build-server-deb: + build-docker-image: runs-on: ubuntu-22.04 - env: - GCS_TAG: server_deb + permissions: + contents: 'read' + id-token: 'write' needs: - build-centos - build-ubuntu @@ -198,107 +200,132 @@ jobs: id: download uses: actions/download-artifact@v4 with: - path: ~/_artifacts - - name: Set up - run: | - sudo apt-get update - sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv python3-mysqldb wget openjdk-8-jdk zip git devscripts libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1 - python3 -m venv --system-site-packages "${HOME}/INSTALL" - "${HOME}/INSTALL/bin/python3" -m pip install --upgrade pip 'setuptools<58.3.1' wheel - - name: Build - run: | - travis/install.sh - mkdir -p grr/config/grr_response_templates/templates - mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates - mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates - travis/build_local_pyindex.sh - travis/build_server_deb.sh - ls -la gcs_upload_dir - - name: Upload installers to GitHub artifacts + pattern: '*installer*' + path: _installers + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REPOSITORY }} + - name: Build and export + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile + # Temporarily add a `testing`-tag to identify this image + # for testing, tag is removed again before uploading to + # github container registry. + tags: | + ${{ env.DOCKER_REPOSITORY }}:testing + ${{ steps.meta.outputs.tags }} + outputs: type=docker,dest=/tmp/grr_base_image.tar + - name: Upload docker image uses: actions/upload-artifact@v4 with: - name: server-deb - path: gcs_upload_dir/ - retention-days: 1 - - test-ubuntu-e2e: - continue-on-error: true # Debug follow up step. + name: grr_base_image + path: /tmp/grr_base_image.tar + retention-days: 3 + + docker-compose-e2e-test: + permissions: + contents: 'read' + id-token: 'write' runs-on: ubuntu-22.04 - env: - GRR_ADMIN_PASS: 'e2e_tests' - APPVEYOR_MYSQL_PASS: 'root' needs: - - build-server-deb + - build-docker-image steps: - uses: actions/checkout@v4 - - name: Set up MySQL - run: | - printf "\n[mysqld]\nmax_allowed_packet=42M\nlog_bin_trust_function_creators=1\n" | sudo tee -a /etc/mysql/my.cnf - sudo /etc/init.d/mysql start - - name: Download installers from GitHub artifacts - id: download + - name: Download artifact uses: actions/download-artifact@v4 with: - name: server-deb - path: _artifacts - - name: Install + name: grr_base_image + path: /tmp + - name: Load image run: | - free -hmw - lscpu - sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh - sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh + docker load --input /tmp/grr_base_image.tar + - name: Start docker compose stack + shell: bash + run: | + docker compose \ + -f docker-compose.yaml \ + -f docker-compose.testing.yaml \ + up -d --wait - name: Test + shell: bash run: | - sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh - sudo -EH ./appveyor/e2e_tests/test_repack.sh - - name: Upload logs and configs to GitHub artifacts - uses: actions/upload-artifact@v4 + docker compose exec grr-client bash /configs/create_fake_user.sh + docker run \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd):/github_workspace \ + -w /github_workspace \ + --entrypoint appveyor/e2e_tests/run_docker_compose_e2e_test.sh \ + ${{ env.DOCKER_REPOSITORY }}:testing \ + $(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' grr-client) + - name: Dump docker compose log output + if: always() + shell: bash + run: | + docker compose logs > /tmp/docker_compose_test.log + - name: Upload docker compose logs if: always() + uses: actions/upload-artifact@v4 with: - name: e2e-test - path: /var/log/grr - retention-days: 1 + name: docker_commpose_test_logs + path: /tmp/docker_compose_test.log + retention-days: 3 + - name: Stop the docker compose stack + if: always() + shell: bash + run: | + docker compose down --volumes - build-push-docker: + push-docker-image: + if: ${{ github.event_name == 'push' }} + env: + REGISTRY: ghcr.io + permissions: write-all runs-on: ubuntu-22.04 needs: - - build-server-deb - # - test-ubuntu-e2e # TODO: Comment back in after debugging is finished. - test-ubuntu - - build-openapi + - docker-compose-e2e-test steps: - uses: actions/checkout@v4 - - name: Download installers from GitHub artifacts - id: download + - name: Download artifact uses: actions/download-artifact@v4 with: - name: server-deb - path: _artifacts - - name: Build Docker image + name: grr_base_image + path: /tmp + - name: Load image run: | - export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3) - ./appveyor/docker_build/build_docker_image.sh - - if: ${{ github.event_name == 'push' }} - name: Login to GitHub Container registry - uses: docker/login-action@v2 + docker load --input /tmp/grr_base_image.tar + - name: Login to GitHub Container registry + uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - if: ${{ github.event_name == 'push' }} - name: Push to GitHub Container registry + - name: Push Docker image run: | - docker push -a ${{ env.DOCKER_REPOSITORY }} - upload: + # Remove the tag used to identify the image for testing again. + docker rmi ${{ env.DOCKER_REPOSITORY }}:testing + docker push --all-tags ${{ env.DOCKER_REPOSITORY }} + + upload-artifacts: if: ${{ github.event_name == 'push' }} permissions: contents: 'read' id-token: 'write' runs-on: ubuntu-22.04 needs: - - build-push-docker + - docker-compose-e2e-test + - test-ubuntu + - build-centos + - build-ubuntu + - build-osx + - build-windows + - build-openapi steps: - uses: actions/checkout@v4 - name: Download installers from GitHub artifacts @@ -319,8 +346,6 @@ jobs: mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx mkdir -p $OUTPUT_DIR/windows/ mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows - mkdir -p $OUTPUT_DIR/server_deb/ - mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb - name: Authenticate uses: 'google-github-actions/auth@v1' with: @@ -335,10 +360,6 @@ jobs: destination: ${{ env.GCS_BUCKET }} # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. parent: false - - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS - run: | - gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true - gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/ - name: Upload OpenAPI to GCS uses: google-github-actions/upload-cloud-storage@v1.0.0 with: @@ -346,4 +367,3 @@ jobs: destination: ${{ env.GCS_BUCKET_OPENAPI }} # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. parent: false -