From 37969c68ce5411f57c7848c393a89d126111cf2d Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 10 Sep 2024 09:37:00 -0400 Subject: [PATCH 1/5] GHA Migration Migrate CI/CD from Travis to GHA --- .github/workflows/dependencies.yml | 14 +++-- .github/workflows/deploy.yml | 86 ++++++++++++++++++++++++++++++ .github/workflows/enforcer.yml | 16 ++++++ .github/workflows/tests.yml | 58 ++++++++++++++++++++ .travis.yml | 55 ------------------- 5 files changed, 171 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/enforcer.yml create mode 100644 .github/workflows/tests.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 60c1d38..b76d580 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -14,9 +14,17 @@ jobs: contents: write pull-requests: write steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + - uses: actions/checkout@v4 with: + token: ${{ steps.app-token.outputs.token }} ref: development + persist-credentials: false - name: Set up Python uses: actions/setup-python@v4 @@ -29,14 +37,14 @@ jobs: - name: Run pre-commit autoupdate run: pre-commit autoupdate - + - name: Run pip-compile run: pip-compile --upgrade - + - name: Create Pull Request uses: peter-evans/create-pull-request@v6.0.2 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.app-token.outputs.token }} base: development branch: dependency-updates delete-branch: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..64677c4 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,86 @@ +name: Deploy + +on: + push: + branches: + - base + - development + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: ${{ github.ref_name }} + + env: + APPLICATION_NAME: pictor + CONTAINER: pictor-web + APPLICATION_PORT: 8012 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Export secrets to environment variables + uses: oNaiPs/secrets-to-env-action@v1.5 + with: + secrets: ${{ toJSON(secrets) }} + + - name: Clone deploy scripts if not present + run: git clone https://github.com/RockefellerArchiveCenter/deploy_scripts.git; + + - name: Substitute environment variables + uses: tvarohohlavy/inplace-envsubst-action@v1.0.0 + with: + files: | + $APPLICATION_NAME/config.py.deploy + appspec.yml.deploy + deploy_scripts/create_apache_config.sh.deploy + deploy_scripts/curl_index.sh.deploy + deploy_scripts/curl_status_endpoint.sh.deploy + deploy_scripts/install_dependencies_django.sh.deploy + deploy_scripts/restart_apachectl.sh.deploy + deploy_scripts/run_management_commands_django.sh.deploy + deploy_scripts/set_permissions.sh.deploy + deploy_scripts/stop_cron.sh.deploy + + - name: Rename deploy files + run: | + mv $APPLICATION_NAME/config.py.deploy $APPLICATION_NAME/config.py + mv appspec.yml.deploy appspec.yml + mv deploy_scripts/create_apache_config.sh.deploy deploy_scripts/create_apache_config.sh + mv deploy_scripts/curl_index.sh.deploy deploy_scripts/curl_index.sh + mv deploy_scripts/curl_status_endpoint.sh.deploy deploy_scripts/curl_status_endpoint.sh + mv deploy_scripts/install_dependencies_django.sh.deploy deploy_scripts/install_dependencies_django.sh + mv deploy_scripts/restart_apachectl.sh.deploy deploy_scripts/restart_apachectl.sh + mv deploy_scripts/run_management_commands_django.sh.deploy deploy_scripts/run_management_commands_django.sh + mv deploy_scripts/set_permissions.sh.deploy deploy_scripts/set_permissions.sh + mv deploy_scripts/stop_cron.sh.deploy deploy_scripts/stop_cron.sh + + - name: Make deploy scripts executable + run: chmod +x deploy_scripts/*.sh + + - name: Create deployment zip + run: sudo deploy_scripts/make_zip_django.sh $DEPLOY_ZIP_DIR $DEPLOY_ZIP_NAME + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE }} + role-skip-session-tagging: true + role-duration-seconds: 900 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Deploy to S3 + run: aws s3 cp $DEPLOY_ZIP_DIR s3://$AWS_BUCKET_NAME --recursive + + - name: Deploy to AWS CodeDeploy + run: aws deploy create-deployment + --region ${{ secrets.AWS_REGION }} + --application-name $APPLICATION_NAME + --deployment-config-name CodeDeployDefault.OneAtATime + --deployment-group-name $DEPLOYMENT_GROUP + --s3-location bucket=$AWS_BUCKET_NAME,bundleType=zip,key=$DEPLOY_ZIP_NAME diff --git a/.github/workflows/enforcer.yml b/.github/workflows/enforcer.yml new file mode 100644 index 0000000..1fc567f --- /dev/null +++ b/.github/workflows/enforcer.yml @@ -0,0 +1,16 @@ +name: 'Check Branch' + +on: + pull_request: + branches: + - base + +jobs: + check_branch: + runs-on: ubuntu-latest + steps: + - name: Check branch + if: github.head_ref != 'development' + run: | + echo "ERROR: You can only merge to base from the development branch." + exit 1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..13db3ab --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,58 @@ +name: Tests + +on: + pull_request: + branches: + - development + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + environment: + name: development + + env: + APPLICATION_NAME: pictor + CONTAINER: pictor-web + APPLICATION_PORT: 8012 + + services: + docker: + image: docker:stable + options: --privileged + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python and cache pip + uses: actions/setup-python@v5.2.0 + with: + python-version: '3.10' + cache: 'pip' + + - name: Copy config file + run: cp ${{ env.APPLICATION_NAME }}/config.py.example ${{ env.APPLICATION_NAME }}/config.py + + - name: Login to Docker + run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + + - name: Start Docker containers + run: docker compose up -d + + - name: Wait for services to be ready + run: ./wait-for-it.sh $CONTAINER:$APPLICATION_PORT -- echo "$CONTAINER is ready" + + - name: Install pre-commit + run: | + pip install "pre-commit===2.13.0" + pre-commit install + + - name: Run pre-commit checks + run: pre-commit run --all-files --show-diff-on-failure + + - name: Run tests + run: docker compose exec -T $CONTAINER python manage.py test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ee188e7..0000000 --- a/.travis.yml +++ /dev/null @@ -1,55 +0,0 @@ -dist: bionic -language: python -python: "3.10" -cache: - directories: - - $HOME/.cache/pip - - $HOME/.cache/pre-commit -services: - - docker -env: - global: - - APPLICATION_NAME: pictor - - CONTAINER: pictor-web -before_install: - - cp ${APPLICATION_NAME}/config.py.example ${APPLICATION_NAME}/config.py - - echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin - - docker-compose up -d -install: - - pip install pre-commit && pre-commit install - - ./wait-for-it.sh $CONTAINER:$APPLICATION_PORT -- docker-compose exec $CONTAINER pip install coverage -script: - - pre-commit run --all-files --show-diff-on-failure - - docker-compose exec $CONTAINER coverage run manage.py test - - docker-compose exec $CONTAINER coverage report --omit=create_derivatives/migrations/* -m -before_deploy: - - if [ ! -d deploy_scripts ]; then git clone https://github.com/RockefellerArchiveCenter/deploy_scripts.git; fi - - sudo deploy_scripts/substitute_env.sh - - sudo deploy_scripts/make_zip_django.sh $DEPLOY_ZIP_DIR $DEPLOY_ZIP_NAME -deploy: - - provider: s3 - access_key_id: $AWS_ACCESS_KEY - secret_access_key: $AWS_SECRET_KEY - local_dir: $DEPLOY_ZIP_DIR - skip_cleanup: true - on: - repo: RockefellerArchiveCenter/${APPLICATION_NAME} - all_branches: true - condition: $TRAVIS_BRANCH =~ ^(base|development)$ - bucket: $AWS_BUCKET_NAME - region: us-east-1 - - provider: codedeploy - bucket: $AWS_BUCKET_NAME - key: $DEPLOY_ZIP_NAME - bundle_type: zip - access_key_id: $AWS_ACCESS_KEY - secret_access_key: $AWS_SECRET_KEY - application: ${APPLICATION_NAME} - deployment_group: ${DEPLOYMENT_GROUP} - region: us-east-1 - on: - repo: RockefellerArchiveCenter/${APPLICATION_NAME} - all_branches: true - condition: $TRAVIS_BRANCH =~ ^(base|development)$ -notifications: - email: false From 0c399986b58937073504ce26c667a5be9c7a154d Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 10 Sep 2024 09:44:12 -0400 Subject: [PATCH 2/5] Update readme Updates build status badge on workflow --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b95def3..c02e49e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A microservice to create image derivatives (JPG2000 and PDF files) and IIIF Mani pictor is part of [Project Electron](https://github.com/RockefellerArchiveCenter/project_electron), an initiative to build sustainable, open and user-centered infrastructure for the archival management of digital records at the [Rockefeller Archive Center](http://rockarch.org/). -[![Build Status](https://travis-ci.com/RockefellerArchiveCenter/pictor.svg?branch=base)](https://travis-ci.com/RockefellerArchiveCenter/pictor) +![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg)] ## Setup From 956395a2a77f844dcdb18d7415abddb523db0ef3 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 10 Sep 2024 09:44:42 -0400 Subject: [PATCH 3/5] fix typo fixes typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c02e49e..32804be 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A microservice to create image derivatives (JPG2000 and PDF files) and IIIF Mani pictor is part of [Project Electron](https://github.com/RockefellerArchiveCenter/project_electron), an initiative to build sustainable, open and user-centered infrastructure for the archival management of digital records at the [Rockefeller Archive Center](http://rockarch.org/). -![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg)] +![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg) ## Setup From d5b52d704a15cb3b808bbab47c0f3272e5224cab Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 10 Sep 2024 09:46:45 -0400 Subject: [PATCH 4/5] Add link to test actions Updates readme to make the badge link to the tests action runs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32804be..f373bef 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A microservice to create image derivatives (JPG2000 and PDF files) and IIIF Mani pictor is part of [Project Electron](https://github.com/RockefellerArchiveCenter/project_electron), an initiative to build sustainable, open and user-centered infrastructure for the archival management of digital records at the [Rockefeller Archive Center](http://rockarch.org/). -![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg) +[![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg)](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml) ## Setup From cc4f6d92f2bc565ce678c8dca8b7d49cfea2029d Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 10 Sep 2024 09:50:03 -0400 Subject: [PATCH 5/5] Remove badge Removes build status badge from readme. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index f373bef..c223715 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ A microservice to create image derivatives (JPG2000 and PDF files) and IIIF Mani pictor is part of [Project Electron](https://github.com/RockefellerArchiveCenter/project_electron), an initiative to build sustainable, open and user-centered infrastructure for the archival management of digital records at the [Rockefeller Archive Center](http://rockarch.org/). -[![Build Status](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml/badge.svg)](https://github.com/RockefellerArchiveCenter/pictor/actions/workflows/tests.yml) - ## Setup Install [git](https://git-scm.com/) and clone the repository