Design doc for Upload/AssetBlob garbage collection #3782
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI for frontend | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| schedule: | |
| - cron: "0 0 * * *" | |
| jobs: | |
| lint-type-check: | |
| defaults: | |
| run: | |
| working-directory: web | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 16 # TODO: use latest version when we update to Vue 3 | |
| - name: Install Vue app | |
| run: yarn install | |
| - name: Lint Vue app | |
| run: yarn run lint --no-fix | |
| - name: Type-check Vue app | |
| run: yarn run type-check | |
| - name: Build Vue app | |
| run: yarn run build | |
| test-e2e-puppeteer: | |
| runs-on: ubuntu-22.04 | |
| services: | |
| postgres: | |
| image: postgres:latest | |
| env: | |
| POSTGRES_DB: django | |
| POSTGRES_PASSWORD: postgres | |
| ports: | |
| - 5432:5432 | |
| rabbitmq: | |
| image: rabbitmq:management | |
| ports: | |
| - 5672:5672 | |
| minio: | |
| # This image does not require any command arguments (which GitHub Actions don't support) | |
| image: bitnami/minio:latest | |
| env: | |
| MINIO_ROOT_USER: minioAccessKey | |
| MINIO_ROOT_PASSWORD: minioSecretKey | |
| ports: | |
| - 9000:9000 | |
| env: | |
| # API server env vars | |
| DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django | |
| DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 | |
| DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey | |
| DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey | |
| DJANGO_STORAGE_BUCKET_NAME: dandi-bucket | |
| DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket | |
| DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs | |
| DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs | |
| DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 | |
| DJANGO_DANDI_API_URL: http://localhost:8000 | |
| DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ | |
| DJANGO_DANDI_DEV_EMAIL: [email protected] | |
| DANDI_ALLOW_LOCALHOST_URLS: 1 | |
| # Web client env vars | |
| VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ | |
| VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ | |
| VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl | |
| # E2E tests env vars | |
| CLIENT_URL: http://localhost:8085 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| cache: 'yarn' | |
| cache-dependency-path: web/yarn.lock | |
| - name: Install web app | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| run: yarn install --frozen-lockfile --prefer-offline | |
| working-directory: web | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install latest version of pip | |
| run: pip install --upgrade pip | |
| - uses: actions/cache@v4 | |
| id: pip-cache | |
| with: | |
| path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} | |
| - name: Install dandi-api dependencies | |
| run: pip install --upgrade --upgrade-strategy eager -e .[dev] | |
| - name: Apply migrations to API server | |
| run: python manage.py migrate | |
| - name: Create any cache tables | |
| run: python manage.py createcachetable | |
| # Populate the database with a dandiset for FileBrowser tests | |
| - name: Create a superuser | |
| run: python manage.py createsuperuser --noinput --email [email protected] | |
| - name: Create Dandiset | |
| run: python manage.py create_dev_dandiset --owner [email protected] | |
| - name: Install E2E tests | |
| run: yarn install --frozen-lockfile | |
| working-directory: e2e/puppeteer | |
| - name: Lint E2E tests | |
| run: yarn run lint --no-fix --max-warnings=0 | |
| working-directory: e2e/puppeteer | |
| - name: Run E2E tests | |
| run: | | |
| # start vue dev server and wait for it to start | |
| yarn --cwd ../../web/ run dev 2> /dev/null & | |
| while ! nc -z localhost 8085; do | |
| sleep 3 | |
| done | |
| # start the dandi-api server | |
| python ../../manage.py runserver & | |
| # run the E2E tests | |
| yarn run test | |
| working-directory: e2e/puppeteer | |
| test-e2e-playwright: | |
| runs-on: ubuntu-22.04 | |
| services: | |
| postgres: | |
| image: postgres:latest | |
| env: | |
| POSTGRES_DB: django | |
| POSTGRES_PASSWORD: postgres | |
| ports: | |
| - 5432:5432 | |
| rabbitmq: | |
| image: rabbitmq:management | |
| ports: | |
| - 5672:5672 | |
| minio: | |
| # This image does not require any command arguments (which GitHub Actions don't support) | |
| image: bitnami/minio:latest | |
| env: | |
| MINIO_ROOT_USER: minioAccessKey | |
| MINIO_ROOT_PASSWORD: minioSecretKey | |
| ports: | |
| - 9000:9000 | |
| env: | |
| # API server env vars | |
| DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django | |
| DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 | |
| DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey | |
| DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey | |
| DJANGO_STORAGE_BUCKET_NAME: dandi-bucket | |
| DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket | |
| DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs | |
| DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs | |
| DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 | |
| DJANGO_DANDI_API_URL: http://localhost:8000 | |
| DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ | |
| DJANGO_DANDI_DEV_EMAIL: [email protected] | |
| DANDI_ALLOW_LOCALHOST_URLS: 1 | |
| # Web client env vars | |
| VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ | |
| VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ | |
| VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install web app | |
| run: yarn install --frozen-lockfile | |
| working-directory: web | |
| - name: Install latest version of pip | |
| run: pip install --upgrade pip | |
| - uses: actions/cache@v4 | |
| id: pip-cache | |
| with: | |
| path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} | |
| - name: Install dandi-api dependencies | |
| run: pip install --upgrade --upgrade-strategy eager -e .[dev] | |
| - name: Apply migrations to API server | |
| run: python manage.py migrate | |
| - name: Create any cache tables | |
| run: python manage.py createcachetable | |
| - name: Install test data | |
| run: python manage.py loaddata playwright | |
| - name: Install dependencies | |
| run: yarn install --frozen-lockfile | |
| working-directory: e2e/playwright | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| working-directory: e2e/playwright | |
| - name: Run Playwright tests | |
| run: | | |
| # start vue dev server | |
| yarn --cwd ../../web/ run dev 2> /dev/null & | |
| while ! nc -z localhost 8085; do | |
| sleep 3 | |
| done | |
| # start the dandi-api server | |
| python ../../manage.py runserver & | |
| # run the tests | |
| npx playwright test | |
| working-directory: e2e/playwright | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: playwright-report | |
| path: e2e/playwright/playwright-report/ | |
| retention-days: 30 |