diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml new file mode 100644 index 000000000..20279a21a --- /dev/null +++ b/.github/workflows/bot.yml @@ -0,0 +1,78 @@ +name: bot + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + pre_job: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + paths_result: ${{ steps.skip_check.outputs.paths_result }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@c449d86cf33a2a6c7a4193264cc2578e2c3266d4 # pin@v4 + with: + paths: '["bot/**", ".github/workflows/**"]' + + test: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + services: + redis: + image: redis:5 + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps port 6379 on service container to the host + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - name: Install poetry + run: | + pipx install poetry==1.1.13 + poetry config virtualenvs.in-project true + - uses: actions/setup-python@v4 + with: + python-version-file: "./bot/.python-version" + cache: poetry + cache-dependency-path: "./bot/poetry.lock" + - name: Install dependencies + working-directory: "./bot" + run: poetry install + - name: Run tests + working-directory: "bot" + run: ./s/test + - name: upload code coverage + working-directory: bot + run: ./s/upload-code-cov + lint: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install poetry + run: | + pipx install poetry==1.1.13 + poetry config virtualenvs.in-project true + - uses: actions/setup-python@v4 + with: + python-version-file: "./bot/.python-version" + cache: poetry + cache-dependency-path: "./bot/poetry.lock" + - name: Install dependencies + working-directory: "./bot" + run: poetry install + - name: Run lints + working-directory: "bot" + run: ./s/lint diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..bcdd9fce7 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,73 @@ +name: docs + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + pre_job: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + paths_result: ${{ steps.skip_check.outputs.paths_result }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@c449d86cf33a2a6c7a4193264cc2578e2c3266d4 # pin@v4 + with: + paths: '["docs/**", ".github/workflows/**"]' + typecheck: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: "docs/package.json" + cache-dependency-path: "docs/yarn.lock" + cache: "yarn" + - name: Install dependencies + working-directory: "docs" + run: yarn install --frozen-lockfile + - name: run typechecker + working-directory: "docs" + run: ./s/typecheck + + fmt: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: "docs/package.json" + cache-dependency-path: "docs/yarn.lock" + cache: "yarn" + - name: Install dependencies + working-directory: "docs" + run: yarn install --frozen-lockfile + - name: Run tests + working-directory: "docs" + run: ./s/fmt-ci + + verify_build: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: "docs/package.json" + cache-dependency-path: "docs/yarn.lock" + cache: "yarn" + - name: Install dependencies + working-directory: "docs" + run: yarn install --frozen-lockfile + - name: Run tests + working-directory: "docs" + run: ./s/build diff --git a/.github/workflows/infrastructure.yml b/.github/workflows/infrastructure.yml new file mode 100644 index 000000000..0936f0441 --- /dev/null +++ b/.github/workflows/infrastructure.yml @@ -0,0 +1,107 @@ +name: infrastructure + +on: + pull_request: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + commit_sha: + description: "SHA to deploy" + required: true + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: false + +# TODO: Skip if no changes in directory +jobs: + lint_shell: + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + - name: Install dependencies + run: | + sudo apt install shellcheck + - name: Run shellcheck + run: ./s/shellcheck + + build_bot_container: + runs-on: ubuntu-latest + if: ${{ github.event_name != 'workflow_dispatch' }} + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:bot" + push: true + file: "Dockerfile" + tags: cdignam/kodiak:${{ github.event.pull_request.head.sha || github.sha }} + cache-from: type=gha,scope=kodiak + cache-to: type=gha,scope=kodiak,mode=max + + build_api_container: + runs-on: ubuntu-latest + if: ${{ github.event_name != 'workflow_dispatch' }} + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:web_api" + push: true + file: "Dockerfile" + tags: cdignam/kodiak-web-api:${{ github.event.pull_request.head.sha || github.sha }} + cache-from: type=gha,scope=kodiak-web-api + cache-to: type=gha,scope=kodiak-web-api,mode=max + build-args: | + GIT_SHA=${{ github.event.pull_request.head.sha || github.sha }} + + build_web_ui_container: + runs-on: ubuntu-latest + if: ${{ github.event_name != 'workflow_dispatch' }} + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:web_ui" + push: true + file: "Dockerfile" + tags: cdignam/kodiak-web-ui:${{ github.event.pull_request.head.sha || github.sha }} + cache-from: type=gha,scope=kodiak-web-ui + cache-to: type=gha,scope=kodiak-web-ui,mode=max + build-args: | + GIT_SHA=${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/web_api.yml b/.github/workflows/web_api.yml new file mode 100644 index 000000000..8670ce01c --- /dev/null +++ b/.github/workflows/web_api.yml @@ -0,0 +1,138 @@ +name: web_api + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + pre_job: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + paths_result: ${{ steps.skip_check.outputs.paths_result }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@c449d86cf33a2a6c7a4193264cc2578e2c3266d4 # pin@v4 + with: + paths: '["web_api/**", ".github/workflows/**"]' + + test: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12 + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:5 + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps port 6379 on service container to the host + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - name: Install poetry + run: | + pipx install poetry==1.7.1 + poetry config virtualenvs.in-project true + - uses: actions/setup-python@v4 + with: + python-version-file: "./web_api/.python-version" + cache: poetry + cache-dependency-path: "./web_api/poetry.lock" + - name: Install dependencies + working-directory: "./web_api" + run: poetry install + - name: Set up environment variables + run: echo "DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/postgres" >> $GITHUB_ENV + - name: Run tests + working-directory: "web_api" + run: ./s/test + - name: upload code coverage + working-directory: web_api + run: ./s/upload-code-cov + lint: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install poetry + run: | + pipx install poetry==1.7.1 + poetry config virtualenvs.in-project true + - uses: actions/setup-python@v4 + with: + python-version-file: "./web_api/.python-version" + cache: poetry + cache-dependency-path: "./web_api/poetry.lock" + - name: Install dependencies + working-directory: "./web_api" + run: poetry install + - name: Run lints + working-directory: "web_api" + run: ./s/lint + + squawk: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12 + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: web_api_test + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 0 + - name: Install poetry + run: | + pipx install poetry==1.7.1 + poetry config virtualenvs.in-project true + - uses: actions/setup-python@v4 + with: + python-version-file: "./web_api/.python-version" + cache: poetry + cache-dependency-path: "./web_api/poetry.lock" + - name: Install dependencies + working-directory: "./web_api" + run: poetry install + - name: Run squawk + working-directory: "./web_api" + run: | + python ./s/squawk.py + - uses: sbdchd/squawk-action@v1 + with: + pattern: "web_api/migrations/*.sql" + version: "latest" diff --git a/.github/workflows/web_ui.yml b/.github/workflows/web_ui.yml new file mode 100644 index 000000000..86e2758b1 --- /dev/null +++ b/.github/workflows/web_ui.yml @@ -0,0 +1,55 @@ +name: web_ui + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + pre_job: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + paths_result: ${{ steps.skip_check.outputs.paths_result }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@c449d86cf33a2a6c7a4193264cc2578e2c3266d4 # pin@v4 + with: + paths: '["web_ui/**", ".github/workflows/**"]' + test: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: "web_ui/package.json" + cache-dependency-path: "web_ui/yarn.lock" + cache: "yarn" + - name: Install dependencies + working-directory: "web_ui" + run: yarn install --frozen-lockfile + - name: Run tests + working-directory: "web_ui" + run: ./s/test + + lint: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: "web_ui/package.json" + cache-dependency-path: "web_ui/yarn.lock" + cache: "yarn" + - name: Install dependencies + working-directory: "web_ui" + run: yarn install --frozen-lockfile + - name: Run tests + working-directory: "web_ui" + run: ./s/lint diff --git a/.gitignore b/.gitignore index 8d17bcbde..675c52e04 100644 --- a/.gitignore +++ b/.gitignore @@ -80,7 +80,7 @@ profile_default/ ipython_config.py # pyenv -.python-version +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. diff --git a/bot/pyproject.toml b/bot/pyproject.toml index 97af0f329..c3c37449c 100644 --- a/bot/pyproject.toml +++ b/bot/pyproject.toml @@ -67,7 +67,7 @@ filterwarnings = [ # an alternative at the moment. # https://github.com/pytest-dev/pytest/issues/3974 "ignore::pytest.PytestDeprecationWarning", - "ignore:SyntaxWarnings", + 'ignore:invalid escape sequence:DeprecationWarning', ] addopts = "--pdbcls IPython.terminal.debugger:TerminalPdb" env = [ diff --git a/bot/s/build b/bot/s/build index 699695e5f..5ada66946 100755 --- a/bot/s/build +++ b/bot/s/build @@ -7,7 +7,7 @@ set -x # https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables LABEL="$CIRCLE_SHA1" -if [ ! -z "$CIRCLE_TAG" ]; then +if [ -n "$CIRCLE_TAG" ]; then LABEL="$CIRCLE_TAG" fi diff --git a/docs/package.json b/docs/package.json index 08f394804..0f5161fae 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,5 +1,9 @@ { "private": true, + "volta": { + "node": "12.4.0", + "yarn": "1.22.17" + }, "devDependencies": { "@types/react": "^16.9.17", "docusaurus": "^1.14.3", diff --git a/web_api/.python-version b/web_api/.python-version new file mode 100644 index 000000000..214b521fe --- /dev/null +++ b/web_api/.python-version @@ -0,0 +1 @@ +3.7.13 diff --git a/web_api/pyproject.toml b/web_api/pyproject.toml index 96c48e3b0..c9a4cc233 100644 --- a/web_api/pyproject.toml +++ b/web_api/pyproject.toml @@ -112,7 +112,7 @@ env = [ "STRIPE_WEBHOOK_SECRET=whsec_someWebhookSecret", "STRIPE_SECRET_KEY=sk_someStripeSecretKey", "STRIPE_PUBLISHABLE_API_KEY=pk_test_someExampleStripeApiKey", - "REDIS_URL=redis://localhost:6379", + "REDIS_URL=redis://127.0.0.1:6379", "DEBUG=1", ] filterwarnings = [ diff --git a/web_api/s/build b/web_api/s/build index e0a46a4f4..0ffb3e7d8 100755 --- a/web_api/s/build +++ b/web_api/s/build @@ -7,7 +7,7 @@ set -x # https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables LABEL="$CIRCLE_SHA1" -if [ ! -z "$CIRCLE_TAG" ]; then +if [ -n "$CIRCLE_TAG" ]; then LABEL="$CIRCLE_TAG" fi diff --git a/web_api/s/squawk.py b/web_api/s/squawk.py index ef649d6d9..d13b54238 100755 --- a/web_api/s/squawk.py +++ b/web_api/s/squawk.py @@ -1,13 +1,11 @@ -#!/usr/bin/env python3 +from __future__ import annotations import logging import os import re import subprocess -from dataclasses import dataclass +import sys from pathlib import Path -from shutil import which -from typing import Mapping, Optional SQUAWK_VERSION = "0.5.0" APP_LABEL = "web_api" @@ -19,14 +17,10 @@ log = logging.getLogger(__file__) -def is_installed(name: str) -> bool: - return which(name) is not None +_MIGRATION_REGEX = re.compile(r"^(\d{4,}_\w+)\.py$") -MIGRATION_REGEX = re.compile(r"^(\d{4,}_\w+)\.py$") - - -def get_migration_id(filepath: str) -> Optional[str]: +def _get_migration_id(filepath: str) -> str | None: """ valid migrations: 0001_initial.py @@ -38,41 +32,22 @@ def get_migration_id(filepath: str) -> Optional[str]: For a valid migration 0001_initial.py, return 0001_initial. """ filename = Path(filepath).name - match = MIGRATION_REGEX.match(filename) + match = _MIGRATION_REGEX.match(filename) if match is None: return None return match.groups()[0] -@dataclass(frozen=True) -class PRInfo: - owner: str - repo: str - pr_number: str - - -def get_pr_info(env: Mapping[str, str]) -> Optional[PRInfo]: - circle_pr = env.get("CIRCLE_PULL_REQUEST") - if circle_pr is None: - return None - _, _, _, owner, repo, _, pr_number = circle_pr.split("/") - - return PRInfo(owner=owner, repo=repo, pr_number=pr_number) - - -def main() -> None: - # circle's built in git checkout code clobbers the `master` ref so we do the - # following to make it not point to the current ref. - # https://discuss.circleci.com/t/git-checkout-of-a-branch-destroys-local-reference-to-master/23781/7 - if os.getenv("CIRCLECI") and os.getenv("CIRCLE_BRANCH") != "master": - subprocess.run(["git", "branch", "-f", "master", "origin/master"], check=True) - +def _get_migration_ids() -> list[tuple[str, str]]: + current_branch = subprocess.run( + ["git", "branch", "--show-current"], capture_output=True, check=True + ) diff_cmd = [ "git", "--no-pager", "diff", "--name-only", - "master...", + f"master...{current_branch}", MIGRATIONS_DIRECTORY, ] @@ -82,13 +57,23 @@ def main() -> None: .stdout.decode() .split() ): - migration_id = get_migration_id(p) + migration_id = _get_migration_id(p) if migration_id is None: continue changed_migrations_ids.append((migration_id, p)) + return changed_migrations_ids - log.info("found migrations: %s", changed_migrations_ids) +def main() -> None: + try: + changed_migrations_ids = _get_migration_ids() + except subprocess.CalledProcessError as e: + print(f"stderr: {e.stderr.decode()}") + print(f"stdout: {e.stdout.decode()}") + print(f"status code: {e.returncode}") + sys.exit(1) + + log.info("found migrations: %s", changed_migrations_ids) # get sqlmigrate to behave os.environ.setdefault("STRIPE_ANNUAL_PLAN_ID", "1") os.environ.setdefault("DEBUG", "1") @@ -125,31 +110,6 @@ def main() -> None: log.info("sql files found: %s", output_files) - if not output_files: - return - - if not is_installed("squawk"): - subprocess.run(["npm", "config", "set", "unsafe-perm", "true"], check=True) - log.info("squawk not found, installing") - subprocess.run( - ["npm", "install", "-g", f"squawk-cli@{SQUAWK_VERSION}"], check=True - ) - - pr_info = get_pr_info(os.environ) - assert pr_info is not None - log.info(pr_info) - - os.environ.setdefault("SQUAWK_GITHUB_PR_NUMBER", pr_info.pr_number) - os.environ.setdefault("SQUAWK_GITHUB_REPO_NAME", pr_info.repo) - os.environ.setdefault("SQUAWK_GITHUB_REPO_OWNER", pr_info.owner) - - res = subprocess.run( - ["squawk", "upload-to-github", *output_files], - capture_output=True, - ) - log.info(res) - res.check_returncode() - if __name__ == "__main__": main() diff --git a/web_ui/package.json b/web_ui/package.json index c55436b46..3de25b90c 100644 --- a/web_ui/package.json +++ b/web_ui/package.json @@ -114,6 +114,10 @@ "devDependencies": { "prettier": "^1.19.1" }, + "volta": { + "node": "12.4.0", + "yarn": "1.22.17" + }, "jest": { "roots": [ "/src" diff --git a/web_ui/s/build b/web_ui/s/build index 2e06a12f0..2b7d88cc5 100755 --- a/web_ui/s/build +++ b/web_ui/s/build @@ -7,7 +7,7 @@ set -x # https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables LABEL="$CIRCLE_SHA1" -if [ ! -z "$CIRCLE_TAG" ]; then +if [ -n "$CIRCLE_TAG" ]; then LABEL="$CIRCLE_TAG" fi