diff --git a/.env b/.env deleted file mode 100644 index 91a59aa422..0000000000 --- a/.env +++ /dev/null @@ -1,8 +0,0 @@ -# Sets the base name for containers and networks for docker-compose to -# "socorro_". This is normally set by the name of this directory, but -# if you clone the repository with a different directory name, then -# you end up with a different project name and then everything is hosed. -# Setting it here fixes that. -COMPOSE_PROJECT_NAME=socorro - -PYTHONUNBUFFERED=true diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 0c542e4bba..14b8d9ce8a 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -34,22 +34,23 @@ jobs: "$GITHUB_REF_NAME" \ "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > version.json + - name: Install just + run: sudo apt-get update && sudo apt-get install just - name: Output version.json run: cat version.json - name: Build Docker images run: | - make build + just build docker compose images - name: Verify requirements.txt contains correct dependencies run: | - docker run --rm local/socorro_app shell ./bin/verify_reqs.sh + just shell ./bin/verify_reqs.sh - name: Run lint check run: | - docker run --rm local/socorro_app shell ./bin/lint.sh + just lint - name: Run tests run: | - make my.env - docker compose run --rm test-ci shell ./bin/test.sh + just test-ci - name: Set Docker image tag to "latest" for updates of the main branch if: github.ref == 'refs/heads/main' diff --git a/.gitignore b/.gitignore index e6773984b1..f44b93d253 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,7 @@ symbols/ .devcontainer-build .cache/ docker-compose.override.yml -my.env +.env # docs things docs/_build/ diff --git a/Makefile b/Makefile deleted file mode 100644 index 5c6c1f2bd4..0000000000 --- a/Makefile +++ /dev/null @@ -1,158 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Include my.env and export it so variables set in there are available -# in the Makefile. -include my.env -export - -# Set these in the environment to override them. This is helpful for -# development if you have file ownership problems because the user -# in the container doesn't match the user on your host. -SOCORRO_UID ?= 10001 -SOCORRO_GID ?= 10001 - -# Set this in the environment to force --no-cache docker builds. -DOCKER_BUILD_OPTS := -ifeq (1, ${NOCACHE}) -DOCKER_BUILD_OPTS := --no-cache -endif - -DOCKER := $(shell which docker) -DC=${DOCKER} compose - -.DEFAULT_GOAL := help -.PHONY: help -help: - @echo "Usage: make RULE" - @echo "" - @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \ - | grep -v grep \ - | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \ - | column -t -s '|' - @echo "" - @echo "See https://socorro.readthedocs.io/ for more documentation." - -my.env: - @if [ ! -f my.env ]; \ - then \ - echo "Copying my.env.dist to my.env..."; \ - cp docker/config/my.env.dist my.env; \ - fi - -.docker-build: - make build - -.devcontainer-build: - make devcontainerbuild - -.PHONY: build -build: my.env ## | Build docker images. - ${DC} build ${DOCKER_BUILD_OPTS} --build-arg userid=${SOCORRO_UID} --build-arg groupid=${SOCORRO_GID} --progress plain app - ${DC} build --progress plain oidcprovider fakesentry gcs-emulator - ${DC} build --progress plain statsd postgresql memcached elasticsearch symbolsserver - touch .docker-build - -.PHONY: devcontainerbuild -devcontainerbuild: my.env ## | Build VS Code development container. - ${DC} build devcontainer - touch .devcontainer-build - -.PHONY: devcontainer -devcontainer: my.env .devcontainer-build ## | Run VS Code development container. - ${DC} up --detach devcontainer - -.PHONY: setup -setup: my.env .docker-build ## | Set up Postgres, Elasticsearch, local Pub/Sub, and local GCS services. - ${DC} run --rm app shell /app/bin/setup_services.sh - -.PHONY: updatedata -updatedata: my.env ## | Add/update necessary database data. - ${DC} run --rm app shell /app/bin/update_data.sh - -.PHONY: run -run: my.env ## | Run processor, webapp, fakesentry, symbolsserver, and required services. - ${DC} up \ - --attach processor \ - --attach webapp \ - --attach fakesentry \ - --attach symbolsserver \ - processor webapp fakesentry symbolsserver - -.PHONY: runservices -runservices: my.env ## | Run service containers (Postgres, Pub/Sub, etc) - ${DC} up -d --remove-orphans \ - elasticsearch \ - gcs-emulator \ - memcached \ - postgresql \ - pubsub \ - statsd \ - symbolsserver - -.PHONY: runsubmitter -runsubmitter: my.env ## | Run stage_submitter and fakecollector - ${DC} up \ - --attach stage_submitter \ - --attach fakecollector \ - stage_submitter fakecollector - -.PHONY: stop -stop: my.env ## | Stop all service containers. - ${DC} stop - -.PHONY: shell -shell: my.env .docker-build ## | Open a shell in the app container. - ${DC} run --rm --entrypoint /bin/bash app - -.PHONY: clean -clean: ## | Remove all build, test, coverage, and Python artifacts. - -rm .docker-build* - -rm -rf .cache - @echo "Skipping deletion of symbols/ in case you have data in there." - -.PHONY: docs -docs: my.env .docker-build ## | Generate Sphinx HTML documetation. - ${DC} run --rm --user ${SOCORRO_UID} app shell make -C docs/ clean - ${DC} run --rm --user ${SOCORRO_UID} app shell make -C docs/ html - -.PHONY: lint -lint: my.env ## | Lint code. - ${DC} run --rm --no-deps app shell ./bin/lint.sh - -.PHONY: lintfix -lintfix: my.env ## | Reformat code. - ${DC} run --rm --no-deps app shell ./bin/lint.sh --fix - -.PHONY: psql -psql: my.env .docker-build ## | Open psql cli. - @echo "NOTE: Password is 'postgres'." - ${DC} run --rm postgresql psql -h postgresql -U postgres -d socorro - -.PHONY: test -test: my.env .docker-build ## | Run unit tests. - ${DC} run --rm test shell ./bin/test.sh - -.PHONY: test-ci -test-ci: my.env .docker-build ## | Run unit tests in CI. - # Run tests in test-ci which doesn't volume mount local directory - ${DC} run --rm test-ci shell ./bin/test.sh - -.PHONY: testshell -testshell: my.env .docker-build ## | Open a shell in the test environment. - ${DC} run --rm --entrypoint /bin/bash test - -.PHONY: rebuildreqs -rebuildreqs: .env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes \ - --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in - -.PHONY: updatereqs -updatereqs: .env .docker-build ## | Update deps in requirements.txt file. - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras --upgrade - -.PHONY: servicestatus -servicestatus: .env .docker-build ## | Check service status - ${DC} run --rm --no-deps app shell service-status diff --git a/bin/lint.sh b/bin/lint.sh index 159a5346f1..fa22ba47c5 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -16,11 +16,18 @@ FILES="socorro-cmd docker socorro webapp bin" PYTHON_VERSION=$(python --version) -if [[ "${1:-}" == "--fix" ]]; then +if [[ "${1:-}" == "--help" ]]; then + echo "Usage: $0 [OPTIONS]" + echo + echo " Lint code" + echo + echo "Options:" + echo " --help Show this message and exit." + echo " --fix Reformat code." +elif [[ "${1:-}" == "--fix" ]]; then echo ">>> ruff fix (${PYTHON_VERSION})" ruff format $FILES ruff check --fix $FILES - else echo ">>> ruff (${PYTHON_VERSION})" ruff check $FILES diff --git a/docker-compose.yml b/docker-compose.yml index 236ba09104..f48f9fa09f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,24 @@ --- +# Sets the base name for containers and networks for docker-compose to +# "socorro_". This is normally set by the name of this directory, but +# if you clone the repository with a different directory name, then +# you end up with a different project name and then everything is hosed. +# Setting it here fixes that. +name: socorro + services: # Socorro app image app: build: context: . dockerfile: docker/Dockerfile + args: + userid: ${USE_UID:-10001} + groupid: ${USE_GID:-10001} image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -65,7 +75,7 @@ services: env_file: - docker/config/local_dev.env - docker/config/test.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -81,7 +91,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -98,7 +108,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -113,7 +123,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -134,7 +144,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - gcs-emulator @@ -178,7 +188,7 @@ services: image: mozilla/socorro_collector:latest env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - gcs-emulator - pubsub @@ -194,7 +204,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env command: ["fakecollector"] ports: - "9000:8000" @@ -205,7 +215,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env command: ["symbolsserver"] stop_signal: SIGINT ports: diff --git a/docker/config/my.env.dist b/docker/config/.env.dist similarity index 95% rename from docker/config/my.env.dist rename to docker/config/.env.dist index c94111304d..36e051ba32 100644 --- a/docker/config/my.env.dist +++ b/docker/config/.env.dist @@ -2,14 +2,16 @@ # environment. Since these are specific to your local development environment, # don't check them in. +PYTHONUNBUFFERED=true + # --------------------------------------------- # Docker settings # --------------------------------------------- # If you want to set the uid and gid of the app user that we use in the # containers, you can do that with these two variables. -# SOCORRO_UID= -# SOCORRO_GID= +# USE_UID= +# USE_GID= # --------------------------------------------- # processor settings diff --git a/justfile b/justfile new file mode 100644 index 0000000000..8e6f02e158 --- /dev/null +++ b/justfile @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +default: + @just --unsorted --list --justfile {{justfile()}} + +env: + @if [ ! -f .env ]; \ + then \ + echo "Copying docker/config/.env.dist to .env..."; \ + cp docker/config/.env.dist .env; \ + fi + +# Build docker images +build *args: env + docker compose build --progress plain {{args}} + +# Set up Postgres, Elasticsearch, local Pub/Sub, and local GCS services. +setup: env + docker compose run --rm app shell /app/bin/setup_services.sh + +# Add/update necessary database data. +update-data: env + docker compose run --rm app shell /app/bin/update_data.sh + +# Run services, defaults to socorro and fakesentry for debugging +run *args='--attach=processor --attach=webapp --attach=fakesentry processor webapp': env + docker compose up {{args}} + +# Run stage submitter and fake collector +run-submitter *args='--attach=stage_submitter --attach=fakecollector': env + docker compose up \ + {{args}} \ + stage_submitter \ + fakecollector + +# Stop service containers. +stop *args: env + docker compose stop {{args}} + +# Remove service containers and networks. +down *args: env + docker compose down {{args}} + +# Open a shell in the app container. +shell *args: env + docker compose run --rm --entrypoint /bin/bash app {{args}} + +# Open a shell in the test environment. +testshell *args: env + docker compose run --rm --entrypoint /bin/bash test {{args}} + +# Remove all build, test, coverage, and Python artifacts. +clean: + -rm -rf .cache + @echo "Skipping deletion of symbols/ in case you have data in there." + +# Generate Sphinx HTML documetation. +docs: env + docker compose run --rm --user ${SOCORRO_UID} app shell make -C docs/ clean + docker compose run --rm --user ${SOCORRO_UID} app shell make -C docs/ html + +# Lint code, or use --fix to reformat and apply auto-fixes for lint. +lint *args: env + docker compose run --rm --no-deps app shell ./bin/lint.sh {{args}} + +# Open psql cli. +psql *args: env + docker compose run --rm postgresql psql postgresql://postgres:postgres@postgresql/socorro {{args}} + +# Run tests. +test *args: env + docker compose run --rm test shell ./bin/test.sh {{args}} + +# Run tests in test-ci which doesn't volume mount local directory. +test-ci *args: env + docker compose run --rm test-ci shell ./bin/test.sh {{args}} + +# Build requirements.txt file after requirements.in changes. +rebuildreqs *args: env + docker compose run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras {{args}} + docker compose run --rm --no-deps app shell pip-compile --generate-hashes \ + --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in + +# Check how far behind different server environments are from main tip. +service-status *args: env + docker compose run --rm --no-deps app shell service-status {{args}}