diff --git a/.env b/.env deleted file mode 100644 index 91a59aa422..0000000000 --- a/.env +++ /dev/null @@ -1,8 +0,0 @@ -# Sets the base name for containers and networks for docker-compose to -# "socorro_". This is normally set by the name of this directory, but -# if you clone the repository with a different directory name, then -# you end up with a different project name and then everything is hosed. -# Setting it here fixes that. -COMPOSE_PROJECT_NAME=socorro - -PYTHONUNBUFFERED=true diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dd0bf17e38..071d39ab42 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,7 +8,9 @@ updates: open-pull-requests-limit: 10 - package-ecosystem: "docker" - directory: "/docker" + directories: + - "/docker" + - "/docker/images/*" schedule: interval: "weekly" open-pull-requests-limit: 10 diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 0c542e4bba..a24c6e6468 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -18,7 +18,10 @@ jobs: contents: read deployments: write id-token: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + # Disable docker compose volume mounts in docker-compose.override.yml + COMPOSE_FILE: docker-compose.yml steps: - uses: actions/checkout@v4 - name: Get info @@ -36,20 +39,21 @@ jobs: "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > version.json - name: Output version.json run: cat version.json + - name: Install just + run: sudo apt-get update && sudo apt-get install -y just - name: Build Docker images run: | - make build + just build docker compose images - name: Verify requirements.txt contains correct dependencies run: | - docker run --rm local/socorro_app shell ./bin/verify_reqs.sh + just verify-reqs - name: Run lint check run: | - docker run --rm local/socorro_app shell ./bin/lint.sh + just lint - name: Run tests run: | - make my.env - docker compose run --rm test-ci shell ./bin/test.sh + just test - name: Set Docker image tag to "latest" for updates of the main branch if: github.ref == 'refs/heads/main' diff --git a/.gitignore b/.gitignore index e6773984b1..834fa45cd1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,8 +24,7 @@ symbols/ .docker-build* .devcontainer-build .cache/ -docker-compose.override.yml -my.env +.env # docs things docs/_build/ diff --git a/Makefile b/Makefile deleted file mode 100644 index e9b97bf9aa..0000000000 --- a/Makefile +++ /dev/null @@ -1,154 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Include my.env and export it so variables set in there are available -# in the Makefile. -include my.env -export - -# Set these in the environment to override them. This is helpful for -# development if you have file ownership problems because the user -# in the container doesn't match the user on your host. -SOCORRO_UID ?= 10001 -SOCORRO_GID ?= 10001 - -# Set this in the environment to force --no-cache docker builds. -DOCKER_BUILD_OPTS := -ifeq (1, ${NOCACHE}) -DOCKER_BUILD_OPTS := --no-cache -endif - -DOCKER := $(shell which docker) -DC=${DOCKER} compose - -.DEFAULT_GOAL := help -.PHONY: help -help: - @echo "Usage: make RULE" - @echo "" - @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \ - | grep -v grep \ - | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \ - | column -t -s '|' - @echo "" - @echo "See https://socorro.readthedocs.io/ for more documentation." - -my.env: - @if [ ! -f my.env ]; \ - then \ - echo "Copying my.env.dist to my.env..."; \ - cp docker/config/my.env.dist my.env; \ - fi - -.docker-build: - make build - -.devcontainer-build: - make devcontainerbuild - -.PHONY: build -build: my.env ## | Build docker images. - ${DC} build ${DOCKER_BUILD_OPTS} --build-arg userid=${SOCORRO_UID} --build-arg groupid=${SOCORRO_GID} --progress plain app - ${DC} build --progress plain oidcprovider fakesentry gcs-emulator - ${DC} build --progress plain statsd postgresql memcached elasticsearch symbolsserver - touch .docker-build - -.PHONY: devcontainerbuild -devcontainerbuild: my.env ## | Build VS Code development container. - ${DC} build devcontainer - touch .devcontainer-build - -.PHONY: devcontainer -devcontainer: my.env .devcontainer-build ## | Run VS Code development container. - ${DC} up --detach devcontainer - -.PHONY: setup -setup: my.env .docker-build ## | Set up Postgres, Elasticsearch, local Pub/Sub, and local GCS services. - ${DC} run --rm app shell /app/bin/setup_services.sh - -.PHONY: updatedata -updatedata: my.env ## | Add/update necessary database data. - ${DC} run --rm app shell /app/bin/update_data.sh - -.PHONY: run -run: my.env ## | Run processor, webapp, fakesentry, symbolsserver, and required services. - ${DC} up \ - --attach processor \ - --attach webapp \ - --attach fakesentry \ - --attach symbolsserver \ - processor webapp fakesentry symbolsserver - -.PHONY: runservices -runservices: my.env ## | Run service containers (Postgres, Pub/Sub, etc) - ${DC} up -d --remove-orphans \ - elasticsearch \ - gcs-emulator \ - memcached \ - postgresql \ - pubsub \ - statsd \ - symbolsserver - -.PHONY: runsubmitter -runsubmitter: my.env ## | Run stage_submitter and fakecollector - ${DC} up \ - --attach stage_submitter \ - --attach fakecollector \ - stage_submitter fakecollector - -.PHONY: stop -stop: my.env ## | Stop all service containers. - ${DC} stop - -.PHONY: shell -shell: my.env .docker-build ## | Open a shell in the app container. - ${DC} run --rm --entrypoint /bin/bash app - -.PHONY: clean -clean: ## | Remove all build, test, coverage, and Python artifacts. - -rm .docker-build* - -rm -rf .cache - @echo "Skipping deletion of symbols/ in case you have data in there." - -.PHONY: docs -docs: my.env .docker-build ## | Generate Sphinx HTML documetation. - ${DC} run --rm --user ${SOCORRO_UID} app shell make -C docs/ clean - ${DC} run --rm --user ${SOCORRO_UID} app shell make -C docs/ html - -.PHONY: lint -lint: my.env ## | Lint code. - ${DC} run --rm --no-deps app shell ./bin/lint.sh - -.PHONY: lintfix -lintfix: my.env ## | Reformat code. - ${DC} run --rm --no-deps app shell ./bin/lint.sh --fix - -.PHONY: psql -psql: my.env .docker-build ## | Open psql cli. - @echo "NOTE: Password is 'postgres'." - ${DC} run --rm postgresql psql -h postgresql -U postgres -d socorro - -.PHONY: test -test: my.env .docker-build ## | Run unit tests. - ${DC} run --rm test shell ./bin/test.sh - -.PHONY: test-ci -test-ci: my.env .docker-build ## | Run unit tests in CI. - # Run tests in test-ci which doesn't volume mount local directory - ${DC} run --rm test-ci shell ./bin/test.sh - -.PHONY: testshell -testshell: my.env .docker-build ## | Open a shell in the test environment. - ${DC} run --rm --entrypoint /bin/bash test - -.PHONY: rebuildreqs -rebuildreqs: .env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes \ - --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in - -.PHONY: updatereqs -updatereqs: .env .docker-build ## | Update deps in requirements.txt file. - ${DC} run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras --upgrade diff --git a/bin/gcs_cli.py b/bin/gcs_cli.py deleted file mode 100755 index a529ce8070..0000000000 --- a/bin/gcs_cli.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Manipulate emulated GCS storage. - -# Usage: ./bin/gcs_cli.py CMD - -import os -from pathlib import Path, PurePosixPath - -import click - -from google.auth.credentials import AnonymousCredentials -from google.cloud import storage -from google.cloud.exceptions import NotFound - - -def get_endpoint_url(): - return os.environ["STORAGE_EMULATOR_HOST"] - - -def get_client(): - project_id = os.environ["STORAGE_PROJECT_ID"] - return storage.Client(credentials=AnonymousCredentials(), project=project_id) - - -@click.group() -def gcs_group(): - """Local dev environment GCS manipulation script""" - - -@gcs_group.command("create") -@click.argument("bucket_name") -def create_bucket(bucket_name): - """Creates a bucket - - Specify BUCKET_NAME. - - """ - # README at https://github.com/fsouza/fake-gcs-server - endpoint_url = get_endpoint_url() - - client = get_client() - - try: - client.get_bucket(bucket_name) - click.echo(f"GCS bucket {bucket_name!r} exists in {endpoint_url!r}.") - except NotFound: - client.create_bucket(bucket_name) - click.echo(f"GCS bucket {bucket_name!r} in {endpoint_url!r} created.") - - -@gcs_group.command("delete") -@click.argument("bucket_name") -def delete_bucket(bucket_name): - """Deletes a bucket - - Specify BUCKET_NAME. - - """ - # README at https://github.com/fsouza/fake-gcs-server/ - endpoint_url = get_endpoint_url() - - client = get_client() - - bucket = None - - try: - bucket = client.get_bucket(bucket_name) - except NotFound: - click.echo(f"GCS bucket {bucket_name!r} at {endpoint_url!r} does not exist.") - return - - # delete blobs before deleting bucket, because bucket.delete(force=True) doesn't - # work if there are more than 256 blobs in the bucket. - for blob in bucket.list_blobs(): - blob.delete() - - bucket.delete() - click.echo(f"GCS bucket {bucket_name!r} at {endpoint_url!r} deleted.") - - -@gcs_group.command("list_buckets") -@click.option("--details/--no-details", default=True, type=bool, help="With details") -def list_buckets(details): - """List GCS buckets""" - - client = get_client() - - buckets = client.list_buckets() - for bucket in buckets: - if details: - # https://cloud.google.com/storage/docs/json_api/v1/buckets#resource-representations - click.echo(f"{bucket.name}\t{bucket.time_created}") - else: - click.echo(f"{bucket.name}") - - -@gcs_group.command("list_objects") -@click.option("--details/--no-details", default=True, type=bool, help="With details") -@click.argument("bucket_name") -def list_objects(bucket_name, details): - """List contents of a bucket""" - - client = get_client() - - try: - client.get_bucket(bucket_name) - except NotFound: - click.echo(f"GCS bucket {bucket_name!r} does not exist.") - return - - blobs = list(client.list_blobs(bucket_name)) - if blobs: - for blob in blobs: - # https://cloud.google.com/storage/docs/json_api/v1/objects#resource-representations - if details: - click.echo(f"{blob.name}\t{blob.size}\t{blob.updated}") - else: - click.echo(f"{blob.name}") - else: - click.echo("No objects in bucket.") - - -@gcs_group.command("upload") -@click.argument("source") -@click.argument("destination") -def upload(source, destination): - """Upload files to a bucket - - SOURCE is a path to a file or directory of files. will recurse on directory trees - - DESTINATION is a path to a file or directory in the bucket. If SOURCE is a - directory or DESTINATION ends with "/", then DESTINATION is treated as a directory. - """ - - client = get_client() - - # remove protocol from destination if present - destination = destination.split("://", 1)[-1] - bucket_name, _, prefix = destination.partition("/") - prefix_path = PurePosixPath(prefix) - - try: - bucket = client.get_bucket(bucket_name) - except NotFound as e: - raise click.ClickException(f"GCS bucket {bucket_name!r} does not exist.") from e - - source_path = Path(source) - if not source_path.exists(): - raise click.ClickException(f"local path {source!r} does not exist.") - source_is_dir = source_path.is_dir() - if source_is_dir: - sources = [p for p in source_path.rglob("*") if not p.is_dir()] - else: - sources = [source_path] - if not sources: - raise click.ClickException(f"No files in directory {source!r}.") - for path in sources: - if source_is_dir: - # source is a directory so treat destination as a directory - key = str(prefix_path / path.relative_to(source_path)) - elif prefix == "" or prefix.endswith("/"): - # source is a file but destination is a directory, preserve file name - key = str(prefix_path / path.name) - else: - key = prefix - blob = bucket.blob(key) - blob.upload_from_filename(path) - click.echo(f"Uploaded gs://{bucket_name}/{key}") - - -def main(argv=None): - argv = argv or [] - gcs_group(argv) - - -if __name__ == "__main__": - gcs_group() diff --git a/bin/license-check.py b/bin/license-check.py deleted file mode 100755 index 05b755288d..0000000000 --- a/bin/license-check.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -""" -This script checks files for license headers. - -This requires Python 3.8+ to run. - -See https://github.com/willkg/socorro-release/#readme for details. - -repo: https://github.com/willkg/socorro-release/ -sha: d19f45bc9eedae34de2905cdd4adf7b9fd03f870 - -""" - -import argparse -import pathlib -import subprocess -import sys - - -DESCRIPTION = ( - "Checks files in specified directory for license headers. " - + "If you don't specify a target, it'll check all files in \"git ls-files\"." -) - -# From https://www.mozilla.org/en-US/MPL/2.0/ -MPLV2 = [ - "This Source Code Form is subject to the terms of the Mozilla Public", - "License, v. 2.0. If a copy of the MPL was not distributed with this", - "file, You can obtain one at https://mozilla.org/MPL/2.0/.", -] - - -LANGUAGE_DATA = {".py": {"comment": ("#",)}} - - -def is_code_file(path: pathlib.Path): - """Determines whether the file is a code file we need to check. - - :param path: the Path for the file - - :returns: True if it's a code file to check, False otherwise. - - """ - if not path.is_file(): - return False - ending: pathlib.Path = path.suffix - return ending in LANGUAGE_DATA - - -def has_license_header(path: pathlib.Path): - """Determines if file at path has an MPLv2 license header. - - :param path: the Path for the file - - :returns: True if it does, False if it doesn't. - - """ - ending: pathlib.Path = path.suffix - comment_indicators = LANGUAGE_DATA[ending]["comment"] - - header = [] - with open(path, "r") as fp: - firstline = True - for line in fp.readlines(): - if firstline and line.startswith("#!"): - firstline = False - continue - - line = line.strip() - # NOTE(willkg): this doesn't handle multiline comments like in C++ - for indicator in comment_indicators: - line = line.strip(indicator) - line = line.strip() - - # Skip blank lines - if not line: - continue - - header.append(line) - if len(header) == len(MPLV2): - if header[: len(MPLV2)] == MPLV2: - return True - else: - break - - return False - - -def main(args): - parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument( - "-l", "--file-only", action="store_true", help="print files only" - ) - parser.add_argument("--verbose", action="store_true", help="verbose output") - parser.add_argument("target", help="file or directory tree to check", nargs="?") - - parsed = parser.parse_args(args) - - if parsed.target: - target = pathlib.Path(parsed.target) - if not target.exists(): - if not parsed.file_only: - print(f"Not a valid file or directory: {target}") - return 1 - - if target.is_file(): - targets = [target] - - elif target.is_dir(): - targets = list(target.rglob("*")) - - else: - ret = subprocess.check_output(["git", "ls-files"]) - targets = [ - pathlib.Path(target.strip()) for target in ret.decode("utf-8").splitlines() - ] - - missing_headers = 0 - - # Iterate through all the files in this target directory - for path in targets: - if parsed.verbose: - print(f"Checking {path}") - if is_code_file(path) and not has_license_header(path): - missing_headers += 1 - if parsed.file_only: - print(str(path)) - else: - print(f"File {path} does not have license header.") - - if missing_headers > 0: - if not parsed.file_only: - print(f"Files with missing headers: {missing_headers}") - print("") - print("Add this:") - print("") - print("\n".join(MPLV2)) - return 1 - - if not parsed.file_only: - print("No files missing headers.") - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/bin/lint.sh b/bin/lint.sh index f292215b62..fa22ba47c5 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -16,11 +16,18 @@ FILES="socorro-cmd docker socorro webapp bin" PYTHON_VERSION=$(python --version) -if [[ "${1:-}" == "--fix" ]]; then +if [[ "${1:-}" == "--help" ]]; then + echo "Usage: $0 [OPTIONS]" + echo + echo " Lint code" + echo + echo "Options:" + echo " --help Show this message and exit." + echo " --fix Reformat code." +elif [[ "${1:-}" == "--fix" ]]; then echo ">>> ruff fix (${PYTHON_VERSION})" ruff format $FILES ruff check --fix $FILES - else echo ">>> ruff (${PYTHON_VERSION})" ruff check $FILES @@ -28,13 +35,13 @@ else echo ">>> license check (${PYTHON_VERSION})" if [[ -d ".git" ]]; then - # If the .git directory exists, we can let license-check.py do + # If the .git directory exists, we can let license-check do # git ls-files. - python bin/license-check.py + license-check else # The .git directory doesn't exist, so run it on all the Python # files in the tree. - python bin/license-check.py . + license-check . fi echo ">>> eslint (js)" diff --git a/bin/process_crashes.sh b/bin/process_crashes.sh index 92acac127f..d4e56ed428 100755 --- a/bin/process_crashes.sh +++ b/bin/process_crashes.sh @@ -9,17 +9,23 @@ # # Usage: ./bin/process_crashes.sh # +# This should be run inside the app container. +# # You can use it with fetch_crashids. For example: # # socorro-cmd fetch_crashids --num=1 | ./bin/process_crashes.sh # -# Make sure to run the processor to do the actual processing. +# You can pause after fetching crash ids and before uploading them to the cloud +# storage bucket. This lets you adjust the crash reports. +# +# ./bin/process_crashes.sh --pause afd59adc-1cfd-4aa3-af2f-6b9ed0241029 # -# Note: This should be called from inside a container. +# Make sure to run the processor to do the actual processing. set -euo pipefail DATADIR=./crashdata_tryit_tmp +PAUSE=0 function cleanup { # Cleans up files generated by the script @@ -29,10 +35,24 @@ function cleanup { # Set up cleanup function to run on script exit trap cleanup EXIT +# If there's at least one argument, check for options +if [[ $# -ne 0 ]]; then + if [[ $1 == "--pause" ]]; then + PAUSE=1 + shift + fi +fi + +# Check for crash ids if [[ $# -eq 0 ]]; then if [ -t 0 ]; then # If stdin is a terminal, then there's no input - echo "Usage: process_crashes.sh CRASHID [CRASHID ...]" + echo "Usage: process_crashes.sh [--pause] CRASHID [CRASHID ...]" + exit 1 + fi + + if [[ "${PAUSE}" == "1" ]]; then + echo "Can't use PAUSE and process crash ids from stdin. Exiting." exit 1 fi @@ -45,6 +65,11 @@ mkdir "${DATADIR}" || echo "${DATADIR} already exists." # Pull down the data for all crashes ./socorro-cmd fetch_crash_data "${DATADIR}" $@ +# Pause (if requested) to let user make changes to crash data +if [[ "${PAUSE}" == "1" ]]; then + read -r -n 1 -p "Pausing... Any key to continue." pauseinput +fi + # Make the bucket and sync contents ./socorro-cmd gcs create "${CRASHSTORAGE_GCS_BUCKET}" ./socorro-cmd gcs upload "${DATADIR}" "${CRASHSTORAGE_GCS_BUCKET}" diff --git a/bin/pubsub_cli.py b/bin/pubsub_cli.py deleted file mode 100755 index 02de6ae156..0000000000 --- a/bin/pubsub_cli.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Pub/Sub manipulation script. -# -# Note: Run this in the base container which has access to Pub/Sub. -# -# Usage: ./bin/pubsub_cli.py [SUBCOMMAND] - -import sys - -import click -from google.cloud import pubsub_v1 -from google.api_core.exceptions import AlreadyExists, NotFound - -from socorro import settings - - -@click.group() -def pubsub_group(): - """Local dev environment Pub/Sub emulator manipulation script.""" - - -@pubsub_group.command("list_topics") -@click.argument("project_id") -@click.pass_context -def list_topics(ctx, project_id): - """List topics for this project.""" - click.echo(f"Listing topics in project {project_id}.") - publisher = pubsub_v1.PublisherClient() - - for topic in publisher.list_topics(project=f"projects/{project_id}"): - click.echo(topic.name) - - -@pubsub_group.command("list_subscriptions") -@click.argument("project_id") -@click.argument("topic_name") -@click.pass_context -def list_subscriptions(ctx, project_id, topic_name): - """List subscriptions for a given topic.""" - click.echo(f"Listing subscriptions in topic {topic_name!r}:") - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) - - for subscription in publisher.list_topic_subscriptions(topic=topic_path): - click.echo(subscription) - - -@pubsub_group.command("create_topic") -@click.argument("project_id") -@click.argument("topic_name") -@click.pass_context -def create_topic(ctx, project_id, topic_name): - """Create topic.""" - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) - - try: - publisher.create_topic(name=topic_path) - click.echo(f"Topic created: {topic_path}") - except AlreadyExists: - click.echo("Topic already created.") - - -@pubsub_group.command("create_subscription") -@click.argument("project_id") -@click.argument("topic_name") -@click.argument("subscription_name") -@click.pass_context -def create_subscription(ctx, project_id, topic_name, subscription_name): - """Create subscription.""" - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) - try: - subscriber.create_subscription( - name=subscription_path, - topic=topic_path, - ack_deadline_seconds=600, - ) - click.echo(f"Subscription created: {subscription_path}") - except AlreadyExists: - click.echo("Subscription already created.") - - -@pubsub_group.command("delete_topic") -@click.argument("project_id") -@click.argument("topic_name") -@click.pass_context -def delete_topic(ctx, project_id, topic_name): - """Delete a topic and all subscriptions.""" - publisher = pubsub_v1.PublisherClient() - subscriber = pubsub_v1.SubscriberClient() - topic_path = publisher.topic_path(project_id, topic_name) - - # Delete all subscriptions - for subscription in publisher.list_topic_subscriptions(topic=topic_path): - click.echo(f"Deleting {subscription} ...") - subscriber.delete_subscription(subscription=subscription) - - # Delete topic - try: - publisher.delete_topic(topic=topic_path) - click.echo(f"Topic deleted: {topic_name}") - except NotFound: - click.echo(f"Topic {topic_name} does not exist.") - - -@pubsub_group.command("publish") -@click.argument("project_id") -@click.argument("topic_name") -@click.argument("crashids", nargs=-1) -@click.pass_context -def publish(ctx, project_id, topic_name, crashids): - """Publish crash_id to a given topic.""" - click.echo(f"Publishing crash ids to topic: {topic_name!r}:") - # configure publisher to group all crashids into a single batch - publisher = pubsub_v1.PublisherClient( - batch_settings=pubsub_v1.types.BatchSettings(max_messages=len(crashids)) - ) - topic_path = publisher.topic_path(project_id, topic_name) - - # Pull crash ids from stdin if there are any - if not crashids and not sys.stdin.isatty(): - crashids = list(click.get_text_stream("stdin").readlines()) - - if not crashids: - raise click.BadParameter( - "No crashids provided.", ctx=ctx, param="crashids", param_hint="crashids" - ) - - # publish all crashes before checking futures to allow for batching - futures = [ - publisher.publish(topic_path, crashid.encode("utf-8"), timeout=5) - for crashid in crashids - ] - for future in futures: - click.echo(future.result()) - - -@pubsub_group.command("pull") -@click.argument("project_id") -@click.argument("subscription_name") -@click.option("--ack/--no-ack", is_flag=True, default=False) -@click.option("--max-messages", default=1, type=int) -@click.pass_context -def pull(ctx, project_id, subscription_name, ack, max_messages): - """Pull crash id from a given subscription.""" - click.echo(f"Pulling crash id from subscription {subscription_name!r}:") - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) - - response = subscriber.pull( - subscription=subscription_path, - max_messages=max_messages, - return_immediately=True, - ) - if not response.received_messages: - return - - ack_ids = [] - for msg in response.received_messages: - click.echo(f"crash id: {msg.message.data}") - ack_ids.append(msg.ack_id) - - if ack: - # Acknowledges the received messages so they will not be sent again. - subscriber.acknowledge(subscription=subscription_path, ack_ids=ack_ids) - - -@pubsub_group.command("create-all") -@click.pass_context -def create_all(ctx): - """Create Pub/Sub queues related to processing.""" - options = settings.QUEUE_PUBSUB["options"] - project_id = options["project_id"] - queues = { - options["standard_topic_name"]: options["standard_subscription_name"], - options["priority_topic_name"]: options["priority_subscription_name"], - options["reprocessing_topic_name"]: options["reprocessing_subscription_name"], - } - for topic_name, subscription_name in queues.items(): - ctx.invoke(create_topic, project_id=project_id, topic_name=topic_name) - ctx.invoke( - create_subscription, - project_id=project_id, - topic_name=topic_name, - subscription_name=subscription_name, - ) - - -@pubsub_group.command("delete-all") -@click.pass_context -def delete_all(ctx): - """Delete Pub/Sub queues related to processing.""" - options = settings.QUEUE_PUBSUB["options"] - project_id = options["project_id"] - for topic_name in ( - options["standard_topic_name"], - options["priority_topic_name"], - options["reprocessing_topic_name"], - ): - ctx.invoke(delete_topic, project_id=project_id, topic_name=topic_name) - - -def main(argv=None): - argv = argv or [] - pubsub_group(argv) - - -if __name__ == "__main__": - pubsub_group() diff --git a/bin/release.py b/bin/release.py deleted file mode 100755 index 6912035dda..0000000000 --- a/bin/release.py +++ /dev/null @@ -1,483 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -""" -This script handles releases for this project. - -This has two subcommands: ``make-bug`` and ``make-tag``. See the help text for -both. - -This requires Python 3.8+ to run. - -Note: If you want to use ``pyproject.toml`` and you're using Python <3.11, this -also requires the tomli library. - -See https://github.com/willkg/socorro-release/#readme for details. - -repo: https://github.com/willkg/socorro-release/ -sha: d19f45bc9eedae34de2905cdd4adf7b9fd03f870 - -""" - -import argparse -import configparser -import datetime -import json -import os -import re -import shlex -import subprocess -import sys -from urllib.parse import urlencode -from urllib.request import urlopen - - -DESCRIPTION = """ -release.py makes it easier to create deploy bugs and push tags to trigger -deploys. - -For help, see: https://github.com/willkg/socorro-release/ -""" - -GITHUB_API = "https://api.github.com/" -BZ_CREATE_URL = "https://bugzilla.mozilla.org/enter_bug.cgi" -BZ_BUG_JSON_URL = "https://bugzilla.mozilla.org/rest/bug/" - -DEFAULT_CONFIG = { - # Bugzilla product and component to write new bugs in - "bugzilla_product": "", - "bugzilla_component": "", - # GitHub user and project name - "github_user": "", - "github_project": "", - # The name of the main branch - "main_branch": "", - # The tag structure using datetime formatting markers - "tag_name_template": "%Y.%m.%d", -} - -LINE = "=" * 80 - -# Recognize "bug-NNNNNNN", "bug NNNNNNN", and multi-bug variants -BUG_RE = re.compile(r"\bbug(?:s?:?\s*|-)([\d\s,\+&#and]+)\b", re.IGNORECASE) - -# Recognize "bug-NNNNNNN" -BUG_HYPHEN_PREFIX_RE = re.compile(r"bug-([\d]+)", re.IGNORECASE) - - -def get_config(): - """Generates configuration. - - This tries to pull configuration from: - - 1. the ``[tool.release]`` table from a ``pyproject.toml`` file, OR - 2. the ``[tool:release]`` section of a ``setup.cfg`` file - - If neither exist, then it uses defaults. - - :returns: configuration dict - - """ - my_config = dict(DEFAULT_CONFIG) - - if os.path.exists("pyproject.toml"): - if sys.version_info >= (3, 11): - import tomllib - else: - try: - import tomli as tomllib - except ImportError: - print( - "For Python <3.11, you need to install tomli to work with pyproject.toml " - + "files." - ) - tomllib = None - - if tomllib is not None: - with open("pyproject.toml", "rb") as fp: - data = tomllib.load(fp) - - config_data = data.get("tool", {}).get("release", {}) - if config_data: - for key, default_val in my_config.items(): - my_config[key] = config_data.get(key, default_val) - return my_config - - if os.path.exists("setup.cfg"): - config = configparser.ConfigParser() - config.read("setup.cfg") - - if "tool:release" in config: - config = config["tool:release"] - for key, default_val in my_config.items(): - my_config[key] = config.get(key, default_val) - - return my_config - - return my_config - - -def find_bugs(line): - """Returns all the bug numbers from the line. - - >>> get_bug_numbers("some line") - [] - >>> get_bug_numbers("bug-1111111: some line") - ["1111111"] - >>> get_bug_numbers("bug 1111111, 2222222: some line") - ["1111111", "2222222"] - - """ - matches = BUG_RE.findall(line) - if not matches: - return [] - bugs = [] - for match in matches: - for part in re.findall(r"\d+", match): - if part: - bugs.append(part) - return bugs - - -def fetch(url, is_json=True): - """Fetch data from a url - - This raises URLError on HTTP request errors. It also raises JSONDecode - errors if it's not valid JSON. - - """ - fp = urlopen(url) - data = fp.read() - if is_json: - return json.loads(data) - return data - - -def fetch_history_from_github(owner, repo, from_rev, main_branch): - url = f"{GITHUB_API}repos/{owner}/{repo}/compare/{from_rev}...{main_branch}" - return fetch(url) - - -def check_output(cmdline, **kwargs): - args = shlex.split(cmdline) - return subprocess.check_output(args, **kwargs).decode("utf-8").strip() - - -def get_remote_name(github_user): - """Figures out the right remote to use - - People name the git remote differently, so this figures out which one to - use. - - :arg str github_user: the github user for the remote name to use - - :returns: the name of the remote - - :raises Exception: if it can't figure out the remote name for the specified - user - - """ - # Figure out remote to push tag to - remote_output = check_output("git remote -v") - - def check_ssh(github_user, remote_url): - return f":{github_user}/" in remote_url - - def check_https(github_user, remote_url): - return f"/{github_user}/" in remote_url - - for line in remote_output.splitlines(): - line = line.split("\t") - if check_ssh(github_user, line[1]) or check_https(github_user, line[1]): - return line[0] - - raise Exception(f"Can't figure out remote name for {github_user}.") - - -def make_tag( - bug_number, - github_project, - github_user, - remote_name, - tag_name, - commits_since_tag, -): - """Tags a release.""" - if bug_number: - resp = fetch(BZ_BUG_JSON_URL + bug_number, is_json=True) - bug_summary = resp["bugs"][0]["summary"] - - input(f">>> Using bug {bug_number}: {bug_summary}. Correct? Ctrl-c to cancel") - - message = ( - f"Tag {tag_name} (bug #{bug_number})\n\n" - + "\n".join(commits_since_tag) - + f"\n\nDeploy bug #{bug_number}" - ) - else: - message = f"Tag {tag_name}\n\n" + "\n".join(commits_since_tag) - - # Print out new tag information - print("") - print(">>> New tag: %s" % tag_name) - print(">>> Tag message:") - print(LINE) - print(message) - print(LINE) - - # Create tag - input(f">>> Ready to tag {tag_name}? Ctrl-c to cancel") - print("") - print(">>> Creating tag...") - subprocess.check_call(["git", "tag", "-s", tag_name, "-m", message]) - - # Push tag - input(f">>> Ready to push to remote {remote_name}? Ctrl-c to cancel") - print("") - print(">>> Pushing...") - subprocess.check_call(["git", "push", "--tags", remote_name, tag_name]) - - if bug_number: - # Show url to tag information on GitHub for bug comment - tag_url = ( - f"https://github.com/{github_user}/{github_project}/releases/tag/{tag_name}" - ) - print("") - print(f">>> Copy and paste this tag url into bug #{bug_number}.") - print(">>> %<-----------------------------------------------") - print(f"{tag_url}") - print(">>> %<-----------------------------------------------") - - -def make_bug( - github_project, - tag_name, - commits_since_tag, - bugs_referenced, - bugzilla_product, - bugzilla_component, -): - """Creates a bug.""" - summary = f"{github_project} deploy: {tag_name}" - print(">>> Creating deploy bug...") - print(">>> Summary") - print(summary) - print() - - description = [ - f"We want to do a deploy for `{github_project}` tagged `{tag_name}`.", - "", - "It consists of the following commits:", - "", - ] - description.extend(commits_since_tag) - if bugs_referenced: - description.append("") - description.append("Bugs referenced:") - description.append("") - for bug in sorted(bugs_referenced): - description.append(f"* bug #{bug}") - description = "\n".join(description) - - print(">>> Description") - print(description) - print() - - if bugzilla_product: - bz_params = { - "priority": "P2", - "bug_type": "task", - "comment": description, - "form_name": "enter_bug", - "short_desc": summary, - } - - bz_params["product"] = bugzilla_product - if bugzilla_component: - bz_params["component"] = bugzilla_component - - bugzilla_link = BZ_CREATE_URL + "?" + urlencode(bz_params) - print(">>> Link to create bug (may not work if it's sufficiently long)") - print(bugzilla_link) - - -def run(): - config = get_config() - - parser = argparse.ArgumentParser(description=DESCRIPTION) - - # Add items that can be configured to argparse as configuration options. - # This makes it possible to specify or override configuration with command - # line arguments. - for key, val in config.items(): - key_arg = key.replace("_", "-") - default_val = val.replace("%", "%%") - parser.add_argument( - f"--{key_arg}", - default=val, - help=f"override configuration {key}; defaults to {default_val!r}", - ) - - subparsers = parser.add_subparsers(dest="cmd") - subparsers.required = True - - subparsers.add_parser("make-bug", help="Make a deploy bug") - make_tag_parser = subparsers.add_parser("make-tag", help="Make a tag and push it") - make_tag_parser.add_argument( - "--with-bug", dest="bug", help="Bug for this deploy if any." - ) - make_tag_parser.add_argument( - "--with-tag", - dest="tag", - help="Tag to use; defaults to figuring out the tag using tag_name_template.", - ) - - args = parser.parse_args() - - github_project = args.github_project - github_user = args.github_user - main_branch = args.main_branch - tag_name_template = args.tag_name_template - - if not github_project or not github_user or not main_branch: - print("main_branch, github_project, and github_user are required.") - print( - "Either set them in pyproject.toml/setup.cfg or specify them as command " - + "line arguments." - ) - return 1 - - # Let's make sure we're up-to-date and on main branch - current_branch = check_output("git rev-parse --abbrev-ref HEAD") - if current_branch != main_branch: - print( - f"Must be on the {main_branch} branch to do this; currently on {current_branch}" - ) - return 1 - - # The current branch can't be dirty - try: - subprocess.check_call("git diff --quiet --ignore-submodules HEAD".split()) - except subprocess.CalledProcessError: - print( - "Can't be \"git dirty\" when we're about to git pull. " - "Stash or commit what you're working on." - ) - return 1 - - remote_name = get_remote_name(github_user) - - # Get existing git tags from remote - check_output( - f"git pull {remote_name} {main_branch} --tags", stderr=subprocess.STDOUT - ) - - # Figure out the most recent tag details - all_tags = check_output("git tag --list --sort=-creatordate").splitlines() - if all_tags: - last_tag = all_tags[0] - last_tag_message = check_output(f'git tag -l --format="%(contents)" {last_tag}') - print(f">>> Last tag was: {last_tag}") - print(">>> Message:") - print(LINE) - print(last_tag_message) - print(LINE) - - resp = fetch_history_from_github( - github_user, github_project, last_tag, main_branch - ) - if resp["status"] != "ahead": - print(f"Nothing to deploy! {resp['status']}") - return - else: - first_commit = check_output("git rev-list --max-parents=0 HEAD") - resp = fetch_history_from_github(github_user, github_project, first_commit) - - bugs_referenced = set() - commits_since_tag = [] - for commit in resp["commits"]: - # Skip merge commits - if len(commit["parents"]) > 1: - continue - - # Use the first 7 characters of the commit sha - sha = commit["sha"][:7] - - # Use the first line of the commit message which is the summary and - # truncate it to 80 characters - summary = commit["commit"]["message"] - summary = summary.splitlines()[0] - summary = summary[:80] - - # Bug 1868455: While GitHub autolinking doesn't suport spaces, Bugzilla - # autolinking doesn't support hyphens. When creating a bug, we want to - # use "bug NNNNNNN" form so Bugzilla autolinking works. - if args.cmd == "make-bug": - summary = BUG_HYPHEN_PREFIX_RE.sub(r"bug \1", summary) - - bugs = find_bugs(summary) - if bugs: - bugs_referenced |= set(bugs) - - # Figure out who did the commit prefering GitHub usernames - who = commit["author"] - if not who: - who = "?" - else: - who = who.get("login", "?") - - commits_since_tag.append("`%s`: %s (%s)" % (sha, summary, who)) - - # Use specified tag or figure out next tag name as YYYY.MM.DD format - if args.cmd == "make-tag" and args.tag: - tag_name = args.tag - else: - tag_name = datetime.datetime.now().strftime(tag_name_template) - - # If there's already a tag, then increment the -N until we find a tag name - # that doesn't exist, yet - existing_tags = check_output(f'git tag -l "{tag_name}*"').splitlines() - if existing_tags: - tag_name_attempt = tag_name - index = 2 - while tag_name_attempt in existing_tags: - tag_name_attempt = f"{tag_name}-{index}" - index += 1 - tag_name = tag_name_attempt - - if args.cmd == "make-bug": - make_bug( - github_project, - tag_name, - commits_since_tag, - bugs_referenced, - args.bugzilla_product, - args.bugzilla_component, - ) - - elif args.cmd == "make-tag": - if args.bugzilla_product and args.bugzilla_component and not args.bug: - print( - "Bugzilla product and component are specified, but you didn't " - + "specify a bug number with --with-bug." - ) - return 1 - make_tag( - args.bug, - github_project, - github_user, - remote_name, - tag_name, - commits_since_tag, - ) - - else: - parser.print_help() - return 1 - - -if __name__ == "__main__": - sys.exit(run()) diff --git a/bin/run_migrations.sh b/bin/run_migrations.sh index 357d811f8d..9caec42bde 100755 --- a/bin/run_migrations.sh +++ b/bin/run_migrations.sh @@ -19,7 +19,7 @@ PRECMD="" # send errors to sentry. if [ -n "${SENTRY_DSN:-}" ]; then echo "SENTRY_DSN defined--enabling sentry." - PRECMD="python bin/sentry-wrap.py wrap-process --timeout=600 --" + PRECMD="sentry-wrap wrap-process --timeout=600 --" else echo "SENTRY_DSN not defined--not enabling sentry." fi diff --git a/bin/run_postdeploy.sh b/bin/run_postdeploy.sh index 5b753d470f..7fb03b703d 100755 --- a/bin/run_postdeploy.sh +++ b/bin/run_postdeploy.sh @@ -19,7 +19,7 @@ PRECMD="" # send errors to sentry. if [ -n "${SENTRY_DSN:-}" ]; then echo "SENTRY_DSN defined--enabling sentry." - PRECMD="python bin/sentry-wrap.py wrap-process --timeout=600 --" + PRECMD="sentry-wrap wrap-process --timeout=600 --" else echo "SENTRY_DSN not defined--not enabling sentry." fi diff --git a/bin/sentry-wrap.py b/bin/sentry-wrap.py deleted file mode 100755 index 4e648c686a..0000000000 --- a/bin/sentry-wrap.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Wraps a command such that if it fails, an error report is sent to the Sentry service -# specified by SENTRY_DSN in the environment. -# -# Usage: python bin/sentry-wrap.py wrap-process -- [CMD] -# Wraps a process in error-reporting Sentry goodness. -# -# Usage: python bin/sentry-wrap.py test-sentry -# Tests Sentry configuration and connection. - - -import os -import shlex -import subprocess -import sys -import time -import traceback - -import click -import sentry_sdk -from sentry_sdk import capture_exception, capture_message - - -@click.group() -def cli_main(): - pass - - -@cli_main.command() -@click.pass_context -def test_sentry(ctx): - sentry_dsn = os.environ.get("SENTRY_DSN") - - if not sentry_dsn: - click.echo("SENTRY_DSN is not defined. Exiting.") - sys.exit(1) - - sentry_sdk.init(sentry_dsn) - capture_message("Sentry test") - click.echo("Success. Check Sentry.") - - -@cli_main.command() -@click.option( - "--timeout", - default=300, - help="Timeout in seconds to wait for process before giving up.", -) -@click.argument("cmd", nargs=-1) -@click.pass_context -def wrap_process(ctx, timeout, cmd): - sentry_dsn = os.environ.get("SENTRY_DSN") - - if not sentry_dsn: - click.echo("SENTRY_DSN is not defined. Exiting.") - sys.exit(1) - - if not cmd: - raise click.UsageError("CMD required") - - start_time = time.time() - - sentry_sdk.init(sentry_dsn) - - cmd = " ".join(cmd) - cmd_args = shlex.split(cmd) - click.echo(f"sentry-wrap: running: {cmd_args}") - - try: - ret = subprocess.run(cmd_args, capture_output=True, timeout=timeout) - if ret.returncode != 0: - sentry_sdk.set_context( - "status", - { - "exit_code": ret.returncode, - "stdout": ret.stdout.decode("utf-8"), - "stderr": ret.stderr.decode("utf-8"), - }, - ) - capture_message(f"Command {cmd!r} failed.") - click.echo(ret.stdout.decode("utf-8"), err=True) - click.echo(ret.stderr.decode("utf-8"), err=True) - time_delta = (time.time() - start_time) / 1000 - click.echo(f"sentry-wrap: fail. {time_delta:.2f}s", err=True) - ctx.exit(1) - - else: - click.echo(ret.stdout.decode("utf-8")) - time_delta = (time.time() - start_time) / 1000 - click.echo(f"sentry-wrap: success! {time_delta:.2f}s") - - except click.exceptions.Exit: - raise - - except Exception as exc: - capture_exception(exc) - click.echo(traceback.format_exc(), err=True) - time_delta = (time.time() - start_time) / 1000 - click.echo(f"sentry-wrap: fail. {time_delta:.2f}s", err=True) - ctx.exit(1) - - -if __name__ == "__main__": - cli_main() diff --git a/bin/service-status.py b/bin/service-status.py deleted file mode 100755 index 7bd5a16174..0000000000 --- a/bin/service-status.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -""" -This script looks at the ``/__version__`` endpoint information and tells you -how far behind different server environments are from main tip. - -This requires Python 3.8+ to run. See help text for more. - -See https://github.com/willkg/socorro-release/#readme for details. - -Note: If you want to use ``pyproject.toml`` and you're using Python <3.11, this -also requires the tomli library. - -repo: https://github.com/willkg/socorro-release/ -sha: d19f45bc9eedae34de2905cdd4adf7b9fd03f870 - -""" - -import argparse -import json -import os -import sys -from urllib.parse import urlparse -from urllib.request import urlopen - - -DESCRIPTION = """ -service-status.py tells you how far behind different server environments -are from main tip. - -For help, see: https://github.com/willkg/socorro-release/ -""" - -DEFAULT_CONFIG = { - # The name of the main branch in the repository - "main_branch": "main", - # List of "label=host" for hosts that have a /__version__ to check - "hosts": [], -} - - -def get_config(): - """Generates configuration. - - This tries to pull configuration from the ``[tool.service-status]`` table - from a ``pyproject.toml`` file. - - If neither exist, then it uses defaults. - - :returns: configuration dict - - """ - my_config = dict(DEFAULT_CONFIG) - - if os.path.exists("pyproject.toml"): - if sys.version_info >= (3, 11): - import tomllib - else: - try: - import tomli as tomllib - except ImportError: - print( - "For Python <3.11, you need to install tomli to work with pyproject.toml " - + "files." - ) - tomllib = None - - if tomllib is not None: - with open("pyproject.toml", "rb") as fp: - data = tomllib.load(fp) - - config_data = data.get("tool", {}).get("service-status", {}) - if config_data: - for key, default_val in my_config.items(): - my_config[key] = config_data.get(key, default_val) - - return my_config - - -def fetch(url, is_json=True): - """Fetch data from a url - - This raises URLError on HTTP request errors. It also raises JSONDecode - errors if it's not valid JSON. - - """ - fp = urlopen(url, timeout=5) - data = fp.read() - if is_json: - return json.loads(data) - return data - - -def fetch_history_from_github(main_branch, user, repo, from_sha): - return fetch( - "https://api.github.com/repos/%s/%s/compare/%s...%s" - % (user, repo, from_sha, main_branch) - ) - - -class StdoutOutput: - def section(self, name): - print("") - print("%s" % name) - print("=" * len(name)) - print("") - - def row(self, *args): - template = "%-13s " * len(args) - print(" " + template % args) - - def print_delta(self, main_branch, user, repo, sha): - resp = fetch_history_from_github(main_branch, user, repo, sha) - # from pprint import pprint - # pprint(resp) - if resp["total_commits"] == 0: - self.row("", "status", "identical") - else: - self.row("", "status", "%s commits" % resp["total_commits"]) - self.row() - self.row( - "", - "https://github.com/%s/%s/compare/%s...%s" - % ( - user, - repo, - sha[:8], - main_branch, - ), - ) - self.row() - for i, commit in enumerate(resp["commits"]): - if len(commit["parents"]) > 1: - # Skip merge commits - continue - - self.row( - "", - commit["sha"][:8], - ("HEAD: " if i == 0 else "") - + "%s (%s)" - % ( - commit["commit"]["message"].splitlines()[0][:60], - (commit["author"] or {}).get("login", "?")[:10], - ), - ) - self.row() - - -def main(): - config = get_config() - - parser = argparse.ArgumentParser(description=DESCRIPTION) - - # Add items that can be configured to argparse as configuration options. - # This makes it possible to specify or override configuration with command - # line arguments. - for key, val in config.items(): - key_arg = key.replace("_", "-") - if isinstance(val, list): - parser.add_argument( - f"--{key_arg}", - default=val, - nargs="+", - metavar="VALUE", - help=f"override configuration {key}; defaults to {val!r}", - ) - else: - default_val = val.replace("%", "%%") - parser.add_argument( - f"--{key_arg}", - default=val, - metavar="VALUE", - help=f"override configuration {key}; defaults to {default_val!r}", - ) - - args = parser.parse_args() - - main_branch = args.main_branch - hosts = args.hosts - - out = StdoutOutput() - - if not hosts: - print("no hosts specified.") - return 1 - - current_section = "" - - for line in hosts: - parts = line.split("=", 1) - if len(parts) == 1: - service = parts[0] - env_name = "environment" - else: - env_name, service = parts - - if current_section != env_name: - out.section(env_name) - current_section = env_name - - service = service.rstrip("/") - resp = fetch(f"{service}/__version__") - commit = resp["commit"] - tag = resp.get("version") or "(none)" - - parsed = urlparse(resp["source"]) - _, user, repo = parsed.path.split("/") - service_name = repo - out.row(service_name, "version", commit, tag) - out.print_delta(main_branch, user, repo, commit) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/setup_services.sh b/bin/setup_services.sh index b0caa5b5e9..8dc46e259e 100755 --- a/bin/setup_services.sh +++ b/bin/setup_services.sh @@ -18,10 +18,10 @@ set -euo pipefail /app/bin/setup_postgres.sh # Delete and create local GCS buckets -/app/socorro-cmd gcs delete "${CRASHSTORAGE_GCS_BUCKET}" -/app/socorro-cmd gcs create "${CRASHSTORAGE_GCS_BUCKET}" -/app/socorro-cmd gcs delete "${TELEMETRY_GCS_BUCKET}" -/app/socorro-cmd gcs create "${TELEMETRY_GCS_BUCKET}" +gcs-cli delete "${CRASHSTORAGE_GCS_BUCKET}" +gcs-cli create "${CRASHSTORAGE_GCS_BUCKET}" +gcs-cli delete "${TELEMETRY_GCS_BUCKET}" +gcs-cli create "${TELEMETRY_GCS_BUCKET}" # Delete and create Elasticsearch indices /app/socorro-cmd legacy_es delete @@ -33,8 +33,16 @@ if [ "${ELASTICSEARCH_MODE^^}" == "PREFER_NEW" ]; then fi # Delete and create Pub/Sub queues -/app/socorro-cmd pubsub delete-all -/app/socorro-cmd pubsub create-all +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_STANDARD_TOPIC_NAME" +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_PRIORITY_TOPIC_NAME" +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_REPROCESSING_TOPIC_NAME" + +pubsub-cli create-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_STANDARD_TOPIC_NAME" +pubsub-cli create-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_PRIORITY_TOPIC_NAME" +pubsub-cli create-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_REPROCESSING_TOPIC_NAME" +pubsub-cli create-subscription "$PUBSUB_PROJECT_ID" "$PUBSUB_STANDARD_TOPIC_NAME" "$PUBSUB_STANDARD_SUBSCRIPTION_NAME" +pubsub-cli create-subscription "$PUBSUB_PROJECT_ID" "$PUBSUB_PRIORITY_TOPIC_NAME" "$PUBSUB_PRIORITY_SUBSCRIPTION_NAME" +pubsub-cli create-subscription "$PUBSUB_PROJECT_ID" "$PUBSUB_REPROCESSING_TOPIC_NAME" "$PUBSUB_REPROCESSING_SUBSCRIPTION_NAME" # Initialize the cronrun bookkeeping for all configured jobs to success /app/webapp/manage.py cronmarksuccess all diff --git a/bin/test.sh b/bin/test.sh index 42e5255353..39e12c389c 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -27,18 +27,20 @@ PYTHON="$(which python)" echo ">>> wait for services to be ready" -urlwait "${DATABASE_URL}" -urlwait "${LEGACY_ELASTICSEARCH_URL}" -urlwait "http://${PUBSUB_EMULATOR_HOST}" 10 -urlwait "${STORAGE_EMULATOR_HOST}/storage/v1/b" 10 -python ./bin/waitfor.py --verbose --codes=200,404 "${SENTRY_DSN}" +waitfor --verbose --conn-only "${DATABASE_URL}" +waitfor --verbose "${LEGACY_ELASTICSEARCH_URL}" +waitfor --verbose "http://${PUBSUB_EMULATOR_HOST}" +waitfor --verbose "${STORAGE_EMULATOR_HOST}/storage/v1/b" +waitfor --verbose --codes={200,404} "${SENTRY_DSN}" # wait for this last because it's slow to start -urlwait "${ELASTICSEARCH_URL}" 30 +waitfor --verbose --timeout=30 "${ELASTICSEARCH_URL}" echo ">>> build queue things and db things" # Clear Pub/Sub for tests -./socorro-cmd pubsub delete-all +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_STANDARD_TOPIC_NAME" +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_PRIORITY_TOPIC_NAME" +pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_REPROCESSING_TOPIC_NAME" # Set up socorro_test db ./socorro-cmd db drop || true diff --git a/bin/waitfor.py b/bin/waitfor.py deleted file mode 100755 index 8e097b70c7..0000000000 --- a/bin/waitfor.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -""" -Given a url, performs GET requests until it gets back an HTTP 200 or exceeds the wait -timeout. - -Usage: bin/waitfor.py [--timeout T] [--verbose] [--codes CODES] URL -""" - -import argparse -import urllib.error -import urllib.request -from urllib.parse import urlsplit -import sys -import time - - -def main(args): - parser = argparse.ArgumentParser( - description=( - "Performs GET requests against given URL until HTTP 200 or exceeds " - "wait timeout." - ) - ) - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--timeout", type=int, default=15, help="Wait timeout") - parser.add_argument( - "--codes", - default="200", - help="Comma-separated list of valid HTTP response codes", - ) - parser.add_argument("url", help="URL to test") - - parsed = parser.parse_args(args) - - ok_codes = [int(code.strip()) for code in parsed.codes.split(",")] - - url = parsed.url - parsed_url = urlsplit(url) - if "@" in parsed_url.netloc: - netloc = parsed_url.netloc - netloc = netloc[netloc.find("@") + 1 :] - parsed_url = parsed_url._replace(netloc=netloc) - url = parsed_url.geturl() - - if parsed.verbose: - print(f"Testing {url} for {ok_codes!r} with timeout {parsed.timeout}...") - - start_time = time.time() - - last_fail = "" - while True: - try: - with urllib.request.urlopen(url, timeout=5) as resp: - if resp.code in ok_codes: - sys.exit(0) - last_fail = f"HTTP status code: {resp.code}" - except TimeoutError as error: - last_fail = f"TimeoutError: {error}" - except urllib.error.URLError as error: - if hasattr(error, "code") and error.code in ok_codes: - sys.exit(0) - last_fail = f"URLError: {error}" - - if parsed.verbose: - print(last_fail) - - time.sleep(0.5) - - delta = time.time() - start_time - if delta > parsed.timeout: - print(f"Failed: {last_fail}, elapsed: {delta:.2f}s") - sys.exit(1) - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000000..3f789a996a --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,37 @@ +--- +# define volumes in docker-compose.override.yml so that can be ignored in CI +services: + app: + volumes: + - .:/app + test: + volumes: + - .:/app + + processor: + volumes: + - .:/app + + crontabber: + volumes: + - .:/app + + webapp: + volumes: + - .:/app + + stage_submitter: + volumes: + - .:/app + + collector: + volumes: + - .:/socorro + + fakecollector: + volumes: + - .:/app + + symbolsserver: + volumes: + - .:/app diff --git a/docker-compose.yml b/docker-compose.yml index e240dab1f9..8f6ab759cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,32 @@ --- +# Sets the base name for containers and networks for docker-compose to +# "socorro_". This is normally set by the name of this directory, but +# if you clone the repository with a different directory name, then +# you end up with a different project name and then everything is hosed. +# Setting it here fixes that. +name: socorro + services: # Socorro app image app: build: context: . - dockerfile: ./docker/Dockerfile + dockerfile: docker/Dockerfile + args: + userid: ${USE_UID:-10001} + groupid: ${USE_GID:-10001} image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd + - gcs-emulator - pubsub - postgresql - legacy-elasticsearch - elasticsearch - volumes: - - .:/app # For development @@ -35,23 +44,6 @@ services: - postgresql - legacy-elasticsearch - elasticsearch - volumes: - - .:/app - - # For running tests in CI - test-ci: - image: local/socorro_app - env_file: - - docker/config/local_dev.env - - docker/config/test.env - depends_on: - - fakesentry - - statsd - - gcs-emulator - - pubsub - - postgresql - - legacy-elasticsearch - - elasticsearch devcontainer: build: @@ -64,7 +56,7 @@ services: env_file: - docker/config/local_dev.env - docker/config/test.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -80,7 +72,7 @@ services: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -90,14 +82,12 @@ services: - elasticsearch - symbolsserver command: ["processor"] - volumes: - - .:/app crontabber: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -105,14 +95,12 @@ services: - legacy-elasticsearch - elasticsearch command: ["crontabber"] - volumes: - - .:/app webapp: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - statsd @@ -126,27 +114,22 @@ services: command: ["webapp", "--dev"] ports: - "8000:8000" - volumes: - - .:/app stage_submitter: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - fakesentry - gcs-emulator - pubsub command: ["stage_submitter"] - volumes: - - .:/app # https://github.com/willkg/kent fakesentry: build: - context: . - dockerfile: ./docker/Dockerfile.fakesentry + context: docker/images/fakesentry image: local/socorro_fakesentry ports: - "8090:8090" @@ -156,8 +139,7 @@ services: # https://hub.docker.com/r/mozilla/oidc-testprovider oidcprovider: build: - context: . - dockerfile: ./docker/Dockerfile.oidcprovider + context: docker/images/oidcprovider image: local/socorro_oidcprovider ports: - "8080:8080" @@ -179,7 +161,7 @@ services: image: mozilla/socorro_collector:latest env_file: - docker/config/local_dev.env - - my.env + - .env depends_on: - gcs-emulator - pubsub @@ -188,31 +170,25 @@ services: - 8000 ports: - "8888:8000" - volumes: - - .:/socorro fakecollector: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env command: ["fakecollector"] ports: - "9000:8000" - volumes: - - .:/app symbolsserver: image: local/socorro_app env_file: - docker/config/local_dev.env - - my.env + - .env command: ["symbolsserver"] stop_signal: SIGINT ports: - "8070:8070" - volumes: - - .:/app # https://hub.docker.com/r/hopsoft/graphite-statsd/ statsd: @@ -232,7 +208,9 @@ services: # https://www.elastic.co/guide/en/elasticsearch/reference/8.15/docker.html elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.2 + build: + context: docker/images/elasticsearch + image: local/socorro_elasticsearch mem_limit: 1g command: - bin/elasticsearch @@ -243,7 +221,9 @@ services: # https://hub.docker.com/_/postgres/ postgresql: - image: postgres:16.4 + build: + context: docker/images/postgres + image: local/socorro_postgres ports: - "8574:5432" environment: @@ -255,8 +235,9 @@ services: # https://cloud.google.com/sdk/docs/downloads-docker # official pubsub emulator pubsub: - # also available as google/cloud-sdk:-emulators - image: gcr.io/google.com/cloudsdktool/google-cloud-cli:463.0.0-emulators + build: + context: docker/images/pubsub-emulator + image: local/socorro_pubsub_emulator command: - gcloud - beta @@ -270,7 +251,9 @@ services: # https://hub.docker.com/_/memcached/ memcached: - image: memcached:1.5.1 + build: + context: docker/images/memcached + image: local/socorro_memcached ports: - "11211:11211" @@ -278,7 +261,7 @@ services: # Fake GCP GCS server for local development and testing gcs-emulator: build: - dockerfile: ./docker/Dockerfile.gcs-emulator + context: docker/images/gcs-emulator image: local/socorro_gcs_emulator command: -port 8001 -scheme http ports: diff --git a/docker/Dockerfile b/docker/Dockerfile index 9152fe5e09..8fe1d3dffc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ # stackwalker # NOTE(smarnach): To upgrade Python to a new minor or major version, see # https://socorro.readthedocs.io/en/latest/dev.html#upgrading-to-a-new-python-version -FROM --platform=linux/amd64 python:3.11.10-slim-bullseye@sha256:08ef1d5c4e0c05244f0971150437c57f6c79863345e549dbaebaf1b61f56bdf5 AS app_amd64 +FROM --platform=linux/amd64 python:3.11.10-slim-bullseye@sha256:d910a25afa706e0b2da4b59990fb59c0495eeab597b5cd777bbdcda8b6530b7e AS app_amd64 # Set up user and group ARG groupid=10001 diff --git a/docker/Dockerfile.fakesentry b/docker/Dockerfile.fakesentry deleted file mode 100644 index 509c83acd6..0000000000 --- a/docker/Dockerfile.fakesentry +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.11.10-slim-bullseye@sha256:08ef1d5c4e0c05244f0971150437c57f6c79863345e549dbaebaf1b61f56bdf5 - -ARG groupid=5000 -ARG userid=5000 - -WORKDIR /app/ - -RUN groupadd -r kent && useradd --no-log-init -r -g kent kent - -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 - -RUN pip install -U 'pip>=20' && \ - pip install --no-cache-dir 'kent==2.0.0' - -USER kent - -ENTRYPOINT ["/usr/local/bin/kent-server"] -CMD ["run"] diff --git a/docker/Dockerfile.gcs-emulator b/docker/Dockerfile.gcs-emulator deleted file mode 100644 index b60eaff348..0000000000 --- a/docker/Dockerfile.gcs-emulator +++ /dev/null @@ -1,4 +0,0 @@ -FROM fsouza/fake-gcs-server:1.50.2@sha256:23996047676fe5312001b828d800670519181c57a5943c2270e7421f3ab3f477 - -# add curl for use in healthcheck -RUN apk add --no-cache curl diff --git a/docker/Dockerfile.oidcprovider b/docker/Dockerfile.oidcprovider deleted file mode 100644 index f95536cce9..0000000000 --- a/docker/Dockerfile.oidcprovider +++ /dev/null @@ -1,6 +0,0 @@ -FROM mozilla/oidc-testprovider:oidc_testprovider-v0.10.9 - -# Modify redirect_urls specified in "fixtures.json" to fit our needs. -COPY ./docker/config/oidcprovider-fixtures.json /code/fixtures.json - -CMD ["./bin/run.sh"] diff --git a/docker/config/my.env.dist b/docker/config/.env.dist similarity index 77% rename from docker/config/my.env.dist rename to docker/config/.env.dist index c94111304d..caf7c824ec 100644 --- a/docker/config/my.env.dist +++ b/docker/config/.env.dist @@ -8,15 +8,8 @@ # If you want to set the uid and gid of the app user that we use in the # containers, you can do that with these two variables. -# SOCORRO_UID= -# SOCORRO_GID= - -# --------------------------------------------- -# processor settings -# --------------------------------------------- - -# Only use 2 threads for the processor -producer_consumer.number_of_threads=2 +# USE_UID= +# USE_GID= # --------------------------------------------- # crash-stats.mozilla.org settings diff --git a/docker/images/elasticsearch/Dockerfile b/docker/images/elasticsearch/Dockerfile new file mode 100644 index 0000000000..4e371df2d3 --- /dev/null +++ b/docker/images/elasticsearch/Dockerfile @@ -0,0 +1 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:8.15.2@sha256:4635ed9a36b5d5f9269a9d1f4d36f6a3774069aba3da95a9ddd68ee04e7c2a56 diff --git a/docker/images/fakesentry/Dockerfile b/docker/images/fakesentry/Dockerfile new file mode 100644 index 0000000000..8915748e87 --- /dev/null +++ b/docker/images/fakesentry/Dockerfile @@ -0,0 +1 @@ +FROM us-docker.pkg.dev/moz-fx-cavendish-prod/cavendish-prod/fakesentry:v2024.11.19-1@sha256:48b73aa08022eeada40fb878dc25ab225f2f91022961ffbce14ae23a7e6bdcd1 diff --git a/docker/images/gcs-emulator/Dockerfile b/docker/images/gcs-emulator/Dockerfile new file mode 100644 index 0000000000..704e760c0c --- /dev/null +++ b/docker/images/gcs-emulator/Dockerfile @@ -0,0 +1 @@ +FROM us-docker.pkg.dev/moz-fx-cavendish-prod/cavendish-prod/gcs-emulator:v2024.11.19-1@sha256:a02a86372ffe26c9f07cb55bf2d1551935389f0469d7e7dd0b69580879a00731 diff --git a/docker/images/memcached/Dockerfile b/docker/images/memcached/Dockerfile new file mode 100644 index 0000000000..7de209db16 --- /dev/null +++ b/docker/images/memcached/Dockerfile @@ -0,0 +1 @@ +FROM memcached:1.5.1@sha256:fc1826e2cb45307c5ac777b18107f14b53f91572b44bc1f856d3ba2e5d115059 diff --git a/docker/images/oidcprovider/Dockerfile b/docker/images/oidcprovider/Dockerfile new file mode 100644 index 0000000000..927196f9b9 --- /dev/null +++ b/docker/images/oidcprovider/Dockerfile @@ -0,0 +1,6 @@ +FROM mozilla/oidc-testprovider:oidc_testprovider-v0.10.10 + +# Modify redirect_urls specified in "fixtures.json" to fit our needs. +COPY fixtures.json /code/fixtures.json + +CMD ["./bin/run.sh"] diff --git a/docker/config/oidcprovider-fixtures.json b/docker/images/oidcprovider/fixtures.json similarity index 100% rename from docker/config/oidcprovider-fixtures.json rename to docker/images/oidcprovider/fixtures.json diff --git a/docker/images/postgres/Dockerfile b/docker/images/postgres/Dockerfile new file mode 100644 index 0000000000..7f6869c85e --- /dev/null +++ b/docker/images/postgres/Dockerfile @@ -0,0 +1 @@ +FROM postgres:16.4@sha256:e62fbf9d3e2b49816a32c400ed2dba83e3b361e6833e624024309c35d334b412 diff --git a/docker/images/pubsub-emulator/Dockerfile b/docker/images/pubsub-emulator/Dockerfile new file mode 100644 index 0000000000..49efbe2a60 --- /dev/null +++ b/docker/images/pubsub-emulator/Dockerfile @@ -0,0 +1,3 @@ +# Define this image outside of docker-compose.yml so that it gets dependabot updates +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:501.0.0-emulators@sha256:4dfc65b1795329f1a94316d3a0eb70a4a37d4813cac0efe05c43b08b2e66d44d +# also available as google/cloud-sdk:-emulators diff --git a/docs/crashstorage.rst b/docs/crashstorage.rst index 3dc5675ca7..67502c2fbe 100644 --- a/docs/crashstorage.rst +++ b/docs/crashstorage.rst @@ -73,7 +73,7 @@ You can see Elasticsearch common options by passing ``--help`` to the processor app and looking at the ``resource.elasticsearch`` options like this:: - $ make shell + $ just shell app@socorro:/app$ python ./socorro/processor/processor_app.py \ --destination.crashstorage_class=socorro.external.es.crashstorage.ESCrashStorage \ --help diff --git a/docs/dev.rst b/docs/dev.rst index 483bdae972..77f4bf8565 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -15,7 +15,7 @@ development environment. Setup quickstart ================ -1. Install required software: Docker, make, and git. +1. Install required software: Docker, just, and git. **Linux**: @@ -26,17 +26,17 @@ Setup quickstart Install `Docker for Mac `_ which will install Docker. - Use `homebrew `_ to install make and git: + Use `homebrew `_ to install just and git: .. code-block:: shell - $ brew install make git + $ brew install just git **Other**: Install `Docker `_. - Install `make `_. + Install `just `_. Install `git `_. @@ -52,14 +52,14 @@ Setup quickstart .. code-block:: shell - $ make my.env + $ just _env - Then edit the file and set the ``SOCORRO_UID`` and ``SOCORRO_GID`` + Then edit the file and set the ``USE_UID`` and ``USE_GID`` variables. These will get used when creating the app user in the base image. - If you ever want different values, change them in ``my.env`` and re-run - ``make build``. + If you ever want different values, change them in ``.env`` and re-run + ``just build``. 4. Build Docker images for Socorro services. @@ -67,23 +67,17 @@ Setup quickstart .. code-block:: shell - $ make build + $ just build That will build the app Docker image required for development. 5. Initialize Postgres, Elasticsearch, Pub/Sub, S3, and SQS. - Then you need to set up services. To do that, run: - - .. code-block:: shell - - $ make runservices - - This starts service containers. Then run: + To do that, run: .. code-block:: shell - $ make setup + $ just setup This creates the Postgres database and sets up tables, stored procedures, integrity rules, types, and a bunch of other things. It also adds a bunch of @@ -107,7 +101,7 @@ Setup quickstart .. code-block:: shell - $ make updatedata + $ just update-data At this point, you should have a basic functional Socorro development @@ -115,7 +109,7 @@ environment that has no crash data in it. .. Note:: - You can run ``make setup`` and ``make updatedata`` any time you want to + You can run ``just setup`` and ``just update-data`` any time you want to throw out all state and re-initialize services. .. Seealso:: @@ -245,7 +239,7 @@ To lint the code: .. code-block:: shell - $ make lint + $ just lint If you hit issues, use ``# noqa``. @@ -253,7 +247,7 @@ To run the reformatter: .. code-block:: shell - $ make lintfix + $ just lint --fix We're using: @@ -304,7 +298,7 @@ Do this: .. code-block:: shell - $ make shell + $ just shell app@socorro:/app$ cd webapp app@socorro:/app/webapp$ ./manage.py makemigration --name "BUGID_desc" APP @@ -334,7 +328,7 @@ For example, to add ``foobar`` version 5: .. code-block:: shell - $ make rebuildreqs + $ just rebuild-reqs to apply the updates to ``requirements.txt`` @@ -342,7 +336,7 @@ For example, to add ``foobar`` version 5: .. code-block:: shell - $ make build + $ just build If there are problems, it'll tell you. @@ -351,7 +345,7 @@ dependencies. To do this, run: .. code-block:: shell - $ make updatereqs + $ just rebuild-reqs --update JavaScript Dependencies @@ -371,7 +365,7 @@ Then rebuild your docker environment: .. code-block:: shell - $ make build + $ just build If there are problems, it'll tell you. @@ -387,7 +381,7 @@ To build the docs, run this: .. code-block:: shell - $ make docs + $ just docs Testing @@ -406,7 +400,7 @@ To run the tests, do: .. code-block:: shell - $ make test + $ just test That runs the ``/app/bin/test.sh`` script in the test container using test configuration. @@ -416,7 +410,7 @@ test container: .. code-block:: shell - $ make testshell + $ just test-shell Then you can run pytest on the Socorro tests or the webapp tests. @@ -499,7 +493,7 @@ build new images: .. code-block:: shell - $ make build + $ just build If there were changes to the database tables, stored procedures, types, @@ -508,8 +502,8 @@ state and re-initialize services: .. code-block:: shell - $ make setup - $ make updatedata + $ just setup + $ just update-data Wiping crash storage and state @@ -520,8 +514,8 @@ data, and reset the state of the system, run: .. code-block:: shell - $ make setup - $ make updatedata + $ just setup + $ just update-data Updating release data @@ -534,7 +528,7 @@ Run: .. code-block:: shell - $ make updatedata + $ just update-data .. _gettingstarted-chapter-configuration: @@ -623,7 +617,7 @@ first run: .. code-block:: shell - $ make devcontainerbuild + $ just build devcontainer Additionally on mac there is the potential that running git from inside any container that mounts the current directory to `/app`, such as the development @@ -643,7 +637,7 @@ pick up changes: .. code-block:: shell - $ make devcontainer + $ just run devcontainer Upgrading to a new Python version @@ -679,7 +673,7 @@ All helper scripts run in the shell in the container: .. code-block:: - $ make shell + $ just shell Some of the scripts require downloading production data from `crash-stats.mozilla.org `_, and it is @@ -713,7 +707,7 @@ Add the API token value to your ``my.env`` file:: SOCORRO_API_TOKEN=apitokenhere -The API token is used by the download scripts (run inside ``$ make shell``), +The API token is used by the download scripts (run inside ``$ just shell``), but not directly by the processor. @@ -739,7 +733,7 @@ You can also use it with ``fetch_crashids``: app@socorro:/app$ socorro-cmd fetch_crashids --num=1 | bin/process_crashes.sh -Run the processor and webapp with ``make run`` to process the crash reports. +Run the processor and webapp with ``just run`` to process the crash reports. If you find this doesn't meet your needs, you can write a shell script using the commands and scripts that ``process_crashes.sh`` uses. They are described @@ -888,7 +882,7 @@ Let's process crashes for Firefox from yesterday. We'd do this: # Set SOCORRO_API_TOKEN in my.env # Start bash in the socorro container - $ make shell + $ just shell # Generate a file of crashids--one per line app@socorro:/app$ socorro-cmd fetch_crashids > crashids.txt @@ -955,6 +949,6 @@ For example:: PGPASSWORD=postgres psql -h localhost -p 8574 -U postgres --no-password socorro -You can also connect with ``make``:: +You can also connect with ``just``:: - make psql + just psql diff --git a/docs/reprocessing.rst b/docs/reprocessing.rst index 224e692342..ebdb449d97 100644 --- a/docs/reprocessing.rst +++ b/docs/reprocessing.rst @@ -29,7 +29,7 @@ In ``my.env``, set ``SOCORRO_REPROCESS_API_TOKEN`` to the token value. For example, this reprocesses a single crash:: - $ make shell + $ just shell app@socorro:app$ socorro-cmd reprocess c2815fd1-e87b-45e9-9630-765060180110 When reprocessing many crashes, it is useful to collect crashids and then @@ -39,12 +39,12 @@ failure. This reprocesses 100 crashes with a specified signature:: - $ make shell + $ just shell app@socorro:app$ socorro-cmd fetch_crashids --signature="some | signature" > crashids app@socorro:app$ cat crashids | socorro-cmd reprocess For more complex crash sets, pass a search URL to generate the list:: - $ make shell + $ just shell app@socorro:app$ socorro-cmd fetch_crashids --num=all --url="https://crash-stats.mozilla.org/search/?product=Sample&date=%3E%3D2019-05-07T22%3A00%3A00.000Z&date=%3C2019-05-07T23%3A00%3A00.000Z" > crashids app@socorro:app$ cat crashids | socorro-cmd reprocess diff --git a/docs/service/cron.rst b/docs/service/cron.rst index 312a93c1ef..c8bc00a039 100644 --- a/docs/service/cron.rst +++ b/docs/service/cron.rst @@ -40,7 +40,7 @@ manage.py cronrun helper commands All commands are accessed in a shell in the app container. For example:: - $ make shell + $ just shell app@socorro:/app$ webapp/manage.py cronrun --help **cronrun** diff --git a/docs/service/processor.rst b/docs/service/processor.rst index deea90a013..fa0e623d6f 100644 --- a/docs/service/processor.rst +++ b/docs/service/processor.rst @@ -52,7 +52,7 @@ processor configuration. To use tools and also ease debugging in the container, you can run a shell:: - $ make shell + $ just shell Then you can start and stop the processor and tweak files and all that jazz. diff --git a/docs/service/stage_submitter.rst b/docs/service/stage_submitter.rst index fbcaae268f..ab1134a978 100644 --- a/docs/service/stage_submitter.rst +++ b/docs/service/stage_submitter.rst @@ -49,8 +49,7 @@ To run the stage submitter and fake collector, do: :: - $ make runservices - $ make runsubmitter + $ just run-submitter After doing this, you can enter a Socorro container shell and use ``bin/process_crash.sh`` to pull down crash data, put it into storage, and @@ -58,7 +57,7 @@ publish the crash id to the standard queue. :: - $ make shell + $ just shell app@socorro:/app$ ./bin/process_crash.sh a206b51a-5955-4704-be1f-cf1ac0240514 diff --git a/docs/tests/system_checklist.rst b/docs/tests/system_checklist.rst index b42ae1b8e7..928730ba48 100644 --- a/docs/tests/system_checklist.rst +++ b/docs/tests/system_checklist.rst @@ -60,7 +60,7 @@ Checklist Local dev environment: - 1. "make shell" + 1. "just shell" 2. "cd webapp/" 3. "./manage.py showmigrations" diff --git a/justfile b/justfile new file mode 100644 index 0000000000..bb7e563b65 --- /dev/null +++ b/justfile @@ -0,0 +1,88 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +_default: + @just --list + +_env: + #!/usr/bin/env sh + if [ ! -f .env ]; then + echo "Copying docker/config/.env.dist to .env..." + cp docker/config/.env.dist .env + fi + +# Build docker images +build *args: _env + docker compose build --progress plain {{args}} + +# Set up Postgres, Elasticsearch, local Pub/Sub, and local GCS services. +setup: _env + docker compose run --rm app shell /app/bin/setup_services.sh + +# Add/update necessary database data. +update-data: _env + docker compose run --rm app shell /app/bin/update_data.sh + +# Run services, defaults to socorro and fakesentry for debugging +run *args='--attach=processor --attach=webapp --attach=fakesentry processor webapp': _env + docker compose up {{args}} + +# Run stage submitter and fake collector +run-submitter *args='--attach=stage_submitter --attach=fakecollector': _env + docker compose up \ + {{args}} \ + stage_submitter \ + fakecollector + +# Stop service containers. +stop *args: + docker compose stop {{args}} + +# Remove service containers and networks. +down *args: + docker compose down {{args}} + +# Open a shell or run a command in the app container. +shell *args='/bin/bash': _env + docker compose run --rm --entrypoint= app {{args}} + +# Open a shell or run a command in the test environment. +test-shell *args='/bin/bash': + docker compose run --rm --entrypoint= test {{args}} + +# Remove all build, test, coverage, and Python artifacts. +clean: + -rm -rf .cache + @echo "Skipping deletion of symbols/ in case you have data in there." + +# Generate Sphinx HTML documetation. +docs: _env + docker compose run --rm app shell make -C docs/ clean + docker compose run --rm app shell make -C docs/ html + +# Lint code, or use --fix to reformat and apply auto-fixes for lint. +lint *args: _env + docker compose run --rm --no-deps app shell ./bin/lint.sh {{args}} + +# Open psql cli. +psql *args: + docker compose run --rm postgresql psql postgresql://postgres:postgres@postgresql/socorro {{args}} + +# Run tests. +test *args: + docker compose run --rm test shell ./bin/test.sh {{args}} + +# Build requirements.txt file after requirements.in changes. +rebuild-reqs *args: _env + docker compose run --rm --no-deps app shell pip-compile --generate-hashes --strip-extras {{args}} + docker compose run --rm --no-deps app shell pip-compile --generate-hashes \ + --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in + +# Verify that the requirements file is built by the version of Python that runs in the container. +verify-reqs: _env + docker compose run --rm --no-deps app shell ./bin/verify_reqs.sh + +# Check how far behind different server environments are from main tip. +service-status *args: _env + docker compose run --rm --no-deps app shell service-status {{args}} diff --git a/requirements.in b/requirements.in index 14260724c0..3295901d77 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ attrs==24.2.0 -boltons==24.0.0 +boltons==24.1.0 click==8.1.7 contextlib2==21.6.0 datadog==0.50.1 @@ -13,11 +13,11 @@ djangorestframework==3.15.2 dj-database-url==2.3.0 dockerflow==2024.4.2 enforce-typing==1.0.0.post1 -everett==3.3.0 -fillmore==2.0.1 +everett==3.4.0 +fillmore==2.1.0 freezegun==1.5.1 -glom==23.5.0 -google-cloud-pubsub==2.26.1 +glom==24.11.0 +google-cloud-pubsub==2.27.1 google-cloud-storage==2.18.2 gunicorn==23.0.0 honcho==2.0.0 @@ -28,7 +28,7 @@ isoweek==1.3.3 jinja2==3.1.4 jsonschema==4.23.0 lxml==5.2.2 -markus[datadog]==5.0.0 +markus[datadog]==5.1.0 markdown-it-py==3.0.0 more-itertools==10.5.0 mozilla-django-oidc==4.0.1 @@ -46,15 +46,15 @@ python-decouple==3.8 PyYAML==6.0.2 requests==2.32.3 requests-mock==1.12.1 -ruff==0.7.1 +ruff==0.7.4 semver==3.0.2 -sentry-sdk==2.8.0 +sentry-sdk==2.17.0 Sphinx==8.1.3 -sphinx_rtd_theme==3.0.1 +sphinx_rtd_theme==3.0.2 statsd==4.0.1 urlwait==1.0 -werkzeug==3.0.6 -whitenoise==6.8.1 +werkzeug==3.1.3 +whitenoise==6.8.2 # NOTE(willkg): Don't need to update this. We don't really use it and we should @@ -78,3 +78,7 @@ elasticsearch-dsl==8.15.3 python-dateutil # via elasticsearch-dsl==0.0.11 six # via elasticsearch-dsl==0.0.11 urllib3>=1.8, <2.0 # via elasticsearch==1.9.0 + +# Mozilla obs-team libraries that are published to GAR instead of pypi +--extra-index-url https://us-python.pkg.dev/moz-fx-cavendish-prod/cavendish-prod-python/simple/ +obs-common==2024.11.14 diff --git a/requirements.txt b/requirements.txt index 5162018294..24ce28d642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ # # pip-compile --generate-hashes --strip-extras # +--extra-index-url https://us-python.pkg.dev/moz-fx-cavendish-prod/cavendish-prod-python/simple/ + alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 @@ -27,9 +29,9 @@ babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via sphinx -boltons==24.0.0 \ - --hash=sha256:7153feccaea1ff2e1472f68d4b57fadb796a2ee49d29f638f1c9cd8fb5ebd916 \ - --hash=sha256:9618695a6ec4f50412e7072e5d78910a00b4111d0b9b549e4a3d60bc321e7807 +boltons==24.1.0 \ + --hash=sha256:4a49b7d57ee055b83a458c8682a2a6f199d263a8aa517098bda9bab813554b87 \ + --hash=sha256:a1776d47fdc387fb730fba1fe245f405ee184ee0be2fb447dd289773a84aed3b # via # -r requirements.in # face @@ -200,6 +202,7 @@ click==8.1.7 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via # -r requirements.in + # obs-common # pip-tools contextlib2==21.6.0 \ --hash=sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f \ @@ -327,26 +330,26 @@ enforce-typing==1.0.0.post1 \ --hash=sha256:90347a61d08e7f7578d9714b4f0fd8abd9b6bc48c8ac8d46d7f290d413afabb7 \ --hash=sha256:d3184dfdbfd7f9520c884986561751a6106c57cdd65d730470645d2d40c47e18 # via -r requirements.in -everett==3.3.0 \ - --hash=sha256:acb7b8f3c5fc9692a8ba14fb257e4649f87bea4856e7406c2e9b6c2cab889105 \ - --hash=sha256:d3ecc55cc1bdf2408ca82bc8db5a3fc588fc4c0f236a6ab9599938f41970b814 +everett==3.4.0 \ + --hash=sha256:f403c4a41764a6301fb31e2558d6e9718999f0eab9e260d986b894fa2e6b6871 \ + --hash=sha256:f8c29c7300702f47b7323b75348e2b86647246694fda7ad410c2a2bfaa980ff7 # via -r requirements.in face==20.1.1 \ --hash=sha256:3790311a7329e4b0d90baee346eecad54b337629576edf3a246683a5f0d24446 \ --hash=sha256:7d59ca5ba341316e58cf72c6aff85cca2541cf5056c4af45cb63af9a814bed3e \ --hash=sha256:ca3a1d8b8b6aa8e61d62a300e9ee24e09c062aceda549e9a640128e4fa0f4559 # via glom -fillmore==2.0.1 \ - --hash=sha256:28c1c47063d116909009e4b31f40098702ce1944df7793e8f47642c7c39a78cf \ - --hash=sha256:a1f194416133b32656bb9918225cd1599359042eb7c63f7c4842ff0b312017ed +fillmore==2.1.0 \ + --hash=sha256:251ed9154ba7f20f5825e4d757db0ad7b1642e72bda7657fe39fe39031cd2092 \ + --hash=sha256:29873e6f7fae15b32ddd01eff7a8418f26ab33c731f3b99a6a07a4c4c8c3625f # via -r requirements.in freezegun==1.5.1 \ --hash=sha256:b29dedfcda6d5e8e083ce71b2b542753ad48cfec44037b3fc79702e2980a89e9 \ --hash=sha256:bf111d7138a8abe55ab48a71755673dbaa4ab87f4cff5634a4442dfec34c15f1 # via -r requirements.in -glom==23.5.0 \ - --hash=sha256:06af5e3486aacc59382ba34e53ebeabd7a9345d78f7dbcbee26f03baa4b83bac \ - --hash=sha256:fe4e9be4dc93c11a99f8277042e4bee95419c02cda4b969f504508b0a1aa6a66 +glom==24.11.0 \ + --hash=sha256:4325f96759a912044af7b6c6bd0dba44ad8c1eb6038aab057329661d2021bb27 \ + --hash=sha256:991db7fcb4bfa9687010aa519b7b541bbe21111e70e58fdd2d7e34bbaa2c1fbd # via -r requirements.in google-api-core==2.17.1 \ --hash=sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e \ @@ -367,14 +370,18 @@ google-cloud-core==2.4.1 \ --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via google-cloud-storage -google-cloud-pubsub==2.26.1 \ - --hash=sha256:932d4434d86af25673082b48d54b318a448d1a7cd718404c33bf008ae9a8bb22 \ - --hash=sha256:d46a302c2c7a008e399f4c04b4be6341d8aa7a537a25810ec8d38a5c125f816d - # via -r requirements.in +google-cloud-pubsub==2.27.1 \ + --hash=sha256:3ca8980c198a847ee464845ab60f05478d4819cf693c9950ee89da96f0b80a41 \ + --hash=sha256:7119dbc5af4b915ecdfa1289919f791a432927eaaa7bbfbeb740e6d7020c181e + # via + # -r requirements.in + # obs-common google-cloud-storage==2.18.2 \ --hash=sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166 \ --hash=sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99 - # via -r requirements.in + # via + # -r requirements.in + # obs-common google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ @@ -805,9 +812,9 @@ markupsafe==2.1.3 \ # via # jinja2 # werkzeug -markus==5.0.0 \ - --hash=sha256:14fe47ebe3d3447cc5eebcd4691f71e7f38dee6338e4eb5d72d64d67f289c6ff \ - --hash=sha256:fd0f0de0914a3ae645cc1eac760dca2fa28943fbbb3671692e09916f7f2e2237 +markus==5.1.0 \ + --hash=sha256:424172efdccc35172b8aadfdcd753412c3ed2b5651c3b3bc9e0b7e7f2e97da52 \ + --hash=sha256:a4ec2d6bb1dcf471638be11a10cb5708de8cc3092ade9cf3b38bb2f651ede33a # via -r requirements.in mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ @@ -825,6 +832,9 @@ oauth2client==4.1.3 \ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 # via -r requirements.in +obs-common==2024.11.14 \ + --hash=sha256:950dc79343eb041efd79d45dc182c3ed047060f50f257a30b626cf3853f093d0 + # via -r requirements.in opentelemetry-api==1.27.0 \ --hash=sha256:953d5871815e7c30c81b56d910c707588000fff7a3ca1c73e6531911d53065e7 \ --hash=sha256:ed673583eaa5f81b5ce5e86ef7cdaf622f88ef65f0b9aab40b843dcae5bef342 @@ -1113,6 +1123,7 @@ requests==2.32.3 \ # google-api-core # google-cloud-storage # mozilla-django-oidc + # obs-common # requests-mock # sphinx requests-mock==1.12.1 \ @@ -1228,36 +1239,37 @@ rsa==4.7.2 \ # via # google-auth # oauth2client -ruff==0.7.1 \ - --hash=sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37 \ - --hash=sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35 \ - --hash=sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c \ - --hash=sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7 \ - --hash=sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a \ - --hash=sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8 \ - --hash=sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99 \ - --hash=sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd \ - --hash=sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565 \ - --hash=sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad \ - --hash=sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378 \ - --hash=sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca \ - --hash=sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250 \ - --hash=sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4 \ - --hash=sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9 \ - --hash=sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112 \ - --hash=sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89 \ - --hash=sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307 +ruff==0.7.4 \ + --hash=sha256:00b4cf3a6b5fad6d1a66e7574d78956bbd09abfd6c8a997798f01f5da3d46a05 \ + --hash=sha256:0d06218747d361d06fd2fdac734e7fa92df36df93035db3dc2ad7aa9852cb109 \ + --hash=sha256:0e92dfb5f00eaedb1501b2f906ccabfd67b2355bdf117fea9719fc99ac2145bc \ + --hash=sha256:11bff065102c3ae9d3ea4dc9ecdfe5a5171349cdd0787c1fc64761212fc9cf1f \ + --hash=sha256:2e32829c429dd081ee5ba39aef436603e5b22335c3d3fff013cd585806a6486a \ + --hash=sha256:3bd726099f277d735dc38900b6a8d6cf070f80828877941983a57bca1cd92172 \ + --hash=sha256:63a569b36bc66fbadec5beaa539dd81e0527cb258b94e29e0531ce41bacc1f20 \ + --hash=sha256:662a63b4971807623f6f90c1fb664613f67cc182dc4d991471c23c541fee62dd \ + --hash=sha256:745775c7b39f914238ed1f1b0bebed0b9155a17cd8bc0b08d3c87e4703b990d6 \ + --hash=sha256:75c53f54904be42dd52a548728a5b572344b50d9b2873d13a3f8c5e3b91f5cac \ + --hash=sha256:7dbdc7d8274e1422722933d1edddfdc65b4336abf0b16dfcb9dedd6e6a517d06 \ + --hash=sha256:80094ecd4793c68b2571b128f91754d60f692d64bc0d7272ec9197fdd09bf9ea \ + --hash=sha256:876f5e09eaae3eb76814c1d3b68879891d6fde4824c015d48e7a7da4cf066a3a \ + --hash=sha256:997512325c6620d1c4c2b15db49ef59543ef9cd0f4aa8065ec2ae5103cedc7e7 \ + --hash=sha256:a4919925e7684a3f18e18243cd6bea7cfb8e968a6eaa8437971f681b7ec51478 \ + --hash=sha256:cd12e35031f5af6b9b93715d8c4f40360070b2041f81273d0527683d5708fce2 \ + --hash=sha256:cfb365c135b830778dda8c04fb7d4280ed0b984e1aec27f574445231e20d6c63 \ + --hash=sha256:e0cea28d0944f74ebc33e9f934238f15c758841f9f5edd180b5315c203293452 # via -r requirements.in semver==3.0.2 \ --hash=sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc \ --hash=sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4 # via -r requirements.in -sentry-sdk==2.8.0 \ - --hash=sha256:6051562d2cfa8087bb8b4b8b79dc44690f8a054762a29c07e22588b1f619bfb5 \ - --hash=sha256:aa4314f877d9cd9add5a0c9ba18e3f27f99f7de835ce36bd150e48a41c7c646f +sentry-sdk==2.17.0 \ + --hash=sha256:625955884b862cc58748920f9e21efdfb8e0d4f98cca4ab0d3918576d5b606ad \ + --hash=sha256:dd0a05352b78ffeacced73a94e86f38b32e2eae15fff5f30ca5abb568a72eacf # via # -r requirements.in # fillmore + # obs-common six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -1279,9 +1291,9 @@ sphinx==8.1.3 \ # sphinxcontrib-jquery # sphinxcontrib-qthelp # sphinxcontrib-serializinghtml -sphinx-rtd-theme==3.0.1 \ - --hash=sha256:921c0ece75e90633ee876bd7b148cfaad136b481907ad154ac3669b6fc957916 \ - --hash=sha256:a4c5745d1b06dfcb80b7704fe532eb765b44065a8fad9851e4258c8804140703 +sphinx-rtd-theme==3.0.2 \ + --hash=sha256:422ccc750c3a3a311de4ae327e82affdaf59eb695ba4936538552f3b00f4ee13 \ + --hash=sha256:b7457bc25dda723b20b086a670b9953c859eab60a2a03ee8eb2bb23e176e5f85 # via -r requirements.in sphinxcontrib-applehelp==1.0.7 \ --hash=sha256:094c4d56209d1734e7d252f6e0b3ccc090bd52ee56807a5d9315b19c122ab15d \ @@ -1338,17 +1350,17 @@ urlwait==1.0 \ --hash=sha256:a9bf2da792fa6983fa93f6360108e16615066ab0f9cfb7f53e5faee5f5dffaac \ --hash=sha256:eae2c20001efc915166cac79c04bac0088ad5787ec64b36f27afd2f359953b2b # via -r requirements.in -werkzeug==3.0.6 \ - --hash=sha256:1bc0c2310d2fbb07b1dd1105eba2f7af72f322e1e455f2f93c993bee8c8a5f17 \ - --hash=sha256:a8dd59d4de28ca70471a34cba79bed5f7ef2e036a76b3ab0835474246eb41f8d +werkzeug==3.1.3 \ + --hash=sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e \ + --hash=sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746 # via -r requirements.in wheel==0.41.3 \ --hash=sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942 \ --hash=sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841 # via pip-tools -whitenoise==6.8.1 \ - --hash=sha256:11042f39f1dcfbb3814726b9364703af6901706582d988e96494cfefdc3a89e2 \ - --hash=sha256:196ba04ca0a80f4a3f99f88381864f218a28b5fb5b44d29feea484d501fa0ba3 +whitenoise==6.8.2 \ + --hash=sha256:486bd7267a375fa9650b136daaec156ac572971acc8bf99add90817a530dd1d4 \ + --hash=sha256:df12dce147a043d1956d81d288c6f0044147c6d2ab9726e5772ac50fb45d2280 # via -r requirements.in wrapt==1.16.0 \ --hash=sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc \ diff --git a/socorro-cmd b/socorro-cmd index d2337a648e..8c1ea4f533 100755 --- a/socorro-cmd +++ b/socorro-cmd @@ -103,8 +103,6 @@ COMMANDS = [ "db": import_path("socorro.scripts.db.db_group"), "es": import_path("es_cli.main"), "legacy_es": import_path("legacy_es_cli.main"), - "pubsub": import_path("pubsub_cli.main"), - "gcs": import_path("gcs_cli.main"), }, ), Group( diff --git a/socorro/external/pubsub/crashqueue.py b/socorro/external/pubsub/crashqueue.py index 4a3d3fef46..0bd954e0b5 100644 --- a/socorro/external/pubsub/crashqueue.py +++ b/socorro/external/pubsub/crashqueue.py @@ -9,6 +9,7 @@ from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient from google.cloud.pubsub_v1.types import BatchSettings, PublisherOptions from more_itertools import chunked +import sentry_sdk from socorro.external.crashqueue_base import CrashQueueBase @@ -260,15 +261,14 @@ def publish(self, queue, crash_ids): for i, future in enumerate(futures): try: future.result() - except Exception: + except Exception as exc: + sentry_sdk.capture_exception(exc) logger.exception( - "Crashid failed to publish: %s %s", + "crashid failed to publish: %s %s", queue, batch[i], ) - failed.append(batch[i]) + failed.append((repr(exc), batch[i])) if failed: - raise CrashIdsFailedToPublish( - f"Crashids failed to publish: {','.join(failed)}" - ) + raise CrashIdsFailedToPublish(f"Crashids failed to publish: {failed!r}") diff --git a/socorro/lib/task_manager.py b/socorro/lib/task_manager.py index 756f32006d..58c8b40a02 100644 --- a/socorro/lib/task_manager.py +++ b/socorro/lib/task_manager.py @@ -9,8 +9,6 @@ LOGGER = logging.getLogger(__name__) -HEARTBEAT_INTERVAL = 60 - def default_task_func(a_param): """Default task function. @@ -21,16 +19,6 @@ def default_task_func(a_param): """ -def default_heartbeat(): - """Runs once a second from the main thread. - - Note: If this raises an exception, it could kill the process or put it in a - weird state. - - """ - LOGGER.info("THUMP") - - def default_iterator(): """Default iterator for tasks. @@ -76,7 +64,6 @@ def __init__( idle_delay=7, quit_on_empty_queue=False, job_source_iterator=default_iterator, - heartbeat_func=default_heartbeat, task_func=default_task_func, ): """ @@ -88,14 +75,12 @@ def __init__( instantiated with a config object can be iterated. The iterator must yield a tuple consisting of a function's tuple of args and, optionally, a mapping of kwargs. Ex: (('a', 17), {'x': 23}) - :arg heartbeat_func: a function to run every second :arg task_func: a function that will accept the args and kwargs yielded by the job_source_iterator """ self.idle_delay = idle_delay self.quit_on_empty_queue = quit_on_empty_queue self.job_source_iterator = job_source_iterator - self.heartbeat_func = heartbeat_func self.task_func = task_func self._pid = os.getpid() @@ -109,7 +94,7 @@ def _get_iterator(self): job_source_iterator can be one of a few things: * a class that can be instantiated and iterated over - * a function that returns an interator + * a function that returns an iterator * an actual iterator/generator * an iterable collection @@ -124,7 +109,7 @@ def _get_iterator(self): def _responsive_sleep(self, seconds, wait_log_interval=0, wait_reason=""): """Responsive sleep that checks for quit flag - When there is litte work to do, the queuing thread sleeps a lot. It can't sleep + When there is little work to do, the queuing thread sleeps a lot. It can't sleep for too long without checking for the quit flag and/or logging about why it is sleeping. @@ -132,7 +117,7 @@ def _responsive_sleep(self, seconds, wait_log_interval=0, wait_reason=""): :arg wait_log_interval: while sleeping, it is helpful if the thread periodically announces itself so that we know that it is still alive. This number is the time in seconds between log entries. - :arg wait_reason: the is for the explaination of why the thread is + :arg wait_reason: the is for the explanation of why the thread is sleeping. This is likely to be a message like: 'there is no work to do'. This was also partially motivated by old versions' of Python inability to @@ -146,14 +131,10 @@ def _responsive_sleep(self, seconds, wait_log_interval=0, wait_reason=""): def blocking_start(self): """This function starts the task manager running to do tasks.""" - next_heartbeat = time.time() + HEARTBEAT_INTERVAL self.logger.debug("threadless start") try: # May never exhaust for job_params in self._get_iterator(): - if time.time() > next_heartbeat: - self.heartbeat_func() - next_heartbeat = time.time() + HEARTBEAT_INTERVAL self.logger.debug("received %r", job_params) if job_params is None: if self.quit_on_empty_queue: diff --git a/socorro/lib/threaded_task_manager.py b/socorro/lib/threaded_task_manager.py index 2b4ea90202..092a877982 100644 --- a/socorro/lib/threaded_task_manager.py +++ b/socorro/lib/threaded_task_manager.py @@ -17,7 +17,6 @@ import time from socorro.lib.task_manager import ( - default_heartbeat, default_iterator, default_task_func, TaskManager, @@ -26,8 +25,6 @@ STOP_TOKEN = (None, None) -HEARTBEAT_INTERVAL = 60 - class ThreadedTaskManager(TaskManager): """Threaded task manager.""" @@ -39,7 +36,6 @@ def __init__( number_of_threads=4, maximum_queue_size=8, job_source_iterator=default_iterator, - heartbeat_func=default_heartbeat, task_func=default_task_func, ): """ @@ -54,7 +50,6 @@ def __init__( instantiated with a config object can be iterated. The iterator must yield a tuple consisting of a function's tuple of args and, optionally, a mapping of kwargs. Ex: (('a', 17), {'x': 23}) - :arg heartbeat_func: a function to run every second :arg task_func: a function that will accept the args and kwargs yielded by the job_source_iterator """ @@ -71,7 +66,6 @@ def __init__( idle_delay=idle_delay, quit_on_empty_queue=quit_on_empty_queue, job_source_iterator=job_source_iterator, - heartbeat_func=heartbeat_func, task_func=task_func, ) self.thread_list = [] # the thread object storage @@ -107,12 +101,8 @@ def wait_for_completion(self): if self.queueing_thread is None: return - next_heartbeat = time.time() + HEARTBEAT_INTERVAL self.logger.debug("waiting to join queueing_thread") while True: - if time.time() > next_heartbeat: - self.heartbeat_func() - next_heartbeat = time.time() + HEARTBEAT_INTERVAL try: self.queueing_thread.join(1.0) if not self.queueing_thread.is_alive(): @@ -149,7 +139,7 @@ def wait_for_empty_queue(self, wait_log_interval=0, wait_reason=""): :arg wait_log_interval: While sleeping, it is helpful if the thread periodically announces itself so that we know that it is still alive. This number is the time in seconds between log entries. - :arg wait_reason: The is for the explaination of why the thread is sleeping. + :arg wait_reason: The is for the explanation of why the thread is sleeping. This is likely to be a message like: 'there is no work to do'. """ diff --git a/socorro/mozilla_rulesets.py b/socorro/mozilla_rulesets.py index 2e990c910c..f8db9e73e2 100644 --- a/socorro/mozilla_rulesets.py +++ b/socorro/mozilla_rulesets.py @@ -68,11 +68,10 @@ CollectorMetadataRule(), # fix ModuleSignatureInfo if it needs fixing ConvertModuleSignatureInfoRule(), - # rules to change the internals of the raw crash - FenixVersionRewriteRule(), - ESRVersionRewrite(), # rules to transform a raw crash into a processed crash CopyFromRawCrashRule(schema=get_schema("processed_crash.schema.yaml")), + FenixVersionRewriteRule(), + ESRVersionRewrite(), SubmittedFromRule(), IdentifierRule(), MinidumpSha256HashRule(), diff --git a/socorro/processor/pipeline.py b/socorro/processor/pipeline.py index c5379385f4..c840ba0191 100644 --- a/socorro/processor/pipeline.py +++ b/socorro/processor/pipeline.py @@ -87,8 +87,8 @@ def process_crash(self, ruleset_name, raw_crash, dumps, processed_crash, tmpdir) # Apply rules; if a rule fails, capture the error and continue onward for rule in ruleset: - with sentry_sdk.push_scope() as scope: - scope.set_extra("rule", rule.name) + with sentry_sdk.new_scope() as scope: + scope.set_context("processor_pipeline", {"rule": rule.name}) try: rule.act( diff --git a/socorro/processor/processor_app.py b/socorro/processor/processor_app.py index 8444e03ac4..9054e25063 100755 --- a/socorro/processor/processor_app.py +++ b/socorro/processor/processor_app.py @@ -27,7 +27,6 @@ from fillmore.libsentry import set_up_sentry from fillmore.scrubber import Scrubber, SCRUB_RULES_DEFAULT -import psutil import sentry_sdk from sentry_sdk.integrations.atexit import AtexitIntegration from sentry_sdk.integrations.dedupe import DedupeIntegration @@ -130,9 +129,14 @@ def transform(self, task, finished_func=(lambda: None)): with METRICS.timer( "processor.process_crash", tags=[f"ruleset:{ruleset_name}"] ): - with sentry_sdk.push_scope() as scope: - scope.set_extra("crash_id", crash_id) - scope.set_extra("ruleset", ruleset_name) + with sentry_sdk.new_scope() as scope: + scope.set_context( + "processor", + { + "crash_id": crash_id, + "ruleset": ruleset_name, + }, + ) # Create temporary directory context with tempfile.TemporaryDirectory(dir=self.temporary_path) as tmpdir: @@ -270,7 +274,6 @@ def _set_up_task_manager(self): manager_settings.update( { "job_source_iterator": self.source_iterator, - "heartbeat_func": self.heartbeat, "task_func": self.transform, } ) @@ -278,65 +281,6 @@ def _set_up_task_manager(self): class_path=manager_class, kwargs=manager_settings ) - def heartbeat(self): - """Runs once a second from the main thread. - - Note: If this raises an exception, it could kill the process or put it in a - weird state. - - """ - try: - processes_by_type = {} - processes_by_status = {} - open_files = 0 - for proc in psutil.process_iter(["cmdline", "status", "open_files"]): - try: - # NOTE(willkg): This is all intertwined with exactly how we run the - # processor in a Docker container. If we ever make changes to that, this - # will change, too. However, even if we never update this, seeing - # "zombie" and "orphaned" as process statuses or seeing lots of - # processes as a type will be really fishy and suggestive that evil is a - # foot. - cmdline = proc.cmdline() or ["unknown"] - - if cmdline[0] in ["/bin/sh", "/bin/bash"]: - proc_type = "shell" - elif cmdline[0] in ["python", "/usr/local/bin/python"]: - proc_type = "python" - elif "stackwalk" in cmdline[0]: - proc_type = "stackwalker" - else: - proc_type = "other" - - open_files_count = len(proc.open_files()) - proc_status = proc.status() - - except psutil.Error: - # For any psutil error, we want to track that we saw a process, but - # the details don't matter - proc_type = "unknown" - proc_status = "unknown" - open_files_count = 0 - - processes_by_type[proc_type] = processes_by_type.get(proc_type, 0) + 1 - processes_by_status[proc_status] = ( - processes_by_status.get(proc_status, 0) + 1 - ) - open_files += open_files_count - - METRICS.gauge("processor.open_files", open_files) - for proc_type, val in processes_by_type.items(): - METRICS.gauge( - "processor.processes_by_type", val, tags=[f"proctype:{proc_type}"] - ) - for status, val in processes_by_status.items(): - METRICS.gauge( - "processor.processes_by_status", val, tags=[f"procstatus:{status}"] - ) - - except Exception as exc: - sentry_sdk.capture_exception(exc) - def close(self): """Clean up the processor on shutdown.""" with suppress(AttributeError): diff --git a/socorro/processor/rules/mozilla.py b/socorro/processor/rules/mozilla.py index a0217dc51b..a931b14377 100644 --- a/socorro/processor/rules/mozilla.py +++ b/socorro/processor/rules/mozilla.py @@ -571,23 +571,25 @@ class FenixVersionRewriteRule(Rule): """ def predicate(self, raw_crash, dumps, processed_crash, tmpdir, status): - is_nightly = (raw_crash.get("Version") or "").startswith("Nightly ") - return raw_crash.get("ProductName") == "Fenix" and is_nightly + is_nightly = (processed_crash.get("version") or "").startswith("Nightly ") + return processed_crash.get("product_name") == "Fenix" and is_nightly def action(self, raw_crash, dumps, processed_crash, tmpdir, status): - status.add_note("Changed version from %r to 0.0a1" % raw_crash.get("Version")) - raw_crash["Version"] = "0.0a1" + if "version" in processed_crash: + version = processed_crash["version"] + status.add_note(f"Changed version from {version!r} to 0.0a1") + processed_crash["version"] = "0.0a1" class ESRVersionRewrite(Rule): def predicate(self, raw_crash, dumps, processed_crash, tmpdir, status): - return raw_crash.get("ReleaseChannel", "") == "esr" + return processed_crash.get("release_channel", "") == "esr" def action(self, raw_crash, dumps, processed_crash, tmpdir, status): - try: - raw_crash["Version"] += "esr" - except KeyError: - status.add_note('"Version" missing from esr release raw_crash') + if "version" in processed_crash: + processed_crash["version"] = processed_crash["version"] + "esr" + else: + status.add_note("'version' missing from esr release processed_crash") class TopMostFilesRule(Rule): @@ -1041,8 +1043,8 @@ def __init__(self): def _error_handler(self, crash_data, exc_info, extra): """Captures errors from signature generation""" - with sentry_sdk.push_scope() as scope: - scope.set_extra("signature_rule", extra["rule"]) + with sentry_sdk.new_scope() as scope: + scope.set_context("signature_generator", {"signature_rule": extra["rule"]}) sentry_sdk.capture_exception(exc_info) def action(self, raw_crash, dumps, processed_crash, tmpdir, status): diff --git a/socorro/signature/generator.py b/socorro/signature/generator.py index 258f3d9117..84dd92dabe 100644 --- a/socorro/signature/generator.py +++ b/socorro/signature/generator.py @@ -18,6 +18,7 @@ SignatureIPCMessageName, SignatureRunWatchDog, SignatureShutdownTimeout, + SigPrintableCharsOnly, SigTruncate, StackOverflowSignature, StackwalkerErrorSignatureRule, @@ -37,6 +38,7 @@ StackOverflowSignature, HungProcess, # NOTE(willkg): These should always come last and in this order + SigPrintableCharsOnly, SigFixWhitespace, SigTruncate, ] diff --git a/socorro/signature/rules.py b/socorro/signature/rules.py index 007e23fe4f..9fca7aebe3 100644 --- a/socorro/signature/rules.py +++ b/socorro/signature/rules.py @@ -918,6 +918,20 @@ def action(self, crash_data, result): return True +class SigPrintableCharsOnly(Rule): + """Remove non-printable characters from signature.""" + + def action(self, crash_data, result): + original_sig = result.signature + sig = "".join( + [c for c in original_sig.strip() if c.isascii() and c.isprintable()] + ) + if sig != original_sig: + result.set_signature(self.name, sig) + result.info(self.name, "unprintable characters removed") + return True + + class SigFixWhitespace(Rule): """Fix whitespace in signatures. diff --git a/socorro/signature/tests/test_rules.py b/socorro/signature/tests/test_rules.py index 9a48175534..eaf82b0f09 100644 --- a/socorro/signature/tests/test_rules.py +++ b/socorro/signature/tests/test_rules.py @@ -1718,6 +1718,30 @@ def test_action_non_ascii_abort_message(self): assert result.signature == "Abort | unknown | hello" +class TestSigPrintableCharsOnly: + @pytest.mark.parametrize( + "signature, expected", + [ + ("everything | fine", "everything | fine"), + # Non-printable null character + ("libxul.so\x00 | frame2", "libxul.so | frame2"), + # Non-ascii emoji + ("libxul.so\U0001f600 | frame2", "libxul.so | frame2"), + ], + ) + def test_whitespace_fixing(self, signature, expected): + rule = rules.SigPrintableCharsOnly() + result = generator.Result() + result.signature = signature + action_result = rule.action({}, result) + assert action_result is True + assert result.signature == expected + if signature != expected: + assert result.notes == [ + "SigPrintableCharsOnly: unprintable characters removed" + ] + + class TestSigFixWhitespace: @pytest.mark.parametrize( "signature, expected", diff --git a/socorro/stage_submitter/submitter.py b/socorro/stage_submitter/submitter.py index 8f51168669..a5ba7a4229 100644 --- a/socorro/stage_submitter/submitter.py +++ b/socorro/stage_submitter/submitter.py @@ -333,11 +333,11 @@ def sample(self, destinations): def process(self, crash): with METRICS.timer("submitter.process"): - with sentry_sdk.push_scope() as scope: + with sentry_sdk.new_scope() as scope: crash_id = crash.crash_id self.logger.debug(f"processing {crash}") - scope.set_extra("crash_id", crash) + scope.set_context("submitter", {"crash_id": crash_id}) # sample and determine destinations destinations = [] diff --git a/socorro/statsd_metrics.yaml b/socorro/statsd_metrics.yaml index f7418c162c..2da71ea327 100644 --- a/socorro/statsd_metrics.yaml +++ b/socorro/statsd_metrics.yaml @@ -144,30 +144,6 @@ socorro.processor.minidumpstackwalk.run: * ``outcome``: either ``success`` or ``fail`` * ``exitcode``: the exit code of the minidump stackwalk process -socorro.processor.open_files: - type: "gauge" - description: | - Gauge of currently open files for all processes running in the container. - -socorro.processor.processes_by_type: - type: "gauge" - description: | - Gauge of processes by type. - - Tags: - - * ``proctype``: one of ``shell``, ``python``, ``stackwalker``, or ``other`` - -socorro.processor.processes_by_status: - type: "gauge" - description: | - Gauge of processes by process status. - - Tags: - - * ``procstatus``: one of ``running``, ``sleeping``, or other process - statuses. - socorro.processor.process_crash: type: "timing" description: | diff --git a/socorro/tests/external/pubsub/test_crashqueue.py b/socorro/tests/external/pubsub/test_crashqueue.py index a25cb80411..4e8b2db9ae 100644 --- a/socorro/tests/external/pubsub/test_crashqueue.py +++ b/socorro/tests/external/pubsub/test_crashqueue.py @@ -8,9 +8,11 @@ import pytest +from socorro import settings +from socorro.external.pubsub.crashqueue import CrashIdsFailedToPublish from socorro.libclass import build_instance_from_settings from socorro.lib.libooid import create_new_ooid -from socorro import settings + # Amount of time to sleep between publish and pull so messages are available PUBSUB_DELAY_PULL = 0.5 @@ -108,3 +110,30 @@ def test_publish_many(self, pubsub_helper, queue): published_crash_ids = pubsub_helper.get_published_crashids(queue) assert set(published_crash_ids) == {crash_id_1, crash_id_2, crash_id_3} + + def test_publish_with_error(self, pubsub_helper, sentry_helper): + queue = "reprocessing" + crash_id = create_new_ooid() + + crashqueue = build_instance_from_settings(settings.QUEUE_PUBSUB) + + # Run teardown_queues in the helper so there's no queue. That will cause an + # error to get thrown by PubSub. + pubsub_helper.teardown_queues() + + with sentry_helper.init() as sentry_client: + try: + crashqueue.publish(queue, [crash_id]) + except CrashIdsFailedToPublish as exc: + print(exc) + + # wait for published messages to become available before pulling + time.sleep(PUBSUB_DELAY_PULL) + + (envelope,) = sentry_client.envelope_payloads + errors = [ + f"{error['type']} {error['value']}" + for error in envelope["exception"]["values"] + ] + + assert "NotFound Topic not found" in errors diff --git a/socorro/tests/processor/rules/test_mozilla.py b/socorro/tests/processor/rules/test_mozilla.py index e589ac2eed..85701b7063 100644 --- a/socorro/tests/processor/rules/test_mozilla.py +++ b/socorro/tests/processor/rules/test_mozilla.py @@ -1273,12 +1273,12 @@ class TestFenixVersionRewriteRule: ], ) def test_predicate(self, tmp_path, product, version, expected): - raw_crash = { - "ProductName": product, - "Version": version, - } + raw_crash = {} dumps = {} - processed_crash = {} + processed_crash = { + "product_name": product, + "version": version, + } status = Status() rule = FenixVersionRewriteRule() @@ -1286,67 +1286,66 @@ def test_predicate(self, tmp_path, product, version, expected): assert ret == expected def test_act(self, tmp_path): - raw_crash = { - "ProductName": "Fenix", - "Version": "Nightly 200315 05:05", - } + raw_crash = {} dumps = {} - processed_crash = {} + processed_crash = { + "product_name": "Fenix", + "version": "Nightly 200315 05:05", + } status = Status() rule = FenixVersionRewriteRule() rule.act(raw_crash, dumps, processed_crash, str(tmp_path), status) - assert raw_crash["Version"] == "0.0a1" + assert processed_crash["version"] == "0.0a1" assert status.notes == ["Changed version from 'Nightly 200315 05:05' to 0.0a1"] class TestESRVersionRewrite: def test_everything_we_hoped_for(self, tmp_path): - raw_crash = copy.deepcopy(canonical_standard_raw_crash) - raw_crash["ReleaseChannel"] = "esr" + raw_crash = {} dumps = {} - processed_crash = {} + processed_crash = { + "release_channel": "esr", + "version": "120.0", + } status = Status() rule = ESRVersionRewrite() rule.act(raw_crash, dumps, processed_crash, str(tmp_path), status) - assert raw_crash["Version"] == "12.0esr" - - # processed_crash should be unchanged - assert processed_crash == {} + assert raw_crash == {} + assert processed_crash["version"] == "120.0esr" def test_this_is_not_the_crash_you_are_looking_for(self, tmp_path): - raw_crash = copy.deepcopy(canonical_standard_raw_crash) - raw_crash["ReleaseChannel"] = "not_esr" + raw_crash = {} dumps = {} - processed_crash = {} + processed_crash = { + "release_channel": "release", + "version": "120.0", + } status = Status() rule = ESRVersionRewrite() rule.act(raw_crash, dumps, processed_crash, str(tmp_path), status) - assert raw_crash["Version"] == "12.0" - - # processed_crash should be unchanged - assert processed_crash == {} + assert raw_crash == {} + assert processed_crash["version"] == "120.0" - def test_this_is_really_broken(self, tmp_path): - raw_crash = copy.deepcopy(canonical_standard_raw_crash) - raw_crash["ReleaseChannel"] = "esr" - del raw_crash["Version"] + def test_no_version(self, tmp_path): + raw_crash = {} dumps = {} - processed_crash = {} + processed_crash = { + "release_channel": "esr", + # no "version" + } status = Status() rule = ESRVersionRewrite() rule.act(raw_crash, dumps, processed_crash, str(tmp_path), status) - assert "Version" not in raw_crash - assert status.notes == ['"Version" missing from esr release raw_crash'] - - # processed_crash should be unchanged - assert processed_crash == {} + assert raw_crash == {} + assert "version" not in processed_crash + assert status.notes == ["'version' missing from esr release processed_crash"] class TestTopMostFilesRule: @@ -2054,7 +2053,7 @@ def predicate(self, raw_crash, processed_crash): # doesn't configure Sentry the way the processor does so we shouldn't test # whether things are scrubbed correctly with sentry_helper.init() as sentry_client: - # Override the regular SigntureGenerator with one with a BadRule + # Override the regular SignatureGenerator with one with a BadRule # in the pipeline rule.generator = SignatureGenerator( ruleset=[BadRule], error_handler=rule._error_handler @@ -2073,10 +2072,11 @@ def predicate(self, raw_crash, processed_crash): assert status.notes == ["BadRule: Rule failed: Cough"] (event,) = sentry_client.envelope_payloads - # NOTE(willkg): Some of the extra bits come from the processor app and since - # we're testing SignatureGenerator in isolation, those don't get added to - # the sentry scope - assert event["extra"] == {"signature_rule": "BadRule", "sys.argv": mock.ANY} + + # Assert that the rule that threw an error is captured in the context. + assert event["contexts"]["signature_generator"] == { + "signature_rule": "BadRule" + } assert event["exception"]["values"][0]["type"] == "Exception" diff --git a/socorro/tests/processor/test_cache_manager.py b/socorro/tests/processor/test_cache_manager.py index 75c3d9ddfc..af4d958ca4 100644 --- a/socorro/tests/processor/test_cache_manager.py +++ b/socorro/tests/processor/test_cache_manager.py @@ -470,9 +470,6 @@ def mock_make_room(*args, **kwargs): (event,) = sentry_client.envelope_payloads - # Drop the "_meta" bit because we don't want to compare that. - del event["_meta"] - # Assert that the event is what we expected differences = diff_structure(event, BROKEN_EVENT) assert differences == [] diff --git a/socorro/tests/processor/test_pipeline.py b/socorro/tests/processor/test_pipeline.py index 42f714ca73..46bca9ca1e 100644 --- a/socorro/tests/processor/test_pipeline.py +++ b/socorro/tests/processor/test_pipeline.py @@ -36,6 +36,9 @@ def action(self, *args, **kwargs): "span_id": ANY, "trace_id": ANY, }, + "processor_pipeline": { + "rule": "socorro.tests.processor.test_pipeline.BadRule", + }, }, "environment": "production", "event_id": ANY, @@ -86,9 +89,6 @@ def action(self, *args, **kwargs): } ] }, - "extra": { - "rule": "socorro.tests.processor.test_pipeline.BadRule", - }, "level": "error", "modules": ANY, "platform": "python", diff --git a/socorro/tests/processor/test_processor_app.py b/socorro/tests/processor/test_processor_app.py index 53e18296c1..dd4fa0562d 100644 --- a/socorro/tests/processor/test_processor_app.py +++ b/socorro/tests/processor/test_processor_app.py @@ -66,21 +66,6 @@ def test_source_iterator(self, processor_settings): assert next(queue) is None assert next(queue) == ((3,), {}) - def test_heartbeat(self, sentry_helper): - """Basic test to make sure it runs, captures metrics, and doesn't error out""" - with sentry_helper.reuse() as sentry_client: - with MetricsMock() as metricsmock: - app = ProcessorApp() - app.heartbeat() - - # Assert it emitted some metrics - metricsmock.assert_gauge("socorro.processor.open_files") - metricsmock.assert_gauge("socorro.processor.processes_by_type") - metricsmock.assert_gauge("socorro.processor.processes_by_status") - - # Assert it didn't throw an exception - assert len(sentry_client.envelopes) == 0 - def test_transform_success(self, processor_settings): app = ProcessorApp() app._set_up_source_and_destination() @@ -176,6 +161,10 @@ def test_transform_unexpected_exception(self, processor_settings): "span_id": ANY, "trace_id": ANY, }, + "processor": { + "crash_id": ANY, + "ruleset": "default", + }, }, "environment": "production", "event_id": ANY, @@ -234,7 +223,6 @@ def test_transform_unexpected_exception(self, processor_settings): } ] }, - "extra": {"crash_id": "930b08ba-e425-49bf-adbd-7c9172220721", "ruleset": "default"}, "level": "error", "modules": ANY, "platform": "python", diff --git a/socorro/tests/test_gcs_cli.py b/socorro/tests/test_gcs_cli.py deleted file mode 100644 index 38db2b4c70..0000000000 --- a/socorro/tests/test_gcs_cli.py +++ /dev/null @@ -1,56 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from uuid import uuid4 - -from click.testing import CliRunner - -from gcs_cli import gcs_group - - -def test_it_runs(): - """Test whether the module loads and spits out help.""" - runner = CliRunner() - result = runner.invoke(gcs_group, ["--help"]) - assert result.exit_code == 0 - - -def test_upload_file_to_root(gcs_helper, tmp_path): - """Test uploading one file to a bucket root.""" - bucket = gcs_helper.create_bucket("test").name - path = tmp_path / uuid4().hex - path.write_text(path.name) - result = CliRunner().invoke( - gcs_group, ["upload", str(path.absolute()), f"gs://{bucket}"] - ) - assert result.exit_code == 0 - assert gcs_helper.download(bucket, path.name) == path.name.encode("utf-8") - - -def test_upload_file_to_dir(gcs_helper, tmp_path): - """Test uploading one file to a directory inside a bucket.""" - bucket = gcs_helper.create_bucket("test").name - path = tmp_path / uuid4().hex - path.write_text(path.name) - result = CliRunner().invoke( - gcs_group, ["upload", str(path.absolute()), f"gs://{bucket}/{path.name}/"] - ) - assert result.exit_code == 0 - assert gcs_helper.download(bucket, f"{path.name}/{path.name}") == path.name.encode( - "utf-8" - ) - - -def test_upload_dir_to_dir(gcs_helper, tmp_path): - """Test uploading a whole directory to a directory inside a bucket.""" - bucket = gcs_helper.create_bucket("test").name - path = tmp_path / uuid4().hex - path.write_text(path.name) - result = CliRunner().invoke( - gcs_group, ["upload", str(tmp_path.absolute()), f"gs://{bucket}/{path.name}"] - ) - assert result.exit_code == 0 - assert gcs_helper.download(bucket, f"{path.name}/{path.name}") == path.name.encode( - "utf-8" - ) diff --git a/socorro/tests/test_pubsub_cli.py b/socorro/tests/test_pubsub_cli.py deleted file mode 100644 index 221fc0eeb6..0000000000 --- a/socorro/tests/test_pubsub_cli.py +++ /dev/null @@ -1,14 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from click.testing import CliRunner - -from pubsub_cli import pubsub_group - - -def test_it_runs(): - """Test whether the module loads and spits out help.""" - runner = CliRunner() - result = runner.invoke(pubsub_group, ["--help"]) - assert result.exit_code == 0 diff --git a/webapp/crashstats/crashstats/jinja2/400.html b/webapp/crashstats/crashstats/jinja2/400.html index 1d005ebdd7..a6046b6f0c 100644 --- a/webapp/crashstats/crashstats/jinja2/400.html +++ b/webapp/crashstats/crashstats/jinja2/400.html @@ -1,4 +1,4 @@ -{% extends "error.html" %} +{% extends "error_base.html" %} {% block page_title %}Bad Request{% endblock %} diff --git a/webapp/crashstats/crashstats/jinja2/404.html b/webapp/crashstats/crashstats/jinja2/404.html index 69805db8a4..2cabd49a9b 100644 --- a/webapp/crashstats/crashstats/jinja2/404.html +++ b/webapp/crashstats/crashstats/jinja2/404.html @@ -1,4 +1,4 @@ -{% extends "error.html" %} +{% extends "error_base.html" %} {% block page_title %}Page Not Found{% endblock %} diff --git a/webapp/crashstats/crashstats/jinja2/500.html b/webapp/crashstats/crashstats/jinja2/500.html index 704628dcca..a1e39d6712 100644 --- a/webapp/crashstats/crashstats/jinja2/500.html +++ b/webapp/crashstats/crashstats/jinja2/500.html @@ -1,4 +1,4 @@ -{% extends "error.html" %} +{% extends "error_base.html" %} {% block page_title %}Internal Server Error{% endblock %} diff --git a/webapp/crashstats/crashstats/jinja2/crashstats/report_index_malformed_raw_crash.html b/webapp/crashstats/crashstats/jinja2/crashstats/report_index_malformed_raw_crash.html new file mode 100644 index 0000000000..58c47e8016 --- /dev/null +++ b/webapp/crashstats/crashstats/jinja2/crashstats/report_index_malformed_raw_crash.html @@ -0,0 +1,22 @@ +{% extends "crashstats_base.html" %} + +{% block content %} +
+
+

Crash Report Malformed

+
+
+
+

+ The crash report you requested is malformed in some way such that it + cannot be shown. +

+

+ If you need to see this crash report, please + submit a bug + describing what happened, and please include the URL for this page. +

+
+
+
+{% endblock %} diff --git a/webapp/crashstats/crashstats/jinja2/crashstats_base.html b/webapp/crashstats/crashstats/jinja2/crashstats_base.html index 51dca6f84d..77bdc41ab1 100644 --- a/webapp/crashstats/crashstats/jinja2/crashstats_base.html +++ b/webapp/crashstats/crashstats/jinja2/crashstats_base.html @@ -24,7 +24,7 @@
- {% include "header_title.html" %} + Mozilla Crash Stats