From 48689975ff1eaa59e8bc69b6fde458fc7e3405b6 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Mon, 27 Nov 2017 14:14:24 -0500 Subject: [PATCH 1/5] Add e2e-tests and system checklist to docs The e2e-tests README is written in Markdown, so this includes all the things we need to use Markdown in the docs. This adds the socorro system test checklist thing we had in my unofficial socorro-zero repository. We've used variations of it several times now. I'm sure there are things that are missing, but it's designed to be an evolving document. --- .gitignore | 5 +- docker/run_build_docs.sh | 7 ++ docs/conf.py | 9 +- docs/index.rst | 2 + docs/requirements.txt | 36 +++++- docs/system_checklist.rst | 231 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 docs/system_checklist.rst diff --git a/.gitignore b/.gitignore index 0f0e7ffced..61356e6662 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ socorro/unittest/database/logs/ socorro/unittest/testlib/logs/ stackwalk/ exploitable/ -docs/_build/ *~ coverage.xml .coverage @@ -52,3 +51,7 @@ e2e-tests/.cache/ .cache/ docker-compose.override.yml my.env + +# docs things +docs/_build/ +docs/e2e_readme.md diff --git a/docker/run_build_docs.sh b/docker/run_build_docs.sh index d46f5e87ea..6812853abe 100755 --- a/docker/run_build_docs.sh +++ b/docker/run_build_docs.sh @@ -5,4 +5,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cd docs + +# Copy the e2e-test README to this directory because otherwise we can't +# pull it in because there isn't a way to do a file include for Markdown +# files. +cp ../e2e-tests/README.md e2e_readme.md + +# Build the docs make html diff --git a/docs/conf.py b/docs/conf.py index 06ce981dcf..35d1513276 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os +import sys import sphinx_rtd_theme # If extensions (or modules to document with autodoc) are in another directory, @@ -39,7 +40,11 @@ # You can specify multiple suffix as a list of strings: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] + +source_parsers = { + '.md': 'recommonmark.parser.CommonMarkParser', +} # The master toctree document. master_doc = 'index' diff --git a/docs/index.rst b/docs/index.rst index f8fa4c2dd9..35579b1910 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,5 +45,7 @@ Contents architecture/* components/* services/* + e2e_readme + system_checklist deploy howto diff --git a/docs/requirements.txt b/docs/requirements.txt index a44b1d53e8..f4ee39bce7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,34 @@ -Sphinx==1.6.5 -sphinx-rtd-theme==0.2.4 +Sphinx==1.6.5 \ + --hash=sha256:fdf77f4f30d84a314c797d67fe7d1b46665e6c48a25699d7bf0610e05a2221d4 \ + --hash=sha256:c6de5dbdbb7a0d7d2757f4389cc00e8f6eb3c49e1772378967a12cfcf2cfe098 +sphinx_rtd_theme==0.2.4 \ + --hash=sha256:62ee4752716e698bad7de8a18906f42d33664128eea06c46b718fc7fbd1a9f5c \ + --hash=sha256:2df74b8ff6fae6965c527e97cca6c6c944886aae474b490e17f92adfbe843417 +future==0.16.0 \ + --hash=sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb +snowballstemmer==1.2.1 \ + --hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \ + --hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 +alabaster==0.7.10 \ + --hash=sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732 \ + --hash=sha256:37cdcb9e9954ed60912ebc1ca12a9d12178c26637abdf124e3cde2341c257fe0 +typing==3.6.2 \ + --hash=sha256:349b1f9c109c84b53ac79ac1d822eaa68fc91d63b321bd9392df15098f746f53 \ + --hash=sha256:63a8255fe7c6269916baa440eb9b6a67139b0b97a01af632e7bd2842e1e02f15 \ + --hash=sha256:d514bd84b284dd3e844f0305ac07511f097e325171f6cc4a20878d11ad771849 +imagesize==0.7.1 \ + --hash=sha256:6ebdc9e0ad188f9d1b2cdd9bc59cbe42bf931875e829e7a595e6b3abdc05cdfb \ + --hash=sha256:0ab2c62b87987e3252f89d30b7cedbec12a01af9274af9ffa48108f2c13c6062 +Babel==2.5.1 \ + --hash=sha256:f20b2acd44f587988ff185d8949c3e208b4b3d5d20fcab7d91fe481ffa435528 \ + --hash=sha256:6007daf714d0cd5524bbe436e2d42b3c20e68da66289559341e48d2cd6d25811 +sphinxcontrib-websupport==1.0.1 \ + --hash=sha256:7a85961326aa3a400cd4ad3c816d70ed6f7c740acd7ce5d78cd0a67825072eb9 \ + --hash=sha256:f4932e95869599b89bf4f80fc3989132d83c9faa5bf633e7b5e0c25dffb75da2 +recommonmark==0.4.0 \ + --hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7 \ + --hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d +# recommonmark 0.4.0 requires CommonMark <= 0.5.4 +CommonMark==0.5.4 \ + --hash=sha256:34d73ec8085923c023930dfc0bcd1c4286e28a2a82de094bb72fabcc0281cbe5 + diff --git a/docs/system_checklist.rst b/docs/system_checklist.rst new file mode 100644 index 0000000000..49be90127b --- /dev/null +++ b/docs/system_checklist.rst @@ -0,0 +1,231 @@ +====================== +Socorro Test Checklist +====================== + +This is a high-level system-wide checklist for making sure Socorro is working +correctly in a specific environment. It's a helpful template for figuring out +what you need to change if you're pushing out a significant change. + +**Note:** This is used infrequently, so if you're about to make a significant change, +you should go through the checklist to make sure the checklist is correct and +that everything is working as expected and fix anything that's wrong, THEN +make your change, then go through the checklist again. + +Lonnen the bear says, "Only you can prevent production fires!" + +Last updated: November 15th, 2017 + + +How to use +========== + +"Significant change" can mean any number of things, so this is just a template. +You should do the following: + +1. Copy and paste the contents of this into a Google doc. + +2. Go through the Google doc, remove things that don't make sense, and add + additional things that are important. + +3. Uplift any changes via PR to this document. + + +Checklist +========= + +:: + + Migrations + ========== + + Make sure we can run migrations + + * Django migrations + + Local dev environment: + + 1. "docker-compose run webapp bash" + 2. "cd webapp-django/" + 3. "./manage.py showmigrations" + + -stage/-prod: + + * Alembic migrations + + Local dev environment: + + 1. "docker-compose run processor bash" + 2. "alembic -c docker/config/alembic.ini current" + + -stage/-prod: + + + Collector (Antenna) + =================== + + Is the collector handling incoming crashes? + + * Check datadog Antenna dashboard for the appropriate environment. + + localdev: Check the logging in the console + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Log into Sentry and check for errors. + + * Submit a crash to the collector. Verify raw crash made it to S3. + + + Processor + ========= + + Is the processor process running? + + * Log into a processor node and watch the processor logs for errors. + + Log file: "/var/log/messages" + + To check for errors: "grep ERROR /var/log/messages | less" + + * Check Datadog "processor.save_raw_and_processed" for appropriate + environment. + + localdev: Check the logging in the console + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + Is the processor saving to ES? Postgres? S3? + + * Check Datadog + "processor.es.ESCrashStorageRedactedJsonDump.save_raw_and_processed.avg" + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Check Datadog + "processor.s3.BotoS3CrashStorage.save_raw_and_processed" for + appropriate environment. + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Check Datadog + "processor.postgres.PostgreSQLCrashStorage.save_raw_and_processed" + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + + Submit a crash or reprocess a crash. Wait a few minutes. Verify the crash was + processed and made it to S3, ES and Postgres. + + **FIXME:** We should write a script that uses envconsul to provide vars and takes + a uuid via the command line and then checks all the things to make sure it's + there. This assumes we don't already have one--we might! + + + Webapp + ====== + + Is the webapp up? + + * Use a browser and check the healthcheck. + + localdev: http://localhost:8000/monitoring/healthcheck/ + stage: https://crash-stats.allizom.org/monitoring/healthcheck/ + prod: https://crash-stats.mozilla.com/monitoring/healthcheck/ + + It should say "ok: true". + + Is the webapp throwing errors? + + * Check Sentry for errors + * Log into webapp node and check logs for errors. + + Log file: "/var/log/messages" + + To check for errors: "grep ERROR /var/log/messages | less" + + * Run QA Selenium tests. + + localdev: ? + stage: In IRC: "webqatestbot build socorro.stage.saucelabs" + prod: In IRC: "webqatestbot build socorro.prod.saucelabs" + + Can we log into the webapp? + + * Log in and check the profile page. + + Is the product home page working? + + * Check the Firefox product home page + + localdev: http://localhost:8000/ + stage: https://crash-stats.allizom.org/home/product/Firefox + prod: https://crash-stats.mozilla.com/home/product/Firefox + + Is super search working? + + * Click "Super Search" and make a search that is not likely to be cached. + For example, filter on a specific date. + + Top Crashers Signature report and Report index + + 1. Browse to Top Crashers + 2. Click on a crash signature to browse to Signature report + 3. Click on a crash id to browse to report index + + Can you upload a symbols file? + + * Download https://github.com/mozilla/socorro/blob/master/webapp-django/crashstats/symbols/tests/sample.zip + to disk + * Log in with a user with permission to upload symbols. + * Go to the symbol upload section. + + stage: https://crash-stats.allizom.org/symbols/upload/web/ + prod: https://crash-stats.mozilla.com/symbols/upload/web/ + + * Try to upload the "sample.zip" file. + * To verify that it worked, go to the public symbols S3 bucket: + + stage: org.mozilla.crash-stats.staging.symbols-public + + and check that there is a "xpcshell.sym" file in the root with a recent + modify date. + + + Crontabber + ========== + + Is crontabber working? + + * Check healthcheck endpoint: + + localdev: http://localhost:8000/monitoring/crontabber/ + stage: https://crash-stats.allizom.org/monitoring/crontabber/ + prod: https://crash-stats.mozilla.com/monitoring/crontabber/ + + It should say ALLGOOD. + + * Check the webapp crontabber-state page: + + localdev: http://localhost:8000/crontabber-state/ + stage: https://crash-stats.allizom.org/crontabber-state/ + prod: https://crash-stats.mozilla.com/crontabber-state/ + + Is crontabber throwing errors? + + * Check Sentry for errors + * Log into admin node and check logs for errors + + Log file: "/var/log/socorro/crontabber" + + To check for errors: "grep ERROR /var/log/messages | less" + + + Stage submitter + =============== + + Is the stage submitter running and sending crashes? + + * Check Datadog dashboard for Antenna on -stage From c8fbb8542258d9737957fe3b355ab9dfc9f16b53 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Mon, 27 Nov 2017 14:19:18 -0500 Subject: [PATCH 2/5] Move tests docs into their own sub-directory --- .gitignore | 2 +- docker/run_build_docs.sh | 2 +- docs/index.rst | 3 +- docs/system_checklist.rst | 231 -------------------------------------- 4 files changed, 3 insertions(+), 235 deletions(-) delete mode 100644 docs/system_checklist.rst diff --git a/.gitignore b/.gitignore index 61356e6662..72b15fe7e7 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,4 @@ my.env # docs things docs/_build/ -docs/e2e_readme.md +docs/tests/e2e_readme.md diff --git a/docker/run_build_docs.sh b/docker/run_build_docs.sh index 6812853abe..01c471eaf1 100755 --- a/docker/run_build_docs.sh +++ b/docker/run_build_docs.sh @@ -9,7 +9,7 @@ cd docs # Copy the e2e-test README to this directory because otherwise we can't # pull it in because there isn't a way to do a file include for Markdown # files. -cp ../e2e-tests/README.md e2e_readme.md +cp ../e2e-tests/README.md tests/e2e_readme.md # Build the docs make html diff --git a/docs/index.rst b/docs/index.rst index 35579b1910..14dce8f382 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,7 +45,6 @@ Contents architecture/* components/* services/* - e2e_readme - system_checklist + tests/* deploy howto diff --git a/docs/system_checklist.rst b/docs/system_checklist.rst deleted file mode 100644 index 49be90127b..0000000000 --- a/docs/system_checklist.rst +++ /dev/null @@ -1,231 +0,0 @@ -====================== -Socorro Test Checklist -====================== - -This is a high-level system-wide checklist for making sure Socorro is working -correctly in a specific environment. It's a helpful template for figuring out -what you need to change if you're pushing out a significant change. - -**Note:** This is used infrequently, so if you're about to make a significant change, -you should go through the checklist to make sure the checklist is correct and -that everything is working as expected and fix anything that's wrong, THEN -make your change, then go through the checklist again. - -Lonnen the bear says, "Only you can prevent production fires!" - -Last updated: November 15th, 2017 - - -How to use -========== - -"Significant change" can mean any number of things, so this is just a template. -You should do the following: - -1. Copy and paste the contents of this into a Google doc. - -2. Go through the Google doc, remove things that don't make sense, and add - additional things that are important. - -3. Uplift any changes via PR to this document. - - -Checklist -========= - -:: - - Migrations - ========== - - Make sure we can run migrations - - * Django migrations - - Local dev environment: - - 1. "docker-compose run webapp bash" - 2. "cd webapp-django/" - 3. "./manage.py showmigrations" - - -stage/-prod: - - * Alembic migrations - - Local dev environment: - - 1. "docker-compose run processor bash" - 2. "alembic -c docker/config/alembic.ini current" - - -stage/-prod: - - - Collector (Antenna) - =================== - - Is the collector handling incoming crashes? - - * Check datadog Antenna dashboard for the appropriate environment. - - localdev: Check the logging in the console - stage: https://app.datadoghq.com/dash/272676/antenna--stage - prod: https://app.datadoghq.com/dash/274773/antenna--prod - - * Log into Sentry and check for errors. - - * Submit a crash to the collector. Verify raw crash made it to S3. - - - Processor - ========= - - Is the processor process running? - - * Log into a processor node and watch the processor logs for errors. - - Log file: "/var/log/messages" - - To check for errors: "grep ERROR /var/log/messages | less" - - * Check Datadog "processor.save_raw_and_processed" for appropriate - environment. - - localdev: Check the logging in the console - stage: https://app.datadoghq.com/dash/272676/antenna--stage - prod: https://app.datadoghq.com/dash/274773/antenna--prod - - Is the processor saving to ES? Postgres? S3? - - * Check Datadog - "processor.es.ESCrashStorageRedactedJsonDump.save_raw_and_processed.avg" - - stage: https://app.datadoghq.com/dash/272676/antenna--stage - prod: https://app.datadoghq.com/dash/274773/antenna--prod - - * Check Datadog - "processor.s3.BotoS3CrashStorage.save_raw_and_processed" for - appropriate environment. - - stage: https://app.datadoghq.com/dash/272676/antenna--stage - prod: https://app.datadoghq.com/dash/274773/antenna--prod - - * Check Datadog - "processor.postgres.PostgreSQLCrashStorage.save_raw_and_processed" - - stage: https://app.datadoghq.com/dash/272676/antenna--stage - prod: https://app.datadoghq.com/dash/274773/antenna--prod - - - Submit a crash or reprocess a crash. Wait a few minutes. Verify the crash was - processed and made it to S3, ES and Postgres. - - **FIXME:** We should write a script that uses envconsul to provide vars and takes - a uuid via the command line and then checks all the things to make sure it's - there. This assumes we don't already have one--we might! - - - Webapp - ====== - - Is the webapp up? - - * Use a browser and check the healthcheck. - - localdev: http://localhost:8000/monitoring/healthcheck/ - stage: https://crash-stats.allizom.org/monitoring/healthcheck/ - prod: https://crash-stats.mozilla.com/monitoring/healthcheck/ - - It should say "ok: true". - - Is the webapp throwing errors? - - * Check Sentry for errors - * Log into webapp node and check logs for errors. - - Log file: "/var/log/messages" - - To check for errors: "grep ERROR /var/log/messages | less" - - * Run QA Selenium tests. - - localdev: ? - stage: In IRC: "webqatestbot build socorro.stage.saucelabs" - prod: In IRC: "webqatestbot build socorro.prod.saucelabs" - - Can we log into the webapp? - - * Log in and check the profile page. - - Is the product home page working? - - * Check the Firefox product home page - - localdev: http://localhost:8000/ - stage: https://crash-stats.allizom.org/home/product/Firefox - prod: https://crash-stats.mozilla.com/home/product/Firefox - - Is super search working? - - * Click "Super Search" and make a search that is not likely to be cached. - For example, filter on a specific date. - - Top Crashers Signature report and Report index - - 1. Browse to Top Crashers - 2. Click on a crash signature to browse to Signature report - 3. Click on a crash id to browse to report index - - Can you upload a symbols file? - - * Download https://github.com/mozilla/socorro/blob/master/webapp-django/crashstats/symbols/tests/sample.zip - to disk - * Log in with a user with permission to upload symbols. - * Go to the symbol upload section. - - stage: https://crash-stats.allizom.org/symbols/upload/web/ - prod: https://crash-stats.mozilla.com/symbols/upload/web/ - - * Try to upload the "sample.zip" file. - * To verify that it worked, go to the public symbols S3 bucket: - - stage: org.mozilla.crash-stats.staging.symbols-public - - and check that there is a "xpcshell.sym" file in the root with a recent - modify date. - - - Crontabber - ========== - - Is crontabber working? - - * Check healthcheck endpoint: - - localdev: http://localhost:8000/monitoring/crontabber/ - stage: https://crash-stats.allizom.org/monitoring/crontabber/ - prod: https://crash-stats.mozilla.com/monitoring/crontabber/ - - It should say ALLGOOD. - - * Check the webapp crontabber-state page: - - localdev: http://localhost:8000/crontabber-state/ - stage: https://crash-stats.allizom.org/crontabber-state/ - prod: https://crash-stats.mozilla.com/crontabber-state/ - - Is crontabber throwing errors? - - * Check Sentry for errors - * Log into admin node and check logs for errors - - Log file: "/var/log/socorro/crontabber" - - To check for errors: "grep ERROR /var/log/messages | less" - - - Stage submitter - =============== - - Is the stage submitter running and sending crashes? - - * Check Datadog dashboard for Antenna on -stage From a8780d548c83c4019d1aef0b370bd41452dd7f5b Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Mon, 27 Nov 2017 14:25:35 -0500 Subject: [PATCH 3/5] Add system checklist for realz --- docs/tests/system_checklist.rst | 231 ++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 docs/tests/system_checklist.rst diff --git a/docs/tests/system_checklist.rst b/docs/tests/system_checklist.rst new file mode 100644 index 0000000000..49be90127b --- /dev/null +++ b/docs/tests/system_checklist.rst @@ -0,0 +1,231 @@ +====================== +Socorro Test Checklist +====================== + +This is a high-level system-wide checklist for making sure Socorro is working +correctly in a specific environment. It's a helpful template for figuring out +what you need to change if you're pushing out a significant change. + +**Note:** This is used infrequently, so if you're about to make a significant change, +you should go through the checklist to make sure the checklist is correct and +that everything is working as expected and fix anything that's wrong, THEN +make your change, then go through the checklist again. + +Lonnen the bear says, "Only you can prevent production fires!" + +Last updated: November 15th, 2017 + + +How to use +========== + +"Significant change" can mean any number of things, so this is just a template. +You should do the following: + +1. Copy and paste the contents of this into a Google doc. + +2. Go through the Google doc, remove things that don't make sense, and add + additional things that are important. + +3. Uplift any changes via PR to this document. + + +Checklist +========= + +:: + + Migrations + ========== + + Make sure we can run migrations + + * Django migrations + + Local dev environment: + + 1. "docker-compose run webapp bash" + 2. "cd webapp-django/" + 3. "./manage.py showmigrations" + + -stage/-prod: + + * Alembic migrations + + Local dev environment: + + 1. "docker-compose run processor bash" + 2. "alembic -c docker/config/alembic.ini current" + + -stage/-prod: + + + Collector (Antenna) + =================== + + Is the collector handling incoming crashes? + + * Check datadog Antenna dashboard for the appropriate environment. + + localdev: Check the logging in the console + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Log into Sentry and check for errors. + + * Submit a crash to the collector. Verify raw crash made it to S3. + + + Processor + ========= + + Is the processor process running? + + * Log into a processor node and watch the processor logs for errors. + + Log file: "/var/log/messages" + + To check for errors: "grep ERROR /var/log/messages | less" + + * Check Datadog "processor.save_raw_and_processed" for appropriate + environment. + + localdev: Check the logging in the console + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + Is the processor saving to ES? Postgres? S3? + + * Check Datadog + "processor.es.ESCrashStorageRedactedJsonDump.save_raw_and_processed.avg" + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Check Datadog + "processor.s3.BotoS3CrashStorage.save_raw_and_processed" for + appropriate environment. + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + * Check Datadog + "processor.postgres.PostgreSQLCrashStorage.save_raw_and_processed" + + stage: https://app.datadoghq.com/dash/272676/antenna--stage + prod: https://app.datadoghq.com/dash/274773/antenna--prod + + + Submit a crash or reprocess a crash. Wait a few minutes. Verify the crash was + processed and made it to S3, ES and Postgres. + + **FIXME:** We should write a script that uses envconsul to provide vars and takes + a uuid via the command line and then checks all the things to make sure it's + there. This assumes we don't already have one--we might! + + + Webapp + ====== + + Is the webapp up? + + * Use a browser and check the healthcheck. + + localdev: http://localhost:8000/monitoring/healthcheck/ + stage: https://crash-stats.allizom.org/monitoring/healthcheck/ + prod: https://crash-stats.mozilla.com/monitoring/healthcheck/ + + It should say "ok: true". + + Is the webapp throwing errors? + + * Check Sentry for errors + * Log into webapp node and check logs for errors. + + Log file: "/var/log/messages" + + To check for errors: "grep ERROR /var/log/messages | less" + + * Run QA Selenium tests. + + localdev: ? + stage: In IRC: "webqatestbot build socorro.stage.saucelabs" + prod: In IRC: "webqatestbot build socorro.prod.saucelabs" + + Can we log into the webapp? + + * Log in and check the profile page. + + Is the product home page working? + + * Check the Firefox product home page + + localdev: http://localhost:8000/ + stage: https://crash-stats.allizom.org/home/product/Firefox + prod: https://crash-stats.mozilla.com/home/product/Firefox + + Is super search working? + + * Click "Super Search" and make a search that is not likely to be cached. + For example, filter on a specific date. + + Top Crashers Signature report and Report index + + 1. Browse to Top Crashers + 2. Click on a crash signature to browse to Signature report + 3. Click on a crash id to browse to report index + + Can you upload a symbols file? + + * Download https://github.com/mozilla/socorro/blob/master/webapp-django/crashstats/symbols/tests/sample.zip + to disk + * Log in with a user with permission to upload symbols. + * Go to the symbol upload section. + + stage: https://crash-stats.allizom.org/symbols/upload/web/ + prod: https://crash-stats.mozilla.com/symbols/upload/web/ + + * Try to upload the "sample.zip" file. + * To verify that it worked, go to the public symbols S3 bucket: + + stage: org.mozilla.crash-stats.staging.symbols-public + + and check that there is a "xpcshell.sym" file in the root with a recent + modify date. + + + Crontabber + ========== + + Is crontabber working? + + * Check healthcheck endpoint: + + localdev: http://localhost:8000/monitoring/crontabber/ + stage: https://crash-stats.allizom.org/monitoring/crontabber/ + prod: https://crash-stats.mozilla.com/monitoring/crontabber/ + + It should say ALLGOOD. + + * Check the webapp crontabber-state page: + + localdev: http://localhost:8000/crontabber-state/ + stage: https://crash-stats.allizom.org/crontabber-state/ + prod: https://crash-stats.mozilla.com/crontabber-state/ + + Is crontabber throwing errors? + + * Check Sentry for errors + * Log into admin node and check logs for errors + + Log file: "/var/log/socorro/crontabber" + + To check for errors: "grep ERROR /var/log/messages | less" + + + Stage submitter + =============== + + Is the stage submitter running and sending crashes? + + * Check Datadog dashboard for Antenna on -stage From 91fdfeabfd4ee34b885d38446ee103cabd19bb76 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Tue, 28 Nov 2017 13:00:59 -0500 Subject: [PATCH 4/5] Redo urls This redoes the urls so that it shows the paths rather than hosts and paths since hosts will change over time. --- docs/tests/system_checklist.rst | 43 +++++++++------------------------ 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/docs/tests/system_checklist.rst b/docs/tests/system_checklist.rst index 49be90127b..43a7cbb863 100644 --- a/docs/tests/system_checklist.rst +++ b/docs/tests/system_checklist.rst @@ -50,6 +50,8 @@ Checklist -stage/-prod: + 1. See Mana + * Alembic migrations Local dev environment: @@ -59,6 +61,8 @@ Checklist -stage/-prod: + 1. See Mana + Collector (Antenna) =================== @@ -129,11 +133,7 @@ Checklist Is the webapp up? - * Use a browser and check the healthcheck. - - localdev: http://localhost:8000/monitoring/healthcheck/ - stage: https://crash-stats.allizom.org/monitoring/healthcheck/ - prod: https://crash-stats.mozilla.com/monitoring/healthcheck/ + * Use a browser and check the healthcheck (/monitoring/healthcheck) It should say "ok: true". @@ -158,11 +158,7 @@ Checklist Is the product home page working? - * Check the Firefox product home page - - localdev: http://localhost:8000/ - stage: https://crash-stats.allizom.org/home/product/Firefox - prod: https://crash-stats.mozilla.com/home/product/Firefox + * Check the Firefox product home page (/ redirects to /home/product/Firefox) Is super search working? @@ -180,18 +176,11 @@ Checklist * Download https://github.com/mozilla/socorro/blob/master/webapp-django/crashstats/symbols/tests/sample.zip to disk * Log in with a user with permission to upload symbols. - * Go to the symbol upload section. - - stage: https://crash-stats.allizom.org/symbols/upload/web/ - prod: https://crash-stats.mozilla.com/symbols/upload/web/ - + * Go to the symbol upload section (/symbols/upload/web) * Try to upload the "sample.zip" file. - * To verify that it worked, go to the public symbols S3 bucket: - - stage: org.mozilla.crash-stats.staging.symbols-public - - and check that there is a "xpcshell.sym" file in the root with a recent - modify date. + * To verify that it worked, go to the public symbols S3 bucket and check + that there is a "xpcshell.sym" file in the root with a recent modify + date. Crontabber @@ -199,19 +188,11 @@ Checklist Is crontabber working? - * Check healthcheck endpoint: - - localdev: http://localhost:8000/monitoring/crontabber/ - stage: https://crash-stats.allizom.org/monitoring/crontabber/ - prod: https://crash-stats.mozilla.com/monitoring/crontabber/ + * Check healthcheck endpoint (/monitoring/crontabber/) It should say ALLGOOD. - * Check the webapp crontabber-state page: - - localdev: http://localhost:8000/crontabber-state/ - stage: https://crash-stats.allizom.org/crontabber-state/ - prod: https://crash-stats.mozilla.com/crontabber-state/ + * Check the webapp crontabber-state page (/crontabber-state/) Is crontabber throwing errors? From 91d3f34205957d4775973586c902f4ea0b53b776 Mon Sep 17 00:00:00 2001 From: Will Kahn-Greene Date: Tue, 28 Nov 2017 15:38:40 -0500 Subject: [PATCH 5/5] Adjust commentary on how to use the checklist --- docs/tests/system_checklist.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/tests/system_checklist.rst b/docs/tests/system_checklist.rst index 43a7cbb863..4973a0d426 100644 --- a/docs/tests/system_checklist.rst +++ b/docs/tests/system_checklist.rst @@ -22,12 +22,13 @@ How to use "Significant change" can mean any number of things, so this is just a template. You should do the following: -1. Copy and paste the contents of this into a Google doc. +1. Copy and paste the contents of this into a Google Doc, Etherpad, or + whatever system you plan to use to keep track of status and outstanding + issues. -2. Go through the Google doc, remove things that don't make sense, and add - additional things that are important. - -3. Uplift any changes via PR to this document. +2. Go through what you copy-and-pasted, remove things that don't make sense, + and add additional things that are important. (Please uplift any changes + via PR to this document that are interesting.) Checklist