From 59422b992fe9a80744089190e6a6940b2143313a Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Thu, 11 May 2023 14:35:32 -0400 Subject: [PATCH] feat!: Remove code not needed after build-jenkins retirement (#362) Much of the code in this repository is no longer needed after the Jenkins server that used to run the edx-platform test suite was retired (the tests were moved to GitHub Actions). Most of what remains is a script used to programmatically create pull requests. --- Makefile | 1 - README.md | 14 +- aws/__init__.py | 0 aws/deregister_amis.py | 102 ---- aws/tests/__init__.py | 0 aws/tests/test_deregister_amis.py | 129 ----- jenkins/admin-scripts/README.md | 3 - .../admin-scripts/delete-old-workers.groovy | 20 - .../admin-scripts/disable-old-workers.groovy | 22 - jenkins/bokchoy_db_pull_request.py | 106 ---- jenkins/build.py | 41 -- jenkins/codecov_response_metrics.py | 251 --------- jenkins/edx_platform_test_notifier.py | 155 ------ jenkins/helpers.py | 22 - jenkins/job.py | 102 ---- jenkins/repo_health_reports.py | 119 ----- jenkins/tests/helpers.py | 119 ----- jenkins/tests/test_bokchoy_db_pull_request.py | 115 ---- jenkins/tests/test_build.py | 30 -- jenkins/tests/test_codecov_analysis.py | 174 ------ jenkins/tests/test_helpers.py | 27 - jenkins/tests/test_job.py | 53 -- jenkins/tests/test_timeout.py | 115 ---- jenkins/timeout.py | 137 ----- jenkins/toggle-spigot.py | 122 ----- lambdas/process_webhooks/README.md | 20 - lambdas/process_webhooks/__init__.py | 0 lambdas/process_webhooks/process_webhooks.py | 205 -------- lambdas/process_webhooks/test/__init__.py | 0 .../test/test_process_webhooks.py | 201 ------- lambdas/restart_jenkins/README.md | 24 - lambdas/restart_jenkins/__init__.py | 0 lambdas/restart_jenkins/restart_jenkins.py | 102 ---- lambdas/send_from_queue/README.md | 19 - lambdas/send_from_queue/__init__.py | 0 lambdas/send_from_queue/send_from_queue.py | 226 -------- lambdas/send_from_queue/test/__init__.py | 0 .../test/test_send_from_queue.py | 106 ---- requirements/aws.in | 3 - requirements/aws.txt | 24 - requirements/base.in | 2 - requirements/base.txt | 33 +- requirements/dev.txt | 32 +- requirements/testing.in | 2 - requirements/testing.txt | 36 +- requirements/travis.in | 3 - requirements/travis.txt | 18 - scripts/create_incr_tickets.py | 186 ------- scripts/setup.sh | 28 - scripts/test.py | 92 ---- tox.ini | 10 +- travis/README.md | 18 - travis/__init__.py | 0 travis/build_info.py | 309 ----------- travis/tests/__init__.py | 0 travis/tests/fixtures/builds_response.list | 8 - travis/tests/test_build_info.py | 496 ------------------ 57 files changed, 14 insertions(+), 4168 deletions(-) delete mode 100644 aws/__init__.py delete mode 100644 aws/deregister_amis.py delete mode 100644 aws/tests/__init__.py delete mode 100644 aws/tests/test_deregister_amis.py delete mode 100644 jenkins/admin-scripts/README.md delete mode 100644 jenkins/admin-scripts/delete-old-workers.groovy delete mode 100644 jenkins/admin-scripts/disable-old-workers.groovy delete mode 100644 jenkins/bokchoy_db_pull_request.py delete mode 100644 jenkins/build.py delete mode 100644 jenkins/codecov_response_metrics.py delete mode 100644 jenkins/edx_platform_test_notifier.py delete mode 100644 jenkins/helpers.py delete mode 100644 jenkins/job.py delete mode 100644 jenkins/repo_health_reports.py delete mode 100644 jenkins/tests/helpers.py delete mode 100644 jenkins/tests/test_bokchoy_db_pull_request.py delete mode 100644 jenkins/tests/test_build.py delete mode 100644 jenkins/tests/test_codecov_analysis.py delete mode 100644 jenkins/tests/test_helpers.py delete mode 100644 jenkins/tests/test_job.py delete mode 100644 jenkins/tests/test_timeout.py delete mode 100644 jenkins/timeout.py delete mode 100644 jenkins/toggle-spigot.py delete mode 100644 lambdas/process_webhooks/README.md delete mode 100644 lambdas/process_webhooks/__init__.py delete mode 100644 lambdas/process_webhooks/process_webhooks.py delete mode 100644 lambdas/process_webhooks/test/__init__.py delete mode 100644 lambdas/process_webhooks/test/test_process_webhooks.py delete mode 100644 lambdas/restart_jenkins/README.md delete mode 100644 lambdas/restart_jenkins/__init__.py delete mode 100644 lambdas/restart_jenkins/restart_jenkins.py delete mode 100644 lambdas/send_from_queue/README.md delete mode 100644 lambdas/send_from_queue/__init__.py delete mode 100644 lambdas/send_from_queue/send_from_queue.py delete mode 100644 lambdas/send_from_queue/test/__init__.py delete mode 100644 lambdas/send_from_queue/test/test_send_from_queue.py delete mode 100644 requirements/aws.in delete mode 100644 requirements/aws.txt delete mode 100644 requirements/travis.in delete mode 100644 requirements/travis.txt delete mode 100644 scripts/create_incr_tickets.py delete mode 100755 scripts/setup.sh delete mode 100644 scripts/test.py delete mode 100644 travis/README.md delete mode 100644 travis/__init__.py delete mode 100644 travis/build_info.py delete mode 100644 travis/tests/__init__.py delete mode 100644 travis/tests/fixtures/builds_response.list delete mode 100644 travis/tests/test_build_info.py diff --git a/Makefile b/Makefile index aa2c7ff0..b8e46c26 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy $(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in pip install -qr requirements/pip.txt pip install -qr requirements/pip-tools.txt - $(PIP_COMPILE) -o requirements/aws.txt requirements/aws.in $(PIP_COMPILE) -o requirements/base.txt requirements/base.in $(PIP_COMPILE) -o requirements/testing.txt requirements/testing.in $(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in diff --git a/README.md b/README.md index 55f51501..b135d61f 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,6 @@ ## About -This repo contains the scripts and tools we use at edX to maintain our build infrastructure, specifically related to managing our Jenkins infrastructure, gathering data on our usage of Travis, and for testing browser performance as part of our [edx-platform](https://github.com/edx/edx-platform) CI. +This repo contains a script to programmatically create GitHub pull requests, used in GitHub Actions such as the ones used to periodically update Python dependencies in most of our repositories. -This repo contains: - -* Scripts that do things like: - * Find all Travis builds on a given github org (like edx) and report on status + counts - * Clean up orphaned jenkins-worker nodes with whom we have lost contact - -* Job infrastructure for our Jenkins instance - -## Developing - -To get a PR merged, file a review request in the SRE Jira portal. (SRE has to perform an additional deployment step after merging.) +It used to also contain the scripts and tools we used at edX to maintain our build infrastructure, specifically related to managing our Jenkins infrastructure, gathering data on our usage of Travis, and for testing browser performance as part of our [edx-platform](https://github.com/edx/edx-platform) CI. This explains the naming choices which seem incongruent with the current repository content. diff --git a/aws/__init__.py b/aws/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/aws/deregister_amis.py b/aws/deregister_amis.py deleted file mode 100644 index 055b4b1f..00000000 --- a/aws/deregister_amis.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Use boto3 to deregister AMIs that match a given tag key-value pair. -This is used by the clean-up-AMIs Jenkins job. - -That tag key-value pair is hardcoded to: - delete_or_keep: delete - -Usage: - If you've defined the AWS credentials as environment variables or in a - .boto file, then use: - `python deregister_amis.py - - Else, you can add the aws keys as arguments to the above command: - ** '--aws-access-key-id' or '-i' - ** '--aws-secret-access-key' or '-s' - - If you don't want to deregister AMIs, but you'd like to know which ones - you'd deregister if you ran the command, then use the --dry-run switch. - -""" -import argparse -import logging -import os -import sys - -import boto3 -from botocore.exceptions import ClientError - -logger = logging.getLogger(__name__) - - -def deregister_amis_by_tag(tag_key, tag_value, dry_run, ec2): - """ - Deregisters AMIs that are found according to tag key/value pairs. - """ - - tag_key_string = f"tag:{tag_key}" - - logger.info("Finding AMIs tagged with {key}: {value}".format( - key=tag_key, - value=tag_value, - )) - try: - filters = [{'Name': tag_key_string, 'Values': [tag_value]}] - amis = ec2.images.filter(Filters=filters) - except ClientError: - logger.exception("An error occurred gathering images.") - raise - - if len(list(amis)) == 0: - logger.info('No images found matching criteria.') - return - for i in amis: - logger.info("Deregistering {image}".format(image=str(i))) - if dry_run: - logger.info("--> Dry run: skipping deregister") - else: - i.deregister() - - -def main(raw_args): # pylint: disable=missing-function-docstring - desc = ( - "Deregister EC2 images that are tagged for 'delete_or_keep' with " - "'delete' as the tag value." - ) - parser = argparse.ArgumentParser(description=desc) - parser.add_argument( - '--dry-run', - action='store_true', - default=False, - help=""" - Do not deregister any AMIs, just list the ones - that are found matching the tag key/value pair. - """ - ) - parser.add_argument( - '--log-level', - dest='log_level', - help="set logging level", - choices=[ - 'DEBUG', 'debug', - 'INFO', 'info', - 'WARNING', 'warning', - 'ERROR', 'error', - 'CRITICAL', 'critical', - ], - default="INFO", - ) - args = parser.parse_args(raw_args) - - # Set logging level - logging.getLogger(__name__).setLevel(args.log_level.upper()) - logging.getLogger('boto3').setLevel(logging.INFO) - logging.getLogger('botocore').setLevel(logging.INFO) - region = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') - ec2 = boto3.resource('ec2', region_name=region) - deregister_amis_by_tag("delete_or_keep", "delete", args.dry_run, ec2) - - -if __name__ == "__main__": - logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s') - main(sys.argv[1:]) diff --git a/aws/tests/__init__.py b/aws/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/aws/tests/test_deregister_amis.py b/aws/tests/test_deregister_amis.py deleted file mode 100644 index efacb016..00000000 --- a/aws/tests/test_deregister_amis.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Tests for testeng-ci/aws. -""" -import os -from unittest import TestCase -from unittest.mock import MagicMock, patch - -import boto3 -from botocore.exceptions import ClientError -from testfixtures import LogCapture - -from aws.deregister_amis import deregister_amis_by_tag, main - - -class MockImages: - """ - Mock boto3 EC2 AMI collection for use in test cases. - """ - def __init__(self, matching_image_exists, filter_raises_error): - self.matching_image_exists = matching_image_exists - self.filter_raises_error = filter_raises_error - self.image = MagicMock() - self.image.__str__.return_value = 'test_ami' - - def filter(self, *args, **kwargs): # lint-amnesty, pylint: disable=missing-function-docstring, unused-argument - if self.filter_raises_error: - raise ClientError(MagicMock(), 'filter') - if self.matching_image_exists: - return [self.image] - return [] - - -class MockEC2: - """ - Mock boto3 EC2 resource implementation for use in test cases. - """ - def __init__(self, matching_image_exists=True, filter_raises_error=False): - self.images = MockImages(matching_image_exists, filter_raises_error) - - -class DeregisterAmisTestCase(TestCase): - """ - TestCase class for testing get_running_instances.py. - """ - - def setUp(self): - self.args = [ - '--log-level', 'INFO', - ] - - @patch('boto3.resource', return_value=MockEC2(matching_image_exists=False)) - def test_main(self, mock_ec2): # lint-amnesty, pylint: disable=unused-argument - """ - Test output of main - """ - with LogCapture() as capture: - main(self.args) - capture.check( - ('aws.deregister_amis', - 'INFO', - 'Finding AMIs tagged with delete_or_keep: delete'), - - ('aws.deregister_amis', - 'INFO', - 'No images found matching criteria.') - ) - - @patch('boto3.resource', return_value=MockEC2()) - def test_main_deregister(self, mock_ec2): # lint-amnesty, pylint: disable=unused-argument - """ - Test that a correctly-tagged AMI is deregistered - """ - with LogCapture() as capture: - main(self.args) - - capture.check( - ('aws.deregister_amis', - 'INFO', - 'Finding AMIs tagged with delete_or_keep: delete'), - - ('aws.deregister_amis', - 'INFO', - 'Deregistering test_ami') - ) - - @patch('boto3.resource', return_value=MockEC2(matching_image_exists=False)) - def test_main_no_deregister(self, mock_ec2): # lint-amnesty, pylint: disable=unused-argument - """ - Test that an AMI without proper tags is not de-registered - """ - with LogCapture() as capture: - main(self.args) - - capture.check( - ('aws.deregister_amis', - 'INFO', - 'Finding AMIs tagged with delete_or_keep: delete'), - - ('aws.deregister_amis', - 'INFO', - 'No images found matching criteria.') - ) - - def test_main_dry_run(self): - """ - Test that a correctly-tagged AMI is NOT deregistered - """ - self.args.append('--dry-run') - mock_ec2 = MockEC2() - with patch('boto3.resource', return_value=mock_ec2): - main(self.args) - mock_ec2.images.image.deregister.assert_not_called() - - -class DeregisterExceptionTestCase(TestCase): - """ - Test exceptions that would be thrown from the script. - """ - @patch('boto3.resource', return_value=MockEC2(filter_raises_error=True)) - def test_cant_get_instances(self, mock_ec2): # lint-amnesty, pylint: disable=unused-argument - region = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') - ec2 = boto3.resource('ec2', region_name=region) - with self.assertRaises(ClientError): - deregister_amis_by_tag( - "foo_tag", - "foo_tag_value", - dry_run=False, - ec2=ec2 - ) diff --git a/jenkins/admin-scripts/README.md b/jenkins/admin-scripts/README.md deleted file mode 100644 index 66ade726..00000000 --- a/jenkins/admin-scripts/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This directory contains scripts which can be run on the Jenkins Script Console (e.g., https://my-jenkins/script) - -These scripts can be used for by an administrator and run in real-time, as the needs arise. diff --git a/jenkins/admin-scripts/delete-old-workers.groovy b/jenkins/admin-scripts/delete-old-workers.groovy deleted file mode 100644 index a70b4694..00000000 --- a/jenkins/admin-scripts/delete-old-workers.groovy +++ /dev/null @@ -1,20 +0,0 @@ -import hudson.model.* -import hudson.node_monitors.* -import hudson.slaves.* -import java.util.concurrent.* - -jenkins = Hudson.instance - -for (slave in jenkins.computers) { - def computer = slave.name - // filter for workers that are offline, are not processing anything, and have some description on why - // they are offline - if (slave.name != "master" & slave.isOffline() & slave.offlineCause != null & slave.countBusy() == 0) { - offCause = slave.getOfflineCause().toString() - // another filter for a specific offline reason. - if (offCause.contains("Time out for last 5 try")) { - println("Deleting " + slave.name + " which has the status: " + slave.offlineCause) - slave.doDoDelete() - } - } -} diff --git a/jenkins/admin-scripts/disable-old-workers.groovy b/jenkins/admin-scripts/disable-old-workers.groovy deleted file mode 100644 index 5a7297f1..00000000 --- a/jenkins/admin-scripts/disable-old-workers.groovy +++ /dev/null @@ -1,22 +0,0 @@ -// This script is built for the script console in Jenkins - -import hudson.model.* -import hudson.node_monitors.* -import hudson.slaves.* -import java.util.concurrent.* - -import hudson.model.*; - -jenkins = Hudson.instance - -for (slave in jenkins.computers) { - def computer = slave.name -// change the name string according to your needs - if (slave.name.contains("jenkins-worker") { - println(slave.name) -// the line below is commented out by default, but will set -// the given worker offline, and assign the string as its -// offline reason -// slave.cliOffline("old worker") -} -} diff --git a/jenkins/bokchoy_db_pull_request.py b/jenkins/bokchoy_db_pull_request.py deleted file mode 100644 index e4ce1639..00000000 --- a/jenkins/bokchoy_db_pull_request.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -This script is to be run inside a Jenkins job after updating bokchoy -db cache files through paver commands on edx-platform. If changes have -been made, this script will generate a PR into master with the updates. -""" -import logging -import os - -import click - -from .github_helpers import GitHubHelper - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel(logging.INFO) - -DB_CACHE_FILEPATH = 'common/test/db_cache' - -FINGERPRINT_FILE = 'bok_choy_migrations.sha1' -BOKCHOY_DB_FILES = [ - 'bok_choy_data_default.json', - 'bok_choy_data_student_module_history.json', - 'bok_choy_migrations_data_default.sql', - 'bok_choy_migrations_data_student_module_history.sql', - 'bok_choy_schema_default.sql', - 'bok_choy_schema_student_module_history.sql', - FINGERPRINT_FILE -] - -github_helper = GitHubHelper() - - -def _read_local_db_file_contents(repo_root, db_file): - """ - Read the contents of a file and return a string of the data. - """ - file_path = os.path.join(DB_CACHE_FILEPATH, db_file) - return github_helper.get_file_contents(repo_root, file_path) - - -# pylint: disable=missing-function-docstring -@click.command() -@click.option( - '--sha', - help="Sha of the merge commit to base the new PR off of", - required=True, -) -@click.option( - '--repo_root', - help="Path to local edx-platform repository that will " - "hold updated database files", - required=True, -) -def main(sha=None, repo_root=None): - logger.info("Authenticating with Github") - github_instance = github_helper.get_github_instance() - repository = github_helper.connect_to_repo(github_instance, "edx-platform") - - all_modified_files = github_helper.get_updated_files_list(repo_root) - bokchoy_db_files_full_path = [os.path.join(DB_CACHE_FILEPATH, db_file) for db_file in BOKCHOY_DB_FILES] - modified_files_list = [file for file in all_modified_files if file in bokchoy_db_files_full_path] - logger.info("modified db files: {}".format(modified_files_list)) - if len(modified_files_list) > 0: - fingerprint = _read_local_db_file_contents(repo_root, FINGERPRINT_FILE) - branch = "refs/heads/testeng/bokchoy_auto_cache_update_{}".format(fingerprint) - - if github_helper.branch_exists(repository, branch): - # If this branch already exists, then there's already a PR - # for this fingerprint. To avoid excessive PR's, exit. - logger.info("Branch name: {} already exists. Exiting.".format(branch)) - else: - user = github_instance.get_user() - commit_sha = github_helper.update_list_of_files( - repository, - repo_root, - modified_files_list, - "Updating Bokchoy testing database cache", - sha, - user.name - ) - github_helper.create_branch(repository, branch, commit_sha) - - logger.info("Checking if there's any old pull requests to delete") - deleted_pulls = github_helper.close_existing_pull_requests(repository, user.login, user.name) - - pr_body = "Bokchoy testing database update" - for num, deleted_pull_number in enumerate(deleted_pulls): - if num == 0: - pr_body += "\n\nDeleted obsolete pull_requests:" - pr_body += "\nhttps://github.com/edx/edx-platform/pull/{}".format(deleted_pull_number) - - logger.info("Creating a new pull request") - github_helper.create_pull_request( - repository, - 'Bokchoy Testing DB Cache update', - pr_body, - 'master', - branch, - team_reviewers=['arbi-bom'] - ) - else: - logger.info("No changes needed") - - -if __name__ == "__main__": - main() diff --git a/jenkins/build.py b/jenkins/build.py deleted file mode 100644 index ad33a5a8..00000000 --- a/jenkins/build.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -A class for working with the build info returned from the jenkins job API -""" -import logging - -logger = logging.getLogger(__name__) - - -class Build(dict): - """ - A class for working with the build info returned from the jenkins job API - - :Args: - build (dict): build data from Jenkins - """ - def __init__(self, build): - self.isbuilding = build.get('building') - - author = None - pr_id = None - actions = build.get('actions', []) - ghprb_class_name = 'org.jenkinsci.plugins.ghprb.GhprbParametersAction' - - for action in actions: - if action.get('_class') == ghprb_class_name: - action_parameters = action.get('parameters') - if action_parameters: - for p in action_parameters: - if p.get('name') == 'ghprbActualCommitAuthorEmail': - author = p.get('value') - if p.get('name') == 'ghprbPullId': - pr_id = p.get('value') - else: - logger.debug( - "Couldn't find build parameters for build #{}".format( - build.get('number') - ) - ) - - self.author = author - self.pr_id = pr_id diff --git a/jenkins/codecov_response_metrics.py b/jenkins/codecov_response_metrics.py deleted file mode 100644 index f8521e08..00000000 --- a/jenkins/codecov_response_metrics.py +++ /dev/null @@ -1,251 +0,0 @@ -""" -# Occasionally, pull requests never get updated with codecov results, which -# can create a lot of difficulty in getting the pr merged. However, this is -# purely anecdotal. - -# This script attempts to determine the extent of the issue. It scans a series -# of repositories that are currently reporting coverage metrics to codecov, -# and collects the length of time it took for codecov to post back a status context -# (if at all). - -# This script should be run periodically in order to get a good understanding -# of the state of codecov response times. -""" -import datetime -import json -import logging -import os -import sys - -from github import Github - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel(logging.INFO) -REPOS = [ - 'edx/bok-choy', - 'edx/completion', - 'edx/course-discovery', - 'edx/credentials', - 'edx/ecommerce', - 'edx/edx-analytics-dashboard', - 'edx/edx-analytics-pipeline', - 'edx/edx-app-android', - 'edx/edx-app-ios', - 'edx/edx-drf-extensions', - 'edx/edx-enterprise', - 'edx/edx-enterprise-data', - 'edx/edx-gomatic', - 'edx/edx-notes-api', - 'edx/edx-platform', - 'edx/edx-proctoring', - 'edx/edx-video-pipeline', - 'edx/edx-video-worker', - 'edx/studio-frontend', - 'edx/XBlock', - 'edx/xqueue', -] - - -def is_recent(activity_time, time_frame=3600): - """ - determine if a timestamp occurred between now (UTC) and time_frame - """ - activity_age = datetime.datetime.utcnow() - activity_time - return activity_age.total_seconds() < time_frame - - -def is_head_recent(pull_request, time_frame=3600): - """ - determine if the head commmit pull request has been pushed between now and - time_frame by seeing if any of the status contexts on the head commit were - posted within said time frame. This seems to be the only way to derive - this information, as the Github API does not serve information about when - a commit is pushed. - """ - try: - head_commit = get_head_commit(pull_request) - except IndexError: - logger.info('{} has no commits. Skipping'.format(pull_request.title)) - return False - statuses = head_commit.get_combined_status().statuses - return any( # pylint: disable=use-a-generator - [is_recent(dt, time_frame) for dt in [s.updated_at for s in statuses]] - ) - - -def get_recent_pull_requests(repo, time_frame=3600): - """ - given a repository, retrieve all pull requests that have received - updated status contexts within a given time frame - """ - recent_pull_requests = [] - for pr in repo.get_pulls(state="all", sort="updated", direction="desc"): - # since the pull requests are sorted by 'updated_at', once we reach - # pull requests that have not been updated in a week, stop searching - if not is_recent(pr.updated_at, 604800): - break - if is_head_recent(pr, time_frame): - recent_pull_requests.append(pr) - - logger.info("Found {} recent pull requests".format( - len(recent_pull_requests) - )) - return recent_pull_requests - - -def get_head_commit(pull_request): - return pull_request.get_commits().reversed[0] - - -def has_context_posted(context_name, statuses): - return context_name in [status.context for status in statuses] - - -def get_context_update_time(context, statuses): - return [status for status in statuses if status.context == context][0].updated_at.replace(microsecond=0) - - -def get_context_state(context, statuses): - return [status for status in statuses if status.context == context][0].state - - -def get_context_age(statuses, codecov_context, trigger_context): - """ - get the age of a given codecov context. This is done by computing - the difference between when the context that triggers codecov was - posted and when codecov results were posted, or, in the case that - they haven't yet, now. - """ - # get the context that should trigger the codecov context - trigger_context_update_time = get_context_update_time( - trigger_context, statuses - ) - - if has_context_posted(codecov_context, statuses): - codecov_context_update_time = get_context_update_time( - codecov_context, statuses - ) - context_age = codecov_context_update_time - trigger_context_update_time - logger.info("'{}' posted {} seconds after {} was posted".format( - codecov_context, context_age, trigger_context - )) - posted = True - else: - current_timestamp = datetime.datetime.utcnow().replace(microsecond=0) - context_age = current_timestamp - trigger_context_update_time - logger.info( - "'{}' has still not posted {} seconds after {} was posted".format( - codecov_context, context_age, trigger_context - ) - ) - posted = False - context_age_in_seconds = int(context_age.total_seconds()) - # occasionally, codecov can be posted back to the pull request before - # travis. This is the case in which a complex travis file runs different - # sharded tasks, one of which submits coverage data. This will result - # in a negative age for the codecov context. Treat these as 0, since - # their impact is not important. - context_age_in_seconds = max(context_age_in_seconds, 0) - return posted, context_age_in_seconds, trigger_context_update_time - - -def gather_codecov_metrics(all_repos, time_frame): - """ - scan all pertinent repos for metrics on how long it took for codecov - to report back following a 'triggering' context posting back to a - pull request. Return a list of JSON objects storing this data. - """ - # pylint: disable=logging-not-lazy - logger.info( - 'Gathering codecov response metrics on pull requests ' + - f'updated within the last {time_frame} seconds' - ) - - results = [] - - for repo in [r for r in all_repos if r.full_name in REPOS]: - - logger.info('Checking {} for recent PRs'.format(repo.full_name)) - prs = get_recent_pull_requests(repo, time_frame=time_frame) - # skip repos not updated within this 'time_frame' - if not prs: - logger.info( - 'No recent pull requests found in {}.'.format(repo.full_name) - ) - continue - - for pr in prs: - pr_title = str(pr.title) - logger.info('Analyzing pr {}'.format(pr_title)) - head_commit = get_head_commit(pr) - head_status = head_commit.get_combined_status().statuses - # mapping of status contexts that generate code coverage data - # and send it to codecov to the codecov status contexts for - # said data - context_map = { - 'continuous-integration/travis-ci/pr': 'codecov/patch', - 'continuous-integration/travis-ci/push': 'codecov/project', - 'jenkins/python': 'codecov/project' - } - for trigger_context, codecov_context in context_map.items(): - # skip prs that have not been posted to by their trigger status - if not has_context_posted(trigger_context, head_status): - logger.info( - "Context '{}' has not posted yet. Skipping".format( - trigger_context - ) - ) - continue - # skip prs in which the trigger context has failed. This means - # that coverage results have not been sent to codecov - if get_context_state(trigger_context, head_status) != 'success': - logger.info("Context '{}' failed. Skipping".format( - trigger_context - )) - continue - posted, context_age, trigger_posted_at = get_context_age( - head_status, codecov_context, trigger_context - ) - result = { - 'repo': repo.full_name, - 'pull_request': pr_title, - 'commit': head_commit.sha, - 'trigger_context_posted_at': str(trigger_posted_at), - 'codecov_received': posted, - 'codecov_received_after': context_age, - 'context': codecov_context - } - results.append(result) - return results - - -# pylint: disable=missing-function-docstring -def main(): - try: - token = os.environ.get('GITHUB_TOKEN') - except KeyError: - logger.error('No value set for GITHUB_TOKEN. Please try again') - sys.exit(1) - - gh = Github(token) - # Only consider pull requests created within this time frame (in seconds) - time_frame = os.environ.get('PULL_REQUEST_TIME_FRAME', 3600) - - all_repos = gh.get_user().get_repos() - results = gather_codecov_metrics(all_repos, time_frame) - - json_data = {'results': results} - outfile_name = 'codecov_metrics.json' - try: - logger.info('Writing results to {}'.format(outfile_name)) - with open(outfile_name, 'w', encoding='utf-8') as outfile: - json.dump(json_data, outfile, separators=(',', ':')) - outfile.write('\n') - except OSError: - logger.error('Unable to write data to {}'.format(outfile_name)) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/jenkins/edx_platform_test_notifier.py b/jenkins/edx_platform_test_notifier.py deleted file mode 100644 index 166adfb8..00000000 --- a/jenkins/edx_platform_test_notifier.py +++ /dev/null @@ -1,155 +0,0 @@ -# pylint: disable=missing-module-docstring -import logging -import sys - -import click -from github import Github - -from .github_helpers import GitHubHelper - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel(logging.INFO) - - -class EdxStatusBot: - """ - A status bot that can perform multiple actions on PRs. - - Looks for lines of the form '{botname}: {action}' in a PR's body to - determine what actions to take. - """ - - DEFAULT_BOT_NAME = 'edx-status-bot' - - # An ordered list of actions that this bot can take. - # - # Each action should correspond to a method name, and each - # action should have a corresponding `action`_marker method, - # e.g. 'ignore_marker', which tells whether the action should - # be taken. - ACTIONS = ('ignore', 'delete_old_comments', 'notify_tests_completed',) - - def __init__(self, token, name=DEFAULT_BOT_NAME): - self.name = name - self.token = token - self.github = Github(self.token) - - def act_on(self, pr): - for action in self.ACTIONS: - take_action = getattr(self, action + '_marker') - if take_action(pr): - getattr(self, action)(pr) - - def ignore(self, pr): - """Ignore taking any further actions on this PR.""" - logger.info( - "PR #{} author doesn't want status updates.".format(pr.number) - ) - sys.exit() - - def ignore_marker(self, pr): - return self._action_str('ignore') in str(pr.body) - - def delete_old_comments(self, pr): # pylint: disable=missing-function-docstring - comments = pr.get_issue_comments() - for comment in comments: - if comment.user.login == self.name: - logger.info( - "Old comment found on PR. Deleting" - ) - comment.delete() - - def delete_old_comments_marker(self, pr): # lint-amnesty, pylint: disable=unused-argument - return True - - def notify_tests_completed(self, pr): - """Post a notification on the PR that tests have finished running.""" - comment = self.generate_notification_message(pr) - try: - pr.create_issue_comment(comment) - except: # pylint: disable=bare-except - logger.error("Failed to add issue comment to PR.") - sys.exit(1) - else: - logger.info("Successfully commented on PR.") - - def get_head_commit(self, pr): - """ - Return the HEAD commit from a given pull request. Occasionally, a pull - request can have no commits (for instance, if it has been reset). In - this case, exit the script, as there is nothing else to do. - """ - commits = pr.get_commits() - if not list(commits): - logger.error("This pull request has no commits. Exiting.") - sys.exit() - head_commit = commits.reversed[0] - return head_commit - - def notify_tests_completed_marker(self, pr): # pylint: disable=missing-function-docstring - head_commit = self.get_head_commit(pr) - for status in head_commit.get_combined_status().statuses: - if status.state == 'pending': - logger.info( - "Other tests are still pending on this PR. Exiting" - ) - break - else: - return True - - def generate_notification_message(self, pr): # pylint: disable=missing-function-docstring - failing_contexts = self.get_failures(pr) - if not failing_contexts: - status_description = "There were no failures." - else: - status_description = "The following contexts failed:\n{}".format( - '\n'.join([f"* {f}" for f in failing_contexts]) - ) - comment = "Your PR has finished running tests. {}".format( - status_description - ) - return comment - - def get_failures(self, pr): - """ return a list of the contexts that recently failed on a given pr """ - head_commit = self.get_head_commit(pr) - return [status.context for status in head_commit.get_combined_status().statuses if status.state in ["failure", - "error"]] - - def _action_str(self, action): - return f'{self.name}: {action}' - - -@click.command() -@click.option( - '--repo', - help="Repository of pull request. Defaults to edx-platform", - default="edx-platform" -) -@click.option( - '--pr_number', - help="The PR number of a pull request. " - "This PR will receive a comment when its tests finish.", - required=True, -) -def main(repo=None, pr_number=None): - """ - Checks a pull request in Github to see if tests are finished. If they - are, it comments on the PR to notify the user. If not, the script exits. - """ - github_helper = GitHubHelper() - bot = EdxStatusBot(token=github_helper.get_github_token()) - repo = github_helper.connect_to_repo(bot.github, repo) - - try: - pr = repo.get_pull(int(pr_number)) - except: # pylint: disable=bare-except - logger.error("Invalid PR number given.") - sys.exit(1) - else: - bot.act_on(pr) - - -if __name__ == "__main__": - main() diff --git a/jenkins/helpers.py b/jenkins/helpers.py deleted file mode 100644 index 5318af10..00000000 --- a/jenkins/helpers.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Helpers for jenkins api -""" - - -def append_url(base, addition): - """ - Add something to a url, ensuring that there are the - right amount of `/`. - - :Args: - base: The original url. - addition: the thing to add to the end of the url - - :Returns: The combined url as a string of the form - `base/addition` - """ - if not base.endswith('/'): - base += '/' - if addition.startswith('/'): - addition = addition[1:] - return base + addition diff --git a/jenkins/job.py b/jenkins/job.py deleted file mode 100644 index 73bc3639..00000000 --- a/jenkins/job.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -A class to interact with a jenkins job API -""" -import logging - -import requests - -from .helpers import append_url - -logger = logging.getLogger(__name__) - - -class JenkinsJob: - - """ - A class for interacting with the jenkins job API - - :Args: - job_url: URL of jenkins job - username: jenkins username - token: jenkins api token - """ - - logging.basicConfig(format='[%(levelname)s] %(message)s') - logger = logging.getLogger(__name__) - logging.getLogger('requests').setLevel('ERROR') - - def __init__(self, job_url, username, token): - self.job_url = job_url - self.auth = (username, token) - - def get_json(self): - """ - Get build data for a given job_url. - - :Returns: - A python dict from the jenkins api response including: - * builds: a list of dicts, each containing: - ** building: Boolean of whether it is actively building - ** timestamp: the time the build started - ** number: the build id number - ** actions: a list of 'actions', from which the only - item used in this script is 'parameters' which can - be used to find the PR number. - """ - api_url = append_url(self.job_url, '/api/json') - - response = requests.get( - api_url, - params={ - "tree": ("builds[building,timestamp," - "number,actions[parameters[*]]]"), - }, - timeout=5, - ) - - response.raise_for_status() - return response.json() - - def update_build_desc(self, build_id, description): - """ - Updates build description. - - :Args: - build_id: id number of build to update - description: the new description - """ - build_url = append_url(self.job_url, str(build_id)) - url = append_url(build_url, "/submitDescription") - - response = requests.post( - url, - auth=self.auth, - params={ - 'description': description, - }, - timeout=5, - ) - - logger.info("Updating description for build #{}. Response: {}".format( - build_id, response.status_code)) - - response.raise_for_status() - return response.ok - - def stop_build(self, build_id): - """ - Stops a build. - - :Args: - build_id: id number of build to abort - """ - build_url = append_url(self.job_url, str(build_id)) - url = append_url(build_url, "/stop") - - response = requests.post(url, auth=self.auth, timeout=5) - - logger.info("Aborting build #{}. Response: {}".format( - build_id, response.status_code)) - - response.raise_for_status() - return response.ok diff --git a/jenkins/repo_health_reports.py b/jenkins/repo_health_reports.py deleted file mode 100644 index 7eacd160..00000000 --- a/jenkins/repo_health_reports.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Script to help create a PR with Repo health report. To be run inside -a Jenkins job that first runs the health checks on a target repo -(see pytest-repo-health and edx-repo-health repos) -""" -import logging - -import click -from github import GithubObject - -from .github_helpers import GitHubHelper - -logging.basicConfig() -LOGGER = logging.getLogger() -LOGGER.setLevel(logging.INFO) - - -@click.command() -@click.option( - '--sha', - help="Sha of the merge commit to base the new PR off of", - required=True, -) -@click.option( - '--repo_root', - help="Path to local repository to run repo health on.", - required=True, -) -@click.option( - '--repo_name', - help="Name of Repo in Github", - required=True, -) -@click.option( - '--org', - help="The github organization for the repository to run make upgrade on.", - required=True, -) -@click.option( - '--user_reviewers', - help="Comma seperated list of Github users to be tagged on pull requests", - default=None -) -@click.option( - '--team_reviewers', - help="Comma seperated list of Github teams to be tagged on pull requests", - default=None -) -def main(sha, repo_root, repo_name, org, user_reviewers, team_reviewers): - """ - Inspect the results of running ``make upgrade`` and create a PR with the - changes if appropriate. - """ - github_helper = GitHubHelper() - - LOGGER.info("Authenticating with Github") - github_instance = github_helper.get_github_instance() - LOGGER.info("Successfully Authenticated with Github") - LOGGER.info("Connecting to repo: {repo_name}".format(repo_name=repo_name)) - repository = github_helper.connect_to_repo(github_instance, repo_name) - LOGGER.info("Successfully connected to repo") - - modified_files_list = github_helper.get_updated_files_list(repo_root) - LOGGER.info("modified files: {}".format(modified_files_list)) - if modified_files_list: - branch = "refs/heads/jenkins/repo_health-{}".format(sha[:7]) - if github_helper.branch_exists(repository, branch): - LOGGER.info("Branch for this sha already exists") - else: - user = github_instance.get_user() - commit_sha = github_helper.update_list_of_files( - repository, - repo_root, - modified_files_list, - "Repo Health Report", - sha, - user.name - ) - github_helper.create_branch(repository, branch, commit_sha) - - LOGGER.info("Checking if there's any old pull requests to delete") - deleted_pulls = github_helper.close_existing_pull_requests(repository, user.login, user.name) - - pr_body = "Review Repo Health Report." - for num, deleted_pull_number in enumerate(deleted_pulls): - if num == 0: - pr_body += "\n\nDeleted obsolete pull_requests:" - pr_body += f"\nhttps://github.com/{org}/{repo_name}/pull/{deleted_pull_number}" - - LOGGER.info("Creating a new pull request") - - # If there are reviewers to be added, split them into python lists - if isinstance(user_reviewers, (str, str)) and user_reviewers: - user_reviewers = user_reviewers.split(',') - else: - user_reviewers = GithubObject.NotSet - - if isinstance(team_reviewers, (str, str)) and team_reviewers: - team_reviewers = team_reviewers.split(',') - else: - team_reviewers = GithubObject.NotSet - - pull_request = github_helper.create_pull_request( - repository, - 'Repo Health Report', - pr_body, - 'master', - branch, - user_reviewers=user_reviewers, - team_reviewers=team_reviewers - ) - LOGGER.info("Merging pull request") - pull_request.merge(commit_message="merging new repo data", merge_method="squash") - else: - LOGGER.info("No changes needed") - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/jenkins/tests/helpers.py b/jenkins/tests/helpers.py deleted file mode 100644 index a40b564e..00000000 --- a/jenkins/tests/helpers.py +++ /dev/null @@ -1,119 +0,0 @@ -# pylint: disable=missing-module-docstring,missing-function-docstring -import datetime -import functools -from unittest.mock import Mock - -from requests import Response - - -def mock_response(status_code, data=None): - response = Response() - response.status_code = status_code - response.json = Mock(return_value=data) - return response - - -def mock_utcnow(func): - class MockDatetime(datetime.datetime): - - @classmethod - def utcnow(cls): - return datetime.datetime.utcfromtimestamp(142009200.0) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - olddatetime = datetime.datetime - datetime.datetime = MockDatetime - ret = func(*args, **kwargs) - datetime.datetime = olddatetime - return ret - - return wrapper - - -class Pr: - - """ - Sample PR dict to use as test data - """ - - def __init__(self, prnum, author='foo@example.com'): - self.prnum = prnum - self.author = author - - @property - def dict(self): - """ - Return the PR object as a dict - """ - return self.__dict__ - - -def sample_data(running_builds, not_running_builds): - """ - Args: - running_builds: (list of dict) A list of dicts, one for each - PR with a running build. Each dict has key/value pairs for - PR number and commit author's email. For example, - [{'pr': '1', 'author': 'foo@example.com'}, {'pr': '1', - 'author': 'foo@example.com'}, {'pr': '2', 'author': - 'bar@example.com'}] indicates that there are currently - 2 builds running for PR #1 and 1 build running for PR #2. - (In this example the same author happened to push commits - twice for PR #1.) The last iterable for PR '1' in the - list will correlate to the currently relevant build. - We will use the array index of the item as the build number. - not_running_builds: (list of dict) A list of dicts for PRs that - have previously run builds. So that all the build numbers - are unique, we will use the length of running_builds plus - the array index of the item as the build number. - Returns: - Python dict of build data. This is in the format expected to - be returned by the jenkins api. - """ - builds = [] - - def mktimestamp(minutes_ago): - first_time = 142009200 * 1000 - build_time = first_time - (minutes_ago * 60000) - return build_time - - for i, build in enumerate(running_builds): - parameters = [ - {'name': 'ghprbPullId', 'value': build.get('prnum')}, - {'name': 'ghprbActualCommitAuthorEmail', 'value': build.get('author')} - ] - actions = [ - { - '_class': 'org.jenkinsci.plugins.ghprb.GhprbParametersAction', - 'parameters': parameters - }, {}, {} - ] - builds.append({ - 'actions': actions, - 'building': True, - 'number': i, - 'timestamp': mktimestamp(i) - }) - - for i, build in enumerate(not_running_builds): - num = i + len(running_builds) - parameters = [ - {'name': 'ghprbPullId', 'value': build.get('prnum')}, - {'name': 'ghprbActualCommitAuthorEmail', 'value': build.get('author')} - ] - actions = [ - { - '_class': 'org.jenkinsci.plugins.ghprb.GhprbParametersAction', - 'parameters': parameters - }, {}, {} - ] - builds.append({ - 'actions': actions, - 'building': False, - 'number': num, - 'timestamp': mktimestamp(num) - }) - - build_data = {'builds': builds} - return build_data diff --git a/jenkins/tests/test_bokchoy_db_pull_request.py b/jenkins/tests/test_bokchoy_db_pull_request.py deleted file mode 100644 index c661c8fb..00000000 --- a/jenkins/tests/test_bokchoy_db_pull_request.py +++ /dev/null @@ -1,115 +0,0 @@ -# pylint: disable=missing-module-docstring,unused-variable,unused-argument -from unittest import TestCase -from unittest.mock import Mock, patch - -from click.testing import CliRunner - -from jenkins.bokchoy_db_pull_request import main - - -class BokchoyPullRequestTestCase(TestCase): - """ - Test Case class for bokchoy_db_pull_request.py - """ - # Create the Cli runner to run the main function with click arguments - runner = CliRunner() - - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_github_instance', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.connect_to_repo', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_updated_files_list', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.branch_exists', - return_value=False) - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_branch', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.update_list_of_files', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_pull_request') - @patch('jenkins.github_helpers.GitHubHelper.delete_branch', - return_value=None) - def test_no_changes( - self, delete_branch_mock, create_pr_mock, create_branch_mock, update_files_mock, - branch_exists_mock, modified_list_mock, repo_mock, authenticate_mock): - """ - Ensure a merge with no changes to db files will not result in any updates. - """ - result = self.runner.invoke( - main, - args=['--sha=123', '--repo_root=../../edx-platform'] - ) - assert not create_branch_mock.called - assert not update_files_mock.called - assert not create_pr_mock.called - - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_github_instance', - return_value=Mock()) - @patch('jenkins.bokchoy_db_pull_request.github_helper.connect_to_repo', - return_value=Mock()) - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_updated_files_list', - return_value=[ - "common/test/db_cache/bok_choy_data_default.json", - "common/test/db_cache/bok_choy_schema_default.sql" - ]) - @patch('jenkins.bokchoy_db_pull_request.github_helper.branch_exists', - return_value=False) - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_branch', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request._read_local_db_file_contents', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.update_list_of_files', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.close_existing_pull_requests', - return_value=[]) - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_pull_request') - @patch('jenkins.github_helpers.GitHubHelper.delete_branch', - return_value=None) - def test_changes( - self, delete_branch_mock, create_pr_mock, close_pr_mock, update_file_mock, read_local_db_mock, - create_branch_mock, branch_exists_mock, modified_list_mock, repo_mock, authenticate_mock - ): - - """ - Ensure a merge with changes to db files will result in the proper updates, a new branch, and a PR. - """ - result = self.runner.invoke( - main, - args=['--sha=123', '--repo_root=../../edx-platform'] - ) - assert create_branch_mock.called - self.assertEqual(create_branch_mock.call_count, 1) - assert update_file_mock.called - self.assertEqual(update_file_mock.call_count, 1) - assert create_pr_mock.called - assert not delete_branch_mock.called - - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_github_instance', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.connect_to_repo', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.branch_exists', - return_value=True) - @patch('jenkins.bokchoy_db_pull_request.github_helper.get_updated_files_list', - return_value="common/test/db_cache/bok_choy_data_default.json\n" - "common/test/db_cache/bok_choy_schema_default.sql") - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_branch', - return_value=None) - @patch('jenkins.bokchoy_db_pull_request.github_helper.create_pull_request') - @patch('jenkins.github_helpers.GitHubHelper.delete_branch', - return_value=None) - def test_branch_exists( - self, delete_branch_mock, create_pr_mock, create_branch_mock, modified_list_mock, - get_branch_mock, repo_mock, authenticate_mock - ): - """ - If the branch for a given fingerprint already exists, make sure the script - doesn't try to create a new branch or create a PR. - """ - result = self.runner.invoke( - main, - args=['--sha=123', '--repo_root=../../edx-platform'] - ) - assert not create_branch_mock.called - assert not create_pr_mock.called - assert not delete_branch_mock.called diff --git a/jenkins/tests/test_build.py b/jenkins/tests/test_build.py deleted file mode 100644 index 4cea6a08..00000000 --- a/jenkins/tests/test_build.py +++ /dev/null @@ -1,30 +0,0 @@ -# pylint: disable=missing-module-docstring -from unittest import TestCase - -from jenkins.build import Build -from jenkins.tests.helpers import Pr, sample_data - - -class BuildTestCase(TestCase): - """ - TestCase class for testing the Build class - """ - - def setUp(self): - self.sample_build_data = sample_data( - [Pr('2', author='bar').dict], - [] - )['builds'][0] - - def test_init_build(self): - build = Build(self.sample_build_data) - self.assertEqual(build.author, 'bar') - self.assertEqual(build.pr_id, '2') - self.assertTrue(build.isbuilding) - - def test_init_build_with_missing_params(self): - self.sample_build_data['actions'][0] = {} - build = Build(self.sample_build_data) - self.assertIsNone(build.author) - self.assertIsNone(build.pr_id) - self.assertTrue(build.isbuilding) diff --git a/jenkins/tests/test_codecov_analysis.py b/jenkins/tests/test_codecov_analysis.py deleted file mode 100644 index 42b52dd8..00000000 --- a/jenkins/tests/test_codecov_analysis.py +++ /dev/null @@ -1,174 +0,0 @@ -# pylint: disable=missing-module-docstring,missing-class-docstring -import datetime -from unittest import TestCase - -from jenkins.codecov_response_metrics import (gather_codecov_metrics, - get_context_age, - get_recent_pull_requests) - - -class MockRepo: - - def __init__(self, full_name, prs): - self.full_name = full_name - self.prs = prs - - def get_pulls(self, state, sort, direction): # lint-amnesty, pylint: disable=unused-argument - return self.prs - - -class MockPR: - - def __init__(self, title, commits, age=10000): - self.title = title - self.commits = commits - self.updated_at = datetime.datetime.utcnow() - datetime.timedelta(seconds=age) - - def get_commits(self): - return self - - @property - def reversed(self): - return self.commits[::-1] - - -class MockCommit: - - def __init__(self, combined_status, sha=123): - self.combined_status = combined_status - self.sha = sha - - def get_combined_status(self): - return self.combined_status - - -class MockCombinedStatus: - - def __init__(self, statuses): - self.statuses = statuses - - -class MockStatus: - - def __init__(self, context, age, state='success'): - self.context = context - self.updated_at = datetime.datetime.utcnow() - datetime.timedelta(seconds=age) - self.state = state - - -class CodeCovTest(TestCase): - - def test_recent_pull_request(self): - - # pull request in which the HEAD commit has no 'recent' status contexts - mocked_old_combined_status = MockCombinedStatus( - [ - MockStatus('A', 1000), - MockStatus('B', 1000), - MockStatus('C', 2000) - ] - ) - mocked_old_commit = MockCommit(mocked_old_combined_status) - mocked_old_pr = MockPR('My PR', [None, None, mocked_old_commit]) - - # pull request in which at least one status context is 'recent' on - # the HEAD commit - mocked_new_combined_status = MockCombinedStatus( - [ - MockStatus('A', 1000), - MockStatus('B', 100), - MockStatus('C', 2000) - ] - ) - mocked_new_commit = MockCommit(mocked_new_combined_status) - mocked_new_pr = MockPR('Test Pr', [None, None, None, mocked_new_commit]) - - mocked_repo = MockRepo('mock/repo', [mocked_old_pr, mocked_new_pr]) - - recent_pull_requests = get_recent_pull_requests(mocked_repo, 500) - self.assertEqual(len(recent_pull_requests), 1) - self.assertEqual(recent_pull_requests[0].title, 'Test Pr') - - def test_context_age_calculation(self): - mocked_combined_status = MockCombinedStatus( - [ - MockStatus('A', 10), - MockStatus('B', 1000), - MockStatus('C', 500), - MockStatus('D', 100) - ] - ) - posted, context_age, _ = get_context_age( - mocked_combined_status.statuses, 'D', 'B' - ) - self.assertTrue(posted) - self.assertEqual(context_age, 900) - - def test_context_age_calculation_not_present(self): - mocked_combined_status = MockCombinedStatus( - [ - MockStatus('A', 10), - MockStatus('B', 100), - MockStatus('C', 500) - ] - ) - posted, context_age, _ = get_context_age( - mocked_combined_status.statuses, 'D', 'B' - ) - self.assertFalse(posted) - self.assertEqual(context_age, 100) - - def test_trigger_contexts_not_present(self): - mocked_combined_status_without_triggers = MockCombinedStatus( - [ - MockStatus('A', 10), - MockStatus('B', 1000), - MockStatus('C', 500), - MockStatus('D', 100) - ] - ) - mocked_combined_status_with_triggers = MockCombinedStatus( - [ - MockStatus('continuous-integration/travis-ci/pr', 2000), - MockStatus('continuous-integration/travis-ci/push', 1000), - MockStatus('codecov/patch', 500), - MockStatus('codecov/project', 100) - ] - ) - mocked_commit_1 = MockCommit( - mocked_combined_status_without_triggers, 1111 - ) - mocked_commit_2 = MockCommit( - mocked_combined_status_with_triggers, 2222 - ) - - mocked_prs = [ - MockPR('pr #1', [None, None, mocked_commit_1]), - MockPR('pr #2', [None, None, None, mocked_commit_2]) - ] - mocked_repos = [MockRepo('edx/ecommerce', mocked_prs)] - - metrics = gather_codecov_metrics(mocked_repos, 5000) - - now = datetime.datetime.utcnow().replace(microsecond=0) - expected_results = [ - { - 'repo': 'edx/ecommerce', - 'pull_request': 'pr #2', - 'commit': 2222, - 'trigger_context_posted_at': str(now - datetime.timedelta(seconds=2000)), - 'codecov_received': True, - 'codecov_received_after': 1500, - 'context': 'codecov/patch' - }, - { - 'repo': 'edx/ecommerce', - 'pull_request': 'pr #2', - 'commit': 2222, - 'trigger_context_posted_at': str(now - datetime.timedelta(seconds=1000)), - 'codecov_received': True, - 'codecov_received_after': 900, - 'context': 'codecov/project' - } - ] - self.assertCountEqual(metrics, expected_results) diff --git a/jenkins/tests/test_helpers.py b/jenkins/tests/test_helpers.py deleted file mode 100644 index 3689fec4..00000000 --- a/jenkins/tests/test_helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -# pylint: disable=missing-module-docstring -from unittest import TestCase - -from jenkins.helpers import append_url - - -# pylint: disable=missing-class-docstring -class HelpersTestCase(TestCase): - - def test_append_url(self): - expected = 'http://my_base_url.com/the_extra_part' - inputs = [ - ('http://my_base_url.com', 'the_extra_part'), - ('http://my_base_url.com', '/the_extra_part'), - ('http://my_base_url.com/', 'the_extra_part'), - ('http://my_base_url.com/', '/the_extra_part'), - ] - - for i in inputs: - returned = append_url(*i) - self.assertEqual( - expected, - returned, - msg="{e} != {r}\nInputs: {i}".format( - e=expected, r=returned, i=str(i) - ) - ) diff --git a/jenkins/tests/test_job.py b/jenkins/tests/test_job.py deleted file mode 100644 index 51062d2b..00000000 --- a/jenkins/tests/test_job.py +++ /dev/null @@ -1,53 +0,0 @@ -# pylint: disable=missing-module-docstring -from unittest import TestCase -from unittest.mock import patch - -from requests.exceptions import HTTPError - -from jenkins.job import JenkinsJob -from jenkins.tests.helpers import Pr, mock_response, sample_data - - -class JenkinsJobTestCase(TestCase): - - """ - TestCase class for testing deduper.py. - """ - - def setUp(self): - self.job_url = 'http://localhost:8080/fakejenkins' - self.user = 'ausername' - self.api_key = 'apikey' - self.job = JenkinsJob(self.job_url, self.user, self.api_key) - - def test_get_json_ok(self): - data = sample_data([Pr('1').dict], []) - with patch('requests.get', return_value=mock_response(200, data)): - response = self.job.get_json() - self.assertEqual(data, response) - - def test_get_json_bad_response(self): - with patch('requests.get', return_value=mock_response(400)): - with self.assertRaises(HTTPError): - self.job.get_json() - - def test_stop_build_ok(self): - with patch('requests.post', return_value=mock_response(200, '')): - response = self.job.stop_build('20') - self.assertTrue(response) - - def test_stop_build_bad_response(self): - with patch('requests.post', return_value=mock_response(400, '')): - with self.assertRaises(HTTPError): - self.job.stop_build('20') - - def test_update_desc_ok(self): - with patch('requests.post', return_value=mock_response(200, '')): - response = self.job.update_build_desc('20', 'new description') - self.assertTrue(response) - - def test_update_desc_bad_response(self): - with patch('requests.post', return_value=mock_response(400, '')): - with self.assertRaises(HTTPError): - self.job.update_build_desc( - '20', 'new description') diff --git a/jenkins/tests/test_timeout.py b/jenkins/tests/test_timeout.py deleted file mode 100644 index c522e63b..00000000 --- a/jenkins/tests/test_timeout.py +++ /dev/null @@ -1,115 +0,0 @@ -# pylint: disable=missing-module-docstring -from unittest import TestCase -from unittest.mock import call, patch - -from requests.exceptions import HTTPError - -from jenkins.job import JenkinsJob -from jenkins.tests.helpers import Pr, mock_utcnow, sample_data -from jenkins.timeout import BuildTimeout, timeout_main - - -class TimeoutTestCase(TestCase): - - """ - TestCase class for testing timeout.py. - """ - - def setUp(self): - self.job_url = 'http://localhost:8080/fakejenkins' - self.user = 'ausername' - self.api_key = 'apikey' - job = JenkinsJob(self.job_url, self.user, self.api_key) - self.timer = BuildTimeout(job, 2) - - @mock_utcnow - def test_get_stuck_builds_3_building(self): - data = sample_data( - [Pr('1').dict, Pr('2').dict, Pr('3').dict], [Pr('4').dict]) - builds = self.timer.get_stuck_builds(data) - self.assertEqual(len(builds), 1) - - @mock_utcnow - def test_get_stuck_builds_1_building(self): - data = sample_data( - [Pr('4').dict], [Pr('1').dict, Pr('2').dict, Pr('3').dict]) - builds = self.timer.get_stuck_builds(data) - self.assertEqual(len(builds), 0) - - @mock_utcnow - def test_get_stuck_builds_none_building(self): - data = sample_data( - [], [Pr('1').dict, Pr('2').dict, Pr('3').dict, Pr('4').dict]) - builds = self.timer.get_stuck_builds(data) - self.assertEqual(len(builds), 0) - - @mock_utcnow - def test_description(self): - expected = ("Build #1 automatically aborted because it has " - "exceeded the timeout of 3 minutes.") - returned = self.timer._aborted_description(3, 1) # pylint: disable=protected-access - self.assertEqual(expected, returned) - - @mock_utcnow - @patch('jenkins.job.JenkinsJob.stop_build', return_value=True) - @patch('jenkins.job.JenkinsJob.update_build_desc', return_value=True) - @patch('jenkins.timeout.BuildTimeout._aborted_description', - return_value='new description') - def test_stop_stuck_builds_with_stuck(self, mock_desc, - update_desc, - stop_build): - sample_build_data = sample_data( - [Pr('0').dict, Pr('1').dict, Pr('2').dict, Pr('3').dict], []) - build_data = self.timer.get_stuck_builds(sample_build_data) - self.timer.stop_stuck_builds(build_data) - - stop_build.assert_has_calls([call(3), call(2)], any_order=True) - - update_desc.assert_has_calls( - [call(2, mock_desc()), call(3, mock_desc())], - any_order=True - ) - - @mock_utcnow - @patch('jenkins.job.JenkinsJob.stop_build', side_effect=HTTPError()) - @patch('jenkins.job.JenkinsJob.update_build_desc', return_value=True) - def test_stop_stuck_builds_failed_to_stop(self, update_desc, stop_build): - sample_build_data = sample_data( - [Pr('1').dict, Pr('2').dict, Pr('3').dict], []) - build_data = self.timer.get_stuck_builds(sample_build_data) - self.timer.stop_stuck_builds(build_data) - stop_build.assert_called_once_with(2) - self.assertFalse(update_desc.called) - - @mock_utcnow - @patch('jenkins.job.JenkinsJob.stop_build', return_value=True) - @patch('jenkins.job.JenkinsJob.update_build_desc', return_value=True) - def test_stop_stuck_builds_none_stuck(self, update_desc, stop_build): - sample_build_data = sample_data( - [Pr('1').dict, Pr('2').dict], [Pr('2').dict]) - build_data = self.timer.get_stuck_builds(sample_build_data) - self.timer.stop_stuck_builds(build_data) - self.assertFalse(stop_build.called) - self.assertFalse(update_desc.called) - - @mock_utcnow - @patch('jenkins.job.JenkinsJob.get_json', return_value=sample_data( - [Pr('1').dict, Pr('2').dict, Pr('2').dict], - [Pr('4').dict, Pr('5').dict, Pr('5').dict])) - @patch('jenkins.job.JenkinsJob.stop_build', return_value=True) - @patch('jenkins.job.JenkinsJob.update_build_desc', return_value=True) - @patch('jenkins.timeout.BuildTimeout._aborted_description', - return_value='new description') - def test_main(self, mock_desc, update_desc, stop_build, get_json): - args = [ - '-t', self.api_key, - '-u', self.user, - '-j', self.job_url, - '--log-level', 'INFO', - '--timeout', '2', - ] - - timeout_main(args) - get_json.assert_called_once_with() - stop_build.assert_called_once_with(2) - update_desc.assert_called_once_with(2, mock_desc()) diff --git a/jenkins/timeout.py b/jenkins/timeout.py deleted file mode 100644 index ae4dab46..00000000 --- a/jenkins/timeout.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -This script is intended to be used to abort builds that are assumed to be -stuck because they have exceeded the expected max time. -""" -import argparse -import datetime -import logging -import sys - -from .job import JenkinsJob - -logger = logging.getLogger(__name__) - - -class BuildTimeout: - - """ - A class for programatically finding and aborting stuck builds. - - :Args: - job: An instance of jenkins_api.job.JenkinsJob - """ - - def __init__(self, job, timeout): - self.job = job - self.timeout = int(timeout) - - @staticmethod - def _aborted_description(timeout, build): - """ - :Args: - timeout: the timeout length in minutes - pr: the PR id - - :Returns: A description (string) - """ - return ("Build #{} automatically aborted because it has exceeded" - " the timeout of {} minutes.".format(build, timeout)) - - def abort_stuck_builds(self): - """ - Find running builds of the job at self.job_url. - If there are builds that have been running for longer than - the set timeout, abort them. It updates the build - description of aborted builds to indicate why they where - stopped. - """ - data = self.job.get_json() - builds = self.get_stuck_builds(data) - self.stop_stuck_builds(builds) - - def get_stuck_builds(self, data): - """ - Return build data for currently running builds. - - :Args: - data: the return value of self.get_json() - - :Returns: - build_data: a list of build numbers as strings - """ - long_running_builds = [] - now = datetime.datetime.utcnow() - - for build in data['builds']: - # Need to divide by 1000 to get time in seconds - start_time = datetime.datetime.utcfromtimestamp( - build['timestamp'] / 1000.0) - time_delta = now - start_time - min_since_start = time_delta.total_seconds() / 60.0 - - if build['building'] and min_since_start >= self.timeout: - long_running_builds.append(build['number']) - - return long_running_builds - - def stop_stuck_builds(self, build_nums): - """ - Finds PRs that are stuck and abort them. - - :Args: - build_data: the data returned by self.get_running_builds() - """ - - lines = [] - for b in build_nums: - lines.append(f"Build #{b} aborted due to timeout.") - desc = self._aborted_description(self.timeout, b) - - try: - self.job.stop_build(b) - self.job.update_build_desc(b, desc) - except Exception as e: # pylint: disable=broad-except - logger.error(e) - - if lines: - out = ("\n---------------------------------" - "\n** Stuck builds found. **" - "\n---------------------------------\n") - out += "\n".join(lines) - logger.info(out) - else: - logger.info("No stuck builds found.") - - -def timeout_main(raw_args): # pylint: disable=missing-function-docstring - # Get args - parser = argparse.ArgumentParser( - description="Programatically abort builds that have been running" - "longer than a specified time") - parser.add_argument('--token', '-t', dest='token', - help='jeknins api token', required=True) - parser.add_argument('--user', '-u', dest='username', - help='jenkins username', required=True) - parser.add_argument('--job', '-j', dest='job_url', - help='URL of jenkins job that uses the GHPRB plugin', - required=True) - parser.add_argument('--timeout', dest='timeout', - help='A time in minutes at which we should consider' - 'a build to be stuck', - required=True) - parser.add_argument('--log-level', dest='log_level', - default="INFO", help="set logging level") - args = parser.parse_args(raw_args) - - # Set logging level - logging.getLogger().setLevel(args.log_level.upper()) - - # Abort builds that exceed timeout - job = JenkinsJob(args.job_url, args.username, args.token) - timer = BuildTimeout(job, args.timeout) - timer.abort_stuck_builds() - - -if __name__ == '__main__': - logging.basicConfig(format='[%(levelname)s] %(message)s') - timeout_main(sys.argv[1:]) diff --git a/jenkins/toggle-spigot.py b/jenkins/toggle-spigot.py deleted file mode 100644 index cfcaf0d2..00000000 --- a/jenkins/toggle-spigot.py +++ /dev/null @@ -1,122 +0,0 @@ -# pylint: disable=missing-module-docstring,missing-function-docstring,bare-except -import logging -import sys -from time import sleep - -import boto3 -import click - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel(logging.INFO) - - -@click.command() -@click.option( - '--spigot_state', - help="Set the state of the spigot. " - "ON: The spigot will send both queued and " - "incoming webhooks to the target url(s). " - "OFF: The spigot will store incoming webhooks " - "in an SQS queue for future processing.", - required=True, - type=click.Choice(['ON', 'OFF']), -) -def main(spigot_state=None): - # Connect to AWS API and Cloudwatch Events - try: - api_client = boto3.client('apigateway') - cloudwatch_client = boto3.client('events') - except: - logger.error( - "Boto was unable to connect to apigateway " - "and/or cloudwatch events" - ) - sys.exit(1) - - # Get the api id and update the state of the spigot - api_id = _get_api_id(api_client) - _update_state( - api_client, - cloudwatch_client, - spigot_state, - api_id - ) - - logger.info( - "The spigot is now: {}".format(spigot_state) - ) - - -def _get_api_id(api_client): - """ - Find the restApiId of the API gateway. - """ - # Rather than hardcode the API's id, find the - # API by its name - api_list = api_client.get_rest_apis() - for api in api_list.get("items"): - if api.get("name") == "edx-tools-webhooks-processing": - api_id = api.get("id") - break - - if api_id: - return api_id - else: - logger.error( - "Could not find an api id for the " - "edx-tools-webhooks-processing API" - ) - sys.exit(1) - - -def _update_state(api_client, cloudwatch_client, spigot_state, api_id): - """ - Update the API stage variable to represent the new state, - and update the send_to_queue lambda trigger accordingly. - """ - update_variable_op = { - 'op': 'replace', - 'path': '/variables/spigot_state', - 'value': spigot_state - } - - if spigot_state == "ON": - api_client.update_stage( - restApiId=api_id, - stageName="prod", - patchOperations=[update_variable_op] - ) - # Sleep for one minute to ensure the API stage variable - # has updated and the dequeuer doesn't start too early. - sleep(60) - try: - cloudwatch_client.enable_rule( - Name="edx-spigot-send-from-queue" - ) - except: - logger.error( - "Could not enable the " - "edx-spigot-send-from-queue event trigger" - ) - sys.exit(1) - elif spigot_state == "OFF": - try: - cloudwatch_client.disable_rule( - Name="edx-spigot-send-from-queue" - ) - except: - logger.error( - "Could not disable the " - "edx-tools-webhooks-processing event trigger" - ) - sys.exit(1) - api_client.update_stage( - restApiId=api_id, - stageName="prod", - patchOperations=[update_variable_op] - ) - - -if __name__ == "__main__": - main() diff --git a/lambdas/process_webhooks/README.md b/lambdas/process_webhooks/README.md deleted file mode 100644 index 90b9639d..00000000 --- a/lambdas/process_webhooks/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# process_webhooks - -Process webhooks and either send them to the specified endpoint(s), or -store them in an SQS queue. - -## Deploying the code - -The s3 bucket must first exist. -To create one using the terraform from the edx-ops/terraform repo: -``` -terraform plan --target aws_s3_bucket.edx-testeng-spigot -terraform apply --target aws_s3_bucket.edx-testeng-spigot -``` - -To zip and upload a new version, using the aws cli: -``` -zip process_webhooks.zip process_webhooks.py -aws s3 cp process_webhooks.zip s3://edx-tools-spigot/ -rm process_webhooks.zip -``` diff --git a/lambdas/process_webhooks/__init__.py b/lambdas/process_webhooks/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lambdas/process_webhooks/process_webhooks.py b/lambdas/process_webhooks/process_webhooks.py deleted file mode 100644 index ff88fa06..00000000 --- a/lambdas/process_webhooks/process_webhooks.py +++ /dev/null @@ -1,205 +0,0 @@ -import json -import logging -import os - -import boto3 -from requests import post - -logger = logging.getLogger() - -# First log the function load message, then change -# the level to be configured via environment variable. -logger.setLevel(logging.INFO) -logger.info('Loading function') - -# Log level is set as a string, default to 'INFO' -log_level = os.environ.get('LOG_LEVEL', 'INFO').upper() -numeric_level = getattr(logging, log_level, None) -if not isinstance(numeric_level, int): - raise ValueError(f'Invalid log level: {log_level}') -logger.setLevel(numeric_level) - - -def _get_target_url(headers): - """ - Get the target URL for the processed hooks from the - OS environment variable. Based on the GitHub event, - add the proper endpoint. - - Return the target URL with the appropriate endpoint - """ - url = os.environ.get('TARGET_URL') - if not url: - raise Exception( - "Environment variable TARGET_URL was not set" - ) - - event_type = headers.get('X-GitHub-Event') - - # Based on the X-Github-Event header, determine the - # proper endpoint for the target url. - # PR's and Issue Comments use the ghprb Jenkins plugin - # Pushes use the Github Jenkins plugin - if event_type in ["issue_comment", "pull_request"]: - endpoint = "ghprbhook/" - elif event_type == "push": - endpoint = "github-webhook/" - elif event_type == "ping": - return None - else: - raise Exception( - "The Spigot does not support webhooks of " - "type: {}".format(event_type) - ) - - return url + "/" + endpoint - - -def _get_target_queue(): - """ - Get the target SQS name for the processed hooks from the - OS environment variable. - - Return the name of the queue - """ - queue_name = os.environ.get('TARGET_QUEUE') - if not queue_name: - raise Exception( - "Environment variable TARGET_QUEUE was not set" - ) - - return queue_name - - -def _add_gh_header(event, headers): - """ - Get the X-GitHub-Event header from the original request - data, add this to the headers, and return the results. - - Raise an error if the GitHub event header is not found. - """ - gh_headers = event.get('headers') - gh_event = gh_headers.get('X-GitHub-Event') - if not gh_event: - msg = f'X-GitHub-Event header was not found in {gh_headers}' - raise ValueError(msg) - - logger.debug(f'GitHub event was: {gh_event}') - headers['X-GitHub-Event'] = gh_event - return headers - - -def _is_from_queue(event): - """ - Check to see if this webhook is being sent from the SQS queue. - This is important to avoid duplicating the hook in the queue - in the event of a failure. - """ - return event.get('from_queue') == "True" - - -def _send_message(url, payload, headers): - """ Send the webhook to the endpoint via an HTTP POST. - Args: - url (str): Target URL for the POST request - payload (dict): Payload to send - headers (dict): Dictionary of headers to send - Returns: - The response from the HTTP POST - """ - response = post(url, json=payload, headers=headers, timeout=(3.05, 30)) - # Trigger the exception block for 4XX and 5XX responses - response.raise_for_status() - return response - - -def _send_to_queue(event, queue_name): - """ - Send the webhook to the SQS queue. - """ - try: - sqs = boto3.resource('sqs') - queue = sqs.get_queue_by_name(QueueName=queue_name) - except: - raise Exception("Unable to find the target queue") - - try: - response = queue.send_message(MessageBody=json.dumps(event)) - except: - raise Exception("The message could not be sent to queue") - - return response - - -def lambda_handler(event, _context): - # Determine if this message is coming from the queue - from_queue = _is_from_queue(event) - - # The header we send should include the original GitHub header, - # and also is set to send the data in the format that Jenkins expects. - header = {'Content-Type': 'application/json'} - - # Add the headers from the event - headers = _add_gh_header(event, header) - logger.debug(f"headers are: '{headers}'") - - # Get the state of the spigot from the api variable - spigot_state = event.get('spigot_state') - logger.info( - f"spigot_state is set to: {spigot_state}" - ) - - if spigot_state == "ON": - # Get the url that the webhook will be sent to - url = _get_target_url(headers) - - if not url: - # If url is None, swallow the hook, since it is just a ping - return ( - "Received a ping webhook. No action required." - ) - - # We had stored the payload to send in the - # 'body' node of the data object. - payload = event.get('body') - logger.debug(f"payload is: '{payload}'") - - # Send it off! - try: - _result = _send_message(url, payload, headers) - except: - if not from_queue: - # The transmission was a failure, if it's not - # already in the queue, add it. - queue_name = _get_target_queue() - _response = _send_to_queue(event, queue_name) - raise Exception( - "There was an error sending the message " - "to the url: {}".format(url) - ) - return ( - f"Webhook successfully sent to url: {url}" - ) - elif spigot_state == "OFF": - # Since the spigot is off, send the event - # to SQS for future processing. However, - # if the message is already in the queue do - # nothing. - if from_queue: - raise Exception( - "The spigot is OFF. No messages should be " - "sent from the queue." - ) - else: - queue_name = _get_target_queue() - _response = _send_to_queue(event, queue_name) - - return ( - f"Webhook successfully sent to queue: {queue_name}" - ) - else: - raise Exception( - "API Gateway stage variable spigot_state " - "was not correctly set. Should be ON or OFF, " - "was: {}".format(spigot_state) - ) diff --git a/lambdas/process_webhooks/test/__init__.py b/lambdas/process_webhooks/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lambdas/process_webhooks/test/test_process_webhooks.py b/lambdas/process_webhooks/test/test_process_webhooks.py deleted file mode 100644 index c6ed93a3..00000000 --- a/lambdas/process_webhooks/test/test_process_webhooks.py +++ /dev/null @@ -1,201 +0,0 @@ -import os -from unittest import TestCase -from unittest.mock import Mock, patch - -from requests import Response - -from ..process_webhooks import (_add_gh_header, _get_target_queue, - _get_target_url, _is_from_queue, _send_message, - lambda_handler) - - -class ProcessWebhooksTestCase(TestCase): - headers = { - 'Content-Type': 'application/json', - 'X-GitHub-Event': '' - } - - @patch.dict(os.environ, {'TARGET_URL': 'http://www.example.com'}) - def test_get_target_url_pr(self): - self.headers['X-GitHub-Event'] = 'pull_request' - url = _get_target_url(self.headers) - self.assertEqual(url, 'http://www.example.com/ghprbhook/') - - @patch.dict(os.environ, {'TARGET_URL': 'http://www.example.com'}) - def test_get_target_url_comment(self): - self.headers['X-GitHub-Event'] = 'issue_comment' - url = _get_target_url(self.headers) - self.assertEqual(url, 'http://www.example.com/ghprbhook/') - - @patch.dict(os.environ, {'TARGET_URL': 'http://www.example.com'}) - def test_get_target_url_push(self): - self.headers['X-GitHub-Event'] = 'push' - url = _get_target_url(self.headers) - self.assertEqual(url, 'http://www.example.com/github-webhook/') - - @patch.dict(os.environ, {'TARGET_URL': 'http://www.example.com'}) - def test_get_target_url_ping(self): - self.headers['X-GitHub-Event'] = 'ping' - url = _get_target_url(self.headers) - self.assertEqual(url, None) - - @patch.dict(os.environ, {'TARGET_URL': 'http://www.example.com'}) - def test_get_target_url_error(self): - self.headers['X-GitHub-Event'] = 'status' - with self.assertRaises(Exception): - url = _get_target_url(self.headers) - - def test_add_gh_header(self): - gh_header = {'X-GitHub-Event': 'push'} - test_data = {'headers': gh_header} - headers = _add_gh_header(test_data, {}) - self.assertEqual(headers, gh_header) - - def test_add_gh_header_exception(self): - gh_header = {} - test_data = {'headers': gh_header} - with self.assertRaises(ValueError): - _add_gh_header(test_data, {}) - - @patch.dict(os.environ, {'TARGET_QUEUE': 'queue_name'}) - def test_get_target_queue(self): - queue = _get_target_queue() - self.assertEqual(queue, 'queue_name') - - def test_is_from_queue_true(self): - event = { - 'from_queue': 'True' - } - from_queue = _is_from_queue(event) - self.assertEqual(from_queue, True) - - def test_is_from_queue_false(self): - event = { - 'from_queue': 'False' - } - from_queue = _is_from_queue(event) - self.assertEqual(from_queue, False) - - -class ProcessWebhooksRequestTestCase(TestCase): - @staticmethod - def mock_response(status_code): - response = Response() - response.status_code = status_code - return response - - def test_send_message_success(self): - with patch( - 'process_webhooks.process_webhooks.post', - return_value=self.mock_response(200) - ): - response = _send_message('http://www.example.com', None, None) - self.assertIsNotNone(response) - self.assertEqual(response.status_code, 200) - - def test_send_message_error(self): - with patch( - 'process_webhooks.process_webhooks.post', - return_value=self.mock_response(500) - ): - with self.assertRaises(Exception): - response = _send_message('http://www.example.com', None, None) - self.assertEqual(response.message, '500 Server Error: None') - - -class LambdaHandlerTestCase(TestCase): - event = { - 'spigot_state': '', - 'body': { - 'zen': 'Non-blocking is better than blocking.', - 'hook_id': 12341234, - 'hook': { - 'type': 'Repository', - 'id': 98765432, - 'events': ['issue_comment', 'pull_request'] - }, - 'repository': {'id': 12341234, 'name': 'foo'}, - 'sender': {'id': 12345678}, - }, - 'headers': {'X-GitHub-Event': 'ping'} - } - - @patch('process_webhooks.process_webhooks._get_target_url', - return_value='http://www.example.com/endpoint/') - @patch('process_webhooks.process_webhooks._send_message', - return_value={}) - def test_lambda_handler_to_target(self, send_msg_mock, _url_mock): - self.event['spigot_state'] = 'ON' - lambda_handler(self.event, None) - send_msg_mock.assert_called_with( - 'http://www.example.com/endpoint/', - self.event.get('body'), - {'Content-Type': 'application/json', 'X-GitHub-Event': 'ping'} - ) - - @patch('process_webhooks.process_webhooks._get_target_url', - return_value='http://www.example.com/endpoint/') - @patch('process_webhooks.process_webhooks._send_message', - side_effect=Exception("Error!")) - @patch('process_webhooks.process_webhooks._is_from_queue', - return_value=False) - @patch('process_webhooks.process_webhooks._get_target_queue', - return_value='queue_name') - @patch('process_webhooks.process_webhooks._send_to_queue', - return_value={}) - def test_lambda_handler_to_target_error( - self, send_queue_mock, _queue_mock, - _from_queue_mock, send_msg_mock, _url_mock - ): - self.event['spigot_state'] = 'ON' - with self.assertRaises(Exception): - lambda_handler(self.event, None) - - send_msg_mock.assert_called_with( - 'http://www.example.com/endpoint/', - self.event.get('body'), - {'Content-Type': 'application/json', 'X-GitHub-Event': 'ping'} - ) - send_queue_mock.assert_called_with( - self.event, - 'queue_name' - ) - - @patch('process_webhooks.process_webhooks._get_target_url', - return_value=None) - @patch('process_webhooks.process_webhooks._send_message', - return_value={}) - def test_lambda_handler_ping(self, send_msg_mock, _url_mock): - self.event['spigot_state'] = 'ON' - lambda_handler(self.event, None) - assert not send_msg_mock.called - - @patch('process_webhooks.process_webhooks._get_target_queue', - return_value='queue_name') - @patch('process_webhooks.process_webhooks._is_from_queue', - return_value=False) - @patch('process_webhooks.process_webhooks._send_to_queue', - return_value={}) - def test_lambda_handler_to_queue( - self, send_queue_mock, _from_queue_mock, _queue_mock - ): - self.event['spigot_state'] = 'OFF' - lambda_handler(self.event, None) - send_queue_mock.assert_called_with( - self.event, - 'queue_name' - ) - - @patch('process_webhooks.process_webhooks._get_target_queue', - return_value='queue_name') - @patch('process_webhooks.process_webhooks._is_from_queue', - return_value=True) - @patch('process_webhooks.process_webhooks._send_to_queue', - return_value={}) - def test_lambda_handler_to_queue_from_queue( - self, send_queue_mock, _from_queue_mock, _queue_mock - ): - self.event['spigot_state'] = 'OFF' - with self.assertRaises(Exception): - lambda_handler(self.event, None) - assert not send_queue_mock.called diff --git a/lambdas/restart_jenkins/README.md b/lambdas/restart_jenkins/README.md deleted file mode 100644 index af839239..00000000 --- a/lambdas/restart_jenkins/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# jenkins_restart - -Post to the safeRestart URL to restart the Jenkins application - -## Deploying the code - -The s3 bucket must first exist. -To create one using the terraform from the edx-ops/terraform repo: -``` -terraform plan --target aws_s3_bucket.edx-tools-jenkins-restart -terraform apply --target aws_s3_bucket.edx-tools-jenkins-restart -``` - -To zip and upload a new version, using the aws cli: -``` -zip restart_jenkins.zip restart_jenkins.py -aws s3 cp restart_jenkins.zip s3://edx-tools-jenkins-restart/ -rm restart_jenkins.zip -``` - -To upload credentials to the credentials bucket: -``` -aws s3 cp jenkins_safe_restart_credentials.json s3://edx-tools-credentials/ -``` diff --git a/lambdas/restart_jenkins/__init__.py b/lambdas/restart_jenkins/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lambdas/restart_jenkins/restart_jenkins.py b/lambdas/restart_jenkins/restart_jenkins.py deleted file mode 100644 index d81a0cb4..00000000 --- a/lambdas/restart_jenkins/restart_jenkins.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import logging -import os - -import botocore.session -from requests import get, post - -CREDENTIALS_BUCKET = "edx-tools-credentials" -CREDENTIALS_FILE = "jenkins_safe_restart_credentials.json" - -logger = logging.getLogger() - -# First log the function load message, then change -# the level to be configured via environment variable. -logger.setLevel(logging.INFO) -logger.info('Loading function') - -# Log level is set as a string, default to 'INFO' -log_level = os.environ.get('LOG_LEVEL', 'INFO').upper() -numeric_level = getattr(logging, log_level, None) -if not isinstance(numeric_level, int): - raise ValueError(f'Invalid log level: {log_level}') -logger.setLevel(numeric_level) - - -def _get_base_url(): - """Get the base URL from the OS environment variable. """ - url = os.environ.get('BASE_URL') - if not url: - raise Exception( - "Environment variable BASE_URL was not set" - ) - # Let the URL be specified with or without a / at the end - return url.rstrip('/') - - -def _get_credentials_from_s3(): - """ - Get jenkins credentials from s3 bucket. - The expected object is a JSON file formatted as: - { - "username": "sampleusername", - "api_token": "sampletoken" - } - """ - session = botocore.session.get_session() - client = session.create_client('s3') - - creds_file = client.get_object( - Bucket=CREDENTIALS_BUCKET, - Key=CREDENTIALS_FILE - ) - creds = json.loads(creds_file['Body'].read()) - - if not creds.get('username') or not creds.get('api_token'): - raise Exception( - 'Credentials file needs both a ' - 'username and api_token attribute' - ) - return (creds['username'], creds['api_token']) - - -def lambda_handler(_event, _context): - jenkins_url = _get_base_url() - auth = _get_credentials_from_s3() - headers = None - - # If CSRF is enabled, you need to get a crumb - # to send in the header of your POST request. - response = get( - f'{jenkins_url}/crumbIssuer/api/json', - auth=auth, - timeout=(3.05, 30) - ) - - # You will get a 404 if CSRF is not enabled, - # in which case you don't need to do anything. - # So only take action if you get a 200. - if response.status_code == 200: - crumb = response.json() - crumb_value = crumb.get('crumb') - crumb_field = crumb.get('crumbRequestField') - headers = {crumb_field: crumb_value} - - response = post( - f'{jenkins_url}/safeRestart', - auth=auth, - headers=headers, - timeout=(3.05, 30) - ) - - # Safe Restart will put the user back at the root. - # If no jobs are running it will restart - # immediately and respond with a 503. - # So we need to raise an error for other 4XX and 5XX - # responses, but not that one. - if response.status_code != 503: - response.raise_for_status() - - -if __name__ == "__main__": - lambda_handler(None, None) diff --git a/lambdas/send_from_queue/README.md b/lambdas/send_from_queue/README.md deleted file mode 100644 index 569872ea..00000000 --- a/lambdas/send_from_queue/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# send_from_queue - -Remove webhooks from the SQS queue and forward them back through the API Gateway. - -## Deploying the code - -The s3 bucket must first exist. -To create one using the terraform from the edx-ops/terraform repo: -``` -terraform plan --target aws_s3_bucket.edx-testeng-spigot -terraform apply --target aws_s3_bucket.edx-testeng-spigot -``` - -To zip and upload a new version, using the aws cli: -``` -zip send_from_queue.zip send_from_queue.py -aws s3 cp send_from_queue.zip s3://edx-tools-spigot/ -rm send_from_queue.zip -``` diff --git a/lambdas/send_from_queue/__init__.py b/lambdas/send_from_queue/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lambdas/send_from_queue/send_from_queue.py b/lambdas/send_from_queue/send_from_queue.py deleted file mode 100644 index eec4dced..00000000 --- a/lambdas/send_from_queue/send_from_queue.py +++ /dev/null @@ -1,226 +0,0 @@ -import json -import logging -import os -import sys - -import boto3 -from requests import post - -logger = logging.getLogger() - -# First log the function load message, then change -# the level to be configured via environment variable. -logger.setLevel(logging.INFO) -logger.info('Loading function') - -# Log level is set as a string, default to 'INFO' -log_level = os.environ.get('LOG_LEVEL', 'INFO').upper() -numeric_level = getattr(logging, log_level, None) -if not isinstance(numeric_level, int): - raise ValueError(f'Invalid log level: {log_level}') -logger.setLevel(numeric_level) - - -def _get_target_queue(): - """ - Get the target SQS name for the processed hooks from the - OS environment variable. - - Return the name of the queue - """ - queue_name = os.environ.get('TARGET_QUEUE') - if not queue_name: - raise Exception( - "Environment variable TARGET_QUEUE was not set" - ) - - return queue_name - - -def _get_queue_object(queue_name): - """ - Connect with SQS via boto and return an object representing - the queue. - """ - try: - sqs_resource = boto3.resource('sqs') - queue_object = sqs_resource.get_queue_by_name(QueueName=queue_name) - except: - raise Exception( - "Unable to connect to the SQS queue" - ) - - return queue_object - - -def _get_api_url(): - """ - Find the url of the API Gateway by getting its id from boto - - Return the url - """ - try: - api_client = boto3.client('apigateway') - api_list = api_client.get_rest_apis() - except: - logger.error( - "Unable to connect to the apigateway" - ) - - for api in api_list.get("items"): - if api.get("name") == "edx-tools-webhooks-processing": - api_id = api.get("id") - break - - if api_id: - # Create the url based on the api id - api_url = ( - "https://{}.execute-api.us-east-1.amazonaws.com" - "/prod/webhooks" - ).format(api_id) - - return api_url - else: - logger.error( - "Could not find an api id for the " - "edx-tools-webhooks-processing API" - ) - sys.exit(1) - - -def _is_queue_empty(queue_name): - """ - Determine whether or not the sqs queue is empty. - """ - try: - sqs_client = boto3.client('sqs') - queue_url_response = sqs_client.get_queue_url( - QueueName=queue_name - ) - queue_url = queue_url_response['QueueUrl'] - except: - raise Exception( - "Unable to get the queue url" - ) - - try: - response = sqs_client.get_queue_attributes( - QueueUrl=queue_url, - AttributeNames=['ApproximateNumberOfMessages'] - ) - # Convert to an integer - num_messages = int( - response['Attributes']['ApproximateNumberOfMessages'] - ) - except: - raise Exception( - "Unable to get ApproximateNumberOfMessages from queue" - ) - - if num_messages == 0: - return True - else: - return False - - -def _get_from_queue(queue_object): - """ - Get webhooks from the SQS queue for processing. - """ - try: - # Get up to 10 messages from the SQS queue. - # 10 is the maximum allowed by this boto method. - # If there are fewer than 10 messages on the queue, - # it will return however many exist. - message_list = queue_object.receive_messages( - MaxNumberOfMessages=10, - WaitTimeSeconds=3 - ) - - return message_list - except: - raise Exception( - "Unable to get messages from the queue" - ) - - -def _delete_from_queue(queue_object, message): - """ - Delete a webhook from the SQS queue. - """ - try: - msg_receipt = message.receipt_handle - msg_id = message.message_id - entry = {'Id': msg_id, 'ReceiptHandle': msg_receipt} - except: - raise Exception( - 'Unable to get necessary message attributes ' - 'for deletion. message_id and ' - 'ReceiptHandle are required' - ) - - try: - response = queue_object.delete_messages(Entries=[entry]) - except: - raise Exception( - f'Unable to delete message {msg_id} from queue' - ) - - -def lambda_handler(event, _context): - # Get the queue name from the env variable and get - # a queue object representing it from boto. - queue_name = _get_target_queue() - queue_object = _get_queue_object(queue_name) - - if _is_queue_empty(queue_name): - empty_msg = "No visible messages in the queue to clear" - logger.debug(empty_msg) - return empty_msg - - # Rather than hardcoding the api url, get it from boto. - # Add the query param so the process_webhooks lambda - # knows the webhook is coming from the queue. - api_url_query = _get_api_url() + "?from_queue=True" - - # The SQS queue is draining - logger.info('Attempting to drain the queue.') - - # Process hooks from the SQS queue until it is empty. If the queue is - # large, this may mean processing items until the lambda times out. - while not _is_queue_empty(queue_name): - # Get messages from the sqs queue - logger.info('Fetching messages from the queue.') - messages = _get_from_queue(queue_object) - - for message in messages: - message_body = json.loads(message.body) - - payload = message_body.get("body") - headers = message_body.get("headers") - - if not payload or not headers: - # The hook is missing either the body or - # the headers. Throw an error - raise Exception( - "Unable to parse the body and headers " - "from the following webhook in the " - "queue. {}".format(message_body) - ) - - # Send the SQS message to the API Gateway - response = post( - api_url_query, - json=payload, - headers=headers, - timeout=(3.05, 30) - ) - - # If there was a problem, raise an error - response.raise_for_status() - - # Otherwise, delete the message since it has been processed - _delete_from_queue(queue_object, message) - - # If this gets reached before a timeout, the queue had been emptied - return "The queue has been cleared." diff --git a/lambdas/send_from_queue/test/__init__.py b/lambdas/send_from_queue/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lambdas/send_from_queue/test/test_send_from_queue.py b/lambdas/send_from_queue/test/test_send_from_queue.py deleted file mode 100644 index 6e6d55dd..00000000 --- a/lambdas/send_from_queue/test/test_send_from_queue.py +++ /dev/null @@ -1,106 +0,0 @@ -import json -import os -from unittest import TestCase -from unittest.mock import Mock, patch - -from requests import Response - -from ..send_from_queue import (_delete_from_queue, _get_api_url, - _get_target_queue, lambda_handler) - - -class SendFromQueueTestCase(TestCase): - @patch.dict(os.environ, {'TARGET_QUEUE': 'queue_name'}) - def test_get_target_queue(self): - queue = _get_target_queue() - self.assertEqual(queue, 'queue_name') - - -class LambdaHandlerTestCase(TestCase): - @staticmethod - def mock_response(status_code): - response = Response() - response.status_code = status_code - return response - - # Create mock class to represent a queue message - class mock_message(): - body = json.dumps( - { - 'body': { - 'zen': 'Non-blocking is better than blocking.', - 'hook_id': 12341234, - 'hook': { - 'type': 'Repository', - 'id': 98765432, - 'events': ['issue_comment', 'pull_request'] - }, - 'repository': {'id': 12341234, 'name': 'foo'}, - 'sender': {'id': 12345678} - }, - 'headers': {'X-GitHub-Event': 'ping'} - } - ) - - @patch('send_from_queue.send_from_queue._get_target_queue', - return_value='queue_name') - @patch('send_from_queue.send_from_queue._get_queue_object', - return_value={}) - @patch('send_from_queue.send_from_queue._is_queue_empty', - return_value=True) - def test_lambda_handler_empty_queue( - self, _queue_empty_mock, _queue_object, _queue_mock - ): - response = lambda_handler(None, None) - self.assertEqual(response, "No visible messages in the queue to clear") - - @patch('send_from_queue.send_from_queue._get_target_queue', - return_value='queue_name') - @patch('send_from_queue.send_from_queue._get_queue_object', - return_value={}) - @patch('send_from_queue.send_from_queue._get_api_url', - return_value='https://api.com') - @patch('send_from_queue.send_from_queue._is_queue_empty', - side_effect=[False, False, True]) - @patch('send_from_queue.send_from_queue._get_from_queue', - return_value=[mock_message()]) - @patch('send_from_queue.send_from_queue._delete_from_queue', - return_value={}) - def test_lambda_handler_with_queue( - self, delete_mock, msg_from_queue_mock, - queue_empty_mock, _api_mock, _queue_object, _queue_mock - ): - with patch( - 'send_from_queue.send_from_queue.post', - return_value=self.mock_response(200) - ): - lambda_handler(None, None) - self.assertEqual(queue_empty_mock.call_count, 3) - self.assertEqual(msg_from_queue_mock.call_count, 1) - self.assertEqual(delete_mock.call_count, 1) - - @patch('send_from_queue.send_from_queue._get_target_queue', - return_value='queue_name') - @patch('send_from_queue.send_from_queue._get_queue_object', - return_value={}) - @patch('send_from_queue.send_from_queue._get_api_url', - return_value='https://api.com') - @patch('send_from_queue.send_from_queue._is_queue_empty', - side_effect=[False, False, True]) - @patch('send_from_queue.send_from_queue._get_from_queue', - return_value=[mock_message()]) - @patch('send_from_queue.send_from_queue._delete_from_queue', - return_value={}) - def test_lambda_handler_with_queue_error( - self, _delete_mock, msg_from_queue_mock, - queue_empty_mock, _api_mock, _queue_object, _queue_mock - ): - with patch( - 'send_from_queue.send_from_queue.post', - return_value=self.mock_response(500) - ): - with self.assertRaises(Exception): - lambda_handler(None, None) - self.assertEqual(queue_empty_mock.call_count, 2) - self.assertEqual(msg_from_queue_mock.call_count, 1) - assert not _delete_mock.called diff --git a/requirements/aws.in b/requirements/aws.in deleted file mode 100644 index dc5c8930..00000000 --- a/requirements/aws.in +++ /dev/null @@ -1,3 +0,0 @@ --c constraints.txt - -boto3 # Used in deregister_amis diff --git a/requirements/aws.txt b/requirements/aws.txt deleted file mode 100644 index c311421c..00000000 --- a/requirements/aws.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# make upgrade -# -boto3==1.26.126 - # via -r requirements/aws.in -botocore==1.29.126 - # via - # boto3 - # s3transfer -jmespath==1.0.1 - # via - # boto3 - # botocore -python-dateutil==2.8.2 - # via botocore -s3transfer==0.6.0 - # via boto3 -six==1.16.0 - # via python-dateutil -urllib3==1.26.15 - # via botocore diff --git a/requirements/base.in b/requirements/base.in index 1ddf9858..4278f101 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,7 +1,5 @@ -c constraints.txt --r aws.txt - GitPython PyGithub packaging # used in create pull request script to compare package versions diff --git a/requirements/base.txt b/requirements/base.txt index 77896aa5..d9d682e6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,14 +4,7 @@ # # make upgrade # -boto3==1.26.126 - # via -r requirements/aws.txt -botocore==1.29.126 - # via - # -r requirements/aws.txt - # boto3 - # s3transfer -certifi==2022.12.7 +certifi==2023.5.7 # via requests cffi==1.15.1 # via @@ -31,11 +24,6 @@ gitpython==3.1.31 # via -r requirements/base.in idna==3.4 # via requests -jmespath==1.0.1 - # via - # -r requirements/aws.txt - # boto3 - # botocore packaging==23.1 # via -r requirements/base.in pycparser==2.21 @@ -46,28 +34,13 @@ pyjwt[crypto]==2.6.0 # via pygithub pynacl==1.5.0 # via pygithub -python-dateutil==2.8.2 - # via - # -r requirements/aws.txt - # botocore requests==2.30.0 # via # -r requirements/base.in # pygithub -s3transfer==0.6.0 - # via - # -r requirements/aws.txt - # boto3 -six==1.16.0 - # via - # -r requirements/aws.txt - # python-dateutil smmap==5.0.0 # via gitdb -urllib3==1.26.15 - # via - # -r requirements/aws.txt - # botocore - # requests +urllib3==2.0.2 + # via requests wrapt==1.15.0 # via deprecated diff --git a/requirements/dev.txt b/requirements/dev.txt index 0af2b645..28b024dd 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,18 +9,11 @@ astroid==2.15.4 # -r requirements/testing.txt # pylint # pylint-celery -boto3==1.26.126 - # via -r requirements/testing.txt -botocore==1.29.126 - # via - # -r requirements/testing.txt - # boto3 - # s3transfer build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2022.12.7 +certifi==2023.5.7 # via # -r requirements/testing.txt # requests @@ -57,8 +50,6 @@ cryptography==40.0.2 # via # -r requirements/testing.txt # pyjwt -ddt==1.6.0 - # via -r requirements/testing.txt deprecated==1.2.13 # via # -r requirements/testing.txt @@ -88,8 +79,6 @@ gitdb==4.0.10 # gitpython gitpython==3.1.31 # via -r requirements/testing.txt -httpretty==1.1.4 - # via -r requirements/testing.txt idna==3.4 # via # -r requirements/testing.txt @@ -106,11 +95,6 @@ jinja2==3.1.2 # via # -r requirements/testing.txt # code-annotations -jmespath==1.0.1 - # via - # -r requirements/testing.txt - # boto3 - # botocore lazy-object-proxy==1.9.0 # via # -r requirements/testing.txt @@ -167,7 +151,7 @@ pyjwt[crypto]==2.6.0 # via # -r requirements/testing.txt # pygithub -pylint==2.17.3 +pylint==2.17.4 # via # -r requirements/testing.txt # edx-lint @@ -201,10 +185,6 @@ pytest==7.3.1 # pytest-cov pytest-cov==4.0.0 # via -r requirements/testing.txt -python-dateutil==2.8.2 - # via - # -r requirements/testing.txt - # botocore python-slugify==8.0.1 # via # -r requirements/testing.txt @@ -217,16 +197,11 @@ requests==2.30.0 # via # -r requirements/testing.txt # pygithub -s3transfer==0.6.0 - # via - # -r requirements/testing.txt - # boto3 six==1.16.0 # via # -r requirements/ci.txt # -r requirements/testing.txt # edx-lint - # python-dateutil # tox smmap==5.0.0 # via @@ -269,10 +244,9 @@ typing-extensions==4.5.0 # -r requirements/testing.txt # astroid # pylint -urllib3==1.26.15 +urllib3==2.0.2 # via # -r requirements/testing.txt - # botocore # requests virtualenv==20.23.0 # via diff --git a/requirements/testing.in b/requirements/testing.in index 4ba8110e..2d7733fa 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -3,9 +3,7 @@ -r base.txt coverage -ddt edx-lint -httpretty mock pycodestyle pytest diff --git a/requirements/testing.txt b/requirements/testing.txt index a295bb48..0db19c72 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -8,14 +8,7 @@ astroid==2.15.4 # via # pylint # pylint-celery -boto3==1.26.126 - # via -r requirements/base.txt -botocore==1.29.126 - # via - # -r requirements/base.txt - # boto3 - # s3transfer -certifi==2022.12.7 +certifi==2023.5.7 # via # -r requirements/base.txt # requests @@ -46,8 +39,6 @@ cryptography==40.0.2 # via # -r requirements/base.txt # pyjwt -ddt==1.6.0 - # via -r requirements/testing.in deprecated==1.2.13 # via # -r requirements/base.txt @@ -64,8 +55,6 @@ gitdb==4.0.10 # gitpython gitpython==3.1.31 # via -r requirements/base.txt -httpretty==1.1.4 - # via -r requirements/testing.in idna==3.4 # via # -r requirements/base.txt @@ -76,11 +65,6 @@ isort==5.12.0 # via pylint jinja2==3.1.2 # via code-annotations -jmespath==1.0.1 - # via - # -r requirements/base.txt - # boto3 - # botocore lazy-object-proxy==1.9.0 # via astroid markupsafe==2.1.2 @@ -111,7 +95,7 @@ pyjwt[crypto]==2.6.0 # via # -r requirements/base.txt # pygithub -pylint==2.17.3 +pylint==2.17.4 # via # edx-lint # pylint-celery @@ -135,10 +119,6 @@ pytest==7.3.1 # pytest-cov pytest-cov==4.0.0 # via -r requirements/testing.in -python-dateutil==2.8.2 - # via - # -r requirements/base.txt - # botocore python-slugify==8.0.1 # via code-annotations pyyaml==6.0 @@ -147,15 +127,8 @@ requests==2.30.0 # via # -r requirements/base.txt # pygithub -s3transfer==0.6.0 - # via - # -r requirements/base.txt - # boto3 six==1.16.0 - # via - # -r requirements/base.txt - # edx-lint - # python-dateutil + # via edx-lint smmap==5.0.0 # via # -r requirements/base.txt @@ -177,10 +150,9 @@ typing-extensions==4.5.0 # via # astroid # pylint -urllib3==1.26.15 +urllib3==2.0.2 # via # -r requirements/base.txt - # botocore # requests wrapt==1.15.0 # via diff --git a/requirements/travis.in b/requirements/travis.in deleted file mode 100644 index d7afac9f..00000000 --- a/requirements/travis.in +++ /dev/null @@ -1,3 +0,0 @@ --c constraints.txt - -requests diff --git a/requirements/travis.txt b/requirements/travis.txt deleted file mode 100644 index 97010167..00000000 --- a/requirements/travis.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -certifi==2021.10.8 - # via requests -charset-normalizer==2.0.7 - # via requests -idna==3.3 - # via requests -requests==2.26.0 - # via -r requirements/travis.in -urllib3==1.25.11 - # via - # -c requirements/constraints.txt - # requests diff --git a/scripts/create_incr_tickets.py b/scripts/create_incr_tickets.py deleted file mode 100644 index 8ceaff2b..00000000 --- a/scripts/create_incr_tickets.py +++ /dev/null @@ -1,186 +0,0 @@ -import os -import sys -from collections import namedtuple - -# We should aspire to create batches of less than 15 python files -# although this is not a strict limit. Setting a target of 10 files, -# will on average yield batches of 10-15. -TARGET_FILE_NUMBER = 10 - - -class Batch: - """ - representation of a `batch` of python files for ticketing purposes - - the `root` of the batch is the greatest common path, given a list of - files - - a batch is considered `blocked` if there exist other batches that - contain files deeper in the path than this batch's root. This value - is used to denote the fact this this batch should not be worked on - or ticketed until the blocking batch has been completed - """ - - def __init__(self, root): - self.root = root - self.files = [] - self.blocked = False - - def __str__(self): - return "{} :: {}".format(self.root, len(self.files)) - - def add(self, file_path): - self.files.append(file_path) - self.rebalance_root() - - def remove(self, file_path): - self.files.remove(file_path) - self.rebalance_root() - - def contains_file(self, file_path): - return file_path in self.files - - def contains_dir(self, dir_path): - return dir_path in self.directories - - @property - def directories(self): - """ - return a list of all of the directories contained within this batch - of files. - """ - directories = list({ - '/'.join(f.split('/')[:-1]) for f in self.files - }) - return sorted(directories) - - @property - def top_level_directories(self): - """ - return a list of all of the top level directories in this batch of - files, that is, all of the directories that are not contained in other - directories in this batch - """ - return [d for d in self.directories if len([x for x in self.directories if x in d]) == 1] - - def rebalance_root(self): - """ - update the root of this batch after a file has been added, in case - their paths differ. For example: - - if this batch had a root of /a/b/c and we add a file from /a/b/d, - the newly balanced root should be /a/b - """ - split_dirs = [d.split('/') for d in self.directories] - new_root = [] - for level in zip(*split_dirs): - if not (all([d == level[0] for d in level])): - break - new_root.append(level[0]) - self.root = '/'.join(new_root) - - def file_count(self): - return len(self.files) - - def blocks(self, dirs): - """ - determine if this batch of files blocks work on another batch of - files. This is the case when a path (contained in this - batch) is a child of a directory in the list `dirs`. - """ - return any([d in self.directories for d in dirs]) - - def base_similar(self, other_root): - """ - determine if this batch has a root that is similar to another- that is, - it is either the same, is a subdirectory, or they share a common parent - """ - if self.root == other_root: - return True - elif self.root.split('/')[:-1] == other_root.split('/')[:-1]: - return True - elif other_root in self.root: - return True - else: - return False - - -def check_if_blocked(batches, root, dirs): - """ - check if any of the batches that have already been grouped are - contained, as sub directories, in a given root and it's children - directories - """ - paths = [os.path.join(root, d) for d in dirs] - return any([b.blocks(paths) for b in batches]) - - -def filter_python_files(files): - """ - given a list of files, extract the python files - """ - return [f for f in files if f.endswith('.py')] - - -def crawl(path, TARGET_FILE_NUMBER): - """ - crawl a given file path, from the deepest node up, collecting and - organizing directories containing python files into `Batches` of less - than `TARGET_FILE_NUMBER` python files. - """ - batches = [] - in_a_batch = False - - for root, dirs, files in os.walk(path, topdown=False): - # skip directories that have no python files - if len(filter_python_files(files)) < 1: - continue - if not in_a_batch: - current_batch = Batch(root) - in_a_batch = True - if not current_batch.base_similar(root): - batches.append(current_batch) - current_batch = Batch(root) - in_a_batch = True - # mark this batch as `blocked` if any of the subdirectories in the - # current node have already been added to the list of batches - if check_if_blocked(batches, root, dirs): - current_batch.blocked = True - python_files = [os.path.join(root, f) for f in filter_python_files(files)] - for file_path in python_files: - current_batch.add(file_path) - if current_batch.file_count() >= TARGET_FILE_NUMBER: - batches.append(current_batch) - in_a_batch = False - if in_a_batch: - batches.append(current_batch) - return batches - - -def main(): - path = sys.argv[1] - ticket_number_seed = int(sys.argv[2]) - batches = crawl(path, TARGET_FILE_NUMBER) - - blocked_batches = [b for b in batches if b.blocked] - ready_batches = [b for b in batches if not b.blocked] - - with open('ready_batches.csv', 'w') as out: - for b in ready_batches: - dirs = ':'.join(b.top_level_directories) - ticket_number = f"INCR-{ticket_number_seed}" - out.write(f'{ticket_number},{b.blocked},{b.file_count()},{dirs}') - out.write('\n') - ticket_number_seed += 1 - - with open('blocked_batches.csv', 'w') as out: - for b in blocked_batches: - dirs = ':'.join(b.top_level_directories) - ticket_number = f"INCR-{ticket_number_seed}" - out.write(f'{ticket_number},{b.blocked},{b.file_count()},{dirs}') - out.write('\n') - ticket_number_seed += 1 - - -if __name__ == '__main__': - main() diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100755 index ff78d459..00000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -mkdir -p happy_path -touch happy_path/a.py -touch happy_path/b.py -touch happy_path/c.py -mkdir -p multi_dir/a -touch multi_dir/a/a1.py -touch multi_dir/a/a2.py -mkdir -p multi_dir/b -touch multi_dir/b/b1.py -mkdir -p multi_dir/c -touch multi_dir/c/c1.py -touch multi_dir/c/c2.py -mkdir -p dependencies/dir/sub-dir -touch dependencies/dir/a.py -touch dependencies/dir/b.py -touch dependencies/dir/sub-dir/c.py -touch dependencies/dir/sub-dir/d.py -touch dependencies/dir/sub-dir/e.py -mkdir -p local/then_this -touch local/then_this/c1.py -mkdir -p local/this_first/dir/sub1 -touch local/this_first/dir/sub1/b1.py -mkdir -p local/this_first/dir/sub2 -touch local/this_first/dir/sub2/a1.py -touch local/this_first/dir/sub2/a2.py -touch local/this_first/dir/sub2/a3.py diff --git a/scripts/test.py b/scripts/test.py deleted file mode 100644 index 26cc6f8f..00000000 --- a/scripts/test.py +++ /dev/null @@ -1,92 +0,0 @@ -# pylint: disable=missing-module-docstring -import os - -from create_incr_tickets import Batch, crawl - -BASE = os.getcwd() - - -def test_dirs(): - - batch = Batch('root') - files = [ - 'food/fruit/berries/blueberries.py', - 'food/fruit/berries/blackberries.py', - 'food/fruit/apples.py', - 'food/fruit/bananas.py', - 'furniture/chair.py', - 'furniture/table.py' - ] - - for f in files: - batch.add(f) - expected_dirs = [ - 'food/fruit', - 'food/fruit/berries', - 'furniture' - ] - assert batch.directories == expected_dirs - - expected_dirs = ['food/fruit', 'furniture'] - assert batch.top_level_directories == expected_dirs - - -def test_rebalanced_root(): - - batch = Batch('food/fruit/berries') - batch.add('food/fruit/berries/raspberries.py') - assert batch.root == 'food/fruit/berries' - batch.add('food/fruit/apples.py') - assert batch.root == 'food/fruit' - batch.add('food/meat/poultry/chicken.py') - assert batch.root == 'food' - - -def test_crawl_happy_path(): - PATH = os.path.join(BASE, 'happy_path') - batches = crawl(PATH, 3) - assert len(batches) == 1 - batch = batches[0] - assert len(batch.files) == 3 - assert batch.root == PATH - - -def test_crawl_multidir(): - PATH = os.path.join(BASE, 'multi_dir') - batches = crawl(PATH, 3) - assert len(batches) == 2 - complete_batch = batches[0] - assert len(complete_batch.files) == 3 - assert complete_batch.root == PATH - incomplete_batch = batches[1] - assert len(incomplete_batch.files) == 2 - assert incomplete_batch.root == PATH + '/a' - - -def test_crawl_w_dependencies(): - PATH = os.path.join(BASE, 'dependencies') - batches = crawl(PATH, 3) - assert len(batches) == 2 - complete_batch = batches[0] - assert len(complete_batch.files) == 3 - assert complete_batch.root == PATH + '/dir/sub-dir' - assert not complete_batch.blocked - incomplete_batch = batches[1] - assert len(incomplete_batch.files) == 2 - assert incomplete_batch.root == PATH + '/dir' - assert incomplete_batch.blocked - - -def test_local_batches(): - PATH = os.path.join(BASE, 'local') - batches = crawl(PATH, 3) - assert len(batches) == 3 - first_batch = batches[0] - assert len(first_batch.files) == 3 - assert first_batch.root == PATH + "/this_first/dir/sub2" - second_batch = batches[1] - assert len(second_batch.files) == 1 - assert second_batch.root == PATH + "/this_first/dir/sub1" - third_batch = batches[2] - assert len(third_batch.files) == 1 - assert third_batch.root == PATH + "/then_this" diff --git a/tox.ini b/tox.ini index 343e2387..972898d6 100644 --- a/tox.ini +++ b/tox.ini @@ -37,18 +37,12 @@ norecursedirs = .* requirements [testenv] deps = -r{toxinidir}/requirements/testing.txt -passenv = - BOTO_CONFIG whitelist_externals = make commands = make clean pycodestyle . - pylint aws jenkins scripts/test.py travis - isort --check-only --diff aws jenkins lambdas scripts travis + pylint jenkins + isort --check-only --diff jenkins make selfcheck - pytest --cov-report term-missing --cov=aws aws pytest --cov-report term-missing --cov=jenkins jenkins - pytest --cov-report term-missing --cov=lambdas lambdas - pytest --cov-report term-missing --cov=travis travis - diff --git a/travis/README.md b/travis/README.md deleted file mode 100644 index 877d31f0..00000000 --- a/travis/README.md +++ /dev/null @@ -1,18 +0,0 @@ -Travis Build Metrics -===== - -Tools contained here are used for obtaining metadata of Travis builds - -build_info ---- -This script will: - -* use the Travis REST API to get information on builds. - -*Usage* - -* create your virtual environment -* pip install -r requirements/travis.txt -* Call script as a python module, e.g. - - `python -m travis.build_info --org MyGithubOrg --log_level debug` diff --git a/travis/__init__.py b/travis/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/travis/build_info.py b/travis/build_info.py deleted file mode 100644 index f3e6942f..00000000 --- a/travis/build_info.py +++ /dev/null @@ -1,309 +0,0 @@ -""" -Gather Travis usage info in real-time using Travis REST APIs - -This only applies to travis instances that do not require -authorization (e.g., travis-ci.org). Auth is a #TODO - -Remove/Replace this implementation for GitHub CI since -Travis is being removed from across edX #TODO -""" - -import argparse -import logging -import sys -from operator import itemgetter - -import requests - -logger = logging.getLogger(__name__) - -BASE_URL = 'https://api.travis-ci.org/' - - -def get_repos(org): - """ - Returns list of active repos in a given org. - """ - repo_list = [] - - # Note: One call is needed if org has <= 100 active repos. - # If org has > 100, this code will need to be modified - # to include 'offset' requests - req = requests.get( - BASE_URL + f'v3/owner/{org}/repos?active=true', - timeout=5, - ) - if req.status_code != 200: - raise requests.HTTPError(req.status_code) - repos = req.json() - try: - for repo in repos.get('repositories'): - repo_list.append(repo['name']) - except KeyError as error: - raise KeyError("Cannot parse response") from error - - return repo_list - - -def get_builds(org, repo, is_finished=False): - """ - Returns list of active builds for a given repo slug - """ - logger.debug('getting builds for repo: %s', repo) - repo_slug = f'{org}/{repo}' - req = requests.get( - BASE_URL + f'repos/{repo_slug}/builds', - timeout=5, - ) - build_list = req.json() - selected_build_list = [] - for build in build_list: - if not is_finished: - if build.get('state') != 'finished': - selected_build_list.append(build) - else: - if build.get('state') == 'finished': - selected_build_list.append(build) - - return selected_build_list - - -def get_last_n_successful_builds(org, repo, number): - """ - Collects the specified number of previous successful builds - for a given repo - """ - finished_builds = get_builds(org, repo, is_finished=True) - successful_builds = [] - for build in finished_builds: - # if build passed, add it to our list - if build['result'] == 0: - successful_builds.append(build) - - # sort in descending order - successful_builds = sorted( - successful_builds, - key=itemgetter('number'), - reverse=True - ) - # if we can't get the specified number of builds, just return everything - if len(successful_builds) < number: - return successful_builds - - return successful_builds[:number] - - -def get_average_build_duration(builds): - """ - returns average build duration in minutes (a whole number) - - """ - durations = [] - for build in builds: - durations.append(int(build['duration'])) - return sum(durations) // len(builds) // 60 # python 3 division here - - -def get_average_duration_org(org, num=5): - """ - Returns dict of repos and average durations - num: dataset size from which to derive the - average (e.g., num=5 would be the average over the last 5 builds) - org: the github org - """ - repos = get_repos(org) - avg_duration_org = [] - for repo in repos: - builds = get_last_n_successful_builds(org, repo, num) - logger.debug('getting average duration for: %s', repo) - avg = get_average_build_duration(builds) - avg_duration_org.append({"repo": repo, "average duration": avg}) - avg_duration_org = sorted( - avg_duration_org, - key=itemgetter("average duration"), - reverse=True - ) - message = '[' - for entry in avg_duration_org: - message += "{'repo': '%s', 'average duration': %s}" % (entry['repo'], entry['average duration']) - message += ']' - logger.info(message) - return avg_duration_org - - -def get_active_jobs(build_id): - """ - Get the jobs for a build - return: list of dicts - """ - jobs = [] - req = requests.get( - BASE_URL + f'v3/build/{build_id}/jobs', - timeout=5, - ) - job_resp = req.json() - for job in job_resp['jobs']: - if job['state'] not in ["passed", "failed"]: - jobs.append(job) - return jobs - - -def active_job_counts(jobs): - """ - Returns counts of total jobs, and - total running jobs for a given list - - This method assumes it has a received - a list of active jobs. - - Possible job states: - * received - * queued - * created - * passed, failed - * started - - """ - job_count = len(jobs) - started_jobs_count = 0 - for job in jobs: - if job['state'] == 'started': - started_jobs_count += 1 - - return job_count, started_jobs_count - - -def repo_active_build_count(builds): - """ - Returns counts of total builds, and total - running builds for a given list of them - - This method assumes it has received a list - of active builds. - - Possible build states: - * created - * started - * finished - - """ - build_count = 0 - started_count = 0 - for build in builds: - build_count += 1 - if build.get('state') == 'started': - started_count += 1 - - return build_count, started_count - - -def get_job_counts(org): - """ - Total job counts (active and total) for - an org - """ - total_job_count = 0 - total_started_job_count = 0 - - repos = get_repos(org) - for repo in repos: - repo_builds = get_builds(org, repo) - repo_jobs = 0 - repo_started_jobs = 0 - for build in repo_builds: - build_jobs = get_active_jobs(build['id']) - total, started = active_job_counts(build_jobs) - total_job_count += total - total_started_job_count += started - repo_jobs += total - repo_started_jobs += started - logger.debug("----> %s", repo) - debug_msg = "total jobs: %d, started jobs: %d" - logger.debug(debug_msg, repo_jobs, repo_started_jobs) - logger.debug('--------') - logger.info('overall_jobs_total=%d', total_job_count) - logger.info('overall_jobs_started=%d', total_started_job_count) - - -def get_build_counts(org): - """ - Find out all the active and waiting builds for a given - Github/Travis org - """ - repos = get_repos(org) - org_build_count = 0 - org_build_started_count = 0 - - for repo in repos: - repo_builds = get_builds(org, repo) - logger.debug("--->%s", repo) - - repo_build_total, num_started = repo_active_build_count(repo_builds) - - debug_string = "total: %d, started: %d" - logger.debug(debug_string, repo_build_total, num_started) - - org_build_count += repo_build_total - org_build_started_count += num_started - - logger.debug('--------') - logger.info("overall_total=%d", org_build_count) - logger.info("overall_started=%d", org_build_started_count) - logger.info( - "overall_queued=%d", org_build_count - org_build_started_count - ) - - -def main(raw_args): - """ - Parse args and execute the script according to those args - """ - desc = ( - "Obtain information on active/waiting Travis builds." - ) - parser = argparse.ArgumentParser(description=desc) - parser.add_argument( - '--org', '-o', - dest='org', - help='Travis org', - required=True - ) - parser.add_argument( - '--task-class', # this is not doing anything for now. - dest='task_class', - help="Select build or job. A build is composed of one or many jobs.", - choices=[ - 'BUILD', 'build', - 'JOB', 'job', - 'DURATION', 'duration' - ], - default="BUILD", - ) - parser.add_argument( - '--log-level', - dest='log_level', - help="set logging level", - choices=[ - 'DEBUG', 'debug', - 'INFO', 'info', - 'WARNING', 'warning', - 'ERROR', 'error', - 'CRITICAL', 'critical', - ], - default="INFO", - ) - args = parser.parse_args(raw_args) - - # Set logging level - logging.getLogger(__name__).setLevel(args.log_level.upper()) - if args.task_class.upper() == 'JOB': - get_job_counts(org=args.org) - elif args.task_class.upper() == 'DURATION': - get_average_duration_org(org=args.org) - else: - get_build_counts(org=args.org) - - -if __name__ == "__main__": - logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s') - main(sys.argv[1:]) diff --git a/travis/tests/__init__.py b/travis/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/travis/tests/fixtures/builds_response.list b/travis/tests/fixtures/builds_response.list deleted file mode 100644 index 5645d5e9..00000000 --- a/travis/tests/fixtures/builds_response.list +++ /dev/null @@ -1,8 +0,0 @@ -[{ "id": 1, "state": "finished", "result": 0, "number": 10, "duration": 6000 }, -{ "id": 2, "state": "started", "number": 11 }, -{ "id": 3, "state": "finished", "result": 0, "number": 12, "duration": 500 }, -{ "id": 4, "state": "finished", "result": 0, "number": 13, "duration": 540 }, -{ "id": 5, "state": "finished", "result": 0, "number": 14, "duration": 600 }, -{ "id": 6, "state": "finished", "result": 0, "number": 15, "duration": 700 }, -{ "id": 7, "state": "queued", "number": 16 }, -{ "id": 8, "state": "finished", "result": 0, "number": 17, "duration": 650 }] \ No newline at end of file diff --git a/travis/tests/test_build_info.py b/travis/tests/test_build_info.py deleted file mode 100644 index 60baafc0..00000000 --- a/travis/tests/test_build_info.py +++ /dev/null @@ -1,496 +0,0 @@ -""" -Tests for testeng-ci/travis/build_info -""" - -import os -from unittest import TestCase -from unittest.mock import patch - -import httpretty -import requests -from ddt import data, ddt -from testfixtures import LogCapture - -from travis.build_info import (BASE_URL, active_job_counts, get_active_jobs, - get_average_build_duration, get_builds, - get_last_n_successful_builds, get_repos, main, - repo_active_build_count) - - -class TestTravisBuildRepoInfo(TestCase): - """ - Test API client for obtaining build information - """ - - def setUp(self): - super().setUp() - self.response_body = """{ - "@type": "repositories", - "repositories": [ - { - "name": "foo" - }, { - "name": "bar" - } - ] - } - """ - - @httpretty.activate - def test_get_active_repos(self): - httpretty.register_uri( - httpretty.GET, - BASE_URL + 'v3/owner/foo/repos', - body=self.response_body, - status=200, - ) - repos = get_repos('foo') - self.assertListEqual(repos, ['foo', 'bar']) - - @httpretty.activate - def test_bad_repos_response(self): - httpretty.register_uri( - httpretty.GET, - BASE_URL + 'v3/owner/foo/repos', - body=self.response_body, - status=404, - ) - with self.assertRaises(requests.HTTPError): - get_repos('foo') - - @httpretty.activate - def test_unparseable_response(self): - httpretty.register_uri( - httpretty.GET, - BASE_URL + 'v3/owner/foo/repos', - body="""{"repositories": [{"no-name": "no"}]}""", - ) - with self.assertRaises(KeyError): - get_repos('foo') - - -class TestTravisActiveBuildInfo(TestCase): - """ - Handle responses for queries on a given repo's builds - """ - - def setUp(self): - super().setUp() - self.url_endpoint = BASE_URL + 'repos/foo/bar-repo/builds' - - @httpretty.activate - def test_good_build_response(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "created"}, - {"id": 2, "state": "started"}]""", - ) - builds = get_builds('foo', 'bar-repo') - self.assertEqual(2, len(builds)) - - @httpretty.activate - def test_active_finished_mix(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "created"}, - {"id": 2, "state": "finished"}]""", - ) - builds = get_builds('foo', 'bar-repo') - self.assertEqual(1, len(builds)) - - @httpretty.activate - def test_all_finished(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "finished"}, - {"id": 2, "state": "finished"}]""", - ) - builds = get_builds('foo', 'bar-repo') - self.assertEqual(0, len(builds)) - - @httpretty.activate - def test_active_build_count(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "started"}, - {"id": 2, "state": "created"}]""", - ) - builds = get_builds('foo', 'bar-repo') - total_count, started_count = repo_active_build_count(builds) - self.assertEqual(2, total_count) - self.assertEqual(1, started_count) - - @httpretty.activate - def test_only_queued_builds(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "created"}, - {"id": 2, "state": "created"}]""", - ) - builds = get_builds('foo', 'bar-repo') - total_count, started_count = repo_active_build_count(builds) - self.assertEqual(2, total_count) - self.assertEqual(0, started_count) - - def test_no_active_builds(self): - builds = [] - total_count, started_count = repo_active_build_count(builds) - self.assertEqual(0, total_count) - self.assertEqual(0, started_count) - - -class TestTravisFinishedBuildInfo(TestCase): - """ - Handle responses for queries on a given repo's builds - """ - - def setUp(self): - super().setUp() - self.url_endpoint = BASE_URL + 'repos/foo/bar-repo/builds' - - @httpretty.activate - def test_vanilla_finished_builds(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "finished"}, - {"id": 2, "state": "finished"}]""", - ) - builds = get_builds('foo', 'bar-repo', is_finished=True) - self.assertEqual(2, len(builds)) - - @httpretty.activate - def test_active_finished_mix(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "created"}, - {"id": 2, "state": "finished"}, - {"id": 3, "state": "started"}]""", - ) - builds = get_builds('foo', 'bar-repo', is_finished=True) - self.assertEqual(1, len(builds)) - - @httpretty.activate - def test_all_finished(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "finished"}, - {"id": 2, "state": "finished"}]""", - ) - builds = get_builds('foo', 'bar-repo', is_finished=True) - self.assertEqual(2, len(builds)) - - @httpretty.activate - def test_all_finished_but_asking_for_active(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "finished"}, - {"id": 2, "state": "finished"}]""", - ) - builds = get_builds('foo', 'bar-repo') - self.assertEqual(0, len(builds)) - - @httpretty.activate - def test_all_active(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body="""[{"id": 1, "state": "started"}, - {"id": 2, "state": "created"}]""", - ) - builds = get_builds('foo', 'bar-repo') - self.assertEqual(2, len(builds)) - - -class TestTravisBuildInfoJobs(TestCase): - """ - Ensure we can get jobs data - """ - - def setUp(self): - super().setUp() - self.url_endpoint = BASE_URL + 'v3/build/11122/jobs' - - @httpretty.activate - def test_jobs_count(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body=""" - {"jobs": - [ - {"id": 1, "state": "received"}, - {"id": 2, "state": "created"} - ] - } - """, - ) - jobs = get_active_jobs(11122) - expected_job_list = [ - {'id': 1, 'state': 'received'}, - {'id': 2, 'state': 'created'} - ] - self.assertListEqual(expected_job_list, jobs) - - @httpretty.activate - def test_jobs_count_with_completed(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body=""" - {"jobs": - [ - {"id": 1, "state": "passed"}, - {"id": 2, "state": "created"}, - {"id": 3, "state": "failed"} - ] - } - """, - ) - jobs = get_active_jobs(11122) - self.assertEqual(1, len(jobs)) - - @httpretty.activate - def test_jobs_count_no_active(self): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body=""" - {"jobs": - [ - {"id": 1, "state": "passed"}, - {"id": 2, "state": "passed"}, - {"id": 3, "state": "failed"} - ] - } - """, - ) - jobs = get_active_jobs(11122) - self.assertEqual(0, len(jobs)) - - def test_active_job_counts_zero_builds(self): - job_count, started_jobs_count = active_job_counts([]) - self.assertTupleEqual((0, 0), (job_count, started_jobs_count)) - - def test_active_job_counts_some(self): - jobs_list = [ - {"id": 1, "state": "queued"}, - {"id": 1, "state": "started"}, - ] - job_count, started_jobs_count = active_job_counts(jobs_list) - self.assertEqual(2, job_count) - self.assertEqual(1, started_jobs_count) - - def test_active_job_counts_various(self): - jobs_list = [ - {"id": 1, "state": "queued"}, - {"id": 1, "state": "created"}, - {"id": 1, "state": "received"}, - ] - job_count, started_jobs_count = active_job_counts(jobs_list) - self.assertEqual(3, job_count) - self.assertEqual(0, started_jobs_count) - - def test_active_job_counts_mult(self): - jobs_list = [ - {"id": 1, "state": "queued"}, - {"id": 1, "state": "started"}, - {"id": 1, "state": "started"}, - ] - job_count, started_jobs_count = active_job_counts(jobs_list) - self.assertEqual(3, job_count) - self.assertEqual(2, started_jobs_count) - - -@ddt -class TestTravisSuccessfulBuilds(TestCase): - """ - Test successful build data capture - """ - - def setUp(self): - super().setUp() - self.url_endpoint = BASE_URL + 'repos/foo/bar-repo/builds' - - @data( - {"requested": 5, "expected": 5}, - {"requested": 7, "expected": 6}, # the max found in the file is 6 - - ) - @httpretty.activate - def test_successful_builds(self, test_data): - httpretty.register_uri( - httpretty.GET, - self.url_endpoint, - body=self._load_mock_builds_response_file( - "fixtures/builds_response.list" - ), - ) - - successful_builds = get_last_n_successful_builds( - 'foo', - 'bar-repo', - test_data['requested'] - ) - self.assertEqual(len(successful_builds), test_data['expected']) - - def _load_mock_builds_response_file(self, filename): - """ - returns the contents of the specified text fixture/file - """ - test_dir = os.path.dirname(__file__) - abs_file = os.path.join(test_dir, filename) - with open(abs_file, encoding='utf-8') as test_file: - contents = test_file.read() - - return contents - - -class TestBuildDurationCalculation(TestCase): - """ - test build duration calculation, such as average - build duration - """ - - def test_average_build_duration_returns_minutes(self): - builds = [ - {'id': 1, 'duration': 60} - ] - self.assertEqual(get_average_build_duration(builds), 1) - - def test_average_duration_whole_numbers(self): - """ - Average number would give a float, but the method - returns a whole number - """ - builds = [ - {'id': 1, 'duration': 600}, - {'id': 2, 'duration': 600}, - {'id': 3, 'duration': 660} - ] - self.assertEqual(get_average_build_duration(builds), 10) - - -class TestTravisBuildInfoMain(TestCase): - """ - Test CLI args, and output, output formatting, etc - """ - - def setUp(self): - super().setUp() - self.org = 'foo' - self.mock_repos = patch( - 'travis.build_info.get_repos', return_value=['bar'] - ) - self.mock_builds = patch( - 'travis.build_info.get_builds', - return_value=[{"id": 1, "state": "started"}] - ) - self.mock_jobs = patch( - 'travis.build_info.get_active_jobs', - return_value=[ - {"id": 1, "state": "passed"}, - {"id": 2, "state": "passed"}, - {"id": 3, "state": "failed"} - ] - - ) - - self.mock_repos.start() - self.mock_builds.start() - self.mock_jobs.start() - self.addCleanup(patch.stopall) - - def test_main(self): - args = [ - '--org', self.org - ] - with LogCapture() as log_capture: - main(args) - log_capture.check( - ('travis.build_info', 'INFO', 'overall_total=1'), - ('travis.build_info', 'INFO', 'overall_started=1'), - ('travis.build_info', 'INFO', 'overall_queued=0') - ) - - def test_main_build_opt_in(self): - args = [ - '--org', self.org, - '--task-class', 'build' - ] - with LogCapture() as log_capture: - main(args) - log_capture.check( - ('travis.build_info', 'INFO', 'overall_total=1'), - ('travis.build_info', 'INFO', 'overall_started=1'), - ('travis.build_info', 'INFO', 'overall_queued=0') - ) - - @patch( - 'travis.build_info.get_builds', - return_value=[ - { - "id": 1, - "state": "finished", - "result": 0, - "number": 10, - "duration": 600 - } - ] - ) - def test_main_duration(self, _mock_builds): - args = [ - '--org', self.org, - '--task-class', 'duration' - ] - with LogCapture() as log_capture: - main(args) - log_capture.check( - ( - 'travis.build_info', - 'INFO', - "[{'repo': 'bar', 'average duration': 10}]" - ) - ) - - def test_main_debug(self): - args = [ - '--org', self.org, - '--log-level', 'debug' - ] - with LogCapture() as log_capture: - main(args) - log_capture.check( - ('travis.build_info', 'DEBUG', '--->bar'), - ('travis.build_info', 'DEBUG', 'total: 1, started: 1'), - ('travis.build_info', 'DEBUG', '--------'), - ('travis.build_info', 'INFO', 'overall_total=1'), - ('travis.build_info', 'INFO', 'overall_started=1'), - ('travis.build_info', 'INFO', 'overall_queued=0') - ) - - def test_main_debug_jobs(self): - args = [ - '--org', self.org, - '--log-level', 'debug', - '--task-class', 'job' - ] - - with LogCapture() as log_capture: - main(args) - log_capture.check( - ('travis.build_info', 'DEBUG', '----> bar'), - ('travis.build_info', 'DEBUG', - 'total jobs: 3, started jobs: 0'), - ('travis.build_info', 'DEBUG', '--------'), - ('travis.build_info', 'INFO', 'overall_jobs_total=3'), - ('travis.build_info', 'INFO', 'overall_jobs_started=0') - )