Skip to content

Commit

Permalink
Handle waffle switch tests separately from other syntehtic tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@bszabo
bszabo committed Jan 10, 2025
1 parent 3ef3412 commit ecc4cb2
Showing 1 changed file with 38 additions and 29 deletions.
67 changes: 38 additions & 29 deletions tubular/scripts/dd_synthetic_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,9 @@ class SyntheticTest:
'''
Attributes for a specific synthetic test and its test run under a datadog CI batch
'''
def __init__(self, name, public_id, serves_as_waffle_flag):
def __init__(self, name, public_id):
self.name = name
self.public_id = public_id
self.serves_as_waffle_flag = serves_as_waffle_flag
self.test_run_id = None
self.success = None

Expand All @@ -26,6 +25,13 @@ class DatadogClient:
DATADOG_SYNTHETIC_TESTS_API_URL = "https://api.datadoghq.com/api/v1/synthetics/tests"
MAX_ALLOWABLE_TIME_SECS = 600 # 10 minutes

WAFFLE_SWITCH_TEST = SyntheticTest(
'''
Test on edx.org
''',
"sad-hqu-h33"
)

def __init__(self, api_key, app_key):
self.api_key = api_key
self.app_key = app_key
Expand All @@ -39,6 +45,7 @@ def trigger_synthetic_tests(self, test_requests: [SyntheticTest]):
:param test_requests: List of tests to be run
:return: None, but saves batch ID and test run IDs as object attributes
'''
requests = list(self.WAFFLE_SWITCH_TEST) + test_requests
self._record_batch_tests(test_requests)
self.trigger_time = time.time() # Key timeouts off of this
logging.info(f'CI batch triggered at time {self.trigger_time}')
Expand All @@ -52,6 +59,21 @@ def trigger_synthetic_tests(self, test_requests: [SyntheticTest]):
except Exception as e:
raise Exception("Datadog error on triggering tests: " + str(e))

def gate_on_waffle_switch(self):
'''
This is a bit hacky, but there's a designated test that's used as a waffle switch might be used in
a Django app. If the test passes, it means that the GoCD pipeline is to pass or fail per the result
of the Datadog synthetic tests that are run. If the test fails, then it means that the Datadog synthetic
test results are to be disregarded. When this is the case, the GoCD pipeline responsible for running the
tests should return with a summary success.
'''
waffle_switch = self._poll_for_test_result(self.WAFFLE_SWITCH_TEST)
if waffle_switch == False:
switch_test_name = self.WAFFLE_SWITCH_TEST.name
logging.warning(
f'*** Datadog Synthetic testing disabled via failing "Waffle flag" test {switch_test_name} ***')
sys.exit(0)

def get_and_record_test_results(self):
'''
Poll for test results for all tests that were run, and save the results
Expand Down Expand Up @@ -92,7 +114,7 @@ def _trigger_batch_tests(self):
"DD-API-KEY": self.api_key,
"DD-APPLICATION-KEY": self.app_key
}
test_public_ids = list(self.tests_by_public_id.keys())
test_public_ids = self.tests_by_public_id.keys()
json_request_body = {"tests": [{"public_id": public_id} for public_id in test_public_ids]}
response = requests.post(url, headers=headers, json=json_request_body)
if response.status_code != 200:
Expand All @@ -114,7 +136,8 @@ def _record_test_run_ids(self, response_body):
for result in response_body['results']:
public_id = result['public_id']
test_run_id = result['result_id']
self.tests_by_public_id[public_id].test_run_id = test_run_id
if public_id != self.WAFFLE_SWITCH_TEST.public_id:
self.tests_by_public_id[public_id].test_run_id = test_run_id

def _poll_for_test_result(self, test):
"""
Expand Down Expand Up @@ -186,73 +209,59 @@ def run_synthetic_tests(enable_automated_rollbacks, slack_notification_channel):
dd_client = DatadogClient(api_key, app_key)

tests_to_run = [
SyntheticTest(
'''
Test on edx.org
''',
"sad-hqu-h33",
True
),
SyntheticTest(
'''
[Synthetics] edX Smoke Test - [Verified student] A verified student can
access a graded course problem
''',
"tck-hrr-ubp",
False
"tck-hrr-ubp"
),
SyntheticTest(
'''
[Synthetics] edX Smoke Test - [Verified student] An enrolled verified student can
access a course’s landing page, course content, and course forum
''',
"zbz-r28-jjx",
False
"zbz-r28-jjx"
),
# SyntheticTest(
# '''
# [Synthetics] edX Smoke Test - [Audit student] An enrolled audit student cannot load
# a graded problem, and sees the upsell screen
# ''',
# "75p-sez-5wg",
# False
# "75p-sez-5wg"
# ),
# SyntheticTest(
# '''
# [Synthetics] edX Smoke Test - [Audit student] An enrolled audit student can access
# a course’s landing page, course content, and course forum
# ''',
# "jvx-2jw-agj",
# False
# "jvx-2jw-agj"
# ),
SyntheticTest(
'''
edX Smoke Test - [Unenrolled student] An unenrolled student cannot load a
course’s landing page, and sees the “Enroll Now” screen
''',
"zkx-36f-kui",
False
"zkx-36f-kui"
),
SyntheticTest(
'''
edX Smoke Test - [Anonymous user] An anonymous user is directed to the
Logistration page (authn.edx.org) when trying to access content behind log-in wall
''',
"6tq-u28-hwa",
False
"6tq-u28-hwa"
),
]
logging.info(f"Running the following tests: {str(tests_to_run)}")

dd_client.trigger_synthetic_tests(tests_to_run)
dd_client.gate_on_waffle_switch() # Exits summarily if test results to be ignored
for test in tests_to_run:
logging.info(f"\t Running test {test.public_id}: {test.name}")
dd_client.get_and_record_test_results()
failed_tests = dd_client.get_failed_tests()

for failed_test in failed_tests:
logging.warning(f'Test {failed_test.name}({failed_test.public_id} failed')
if failed_test.serves_as_waffle_flag:
logging.warning(f'*** Datadog Synthetic testing disabled via failing "Waffle flag" test {failed_test.name} ***')
sys.exit(0) # Failure by the "waffle flag" test is a signal that synthetic test results are to be
# ignored; that is, all synthetic tests are to be treated as having passed.
logging.warning(f'Test failed: {failed_test.public_id} -- {failed_test.name}')

task_failed_code = 1 if failed_tests else 0

Expand Down

0 comments on commit ecc4cb2

Please sign in to comment.