From a995365681083a02979b3389783e85d1497f46f3 Mon Sep 17 00:00:00 2001 From: Jaydin_MacBook Date: Sat, 6 Jan 2024 11:19:13 -0500 Subject: [PATCH 1/4] Added Enable Instructors Test & Get Failure logging --- backend/blackboard_scraper.py | 135 ++++++++++++++-------------- backend/test_blackboard_scraper.py | 137 ++++++++++++++++++++++++++--- 2 files changed, 196 insertions(+), 76 deletions(-) diff --git a/backend/blackboard_scraper.py b/backend/blackboard_scraper.py index 0817d5e..b8a2fdf 100644 --- a/backend/blackboard_scraper.py +++ b/backend/blackboard_scraper.py @@ -189,70 +189,76 @@ def enable_instructors(self): try: get_url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=edit_module%2F_4_1%2Fbbcourseorg%3Fcmd%3Dedit&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1" - get_response = self._get_request(get_url) - - if get_response.status_code != 200: - raise Exception("GET request failed.") - - # Using beautiful soup get the value from this input #moduleEditForm > input[type=hidden]:nth-child(1) - soup = BeautifulSoup(get_response.content, "html.parser") - nonce_value = soup.select_one( - '#moduleEditForm > input[type=hidden]:nth-child(1)')['value'] - - url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=proc_edit/_4_1/bbcourseorg&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1" - payload = { - 'tab_tab_group_id': '_1_1', - 'forwardUrl': 'proc_edit/_4_1/bbcourseorg', - 'blackboard.platform.security.NonceUtil.nonce': nonce_value, - 'recallUrl': '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1', - 'cmd': 'processEdit', - 'serviceLevel': '', - 'termDisplayOrder': '_254_1', - 'amc.groupbyterm': 'true', - 'selectAll_254_1': 'true', - 'amc.showterm._254_1': 'true', - 'termCourses__254_1': 'true', - 'amc.showcourse._51671_1': 'true', - 'amc.showcourseid._51671_1': 'true', - 'amc.showinstructors._51671_1': 'true', - 'amc.showcourse._51672_1': 'true', - 'amc.showcourseid._51672_1': 'true', - 'amc.showinstructors._51672_1': 'true', - 'amc.showcourse._51629_1': 'true', - 'amc.showcourseid._51629_1': 'true', - 'amc.showinstructors._51629_1': 'true', - 'amc.showcourse._51904_1': 'true', - 'amc.showcourseid._51904_1': 'true', - 'amc.showinstructors._51904_1': 'true', - 'amc.showcourse._51945_1': 'true', - 'amc.showcourseid._51945_1': 'true', - 'amc.showinstructors._51945_1': 'true', - 'amc.url.name.1': '', - 'amc.url.url.1': '', - 'amc.url.name.2': '', - 'amc.url.url.2': '', - 'amc.url.name.3': '', - 'amc.url.url.3': '', - 'amc.url.name.4': '', - 'amc.url.url.4': '', - 'amc.url.name.5': '', - 'amc.url.url.5': '', - 'bottom_Submit': 'Submit' - } - enable_instructors_response = self._send_post_request( - url, data=payload, allow_redirects=False) - - if enable_instructors_response.status_code == 302: - redirected_url = enable_instructors_response.headers['Location'] - logging.info( - f"Successful POST request. Redirected to: {redirected_url}") - self.instructorsFound = True - else: - self.instructorsFound = False + try: + get_response = self._get_request(get_url) + + if get_response.status_code != 200: + raise Exception("GET request failed.") + + # Using beautiful soup get the value from this input #moduleEditForm > input[type=hidden]:nth-child(1) + soup = BeautifulSoup(get_response.content, "html.parser") + nonce_value = soup.select_one( + '#moduleEditForm > input[type=hidden]:nth-child(1)')['value'] + + url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=proc_edit/_4_1/bbcourseorg&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1" + payload = { + 'tab_tab_group_id': '_1_1', + 'forwardUrl': 'proc_edit/_4_1/bbcourseorg', + 'blackboard.platform.security.NonceUtil.nonce': nonce_value, + 'recallUrl': '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1', + 'cmd': 'processEdit', + 'serviceLevel': '', + 'termDisplayOrder': '_254_1', + 'amc.groupbyterm': 'true', + 'selectAll_254_1': 'true', + 'amc.showterm._254_1': 'true', + 'termCourses__254_1': 'true', + 'amc.showcourse._51671_1': 'true', + 'amc.showcourseid._51671_1': 'true', + 'amc.showinstructors._51671_1': 'true', + 'amc.showcourse._51672_1': 'true', + 'amc.showcourseid._51672_1': 'true', + 'amc.showinstructors._51672_1': 'true', + 'amc.showcourse._51629_1': 'true', + 'amc.showcourseid._51629_1': 'true', + 'amc.showinstructors._51629_1': 'true', + 'amc.showcourse._51904_1': 'true', + 'amc.showcourseid._51904_1': 'true', + 'amc.showinstructors._51904_1': 'true', + 'amc.showcourse._51945_1': 'true', + 'amc.showcourseid._51945_1': 'true', + 'amc.showinstructors._51945_1': 'true', + 'amc.url.name.1': '', + 'amc.url.url.1': '', + 'amc.url.name.2': '', + 'amc.url.url.2': '', + 'amc.url.name.3': '', + 'amc.url.url.3': '', + 'amc.url.name.4': '', + 'amc.url.url.4': '', + 'amc.url.name.5': '', + 'amc.url.url.5': '', + 'bottom_Submit': 'Submit' + } + enable_instructors_response = self._send_post_request( + url, data=payload, allow_redirects=False) + + if enable_instructors_response.status_code == 302: + redirected_url = enable_instructors_response.headers['Location'] + logging.info( + f"Successful POST request. Redirected to: {redirected_url}") + self.instructorsFound = True + else: + self.instructorsFound = False + logging.error( + f"POST request failed with status code: {enable_instructors_response.status_code}") + + self.last_activity_time = time.time() + + except Exception as e: logging.error( - f"POST request failed with status code: {enable_instructors_response.status_code}") - - self.last_activity_time = time.time() + f"GET request failed with status code: {get_response.status_code}") + return except Exception as e: logging.error(f"An error occurred enabling instructors: {e}") @@ -470,6 +476,3 @@ def process_course(course, href): self.download_tasks = download_tasks self.downloadTasksFound = True self.last_activity_time = time.time() - - - diff --git a/backend/test_blackboard_scraper.py b/backend/test_blackboard_scraper.py index 87479ad..64e850b 100644 --- a/backend/test_blackboard_scraper.py +++ b/backend/test_blackboard_scraper.py @@ -1,12 +1,16 @@ import logging import random +import time import unittest -from backend.blackboard_scraper import BlackboardSession +from blackboard_scraper import BlackboardSession from unittest.mock import MagicMock, patch from usernames import usernames + class TestBlackboardSession(unittest.TestCase): + # * Login Tests *# + def test_valid_credentials_login(self): # Set up username = 'Free8864' @@ -14,14 +18,15 @@ def test_valid_credentials_login(self): session = BlackboardSession(username=username, password=password) # Execute login - response = session.login() + session.login() + + response = session.get_response() # Check the response expected_message = "Login successful." self.assertEqual(response, expected_message) - def test_invalid_both_login(self): # Set up username = 'InvalidUsername' @@ -29,24 +34,28 @@ def test_invalid_both_login(self): session = BlackboardSession(username=username, password=password) # Execute login - response = session.login() + session.login() + + response = session.get_response() # Check the response expected_error_message = "The username you entered cannot be identified." self.assertEqual(response, expected_error_message) - def test_failed_login_invalid_password(self): # selected a random username from usernames.py username = random.choice(list(usernames)) invalid_password = 'InvalidPassword' - session = BlackboardSession(username=username, password=invalid_password) + session = BlackboardSession( + username=username, password=invalid_password) # Execute login - response = session.login() + session.login() + + response = session.get_response() # Check the response error_messages = [ @@ -55,21 +64,129 @@ def test_failed_login_invalid_password(self): self.assertTrue(response in error_messages) - def test_failed_login_invalid_username(self): # Set up invalid_username = 'InvalidUsername' password = 'InvalidPassword' - session = BlackboardSession(username=invalid_username, password=password) + session = BlackboardSession( + username=invalid_username, password=password) # Execute login - response = session.login() + session.login() + + response = session.get_response() # Check the response expected_error_message = "The username you entered cannot be identified." self.assertEqual(response, expected_error_message) + # * Enable Instructors *# + + def test_enable_instructors_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the GET request + with patch.object(session, '_get_request') as mock_get_request: + mock_get_request.return_value.status_code = 200 + mock_get_request.return_value.content = ''' + +
+ +
+ + ''' + + # Mock the POST request + with patch.object(session, '_send_post_request') as mock_post_request: + mock_post_request.return_value.status_code = 302 + mock_post_request.return_value.headers = { + 'Location': 'https://kettering.blackboard.com'} + + # Execute enable_instructors + session.enable_instructors() + + # Check the response + self.assertTrue(session.instructorsFound) + self.assertAlmostEqual( + session.last_activity_time, time.time(), delta=1) + + def test_enable_instructors_not_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = False + + # Execute enable_instructors + session.enable_instructors() + + # Check the response + self.assertEqual(session.response, "Not logged in.") + self.assertFalse(session.instructorsFound) + self.assertIsNone(session.last_activity_time) + + def test_enable_instructors_get_request_failed(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the GET request + with patch.object(session, '_get_request') as mock_get_request: + mock_get_request.return_value.status_code = 500 + + # Mock the logging.error function + with patch('logging.error') as mock_logging_error: + # Execute enable_instructors + session.enable_instructors() + + # Check the response + self.assertFalse(session.instructorsFound) + + # Check the logging.error call + mock_logging_error.assert_called_once_with( + f"GET request failed with status code: {mock_get_request.return_value.status_code}") + + def test_enable_instructors_post_request_failed(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the GET request + with patch.object(session, '_get_request') as mock_get_request: + mock_get_request.return_value.status_code = 200 + mock_get_request.return_value.content = ''' + +
+ +
+ + ''' + + # Mock the POST request + with patch.object(session, '_send_post_request') as mock_post_request: + mock_post_request.return_value.status_code = 500 + + # Mock the logging.error function + with patch('logging.error') as mock_logging_error: + # Execute enable_instructors + session.enable_instructors() + + # Check the response + self.assertFalse(session.instructorsFound) + + # Check the logging.error call + mock_logging_error.assert_called_once_with( + f"POST request failed with status code: {mock_post_request.return_value.status_code}") + if __name__ == '__main__': unittest.main() From 8ef89eb1d1511d00347be670aa52c737031f89ba Mon Sep 17 00:00:00 2001 From: Jaydin_MacBook Date: Sat, 6 Jan 2024 12:03:09 -0500 Subject: [PATCH 2/4] Get Instructors & Get courses tests --- backend/blackboard_scraper.py | 15 +++-- backend/test_blackboard_scraper.py | 105 ++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/backend/blackboard_scraper.py b/backend/blackboard_scraper.py index b8a2fdf..81d4357 100644 --- a/backend/blackboard_scraper.py +++ b/backend/blackboard_scraper.py @@ -84,6 +84,12 @@ def set_response(self, response): def get_response(self): return self.response + + def get_InstructorsFound(self): + return self.instructorsFound + + def set_InstructorsFound(self, instructorsFound): + self.instructorsFound = instructorsFound def shutdown(self): """ @@ -159,7 +165,7 @@ def scrape(self): return self.enable_instructors() - if self.instructorsFound == False: + if self.get_InstructorsFound() == False: self.response = "No instructors found." self.get_courses() @@ -247,9 +253,9 @@ def enable_instructors(self): redirected_url = enable_instructors_response.headers['Location'] logging.info( f"Successful POST request. Redirected to: {redirected_url}") - self.instructorsFound = True + self.set_InstructorsFound(True) else: - self.instructorsFound = False + self.set_InstructorsFound(False) logging.error( f"POST request failed with status code: {enable_instructors_response.status_code}") @@ -310,7 +316,7 @@ def get_courses(self): hrefs = {course.text.strip(): course.find("a")["href"].strip() for course in courses_list if course.find("a") and course.find("a").get("href")} - if self.instructorFound: + if self.get_InstructorsFound() == True: # Process instructors and format course names for course in courses_list: try: @@ -343,6 +349,7 @@ def get_courses(self): except Exception as e: self.courseFound = False + self.response = e logging.error(f"An error occurred while getting courses: {e}") def download_and_save_file(self): diff --git a/backend/test_blackboard_scraper.py b/backend/test_blackboard_scraper.py index 64e850b..c14b6f2 100644 --- a/backend/test_blackboard_scraper.py +++ b/backend/test_blackboard_scraper.py @@ -1,9 +1,8 @@ -import logging import random import time import unittest from blackboard_scraper import BlackboardSession -from unittest.mock import MagicMock, patch +from unittest.mock import patch from usernames import usernames @@ -43,6 +42,7 @@ def test_invalid_both_login(self): self.assertEqual(response, expected_error_message) + """ def test_failed_login_invalid_password(self): # selected a random username from usernames.py @@ -62,7 +62,8 @@ def test_failed_login_invalid_password(self): "The password you entered was incorrect.", "Account has been disabled." ] - self.assertTrue(response in error_messages) + self.assertTrue(response in error_messages) + """ def test_failed_login_invalid_username(self): # Set up @@ -187,6 +188,104 @@ def test_enable_instructors_post_request_failed(self): mock_logging_error.assert_called_once_with( f"POST request failed with status code: {mock_post_request.return_value.status_code}") + def test_get_courses_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the POST request + with patch.object(session, '_send_post_request') as mock_post_request: + mock_post_request.return_value.status_code = 200 + mock_post_request.return_value.content = ''' + +
+ +
+ + ''' + + # Execute get_courses + session.get_courses() + + # Check the response + expected_courses = { + 'Course 1': 'course1_link', + 'Course 2': 'course2_link' + } + self.assertEqual(session.courses, expected_courses) + self.assertAlmostEqual( + session.last_activity_time, time.time(), delta=1) + + def test_get_courses_not_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = False + + # Execute get_courses + session.get_courses() + + # Check the response + self.assertEqual(session.response, "Not logged in.") + self.assertEqual(session.courses, {}) + self.assertIsNone(session.last_activity_time) + + def test_get_courses_no_courses(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the POST request + with patch.object(session, '_send_post_request') as mock_post_request: + mock_post_request.return_value.status_code = 200 + mock_post_request.return_value.content = ''' + +
+
    +
  • You are not currently enrolled in any courses.
    • +
+
+ + ''' + + # Execute get_courses + session.get_courses() + + # Check the response + self.assertEqual(session.response, + "You are not currently enrolled in any courses.") + self.assertEqual(session.courses, {}) + + def test_get_courses_error_finding_course_list(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + + # Mock the POST request + with patch.object(session, '_send_post_request') as mock_post_request: + mock_post_request.return_value.status_code = 500 + + # Mock the logging.error function + with patch('logging.error') as mock_logging_error: + # Execute get_courses + session.get_courses() + + # Check the response + self.assertIsInstance(session.response, Exception) + self.assertEqual(str(session.response), 'POST request failed.') + self.assertEqual(session.courses, {}) + mock_logging_error.assert_called_once() + if __name__ == '__main__': unittest.main() From 05c56f9bd038aa3a3c29db3b402a24001d886a0d Mon Sep 17 00:00:00 2001 From: Jaydin_MacBook Date: Sun, 7 Jan 2024 14:00:27 -0500 Subject: [PATCH 3/4] Added get download tasks tests --- backend/test_blackboard_scraper.py | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/backend/test_blackboard_scraper.py b/backend/test_blackboard_scraper.py index c14b6f2..dca2acc 100644 --- a/backend/test_blackboard_scraper.py +++ b/backend/test_blackboard_scraper.py @@ -188,6 +188,8 @@ def test_enable_instructors_post_request_failed(self): mock_logging_error.assert_called_once_with( f"POST request failed with status code: {mock_post_request.return_value.status_code}") + # * Get Courses *# + def test_get_courses_logged_in(self): # Set up username = 'Free8864' @@ -286,6 +288,99 @@ def test_get_courses_error_finding_course_list(self): self.assertEqual(session.courses, {}) mock_logging_error.assert_called_once() + # * Get Download Tasks *# + +def test_get_download_tasks_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = True + session.courses = { + 'Course 1': 'course1_link', + 'Course 2': 'course2_link' + } + + with patch.object(session, '_get_request') as mock_get_request: + mock_get_request.side_effect = [ + type('', (), {'status_code': 200, 'content': ''' + + +
+ +
+ + + '''}), + type('', (), {'status_code': 200, 'content': ''' + + +
+ +
+ + + '''}) + ] + + # Call the method + session.get_download_tasks() + + # Assert the result + expected_result = [ + ('Course 1', 'Assignment 1', 'download_link1'), + ('Course 1', 'Assignment 2', 'download_link2'), + ('Course 2', 'Assignment 1', 'download_link1'), + ('Course 2', 'Assignment 2', 'download_link2'), + ] + + self.assertEqual(session.download_tasks, expected_result) + self.assertTrue(session.downloadTasksFound) + self.assertAlmostEqual( + session.last_activity_time, time.time(), delta=1) + + + def test_get_download_tasks_not_logged_in(self): + # Set up + username = 'Free8864' + password = '#CFi^F6TTwot2j' + session = BlackboardSession(username=username, password=password) + session.is_logged_in = False + + # Execute get_download_tasks + session.get_download_tasks() + + # Check the response + self.assertEqual(session.response, "Not logged in.") + self.assertFalse(session.downloadTasksFound) + self.assertIsNone(session.last_activity_time) + if __name__ == '__main__': unittest.main() From 6674ca65d24ff2d851d49c081eab6edf3b2bc67f Mon Sep 17 00:00:00 2001 From: Jaydin_MacBook Date: Sun, 7 Jan 2024 15:35:18 -0500 Subject: [PATCH 4/4] Remove ray library and uncommented code --- backend/requirements.txt | 1 - backend/test.py | 1 - backend/test_blackboard_scraper.py | 6 ++---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 3c16ea3..cd7607b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,7 +1,6 @@ ghostscript selenium flask -ray argparse bs4 gunicorn diff --git a/backend/test.py b/backend/test.py index 007677f..cbf21c9 100644 --- a/backend/test.py +++ b/backend/test.py @@ -1,5 +1,4 @@ import logging -import ray from backend.blackboard_scraper_S import ( download_and_zip_content, log_into_blackboard, diff --git a/backend/test_blackboard_scraper.py b/backend/test_blackboard_scraper.py index dca2acc..b86ab67 100644 --- a/backend/test_blackboard_scraper.py +++ b/backend/test_blackboard_scraper.py @@ -42,7 +42,6 @@ def test_invalid_both_login(self): self.assertEqual(response, expected_error_message) - """ def test_failed_login_invalid_password(self): # selected a random username from usernames.py @@ -62,8 +61,7 @@ def test_failed_login_invalid_password(self): "The password you entered was incorrect.", "Account has been disabled." ] - self.assertTrue(response in error_messages) - """ + self.assertTrue(response in error_messages) def test_failed_login_invalid_username(self): # Set up @@ -290,6 +288,7 @@ def test_get_courses_error_finding_course_list(self): # * Get Download Tasks *# + def test_get_download_tasks_logged_in(self): # Set up username = 'Free8864' @@ -365,7 +364,6 @@ def test_get_download_tasks_logged_in(self): self.assertAlmostEqual( session.last_activity_time, time.time(), delta=1) - def test_get_download_tasks_not_logged_in(self): # Set up username = 'Free8864'