Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create Unit-Tests.yml #23

Merged
merged 4 commits into from
Jan 7, 2024
Merged

Create Unit-Tests.yml #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9c15375
Create Unit-Tests.yml
TheManWhoLikesToCode Jan 7, 2024
7ab6ce4
Update requirements.txt
TheManWhoLikesToCode Jan 7, 2024
9c37376
Update test_blackboard_scraper.py
TheManWhoLikesToCode Jan 7, 2024
dd0a15d
Update blackboard_scraper.py
TheManWhoLikesToCode Jan 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/workflows/Unit-Tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: Run Blackboard Session Tests

on:
push:
branches:
- main
pull_request:
branches:
- main

jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2

- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.x'

- name: Install backend dependencies
run: |
python -m pip install --upgrade pip
pip install -r backend/requirements.txt

- name: Run tests
run: |
python backend/test_blackboard_scraper.py
146 changes: 78 additions & 68 deletions backend/blackboard_scraper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ def set_response(self, response):

def get_response(self):
return self.response

def get_InstructorsFound(self):
return self.instructorsFound

def set_InstructorsFound(self, instructorsFound):
self.instructorsFound = instructorsFound

def shutdown(self):
"""
Expand Down Expand Up @@ -159,7 +165,7 @@ def scrape(self):
return

self.enable_instructors()
if self.instructorsFound == False:
if self.get_InstructorsFound() == False:
self.response = "No instructors found."

self.get_courses()
Expand Down Expand Up @@ -189,70 +195,76 @@ def enable_instructors(self):

try:
get_url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=edit_module%2F_4_1%2Fbbcourseorg%3Fcmd%3Dedit&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1"
get_response = self._get_request(get_url)

if get_response.status_code != 200:
raise Exception("GET request failed.")

# Using beautiful soup get the value from this input #moduleEditForm > input[type=hidden]:nth-child(1)
soup = BeautifulSoup(get_response.content, "html.parser")
nonce_value = soup.select_one(
'#moduleEditForm > input[type=hidden]:nth-child(1)')['value']

url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=proc_edit/_4_1/bbcourseorg&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1"
payload = {
'tab_tab_group_id': '_1_1',
'forwardUrl': 'proc_edit/_4_1/bbcourseorg',
'blackboard.platform.security.NonceUtil.nonce': nonce_value,
'recallUrl': '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1',
'cmd': 'processEdit',
'serviceLevel': '',
'termDisplayOrder': '_254_1',
'amc.groupbyterm': 'true',
'selectAll_254_1': 'true',
'amc.showterm._254_1': 'true',
'termCourses__254_1': 'true',
'amc.showcourse._51671_1': 'true',
'amc.showcourseid._51671_1': 'true',
'amc.showinstructors._51671_1': 'true',
'amc.showcourse._51672_1': 'true',
'amc.showcourseid._51672_1': 'true',
'amc.showinstructors._51672_1': 'true',
'amc.showcourse._51629_1': 'true',
'amc.showcourseid._51629_1': 'true',
'amc.showinstructors._51629_1': 'true',
'amc.showcourse._51904_1': 'true',
'amc.showcourseid._51904_1': 'true',
'amc.showinstructors._51904_1': 'true',
'amc.showcourse._51945_1': 'true',
'amc.showcourseid._51945_1': 'true',
'amc.showinstructors._51945_1': 'true',
'amc.url.name.1': '',
'amc.url.url.1': '',
'amc.url.name.2': '',
'amc.url.url.2': '',
'amc.url.name.3': '',
'amc.url.url.3': '',
'amc.url.name.4': '',
'amc.url.url.4': '',
'amc.url.name.5': '',
'amc.url.url.5': '',
'bottom_Submit': 'Submit'
}
enable_instructors_response = self._send_post_request(
url, data=payload, allow_redirects=False)

if enable_instructors_response.status_code == 302:
redirected_url = enable_instructors_response.headers['Location']
logging.info(
f"Successful POST request. Redirected to: {redirected_url}")
self.instructorsFound = True
else:
self.instructorsFound = False
try:
get_response = self._get_request(get_url)

if get_response.status_code != 200:
raise Exception("GET request failed.")

# Using beautiful soup get the value from this input #moduleEditForm > input[type=hidden]:nth-child(1)
soup = BeautifulSoup(get_response.content, "html.parser")
nonce_value = soup.select_one(
'#moduleEditForm > input[type=hidden]:nth-child(1)')['value']

url = "https://kettering.blackboard.com/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1&forwardUrl=proc_edit/_4_1/bbcourseorg&recallUrl=%2Fwebapps%2Fportal%2Fexecute%2Ftabs%2FtabAction%3Ftab_tab_group_id%3D_1_1"
payload = {
'tab_tab_group_id': '_1_1',
'forwardUrl': 'proc_edit/_4_1/bbcourseorg',
'blackboard.platform.security.NonceUtil.nonce': nonce_value,
'recallUrl': '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_1_1',
'cmd': 'processEdit',
'serviceLevel': '',
'termDisplayOrder': '_254_1',
'amc.groupbyterm': 'true',
'selectAll_254_1': 'true',
'amc.showterm._254_1': 'true',
'termCourses__254_1': 'true',
'amc.showcourse._51671_1': 'true',
'amc.showcourseid._51671_1': 'true',
'amc.showinstructors._51671_1': 'true',
'amc.showcourse._51672_1': 'true',
'amc.showcourseid._51672_1': 'true',
'amc.showinstructors._51672_1': 'true',
'amc.showcourse._51629_1': 'true',
'amc.showcourseid._51629_1': 'true',
'amc.showinstructors._51629_1': 'true',
'amc.showcourse._51904_1': 'true',
'amc.showcourseid._51904_1': 'true',
'amc.showinstructors._51904_1': 'true',
'amc.showcourse._51945_1': 'true',
'amc.showcourseid._51945_1': 'true',
'amc.showinstructors._51945_1': 'true',
'amc.url.name.1': '',
'amc.url.url.1': '',
'amc.url.name.2': '',
'amc.url.url.2': '',
'amc.url.name.3': '',
'amc.url.url.3': '',
'amc.url.name.4': '',
'amc.url.url.4': '',
'amc.url.name.5': '',
'amc.url.url.5': '',
'bottom_Submit': 'Submit'
}
enable_instructors_response = self._send_post_request(
url, data=payload, allow_redirects=False)

if enable_instructors_response.status_code == 302:
redirected_url = enable_instructors_response.headers['Location']
logging.info(
f"Successful POST request. Redirected to: {redirected_url}")
self.set_InstructorsFound(True)
else:
self.set_InstructorsFound(False)
logging.error(
f"POST request failed with status code: {enable_instructors_response.status_code}")

self.last_activity_time = time.time()

except Exception as e:
logging.error(
f"POST request failed with status code: {enable_instructors_response.status_code}")

self.last_activity_time = time.time()
f"GET request failed with status code: {get_response.status_code}")
return

except Exception as e:
logging.error(f"An error occurred enabling instructors: {e}")
Expand Down Expand Up @@ -304,7 +316,7 @@ def get_courses(self):
hrefs = {course.text.strip(): course.find("a")["href"].strip()
for course in courses_list if course.find("a") and course.find("a").get("href")}

if self.instructorFound:
if self.get_InstructorsFound() == True:
# Process instructors and format course names
for course in courses_list:
try:
Expand Down Expand Up @@ -337,6 +349,7 @@ def get_courses(self):

except Exception as e:
self.courseFound = False
self.response = e
logging.error(f"An error occurred while getting courses: {e}")

def download_and_save_file(self):
Expand Down Expand Up @@ -470,6 +483,3 @@ def process_course(course, href):
self.download_tasks = download_tasks
self.downloadTasksFound = True
self.last_activity_time = time.time()



3 changes: 1 addition & 2 deletions backend/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
ghostscript
selenium
flask
ray
argparse
bs4
gunicorn
flask_cors
flask_apscheduler
pydrive2
pydrive2
Loading
Loading