From 90944519271a22fdd0a3373abc13e313dd54ef2c Mon Sep 17 00:00:00 2001 From: alisonmyers Date: Tue, 29 Sep 2020 17:08:00 -0700 Subject: [PATCH] added pagination to student call --- extend_quiz.py | 27 +++++++++++++++++---------- src/helpers.py | 18 +++++++++++++++--- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/extend_quiz.py b/extend_quiz.py index b255bfe..eed7da5 100644 --- a/extend_quiz.py +++ b/extend_quiz.py @@ -68,17 +68,24 @@ def dl_students(course_id, AUTH_HEADER, API_URL): None (creates csv from data) ''' student_list = _get_students(course_id, AUTH_HEADER, API_URL) - + #print(student_list) # Default for extra_time and extra_attempts is null as requested - df = pd.DataFrame(columns=['name','SIS_id','canvas_id','extra_time','extra_attempts']) - for student in student_list: - df = df.append({ - 'name':student['name'], - 'SIS_id': student['sis_user_id'], - 'canvas_id': student['id'], - 'extra_time': None, - 'extra_attempts': None - }, ignore_index=True) + df = pd.DataFrame(student_list) + df = df[['name','sis_user_id','id']] + df['extra_time'] = '' + df['extra_attempts'] ='' + df = df.rename(columns={"sis_user_id": "SIS_id", "id":"canvas_id"}) + df = df[['name','SIS_id','canvas_id','extra_time','extra_attempts']] + #print(df) + # df = pd.DataFrame(columns=['name','SIS_id','canvas_id','extra_time','extra_attempts']) + # for student in student_list: + # df = df.append({ + # 'name':student['name'], + # 'SIS_id': student['sis_user_id'], + # 'canvas_id': student['id'], + # 'extra_time': None, + # 'extra_attempts': None + # }, ignore_index=True) path = os.path.join(INPUT, 'input-all_course_students.csv') df.to_csv(path, index=False) diff --git a/src/helpers.py b/src/helpers.py index 8eb9ed3..0eaf308 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -60,9 +60,20 @@ def _get_quiz(course_obj, quiz_id): except Exception as qe: print_error(f'ERROR: Could not find quiz [ID: {quiz_id}]. Please check course id.') raise + +def _paginate_list(sub_list, auth_header): + json_list = pd.read_json(sub_list.text) + + while sub_list.links['current']['url'] != sub_list.links['last']['url']: + sub_list = requests.get(sub_list.links['next']['url'], + headers=auth_header) + admin_sub_table = pd.read_json(sub_list.text) + json_list= pd.concat([json_list, admin_sub_table], sort=True) + json_list=json_list.reset_index(drop=True) + return json_list -def _get_students(course_id, auth_header, API_URL): +def _get_students(course_id, AUTH_HEADER, API_URL): ''' Function gets a list of students for a course Parameters: @@ -80,8 +91,9 @@ def _get_students(course_id, auth_header, API_URL): # Above code doesn't return student SIS ID, forced to use request lib temporarily try: url = "{}api/v1/courses/{}/users".format(API_URL, course_id) - student_list = requests.get(url, headers=auth_header, params={'enrollment_type[]':'student'}) - student_list = json.loads(student_list.text) + student_list = requests.get(url, headers=AUTH_HEADER, params={'enrollment_type[]':'student', 'per_page':50}) + student_list = _paginate_list(student_list, AUTH_HEADER) + #student_list = json.loads(student_list.text) return(student_list) except Exception as se: # print(str(se))