From 9d01d13eaa0494a1b4edd81fc0a86ce088a69d7f Mon Sep 17 00:00:00 2001 From: soumitradev Date: Mon, 7 Aug 2023 21:30:00 +0530 Subject: [PATCH] feat: temporarily rewrite how ingestion overwriting is handled --- src/ingestJSON.ts | 358 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 282 insertions(+), 76 deletions(-) diff --git a/src/ingestJSON.ts b/src/ingestJSON.ts index 0d8648e2..bdd15df9 100644 --- a/src/ingestJSON.ts +++ b/src/ingestJSON.ts @@ -101,40 +101,238 @@ export const ingestJSON = async ( latestCourse.semester == semester && forceOverwrite ) { - console.log( - `WARNING: overwriting will wipe all courses, sections, and timetables for that combination of academic year and semester` - ); - console.log( - `deleting courses from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}...` - ); - const deletedCourseResult = await queryRunner.manager - .createQueryBuilder() - .delete() - .from(Course) - .where("acad_year = :year", { year: latestCourse.acadYear }) - .andWhere("semester = :semester", { - semester: latestCourse.semester, + // console.log( + // `WARNING: overwriting will wipe all courses, sections, and timetables for that combination of academic year and semester` + // ); + // console.log( + // `deleting courses from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}...` + // ); + // const deletedCourseResult = await queryRunner.manager + // .createQueryBuilder() + // .delete() + // .from(Course) + // .where("acad_year = :year", { year: latestCourse.acadYear }) + // .andWhere("semester = :semester", { + // semester: latestCourse.semester, + // }) + // .execute(); + // console.log( + // `deleted ${deletedCourseResult.affected} courses from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}!` + // ); + // console.log( + // `deleting timetables from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}...` + // ); + // const deletedTimetableResult = await queryRunner.manager + // .createQueryBuilder() + // .delete() + // .from(Timetable) + // .where("acad_year = :year", { year: latestCourse.acadYear }) + // .andWhere("semester = :semester", { + // semester: latestCourse.semester, + // }) + // .execute(); + // console.log( + // `deleted ${deletedTimetableResult.affected} timetables from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}!` + // ); + const courses = timetableJSON.courses as CoursesJSON; + + console.log("constructing courses..."); + const courseValues = await Promise.all( + Object.keys(courses).map(async (courseCode) => { + const midsemTimes = courses[courseCode].exams_iso[0].midsem; + const compreTimes = courses[courseCode].exams_iso[0].compre; + const existingCourseID = await queryRunner.manager + .createQueryBuilder() + .select() + .from(Course, "course") + .where("course.code = :code", { code: courseCode }) + .execute(); + if (existingCourseID[0] !== undefined) { + return { + id: existingCourseID[0].id, + code: courseCode, + name: courses[courseCode].course_name, + acadYear: year, + semester: semester, + archived: false, + midsemStartTime: midsemTimes + ? new Date(midsemTimes.split("|")[0]) + : null, + midsemEndTime: midsemTimes + ? new Date(midsemTimes.split("|")[1]) + : null, + compreStartTime: compreTimes + ? new Date(compreTimes.split("|")[0]) + : null, + compreEndTime: compreTimes + ? new Date(compreTimes.split("|")[1]) + : null, + } as QueryDeepPartialEntity; + } else { + const courseInsertResult = await queryRunner.manager + .createQueryBuilder() + .insert() + .into(Course) + .values({ + code: courseCode, + name: courses[courseCode].course_name, + acadYear: year, + semester: semester, + archived: false, + midsemStartTime: midsemTimes + ? new Date(midsemTimes.split("|")[0]) + : null, + midsemEndTime: midsemTimes + ? new Date(midsemTimes.split("|")[1]) + : null, + compreStartTime: compreTimes + ? new Date(compreTimes.split("|")[0]) + : null, + compreEndTime: compreTimes + ? new Date(compreTimes.split("|")[1]) + : null, + } as QueryDeepPartialEntity) + .execute(); + console.log("INSERTED " + courseCode); + return { + id: courseInsertResult.identifiers[0].id, + code: courseCode, + name: courses[courseCode].course_name, + acadYear: year, + semester: semester, + archived: false, + midsemStartTime: midsemTimes + ? new Date(midsemTimes.split("|")[0]) + : null, + midsemEndTime: midsemTimes + ? new Date(midsemTimes.split("|")[1]) + : null, + compreStartTime: compreTimes + ? new Date(compreTimes.split("|")[0]) + : null, + compreEndTime: compreTimes + ? new Date(compreTimes.split("|")[1]) + : null, + } as QueryDeepPartialEntity; + } }) - .execute(); - console.log( - `deleted ${deletedCourseResult.affected} courses from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}!` ); - console.log( - `deleting timetables from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}...` - ); - const deletedTimetableResult = await queryRunner.manager - .createQueryBuilder() - .delete() - .from(Timetable) - .where("acad_year = :year", { year: latestCourse.acadYear }) - .andWhere("semester = :semester", { - semester: latestCourse.semester, - }) - .execute(); - console.log( - `deleted ${deletedTimetableResult.affected} timetables from academic year ${latestCourse.acadYear} and semester ${latestCourse.semester}!` + console.log("updating courses..."); + const courseInsertResult = await queryRunner.manager.query( + `UPDATE public.course AS c SET + code = u.code, + name = u.name, + acad_year = u.acad_year, + semester = u.semester, + archived = u.archived, + midsem_start_time = u.midsem_start_time, + midsem_end_time = u.midsem_end_time, + compre_start_time = u.compre_start_time, + compre_end_time = u.compre_end_time + FROM (VALUES + ${courseValues + .map((c) => { + const escapedName = (c.name as string) + .replace(/\n/g, " ") + .replace(/'/g, "''"); + return `('${c.id}', '${c.code}', '${escapedName}', ${ + c.acadYear + }, ${c.semester}, ${c.archived}, ${ + c.midsemStartTime === null + ? "NULL" + : `to_timestamp(${( + c.midsemStartTime as Date + ).getTime()} / 1000.0)` + },${ + c.midsemEndTime === null + ? "NULL" + : `to_timestamp(${( + c.midsemEndTime as Date + ).getTime()} / 1000.0)` + },${ + c.compreStartTime === null + ? "NULL" + : `to_timestamp(${( + c.compreStartTime as Date + ).getTime()} / 1000.0)` + },${ + c.compreEndTime === null + ? "NULL" + : `to_timestamp(${( + c.compreEndTime as Date + ).getTime()} / 1000.0)` + })`; + }) + .join(", ")} + ) AS u(id, code, name, acad_year, semester, archived, midsem_start_time, midsem_end_time, compre_start_time, compre_end_time) + WHERE u.id = c.id::text;` ); + console.log(courseInsertResult[1] + " courses updated!"); + + // console.log("constructing sections..."); + // // grab the IDs of courses because we will use them for relations + // const courseIDs = courseInsertResult.identifiers.map( + // (idObj) => idObj.id as string + // ); + // for (let i = 0; i < courseValues.length; i++) { + // // for each course, loop over the sections, and add the room-timings to an array + // Object.keys(courses[courseValues[i].code as string].sections).forEach( + // (courseSectionCode) => { + // const roomTimes = []; + // const scheduleObjs = + // courses[courseValues[i].code as string].sections[ + // courseSectionCode + // ].schedule; + // for (let j = 0; j < scheduleObjs.length; j++) { + // for (let k = 0; k < scheduleObjs[j].days.length; k++) { + // for (let l = 0; l < scheduleObjs[j].hours.length; l++) { + // roomTimes.push( + // `${courseValues[i].code}:${scheduleObjs[j].room}:${scheduleObjs[j].days[k]}:${scheduleObjs[j].hours[l]}` + // ); + // } + // } + // } + // // push the section to an array, so we can insert all sections at once + // sectionValues.push({ + // courseId: courseIDs[i], + // instructors: + // courses[courseValues[i].code as string].sections[ + // courseSectionCode + // ].instructor, + // type: courseSectionCode.slice(0, 1), + // number: parseInt(courseSectionCode.slice(1)), + // roomTime: roomTimes, + // } as QueryDeepPartialEntity
); + // } + // ); + // } + + // console.log("inserting sections..."); + // const sectionInsertResult = await queryRunner.manager + // .createQueryBuilder() + // .insert() + // .into(Section) + // .values(sectionValues) + // .execute(); + // console.log("sections inserted!"); + + // await queryRunner.commitTransaction(); + // // show summary of the transaction + // console.log("================= SUMMARY ================="); + // console.log(`courses archived: ${archivedCoursesUpdateCount}`); + // console.log(`timetables archived: ${archivedTimetablesUpdateCount}`); + // console.log( + // `courses inserted: ${courseInsertResult.identifiers.length}` + // ); + // console.log( + // `sections inserted: ${sectionInsertResult.identifiers.length}` + // ); + // const endhrTime = process.hrtime(); + // const endTime = endhrTime[0] * 1000 + endhrTime[1] / 1000000; + // console.log( + // `Ingestion took ${Math.round((endTime - startTime) * 1000) / 1000}ms` + // ); } else { console.log("marking all old courses as archived..."); const archivedCoursesUpdateResult = await queryRunner.manager @@ -162,57 +360,65 @@ export const ingestJSON = async ( archivedTimetablesUpdateCount = archivedTimetablesUpdateResult.affected ?? 0; console.log("marked old timetables as archived!"); - } - console.log("checking if all existing courses are archived..."); - const allCoursesCountResult = await queryRunner.manager - .createQueryBuilder() - .select() - .from(Course, "course") - .getCount(); - const archivedCoursesCountResult = await queryRunner.manager - .createQueryBuilder() - .select() - .from(Course, "course") - .where("archived = :archived", { archived: true }) - .getCount(); - console.log( - `${archivedCoursesCountResult}/${allCoursesCountResult} courses are archived` - ); - if (archivedCoursesCountResult != allCoursesCountResult) { - console.error( - `error: not all courses in db are archived; db state inconsistent` - ); - throw Error( - `error: not all courses in db are archived; db state inconsistent` + console.log("checking if all existing courses are archived..."); + const allCoursesCountResult = await queryRunner.manager + .createQueryBuilder() + .select() + .from(Course, "course") + .getCount(); + const archivedCoursesCountResult = await queryRunner.manager + .createQueryBuilder() + .select() + .from(Course, "course") + .where("archived = :archived", { archived: true }) + .getCount(); + console.log( + `${archivedCoursesCountResult}/${allCoursesCountResult} courses are archived` ); - } - console.log("finished checking courses!"); + if (archivedCoursesCountResult != allCoursesCountResult) { + console.error( + `error: not all courses in db are archived; db state inconsistent` + ); + throw Error( + `error: not all courses in db are archived; db state inconsistent` + ); + } + console.log("finished checking courses!"); - console.log("checking if all existing timetables are archived..."); - const allTimetablesCountResult = await queryRunner.manager - .createQueryBuilder() - .select() - .from(Timetable, "timetable") - .getCount(); - const archivedTimetablesCountResult = await queryRunner.manager - .createQueryBuilder() - .select() - .from(Timetable, "timetable") - .where("archived = :archived", { archived: true }) - .getCount(); - console.log( - `${archivedTimetablesCountResult}/${allTimetablesCountResult} timetables are archived` - ); - if (archivedTimetablesCountResult != allTimetablesCountResult) { - console.error( - `error: not all timetables in db are archived; db state inconsistent` - ); - throw Error( - `error: not all timetables in db are archived; db state inconsistent` + console.log("checking if all existing timetables are archived..."); + const allTimetablesCountResult = await queryRunner.manager + .createQueryBuilder() + .select() + .from(Timetable, "timetable") + .getCount(); + const archivedTimetablesCountResult = await queryRunner.manager + .createQueryBuilder() + .select() + .from(Timetable, "timetable") + .where("archived = :archived", { archived: true }) + .getCount(); + console.log( + `${archivedTimetablesCountResult}/${allTimetablesCountResult} timetables are archived` ); + if (archivedTimetablesCountResult != allTimetablesCountResult) { + console.error( + `error: not all timetables in db are archived; db state inconsistent` + ); + throw Error( + `error: not all timetables in db are archived; db state inconsistent` + ); + } + console.log("finished checking timetables!"); } - console.log("finished checking timetables!"); + } + if ( + latestCourse && + latestCourse.acadYear == year && + latestCourse.semester == semester && + forceOverwrite + ) { + return; } const courses = timetableJSON.courses as CoursesJSON;