Initial commit

.gitignore

@@ -0,0 +1 @@
+node_modules

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "tau",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/lodash": "^4.14.136",
+    "@types/node": "^12.6.9",
+    "@types/node-fetch": "^2.5.0",
+    "@types/url-join": "^4.0.0",
+    "lodash": "^4.17.15",
+    "node-fetch": "^2.6.0",
+    "ts-node": "^8.3.0",
+    "typescript": "^3.5.3",
+    "url-join": "^4.0.1"
+  }
+}

src/course.ts

@@ -0,0 +1,47 @@
+export type Course = {
+    id: string,
+    chaptersCount: number,
+    status?: string,
+    credits: number,
+    courseId: string,
+    category: string,
+    groupName: string,
+    title: string,
+    titleSlug: string,
+    teacher: {
+        twitter: string,
+        name: string,
+        photoURL: string,
+        profilePath: string
+    },
+    level: string,
+    type: string,
+    abstract: string,
+    sortOrder: number,
+    group: string,
+    releaseDate: string
+    chapters?: Chapter[]
+}
+
+export type Chapter = {
+    chapterId: string,
+    questions?: Question[]
+}
+
+export type Question = {
+    answers: string[],
+    id: string,
+    question: string,
+    type: string,
+    correctAnswerIndex?: number
+}
+
+export type SubmitQuestionsResult = {
+    attempts: number,
+    chapterId: string,
+    courseId: string,
+    credits: number,
+    failed: string[],
+    passed: string[],
+    result: boolean
+}

src/scraper.ts

@@ -0,0 +1,107 @@
+import fetch from "node-fetch";
+import urljoin from "url-join";
+import { Course, Question, SubmitQuestionsResult, Chapter } from "./course";
+import * as fs from "fs";
+import _, { concat } from "lodash";
+import { authenticate } from "./utils"
+
+const EMAIL = "";
+const PASSWORD = "";
+
+(async () => {
+    const { baseUrl, token } = await authenticate(EMAIL, PASSWORD);
+
+    const courses = await getLiveCourses(baseUrl);
+
+    const coursesWithAnswers = await Promise.all(courses.map(async course => {
+        const courseWithQuesions = await getChaptersAndQuestions(baseUrl, course);
+        const courseWithAnswers = await getAnswers(baseUrl, token, courseWithQuesions);
+        return courseWithAnswers;
+    }));
+    // for await (const course of courses) {
+    //     const courseWithQuesions = await getChaptersAndQuestions(baseUrl, course);
+    //     const courseWithAnswers = await getAnswers(baseUrl, token, courseWithQuesions);
+    // }
+    fs.writeFileSync("courses.json", JSON.stringify(coursesWithAnswers));
+})();
+
+async function getLiveCourses(baseUrl: string) {
+    const endpoint = urljoin(baseUrl, "getCourses");
+    const response = await fetch(endpoint);
+    const courses = JSON.parse(await response.text()) as Course[];
+    const liveCourses = courses.filter(course => course.status && course.status == "live");
+
+    if (!liveCourses) {
+        new Error("No live courses found");
+    }
+
+    return liveCourses;
+}
+
+async function getChaptersAndQuestions(baseUrl: string, course: Course) {
+    const endpoint = urljoin("https://testautomationu.applitools.com", course.titleSlug);
+    const response = await fetch(endpoint);
+    const html = await response.text();
+    const matches1 = html.match(/(?<=>Chapter ).+?(?=\ |\.)/g);
+    const matches2 = html.match(/(?<=>Chapter ).+?(?=[a-z]|\ |\.)/g);
+
+    if (matches1 && matches2) {
+        const matches = matches1.concat(matches2);
+        if (matches.length > 0) {
+            const chapters = await Promise.all(_.uniq(matches).map(async match => {
+                const questions = await getQuestionsForChapter(baseUrl, course, `chapter${match}`);
+                if (questions.length > 0) {
+                    return { chapterId: `chapter${match}`, questions } as Chapter;
+                }
+            }));
+            const filteredChapters = chapters.filter(chapter => chapter != null) as Chapter[];
+            if (filteredChapters && filteredChapters.length > 0) {
+                course.chapters = filteredChapters;
+            }
+        }
+    } else {
+        throw Error(`Unable to get chapters from the ${course.courseId} introduction page.`);
+    }
+    return course;
+}
+
+async function getAnswers(baseUrl: string, token: string, course: Course) {
+    if (course.chapters) {
+        for await (const chapter of course.chapters) {
+            if (chapter.questions) {
+                for await (const question of chapter.questions) {
+                    console.log(`Scraping: ${course.courseId}\t${chapter.chapterId}\t${question.id}`);
+                    const answerIndex = await getAnswerForQuestion(baseUrl, token, course, chapter.chapterId, question);
+                    question.correctAnswerIndex = answerIndex;
+                }
+            }
+        }
+    }
+
+    return course;
+}
+
+async function getQuestionsForChapter(baseUrl: string, course: Course, chapter: string) {
+    const endpoint = urljoin(baseUrl, "quizzes", course.courseId, chapter);
+    const response = await fetch(endpoint);
+    return JSON.parse(await response.text()) as Question[]
+}
+
+async function getAnswerForQuestion(baseUrl: string, token: string, course: Course, chapter: string, question: Question) {
+    const endpoint = urljoin(baseUrl, "validateQuizzes", course.courseId, chapter)
+
+    for (let counter = 0; counter < question.answers.length; counter++) {
+        const response = await fetch(endpoint, {
+            method: "POST", headers: {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${token}`
+            },
+            body: JSON.stringify({ [question.id]: counter })
+        });
+        const json = JSON.parse(await response.text()) as SubmitQuestionsResult;
+        if (json.passed.includes(question.id)) {
+            return counter;
+        }
+    }
+}

src/submitter.ts

@@ -0,0 +1,67 @@
+import { authenticate } from "./utils";
+import * as fs from "fs";
+import { Course, SubmitQuestionsResult } from "./course";
+import urljoin from "url-join";
+import fetch from "node-fetch";
+
+const EMAIL = "";
+const PASSWORD = "";
+
+(async () => {
+    const { baseUrl, token } = await authenticate(EMAIL, PASSWORD);
+    const courses = JSON.parse(fs.readFileSync("courses.json", { encoding: "utf-8" })) as Course[];
+
+    // await Promise.all(courses.map(async course => {
+    //     await submitQuestions(baseUrl, token, course);
+    //     await generateCertificate(baseUrl, token, course);
+    // }));
+
+    for await(const course of courses) {
+        await submitQuestions(baseUrl, token, course);
+        await generateCertificate(baseUrl, token, course);
+    }
+})()
+
+async function submitQuestions(baseUrl: string, token: string, course: Course) {
+    if (course.chapters) {
+        for await (const chapter of course.chapters) {
+            if (chapter.questions && chapter.questions.length > 0) {
+                let answers= {};
+
+                for await (const question of chapter.questions) {
+                    if (question.correctAnswerIndex !== undefined) {
+                        Object.assign(answers, {[question.id]: question.correctAnswerIndex.toString()})
+                    }
+                }
+
+                const endpoint = urljoin(baseUrl, "validateQuizzes", course.courseId, chapter.chapterId)
+                console.log(`Submitting ${course.courseId}\t${chapter.chapterId} with\t${JSON.stringify(answers)}`)
+                const response = await fetch(endpoint, {
+                    method: "POST", headers: {
+                        'Accept': 'application/json',
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${token}`
+                    },
+                    body: JSON.stringify(answers)
+                });
+                const json = JSON.parse(await response.text()) as SubmitQuestionsResult;
+                if(!json.result) {
+                    throw Error(`Something went wrong whils submitting questions for ${course.courseId} ${chapter.chapterId}.\n${json}`)
+                }
+            }
+        }
+    }
+}
+
+async function generateCertificate(baseUrl: string, token: string, course: Course) {
+    const endpoint = urljoin(baseUrl, "generateUploadCertificate");
+    console.log(`Generating certificate for ${course.courseId}`)
+    await fetch(endpoint, {
+        method: "POST", headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${token}`
+        },
+        body: JSON.stringify({"courseId": course.courseId})
+    });
+}

src/utils.ts

@@ -0,0 +1,24 @@
+import fetch from "node-fetch";
+
+async function getTAUInfo() {
+    const response = await fetch("https://testautomationu.applitools.com");
+    const html = await response.text();
+    const baseUrlMatch = html.match(/(?<=let serverURL = ").*(?=")/);
+    const apiKeyMatch = html.match(/(?<=apiKey: ").*(?=")/);
+    if (!baseUrlMatch || !apiKeyMatch) {
+        throw Error("Unable to find required site info.");
+    }
+    return { baseUrl: baseUrlMatch[0], apiKey: apiKeyMatch[0] };
+}
+
+async function getBearerToken(apiKey: string, email:string, password: string) {
+    const response = await fetch(`https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key=${apiKey}`,
+        { method: "POST", body: JSON.stringify({ email, password, returnSecureToken: true }) });
+    return JSON.parse(await response.text()).idToken;
+}
+
+export async function authenticate(email:string, password: string) {
+    const { apiKey, baseUrl } = await getTAUInfo();
+    const token = await getBearerToken(apiKey, email, password);
+    return { baseUrl, token };
+}