From 9b8754c5fc32f167f40cd73daee4cdabdf5db0b2 Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Wed, 6 Mar 2019 17:58:34 +0800 Subject: [PATCH] Replace all Promise with async function, & standardlize codes --- src/builder.ts | 140 ++++++++++++---------------- src/pages/build-with-github-repo.ts | 76 ++++++++------- src/pages/build-with-zip.ts | 8 +- src/pages/check-status.ts | 6 +- src/pages/result.ts | 6 +- src/utils/exec.ts | 43 ++++----- src/utils/queue.ts | 5 - 7 files changed, 124 insertions(+), 160 deletions(-) diff --git a/src/builder.ts b/src/builder.ts index 7add386..dc09ccd 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -5,14 +5,6 @@ import exec, { ExecError } from "./utils/exec"; import { WORKSPACE, TEMP_DIR, BUILDER_CONFIG_NAME, OUTPUT_DIR } from "./config"; import Queue from "./utils/queue"; -class BuildQueue extends Queue { - constructor() { super(); } - public push(jobId: string) { - super.push(jobId); - console.timeLog("Added job(" + jobId + ")"); - Builder.notify(); - } -} export class JobPool { private static pool = new Map(); public static add(job: Job) { @@ -25,26 +17,7 @@ export class JobPool { return JobPool.pool.has(jobId); } } -const buildQueue = new BuildQueue(); - -export function addBuildQueue(job: Job) { - console.timeLog("Job(" + job.id + ") going to be added into build queue"); - // job will be added to build queue by itself after it is ready (in constructor) - job.status = JobStatus.waiting; - buildQueue.push(job.id); -} - -export enum BuildType { - "github-repo" = "github-repo", - "source-upload" = "source-upload" -} -export enum JobStatus { - preparing = "preparing", - waiting = "waiting", - building = "building", - done = "done", - failed = "failed" -} +type JobStatus = "preparing" | "waiting" | "building" | "done" | "failed"; export class Job { private _id: string; private _extraInfo: { [key: string]: string | number } = {}; @@ -55,11 +28,11 @@ export class Job { public status: JobStatus; public constructor() { - // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} + fs.ensureDirSync(TEMP_DIR); let jobDir = fs.mkdtempSync(TEMP_DIR + "/"); this._id = jobDir.substring(jobDir.lastIndexOf("/") + 1); - this._extraInfo.startTimestamp = Date.now(); - this.status = JobStatus.preparing; + this.status = "preparing"; + this.attachInfo("startTimestamp", Date.now()); JobPool.add(this); } @@ -68,64 +41,75 @@ export class Job { } } +class BuildQueue { + private static queue = new Queue(); + public static push(jobId: string) { + BuildQueue.queue.push(jobId); + console.timeLog("Added job(" + jobId + ")"); + Builder.notify(); + } + public static isEmpty() { + return BuildQueue.queue.isEmpty(); + } + public static pop() { + return BuildQueue.queue.pop(); + } +} +export function pushBuildQueue(job: Job) { + job.status = "waiting"; + BuildQueue.push(job.id); +} + class Builder { private static builderAvailable = true; - public static notify() { - if (Builder.builderAvailable && !buildQueue.isEmpty()) { + public static async notify() { + if (Builder.builderAvailable && !BuildQueue.isEmpty()) { Builder.builderAvailable = false; - let jobId = buildQueue.pop(); - JobPool.get(jobId).status = JobStatus.building; - Builder.cleanWorkspace() - .then(() => Builder.buildJob(jobId)); + let jobId = BuildQueue.pop(); + JobPool.get(jobId).status = "building"; + Builder.buildJob(jobId); } } - private static cleanWorkspace() { - return new Promise(resolve => { - exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f") - .then(stdout => { - console.timeLog("Workspace cleaned"); - resolve(); - }); - }) - } private static async buildJob(jobId: string) { + await exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f"); + console.timeLog("Workspace cleaned"); + + let job = JobPool.get(jobId); + console.timeLog("Going to build job(" + jobId + ")"); + let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8")); + let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/"; + fs.ensureDirSync(targetPath); + fs.emptyDirSync(targetPath); + fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath); + console.log("Copied: " + targetPath); + + console.log("Compile started: job(" + jobId + ")"); try { - let job = JobPool.get(jobId); - console.timeLog("Going to build job(" + jobId + ")"); - let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8")); - let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/"; - fs.ensureDirSync(targetPath); - fs.emptyDirSync(targetPath); - fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath); - console.log("Copied: " + targetPath); - console.log("Compile started: job(" + jobId + ")"); - await exec("cd " + WORKSPACE + "/appinventor && ant extensions", true); - - let zip = new AdmZip(); - zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); - zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); - let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; - zip.writeZip(zipPath); - JobPool.get(jobId).status = JobStatus.done; - console.log("Done job(" + jobId + "): " + zipPath); - Builder.builderAvailable = true; - Builder.notify(); - } catch (reason) { - JobPool.get(jobId).status = JobStatus.failed; - if (reason instanceof ExecError) { - let err = reason; - // Notice that it would not work on windows - let stdout = err.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); - let stderr = err.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); - JobPool.get(jobId).attachInfo("failInfo", err.message + ": code(" + err.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr); - } else { - JobPool.get(jobId).attachInfo("failInfo", reason); - } - console.log("Job(" + jobId + ") build failed", reason); + } catch (e) { + e = e; + JobPool.get(jobId).status = "failed"; + // Notice that it would not work on windows + let stdout = e.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + let stderr = e.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + JobPool.get(jobId).attachInfo("failInfo", + e.message + ": code(" + e.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr); + console.log("Job(" + jobId + ") build failed in part of `ant extensions`"); Builder.builderAvailable = true; Builder.notify(); } + + let zip = new AdmZip(); + zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); + zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); + let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; + zip.writeZip(zipPath); + + JobPool.get(jobId).status = "done"; + console.log("Done job(" + jobId + "): " + zipPath); + + Builder.builderAvailable = true; + Builder.notify(); } } \ No newline at end of file diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index c83df77..9ed9100 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -7,7 +7,7 @@ import { URLSearchParams } from "url"; import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config"; import { responseSuccess, responseError } from "../index"; -import { addBuildQueue, Job, JobPool, BuildType, JobStatus } from "../builder"; +import { pushBuildQueue, Job, JobPool } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (request.method == "GET") { @@ -23,8 +23,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS return; } - } else if (request.method == "POST") { - // webhook + } else if (request.method == "POST") { // webhook let indexOfEvent = request.rawHeaders.indexOf("X-GitHub-Event"); let event: string; if (indexOfEvent != -1 && (indexOfEvent + 1) < request.rawHeaders.length) { @@ -68,11 +67,11 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS } } -function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) { +async function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) { let job = new Job(); let jobId = job.id; - job.attachInfo("buildType", BuildType["github-repo"]); + job.attachInfo("buildType", "github-repo"); job.attachInfo("owner", owner); job.attachInfo("repoName", repoName); job.attachInfo("ref", ref); @@ -82,38 +81,37 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin jobId: jobId }); - // Downlaod archieve - new Github().repos.getArchiveLink({ - owner: owner, - repo: repoName, - archive_format: "zipball", - ref: ref - }) - .then(archieveResponse => new Promise((resolve, reject) => { - let zip = new Admzip( archieveResponse.data); - if (zip.getEntries().length == 0) { - reject("No source found in archieve downloaded."); - return; - } - let entryDir = zip.getEntries()[0].entryName; - zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/"); - fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, - TEMP_DIR + "/" + jobId + "/src"); - fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); - console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src"); - addBuildQueue(job); - })) - .catch(reason => { - JobPool.get(jobId).status = JobStatus.failed; - if (Object.getPrototypeOf(reason) == Error.prototype) { - let err = reason; - JobPool.get(jobId).attachInfo("failInfo", - err.name == "HttpError" - ? "Cannot load source from github, please check if the ref exists in the specified repo." - : err.message); - } else { - JobPool.get(jobId).attachInfo("failInfo", reason.toString()); - } - console.log("Fail prepare source of job(" + jobId + ")", reason); - }); + // Downlaod archive + let archiveResponse: Github.Response; + try { + archiveResponse = await new Github().repos.getArchiveLink({ + owner: owner, + repo: repoName, + archive_format: "zipball", + ref: ref + }); + } catch (e) { + let err = e; + JobPool.get(jobId).status = "failed"; + JobPool.get(jobId).attachInfo("failInfo", + err.name == "HttpError" + ? "Cannot load source from github, please check if the ref exists in the specified repo." + : err.message); + console.log("Fail prepare source of job(" + jobId + ")", e); + return; + } + + // Extract responding archive + let zip = new Admzip( archiveResponse.data); + if (zip.getEntries().length == 0) { + throw "No source found in archive downloaded."; + } + let entryDir = zip.getEntries()[0].entryName; + zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/"); + fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, + TEMP_DIR + "/" + jobId + "/src"); + fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); + console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src"); + + pushBuildQueue(job); } \ No newline at end of file diff --git a/src/pages/build-with-zip.ts b/src/pages/build-with-zip.ts index 67e497c..9110f38 100644 --- a/src/pages/build-with-zip.ts +++ b/src/pages/build-with-zip.ts @@ -1,4 +1,3 @@ -import * as fs from "fs-extra"; import * as AdmZip from "adm-zip"; import * as formidable from "formidable" @@ -7,7 +6,7 @@ import { URLSearchParams } from "url"; import { responseError, responseSuccess } from "../index"; import { ENABLE_REPO_WHITELIST, TEMP_DIR } from "../config"; -import { Job, addBuildQueue, BuildType } from "../builder"; +import { Job, pushBuildQueue } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (ENABLE_REPO_WHITELIST) { @@ -15,9 +14,8 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS } else { let job = new Job(); + job.attachInfo("buildType", "source-upload"); - job.attachInfo("buildType", BuildType["source-upload"]); - let type = request.headers["content-type"] || ""; if (!type.includes("multipart/form-data")) { console.log("Request content type: " + type); @@ -35,7 +33,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS } let zip = new AdmZip(file.path); zip.extractAllTo(jobDir + "/src/"); - addBuildQueue(job); + pushBuildQueue(job); responseSuccess(response, { msg: "Job added.", jobId: job.id diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts index 188dece..fd3e7ad 100644 --- a/src/pages/check-status.ts +++ b/src/pages/check-status.ts @@ -1,9 +1,9 @@ +import * as fs from "fs-extra"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; import { responseSuccess, responseError } from "../index"; -import { JobPool, JobStatus } from "../builder"; -import * as fs from "fs-extra"; +import { JobPool } from "../builder"; import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { @@ -20,7 +20,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS ret[key] = job.extraInfo[key]; } } else { - ret.status = JobStatus.done; + ret.status = "done"; } responseSuccess(response, ret); } \ No newline at end of file diff --git a/src/pages/result.ts b/src/pages/result.ts index 4d66edf..d7f6b4e 100644 --- a/src/pages/result.ts +++ b/src/pages/result.ts @@ -3,7 +3,7 @@ import * as fs from "fs-extra"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; -import { JobPool, JobStatus } from "../builder"; +import { JobPool } from "../builder"; import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { @@ -14,8 +14,8 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS response.end("Job does not exist."); return; } - let status = JobPool.has(jobId) ? JobPool.get(jobId).status : JobStatus.done; - if (status != JobStatus.done) { + let status = JobPool.has(jobId) ? JobPool.get(jobId).status : "done"; + if (status != "done") { console.log("Response end with 404 job is not ready yet"); response.writeHead(404); response.end("Job not ready yet."); diff --git a/src/utils/exec.ts b/src/utils/exec.ts index 4ec11c8..1148ced 100644 --- a/src/utils/exec.ts +++ b/src/utils/exec.ts @@ -1,40 +1,29 @@ -import { exec } from "shelljs"; +import { exec, ExecOutputReturnValue } from "shelljs"; -/** - * Execute a command with promise returns - * @param command - * @returns a Promise. - * With then(stdout => {execute when result code is 0}) - * and catch((response = [code, stderr]) => {execute when code other than 0 returned}) - */ -export default (command: string, silent = false) => { - return new Promise((resolve, reject) => { - exec(command, { silent: silent }, (code, stdout, stderr) => { - if (code == 0) { - resolve(stdout); - } else { - reject(new ExecError(code, stdout, stderr)); - } - }); +export default async (command: string, silent = false) => { + let result = exec(command, { + silent: silent, + async: false // ensure it returns ExecOutputReturnValue }); + if (result.code == 0) { + return result.stdout; + } else { + throw new ExecError(result); + } } export class ExecError extends Error { - private _code: number; - private _stdout: string; - private _stderr: string; - constructor(code: number, stdout: string, stderr: string) { + private _execOutputReturnValue: ExecOutputReturnValue; + constructor(execOutputReturnValue: ExecOutputReturnValue) { super("Command executing error occured"); - this._code = code; - this._stdout = stdout; - this._stderr = stderr; + this._execOutputReturnValue = execOutputReturnValue; } get code() { - return this._code; + return this._execOutputReturnValue.code; } get stdout() { - return this._stdout; + return this._execOutputReturnValue.stdout; } get stderr() { - return this._stderr; + return this._execOutputReturnValue.stderr; } } \ No newline at end of file diff --git a/src/utils/queue.ts b/src/utils/queue.ts index c066e7c..1df06ed 100644 --- a/src/utils/queue.ts +++ b/src/utils/queue.ts @@ -1,5 +1,4 @@ export default class Queue { - private elements: Array; public constructor() { @@ -9,19 +8,15 @@ export default class Queue { public push(o: T) { this.elements.unshift(o); } - public pop(): T { return this.elements.pop(); } - public size(): number { return this.elements.length; } - public isEmpty(): boolean { return this.size() == 0; } - public clear() { delete this.elements; this.elements = new Array();