From f1ed47cd81f56fc388704a64d78dc4695f3961be Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Sun, 3 Mar 2019 23:00:08 +0800 Subject: [PATCH 01/10] Implemented github repo building with webhook --- package.json | 2 +- src/builder.ts | 9 ++- src/config.ts | 14 ++--- src/index.ts | 15 +++-- src/pages/build-with-github-repo.ts | 87 ++++++++++++++++++----------- static/index.html | 6 +- 6 files changed, 82 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 0ec9e40..7cd3dec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "extension-builder", - "version": "1.0.0", + "version": "1.1.0", "description": "", "main": "index.js", "repository": { diff --git a/src/builder.ts b/src/builder.ts index 80bef19..c4f9069 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -36,7 +36,14 @@ export function addBuildQueue(jobId: string) { interface JobConfig { package: string } -class Job { +export class Job { + + public static generateJobId() { + // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} + let jobId = fs.mkdtempSync(TEMP_DIR + "/"); + return jobId.substring(jobId.lastIndexOf("/") + 1); + } + private _id: string; private _config: JobConfig; diff --git a/src/config.ts b/src/config.ts index 9972e78..5bc4887 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,22 +13,22 @@ export const EMPTY_TEMP_DIR_BEFORE_BUILD = false; interface WhiteList { owner: string; repoName: string; - branch: string | string[]; + nodes: string | string[]; // Nodes includes branchs, commits and tags. Can be "*" for any } const REPO_WHITELIST: WhiteList[] = [ - { owner: "OpenSourceAIX", repoName: "ColinTreeListView", branch: "extension-builder-test" } + { owner: "OpenSourceAIX", repoName: "ColinTreeListView", nodes: "extension-builder-test" } ]; -export function inWhitelist(owner: string, repoName: string, branch = "master") { +export function inWhitelist(owner: string, repoName: string, codeNode = "") { for (let i in REPO_WHITELIST) { if (REPO_WHITELIST.hasOwnProperty(i)) { let item = REPO_WHITELIST[i]; if (owner == item.owner && repoName == item.repoName) { - let acceptBranchs = item.branch; - if (acceptBranchs == "*") { + let acceptNodes = item.nodes; + if (acceptNodes == "*") { return true; } else { - acceptBranchs = typeof(acceptBranchs)=="string" ? [ acceptBranchs ] : acceptBranchs; - return acceptBranchs.includes(branch); + acceptNodes = typeof(acceptNodes)=="string" ? [ acceptNodes ] : acceptNodes; + return acceptNodes.includes(codeNode); } } } diff --git a/src/index.ts b/src/index.ts index ed54537..0453133 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,22 +52,27 @@ function startServer() { let params = new url.URLSearchParams(requestUrl.query); console.timeLog("Processing request: " + request.url); switch (requestUrl.pathname) { - case "/build-with-github-repo": + case "/build-with-github-repo": { handleBuildWithGithubRepo(request, response, params); return; - case "/build-with-zip": + } + case "/build-with-zip": { handleBuildWithZip(request, response, params); return; - case "/check-status": + } + case "/check-status": { handleCheckStatus(request, response, params); return; - case "/result": + } + case "/result": { handleResult(request, response, params); return; - default: + } + default: { if (handleStaticFile(response, requestUrl.pathname)) { return; } + } } responseError(response, 404, "404 Not found."); } catch (error) { diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 8c7d008..d987a9c 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -7,62 +7,81 @@ import { URLSearchParams } from "url"; import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config"; import { responseSuccess, responseError } from "../index"; -import { addBuildQueue } from "../builder"; +import { addBuildQueue, Job } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (request.method == "GET") { let owner = params.get("owner"); let repoName = params.get("repoName"); - let branch = params.has("branch") ? params.get("branch") : "master"; - - console.timeLog("Job info received: repo= " + owner + "/" + repoName + " branch= " + branch); - - if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, branch)) { - // 1. white list is not enabled - // 2. is in the white list - - // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} - let jobId = fs.mkdtempSync(TEMP_DIR + "/"); - jobId = jobId.substring(jobId.lastIndexOf("/") + 1); - - responseSuccess(response, { - msg: "Build started.", - jobId: jobId - }); - - getZip(jobId, owner, repoName, branch) - .then(zipName => prepareSource(jobId, zipName)) - .then(() => addBuildQueue(jobId)) - .catch(reason => { - console.error(reason); - }); + let codeNode = params.has("codeNode") ? params.get("codeNode") : "master"; + console.timeLog("Job info received: repo= " + owner + "/" + repoName + " codeNode= " + codeNode); + if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, codeNode)) { + startGithubJob(response, owner, repoName, codeNode); return; } else { - responseError(response, 403, "Whitelist is enabled & your repo is not in it."); + responseError(response, 403, "Whitelist is enabled & your repo (or codeNode) is not in it."); return; } } else if (request.method == "POST") { // webhook + let event = request.headers["X-GitHub-Event"]; + if (!["push", "release"].includes(event)) { + responseError(response, 403, "Does not support event type: " + event); + return; + } let content = ""; - request.on("data", chunk => { content += chunk; }); - request.on("end", () => { - responseSuccess(response, { content: content }); + let playload = JSON.parse(content); + let commitOrTag: string; + switch (event) { + case "push": + commitOrTag = playload.head_commit; + break; + case "release": + commitOrTag = playload.release.tag_name; + break; + } + let owner = playload.repository.owner.name; + let repoName = playload.repository.name; + if (typeof(owner)!="string" || typeof(repoName)!="string" || commitOrTag) { + responseError(response, 403, "Cannot be built, at least one of owner, repoName or commit is not string."); + return; + } + if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName)) { + startGithubJob(response, owner, repoName, commitOrTag); + return; + } else { + responseError(response, 403, "Whitelist is enabled & your repo is not in it."); + return; + } }); - return; } } -function getZip(jobId: string, owner: string, repoName: string, branch: string) { +function startGithubJob(response: ServerResponse, owner: string, repoName: string, codeNode: string) { + let jobId = Job.generateJobId(); + responseSuccess(response, { + msg: "Build started.", + jobId: jobId + }); + getZip(jobId, owner, repoName, codeNode) + .then(zipName => prepareSource(jobId, zipName)) + .then(() => addBuildQueue(jobId)) + .catch(reason => { + console.error(reason); + }); +} + +function getZip(jobId: string, owner: string, repoName: string, codeNode: string) { return new Promise(resolve => { - // https://github.com/{owner}/{repoName}/archive/{branch}.zip - // redirecting to -> https://codeload.github.com/{owner}/{repoName}/zip/{branch} - let requestUrl = "https://codeload.github.com/" + owner + "/" + repoName + "/zip/" + branch; - let zipName = owner + "_" + repoName + "_" + branch; + // https://github.com/{owner}/{repoName}/archive/{codeNode}.zip + // redirecting to -> https://codeload.github.com/{owner}/{repoName}/zip/{codeNode} + let requestUrl = "https://codeload.github.com/" + owner + "/" + repoName + "/zip/" + codeNode; + let zipName = owner + "_" + repoName + "_" + codeNode; let destZipPath = TEMP_DIR + "/" + jobId + "/" + zipName + ".zip"; https.get(requestUrl, response => { var stream = response.pipe(fs.createWriteStream(destZipPath)); diff --git a/static/index.html b/static/index.html index 5254800..824dccb 100644 --- a/static/index.html +++ b/static/index.html @@ -34,7 +34,7 @@ data: { owner: $("#owner").val(), repoName: $("#repoName").val(), - branch: $("#branch").val() + codeNode: $("#codeNode").val() }, dataType: "json", success: json => { @@ -56,7 +56,7 @@ }); $("#previewRepo").click(() => { window.open( - "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#branch").val(), + "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#codeNode").val(), "_blank").focus(); return false; }) @@ -73,7 +73,7 @@ Repo Name - Branch + Code node(branch, commit or tag) From 797264ba7055af4b9b03bb560156dc7e682000ad Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Sun, 3 Mar 2019 23:20:17 +0800 Subject: [PATCH 02/10] Fix header reading issue & standardlize logs --- src/index.ts | 4 +++- src/pages/build-with-github-repo.ts | 7 ++++++- src/pages/result.ts | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 0453133..9053bcd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,10 +11,12 @@ import handleResult from "./pages/result"; export const CONTENT_TYPE_JSON = {"Content-Type": "application/json"}; export function responseSuccess(response: http.ServerResponse, info: {}) { + console.log("Response end with 200: " + info); response.writeHead(200, CONTENT_TYPE_JSON); response.end(JSON.stringify(info)); } export function responseError(response: http.ServerResponse, code: number, msg: string) { + console.log("Response end with " + code + ": " + msg); response.writeHead(code, CONTENT_TYPE_JSON); response.end(JSON.stringify({ msg: msg })); } @@ -39,7 +41,7 @@ function handleStaticFile(response: http.ServerResponse, pathname: string): bool } pathname = staticDir + pathname; let mime = mimeTypes.lookup(pathname); - console.log("Returning static file(" + pathname + ") mime(" + mime + ")"); + console.log("Response end with 200: static file(" + pathname + ") mime(" + mime + ")"); response.writeHead(200, { "Content-Type": mime!==false ? mime : "application/octet-stream" }); fs.createReadStream(pathname).pipe(response, { end: true }); return true; diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index d987a9c..260b429 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -25,7 +25,12 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS } else if (request.method == "POST") { // webhook - let event = request.headers["X-GitHub-Event"]; + let indexOfEvent = request.rawHeaders.indexOf("X-GitHub-Event"); + let event: string; + if (indexOfEvent != -1 && (indexOfEvent + 1) < request.rawHeaders.length) { + event = request.rawHeaders[indexOfEvent + 1]; + } + console.log("Github event = " + event); if (!["push", "release"].includes(event)) { responseError(response, 403, "Does not support event type: " + event); return; diff --git a/src/pages/result.ts b/src/pages/result.ts index e7c9d23..802bc3f 100644 --- a/src/pages/result.ts +++ b/src/pages/result.ts @@ -9,18 +9,21 @@ import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { let jobId = params.get("jobId"); if (!JobPool.has(jobId)) { + console.log("Response end with 404: job not exist"); response.writeHead(404); response.end("Job does not exist."); return; } let status = JobPool.get(jobId).status; if (status != "done") { + console.log("Response end with 404 job is not ready yet"); response.writeHead(404); response.end("Job not ready yet."); return; } else { let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; var stat = fs.statSync(zipPath); + console.log("Response end with 200 (build result will be sent)"); response.writeHead(200, { "Content-Type": "application/zip", "Content-Length": stat.size From 29d531b05f27013b400dd085632ae2ff8b47382a Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Sun, 3 Mar 2019 23:25:30 +0800 Subject: [PATCH 03/10] Fix commit id is not retrieved properly --- src/pages/build-with-github-repo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 260b429..6f70ddd 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -44,7 +44,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS let commitOrTag: string; switch (event) { case "push": - commitOrTag = playload.head_commit; + commitOrTag = playload.head_commit.id; break; case "release": commitOrTag = playload.release.tag_name; From 49025607f225affbc35decdd194506ab7d96fd30 Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Mon, 4 Mar 2019 01:48:09 +0800 Subject: [PATCH 04/10] Optmised variable names, code logic & Changed way load source from github --- package-lock.json | 228 +++++++++++++++++++++++++++- package.json | 1 + src/builder.ts | 71 +++++---- src/config.ts | 14 +- src/index.ts | 3 +- src/pages/build-with-github-repo.ts | 86 +++++------ static/index.html | 6 +- 7 files changed, 316 insertions(+), 93 deletions(-) diff --git a/package-lock.json b/package-lock.json index d28b204..c40d8c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,47 @@ { "name": "extension-builder", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "@octokit/endpoint": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-3.1.3.tgz", + "integrity": "sha512-vAWzeoj9Lzpl3V3YkWKhGzmDUoMfKpyxJhpq74/ohMvmLXDoEuAGnApy/7TRi3OmnjyX2Lr+e9UGGAD0919ohA==", + "requires": { + "deepmerge": "3.2.0", + "is-plain-object": "2.0.4", + "universal-user-agent": "2.0.3", + "url-template": "2.0.8" + } + }, + "@octokit/request": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-2.4.0.tgz", + "integrity": "sha512-Bm2P0duVRUeKhyepNyFg5GX+yhCK71fqdtpsw5Rz+PQPjSha8HYwPMF5QfpzpD8b6/Xl3xhTgu3V90W362gZ1A==", + "requires": { + "@octokit/endpoint": "3.1.3", + "is-plain-object": "2.0.4", + "node-fetch": "2.3.0", + "universal-user-agent": "2.0.3" + } + }, + "@octokit/rest": { + "version": "16.16.3", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-16.16.3.tgz", + "integrity": "sha512-8v5xyqXZwQbQ1WsTLU3G25nAlcKYEgIXzDeqLgTFpbzzJXcey0C8Mcs/LZiAgU8dDINZtO2dAPgd1cVKgK9DQw==", + "requires": { + "@octokit/request": "2.4.0", + "before-after-hook": "1.3.2", + "btoa-lite": "1.0.0", + "lodash.get": "4.4.2", + "lodash.set": "4.3.2", + "lodash.uniq": "4.5.0", + "octokit-pagination-methods": "1.1.0", + "universal-user-agent": "2.0.3", + "url-template": "2.0.8" + } + }, "@types/adm-zip": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.32.tgz", @@ -77,6 +115,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "before-after-hook": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.3.2.tgz", + "integrity": "sha512-zyPgY5dgbf99c0uGUjhY4w+mxqEGxPKg9RQDl34VvrVh2bM31lFN+mwR1ZHepq/KA3VCPk1gwJZL6IIJqjLy2w==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -86,11 +129,47 @@ "concat-map": "0.0.1" } }, + "btoa-lite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", + "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "requires": { + "nice-try": "1.0.5", + "path-key": "2.0.1", + "semver": "5.6.0", + "shebang-command": "1.2.0", + "which": "1.3.1" + } + }, + "deepmerge": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.2.0.tgz", + "integrity": "sha512-6+LuZGU7QCNUnAJyX8cIrlzoEgggTM6B7mm+znKOX4t5ltluT9KLjN6g61ECMS0LTsLW7yDpNoxhix5FZcrIow==" + }, + "execa": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", + "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "requires": { + "cross-spawn": "6.0.5", + "get-stream": "3.0.0", + "is-stream": "1.1.0", + "npm-run-path": "2.0.2", + "p-finally": "1.0.0", + "signal-exit": "3.0.2", + "strip-eof": "1.0.0" + } + }, "fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -106,6 +185,11 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + }, "glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", @@ -143,6 +227,29 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz", "integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==" }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "requires": { + "isobject": "3.0.1" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" + }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", @@ -151,6 +258,26 @@ "graceful-fs": "4.1.15" } }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, + "macos-release": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.0.0.tgz", + "integrity": "sha512-iCM3ZGeqIzlrH7KxYK+fphlJpCCczyHXc+HhRVbEu9uNTCrzYJjvvtefzeKTCVHd5AP/aD/fzC80JZ4ZP+dQ/A==" + }, "mime-db": { "version": "1.38.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", @@ -172,6 +299,29 @@ "brace-expansion": "1.1.11" } }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" + }, + "node-fetch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz", + "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==" + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "requires": { + "path-key": "2.0.1" + } + }, + "octokit-pagination-methods": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz", + "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -180,11 +330,30 @@ "wrappy": "1.0.2" } }, + "os-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.0.0.tgz", + "integrity": "sha512-7c74tib2FsdFbQ3W+qj8Tyd1R3Z6tuVRNNxXjJcZ4NgjIEQU9N/prVMqcW29XZPXGACqaXN3jq58/6hoaoXH6g==", + "requires": { + "macos-release": "2.0.0", + "windows-release": "3.1.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" + }, "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", @@ -206,6 +375,24 @@ "path-parse": "1.0.6" } }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "requires": { + "shebang-regex": "1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" + }, "shelljs": { "version": "0.8.3", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.3.tgz", @@ -216,11 +403,50 @@ "rechoir": "0.6.2" } }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" + }, + "universal-user-agent": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.3.tgz", + "integrity": "sha512-eRHEHhChCBHrZsA4WEhdgiOKgdvgrMIHwnwnqD0r5C6AO8kwKcG7qSku3iXdhvHL3YvsS9ZkSGN8h/hIpoFC8g==", + "requires": { + "os-name": "3.0.0" + } + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "2.0.0" + } + }, + "windows-release": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.1.0.tgz", + "integrity": "sha512-hBb7m7acFgQPQc222uEQTmdcGLeBmQLNLFIh0rDk3CwFOBrfjefLzEfEfmpMq8Af/n/GnFf3eYf203FY1PmudA==", + "requires": { + "execa": "0.10.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 7cd3dec..09efbf2 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ }, "homepage": "https://github.com/ColinTree-bot/extension-builder#readme", "dependencies": { + "@octokit/rest": "^16.16.3", "adm-zip": "^0.4.13", "fs-extra": "^7.0.1", "mime-types": "^2.1.22", diff --git a/src/builder.ts b/src/builder.ts index c4f9069..ae090f9 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -15,8 +15,8 @@ class BuildQueue extends Queue { } export class JobPool { private static pool = new Map(); - public static add(jobId: string, job: Job) { - JobPool.pool.set(jobId, job); + public static add(job: Job) { + JobPool.pool.set(job.id, job); } public static get(jobId: string): Job { return JobPool.pool.get(jobId); @@ -27,48 +27,44 @@ export class JobPool { } const buildQueue = new BuildQueue(); -export function addBuildQueue(jobId: string) { - console.timeLog("Job(" + jobId + ") going to be added into build queue"); +export function addBuildQueue(job: Job) { + console.timeLog("Job(" + job.id + ") going to be added into build queue"); // job will be added to build queue by itself after it is ready (in constructor) - JobPool.add(jobId, new Job(jobId)); + job.status = JobStatus.waiting; + buildQueue.push(job.id); } -interface JobConfig { - package: string +export enum BuildType { + "github-repo" = "github-repo", + "source-upload" = "source-upload" +} +export enum JobStatus { + preparing = "preparing", + waiting = "waiting", + building = "building", + done = "done", + failed = "failed" } export class Job { - - public static generateJobId() { - // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} - let jobId = fs.mkdtempSync(TEMP_DIR + "/"); - return jobId.substring(jobId.lastIndexOf("/") + 1); - } - private _id: string; - private _config: JobConfig; + private _extraInfo: { [key: string]: string | number } = {}; get id() { return this._id; } - get config() { return this._config; } + get extraInfo() { return this._extraInfo; } - public status: "waiting" | "building" | "done" | "failed"; + public status: JobStatus; - public constructor(jobId: string) { - this._id = jobId; - this.status = "waiting"; - this.loadConfig() - .then(config => { - this._config = config; - buildQueue.push(jobId); - }); + public constructor() { + // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} + let jobDir = fs.mkdtempSync(TEMP_DIR + "/"); + this._id = jobDir.substring(jobDir.lastIndexOf("/") + 1); + this._extraInfo.startTimestamp = Date.now(); + this.status = JobStatus.preparing; + JobPool.add(this); } - private loadConfig() { - return new Promise(resolve => { - fs.readFile(TEMP_DIR + "/" + this.id + "/src/" + BUILDER_CONFIG_NAME, - "utf8", (err, data) => { - if (err) throw err; - resolve(JSON.parse(data)); - }); - }); + + public attachInfo(key: string, value: string | number) { + this._extraInfo[key] = value; } } @@ -79,7 +75,7 @@ class Builder { if (Builder.builderAvailable && !buildQueue.isEmpty()) { Builder.builderAvailable = false; let jobId = buildQueue.pop(); - JobPool.get(jobId).status = "building"; + JobPool.get(jobId).status = JobStatus.building; Builder.cleanWorkspace() .then(() => Builder.buildJob(jobId)); } @@ -97,7 +93,7 @@ class Builder { return new Promise(resolve => { let job = JobPool.get(jobId); console.timeLog("Going to build job(" + jobId + ")"); - let config = job.config; + let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8")); let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/"; fs.ensureDirSync(targetPath); fs.emptyDirSync(targetPath); @@ -108,16 +104,17 @@ class Builder { .then(stdout => { let zip = new AdmZip(); zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); + zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; zip.writeZip(zipPath); - JobPool.get(jobId).status = "done"; + JobPool.get(jobId).status = JobStatus.done; console.log("Done job(" + jobId + "): " + zipPath); Builder.builderAvailable = true; Builder.notify(); resolve(); }) .catch(reason => { - JobPool.get(jobId).status = "failed"; + JobPool.get(jobId).status = JobStatus.failed; console.log("Job(" + jobId + ") build failed", reason); Builder.builderAvailable = true; Builder.notify(); diff --git a/src/config.ts b/src/config.ts index 5bc4887..6817f06 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,22 +13,22 @@ export const EMPTY_TEMP_DIR_BEFORE_BUILD = false; interface WhiteList { owner: string; repoName: string; - nodes: string | string[]; // Nodes includes branchs, commits and tags. Can be "*" for any + refs: string | string[]; // refs includes branchs, commits and tags. Can be "*" for any } const REPO_WHITELIST: WhiteList[] = [ - { owner: "OpenSourceAIX", repoName: "ColinTreeListView", nodes: "extension-builder-test" } + { owner: "OpenSourceAIX", repoName: "ColinTreeListView", refs: "extension-builder-test" } ]; -export function inWhitelist(owner: string, repoName: string, codeNode = "") { +export function inWhitelist(owner: string, repoName: string, coderef = "") { for (let i in REPO_WHITELIST) { if (REPO_WHITELIST.hasOwnProperty(i)) { let item = REPO_WHITELIST[i]; if (owner == item.owner && repoName == item.repoName) { - let acceptNodes = item.nodes; - if (acceptNodes == "*") { + let acceptRefs = item.refs; + if (acceptRefs == "*") { return true; } else { - acceptNodes = typeof(acceptNodes)=="string" ? [ acceptNodes ] : acceptNodes; - return acceptNodes.includes(codeNode); + acceptRefs = typeof(acceptRefs)=="string" ? [ acceptRefs ] : acceptRefs; + return acceptRefs.includes(coderef); } } } diff --git a/src/index.ts b/src/index.ts index 9053bcd..526a8ef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,9 +11,10 @@ import handleResult from "./pages/result"; export const CONTENT_TYPE_JSON = {"Content-Type": "application/json"}; export function responseSuccess(response: http.ServerResponse, info: {}) { + info = JSON.stringify(info); console.log("Response end with 200: " + info); response.writeHead(200, CONTENT_TYPE_JSON); - response.end(JSON.stringify(info)); + response.end(info); } export function responseError(response: http.ServerResponse, code: number, msg: string) { console.log("Response end with " + code + ": " + msg); diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 6f70ddd..6e0b9df 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -1,25 +1,25 @@ -import * as https from "https"; import * as fs from "fs-extra"; import * as Admzip from "adm-zip"; +import * as Github from "@octokit/rest"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config"; import { responseSuccess, responseError } from "../index"; -import { addBuildQueue, Job } from "../builder"; +import { addBuildQueue, Job, JobPool, BuildType, JobStatus } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (request.method == "GET") { let owner = params.get("owner"); let repoName = params.get("repoName"); - let codeNode = params.has("codeNode") ? params.get("codeNode") : "master"; - console.timeLog("Job info received: repo= " + owner + "/" + repoName + " codeNode= " + codeNode); - if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, codeNode)) { - startGithubJob(response, owner, repoName, codeNode); + let ref = params.has("ref") ? params.get("ref") : "master"; + console.timeLog("Job info received: repo= " + owner + "/" + repoName + " ref= " + ref); + if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, ref)) { + startGithubJob(response, owner, repoName, ref); return; } else { - responseError(response, 403, "Whitelist is enabled & your repo (or codeNode) is not in it."); + responseError(response, 403, "Whitelist is enabled & your repo (or ref) is not in it."); return; } @@ -50,8 +50,9 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS commitOrTag = playload.release.tag_name; break; } - let owner = playload.repository.owner.name; + let owner = playload.repository.owner.login; let repoName = playload.repository.name; + console.log("Repo = " + owner + "/" + repoName + " commitOrTag = " + commitOrTag); if (typeof(owner)!="string" || typeof(repoName)!="string" || commitOrTag) { responseError(response, 403, "Cannot be built, at least one of owner, repoName or commit is not string."); return; @@ -67,51 +68,48 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS } } -function startGithubJob(response: ServerResponse, owner: string, repoName: string, codeNode: string) { - let jobId = Job.generateJobId(); +function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) { + let job = new Job(); + let jobId = job.id; + + job.attachInfo("buildType", BuildType["github-repo"]); + job.attachInfo("owner", owner); + job.attachInfo("repoName", repoName); + job.attachInfo("ref", ref); + responseSuccess(response, { msg: "Build started.", jobId: jobId }); - getZip(jobId, owner, repoName, codeNode) - .then(zipName => prepareSource(jobId, zipName)) - .then(() => addBuildQueue(jobId)) - .catch(reason => { - console.error(reason); - }); -} - -function getZip(jobId: string, owner: string, repoName: string, codeNode: string) { - return new Promise(resolve => { - // https://github.com/{owner}/{repoName}/archive/{codeNode}.zip - // redirecting to -> https://codeload.github.com/{owner}/{repoName}/zip/{codeNode} - let requestUrl = "https://codeload.github.com/" + owner + "/" + repoName + "/zip/" + codeNode; - let zipName = owner + "_" + repoName + "_" + codeNode; - let destZipPath = TEMP_DIR + "/" + jobId + "/" + zipName + ".zip"; - https.get(requestUrl, response => { - var stream = response.pipe(fs.createWriteStream(destZipPath)); - stream.on("finish", () => { - console.timeLog("Downloaded: " + destZipPath); - resolve(zipName); - }); - }); - }); -} -function prepareSource(jobId: string, zipPath: string) { - return new Promise(resolve => { - let zip = new Admzip(TEMP_DIR + "/" + jobId + "/" + zipPath + ".zip"); - let entryDir: string; - zip.getEntries().forEach(entry => { - if (!entryDir) { - entryDir = entry.entryName; - } - }); - zip.extractEntryTo(entryDir, TEMP_DIR + "/" + jobId + "/rawComponentSource/"); + // Downlaod archieve + new Github().repos.getArchiveLink({ + owner: owner, + repo: repoName, + archive_format: "zipball", + ref: ref + }) + // Load source + .then(archieveResponse => new Promise((resolve, reject) => { + let zip = new Admzip( archieveResponse.data); + if (zip.getEntries().length == 0) { + reject("No source found in archieve downloaded."); + return; + } + let entryDir = zip.getEntries()[0].entryName; + zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/"); fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, TEMP_DIR + "/" + jobId + "/src"); fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); console.timeLog("Unzipped & moved to " + TEMP_DIR + "/" + jobId + "/src"); resolve(); + })) + // Add to build queue + .then(() => + addBuildQueue(job) + ) + .catch(reason => { + JobPool.get(jobId).status = JobStatus.failed; + console.error(reason); }); } \ No newline at end of file diff --git a/static/index.html b/static/index.html index 824dccb..e4b9a73 100644 --- a/static/index.html +++ b/static/index.html @@ -34,7 +34,7 @@ data: { owner: $("#owner").val(), repoName: $("#repoName").val(), - codeNode: $("#codeNode").val() + ref: $("#ref").val() }, dataType: "json", success: json => { @@ -56,7 +56,7 @@ }); $("#previewRepo").click(() => { window.open( - "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#codeNode").val(), + "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#ref").val(), "_blank").focus(); return false; }) @@ -73,7 +73,7 @@ Repo Name - Code node(branch, commit or tag) + Ref From 9f6ff05423d4316189e0dddf0b7318c25d9622fb Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Mon, 4 Mar 2019 01:57:01 +0800 Subject: [PATCH 05/10] Update log info --- src/pages/build-with-github-repo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 6e0b9df..b504f25 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -101,7 +101,7 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, TEMP_DIR + "/" + jobId + "/src"); fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); - console.timeLog("Unzipped & moved to " + TEMP_DIR + "/" + jobId + "/src"); + console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src"); resolve(); })) // Add to build queue From e75c2e7cd82e981b60615edff7aa981e21d90b4e Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Mon, 4 Mar 2019 14:11:12 +0800 Subject: [PATCH 06/10] Done build-with-zip --- package-lock.json | 15 ++++++++++ package.json | 2 ++ src/config.ts | 2 +- src/pages/build-with-github-repo.ts | 2 +- src/pages/build-with-zip.ts | 43 ++++++++++++++++++++++++++--- static/index.html | 42 ++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index c40d8c9..fe5117f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -57,6 +57,16 @@ "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==", "dev": true }, + "@types/formidable": { + "version": "1.0.31", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-1.0.31.tgz", + "integrity": "sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q==", + "dev": true, + "requires": { + "@types/events": "3.0.0", + "@types/node": "11.9.6" + } + }, "@types/fs-extra": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.5.tgz", @@ -170,6 +180,11 @@ "strip-eof": "1.0.0" } }, + "formidable": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz", + "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==" + }, "fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", diff --git a/package.json b/package.json index 09efbf2..7e35298 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@octokit/rest": "^16.16.3", "adm-zip": "^0.4.13", + "formidable": "^1.2.1", "fs-extra": "^7.0.1", "mime-types": "^2.1.22", "shelljs": "^0.8.3" @@ -25,6 +26,7 @@ }, "devDependencies": { "@types/adm-zip": "^0.4.32", + "@types/formidable": "^1.0.31", "@types/fs-extra": "^5.0.5", "@types/mime-types": "^2.1.0", "@types/shelljs": "^0.8.3" diff --git a/src/config.ts b/src/config.ts index 6817f06..0fbcdc3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,7 @@ import * as os from "os"; export const WORKSPACE = "/usr/workspace/"; -export const ENABLE_REPO_WHITELIST = true; +export const ENABLE_REPO_WHITELIST = false; export const PORT = 8048; export const TEMP_DIR = os.tmpdir() + "/extension-builder/"; export const BUILDER_CONFIG_NAME = "builder-config.json"; diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index b504f25..99b10c3 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -78,7 +78,7 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin job.attachInfo("ref", ref); responseSuccess(response, { - msg: "Build started.", + msg: "Job added.", jobId: jobId }); diff --git a/src/pages/build-with-zip.ts b/src/pages/build-with-zip.ts index e327f22..67e497c 100644 --- a/src/pages/build-with-zip.ts +++ b/src/pages/build-with-zip.ts @@ -1,15 +1,50 @@ +import * as fs from "fs-extra"; +import * as AdmZip from "adm-zip"; +import * as formidable from "formidable" + import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; -import { responseError } from "../index"; -import { ENABLE_REPO_WHITELIST } from "../config"; +import { responseError, responseSuccess } from "../index"; +import { ENABLE_REPO_WHITELIST, TEMP_DIR } from "../config"; +import { Job, addBuildQueue, BuildType } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (ENABLE_REPO_WHITELIST) { responseError(response, 403, "Currently in white list mode, build with zip is disabled"); - return; + } else { - responseError(response, 501, "build with zip is not implemented yet."); + let job = new Job(); + + job.attachInfo("buildType", BuildType["source-upload"]); + + let type = request.headers["content-type"] || ""; + if (!type.includes("multipart/form-data")) { + console.log("Request content type: " + type); + responseError(response, 400, "Please use multipart/form-data format"); + return; + } + + let jobDir = TEMP_DIR + "/" + job.id + "/"; + let form = new formidable.IncomingForm(); + form.uploadDir = jobDir; + form.keepExtensions = true; + form.on("file", (name, file) => { + if (name != "source") { + return; + } + let zip = new AdmZip(file.path); + zip.extractAllTo(jobDir + "/src/"); + addBuildQueue(job); + responseSuccess(response, { + msg: "Job added.", + jobId: job.id + }); + }); + form.on("error", err => { + responseError(response, 500, err); + }) + form.parse(request); return; } } \ No newline at end of file diff --git a/static/index.html b/static/index.html index e4b9a73..5c65478 100644 --- a/static/index.html +++ b/static/index.html @@ -16,6 +16,9 @@ if (xhr.status == 404) { alert("404! maybe server is too busy to handle our request"); return; + } else if (xhr.status != 200) { + alert(xhr.status + "!" + xhr.responseJSON.msg); + return; } $("#status").text(xhr.responseJSON.status); if (xhr.responseJSON.status == "done") { @@ -49,6 +52,36 @@ }, error: xhr => { console.log(xhr.responseJSON); + alert(xhr.responseJSON); + } + }); + $("[type=submit]").prop("disabled", true); + return false; + }); + $("#buildWithZip").submit(() => { + let formData = new FormData(); + formData.append("source", $("#zip")[0].files[0]); + $.ajax({ + url: "/build-with-zip", + type: "POST", + cache: false, + data: formData, + processData: false, + contentType: false, + dataType: "json", + success: json => { + let jobId = json["jobId"]; + $("#jobId").text(jobId); + $("#status").text("submitted"); + checkStatus(jobId, () => { + $("#download") + .attr("href", "/result?jobId=" + jobId) + .parent().show(); + }); + }, + error: xhr => { + console.log(xhr.responseJSON); + alert(xhr.responseJSON); } }); $("[type=submit]").prop("disabled", true); @@ -81,6 +114,15 @@ +
+ +
+ + +
+ +
+

jobId:

From 333122e4cf12cf670c458667ef67c87eb556a2ac Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Tue, 5 Mar 2019 19:33:45 +0800 Subject: [PATCH 07/10] Handled check-status &result when job not exist in pool but result exist --- src/pages/check-status.ts | 9 +++++---- src/pages/result.ts | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts index 52bb500..33bc406 100644 --- a/src/pages/check-status.ts +++ b/src/pages/check-status.ts @@ -2,14 +2,15 @@ import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; import { responseSuccess, responseError } from "../index"; -import { JobPool } from "../builder"; +import { JobPool, JobStatus } from "../builder"; +import * as fs from "fs-extra"; +import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { let jobId = params.get("jobId"); - if (!JobPool.has(jobId)) { - // TODO: try harder to find this job in "build-result"? + if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) { responseError(response, 404, "Specified job does not exist in job pool."); return; } - responseSuccess(response, { status: JobPool.get(jobId).status }); + responseSuccess(response, { status: JobPool.has(jobId) ? JobPool.get(jobId).status : JobStatus.done }); } \ No newline at end of file diff --git a/src/pages/result.ts b/src/pages/result.ts index 802bc3f..4d66edf 100644 --- a/src/pages/result.ts +++ b/src/pages/result.ts @@ -3,19 +3,19 @@ import * as fs from "fs-extra"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; -import { JobPool } from "../builder"; +import { JobPool, JobStatus } from "../builder"; import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { let jobId = params.get("jobId"); - if (!JobPool.has(jobId)) { + if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) { console.log("Response end with 404: job not exist"); response.writeHead(404); response.end("Job does not exist."); return; } - let status = JobPool.get(jobId).status; - if (status != "done") { + let status = JobPool.has(jobId) ? JobPool.get(jobId).status : JobStatus.done; + if (status != JobStatus.done) { console.log("Response end with 404 job is not ready yet"); response.writeHead(404); response.end("Job not ready yet."); From aa64ab8dd9606831336bf25d6c164a26bd87c205 Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Tue, 5 Mar 2019 22:36:09 +0800 Subject: [PATCH 08/10] Handled some build error --- src/builder.ts | 50 ++++++++++++++++------------- src/pages/build-with-github-repo.ts | 18 +++++++---- src/pages/check-status.ts | 12 ++++++- src/utils/exec.ts | 22 ++++++++++++- static/index.html | 4 ++- 5 files changed, 73 insertions(+), 33 deletions(-) diff --git a/src/builder.ts b/src/builder.ts index ae090f9..52be257 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -1,7 +1,7 @@ import * as fs from "fs-extra"; import * as AdmZip from "adm-zip" -import exec from "./utils/exec"; +import exec, { ExecError } from "./utils/exec"; import { WORKSPACE, TEMP_DIR, BUILDER_CONFIG_NAME, OUTPUT_DIR } from "./config"; import Queue from "./utils/queue"; @@ -89,8 +89,8 @@ class Builder { }); }) } - private static buildJob(jobId: string) { - return new Promise(resolve => { + private static async buildJob(jobId: string) { + try { let job = JobPool.get(jobId); console.timeLog("Going to build job(" + jobId + ")"); let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8")); @@ -100,25 +100,29 @@ class Builder { fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath); console.log("Copied: " + targetPath); console.log("Compile started: job(" + jobId + ")"); - exec("cd " + WORKSPACE + "/appinventor && ant extensions", true) - .then(stdout => { - let zip = new AdmZip(); - zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); - zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); - let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; - zip.writeZip(zipPath); - JobPool.get(jobId).status = JobStatus.done; - console.log("Done job(" + jobId + "): " + zipPath); - Builder.builderAvailable = true; - Builder.notify(); - resolve(); - }) - .catch(reason => { - JobPool.get(jobId).status = JobStatus.failed; - console.log("Job(" + jobId + ") build failed", reason); - Builder.builderAvailable = true; - Builder.notify(); - }); - }); + + await exec("cd " + WORKSPACE + "/appinventor && ant extensions", true); + + let zip = new AdmZip(); + zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); + zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); + let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; + zip.writeZip(zipPath); + JobPool.get(jobId).status = JobStatus.done; + console.log("Done job(" + jobId + "): " + zipPath); + Builder.builderAvailable = true; + Builder.notify(); + } catch (reason) { + JobPool.get(jobId).status = JobStatus.failed; + if (reason instanceof ExecError) { + let err = reason; + JobPool.get(jobId).attachInfo("failInfo", err.message + ": code(" + err.code + ") stdout:\n" + err.stdout + "\n\nstderr:\n" + err.stderr); + } else { + JobPool.get(jobId).attachInfo("failInfo", reason); + } + console.log("Job(" + jobId + ") build failed", reason); + Builder.builderAvailable = true; + Builder.notify(); + } } } \ No newline at end of file diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 99b10c3..c83df77 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -89,7 +89,6 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin archive_format: "zipball", ref: ref }) - // Load source .then(archieveResponse => new Promise((resolve, reject) => { let zip = new Admzip( archieveResponse.data); if (zip.getEntries().length == 0) { @@ -102,14 +101,19 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin TEMP_DIR + "/" + jobId + "/src"); fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src"); - resolve(); + addBuildQueue(job); })) - // Add to build queue - .then(() => - addBuildQueue(job) - ) .catch(reason => { JobPool.get(jobId).status = JobStatus.failed; - console.error(reason); + if (Object.getPrototypeOf(reason) == Error.prototype) { + let err = reason; + JobPool.get(jobId).attachInfo("failInfo", + err.name == "HttpError" + ? "Cannot load source from github, please check if the ref exists in the specified repo." + : err.message); + } else { + JobPool.get(jobId).attachInfo("failInfo", reason.toString()); + } + console.log("Fail prepare source of job(" + jobId + ")", reason); }); } \ No newline at end of file diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts index 33bc406..188dece 100644 --- a/src/pages/check-status.ts +++ b/src/pages/check-status.ts @@ -12,5 +12,15 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS responseError(response, 404, "Specified job does not exist in job pool."); return; } - responseSuccess(response, { status: JobPool.has(jobId) ? JobPool.get(jobId).status : JobStatus.done }); + let ret: { [key: string]: string | number } = {}; + if (JobPool.has(jobId)) { + let job = JobPool.get(jobId); + ret.status = job.status; + for (let key in job.extraInfo) { + ret[key] = job.extraInfo[key]; + } + } else { + ret.status = JobStatus.done; + } + responseSuccess(response, ret); } \ No newline at end of file diff --git a/src/utils/exec.ts b/src/utils/exec.ts index 7db2582..4ec11c8 100644 --- a/src/utils/exec.ts +++ b/src/utils/exec.ts @@ -13,8 +13,28 @@ export default (command: string, silent = false) => { if (code == 0) { resolve(stdout); } else { - reject([code, stderr]); + reject(new ExecError(code, stdout, stderr)); } }); }); +} +export class ExecError extends Error { + private _code: number; + private _stdout: string; + private _stderr: string; + constructor(code: number, stdout: string, stderr: string) { + super("Command executing error occured"); + this._code = code; + this._stdout = stdout; + this._stderr = stderr; + } + get code() { + return this._code; + } + get stdout() { + return this._stdout; + } + get stderr() { + return this._stderr; + } } \ No newline at end of file diff --git a/static/index.html b/static/index.html index 5c65478..4e7c3fe 100644 --- a/static/index.html +++ b/static/index.html @@ -24,7 +24,9 @@ if (xhr.responseJSON.status == "done") { callback_whenDone(); } else { - checkStatus(jobId, callback_whenDone, times + 1); + if (xhr.responseJSON.status != "failed") { + checkStatus(jobId, callback_whenDone, times + 1); + } } } }); From 076271c5ce6587f811eb6e3bd28f27f490bb50f1 Mon Sep 17 00:00:00 2001 From: ColinTree <502470184@qq.com> Date: Tue, 5 Mar 2019 22:48:11 +0800 Subject: [PATCH 09/10] Fixed WORKSPACE path leaking --- src/builder.ts | 5 ++++- static/index.html | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/builder.ts b/src/builder.ts index 52be257..7add386 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -116,7 +116,10 @@ class Builder { JobPool.get(jobId).status = JobStatus.failed; if (reason instanceof ExecError) { let err = reason; - JobPool.get(jobId).attachInfo("failInfo", err.message + ": code(" + err.code + ") stdout:\n" + err.stdout + "\n\nstderr:\n" + err.stderr); + // Notice that it would not work on windows + let stdout = err.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + let stderr = err.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + JobPool.get(jobId).attachInfo("failInfo", err.message + ": code(" + err.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr); } else { JobPool.get(jobId).attachInfo("failInfo", reason); } diff --git a/static/index.html b/static/index.html index 4e7c3fe..7e5cac7 100644 --- a/static/index.html +++ b/static/index.html @@ -24,7 +24,9 @@ if (xhr.responseJSON.status == "done") { callback_whenDone(); } else { - if (xhr.responseJSON.status != "failed") { + if (xhr.responseJSON.status == "failed") { + $("#output").text(xhr.responseJSON.failInfo); + } else { checkStatus(jobId, callback_whenDone, times + 1); } } @@ -133,6 +135,8 @@ Status:

+

+
 

   Download
 


From 9b8754c5fc32f167f40cd73daee4cdabdf5db0b2 Mon Sep 17 00:00:00 2001
From: ColinTree <502470184@qq.com>
Date: Wed, 6 Mar 2019 17:58:34 +0800
Subject: [PATCH 10/10] Replace all Promise with async function, & standardlize
 codes

---
 src/builder.ts                      | 140 ++++++++++++----------------
 src/pages/build-with-github-repo.ts |  76 ++++++++-------
 src/pages/build-with-zip.ts         |   8 +-
 src/pages/check-status.ts           |   6 +-
 src/pages/result.ts                 |   6 +-
 src/utils/exec.ts                   |  43 ++++-----
 src/utils/queue.ts                  |   5 -
 7 files changed, 124 insertions(+), 160 deletions(-)

diff --git a/src/builder.ts b/src/builder.ts
index 7add386..dc09ccd 100644
--- a/src/builder.ts
+++ b/src/builder.ts
@@ -5,14 +5,6 @@ import exec, { ExecError } from "./utils/exec";
 import { WORKSPACE, TEMP_DIR, BUILDER_CONFIG_NAME, OUTPUT_DIR } from "./config";
 import Queue from "./utils/queue";
 
-class BuildQueue extends Queue {
-  constructor() { super(); }
-  public push(jobId: string) {
-    super.push(jobId);
-    console.timeLog("Added job(" + jobId + ")");
-    Builder.notify();
-  }
-}
 export class JobPool {
   private static pool = new Map();
   public static add(job: Job) {
@@ -25,26 +17,7 @@ export class JobPool {
     return JobPool.pool.has(jobId);
   }
 }
-const buildQueue = new BuildQueue();
-
-export function addBuildQueue(job: Job) {
-  console.timeLog("Job(" + job.id + ") going to be added into build queue");
-  // job will be added to build queue by itself after it is ready (in constructor)
-  job.status = JobStatus.waiting;
-  buildQueue.push(job.id);
-}
-
-export enum BuildType {
-  "github-repo" = "github-repo",
-  "source-upload" = "source-upload"
-}
-export enum JobStatus {
-  preparing = "preparing",
-  waiting = "waiting",
-  building = "building",
-  done = "done",
-  failed = "failed"
-}
+type JobStatus = "preparing" | "waiting" | "building" | "done" | "failed";
 export class Job {
   private _id: string;
   private _extraInfo: { [key: string]: string | number } = {};
@@ -55,11 +28,11 @@ export class Job {
   public status: JobStatus;
 
   public constructor() {
-    // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId}
+    fs.ensureDirSync(TEMP_DIR);
     let jobDir = fs.mkdtempSync(TEMP_DIR + "/");
     this._id = jobDir.substring(jobDir.lastIndexOf("/") + 1);
-    this._extraInfo.startTimestamp = Date.now();
-    this.status = JobStatus.preparing;
+    this.status = "preparing";
+    this.attachInfo("startTimestamp", Date.now());
     JobPool.add(this);
   }
 
@@ -68,64 +41,75 @@ export class Job {
   }
 }
 
+class BuildQueue {
+  private static queue = new Queue();
+  public static push(jobId: string) {
+    BuildQueue.queue.push(jobId);
+    console.timeLog("Added job(" + jobId + ")");
+    Builder.notify();
+  }
+  public static isEmpty() {
+    return BuildQueue.queue.isEmpty();
+  }
+  public static pop() {
+    return BuildQueue.queue.pop();
+  }
+}
+export function pushBuildQueue(job: Job) {
+  job.status = "waiting";
+  BuildQueue.push(job.id);
+}
+
 class Builder {
   private static builderAvailable = true;
 
-  public static notify() {
-    if (Builder.builderAvailable && !buildQueue.isEmpty()) {
+  public static async notify() {
+    if (Builder.builderAvailable && !BuildQueue.isEmpty()) {
       Builder.builderAvailable = false;
-      let jobId = buildQueue.pop();
-      JobPool.get(jobId).status = JobStatus.building;
-      Builder.cleanWorkspace()
-      .then(() => Builder.buildJob(jobId));
+      let jobId = BuildQueue.pop();
+      JobPool.get(jobId).status = "building";
+      Builder.buildJob(jobId);
     }
   }
-  private static cleanWorkspace() {
-    return new Promise(resolve => {
-      exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f")
-      .then(stdout => {
-        console.timeLog("Workspace cleaned");
-        resolve();
-      });
-    })
-  }
   private static async buildJob(jobId: string) {
+    await exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f");
+    console.timeLog("Workspace cleaned");
+
+    let job = JobPool.get(jobId);
+    console.timeLog("Going to build job(" + jobId + ")");
+    let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8"));
+    let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/";
+    fs.ensureDirSync(targetPath);
+    fs.emptyDirSync(targetPath);
+    fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath);
+    console.log("Copied: " + targetPath);
+
+    console.log("Compile started: job(" + jobId + ")");
     try {
-      let job = JobPool.get(jobId);
-      console.timeLog("Going to build job(" + jobId + ")");
-      let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8"));
-      let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/";
-      fs.ensureDirSync(targetPath);
-      fs.emptyDirSync(targetPath);
-      fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath);
-      console.log("Copied: " + targetPath);
-      console.log("Compile started: job(" + jobId + ")");
-      
       await exec("cd " + WORKSPACE + "/appinventor && ant extensions", true);
-      
-      let zip = new AdmZip();
-      zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions");
-      zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo)));
-      let zipPath = OUTPUT_DIR + "/" + jobId + ".zip";
-      zip.writeZip(zipPath);
-      JobPool.get(jobId).status = JobStatus.done;
-      console.log("Done job(" + jobId + "): " + zipPath);
-      Builder.builderAvailable = true;
-      Builder.notify();
-    } catch (reason) {
-      JobPool.get(jobId).status = JobStatus.failed;
-      if (reason instanceof ExecError) {
-        let err =  reason;
-        // Notice that it would not work on windows
-        let stdout = err.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
-        let stderr = err.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
-        JobPool.get(jobId).attachInfo("failInfo", err.message + ": code(" + err.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr);
-      } else {
-        JobPool.get(jobId).attachInfo("failInfo", reason);
-      }
-      console.log("Job(" + jobId + ") build failed", reason);
+    } catch (e) {
+      e =  e;
+      JobPool.get(jobId).status = "failed";
+      // Notice that it would not work on windows
+      let stdout = e.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
+      let stderr = e.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
+      JobPool.get(jobId).attachInfo("failInfo",
+          e.message + ": code(" + e.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr);
+      console.log("Job(" + jobId + ") build failed in part of `ant extensions`");
       Builder.builderAvailable = true;
       Builder.notify();
     }
+
+    let zip = new AdmZip();
+    zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions");
+    zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo)));
+    let zipPath = OUTPUT_DIR + "/" + jobId + ".zip";
+    zip.writeZip(zipPath);
+
+    JobPool.get(jobId).status = "done";
+    console.log("Done job(" + jobId + "): " + zipPath);
+
+    Builder.builderAvailable = true;
+    Builder.notify();
   }
 }
\ No newline at end of file
diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts
index c83df77..9ed9100 100644
--- a/src/pages/build-with-github-repo.ts
+++ b/src/pages/build-with-github-repo.ts
@@ -7,7 +7,7 @@ import { URLSearchParams } from "url";
 
 import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config";
 import { responseSuccess, responseError } from "../index";
-import { addBuildQueue, Job, JobPool, BuildType, JobStatus } from "../builder";
+import { pushBuildQueue, Job, JobPool } from "../builder";
 
 export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
   if (request.method == "GET") {
@@ -23,8 +23,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
       return;
     }
 
-  } else if (request.method == "POST") {
-    // webhook
+  } else if (request.method == "POST") { // webhook
     let indexOfEvent = request.rawHeaders.indexOf("X-GitHub-Event");
     let event: string;
     if (indexOfEvent != -1 && (indexOfEvent + 1) < request.rawHeaders.length) {
@@ -68,11 +67,11 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
   }
 }
 
-function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) {
+async function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) {
   let job = new Job();
   let jobId = job.id;
 
-  job.attachInfo("buildType", BuildType["github-repo"]);
+  job.attachInfo("buildType", "github-repo");
   job.attachInfo("owner", owner);
   job.attachInfo("repoName", repoName);
   job.attachInfo("ref", ref);
@@ -82,38 +81,37 @@ function startGithubJob(response: ServerResponse, owner: string, repoName: strin
     jobId: jobId
   });
 
-  // Downlaod archieve
-  new Github().repos.getArchiveLink({
-    owner: owner,
-    repo: repoName,
-    archive_format: "zipball",
-    ref: ref
-  })
-  .then(archieveResponse => new Promise((resolve, reject) => {
-    let zip = new Admzip( archieveResponse.data);
-    if (zip.getEntries().length == 0) {
-      reject("No source found in archieve downloaded.");
-      return;
-    }
-    let entryDir = zip.getEntries()[0].entryName;
-    zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/");
-    fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir,
-                TEMP_DIR + "/" + jobId + "/src");
-    fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource");
-    console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src");
-    addBuildQueue(job);
-  }))
-  .catch(reason => {
-    JobPool.get(jobId).status = JobStatus.failed;
-    if (Object.getPrototypeOf(reason) == Error.prototype) {
-      let err =  reason;
-      JobPool.get(jobId).attachInfo("failInfo",
-          err.name == "HttpError"
-          ? "Cannot load source from github, please check if the ref exists in the specified repo."
-          : err.message);
-    } else {
-      JobPool.get(jobId).attachInfo("failInfo", reason.toString());
-    }
-    console.log("Fail prepare source of job(" + jobId + ")", reason);
-  });
+  // Downlaod archive
+  let archiveResponse: Github.Response;
+  try {
+    archiveResponse = await new Github().repos.getArchiveLink({
+      owner: owner,
+      repo: repoName,
+      archive_format: "zipball",
+      ref: ref
+    });
+  } catch (e) {
+    let err =  e;
+    JobPool.get(jobId).status = "failed";
+    JobPool.get(jobId).attachInfo("failInfo",
+        err.name == "HttpError"
+        ? "Cannot load source from github, please check if the ref exists in the specified repo."
+        : err.message);
+    console.log("Fail prepare source of job(" + jobId + ")", e);
+    return;
+  }
+
+  // Extract responding archive
+  let zip = new Admzip( archiveResponse.data);
+  if (zip.getEntries().length == 0) {
+    throw "No source found in archive downloaded.";
+  }
+  let entryDir = zip.getEntries()[0].entryName;
+  zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/");
+  fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir,
+              TEMP_DIR + "/" + jobId + "/src");
+  fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource");
+  console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src");
+
+  pushBuildQueue(job);
 }
\ No newline at end of file
diff --git a/src/pages/build-with-zip.ts b/src/pages/build-with-zip.ts
index 67e497c..9110f38 100644
--- a/src/pages/build-with-zip.ts
+++ b/src/pages/build-with-zip.ts
@@ -1,4 +1,3 @@
-import * as fs from "fs-extra";
 import * as AdmZip from "adm-zip";
 import * as formidable from "formidable"
 
@@ -7,7 +6,7 @@ import { URLSearchParams } from "url";
 
 import { responseError, responseSuccess } from "../index";
 import { ENABLE_REPO_WHITELIST, TEMP_DIR } from "../config";
-import { Job, addBuildQueue, BuildType } from "../builder";
+import { Job, pushBuildQueue } from "../builder";
 
 export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
   if (ENABLE_REPO_WHITELIST) {
@@ -15,9 +14,8 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
 
   } else {
     let job = new Job();
+    job.attachInfo("buildType", "source-upload");
 
-    job.attachInfo("buildType", BuildType["source-upload"]);
-    
     let type = request.headers["content-type"] || "";
     if (!type.includes("multipart/form-data")) {
       console.log("Request content type: " + type);
@@ -35,7 +33,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
       }
       let zip = new AdmZip(file.path);
       zip.extractAllTo(jobDir + "/src/");
-      addBuildQueue(job);
+      pushBuildQueue(job);
       responseSuccess(response, {
         msg: "Job added.",
         jobId: job.id
diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts
index 188dece..fd3e7ad 100644
--- a/src/pages/check-status.ts
+++ b/src/pages/check-status.ts
@@ -1,9 +1,9 @@
+import * as fs from "fs-extra";
 import { IncomingMessage, ServerResponse } from "http";
 import { URLSearchParams } from "url";
 
 import { responseSuccess, responseError } from "../index";
-import { JobPool, JobStatus } from "../builder";
-import * as fs from "fs-extra";
+import { JobPool } from "../builder";
 import { OUTPUT_DIR } from "../config";
 
 export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
@@ -20,7 +20,7 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
       ret[key] = job.extraInfo[key];
     }
   } else {
-    ret.status = JobStatus.done;
+    ret.status = "done";
   }
   responseSuccess(response, ret);
 }
\ No newline at end of file
diff --git a/src/pages/result.ts b/src/pages/result.ts
index 4d66edf..d7f6b4e 100644
--- a/src/pages/result.ts
+++ b/src/pages/result.ts
@@ -3,7 +3,7 @@ import * as fs from "fs-extra";
 import { IncomingMessage, ServerResponse } from "http";
 import { URLSearchParams } from "url";
 
-import { JobPool, JobStatus } from "../builder";
+import { JobPool } from "../builder";
 import { OUTPUT_DIR } from "../config";
 
 export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
@@ -14,8 +14,8 @@ export default (request: IncomingMessage, response: ServerResponse, params: URLS
     response.end("Job does not exist.");
     return;
   }
-  let status = JobPool.has(jobId) ? JobPool.get(jobId).status : JobStatus.done;
-  if (status != JobStatus.done) {
+  let status = JobPool.has(jobId) ? JobPool.get(jobId).status : "done";
+  if (status != "done") {
     console.log("Response end with 404 job is not ready yet");
     response.writeHead(404);
     response.end("Job not ready yet.");
diff --git a/src/utils/exec.ts b/src/utils/exec.ts
index 4ec11c8..1148ced 100644
--- a/src/utils/exec.ts
+++ b/src/utils/exec.ts
@@ -1,40 +1,29 @@
-import { exec } from "shelljs";
+import { exec, ExecOutputReturnValue } from "shelljs";
 
-/**
- * Execute a command with promise returns
- * @param command
- * @returns a Promise.
- *    With then(stdout => {execute when result code is 0})
- *    and catch((response = [code, stderr]) => {execute when code other than 0 returned})
- */
-export default (command: string, silent = false) => {
-  return new Promise((resolve, reject) => {
-    exec(command, { silent: silent }, (code, stdout, stderr) => {
-      if (code == 0) {
-        resolve(stdout);
-      } else {
-        reject(new ExecError(code, stdout, stderr));
-      }
-    });
+export default async (command: string, silent = false) => {
+  let result =  exec(command, {
+    silent: silent,
+    async: false // ensure it returns ExecOutputReturnValue
   });
+  if (result.code == 0) {
+    return result.stdout;
+  } else {
+    throw new ExecError(result);
+  }
 }
 export class ExecError extends Error {
-  private _code: number;
-  private _stdout: string;
-  private _stderr: string;
-  constructor(code: number, stdout: string, stderr: string) {
+  private _execOutputReturnValue: ExecOutputReturnValue;
+  constructor(execOutputReturnValue: ExecOutputReturnValue) {
     super("Command executing error occured");
-    this._code = code;
-    this._stdout = stdout;
-    this._stderr = stderr;
+    this._execOutputReturnValue = execOutputReturnValue;
   }
   get code() {
-    return this._code;
+    return this._execOutputReturnValue.code;
   }
   get stdout() {
-    return this._stdout;
+    return this._execOutputReturnValue.stdout;
   }
   get stderr() {
-    return this._stderr;
+    return this._execOutputReturnValue.stderr;
   }
 }
\ No newline at end of file
diff --git a/src/utils/queue.ts b/src/utils/queue.ts
index c066e7c..1df06ed 100644
--- a/src/utils/queue.ts
+++ b/src/utils/queue.ts
@@ -1,5 +1,4 @@
 export default class Queue {
-
   private elements: Array;
 
   public constructor() {
@@ -9,19 +8,15 @@ export default class Queue {
   public push(o: T) {
     this.elements.unshift(o);
   }
-
   public pop(): T {
     return this.elements.pop();
   }
-
   public size(): number {
     return this.elements.length;
   }
-
   public isEmpty(): boolean {
     return this.size() == 0;
   }
-
   public clear() {
     delete this.elements;
     this.elements = new Array();