diff --git a/package-lock.json b/package-lock.json index d28b204..fe5117f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,47 @@ { "name": "extension-builder", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "@octokit/endpoint": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-3.1.3.tgz", + "integrity": "sha512-vAWzeoj9Lzpl3V3YkWKhGzmDUoMfKpyxJhpq74/ohMvmLXDoEuAGnApy/7TRi3OmnjyX2Lr+e9UGGAD0919ohA==", + "requires": { + "deepmerge": "3.2.0", + "is-plain-object": "2.0.4", + "universal-user-agent": "2.0.3", + "url-template": "2.0.8" + } + }, + "@octokit/request": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-2.4.0.tgz", + "integrity": "sha512-Bm2P0duVRUeKhyepNyFg5GX+yhCK71fqdtpsw5Rz+PQPjSha8HYwPMF5QfpzpD8b6/Xl3xhTgu3V90W362gZ1A==", + "requires": { + "@octokit/endpoint": "3.1.3", + "is-plain-object": "2.0.4", + "node-fetch": "2.3.0", + "universal-user-agent": "2.0.3" + } + }, + "@octokit/rest": { + "version": "16.16.3", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-16.16.3.tgz", + "integrity": "sha512-8v5xyqXZwQbQ1WsTLU3G25nAlcKYEgIXzDeqLgTFpbzzJXcey0C8Mcs/LZiAgU8dDINZtO2dAPgd1cVKgK9DQw==", + "requires": { + "@octokit/request": "2.4.0", + "before-after-hook": "1.3.2", + "btoa-lite": "1.0.0", + "lodash.get": "4.4.2", + "lodash.set": "4.3.2", + "lodash.uniq": "4.5.0", + "octokit-pagination-methods": "1.1.0", + "universal-user-agent": "2.0.3", + "url-template": "2.0.8" + } + }, "@types/adm-zip": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.32.tgz", @@ -19,6 +57,16 @@ "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==", "dev": true }, + "@types/formidable": { + "version": "1.0.31", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-1.0.31.tgz", + "integrity": "sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q==", + "dev": true, + "requires": { + "@types/events": "3.0.0", + "@types/node": "11.9.6" + } + }, "@types/fs-extra": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.5.tgz", @@ -77,6 +125,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "before-after-hook": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.3.2.tgz", + "integrity": "sha512-zyPgY5dgbf99c0uGUjhY4w+mxqEGxPKg9RQDl34VvrVh2bM31lFN+mwR1ZHepq/KA3VCPk1gwJZL6IIJqjLy2w==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -86,11 +139,52 @@ "concat-map": "0.0.1" } }, + "btoa-lite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", + "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "requires": { + "nice-try": "1.0.5", + "path-key": "2.0.1", + "semver": "5.6.0", + "shebang-command": "1.2.0", + "which": "1.3.1" + } + }, + "deepmerge": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.2.0.tgz", + "integrity": "sha512-6+LuZGU7QCNUnAJyX8cIrlzoEgggTM6B7mm+znKOX4t5ltluT9KLjN6g61ECMS0LTsLW7yDpNoxhix5FZcrIow==" + }, + "execa": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", + "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "requires": { + "cross-spawn": "6.0.5", + "get-stream": "3.0.0", + "is-stream": "1.1.0", + "npm-run-path": "2.0.2", + "p-finally": "1.0.0", + "signal-exit": "3.0.2", + "strip-eof": "1.0.0" + } + }, + "formidable": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz", + "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==" + }, "fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -106,6 +200,11 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + }, "glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", @@ -143,6 +242,29 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz", "integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==" }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "requires": { + "isobject": "3.0.1" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" + }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", @@ -151,6 +273,26 @@ "graceful-fs": "4.1.15" } }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, + "macos-release": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.0.0.tgz", + "integrity": "sha512-iCM3ZGeqIzlrH7KxYK+fphlJpCCczyHXc+HhRVbEu9uNTCrzYJjvvtefzeKTCVHd5AP/aD/fzC80JZ4ZP+dQ/A==" + }, "mime-db": { "version": "1.38.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", @@ -172,6 +314,29 @@ "brace-expansion": "1.1.11" } }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" + }, + "node-fetch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz", + "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==" + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "requires": { + "path-key": "2.0.1" + } + }, + "octokit-pagination-methods": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz", + "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -180,11 +345,30 @@ "wrappy": "1.0.2" } }, + "os-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.0.0.tgz", + "integrity": "sha512-7c74tib2FsdFbQ3W+qj8Tyd1R3Z6tuVRNNxXjJcZ4NgjIEQU9N/prVMqcW29XZPXGACqaXN3jq58/6hoaoXH6g==", + "requires": { + "macos-release": "2.0.0", + "windows-release": "3.1.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" + }, "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", @@ -206,6 +390,24 @@ "path-parse": "1.0.6" } }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "requires": { + "shebang-regex": "1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" + }, "shelljs": { "version": "0.8.3", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.3.tgz", @@ -216,11 +418,50 @@ "rechoir": "0.6.2" } }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" + }, + "universal-user-agent": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.3.tgz", + "integrity": "sha512-eRHEHhChCBHrZsA4WEhdgiOKgdvgrMIHwnwnqD0r5C6AO8kwKcG7qSku3iXdhvHL3YvsS9ZkSGN8h/hIpoFC8g==", + "requires": { + "os-name": "3.0.0" + } + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "2.0.0" + } + }, + "windows-release": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.1.0.tgz", + "integrity": "sha512-hBb7m7acFgQPQc222uEQTmdcGLeBmQLNLFIh0rDk3CwFOBrfjefLzEfEfmpMq8Af/n/GnFf3eYf203FY1PmudA==", + "requires": { + "execa": "0.10.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 0ec9e40..7e35298 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "extension-builder", - "version": "1.0.0", + "version": "1.1.0", "description": "", "main": "index.js", "repository": { @@ -14,7 +14,9 @@ }, "homepage": "https://github.com/ColinTree-bot/extension-builder#readme", "dependencies": { + "@octokit/rest": "^16.16.3", "adm-zip": "^0.4.13", + "formidable": "^1.2.1", "fs-extra": "^7.0.1", "mime-types": "^2.1.22", "shelljs": "^0.8.3" @@ -24,6 +26,7 @@ }, "devDependencies": { "@types/adm-zip": "^0.4.32", + "@types/formidable": "^1.0.31", "@types/fs-extra": "^5.0.5", "@types/mime-types": "^2.1.0", "@types/shelljs": "^0.8.3" diff --git a/src/builder.ts b/src/builder.ts index 80bef19..dc09ccd 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -1,22 +1,14 @@ import * as fs from "fs-extra"; import * as AdmZip from "adm-zip" -import exec from "./utils/exec"; +import exec, { ExecError } from "./utils/exec"; import { WORKSPACE, TEMP_DIR, BUILDER_CONFIG_NAME, OUTPUT_DIR } from "./config"; import Queue from "./utils/queue"; -class BuildQueue extends Queue { - constructor() { super(); } - public push(jobId: string) { - super.push(jobId); - console.timeLog("Added job(" + jobId + ")"); - Builder.notify(); - } -} export class JobPool { private static pool = new Map(); - public static add(jobId: string, job: Job) { - JobPool.pool.set(jobId, job); + public static add(job: Job) { + JobPool.pool.set(job.id, job); } public static get(jobId: string): Job { return JobPool.pool.get(jobId); @@ -25,96 +17,99 @@ export class JobPool { return JobPool.pool.has(jobId); } } -const buildQueue = new BuildQueue(); - -export function addBuildQueue(jobId: string) { - console.timeLog("Job(" + jobId + ") going to be added into build queue"); - // job will be added to build queue by itself after it is ready (in constructor) - JobPool.add(jobId, new Job(jobId)); -} - -interface JobConfig { - package: string -} -class Job { +type JobStatus = "preparing" | "waiting" | "building" | "done" | "failed"; +export class Job { private _id: string; - private _config: JobConfig; + private _extraInfo: { [key: string]: string | number } = {}; get id() { return this._id; } - get config() { return this._config; } + get extraInfo() { return this._extraInfo; } - public status: "waiting" | "building" | "done" | "failed"; + public status: JobStatus; - public constructor(jobId: string) { - this._id = jobId; - this.status = "waiting"; - this.loadConfig() - .then(config => { - this._config = config; - buildQueue.push(jobId); - }); + public constructor() { + fs.ensureDirSync(TEMP_DIR); + let jobDir = fs.mkdtempSync(TEMP_DIR + "/"); + this._id = jobDir.substring(jobDir.lastIndexOf("/") + 1); + this.status = "preparing"; + this.attachInfo("startTimestamp", Date.now()); + JobPool.add(this); } - private loadConfig() { - return new Promise(resolve => { - fs.readFile(TEMP_DIR + "/" + this.id + "/src/" + BUILDER_CONFIG_NAME, - "utf8", (err, data) => { - if (err) throw err; - resolve(JSON.parse(data)); - }); - }); + + public attachInfo(key: string, value: string | number) { + this._extraInfo[key] = value; } } +class BuildQueue { + private static queue = new Queue(); + public static push(jobId: string) { + BuildQueue.queue.push(jobId); + console.timeLog("Added job(" + jobId + ")"); + Builder.notify(); + } + public static isEmpty() { + return BuildQueue.queue.isEmpty(); + } + public static pop() { + return BuildQueue.queue.pop(); + } +} +export function pushBuildQueue(job: Job) { + job.status = "waiting"; + BuildQueue.push(job.id); +} + class Builder { private static builderAvailable = true; - public static notify() { - if (Builder.builderAvailable && !buildQueue.isEmpty()) { + public static async notify() { + if (Builder.builderAvailable && !BuildQueue.isEmpty()) { Builder.builderAvailable = false; - let jobId = buildQueue.pop(); + let jobId = BuildQueue.pop(); JobPool.get(jobId).status = "building"; - Builder.cleanWorkspace() - .then(() => Builder.buildJob(jobId)); + Builder.buildJob(jobId); } } - private static cleanWorkspace() { - return new Promise(resolve => { - exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f") - .then(stdout => { - console.timeLog("Workspace cleaned"); - resolve(); - }); - }) - } - private static buildJob(jobId: string) { - return new Promise(resolve => { - let job = JobPool.get(jobId); - console.timeLog("Going to build job(" + jobId + ")"); - let config = job.config; - let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/"; - fs.ensureDirSync(targetPath); - fs.emptyDirSync(targetPath); - fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath); - console.log("Copied: " + targetPath); - console.log("Compile started: job(" + jobId + ")"); - exec("cd " + WORKSPACE + "/appinventor && ant extensions", true) - .then(stdout => { - let zip = new AdmZip(); - zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); - let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; - zip.writeZip(zipPath); - JobPool.get(jobId).status = "done"; - console.log("Done job(" + jobId + "): " + zipPath); - Builder.builderAvailable = true; - Builder.notify(); - resolve(); - }) - .catch(reason => { - JobPool.get(jobId).status = "failed"; - console.log("Job(" + jobId + ") build failed", reason); - Builder.builderAvailable = true; - Builder.notify(); - }); - }); + private static async buildJob(jobId: string) { + await exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f"); + console.timeLog("Workspace cleaned"); + + let job = JobPool.get(jobId); + console.timeLog("Going to build job(" + jobId + ")"); + let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8")); + let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/"; + fs.ensureDirSync(targetPath); + fs.emptyDirSync(targetPath); + fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath); + console.log("Copied: " + targetPath); + + console.log("Compile started: job(" + jobId + ")"); + try { + await exec("cd " + WORKSPACE + "/appinventor && ant extensions", true); + } catch (e) { + e = e; + JobPool.get(jobId).status = "failed"; + // Notice that it would not work on windows + let stdout = e.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + let stderr = e.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/"); + JobPool.get(jobId).attachInfo("failInfo", + e.message + ": code(" + e.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr); + console.log("Job(" + jobId + ") build failed in part of `ant extensions`"); + Builder.builderAvailable = true; + Builder.notify(); + } + + let zip = new AdmZip(); + zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions"); + zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo))); + let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; + zip.writeZip(zipPath); + + JobPool.get(jobId).status = "done"; + console.log("Done job(" + jobId + "): " + zipPath); + + Builder.builderAvailable = true; + Builder.notify(); } } \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 9972e78..0fbcdc3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,7 @@ import * as os from "os"; export const WORKSPACE = "/usr/workspace/"; -export const ENABLE_REPO_WHITELIST = true; +export const ENABLE_REPO_WHITELIST = false; export const PORT = 8048; export const TEMP_DIR = os.tmpdir() + "/extension-builder/"; export const BUILDER_CONFIG_NAME = "builder-config.json"; @@ -13,22 +13,22 @@ export const EMPTY_TEMP_DIR_BEFORE_BUILD = false; interface WhiteList { owner: string; repoName: string; - branch: string | string[]; + refs: string | string[]; // refs includes branchs, commits and tags. Can be "*" for any } const REPO_WHITELIST: WhiteList[] = [ - { owner: "OpenSourceAIX", repoName: "ColinTreeListView", branch: "extension-builder-test" } + { owner: "OpenSourceAIX", repoName: "ColinTreeListView", refs: "extension-builder-test" } ]; -export function inWhitelist(owner: string, repoName: string, branch = "master") { +export function inWhitelist(owner: string, repoName: string, coderef = "") { for (let i in REPO_WHITELIST) { if (REPO_WHITELIST.hasOwnProperty(i)) { let item = REPO_WHITELIST[i]; if (owner == item.owner && repoName == item.repoName) { - let acceptBranchs = item.branch; - if (acceptBranchs == "*") { + let acceptRefs = item.refs; + if (acceptRefs == "*") { return true; } else { - acceptBranchs = typeof(acceptBranchs)=="string" ? [ acceptBranchs ] : acceptBranchs; - return acceptBranchs.includes(branch); + acceptRefs = typeof(acceptRefs)=="string" ? [ acceptRefs ] : acceptRefs; + return acceptRefs.includes(coderef); } } } diff --git a/src/index.ts b/src/index.ts index ed54537..526a8ef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,10 +11,13 @@ import handleResult from "./pages/result"; export const CONTENT_TYPE_JSON = {"Content-Type": "application/json"}; export function responseSuccess(response: http.ServerResponse, info: {}) { + info = JSON.stringify(info); + console.log("Response end with 200: " + info); response.writeHead(200, CONTENT_TYPE_JSON); - response.end(JSON.stringify(info)); + response.end(info); } export function responseError(response: http.ServerResponse, code: number, msg: string) { + console.log("Response end with " + code + ": " + msg); response.writeHead(code, CONTENT_TYPE_JSON); response.end(JSON.stringify({ msg: msg })); } @@ -39,7 +42,7 @@ function handleStaticFile(response: http.ServerResponse, pathname: string): bool } pathname = staticDir + pathname; let mime = mimeTypes.lookup(pathname); - console.log("Returning static file(" + pathname + ") mime(" + mime + ")"); + console.log("Response end with 200: static file(" + pathname + ") mime(" + mime + ")"); response.writeHead(200, { "Content-Type": mime!==false ? mime : "application/octet-stream" }); fs.createReadStream(pathname).pipe(response, { end: true }); return true; @@ -52,22 +55,27 @@ function startServer() { let params = new url.URLSearchParams(requestUrl.query); console.timeLog("Processing request: " + request.url); switch (requestUrl.pathname) { - case "/build-with-github-repo": + case "/build-with-github-repo": { handleBuildWithGithubRepo(request, response, params); return; - case "/build-with-zip": + } + case "/build-with-zip": { handleBuildWithZip(request, response, params); return; - case "/check-status": + } + case "/check-status": { handleCheckStatus(request, response, params); return; - case "/result": + } + case "/result": { handleResult(request, response, params); return; - default: + } + default: { if (handleStaticFile(response, requestUrl.pathname)) { return; } + } } responseError(response, 404, "404 Not found."); } catch (error) { diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts index 8c7d008..9ed9100 100644 --- a/src/pages/build-with-github-repo.ts +++ b/src/pages/build-with-github-repo.ts @@ -1,93 +1,117 @@ -import * as https from "https"; import * as fs from "fs-extra"; import * as Admzip from "adm-zip"; +import * as Github from "@octokit/rest"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config"; import { responseSuccess, responseError } from "../index"; -import { addBuildQueue } from "../builder"; +import { pushBuildQueue, Job, JobPool } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (request.method == "GET") { let owner = params.get("owner"); let repoName = params.get("repoName"); - let branch = params.has("branch") ? params.get("branch") : "master"; - - console.timeLog("Job info received: repo= " + owner + "/" + repoName + " branch= " + branch); - - if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, branch)) { - // 1. white list is not enabled - // 2. is in the white list - - // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId} - let jobId = fs.mkdtempSync(TEMP_DIR + "/"); - jobId = jobId.substring(jobId.lastIndexOf("/") + 1); - - responseSuccess(response, { - msg: "Build started.", - jobId: jobId - }); - - getZip(jobId, owner, repoName, branch) - .then(zipName => prepareSource(jobId, zipName)) - .then(() => addBuildQueue(jobId)) - .catch(reason => { - console.error(reason); - }); + let ref = params.has("ref") ? params.get("ref") : "master"; + console.timeLog("Job info received: repo= " + owner + "/" + repoName + " ref= " + ref); + if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, ref)) { + startGithubJob(response, owner, repoName, ref); return; } else { - responseError(response, 403, "Whitelist is enabled & your repo is not in it."); + responseError(response, 403, "Whitelist is enabled & your repo (or ref) is not in it."); return; } - } else if (request.method == "POST") { - // webhook + } else if (request.method == "POST") { // webhook + let indexOfEvent = request.rawHeaders.indexOf("X-GitHub-Event"); + let event: string; + if (indexOfEvent != -1 && (indexOfEvent + 1) < request.rawHeaders.length) { + event = request.rawHeaders[indexOfEvent + 1]; + } + console.log("Github event = " + event); + if (!["push", "release"].includes(event)) { + responseError(response, 403, "Does not support event type: " + event); + return; + } let content = ""; - request.on("data", chunk => { content += chunk; }); - request.on("end", () => { - responseSuccess(response, { content: content }); + let playload = JSON.parse(content); + let commitOrTag: string; + switch (event) { + case "push": + commitOrTag = playload.head_commit.id; + break; + case "release": + commitOrTag = playload.release.tag_name; + break; + } + let owner = playload.repository.owner.login; + let repoName = playload.repository.name; + console.log("Repo = " + owner + "/" + repoName + " commitOrTag = " + commitOrTag); + if (typeof(owner)!="string" || typeof(repoName)!="string" || commitOrTag) { + responseError(response, 403, "Cannot be built, at least one of owner, repoName or commit is not string."); + return; + } + if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName)) { + startGithubJob(response, owner, repoName, commitOrTag); + return; + } else { + responseError(response, 403, "Whitelist is enabled & your repo is not in it."); + return; + } }); - return; } } -function getZip(jobId: string, owner: string, repoName: string, branch: string) { - return new Promise(resolve => { - // https://github.com/{owner}/{repoName}/archive/{branch}.zip - // redirecting to -> https://codeload.github.com/{owner}/{repoName}/zip/{branch} - let requestUrl = "https://codeload.github.com/" + owner + "/" + repoName + "/zip/" + branch; - let zipName = owner + "_" + repoName + "_" + branch; - let destZipPath = TEMP_DIR + "/" + jobId + "/" + zipName + ".zip"; - https.get(requestUrl, response => { - var stream = response.pipe(fs.createWriteStream(destZipPath)); - stream.on("finish", () => { - console.timeLog("Downloaded: " + destZipPath); - resolve(zipName); - }); - }); +async function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) { + let job = new Job(); + let jobId = job.id; + + job.attachInfo("buildType", "github-repo"); + job.attachInfo("owner", owner); + job.attachInfo("repoName", repoName); + job.attachInfo("ref", ref); + + responseSuccess(response, { + msg: "Job added.", + jobId: jobId }); -} -function prepareSource(jobId: string, zipPath: string) { - return new Promise(resolve => { - let zip = new Admzip(TEMP_DIR + "/" + jobId + "/" + zipPath + ".zip"); - let entryDir: string; - zip.getEntries().forEach(entry => { - if (!entryDir) { - entryDir = entry.entryName; - } + // Downlaod archive + let archiveResponse: Github.Response; + try { + archiveResponse = await new Github().repos.getArchiveLink({ + owner: owner, + repo: repoName, + archive_format: "zipball", + ref: ref }); - zip.extractEntryTo(entryDir, TEMP_DIR + "/" + jobId + "/rawComponentSource/"); - fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, - TEMP_DIR + "/" + jobId + "/src"); - fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); - console.timeLog("Unzipped & moved to " + TEMP_DIR + "/" + jobId + "/src"); - resolve(); - }); + } catch (e) { + let err = e; + JobPool.get(jobId).status = "failed"; + JobPool.get(jobId).attachInfo("failInfo", + err.name == "HttpError" + ? "Cannot load source from github, please check if the ref exists in the specified repo." + : err.message); + console.log("Fail prepare source of job(" + jobId + ")", e); + return; + } + + // Extract responding archive + let zip = new Admzip( archiveResponse.data); + if (zip.getEntries().length == 0) { + throw "No source found in archive downloaded."; + } + let entryDir = zip.getEntries()[0].entryName; + zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/"); + fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir, + TEMP_DIR + "/" + jobId + "/src"); + fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource"); + console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src"); + + pushBuildQueue(job); } \ No newline at end of file diff --git a/src/pages/build-with-zip.ts b/src/pages/build-with-zip.ts index e327f22..9110f38 100644 --- a/src/pages/build-with-zip.ts +++ b/src/pages/build-with-zip.ts @@ -1,15 +1,48 @@ +import * as AdmZip from "adm-zip"; +import * as formidable from "formidable" + import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; -import { responseError } from "../index"; -import { ENABLE_REPO_WHITELIST } from "../config"; +import { responseError, responseSuccess } from "../index"; +import { ENABLE_REPO_WHITELIST, TEMP_DIR } from "../config"; +import { Job, pushBuildQueue } from "../builder"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { if (ENABLE_REPO_WHITELIST) { responseError(response, 403, "Currently in white list mode, build with zip is disabled"); - return; + } else { - responseError(response, 501, "build with zip is not implemented yet."); + let job = new Job(); + job.attachInfo("buildType", "source-upload"); + + let type = request.headers["content-type"] || ""; + if (!type.includes("multipart/form-data")) { + console.log("Request content type: " + type); + responseError(response, 400, "Please use multipart/form-data format"); + return; + } + + let jobDir = TEMP_DIR + "/" + job.id + "/"; + let form = new formidable.IncomingForm(); + form.uploadDir = jobDir; + form.keepExtensions = true; + form.on("file", (name, file) => { + if (name != "source") { + return; + } + let zip = new AdmZip(file.path); + zip.extractAllTo(jobDir + "/src/"); + pushBuildQueue(job); + responseSuccess(response, { + msg: "Job added.", + jobId: job.id + }); + }); + form.on("error", err => { + responseError(response, 500, err); + }) + form.parse(request); return; } } \ No newline at end of file diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts index 52bb500..fd3e7ad 100644 --- a/src/pages/check-status.ts +++ b/src/pages/check-status.ts @@ -1,15 +1,26 @@ +import * as fs from "fs-extra"; import { IncomingMessage, ServerResponse } from "http"; import { URLSearchParams } from "url"; import { responseSuccess, responseError } from "../index"; import { JobPool } from "../builder"; +import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { let jobId = params.get("jobId"); - if (!JobPool.has(jobId)) { - // TODO: try harder to find this job in "build-result"? + if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) { responseError(response, 404, "Specified job does not exist in job pool."); return; } - responseSuccess(response, { status: JobPool.get(jobId).status }); + let ret: { [key: string]: string | number } = {}; + if (JobPool.has(jobId)) { + let job = JobPool.get(jobId); + ret.status = job.status; + for (let key in job.extraInfo) { + ret[key] = job.extraInfo[key]; + } + } else { + ret.status = "done"; + } + responseSuccess(response, ret); } \ No newline at end of file diff --git a/src/pages/result.ts b/src/pages/result.ts index e7c9d23..d7f6b4e 100644 --- a/src/pages/result.ts +++ b/src/pages/result.ts @@ -8,19 +8,22 @@ import { OUTPUT_DIR } from "../config"; export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => { let jobId = params.get("jobId"); - if (!JobPool.has(jobId)) { + if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) { + console.log("Response end with 404: job not exist"); response.writeHead(404); response.end("Job does not exist."); return; } - let status = JobPool.get(jobId).status; + let status = JobPool.has(jobId) ? JobPool.get(jobId).status : "done"; if (status != "done") { + console.log("Response end with 404 job is not ready yet"); response.writeHead(404); response.end("Job not ready yet."); return; } else { let zipPath = OUTPUT_DIR + "/" + jobId + ".zip"; var stat = fs.statSync(zipPath); + console.log("Response end with 200 (build result will be sent)"); response.writeHead(200, { "Content-Type": "application/zip", "Content-Length": stat.size diff --git a/src/utils/exec.ts b/src/utils/exec.ts index 7db2582..1148ced 100644 --- a/src/utils/exec.ts +++ b/src/utils/exec.ts @@ -1,20 +1,29 @@ -import { exec } from "shelljs"; +import { exec, ExecOutputReturnValue } from "shelljs"; -/** - * Execute a command with promise returns - * @param command - * @returns a Promise. - * With then(stdout => {execute when result code is 0}) - * and catch((response = [code, stderr]) => {execute when code other than 0 returned}) - */ -export default (command: string, silent = false) => { - return new Promise((resolve, reject) => { - exec(command, { silent: silent }, (code, stdout, stderr) => { - if (code == 0) { - resolve(stdout); - } else { - reject([code, stderr]); - } - }); +export default async (command: string, silent = false) => { + let result = exec(command, { + silent: silent, + async: false // ensure it returns ExecOutputReturnValue }); + if (result.code == 0) { + return result.stdout; + } else { + throw new ExecError(result); + } +} +export class ExecError extends Error { + private _execOutputReturnValue: ExecOutputReturnValue; + constructor(execOutputReturnValue: ExecOutputReturnValue) { + super("Command executing error occured"); + this._execOutputReturnValue = execOutputReturnValue; + } + get code() { + return this._execOutputReturnValue.code; + } + get stdout() { + return this._execOutputReturnValue.stdout; + } + get stderr() { + return this._execOutputReturnValue.stderr; + } } \ No newline at end of file diff --git a/src/utils/queue.ts b/src/utils/queue.ts index c066e7c..1df06ed 100644 --- a/src/utils/queue.ts +++ b/src/utils/queue.ts @@ -1,5 +1,4 @@ export default class Queue { - private elements: Array; public constructor() { @@ -9,19 +8,15 @@ export default class Queue { public push(o: T) { this.elements.unshift(o); } - public pop(): T { return this.elements.pop(); } - public size(): number { return this.elements.length; } - public isEmpty(): boolean { return this.size() == 0; } - public clear() { delete this.elements; this.elements = new Array(); diff --git a/static/index.html b/static/index.html index 5254800..7e5cac7 100644 --- a/static/index.html +++ b/static/index.html @@ -16,12 +16,19 @@ if (xhr.status == 404) { alert("404! maybe server is too busy to handle our request"); return; + } else if (xhr.status != 200) { + alert(xhr.status + "!" + xhr.responseJSON.msg); + return; } $("#status").text(xhr.responseJSON.status); if (xhr.responseJSON.status == "done") { callback_whenDone(); } else { - checkStatus(jobId, callback_whenDone, times + 1); + if (xhr.responseJSON.status == "failed") { + $("#output").text(xhr.responseJSON.failInfo); + } else { + checkStatus(jobId, callback_whenDone, times + 1); + } } } }); @@ -34,7 +41,7 @@ data: { owner: $("#owner").val(), repoName: $("#repoName").val(), - branch: $("#branch").val() + ref: $("#ref").val() }, dataType: "json", success: json => { @@ -49,6 +56,36 @@ }, error: xhr => { console.log(xhr.responseJSON); + alert(xhr.responseJSON); + } + }); + $("[type=submit]").prop("disabled", true); + return false; + }); + $("#buildWithZip").submit(() => { + let formData = new FormData(); + formData.append("source", $("#zip")[0].files[0]); + $.ajax({ + url: "/build-with-zip", + type: "POST", + cache: false, + data: formData, + processData: false, + contentType: false, + dataType: "json", + success: json => { + let jobId = json["jobId"]; + $("#jobId").text(jobId); + $("#status").text("submitted"); + checkStatus(jobId, () => { + $("#download") + .attr("href", "/result?jobId=" + jobId) + .parent().show(); + }); + }, + error: xhr => { + console.log(xhr.responseJSON); + alert(xhr.responseJSON); } }); $("[type=submit]").prop("disabled", true); @@ -56,7 +93,7 @@ }); $("#previewRepo").click(() => { window.open( - "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#branch").val(), + "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#ref").val(), "_blank").focus(); return false; }) @@ -73,7 +110,7 @@ Repo Name - Branch + Ref @@ -81,6 +118,15 @@ +
+ +
+ + +
+ +
+

jobId:

@@ -89,6 +135,8 @@ Status:

+

+
 

Download