diff --git a/package-lock.json b/package-lock.json
index d28b204..fe5117f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,9 +1,47 @@
{
"name": "extension-builder",
- "version": "1.0.0",
+ "version": "1.1.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+ "@octokit/endpoint": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-3.1.3.tgz",
+ "integrity": "sha512-vAWzeoj9Lzpl3V3YkWKhGzmDUoMfKpyxJhpq74/ohMvmLXDoEuAGnApy/7TRi3OmnjyX2Lr+e9UGGAD0919ohA==",
+ "requires": {
+ "deepmerge": "3.2.0",
+ "is-plain-object": "2.0.4",
+ "universal-user-agent": "2.0.3",
+ "url-template": "2.0.8"
+ }
+ },
+ "@octokit/request": {
+ "version": "2.4.0",
+ "resolved": "https://registry.npmjs.org/@octokit/request/-/request-2.4.0.tgz",
+ "integrity": "sha512-Bm2P0duVRUeKhyepNyFg5GX+yhCK71fqdtpsw5Rz+PQPjSha8HYwPMF5QfpzpD8b6/Xl3xhTgu3V90W362gZ1A==",
+ "requires": {
+ "@octokit/endpoint": "3.1.3",
+ "is-plain-object": "2.0.4",
+ "node-fetch": "2.3.0",
+ "universal-user-agent": "2.0.3"
+ }
+ },
+ "@octokit/rest": {
+ "version": "16.16.3",
+ "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-16.16.3.tgz",
+ "integrity": "sha512-8v5xyqXZwQbQ1WsTLU3G25nAlcKYEgIXzDeqLgTFpbzzJXcey0C8Mcs/LZiAgU8dDINZtO2dAPgd1cVKgK9DQw==",
+ "requires": {
+ "@octokit/request": "2.4.0",
+ "before-after-hook": "1.3.2",
+ "btoa-lite": "1.0.0",
+ "lodash.get": "4.4.2",
+ "lodash.set": "4.3.2",
+ "lodash.uniq": "4.5.0",
+ "octokit-pagination-methods": "1.1.0",
+ "universal-user-agent": "2.0.3",
+ "url-template": "2.0.8"
+ }
+ },
"@types/adm-zip": {
"version": "0.4.32",
"resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.32.tgz",
@@ -19,6 +57,16 @@
"integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==",
"dev": true
},
+ "@types/formidable": {
+ "version": "1.0.31",
+ "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-1.0.31.tgz",
+ "integrity": "sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q==",
+ "dev": true,
+ "requires": {
+ "@types/events": "3.0.0",
+ "@types/node": "11.9.6"
+ }
+ },
"@types/fs-extra": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.5.tgz",
@@ -77,6 +125,11 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
+ "before-after-hook": {
+ "version": "1.3.2",
+ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-1.3.2.tgz",
+ "integrity": "sha512-zyPgY5dgbf99c0uGUjhY4w+mxqEGxPKg9RQDl34VvrVh2bM31lFN+mwR1ZHepq/KA3VCPk1gwJZL6IIJqjLy2w=="
+ },
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -86,11 +139,52 @@
"concat-map": "0.0.1"
}
},
+ "btoa-lite": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz",
+ "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc="
+ },
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
+ "cross-spawn": {
+ "version": "6.0.5",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
+ "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+ "requires": {
+ "nice-try": "1.0.5",
+ "path-key": "2.0.1",
+ "semver": "5.6.0",
+ "shebang-command": "1.2.0",
+ "which": "1.3.1"
+ }
+ },
+ "deepmerge": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.2.0.tgz",
+ "integrity": "sha512-6+LuZGU7QCNUnAJyX8cIrlzoEgggTM6B7mm+znKOX4t5ltluT9KLjN6g61ECMS0LTsLW7yDpNoxhix5FZcrIow=="
+ },
+ "execa": {
+ "version": "0.10.0",
+ "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz",
+ "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==",
+ "requires": {
+ "cross-spawn": "6.0.5",
+ "get-stream": "3.0.0",
+ "is-stream": "1.1.0",
+ "npm-run-path": "2.0.2",
+ "p-finally": "1.0.0",
+ "signal-exit": "3.0.2",
+ "strip-eof": "1.0.0"
+ }
+ },
+ "formidable": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz",
+ "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg=="
+ },
"fs-extra": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
@@ -106,6 +200,11 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
+ "get-stream": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+ "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
+ },
"glob": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
@@ -143,6 +242,29 @@
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz",
"integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw=="
},
+ "is-plain-object": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+ "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+ "requires": {
+ "isobject": "3.0.1"
+ }
+ },
+ "is-stream": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
+ },
+ "isexe": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
+ },
+ "isobject": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
+ },
"jsonfile": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
@@ -151,6 +273,26 @@
"graceful-fs": "4.1.15"
}
},
+ "lodash.get": {
+ "version": "4.4.2",
+ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
+ "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
+ },
+ "lodash.set": {
+ "version": "4.3.2",
+ "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz",
+ "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM="
+ },
+ "lodash.uniq": {
+ "version": "4.5.0",
+ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
+ "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
+ },
+ "macos-release": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.0.0.tgz",
+ "integrity": "sha512-iCM3ZGeqIzlrH7KxYK+fphlJpCCczyHXc+HhRVbEu9uNTCrzYJjvvtefzeKTCVHd5AP/aD/fzC80JZ4ZP+dQ/A=="
+ },
"mime-db": {
"version": "1.38.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz",
@@ -172,6 +314,29 @@
"brace-expansion": "1.1.11"
}
},
+ "nice-try": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
+ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
+ },
+ "node-fetch": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
+ "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
+ },
+ "npm-run-path": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
+ "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
+ "requires": {
+ "path-key": "2.0.1"
+ }
+ },
+ "octokit-pagination-methods": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz",
+ "integrity": "sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ=="
+ },
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -180,11 +345,30 @@
"wrappy": "1.0.2"
}
},
+ "os-name": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.0.0.tgz",
+ "integrity": "sha512-7c74tib2FsdFbQ3W+qj8Tyd1R3Z6tuVRNNxXjJcZ4NgjIEQU9N/prVMqcW29XZPXGACqaXN3jq58/6hoaoXH6g==",
+ "requires": {
+ "macos-release": "2.0.0",
+ "windows-release": "3.1.0"
+ }
+ },
+ "p-finally": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+ "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
+ },
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
+ "path-key": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
+ "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
+ },
"path-parse": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
@@ -206,6 +390,24 @@
"path-parse": "1.0.6"
}
},
+ "semver": {
+ "version": "5.6.0",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+ "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
+ },
+ "shebang-command": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
+ "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+ "requires": {
+ "shebang-regex": "1.0.0"
+ }
+ },
+ "shebang-regex": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
+ "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
+ },
"shelljs": {
"version": "0.8.3",
"resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.3.tgz",
@@ -216,11 +418,50 @@
"rechoir": "0.6.2"
}
},
+ "signal-exit": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
+ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
+ },
+ "strip-eof": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
+ "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
+ },
+ "universal-user-agent": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-2.0.3.tgz",
+ "integrity": "sha512-eRHEHhChCBHrZsA4WEhdgiOKgdvgrMIHwnwnqD0r5C6AO8kwKcG7qSku3iXdhvHL3YvsS9ZkSGN8h/hIpoFC8g==",
+ "requires": {
+ "os-name": "3.0.0"
+ }
+ },
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
},
+ "url-template": {
+ "version": "2.0.8",
+ "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz",
+ "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE="
+ },
+ "which": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+ "requires": {
+ "isexe": "2.0.0"
+ }
+ },
+ "windows-release": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.1.0.tgz",
+ "integrity": "sha512-hBb7m7acFgQPQc222uEQTmdcGLeBmQLNLFIh0rDk3CwFOBrfjefLzEfEfmpMq8Af/n/GnFf3eYf203FY1PmudA==",
+ "requires": {
+ "execa": "0.10.0"
+ }
+ },
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
diff --git a/package.json b/package.json
index 0ec9e40..7e35298 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "extension-builder",
- "version": "1.0.0",
+ "version": "1.1.0",
"description": "",
"main": "index.js",
"repository": {
@@ -14,7 +14,9 @@
},
"homepage": "https://github.com/ColinTree-bot/extension-builder#readme",
"dependencies": {
+ "@octokit/rest": "^16.16.3",
"adm-zip": "^0.4.13",
+ "formidable": "^1.2.1",
"fs-extra": "^7.0.1",
"mime-types": "^2.1.22",
"shelljs": "^0.8.3"
@@ -24,6 +26,7 @@
},
"devDependencies": {
"@types/adm-zip": "^0.4.32",
+ "@types/formidable": "^1.0.31",
"@types/fs-extra": "^5.0.5",
"@types/mime-types": "^2.1.0",
"@types/shelljs": "^0.8.3"
diff --git a/src/builder.ts b/src/builder.ts
index 80bef19..dc09ccd 100644
--- a/src/builder.ts
+++ b/src/builder.ts
@@ -1,22 +1,14 @@
import * as fs from "fs-extra";
import * as AdmZip from "adm-zip"
-import exec from "./utils/exec";
+import exec, { ExecError } from "./utils/exec";
import { WORKSPACE, TEMP_DIR, BUILDER_CONFIG_NAME, OUTPUT_DIR } from "./config";
import Queue from "./utils/queue";
-class BuildQueue extends Queue {
- constructor() { super(); }
- public push(jobId: string) {
- super.push(jobId);
- console.timeLog("Added job(" + jobId + ")");
- Builder.notify();
- }
-}
export class JobPool {
private static pool = new Map();
- public static add(jobId: string, job: Job) {
- JobPool.pool.set(jobId, job);
+ public static add(job: Job) {
+ JobPool.pool.set(job.id, job);
}
public static get(jobId: string): Job {
return JobPool.pool.get(jobId);
@@ -25,96 +17,99 @@ export class JobPool {
return JobPool.pool.has(jobId);
}
}
-const buildQueue = new BuildQueue();
-
-export function addBuildQueue(jobId: string) {
- console.timeLog("Job(" + jobId + ") going to be added into build queue");
- // job will be added to build queue by itself after it is ready (in constructor)
- JobPool.add(jobId, new Job(jobId));
-}
-
-interface JobConfig {
- package: string
-}
-class Job {
+type JobStatus = "preparing" | "waiting" | "building" | "done" | "failed";
+export class Job {
private _id: string;
- private _config: JobConfig;
+ private _extraInfo: { [key: string]: string | number } = {};
get id() { return this._id; }
- get config() { return this._config; }
+ get extraInfo() { return this._extraInfo; }
- public status: "waiting" | "building" | "done" | "failed";
+ public status: JobStatus;
- public constructor(jobId: string) {
- this._id = jobId;
- this.status = "waiting";
- this.loadConfig()
- .then(config => {
- this._config = config;
- buildQueue.push(jobId);
- });
+ public constructor() {
+ fs.ensureDirSync(TEMP_DIR);
+ let jobDir = fs.mkdtempSync(TEMP_DIR + "/");
+ this._id = jobDir.substring(jobDir.lastIndexOf("/") + 1);
+ this.status = "preparing";
+ this.attachInfo("startTimestamp", Date.now());
+ JobPool.add(this);
}
- private loadConfig() {
- return new Promise(resolve => {
- fs.readFile(TEMP_DIR + "/" + this.id + "/src/" + BUILDER_CONFIG_NAME,
- "utf8", (err, data) => {
- if (err) throw err;
- resolve(JSON.parse(data));
- });
- });
+
+ public attachInfo(key: string, value: string | number) {
+ this._extraInfo[key] = value;
}
}
+class BuildQueue {
+ private static queue = new Queue();
+ public static push(jobId: string) {
+ BuildQueue.queue.push(jobId);
+ console.timeLog("Added job(" + jobId + ")");
+ Builder.notify();
+ }
+ public static isEmpty() {
+ return BuildQueue.queue.isEmpty();
+ }
+ public static pop() {
+ return BuildQueue.queue.pop();
+ }
+}
+export function pushBuildQueue(job: Job) {
+ job.status = "waiting";
+ BuildQueue.push(job.id);
+}
+
class Builder {
private static builderAvailable = true;
- public static notify() {
- if (Builder.builderAvailable && !buildQueue.isEmpty()) {
+ public static async notify() {
+ if (Builder.builderAvailable && !BuildQueue.isEmpty()) {
Builder.builderAvailable = false;
- let jobId = buildQueue.pop();
+ let jobId = BuildQueue.pop();
JobPool.get(jobId).status = "building";
- Builder.cleanWorkspace()
- .then(() => Builder.buildJob(jobId));
+ Builder.buildJob(jobId);
}
}
- private static cleanWorkspace() {
- return new Promise(resolve => {
- exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f")
- .then(stdout => {
- console.timeLog("Workspace cleaned");
- resolve();
- });
- })
- }
- private static buildJob(jobId: string) {
- return new Promise(resolve => {
- let job = JobPool.get(jobId);
- console.timeLog("Going to build job(" + jobId + ")");
- let config = job.config;
- let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/";
- fs.ensureDirSync(targetPath);
- fs.emptyDirSync(targetPath);
- fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath);
- console.log("Copied: " + targetPath);
- console.log("Compile started: job(" + jobId + ")");
- exec("cd " + WORKSPACE + "/appinventor && ant extensions", true)
- .then(stdout => {
- let zip = new AdmZip();
- zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions");
- let zipPath = OUTPUT_DIR + "/" + jobId + ".zip";
- zip.writeZip(zipPath);
- JobPool.get(jobId).status = "done";
- console.log("Done job(" + jobId + "): " + zipPath);
- Builder.builderAvailable = true;
- Builder.notify();
- resolve();
- })
- .catch(reason => {
- JobPool.get(jobId).status = "failed";
- console.log("Job(" + jobId + ") build failed", reason);
- Builder.builderAvailable = true;
- Builder.notify();
- });
- });
+ private static async buildJob(jobId: string) {
+ await exec("cd " + WORKSPACE + " && git reset --hard HEAD && git clean -f");
+ console.timeLog("Workspace cleaned");
+
+ let job = JobPool.get(jobId);
+ console.timeLog("Going to build job(" + jobId + ")");
+ let config = JSON.parse(fs.readFileSync(TEMP_DIR + "/" + jobId + "/src/" + BUILDER_CONFIG_NAME, "utf8"));
+ let targetPath = WORKSPACE + "/appinventor/components/src/" + config.package.split(".").join("/") + "/";
+ fs.ensureDirSync(targetPath);
+ fs.emptyDirSync(targetPath);
+ fs.copySync(TEMP_DIR + "/" + jobId + "/src/", targetPath);
+ console.log("Copied: " + targetPath);
+
+ console.log("Compile started: job(" + jobId + ")");
+ try {
+ await exec("cd " + WORKSPACE + "/appinventor && ant extensions", true);
+ } catch (e) {
+ e = e;
+ JobPool.get(jobId).status = "failed";
+ // Notice that it would not work on windows
+ let stdout = e.stdout.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
+ let stderr = e.stderr.split(WORKSPACE).join("%SERVER_WORKSPACE%/");
+ JobPool.get(jobId).attachInfo("failInfo",
+ e.message + ": code(" + e.code + ") stdout:\n" + stdout + "\n\nstderr:\n" + stderr);
+ console.log("Job(" + jobId + ") build failed in part of `ant extensions`");
+ Builder.builderAvailable = true;
+ Builder.notify();
+ }
+
+ let zip = new AdmZip();
+ zip.addLocalFolder(WORKSPACE + "/appinventor/components/build/extensions");
+ zip.addFile("build-info.json", new Buffer(JSON.stringify(job.extraInfo)));
+ let zipPath = OUTPUT_DIR + "/" + jobId + ".zip";
+ zip.writeZip(zipPath);
+
+ JobPool.get(jobId).status = "done";
+ console.log("Done job(" + jobId + "): " + zipPath);
+
+ Builder.builderAvailable = true;
+ Builder.notify();
}
}
\ No newline at end of file
diff --git a/src/config.ts b/src/config.ts
index 9972e78..0fbcdc3 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -1,7 +1,7 @@
import * as os from "os";
export const WORKSPACE = "/usr/workspace/";
-export const ENABLE_REPO_WHITELIST = true;
+export const ENABLE_REPO_WHITELIST = false;
export const PORT = 8048;
export const TEMP_DIR = os.tmpdir() + "/extension-builder/";
export const BUILDER_CONFIG_NAME = "builder-config.json";
@@ -13,22 +13,22 @@ export const EMPTY_TEMP_DIR_BEFORE_BUILD = false;
interface WhiteList {
owner: string;
repoName: string;
- branch: string | string[];
+ refs: string | string[]; // refs includes branchs, commits and tags. Can be "*" for any
}
const REPO_WHITELIST: WhiteList[] = [
- { owner: "OpenSourceAIX", repoName: "ColinTreeListView", branch: "extension-builder-test" }
+ { owner: "OpenSourceAIX", repoName: "ColinTreeListView", refs: "extension-builder-test" }
];
-export function inWhitelist(owner: string, repoName: string, branch = "master") {
+export function inWhitelist(owner: string, repoName: string, coderef = "") {
for (let i in REPO_WHITELIST) {
if (REPO_WHITELIST.hasOwnProperty(i)) {
let item = REPO_WHITELIST[i];
if (owner == item.owner && repoName == item.repoName) {
- let acceptBranchs = item.branch;
- if (acceptBranchs == "*") {
+ let acceptRefs = item.refs;
+ if (acceptRefs == "*") {
return true;
} else {
- acceptBranchs = typeof(acceptBranchs)=="string" ? [ acceptBranchs ] : acceptBranchs;
- return acceptBranchs.includes(branch);
+ acceptRefs = typeof(acceptRefs)=="string" ? [ acceptRefs ] : acceptRefs;
+ return acceptRefs.includes(coderef);
}
}
}
diff --git a/src/index.ts b/src/index.ts
index ed54537..526a8ef 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -11,10 +11,13 @@ import handleResult from "./pages/result";
export const CONTENT_TYPE_JSON = {"Content-Type": "application/json"};
export function responseSuccess(response: http.ServerResponse, info: {}) {
+ info = JSON.stringify(info);
+ console.log("Response end with 200: " + info);
response.writeHead(200, CONTENT_TYPE_JSON);
- response.end(JSON.stringify(info));
+ response.end(info);
}
export function responseError(response: http.ServerResponse, code: number, msg: string) {
+ console.log("Response end with " + code + ": " + msg);
response.writeHead(code, CONTENT_TYPE_JSON);
response.end(JSON.stringify({ msg: msg }));
}
@@ -39,7 +42,7 @@ function handleStaticFile(response: http.ServerResponse, pathname: string): bool
}
pathname = staticDir + pathname;
let mime = mimeTypes.lookup(pathname);
- console.log("Returning static file(" + pathname + ") mime(" + mime + ")");
+ console.log("Response end with 200: static file(" + pathname + ") mime(" + mime + ")");
response.writeHead(200, { "Content-Type": mime!==false ? mime : "application/octet-stream" });
fs.createReadStream(pathname).pipe(response, { end: true });
return true;
@@ -52,22 +55,27 @@ function startServer() {
let params = new url.URLSearchParams(requestUrl.query);
console.timeLog("Processing request: " + request.url);
switch (requestUrl.pathname) {
- case "/build-with-github-repo":
+ case "/build-with-github-repo": {
handleBuildWithGithubRepo(request, response, params);
return;
- case "/build-with-zip":
+ }
+ case "/build-with-zip": {
handleBuildWithZip(request, response, params);
return;
- case "/check-status":
+ }
+ case "/check-status": {
handleCheckStatus(request, response, params);
return;
- case "/result":
+ }
+ case "/result": {
handleResult(request, response, params);
return;
- default:
+ }
+ default: {
if (handleStaticFile(response, requestUrl.pathname)) {
return;
}
+ }
}
responseError(response, 404, "404 Not found.");
} catch (error) {
diff --git a/src/pages/build-with-github-repo.ts b/src/pages/build-with-github-repo.ts
index 8c7d008..9ed9100 100644
--- a/src/pages/build-with-github-repo.ts
+++ b/src/pages/build-with-github-repo.ts
@@ -1,93 +1,117 @@
-import * as https from "https";
import * as fs from "fs-extra";
import * as Admzip from "adm-zip";
+import * as Github from "@octokit/rest";
import { IncomingMessage, ServerResponse } from "http";
import { URLSearchParams } from "url";
import { ENABLE_REPO_WHITELIST, TEMP_DIR, inWhitelist } from "../config";
import { responseSuccess, responseError } from "../index";
-import { addBuildQueue } from "../builder";
+import { pushBuildQueue, Job, JobPool } from "../builder";
export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
if (request.method == "GET") {
let owner = params.get("owner");
let repoName = params.get("repoName");
- let branch = params.has("branch") ? params.get("branch") : "master";
-
- console.timeLog("Job info received: repo= " + owner + "/" + repoName + " branch= " + branch);
-
- if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, branch)) {
- // 1. white list is not enabled
- // 2. is in the white list
-
- // fs.mkdtempSync(TEMP_DIR) => {TEMP_DIR}/{jobId}
- let jobId = fs.mkdtempSync(TEMP_DIR + "/");
- jobId = jobId.substring(jobId.lastIndexOf("/") + 1);
-
- responseSuccess(response, {
- msg: "Build started.",
- jobId: jobId
- });
-
- getZip(jobId, owner, repoName, branch)
- .then(zipName => prepareSource(jobId, zipName))
- .then(() => addBuildQueue(jobId))
- .catch(reason => {
- console.error(reason);
- });
+ let ref = params.has("ref") ? params.get("ref") : "master";
+ console.timeLog("Job info received: repo= " + owner + "/" + repoName + " ref= " + ref);
+ if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName, ref)) {
+ startGithubJob(response, owner, repoName, ref);
return;
} else {
- responseError(response, 403, "Whitelist is enabled & your repo is not in it.");
+ responseError(response, 403, "Whitelist is enabled & your repo (or ref) is not in it.");
return;
}
- } else if (request.method == "POST") {
- // webhook
+ } else if (request.method == "POST") { // webhook
+ let indexOfEvent = request.rawHeaders.indexOf("X-GitHub-Event");
+ let event: string;
+ if (indexOfEvent != -1 && (indexOfEvent + 1) < request.rawHeaders.length) {
+ event = request.rawHeaders[indexOfEvent + 1];
+ }
+ console.log("Github event = " + event);
+ if (!["push", "release"].includes(event)) {
+ responseError(response, 403, "Does not support event type: " + event);
+ return;
+ }
let content = "";
-
request.on("data", chunk => {
content += chunk;
});
-
request.on("end", () => {
- responseSuccess(response, { content: content });
+ let playload = JSON.parse(content);
+ let commitOrTag: string;
+ switch (event) {
+ case "push":
+ commitOrTag = playload.head_commit.id;
+ break;
+ case "release":
+ commitOrTag = playload.release.tag_name;
+ break;
+ }
+ let owner = playload.repository.owner.login;
+ let repoName = playload.repository.name;
+ console.log("Repo = " + owner + "/" + repoName + " commitOrTag = " + commitOrTag);
+ if (typeof(owner)!="string" || typeof(repoName)!="string" || commitOrTag) {
+ responseError(response, 403, "Cannot be built, at least one of owner, repoName or commit is not string.");
+ return;
+ }
+ if (!ENABLE_REPO_WHITELIST || inWhitelist(owner, repoName)) {
+ startGithubJob(response, owner, repoName, commitOrTag);
+ return;
+ } else {
+ responseError(response, 403, "Whitelist is enabled & your repo is not in it.");
+ return;
+ }
});
- return;
}
}
-function getZip(jobId: string, owner: string, repoName: string, branch: string) {
- return new Promise(resolve => {
- // https://github.com/{owner}/{repoName}/archive/{branch}.zip
- // redirecting to -> https://codeload.github.com/{owner}/{repoName}/zip/{branch}
- let requestUrl = "https://codeload.github.com/" + owner + "/" + repoName + "/zip/" + branch;
- let zipName = owner + "_" + repoName + "_" + branch;
- let destZipPath = TEMP_DIR + "/" + jobId + "/" + zipName + ".zip";
- https.get(requestUrl, response => {
- var stream = response.pipe(fs.createWriteStream(destZipPath));
- stream.on("finish", () => {
- console.timeLog("Downloaded: " + destZipPath);
- resolve(zipName);
- });
- });
+async function startGithubJob(response: ServerResponse, owner: string, repoName: string, ref: string) {
+ let job = new Job();
+ let jobId = job.id;
+
+ job.attachInfo("buildType", "github-repo");
+ job.attachInfo("owner", owner);
+ job.attachInfo("repoName", repoName);
+ job.attachInfo("ref", ref);
+
+ responseSuccess(response, {
+ msg: "Job added.",
+ jobId: jobId
});
-}
-function prepareSource(jobId: string, zipPath: string) {
- return new Promise(resolve => {
- let zip = new Admzip(TEMP_DIR + "/" + jobId + "/" + zipPath + ".zip");
- let entryDir: string;
- zip.getEntries().forEach(entry => {
- if (!entryDir) {
- entryDir = entry.entryName;
- }
+ // Downlaod archive
+ let archiveResponse: Github.Response;
+ try {
+ archiveResponse = await new Github().repos.getArchiveLink({
+ owner: owner,
+ repo: repoName,
+ archive_format: "zipball",
+ ref: ref
});
- zip.extractEntryTo(entryDir, TEMP_DIR + "/" + jobId + "/rawComponentSource/");
- fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir,
- TEMP_DIR + "/" + jobId + "/src");
- fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource");
- console.timeLog("Unzipped & moved to " + TEMP_DIR + "/" + jobId + "/src");
- resolve();
- });
+ } catch (e) {
+ let err = e;
+ JobPool.get(jobId).status = "failed";
+ JobPool.get(jobId).attachInfo("failInfo",
+ err.name == "HttpError"
+ ? "Cannot load source from github, please check if the ref exists in the specified repo."
+ : err.message);
+ console.log("Fail prepare source of job(" + jobId + ")", e);
+ return;
+ }
+
+ // Extract responding archive
+ let zip = new Admzip( archiveResponse.data);
+ if (zip.getEntries().length == 0) {
+ throw "No source found in archive downloaded.";
+ }
+ let entryDir = zip.getEntries()[0].entryName;
+ zip.extractAllTo(TEMP_DIR + "/" + jobId + "/rawComponentSource/");
+ fs.moveSync(TEMP_DIR + "/" + jobId + "/rawComponentSource/" + entryDir,
+ TEMP_DIR + "/" + jobId + "/src");
+ fs.rmdirSync(TEMP_DIR + "/" + jobId + "/rawComponentSource");
+ console.timeLog("Source extracted to " + TEMP_DIR + "/" + jobId + "/src");
+
+ pushBuildQueue(job);
}
\ No newline at end of file
diff --git a/src/pages/build-with-zip.ts b/src/pages/build-with-zip.ts
index e327f22..9110f38 100644
--- a/src/pages/build-with-zip.ts
+++ b/src/pages/build-with-zip.ts
@@ -1,15 +1,48 @@
+import * as AdmZip from "adm-zip";
+import * as formidable from "formidable"
+
import { IncomingMessage, ServerResponse } from "http";
import { URLSearchParams } from "url";
-import { responseError } from "../index";
-import { ENABLE_REPO_WHITELIST } from "../config";
+import { responseError, responseSuccess } from "../index";
+import { ENABLE_REPO_WHITELIST, TEMP_DIR } from "../config";
+import { Job, pushBuildQueue } from "../builder";
export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
if (ENABLE_REPO_WHITELIST) {
responseError(response, 403, "Currently in white list mode, build with zip is disabled");
- return;
+
} else {
- responseError(response, 501, "build with zip is not implemented yet.");
+ let job = new Job();
+ job.attachInfo("buildType", "source-upload");
+
+ let type = request.headers["content-type"] || "";
+ if (!type.includes("multipart/form-data")) {
+ console.log("Request content type: " + type);
+ responseError(response, 400, "Please use multipart/form-data format");
+ return;
+ }
+
+ let jobDir = TEMP_DIR + "/" + job.id + "/";
+ let form = new formidable.IncomingForm();
+ form.uploadDir = jobDir;
+ form.keepExtensions = true;
+ form.on("file", (name, file) => {
+ if (name != "source") {
+ return;
+ }
+ let zip = new AdmZip(file.path);
+ zip.extractAllTo(jobDir + "/src/");
+ pushBuildQueue(job);
+ responseSuccess(response, {
+ msg: "Job added.",
+ jobId: job.id
+ });
+ });
+ form.on("error", err => {
+ responseError(response, 500, err);
+ })
+ form.parse(request);
return;
}
}
\ No newline at end of file
diff --git a/src/pages/check-status.ts b/src/pages/check-status.ts
index 52bb500..fd3e7ad 100644
--- a/src/pages/check-status.ts
+++ b/src/pages/check-status.ts
@@ -1,15 +1,26 @@
+import * as fs from "fs-extra";
import { IncomingMessage, ServerResponse } from "http";
import { URLSearchParams } from "url";
import { responseSuccess, responseError } from "../index";
import { JobPool } from "../builder";
+import { OUTPUT_DIR } from "../config";
export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
let jobId = params.get("jobId");
- if (!JobPool.has(jobId)) {
- // TODO: try harder to find this job in "build-result"?
+ if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) {
responseError(response, 404, "Specified job does not exist in job pool.");
return;
}
- responseSuccess(response, { status: JobPool.get(jobId).status });
+ let ret: { [key: string]: string | number } = {};
+ if (JobPool.has(jobId)) {
+ let job = JobPool.get(jobId);
+ ret.status = job.status;
+ for (let key in job.extraInfo) {
+ ret[key] = job.extraInfo[key];
+ }
+ } else {
+ ret.status = "done";
+ }
+ responseSuccess(response, ret);
}
\ No newline at end of file
diff --git a/src/pages/result.ts b/src/pages/result.ts
index e7c9d23..d7f6b4e 100644
--- a/src/pages/result.ts
+++ b/src/pages/result.ts
@@ -8,19 +8,22 @@ import { OUTPUT_DIR } from "../config";
export default (request: IncomingMessage, response: ServerResponse, params: URLSearchParams) => {
let jobId = params.get("jobId");
- if (!JobPool.has(jobId)) {
+ if (!JobPool.has(jobId) && !fs.existsSync(OUTPUT_DIR + "/" + jobId + ".zip")) {
+ console.log("Response end with 404: job not exist");
response.writeHead(404);
response.end("Job does not exist.");
return;
}
- let status = JobPool.get(jobId).status;
+ let status = JobPool.has(jobId) ? JobPool.get(jobId).status : "done";
if (status != "done") {
+ console.log("Response end with 404 job is not ready yet");
response.writeHead(404);
response.end("Job not ready yet.");
return;
} else {
let zipPath = OUTPUT_DIR + "/" + jobId + ".zip";
var stat = fs.statSync(zipPath);
+ console.log("Response end with 200 (build result will be sent)");
response.writeHead(200, {
"Content-Type": "application/zip",
"Content-Length": stat.size
diff --git a/src/utils/exec.ts b/src/utils/exec.ts
index 7db2582..1148ced 100644
--- a/src/utils/exec.ts
+++ b/src/utils/exec.ts
@@ -1,20 +1,29 @@
-import { exec } from "shelljs";
+import { exec, ExecOutputReturnValue } from "shelljs";
-/**
- * Execute a command with promise returns
- * @param command
- * @returns a Promise.
- * With then(stdout => {execute when result code is 0})
- * and catch((response = [code, stderr]) => {execute when code other than 0 returned})
- */
-export default (command: string, silent = false) => {
- return new Promise((resolve, reject) => {
- exec(command, { silent: silent }, (code, stdout, stderr) => {
- if (code == 0) {
- resolve(stdout);
- } else {
- reject([code, stderr]);
- }
- });
+export default async (command: string, silent = false) => {
+ let result = exec(command, {
+ silent: silent,
+ async: false // ensure it returns ExecOutputReturnValue
});
+ if (result.code == 0) {
+ return result.stdout;
+ } else {
+ throw new ExecError(result);
+ }
+}
+export class ExecError extends Error {
+ private _execOutputReturnValue: ExecOutputReturnValue;
+ constructor(execOutputReturnValue: ExecOutputReturnValue) {
+ super("Command executing error occured");
+ this._execOutputReturnValue = execOutputReturnValue;
+ }
+ get code() {
+ return this._execOutputReturnValue.code;
+ }
+ get stdout() {
+ return this._execOutputReturnValue.stdout;
+ }
+ get stderr() {
+ return this._execOutputReturnValue.stderr;
+ }
}
\ No newline at end of file
diff --git a/src/utils/queue.ts b/src/utils/queue.ts
index c066e7c..1df06ed 100644
--- a/src/utils/queue.ts
+++ b/src/utils/queue.ts
@@ -1,5 +1,4 @@
export default class Queue {
-
private elements: Array;
public constructor() {
@@ -9,19 +8,15 @@ export default class Queue {
public push(o: T) {
this.elements.unshift(o);
}
-
public pop(): T {
return this.elements.pop();
}
-
public size(): number {
return this.elements.length;
}
-
public isEmpty(): boolean {
return this.size() == 0;
}
-
public clear() {
delete this.elements;
this.elements = new Array();
diff --git a/static/index.html b/static/index.html
index 5254800..7e5cac7 100644
--- a/static/index.html
+++ b/static/index.html
@@ -16,12 +16,19 @@
if (xhr.status == 404) {
alert("404! maybe server is too busy to handle our request");
return;
+ } else if (xhr.status != 200) {
+ alert(xhr.status + "!" + xhr.responseJSON.msg);
+ return;
}
$("#status").text(xhr.responseJSON.status);
if (xhr.responseJSON.status == "done") {
callback_whenDone();
} else {
- checkStatus(jobId, callback_whenDone, times + 1);
+ if (xhr.responseJSON.status == "failed") {
+ $("#output").text(xhr.responseJSON.failInfo);
+ } else {
+ checkStatus(jobId, callback_whenDone, times + 1);
+ }
}
}
});
@@ -34,7 +41,7 @@
data: {
owner: $("#owner").val(),
repoName: $("#repoName").val(),
- branch: $("#branch").val()
+ ref: $("#ref").val()
},
dataType: "json",
success: json => {
@@ -49,6 +56,36 @@
},
error: xhr => {
console.log(xhr.responseJSON);
+ alert(xhr.responseJSON);
+ }
+ });
+ $("[type=submit]").prop("disabled", true);
+ return false;
+ });
+ $("#buildWithZip").submit(() => {
+ let formData = new FormData();
+ formData.append("source", $("#zip")[0].files[0]);
+ $.ajax({
+ url: "/build-with-zip",
+ type: "POST",
+ cache: false,
+ data: formData,
+ processData: false,
+ contentType: false,
+ dataType: "json",
+ success: json => {
+ let jobId = json["jobId"];
+ $("#jobId").text(jobId);
+ $("#status").text("submitted");
+ checkStatus(jobId, () => {
+ $("#download")
+ .attr("href", "/result?jobId=" + jobId)
+ .parent().show();
+ });
+ },
+ error: xhr => {
+ console.log(xhr.responseJSON);
+ alert(xhr.responseJSON);
}
});
$("[type=submit]").prop("disabled", true);
@@ -56,7 +93,7 @@
});
$("#previewRepo").click(() => {
window.open(
- "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#branch").val(),
+ "https://github.com/" + $("#owner").val() + "/" + $("#repoName").val() + "/tree/" + $("#ref").val(),
"_blank").focus();
return false;
})
@@ -73,7 +110,7 @@
Repo Name | |
- Branch | |
+ Ref | |
@@ -81,6 +118,15 @@
+
+
+
+
+
+
jobId:
@@ -89,6 +135,8 @@
Status:
+
+
Download