From 1008d5c84870cbf0d54be2fde3910993e0d0e2ce Mon Sep 17 00:00:00 2001 From: Gilad S Date: Wed, 12 Apr 2023 13:32:26 +0000 Subject: [PATCH 1/4] feat: overcome download connection errors --- index.js | 55 ++++++++++++++++++++++++++-------------------------- package.json | 2 +- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/index.js b/index.js index ea1cf3a..8bb131f 100644 --- a/index.js +++ b/index.js @@ -14,7 +14,7 @@ const term = require( 'terminal-kit' ).terminal; const Downloader = require("nodejs-file-downloader"); const semver = require('semver'); //const _7z = require('7zip-min'); -const axios = require('axios') +const EasyDl = require("easydl"); const platform = os.platform() const shell = platform === 'win32' ? 'powershell.exe' : 'bash'; const L = require("./llama") @@ -102,33 +102,34 @@ class Dalai { } return encodedStr; } - down(url, dest, headers) { - return new Promise((resolve, reject) => { - const task = path.basename(dest) - this.startProgress(task) - axios({ - url, - method: 'GET', - responseType: 'stream', - maxContentLength: Infinity, - headers, - onDownloadProgress: progressEvent => { - const progress = (progressEvent.loaded / progressEvent.total) * 100; - this.progress(task, progress) - } + async down(url, dest, headers) { + const task = path.basename(dest); + + const download = new EasyDl(url, dest, { + connections: 10, + maxRetry: 30, + existBehavior: "overwrite", + httpOptions: { + headers: headers || {} + } + }); - }).then(response => { - const writer = fs.createWriteStream(dest); - response.data.pipe(writer); - writer.on('finish', () => { - this.progressBar.update(1); - term("\n") - resolve() - }); - }).catch(error => { - reject(error) - }); - }) + download.on('progress', (progressReport) => { + this.progress(task, progressReport.total.percentage); + }); + + let recentError = null; + download.on("error", (error) => { + recentError = error; + }); + + const success = await download.wait(); + + if (!success) + throw recentError || new Error("Download failed"); + + this.progressBar.update(1); + term("\n"); } async python () { // install self-contained python => only for windows for now diff --git a/package.json b/package.json index 8839c99..768370f 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "postinstall": "node setup" }, "dependencies": { - "axios": "^1.3.4", + "easydl": "^1.0.3", "ejs": "^3.1.8", "express": "^4.18.2", "isomorphic-git": "^1.22.0", From 9d2d14a34774e0958f6cd6b86e0a212fa329b539 Mon Sep 17 00:00:00 2001 From: Gilad S Date: Wed, 12 Apr 2023 13:52:35 +0000 Subject: [PATCH 2/4] fix: bugs --- index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 8bb131f..d735498 100644 --- a/index.js +++ b/index.js @@ -103,7 +103,8 @@ class Dalai { return encodedStr; } async down(url, dest, headers) { - const task = path.basename(dest); + const task = path.basename(dest) + this.startProgress(task) const download = new EasyDl(url, dest, { connections: 10, @@ -123,10 +124,12 @@ class Dalai { recentError = error; }); - const success = await download.wait(); + download.start(); - if (!success) - throw recentError || new Error("Download failed"); + await new Promise((accept, reject) => { + download.once("end", () => accept()); + download.once("close", () => reject(recentError) || new Error("download failed")); + }); this.progressBar.update(1); term("\n"); From abd311909df447471d5ce5b13c218d43bca5a7ce Mon Sep 17 00:00:00 2001 From: Gilad S Date: Wed, 12 Apr 2023 15:08:30 +0000 Subject: [PATCH 3/4] feat: download all llama files in parallel --- index.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++++---- llama.js | 24 +++++++++++++++----- package.json | 2 ++ 3 files changed, 79 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index d735498..262b037 100644 --- a/index.js +++ b/index.js @@ -15,11 +15,13 @@ const Downloader = require("nodejs-file-downloader"); const semver = require('semver'); //const _7z = require('7zip-min'); const EasyDl = require("easydl"); +const chalk = require("chalk"); const platform = os.platform() const shell = platform === 'win32' ? 'powershell.exe' : 'bash'; const L = require("./llama") const A = require("./alpaca") const TorrentDownloader = require("./torrent") +const cliProgress = require("cli-progress"); const exists = s => new Promise(r=>fs.access(s, fs.constants.F_OK, e => r(!e))) const escapeNewLine = (platform, arg) => platform === 'win32' ? arg.replaceAll(/\n/g, "\\n").replaceAll(/\r/g, "\\r") : arg const escapeDoubleQuotes = (platform, arg) => platform === 'win32' ? arg.replaceAll(/"/g, '`"') : arg.replaceAll(/"/g, '\\"') @@ -107,7 +109,7 @@ class Dalai { this.startProgress(task) const download = new EasyDl(url, dest, { - connections: 10, + connections: 5, maxRetry: 30, existBehavior: "overwrite", httpOptions: { @@ -115,7 +117,7 @@ class Dalai { } }); - download.on('progress', (progressReport) => { + download.on("progress", (progressReport) => { this.progress(task, progressReport.total.percentage); }); @@ -124,16 +126,68 @@ class Dalai { recentError = error; }); - download.start(); - await new Promise((accept, reject) => { download.once("end", () => accept()); - download.once("close", () => reject(recentError) || new Error("download failed")); + download.once("close", () => reject(recentError || new Error("download failed"))); + + download.start(); }); this.progressBar.update(1); term("\n"); } + async multiDownload(items) { + const multibar = new cliProgress.MultiBar({ + clearOnComplete: false, + hideCursor: true, + format: `${chalk.bold("{filename}")} ${chalk.yellow("{percentage}%")} ${chalk.cyan("{bar}")} ${chalk.grey("{eta_formatted}")}`, + }, cliProgress.Presets.shades_classic); + + async function downloadFile(url, dest, headers) { + const bar = multibar.create(100, 0); + bar.update(0, { + filename: path.basename(dest) + }); + + const download = new EasyDl(url, dest, { + connections: 5, + maxRetry: 30, + existBehavior: "overwrite", + httpOptions: { + headers: headers || {} + } + }); + + download.on("progress", (progressReport) => { + bar.update(Math.floor(progressReport.total.percentage)); + }); + + let recentError = null; + download.on("error", (error) => { + recentError = error; + }); + + await new Promise((accept, reject) => { + download.once("end", () => accept()); + download.once("close", () => reject(recentError || new Error("download failed"))); + + download.start(); + }); + + bar.update(100); + bar.stop(); + } + + const downloads = []; + for (const item of items) { + const { url, dest, headers } = item; + downloads.push(downloadFile(url, dest, headers)); + } + + await Promise.all(downloads); + multibar.stop(); + term("\n"); + } async python () { // install self-contained python => only for windows for now // 1. download diff --git a/llama.js b/llama.js index 6e54625..0ed76f7 100644 --- a/llama.js +++ b/llama.js @@ -138,6 +138,7 @@ npx dalai install 7B 13B const resolvedPath = path.resolve(this.home, "models", model) await fs.promises.mkdir(resolvedPath, { recursive: true }).catch((e) => { }) + const filesToDownload = []; for(let file of files) { if (fs.existsSync(path.resolve(resolvedPath, file))) { console.log(`Skip file download, it already exists: ${file}`) @@ -145,12 +146,18 @@ npx dalai install 7B 13B } const url = `https://agi.gpt4.org/llama/LLaMA/${model}/${file}` - await this.root.down(url, path.resolve(resolvedPath, file), { - "User-Agent": "Mozilla/5.0" - }) + filesToDownload.push({ + url, + dest: path.resolve(resolvedPath, file), + headers: { + "User-Agent": "Mozilla/5.0" + } + }); } + await this.root.multiDownload(filesToDownload); const files2 = ["tokenizer_checklist.chk", "tokenizer.model"] + const filesToDownload2 = []; for(let file of files2) { // if (fs.existsSync(path.resolve(this.home, "models", file))) { // console.log(`Skip file download, it already exists: ${file}`) @@ -158,10 +165,15 @@ npx dalai install 7B 13B // } const url = `https://agi.gpt4.org/llama/LLaMA/${file}` const dir = path.resolve(this.home, "models") - await this.root.down(url, path.resolve(dir, file), { - "User-Agent": "Mozilla/5.0" - }) + filesToDownload2.push({ + url, + dest: path.resolve(dir, file), + headers: { + "User-Agent": "Mozilla/5.0" + } + }); } + await this.root.multiDownload(filesToDownload2); } } diff --git a/package.json b/package.json index 768370f..ea0c011 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "postinstall": "node setup" }, "dependencies": { + "chalk": "^4.1.2", + "cli-progress": "^3.12.0", "easydl": "^1.0.3", "ejs": "^3.1.8", "express": "^4.18.2", From 4aa1758de5c83217b02da646f3d5f7d8a0327a71 Mon Sep 17 00:00:00 2001 From: Gilad S Date: Wed, 12 Apr 2023 15:31:33 +0000 Subject: [PATCH 4/4] fix: bugs --- index.js | 15 ++++++++++++--- package.json | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 262b037..5cf909f 100644 --- a/index.js +++ b/index.js @@ -16,6 +16,7 @@ const semver = require('semver'); //const _7z = require('7zip-min'); const EasyDl = require("easydl"); const chalk = require("chalk"); +const bytes = require("bytes"); const platform = os.platform() const shell = platform === 'win32' ? 'powershell.exe' : 'bash'; const L = require("./llama") @@ -140,13 +141,19 @@ class Dalai { const multibar = new cliProgress.MultiBar({ clearOnComplete: false, hideCursor: true, - format: `${chalk.bold("{filename}")} ${chalk.yellow("{percentage}%")} ${chalk.cyan("{bar}")} ${chalk.grey("{eta_formatted}")}`, + autopadding: true, + format: `${chalk.bold("{filename}")} ${chalk.yellow("{percentage}%")} ${chalk.cyan("{bar}")} {speed}${chalk.grey("{eta_formatted}")}`, }, cliProgress.Presets.shades_classic); + const longestFileName = items.reduce((acc, item) => { + return Math.max(acc, path.basename(item.dest).trim().length); + }, 0); + async function downloadFile(url, dest, headers) { const bar = multibar.create(100, 0); bar.update(0, { - filename: path.basename(dest) + speed: "", + filename: path.basename(dest).trim().padEnd(longestFileName, " "), }); const download = new EasyDl(url, dest, { @@ -159,7 +166,9 @@ class Dalai { }); download.on("progress", (progressReport) => { - bar.update(Math.floor(progressReport.total.percentage)); + bar.update(progressReport.total.percentage, { + speed: Number.isFinite(progressReport.total.speed) ? chalk.blue((bytes(progressReport.total.speed) + "/s").padEnd(10)) + chalk.grey(" | ") : "" + }); }); let recentError = null; diff --git a/package.json b/package.json index ea0c011..8a23d6e 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "postinstall": "node setup" }, "dependencies": { + "bytes": "^3.1.2", "chalk": "^4.1.2", "cli-progress": "^3.12.0", "easydl": "^1.0.3",