diff --git a/README.md b/README.md index 92768c3..1fe706c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ console.log(downloader.writeStream.result); // Uint8Array ### Custom stream You can use a custom stream + ```ts import {downloadFileBrowser} from "ipull/dist/browser.js"; @@ -85,7 +86,7 @@ console.log(downloader.writeStream.result.length === 0); // true, because we wri ``` Usage: ipull [options] [files...] -Pull/copy files from remote server/local directory +Pull/copy files from a remote server/local directory Arguments: files Files to pull/copy @@ -114,8 +115,7 @@ ipull set .zip ~/Downloads/zips ### Download file from parts -Download a file from multiple parts, and merge them into a single file. - +Consolidate multiple files parts into one file. Beneficial for downloading large files from servers that limit file size. (e.g. HuggingFace models) ```ts @@ -203,6 +203,8 @@ finish If a network/file-system error occurs, the download will automatically retry with [async-retry](https://www.npmjs.com/package/async-retry) +If the maximum reties was reached the download will fail and an error will be thrown from the `download()` call: + ```ts import {downloadFile} from 'ipull'; @@ -245,7 +247,12 @@ downloader.on("progress", (progress) => { ### Download multiple files -If you want to download multiple files, you can use the `downloadSequence` function +If you want to download multiple files, you can use the `downloadSequence` function. + +By default, it will download files one by one, but you can set the `parallel` option to download them in parallel. +It is better to download one file at a time if you are downloading from the same server (as it may limit the number of +connections). + ```ts import {downloadFile, downloadSequence} from "ipull"; diff --git a/package-lock.json b/package-lock.json index 97efad8..3cb4da9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,10 +16,10 @@ "commander": "^10.0.0", "eventemitter3": "^5.0.1", "fs-extra": "^11.1.1", - "level": "^8.0.0", "lifecycle-utils": "^1.3.1", "lodash.debounce": "^4.0.8", "log-update": "^6.0.0", + "lowdb": "^7.0.1", "pretty-bytes": "^6.1.0", "pretty-ms": "^8.0.0" }, @@ -2504,23 +2504,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/abstract-level": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-1.0.3.tgz", - "integrity": "sha512-t6jv+xHy+VYwc4xqZMn2Pa9DjcdzvzZmQGRjTFc8spIbRGHgBrEKbPq+rYXc7CCo0lxgYvSgKVg9qZAhpVQSjA==", - "dependencies": { - "buffer": "^6.0.3", - "catering": "^2.1.0", - "is-buffer": "^2.0.5", - "level-supports": "^4.0.0", - "level-transcoder": "^1.0.1", - "module-error": "^1.0.1", - "queue-microtask": "^1.2.3" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -2849,25 +2832,6 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/before-after-hook": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", @@ -2902,40 +2866,6 @@ "node": ">=8" } }, - "node_modules/browser-level": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-1.0.1.tgz", - "integrity": "sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==", - "dependencies": { - "abstract-level": "^1.0.2", - "catering": "^2.1.1", - "module-error": "^1.0.2", - "run-parallel-limit": "^1.1.0" - } - }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/builtin-modules": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", @@ -3027,14 +2957,6 @@ "cdl": "bin/cdl.js" } }, - "node_modules/catering": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz", - "integrity": "sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==", - "engines": { - "node": ">=6" - } - }, "node_modules/chai": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/chai/-/chai-4.4.1.tgz", @@ -3076,22 +2998,6 @@ "node": "*" } }, - "node_modules/classic-level": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-1.3.0.tgz", - "integrity": "sha512-iwFAJQYtqRTRM0F6L8h4JCt00ZSGdOyqh7yVrhhjrOpFhmBjNlRUey64MCiyo6UmQHMJ+No3c81nujPv+n9yrg==", - "hasInstallScript": true, - "dependencies": { - "abstract-level": "^1.0.2", - "catering": "^2.1.0", - "module-error": "^1.0.1", - "napi-macros": "^2.2.2", - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/clean-stack": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz", @@ -4974,25 +4880,6 @@ "url": "https://github.com/sponsors/typicode" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -5151,28 +5038,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-buffer": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", - "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "engines": { - "node": ">=4" - } - }, "node_modules/is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", @@ -5607,42 +5472,6 @@ "node": ">=0.10.0" } }, - "node_modules/level": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz", - "integrity": "sha512-ypf0jjAk2BWI33yzEaaotpq7fkOPALKAgDBxggO6Q9HGX2MRXn0wbP1Jn/tJv1gtL867+YOjOB49WaUF3UoJNQ==", - "dependencies": { - "browser-level": "^1.0.1", - "classic-level": "^1.2.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/level" - } - }, - "node_modules/level-supports": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-4.0.1.tgz", - "integrity": "sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==", - "engines": { - "node": ">=12" - } - }, - "node_modules/level-transcoder": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-transcoder/-/level-transcoder-1.0.1.tgz", - "integrity": "sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==", - "dependencies": { - "buffer": "^6.0.3", - "module-error": "^1.0.1" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5933,6 +5762,20 @@ "get-func-name": "^2.0.1" } }, + "node_modules/lowdb": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-7.0.1.tgz", + "integrity": "sha512-neJAj8GwF0e8EpycYIDFqEPcx9Qz4GUho20jWFR7YiFeXzF1YMLdxB36PypcTSPMA+4+LvgyMacYhlr18Zlymw==", + "dependencies": { + "steno": "^4.0.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/lru-cache": { "version": "7.18.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", @@ -6293,14 +6136,6 @@ "node": ">=0.10.0" } }, - "node_modules/module-error": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", - "integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==", - "engines": { - "node": ">=10" - } - }, "node_modules/mrmime": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", @@ -6334,11 +6169,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/napi-macros": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", - "integrity": "sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==" - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -6366,16 +6196,6 @@ "lodash": "^4.17.21" } }, - "node_modules/node-gyp-build": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", - "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", @@ -10124,6 +9944,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, "funding": [ { "type": "github", @@ -10654,28 +10475,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/run-parallel-limit": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz", - "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, "node_modules/safe-array-concat": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.0.tgz", @@ -11260,6 +11059,17 @@ "integrity": "sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==", "dev": true }, + "node_modules/steno": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/steno/-/steno-4.0.2.tgz", + "integrity": "sha512-yhPIQXjrlt1xv7dyPQg2P17URmXbuM5pdGkpiMB3RenprfiBlvK415Lctfe0eshk90oA7/tNq7WEiMK8RSP39A==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/stream-combiner2": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz", diff --git a/package.json b/package.json index 6583a08..7f30b99 100644 --- a/package.json +++ b/package.json @@ -102,10 +102,10 @@ "commander": "^10.0.0", "eventemitter3": "^5.0.1", "fs-extra": "^11.1.1", - "level": "^8.0.0", "lifecycle-utils": "^1.3.1", "lodash.debounce": "^4.0.8", "log-update": "^6.0.0", + "lowdb": "^7.0.1", "pretty-bytes": "^6.1.0", "pretty-ms": "^8.0.0" } diff --git a/src/cli/cli.ts b/src/cli/cli.ts index f983621..7481da5 100644 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -8,7 +8,6 @@ import findDownloadDir, {downloadToDirectory, findFileName} from "./utils/find-d const pullCommand = new Command(); pullCommand - .version(packageJson.version) .description("Pull/copy files from remote server/local directory") .argument("[files...]", "Files to pull/copy") .option("-s --save [path]", "Save location (directory/file)") @@ -37,7 +36,8 @@ pullCommand }); await downloader.download(); } - }); + }) + .version(packageJson.version); pullCommand.addCommand(setCommand); pullCommand.parse(); diff --git a/src/cli/commands/set.ts b/src/cli/commands/set.ts index 156b5f8..bf2587a 100644 --- a/src/cli/commands/set.ts +++ b/src/cli/commands/set.ts @@ -20,10 +20,16 @@ setCommand.description("Set download locations") .addHelpText("afterAll", HELP_TEXT) .action(async (path, value, {delete: deleteSetting}) => { if (deleteSetting) { - await AppDB.del(path); + await AppDB.update(data => { + delete data[path]; + }); + console.log(`Deleted ${path}`); return; } - await AppDB.put(path, value); + + await AppDB.update(data => { + data[path] = value; + }); console.log(`${value} set to ${path}`); }); diff --git a/src/cli/utils/find-download-dir.ts b/src/cli/utils/find-download-dir.ts index cf37863..f4f0ea8 100644 --- a/src/cli/utils/find-download-dir.ts +++ b/src/cli/utils/find-download-dir.ts @@ -1,12 +1,12 @@ import path from "path"; import fs from "fs-extra"; -import {getWithDefault} from "../../settings/settings.js"; +import {AppDB} from "../../settings/settings.js"; const DEFAULT_DOWNLOAD_DIR = process.cwd(); export default async function findDownloadDir(fileName?: string) { - const downloadLocation = await getWithDefault(path.extname(fileName || "")); - const defaultLocation = await getWithDefault("default"); + const downloadLocation = AppDB.data[path.extname(fileName || "")]; + const defaultLocation = AppDB.data["default"]; return downloadLocation || defaultLocation || DEFAULT_DOWNLOAD_DIR; } diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 72838f8..1ca59b1 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -1,21 +1,13 @@ -import {Level} from "level"; +import {JSONFilePreset} from "lowdb/node"; import {DB_PATH} from "../const.js"; +import {Low} from "lowdb"; -const NOT_FOUND_STATUS = 404; -export const AppDB = new Level( - DB_PATH, - {valueEncoding: "json"} -); +const AppDB = await JSONFilePreset(DB_PATH, {} as Record); +await AppDB.read(); -export async function getWithDefault(name: string, defaultValue = null) { - try { - return await AppDB.get(name); - } catch (error: any) { - if (error.status === NOT_FOUND_STATUS) { - return defaultValue; - } - throw error; - } -} +export { + AppDB, + Low +};