Beta (#5)

* feat: new download engine BREAKING CHANGE: New API, Support for browser download * docs(README): fetchers * fix: better tests * fix: tests * fix(package): script * docs(README): badge * feat: multi file download cli, new cli look BREAKING CHANGE: new api * feat: events for child download * feat: default download info * fix: fetch strategy * fix: warning * feat: if-range skip * fix: none async events * feat: better errors * fix: tests * fix: tests imports * fix: one stream transfer * feat: loading animation * fix: custom progress bar * fix: docs * feat: best defaults * fix: tests timeout * feat: remove levelDB * fix: lowdb read * feat: fancy CLI UI (#6) * feat: fancy transfer CLI UI * fix: bugs * fix: update `package-lock.json` after pull * fix: cli download files * feat: stateful multi-connection download --------- Co-authored-by: Gilad S <[email protected]>
ido-pluto · Mar 12, 2024 · b10958f · b10958f
1 parent e7d4163
commit b10958f
Show file tree

Hide file tree

Showing 66 changed files with 2,633 additions and 1,200 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -187,23 +187,6 @@
         "no-var": [
             "error"
         ],
-        "import/order": [
-            "error",
-            {
-                "groups": [
-                    "builtin",
-                    "external",
-                    "internal",
-                    "parent",
-                    "sibling",
-                    "index",
-                    "type",
-                    "object",
-                    "unknown"
-                ],
-                "warnOnUnassignedImports": true
-            }
-        ],
         "n/file-extension-in-import": [
             "error",
             "always"
@@ -264,14 +247,6 @@
         "eol-last": [
             "warn",
             "always"
-        ],
-        "max-len": [
-            "warn",
-            {
-                "code": 140,
-                "tabWidth": 4,
-                "ignoreStrings": true
-            }
         ]
     }
 }
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ node_modules
 
 /.env
 /.eslintcache
+/test/utils/files/big-image.jpg
diff --git a/README.md b/README.md
@@ -1,10 +1,9 @@
 <div align="center">
     <h1>iPull</h1>
-    <img src="assets/ipull-logo-rounded.png" height="200px" />
+    <img src="./assets/ipull-logo-rounded.png" height="200px" />
 </div>
 
 <div align="center">
-<br/>
 
 [![Build](https://github.com/ido-pluto/ipull/actions/workflows/build.yml/badge.svg)](https://github.com/ido-pluto/ipull/actions/workflows/build.yml)
 [![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/ipull)
@@ -13,14 +12,15 @@
 [![Version](https://badgen.net/npm/v/ipull)](https://www.npmjs.com/package/ipull)
 
 </div>
+<br />
 
 > Super fast file downloader with multiple connections
 
 ```bash
 npx ipull http://example.com/file.large
 ```
 
-![pull-example](https://github.com/ido-pluto/ipull/blob/main/assets/pull-file.gif)
+![pull-example](./assets/pull-file.gif)
 
 ## Features
 
@@ -31,75 +31,62 @@ npx ipull http://example.com/file.large
 - CLI Progress bar
 - Download statistics (speed, time left, etc.)
 
-## NodeJS API
+### NodeJS API
 
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = downloadFile('https://example.com/file.large', {
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
     directory: './this/path',
-    fileName: 'file.large', // optional
-    cliProgress: true // Show progress bar in the CLI (default: true)
+    cliProgress: true // Show progress bar in the CLI (default: false)
 });
 
 await downloader.download();
 ```
 
-### Events
-
-```ts
-import {downloadFile} from 'ipull';
-
-const downloader = downloadFile('https://example.com/file.large', {
-    onInit(engine) {
-    }, // retrive the file size and other details
-    onStart(engine) {
-    }, // download has started
-    onProgress(engine) {
-        console.log(`Time left: ${engine.timeLeft}, download speed: ${engine.speed}`)
-    },
-    onFinished(engine) {
-    }, // download has finished (file is still open)
-    onClosed(engine) {
-    } // download has finished and the file is closed
-});
-```
-
 ## Browser support
 
 Download a file in the browser using multiple connections
 
 ```ts
-import {downloadFileBrowserMemory} from "ipull/dist/browser.js";
+import {downloadFileBrowser} from "ipull/dist/browser.js";
 
-const {downloader, memory} = await downloadFileBrowserMemory(DOWNLOAD_URL, {
-    acceptRangeAlwaysTrue: true // cors origin request will not return the range header, but we can force it to be true (multipart download)
+const downloader = await downloadFileBrowser({
+    url: 'https://example.com/file.large',
+    acceptRangeIsKnown: true // cors origin request will not return the range header, but we can force it to be true (multi-connection download)
 });
 
 await downloader.download();
-image.src = memory.createBlobURL();
+image.src = downloader.writeStream.resultAsBlobURL();
+
+console.log(downloader.writeStream.result); // Uint8Array
 ```
 
 ### Custom stream
 
+You can use a custom stream
+
 ```ts
 import {downloadFileBrowser} from "ipull/dist/browser.js";
 
-const downloader = await downloadFileBrowser(DOWNLOAD_URL, {
+const downloader = await downloadFileBrowser({
+    url: 'https://example.com/file.large',
     onWrite: (cursor: number, buffer: Uint8Array, options) => {
         console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
     }
 });
 
 await downloader.download();
+console.log(downloader.writeStream.result.length === 0); // true, because we write to a custom stream
 ```
 
 ## CLI
 
 ```
 Usage: ipull [options] [files...]
 
-Pull/copy files from remote server/local directory
+Pull/copy files from a remote server/local directory
 
 Arguments:
   files                         Files to pull/copy
@@ -128,20 +115,20 @@ ipull set .zip ~/Downloads/zips
 
 ### Download file from parts
 
-Download a file from multiple parts, and merge them into a single file.
-
+Consolidate multiple files parts into one file.
 Beneficial for downloading large files from servers that limit file size. (e.g. HuggingFace models)
 
 ```ts
 import {downloadFile} from 'ipull';
 
 const downloadParts = [
-    "https://example.com/file.large1",
-    "https://example.com/file.large2",
-    "https://example.com/file.large3",
+    "https://example.com/file.large-part-1",
+    "https://example.com/file.large-part-2",
+    "https://example.com/file.large-part-3",
 ];
 
-const downloader = downloadFile(downloadParts, {
+const downloader = await downloadFile({
+    partsURL: downloadParts,
     directory: './this/path',
     filename: 'file.large'
 });
@@ -158,8 +145,9 @@ You can set custom headers for the download request
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = downloadFile('https://example.com/file.large', {
-    directory: './this/path',
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
+    savePath: './this/path/file.large',
     headers: {
         'Authorization': 'Bearer token'
     }
@@ -168,33 +156,20 @@ const downloader = downloadFile('https://example.com/file.large', {
 await downloader.download();
 ```
 
-### Copy file
-
-Copy file from local directory to another directory with CLI progress bar
-
-```ts
-import {copyFile} from 'ipull';
-
-const downloader = await copyFile('path/to/file', {
-    directory: './this/path'
-});
-await downloader.download();
-
-```
-
 ### Abort download
 
 You can cancel the download by calling the `abort` method
 
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = downloadFile('https://example.com/file.large', {
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
     directory: './this/path'
 });
 
 setTimeout(() => {
-    downloader.abort();
+    downloader.close();
 }, 5_000);
 
 await downloader.download();
@@ -205,7 +180,8 @@ await downloader.download();
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = downloadFile('https://example.com/file.large', {
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
     directory: './this/path'
 });
 
@@ -227,14 +203,14 @@ finish
 If a network/file-system error occurs, the download will automatically retry
 with [async-retry](https://www.npmjs.com/package/async-retry)
 
+If the maximum reties was reached the download will fail and an error will be thrown from the `download()` call:
+
 ```ts
 import {downloadFile} from 'ipull';
 
-const downloader = downloadFile('https://example.com/file.large', {
-    directory: './this/path',
-    retry: {
-        retries: 20 // default: 10
-    }
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
+    directory: './this/path'
 });
 
 try {
@@ -244,56 +220,81 @@ try {
 }
 ```
 
-<details>
-<summary>
-<h3>Custom Downloader (click to expand)
-</h3>
-</summary>
-In this example, there will be one progress bar for all the files
+### Listening to events
+
+Events are emitted using the `EventEmitter` pattern and can be listened to using the `on` method
 
 ```ts
-import {TransferCli, DownloadEngineNodejs, TransferStatistics} from "ipull";
-
-const cli = new TransferCli();
-const statistics = new TransferStatistics();
-
-const filesToDownload = ["https://example.com/file1.large", "https://example.com/file2.large", "https://example.com/file3.large"];
-let totalSize = 0;
-let bytesDownloaded = 0;
-
-const downloadsPromise = filesToDownload.map((url, index) => {
-    return DownloadEngineNodejs.fromParts(url, {
-        onInit(engine) {
-            totalSize += engine.file.totalSize;
-        },
-        onProgress(progress) {
-            const status = statistics.updateProgress(bytesDownloaded + progress.bytesDownloaded, totalSize);
-
-            cli.updateProgress({
-                ...status,
-                ...progress,
-                objectType: `${index}/${filesToDownload.length}`
-            });
-        },
-        onClosed(engine) {
-            bytesDownloaded += engine.file.totalSize
-        }
-    });
+interface DownloadEngineEvents {
+    start: [];
+    paused: [];
+    resumed: [];
+    progress: [FormattedStatus];
+    save: [DownloadProgressInfo];
+    finished: [];
+    closed: [];
+}
+
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
+    directory: './this/path'
 });
 
-for (const downloader of await Promise.all(downloadsPromise)) {
-    await downloader.download();
-}
+downloader.on("progress", (progress) => {
+    console.log(`Downloaded ${progress.transferred} bytes`);
+});
+```
+
+### Download multiple files
+
+If you want to download multiple files, you can use the `downloadSequence` function.
+
+By default, it will download files one by one, but you can set the `parallel` option to download them in parallel.
+It is better to download one file at a time if you are downloading from the same server (as it may limit the number of
+connections).
+
+```ts
+import {downloadFile, downloadSequence} from "ipull";
+
+const downloader = await downloadSequence(
+    {
+        cliProgress: true,
+    },
+    downloadFile({
+        url: "https://example.com/file1.large",
+        directory: "."
+    }),
+    downloadFile({
+        url: "https://example.com/file2.large",
+        directory: "."
+    }),
+);
+
+await downloader.download();
 ```
 
-![custom-progress-bar](assets/custom-progress.png)
+### Custom progress bar
+
+```ts
+import {downloadFile, FormattedStatus} from "ipull";
 
-</details>
+function createProgressBar({fileName, ...data}: FormattedStatus) {
+    return `${fileName} ${JSON.stringify(data)}`;
+}
+
+const downloader = await downloadFile({
+    url: "https://example.com/file.large",
+    directory: "./this/path",
+    cliStyle: createProgressBar
+});
+
+await downloader.download();
+```
 
 <br />
 
 <div align="center" width="360">
-    <img alt="Star please" src="assets/star-please.png" width="360" margin="auto" />
+    <img alt="Star please" src="./assets/star-please.png" width="360" margin="auto" />
     <br/>
     <p align="right">
         <i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;

diff --git a/assets/custom-progress.png b/assets/custom-progress.png
diff --git a/assets/pull-file.gif b/assets/pull-file.gif
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,3 +8,4 @@ node_modules

		/.env
		/.eslintcache
		/test/utils/files/big-image.jpg