diff --git a/assets/pull-file.gif b/assets/pull-file.gif index a24e99c..06950d4 100644 Binary files a/assets/pull-file.gif and b/assets/pull-file.gif differ diff --git a/package-lock.json b/package-lock.json index dc71cec..4985516 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,8 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@supercharge/promise-pool": "^3.1.1", "async-retry": "^1.3.3", + "axios": "^1.6.7", "chalk": "^5.3.0", "cli-spinners": "^2.9.2", "commander": "^10.0.0", @@ -2078,14 +2078,6 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, - "node_modules/@supercharge/promise-pool": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-3.1.1.tgz", - "integrity": "sha512-TgCm6jVqMPv+OgD5uBNND/CkCwNDdXPQlcprtnXsWSBpTCy0q5CI6vRj+jsUiXE1xeRaKIX4UeaYJqzZBL92sg==", - "engines": { - "node": ">=8" - } - }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -2819,6 +2811,11 @@ "retry": "0.13.1" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", @@ -2831,6 +2828,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz", + "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==", + "dependencies": { + "follow-redirects": "^1.15.4", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -3123,6 +3130,17 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", @@ -3440,6 +3458,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", @@ -4403,6 +4429,25 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -4412,6 +4457,19 @@ "is-callable": "^1.1.3" } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -6092,6 +6150,25 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", @@ -9964,6 +10041,11 @@ "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", "dev": true }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", diff --git a/package.json b/package.json index f048e04..756dbd5 100644 --- a/package.json +++ b/package.json @@ -119,8 +119,8 @@ "xmlhttprequest-ssl": "^2.1.1" }, "dependencies": { - "@supercharge/promise-pool": "^3.1.1", "async-retry": "^1.3.3", + "axios": "^1.6.7", "chalk": "^5.3.0", "cli-spinners": "^2.9.2", "commander": "^10.0.0", diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index b9644a7..6389935 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -1,12 +1,15 @@ import DownloadEngineBrowser, {DownloadEngineOptionsBrowser} from "./download-engine/engine/download-engine-browser.js"; import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js"; +export const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3; + export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser; /** * Download one file in the browser environment. */ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) { + options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_BROWSER; return await DownloadEngineBrowser.createFromOptions(options); } diff --git a/src/download/download-engine/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts similarity index 65% rename from src/download/download-engine/download-engine-file.ts rename to src/download/download-engine/download-file/download-engine-file.ts index f44f6ca..71cf829 100644 --- a/src/download/download-engine/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -1,11 +1,11 @@ -import {PromisePool, Stoppable} from "@supercharge/promise-pool"; import ProgressStatusFile, {ProgressStatus} from "./progress-status-file.js"; -import {ChunkStatus, DownloadFile, SaveProgressInfo} from "./types.js"; -import BaseDownloadEngineFetchStream from "./streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js"; -import BaseDownloadEngineWriteStream from "./streams/download-engine-write-stream/base-download-engine-write-stream.js"; +import {ChunkStatus, DownloadFile, SaveProgressInfo} from "../types.js"; +import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js"; +import BaseDownloadEngineWriteStream from "../streams/download-engine-write-stream/base-download-engine-write-stream.js"; import retry from "async-retry"; import {EventEmitter} from "eventemitter3"; import {withLock} from "lifecycle-utils"; +import DownloadProgram from "./download-program.js"; export type DownloadEngineFileOptions = { chunkSize?: number; @@ -47,12 +47,12 @@ export default class DownloadEngineFile extends EventEmitter this._progress.part || !this._activePart.acceptRange) { this._progress.part = i; this._progress.chunkSize = this.options.chunkSize; + this._progress.parallelStreams = this.options.parallelStreams; this._progress.chunks = this._emptyChunksForPart(i); } - this._activeStreamBytes = {}; - if (this._activePart.acceptRange && this.options.parallelStreams > 1) { - await this._downloadPartParallelStream(); - } else { - await this._downloadWithoutParallelStreams(); + if (!this._activePart.acceptRange) { + this._progress.parallelStreams = 1; } + + this._activeStreamBytes = {}; + const downloadProgram = new DownloadProgram(this._progress, this._downloadSlice.bind(this)); + await downloadProgram.download(); } - await this._saveProgress(); this._progressStatus.finished(); + await this._saveProgress(); this.emit("finished"); await this.options.onFinishAsync?.(); } - protected async _downloadWithoutParallelStreams() { - const startIndex = this._progress.chunks.findIndex(status => status !== ChunkStatus.COMPLETE); - if (startIndex === -1) return; - const startByteDownloaded = startIndex * this._progress.chunkSize; - + protected async _downloadSlice(startChunk: number, endChunk: number) { const fetchState = this.options.fetchStream.withSubState({ chunkSize: this._progress.chunkSize, - start: startByteDownloaded, - end: this.downloadSize, + startChunk, + endChunk, + totalSize: this.downloadSize, url: this._activePart.downloadURL!, rangeSupport: this._activePart.acceptRange, onProgress: (length: number) => { - this._activeStreamBytes[0] = length; + this._activeStreamBytes[startChunk] = length; this._sendProgressDownloadPart(); } }); - await fetchState.fetchChunks((chunk, index) => { + this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS; + await fetchState.fetchChunks((chunks, writePosition, index) => { if (this._closed) return; - const byteDownloaded = startByteDownloaded + index * this._progress.chunkSize; - this.options.writeStream.write(byteDownloaded, chunk); - this._progress.chunks[startIndex + index] = ChunkStatus.COMPLETE; - this._activeStreamBytes[0] = 0; - this._saveProgress(); - }); - } + for (const chunk of chunks) { + this.options.writeStream.write(writePosition, chunk); + writePosition += chunk.length; + } - protected async _downloadPartParallelStream() { - try { - await PromisePool.withConcurrency(this.options.parallelStreams) - .for(this._progress.chunks) - .process(async (status, index, pool) => { - await this.options.fetchStream.paused; - this._activePool = pool; - if (status !== ChunkStatus.NOT_STARTED) { - return; - } - this._activeStreamBytes[index] = 0; - this._progress.chunks[index] = ChunkStatus.IN_PROGRESS; - - const start = index * this._progress.chunkSize; - const end = Math.min(start + this._progress.chunkSize, this._activePart.size); - const buffer = await this.options.fetchStream.fetchBytes(this._activePart.downloadURL!, start, end, (length: number) => { - this._activeStreamBytes[index] = length; - this._sendProgressDownloadPart(); - }); - - await this.options.writeStream.write(start, buffer); - this._progress.chunks[index] = ChunkStatus.COMPLETE; - delete this._activeStreamBytes[index]; - - this._saveProgress(); - }); - } finally { - this._activePool = undefined; - } + this._progress.chunks[index] = ChunkStatus.COMPLETE; + delete this._activeStreamBytes[startChunk]; + void this._saveProgress(); + + if (this._progress.chunks[index + 1] != ChunkStatus.NOT_STARTED) { + return fetchState.close(); + } + + this._progress.chunks[index + 1] = ChunkStatus.IN_PROGRESS; + }); + delete this._activeStreamBytes[startChunk]; } protected async _saveProgress() { @@ -219,7 +199,6 @@ export default class DownloadEngineFile extends EventEmitter Promise; + + public constructor(_savedProgress: SaveProgressInfo, _downloadSlice: (startChunk: number, endChunk: number) => Promise) { + this._downloadSlice = _downloadSlice; + this._savedProgress = _savedProgress; + this._findChunksSlices(); + } + + public async download() { + if (this._savedProgress.parallelStreams === 1) { + return this._downloadSlice(0, this._savedProgress.chunks.length); + } + + const activeDownloads: Promise[] = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const slice = this._createOneSlice(); + if (slice == null) break; + + while (activeDownloads.length >= this._savedProgress.parallelStreams) { + await Promise.race(activeDownloads); + } + + const promise = this._downloadSlice(slice.start, slice.end); + activeDownloads.push(promise); + promise.then(() => { + activeDownloads.splice(activeDownloads.indexOf(promise), 1); + }); + } + + await Promise.all(activeDownloads); + } + + private _createOneSlice(): ProgramSlice | null { + const slice = this._findChunksSlices()[0]; + if (!slice) return null; + const length = slice.end - slice.start; + return {start: Math.floor(slice.start + length / 2), end: slice.end}; + } + + private _findChunksSlices() { + const chunksSlices: ProgramSlice[] = []; + + let start = 0; + let currentIndex = 0; + for (const chunk of this._savedProgress.chunks) { + if (chunk !== ChunkStatus.NOT_STARTED) { + if (start === currentIndex) { + start = ++currentIndex; + continue; + } + chunksSlices.push({start, end: currentIndex}); + start = ++currentIndex; + continue; + } + + currentIndex++; + } + + if (start !== currentIndex) { + chunksSlices.push({start, end: currentIndex}); + } + + return chunksSlices.sort((a, b) => (b.end - b.start) - (a.end - a.start)); + } +} diff --git a/src/download/download-engine/progress-status-file.ts b/src/download/download-engine/download-file/progress-status-file.ts similarity index 79% rename from src/download/download-engine/progress-status-file.ts rename to src/download/download-engine/download-file/progress-status-file.ts index b25d971..6258561 100644 --- a/src/download/download-engine/progress-status-file.ts +++ b/src/download/download-engine/download-file/progress-status-file.ts @@ -48,6 +48,19 @@ export default class ProgressStatusFile { } public createStatus(downloadPart: number, transferredBytes: number): ProgressStatusFile { - return new ProgressStatusFile(this.totalBytes, this.totalDownloadParts, this.fileName, this.comment, this.transferAction, downloadPart, transferredBytes); + const newStatus = new ProgressStatusFile( + this.totalBytes, + this.totalDownloadParts, + this.fileName, + this.comment, + this.transferAction, + downloadPart, + transferredBytes + ); + + newStatus.startTime = this.startTime; + newStatus.endTime = this.endTime; + + return newStatus; } } diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index 8de55a2..6934576 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -1,5 +1,5 @@ import {DownloadFile, SaveProgressInfo} from "../types.js"; -import DownloadEngineFile, {DownloadEngineFileOptions} from "../download-engine-file.js"; +import DownloadEngineFile, {DownloadEngineFileOptions} from "../download-file/download-engine-file.js"; import BaseDownloadEngineFetchStream, {BaseDownloadEngineFetchStreamOptions} from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js"; import UrlInputError from "./error/url-input-error.js"; import {EventEmitter} from "eventemitter3"; diff --git a/src/download/download-engine/engine/download-engine-browser.ts b/src/download/download-engine/engine/download-engine-browser.ts index e43d204..d961870 100644 --- a/src/download/download-engine/engine/download-engine-browser.ts +++ b/src/download/download-engine/engine/download-engine-browser.ts @@ -1,5 +1,5 @@ import {SaveProgressInfo} from "../types.js"; -import DownloadEngineFile from "../download-engine-file.js"; +import DownloadEngineFile from "../download-file/download-engine-file.js"; import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js"; import DownloadEngineFetchStreamXhr from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.js"; import DownloadEngineWriteStreamBrowser, {DownloadEngineWriteStreamBrowserWriter} from "../streams/download-engine-write-stream/download-engine-write-stream-browser.js"; diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 9379cb7..241c868 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -2,7 +2,7 @@ import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engi import {EventEmitter} from "eventemitter3"; import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer-visualize/progress-statistics-builder.js"; import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; -import DownloadEngineFile from "../download-engine-file.js"; +import DownloadEngineFile from "../download-file/download-engine-file.js"; import {createFormattedStatus, FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineFile; diff --git a/src/download/download-engine/engine/download-engine-nodejs.ts b/src/download/download-engine/engine/download-engine-nodejs.ts index b037f4d..319faa3 100644 --- a/src/download/download-engine/engine/download-engine-nodejs.ts +++ b/src/download/download-engine/engine/download-engine-nodejs.ts @@ -1,6 +1,6 @@ import path from "path"; import {DownloadFile} from "../types.js"; -import DownloadEngineFile from "../download-engine-file.js"; +import DownloadEngineFile from "../download-file/download-engine-file.js"; import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js"; import DownloadEngineWriteStreamNodejs from "../streams/download-engine-write-stream/download-engine-write-stream-nodejs.js"; import DownloadEngineFetchStreamLocalFile from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js"; diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index d8f9709..022d47b 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -15,19 +15,23 @@ export type BaseDownloadEngineFetchStreamOptions = { export type FetchSubState = { url: string, - start: number, - end: number, + startChunk: number, + endChunk: number, + totalSize: number, chunkSize: number, rangeSupport?: boolean, - onProgress?: (length: number) => void + onProgress?: (length: number) => void, }; export type BaseDownloadEngineFetchStreamEvents = { paused: () => void resumed: () => void aborted: () => void + errorCountIncreased: (errorCount: number, error: Error) => void }; +export type WriteCallback = (data: Uint8Array[], position: number, index: number) => void; + export default abstract class BaseDownloadEngineFetchStream extends EventEmitter { public readonly abstract transferAction: string; public readonly options: Partial = {}; @@ -35,6 +39,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter public paused?: Promise; public aborted = false; protected _pausedResolve?: () => void; + public errorCount = {value: 0}; constructor(options: Partial = {}) { super(); @@ -42,6 +47,14 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter this.initEvents(); } + protected get _startSize() { + return this.state.startChunk * this.state.chunkSize; + } + + protected get _endSize() { + return Math.min(this.state.endChunk * this.state.chunkSize, this.state.totalSize); + } + protected initEvents() { this.on("aborted", () => { this.aborted = true; @@ -63,24 +76,34 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter abstract withSubState(state: FetchSubState): this; - public async fetchBytes(url: string, start: number, end: number, onProgress?: (length: number) => void) { - return await retry(async () => { - return await this.fetchBytesWithoutRetry(url, start, end, onProgress); - }, this.options.retry); - } + protected cloneState(state: FetchSubState, fetchStream: Fetcher): Fetcher { + fetchStream.state = state; + fetchStream.errorCount = this.errorCount; + fetchStream.on("errorCountIncreased", this.emit.bind(this, "errorCountIncreased")); + + this.on("aborted", fetchStream.emit.bind(fetchStream, "aborted")); + this.on("paused", fetchStream.emit.bind(fetchStream, "paused")); + this.on("resumed", fetchStream.emit.bind(fetchStream, "resumed")); - protected abstract fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: (length: number) => void): Promise; + return fetchStream; + } public async fetchDownloadInfo(url: string): Promise<{ length: number, acceptRange: boolean }> { return this.options.defaultFetchDownloadInfo ?? await retry(async () => { - return await this.fetchDownloadInfoWithoutRetry(url); + try { + return await this.fetchDownloadInfoWithoutRetry(url); + } catch (error: any) { + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); + throw error; + } }, this.options.retry); } protected abstract fetchDownloadInfoWithoutRetry(url: string): Promise<{ length: number, acceptRange: boolean }>; - public async fetchChunks(callback: (data: Uint8Array, index: number) => void) { - let lastStartLocation = this.state.start; + public async fetchChunks(callback: WriteCallback) { + let lastStartLocation = this.state.startChunk; let retryResolvers = retryAsyncStatementSimple(this.options.retry); // eslint-disable-next-line no-constant-condition @@ -88,16 +111,19 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter try { return await this.fetchWithoutRetryChunks(callback); } catch (error: any) { - if (lastStartLocation !== this.state.start) { - lastStartLocation = this.state.start; + if (error?.name === "AbortError") return; + if (lastStartLocation !== this.state.startChunk) { + lastStartLocation = this.state.startChunk; retryResolvers = retryAsyncStatementSimple(this.options.retry); } + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); await retryResolvers(error); } } } - protected abstract fetchWithoutRetryChunks(callback: (data: Uint8Array, index: number) => void): Promise | void; + protected abstract fetchWithoutRetryChunks(callback: WriteCallback): Promise | void; public close(): void | Promise { this.emit("aborted"); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 35c562b..33ef335 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -1,7 +1,5 @@ -import BaseDownloadEngineFetchStream, {FetchSubState} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, {FetchSubState, WriteCallback} from "./base-download-engine-fetch-stream.js"; import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; -import FetchStreamError from "./errors/fetch-stream-error.js"; -import {promiseWithResolvers} from "./utils/retry-async-statement.js"; import SmartChunkSplit from "./utils/smart-chunk-split.js"; type GetNextChunk = () => Promise> | ReadableStreamReadResult; @@ -10,44 +8,38 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe withSubState(state: FetchSubState): this { const fetchStream = new DownloadEngineFetchStreamFetch(this.options); - fetchStream.state = state; - - return fetchStream as this; - } - - protected override async fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: ((length: number) => void) | undefined): Promise { - const {promise, resolve, reject} = promiseWithResolvers(); - await this.withSubState({url, start, end, onProgress, chunkSize: end - start, rangeSupport: true}) - .fetchWithoutRetryChunks(resolve); - reject(new FetchStreamError("No chunks received")); - - return await promise; + return this.cloneState(state, fetchStream) as this; } - protected override async fetchWithoutRetryChunks(callback: (data: Uint8Array, index: number) => void) { + protected override async fetchWithoutRetryChunks(callback: WriteCallback) { + const controller = new AbortController(); const response = await fetch(this.appendToURL(this.state.url), { headers: { accept: "*/*", ...this.options.headers, - range: `bytes=${this.state.start}-${this.state.end! - 1}` - } + range: `bytes=${this._startSize}-${this._endSize - 1}` + }, + signal: controller.signal + }); + + this.on("aborted", () => { + controller.abort(); }); const contentLength = parseInt(response.headers.get("content-length")!); - const expectedContentLength = this.state.end - this.state.start; + const expectedContentLength = this._endSize - this._startSize; if (contentLength !== expectedContentLength) { throw new InvalidContentLengthError(expectedContentLength, contentLength); } const reader = response.body!.getReader(); - return await this.chunkGenerator(callback, () => reader.read()); } protected override async fetchDownloadInfoWithoutRetry(url: string): Promise<{ length: number; acceptRange: boolean; }> { const response = await fetch(url, { method: "HEAD", - ...this.options.headers + headers: this.options.headers }); const length = parseInt(response.headers.get("content-length")!); @@ -59,7 +51,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe }; } - async chunkGenerator(callback: (data: Uint8Array, index: number) => void, getNextChunk: GetNextChunk) { + async chunkGenerator(callback: WriteCallback, getNextChunk: GetNextChunk) { const smartSplit = new SmartChunkSplit(callback, this.state); // eslint-disable-next-line no-constant-condition @@ -69,7 +61,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe if (done || this.aborted) break; smartSplit.addChunk(value); - this.state.onProgress?.(smartSplit.leftOverLength); + this.state.onProgress?.(smartSplit.savedLength); } smartSplit.sendLeftovers(); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts index 4540699..5c99b14 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts @@ -2,9 +2,9 @@ import fs, {FileHandle} from "fs/promises"; import {withLock} from "lifecycle-utils"; import retry from "async-retry"; import fsExtra from "fs-extra"; -import BaseDownloadEngineFetchStream, {FetchSubState} from "./base-download-engine-fetch-stream.js"; -import {promiseWithResolvers} from "./utils/retry-async-statement.js"; +import BaseDownloadEngineFetchStream, {FetchSubState, WriteCallback} from "./base-download-engine-fetch-stream.js"; import SmartChunkSplit from "./utils/smart-chunk-split.js"; +import streamResponse from "./utils/stream-response.js"; const OPEN_MODE = "r"; @@ -15,9 +15,7 @@ export default class DownloadEngineFetchStreamLocalFile extends BaseDownloadEngi override withSubState(state: FetchSubState): this { const fetchStream = new DownloadEngineFetchStreamLocalFile(this.options); - fetchStream.state = state; - - return fetchStream as this; + return this.cloneState(state, fetchStream) as this; } private async _ensureFileOpen(path: string) { @@ -34,54 +32,15 @@ export default class DownloadEngineFetchStreamLocalFile extends BaseDownloadEngi }); } - protected override async fetchWithoutRetryChunks(callback: (data: Uint8Array, index: number) => void) { + protected override async fetchWithoutRetryChunks(callback: WriteCallback): Promise { const file = await this._ensureFileOpen(this.state.url); - const {promise, resolve, reject} = promiseWithResolvers(); - - const smartSplit = new SmartChunkSplit(callback, this.state); const stream = file.createReadStream({ - start: this.state.start, - end: this.state.end, + start: this._startSize, + end: this._endSize, autoClose: true }); - stream.on("data", (chunk) => { - smartSplit.addChunk(new Uint8Array(chunk as Buffer)); - this.state.onProgress?.(smartSplit.leftOverLength); - }); - - stream.on("close", () => { - smartSplit.sendLeftovers(); - resolve(); - }); - - stream.on("error", (error) => { - reject(error); - }); - - const pause = stream.pause.bind(stream); - const resume = stream.resume.bind(stream); - const close = stream.destroy.bind(stream); - - this.on("paused", pause); - this.on("resumed", resume); - this.on("aborted", close); - - try { - await promise; - } finally { - this.off("paused", pause); - this.off("resumed", resume); - this.off("aborted", close); - stream.destroy(); - } - } - - protected override async fetchBytesWithoutRetry(path: string, start: number, end: number) { - const file = await this._ensureFileOpen(path); - const buffer = Buffer.alloc(end - start); - await file.read(buffer, 0, buffer.byteLength, start); - return buffer; + return await streamResponse(stream, this, new SmartChunkSplit(callback, this.state), this.state.onProgress); } protected override async fetchDownloadInfoWithoutRetry(path: string): Promise<{ length: number; acceptRange: boolean }> { diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index bba0950..c032205 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -1,8 +1,9 @@ -import BaseDownloadEngineFetchStream, {FetchSubState} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, {FetchSubState, WriteCallback} from "./base-download-engine-fetch-stream.js"; import EmptyResponseError from "./errors/empty-response-error.js"; import StatusCodeError from "./errors/status-code-error.js"; import XhrError from "./errors/xhr-error.js"; import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; +import retry from "async-retry"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { @@ -10,12 +11,16 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc withSubState(state: FetchSubState): this { const fetchStream = new DownloadEngineFetchStreamXhr(this.options); - fetchStream.state = state; + return this.cloneState(state, fetchStream) as this; + } - return fetchStream as this; + public async fetchBytes(url: string, start: number, end: number, onProgress?: (length: number) => void) { + return await retry(async () => { + return await this.fetchBytesWithoutRetry(url, start, end, onProgress); + }, this.options.retry); } - protected override fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: (length: number) => void): Promise { + protected fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: (length: number) => void): Promise { return new Promise((resolve, reject) => { const headers = { accept: "*/*", @@ -60,10 +65,14 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc }; xhr.send(); + + this.on("aborted", () => { + xhr.abort(); + }); }); } - public override async fetchChunks(callback: (data: Uint8Array, index: number) => void) { + public override async fetchChunks(callback: WriteCallback) { if (this.state.rangeSupport) { return await this._fetchChunksRangeSupport(callback); } @@ -75,23 +84,20 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc throw new Error("Method not needed, use fetchChunks instead."); } - protected async _fetchChunksRangeSupport(callback: (data: Uint8Array, index: number) => void) { - let index = 0; - while (this.state.start < this.state.end) { + protected async _fetchChunksRangeSupport(callback: WriteCallback) { + while (this._startSize < this._endSize) { await this.paused; if (this.aborted) return; - const end = Math.min(this.state.end, this.state.start + this.state.chunkSize); - const chunk = await this.fetchBytes(this.state.url, this.state.start, end, this.state.onProgress); - this.state.start += chunk.length; - callback(chunk, index++); + const chunk = await this.fetchBytes(this.state.url, this._startSize, this._endSize, this.state.onProgress); + callback([chunk], this._startSize, this.state.startChunk++); } } - protected async _fetchChunksWithoutRange(callback: (data: Uint8Array, index: number) => void) { + protected async _fetchChunksWithoutRange(callback: WriteCallback) { const relevantContent = await (async (): Promise => { - const result = await this.fetchBytes(this.state.url, 0, this.state.end, this.state.onProgress); - return result.slice(this.state.start, this.state.end); + const result = await this.fetchBytes(this.state.url, 0, this._endSize, this.state.onProgress); + return result.slice(this._startSize, this._endSize); })(); let totalReceivedLength = 0; @@ -105,7 +111,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc const chunk = relevantContent.slice(start, end); totalReceivedLength += chunk.byteLength; - callback(chunk, index++); + callback([chunk], index * this.state.chunkSize, index++); } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts index 3d4710e..caec905 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts @@ -1,43 +1,51 @@ +import {WriteCallback} from "../base-download-engine-fetch-stream.js"; + export type SmartChunkSplitOptions = { chunkSize: number; + startChunk: number; }; export default class SmartChunkSplit { - private readonly _callback: (data: Uint8Array, index: number) => void; + private readonly _callback: WriteCallback; private readonly _options: SmartChunkSplitOptions; - private _counter: number = 0; - private _chunk: Uint8Array = new Uint8Array(0); + private _bytesWriteLocation: number; + private _bytesLeftovers: number = 0; + private _chunks: Uint8Array[] = []; - public constructor(_callback: (data: Uint8Array, index: number) => void, _options: SmartChunkSplitOptions) { + public constructor(_callback: WriteCallback, _options: SmartChunkSplitOptions) { this._options = _options; this._callback = _callback; + this._bytesWriteLocation = _options.startChunk * _options.chunkSize; } - public addChunk(data: Uint8Array) { - const oldData = this._chunk; - this._chunk = new Uint8Array(oldData.length + data.length); - this._chunk.set(oldData); - this._chunk.set(data, oldData.length); - + this._chunks.push(data); this._sendChunk(); } - public get leftOverLength() { - return this._chunk.length; + public get savedLength() { + return this._bytesLeftovers + this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); } public sendLeftovers() { - if (this._chunk.length > 0) { - this._callback(this._chunk, this._counter++); + if (this.savedLength > 0) { + this._callback(this._chunks, this._bytesWriteLocation, this._options.startChunk++); } } private _sendChunk() { - while (this._chunk.length >= this._options.chunkSize) { - const chunk = this._chunk.slice(0, this._options.chunkSize); - this._chunk = this._chunk.slice(this._options.chunkSize); - this._callback(chunk, this._counter++); + while (this._chunks.length && this.savedLength >= this._options.chunkSize) { + let sendLength = this._bytesLeftovers; + for (let i = 0; i < this._chunks.length; i++) { + sendLength += this._chunks[i].byteLength; + if (sendLength >= this._options.chunkSize) { + this._callback(this._chunks.slice(0, i + 1), this._bytesWriteLocation, this._options.startChunk++); + this._chunks = this._chunks.slice(i + 1); + this._bytesWriteLocation += sendLength - this._bytesLeftovers; + this._bytesLeftovers = sendLength - this._options.chunkSize; + break; + } + } } } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts new file mode 100644 index 0000000..147c7b0 --- /dev/null +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts @@ -0,0 +1,48 @@ +import {promiseWithResolvers} from "./retry-async-statement.js"; +import SmartChunkSplit from "./smart-chunk-split.js"; +import BaseDownloadEngineFetchStream from "../base-download-engine-fetch-stream.js"; + +type IStreamResponse = { + on(event: "data", listener: (chunk: Uint8Array) => void): IStreamResponse; + on(event: "close", listener: () => void): IStreamResponse; + on(event: "error", listener: (error: Error) => void): IStreamResponse; + pause(): void; + resume(): void; + destroy(): void; +}; + +export default async function streamResponse(stream: IStreamResponse, downloadEngine: BaseDownloadEngineFetchStream, smartSplit: SmartChunkSplit, onProgress?: (leftOverLength: number) => void): Promise { + const {promise, resolve, reject} = promiseWithResolvers(); + + stream.on("data", (chunk) => { + smartSplit.addChunk(chunk); + onProgress?.(smartSplit.savedLength); + }); + + stream.on("close", () => { + smartSplit.sendLeftovers(); + smartSplit.sendLeftovers(); + resolve(); + }); + + stream.on("error", (error) => { + reject(error); + }); + + const pause = stream.pause.bind(stream); + const resume = stream.resume.bind(stream); + const close = stream.destroy.bind(stream); + + downloadEngine.on("paused", pause); + downloadEngine.on("resumed", resume); + downloadEngine.on("aborted", close); + + try { + await promise; + } finally { + downloadEngine.off("paused", pause); + downloadEngine.off("resumed", resume); + downloadEngine.off("aborted", close); + stream.destroy(); + } +} diff --git a/src/download/download-engine/types.ts b/src/download/download-engine/types.ts index 3e9b080..f1a96e0 100644 --- a/src/download/download-engine/types.ts +++ b/src/download/download-engine/types.ts @@ -14,6 +14,7 @@ export type SaveProgressInfo = { part: number, chunks: ChunkStatus[], chunkSize: number, + parallelStreams: number }; export type DownloadFile = { diff --git a/src/download/node-download.ts b/src/download/node-download.ts index 9c42439..813b81c 100644 --- a/src/download/node-download.ts +++ b/src/download/node-download.ts @@ -5,6 +5,9 @@ import TransferCli, {TransferCliOptions} from "./transfer-visualize/transfer-cli import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.js"; import {CliFormattedStatus} from "./transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.js"; +export const DEFAULT_PARALLEL_STREAMS_FOR_NODEJS = 3; +export const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "fancy"; + export type CliProgressDownloadEngineOptions = { truncateName?: boolean | number; cliProgress?: boolean; @@ -25,11 +28,9 @@ function createCliProgressForDownloadEngine(options: CliProgressDownloadEngineOp cliOptions.name = options.cliName; } - if (options.cliStyle) { - cliOptions.createProgressBar = typeof options.cliStyle === "function" ? - options.cliStyle : - switchCliProgressStyle(options.cliStyle, {truncateName: options.truncateName}); - } + cliOptions.createProgressBar = typeof options.cliStyle === "function" ? + options.cliStyle : + switchCliProgressStyle(options.cliStyle ?? DEFAULT_CLI_STYLE, {truncateName: options.truncateName}); return new TransferCli(cliOptions); } @@ -45,6 +46,7 @@ export async function downloadFile(options: DownloadFileOptions) { cli = createCliProgressForDownloadEngine(options); cli.loadingAnimation.start(); } + options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_NODEJS; const downloader = await DownloadEngineNodejs.createFromOptions(options); diff --git a/src/download/transfer-visualize/format-transfer-status.ts b/src/download/transfer-visualize/format-transfer-status.ts index d603061..efe91bb 100644 --- a/src/download/transfer-visualize/format-transfer-status.ts +++ b/src/download/transfer-visualize/format-transfer-status.ts @@ -1,7 +1,7 @@ import {TransferProgressInfo} from "./transfer-statistics.js"; import prettyBytes, {Options as PrettyBytesOptions} from "pretty-bytes"; import prettyMilliseconds, {Options as PrettyMsOptions} from "pretty-ms"; -import {ProgressStatus} from "../download-engine/progress-status-file.js"; +import {ProgressStatus} from "../download-engine/download-file/progress-status-file.js"; export type CliInfoStatus = TransferProgressInfo & { fileName?: string, diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index b71e4e9..46d8ba4 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -3,7 +3,7 @@ import {EventEmitter} from "eventemitter3"; import TransferStatistics from "./transfer-statistics.js"; import DownloadEngineMultiDownload from "../download-engine/engine/download-engine-multi-download.js"; import {createFormattedStatus, FormattedStatus} from "./format-transfer-status.js"; -import DownloadEngineFile from "../download-engine/download-engine-file.js"; +import DownloadEngineFile from "../download-engine/download-file/download-engine-file.js"; export type ProgressStatusWithIndex = FormattedStatus & { index: number, diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts index bf2eebb..83f4cb3 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts @@ -149,8 +149,9 @@ export default class FancyTransferCliProgressBar { const wasSuccessful = this.status.percentage === 100; const {endTime, startTime} = this.status; + const downloadTime = (endTime || Date.now()) - startTime; const finishedText = wasSuccessful - ? `downloaded ${this.status.formatTransferred} in ${prettyMilliseconds(endTime - startTime, PRETTY_MS_OPTIONS)}` + ? `downloaded ${this.status.formatTransferred} in ${prettyMilliseconds(downloadTime, PRETTY_MS_OPTIONS)}` : `failed downloading after ${prettyMilliseconds(endTime - startTime, PRETTY_MS_OPTIONS)}`; return renderDataLine([{ diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.ts index 432d321..8228813 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.ts @@ -3,7 +3,7 @@ import FancyTransferCliProgressBar from "./fancy-transfer-cli-progress-bar.js"; export type AvailableCLIProgressStyle = "basic" | "fancy"; -export default function switchCliProgressStyle(cliStyle: AvailableCLIProgressStyle, {truncateName}: {truncateName?: boolean | number}) { +export default function switchCliProgressStyle(cliStyle: AvailableCLIProgressStyle, {truncateName}: { truncateName?: boolean | number }) { switch (cliStyle) { case "basic": return BaseTransferCliProgressBar.createLineRenderer({truncateName}); diff --git a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts index 37140cc..4902e66 100644 --- a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts +++ b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts @@ -1,9 +1,10 @@ import logUpdate from "log-update"; import debounce from "lodash.debounce"; -import BaseTransferCliProgressBar, {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js"; +import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js"; import cliSpinners from "cli-spinners"; import CliSpinnersLoadingAnimation from "./loading-animation/cli-spinners-loading-animation.js"; import {FormattedStatus} from "../format-transfer-status.js"; +import switchCliProgressStyle from "./progress-bars/switch-cli-progress-style.js"; export type TransferCliOptions = { action?: string, @@ -20,7 +21,7 @@ export const DEFAULT_TRANSFER_CLI_OPTIONS: TransferCliOptions = { truncateName: true, debounceWait: 20, maxDebounceWait: 100, - createProgressBar: BaseTransferCliProgressBar.createLineRenderer({truncateName: false}), + createProgressBar: switchCliProgressStyle("basic", {truncateName: true}), loadingAnimation: "dots", loadingText: "Gathering information" }; @@ -33,7 +34,7 @@ export default class TransferCli { public constructor(options: Partial) { this.options = {...DEFAULT_TRANSFER_CLI_OPTIONS, ...options}; - this._logUpdate = debounce(this._logUpdate.bind(this), this.options.debounceWait, { + this.updateStatues = debounce(this.updateStatues.bind(this), this.options.debounceWait, { maxWait: this.options.maxDebounceWait }); diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 7ba1107..2a139f2 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -2,10 +2,8 @@ import {JSONFilePreset} from "lowdb/node"; import {DB_PATH} from "../const.js"; import {Low} from "lowdb"; - const AppDB = await JSONFilePreset(DB_PATH, {} as Record); - export { AppDB, Low diff --git a/test/copy-file.test.ts b/test/copy-file.test.ts index 4b0df0b..2829d45 100644 --- a/test/copy-file.test.ts +++ b/test/copy-file.test.ts @@ -1,12 +1,9 @@ import {describe, test} from "vitest"; import fs from "fs-extra"; -import DownloadEngineWriteStreamBrowser from "../src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js"; -import {createDownloadFile, ensureLocalFile, TEXT_FILE_EXAMPLE} from "./utils/download.js"; +import {ensureLocalFile, TEXT_FILE_EXAMPLE} from "./utils/download.js"; import {fileHash} from "./utils/hash.js"; import {copyFileInfo} from "./utils/copy.js"; import {downloadFile} from "../src/index.js"; -import DownloadEngineFetchStreamLocalFile from "../src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js"; -import DownloadEngineFile from "../src/download/download-engine/download-engine-file.js"; describe("File Copy", async () => { @@ -51,25 +48,4 @@ describe("File Copy", async () => { context.expect(copiedFileHash) .toBe(originalFileHash); }); - - - test.concurrent("Total bytes written", async (context) => { - let totalBytesWritten = 0; - const fetchStream = new DownloadEngineFetchStreamLocalFile(); - const writeStream = new DownloadEngineWriteStreamBrowser((cursor, data) => { - totalBytesWritten += data.byteLength; - }); - - const file = await createDownloadFile(TEXT_FILE_EXAMPLE, fetchStream); - const downloader = new DownloadEngineFile(file, { - chunkSize: 4, - parallelStreams: 8, - fetchStream, - writeStream - }); - - await downloader.download(); - context.expect(totalBytesWritten) - .toBe(file.totalSize); - }); }, {timeout: 1000 * 60 * 3}); diff --git a/test/download.test.ts b/test/download.test.ts index 7b1bbde..772427c 100644 --- a/test/download.test.ts +++ b/test/download.test.ts @@ -4,7 +4,7 @@ import DownloadEngineWriteStreamBrowser from "../src/download/download-engine/st import {BIG_IMAGE} from "./utils/files.js"; import {createDownloadFile} from "./utils/download.js"; import DownloadEngineFetchStreamFetch from "../src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js"; -import DownloadEngineFile from "../src/download/download-engine/download-engine-file.js"; +import DownloadEngineFile from "../src/download/download-engine/download-file/download-engine-file.js"; describe("File Download", () => { test.concurrent("Parallel connection download", async (context) => {