feat(ci): progress bar for ci

package.json

@@ -130,6 +130,7 @@
     "@tinyhttp/content-disposition": "^2.2.0",
     "async-retry": "^1.3.3",
     "chalk": "^5.3.0",
+    "ci-info": "^4.0.0",
     "cli-spinners": "^2.9.2",
     "commander": "^10.0.0",
     "eventemitter3": "^5.0.1",

src/cli/cli.ts

@@ -5,6 +5,7 @@ import {packageJson} from "../const.js";
 import {downloadFile, downloadSequence} from "../download/node-download.js";
 import {setCommand} from "./commands/set.js";
 import findDownloadDir, {downloadToDirectory, findFileName} from "./utils/find-download-dir.js";
+import {AvailableCLIProgressStyle} from "../download/transfer-visualize/transfer-cli/progress-bars/switch-cli-progress-style.js";
 
 
 const pullCommand = new Command();
@@ -13,9 +14,16 @@ pullCommand
     .argument("[files...]", "Files to pull/copy")
     .option("-s --save [path]", "Save location (directory/file)")
     .option("-c --connections [number]", "Number of parallel connections", "4")
+    .addOption(new Option("-st --style [type]", "The style of the CLI progress bar").choices(["basic", "fancy", "ci", "summary"]))
     .addOption(new Option("-p --program [type]", "The download strategy").choices(["stream", "chunks"]))
     .option("-t --truncate-name", "Truncate file names in the CLI status to make them appear shorter")
-    .action(async (files: string[] = [], {save: saveLocation, truncateName, number, program}: { save?: string, truncateName?: boolean, number: string, program: string }) => {
+    .action(async (files: string[] = [], {save: saveLocation, truncateName, number, program, style}: {
+        save?: string,
+        truncateName?: boolean,
+        number: string,
+        program: string,
+        style: AvailableCLIProgressStyle
+    }) => {
         if (files.length === 0) {
             pullCommand.outputHelp();
             process.exit(0);
@@ -41,7 +49,7 @@ pullCommand
         const downloader = await downloadSequence({
             truncateName,
             cliProgress: true,
-            cliStyle: "fancy"
+            cliStyle: style
         }, ...fileDownloads);
         await downloader.download();
     })

src/download/download-engine/download-file/progress-status-file.ts

@@ -23,6 +23,7 @@ export enum DownloadStatus {
 
 export enum DownloadFlags {
     Existing = "Existing",
+    DownloadSequence = "DownloadSequence"
 }
 
 export default class ProgressStatusFile {

src/download/download-engine/engine/download-engine-multi-download.ts

@@ -4,6 +4,7 @@ import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer
 import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js";
 import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
 import {concurrency} from "./utils/concurrency.js";
+import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js";
 
 const DEFAULT_PARALLEL_DOWNLOADS = 1;
 
@@ -44,6 +45,8 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
     }
 
     protected _init() {
+        this._progressStatisticsBuilder.downloadStatus = DownloadStatus.NotStarted;
+
         this._changeEngineFinishDownload();
         for (const [index, engine] of Object.entries(this.downloads)) {
             const numberIndex = Number(index);
@@ -55,6 +58,7 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
 
         this._progressStatisticsBuilder.add(...this.downloads);
         this._progressStatisticsBuilder.on("progress", progress => {
+            progress.downloadFlags = progress.downloadFlags.concat([DownloadFlags.DownloadSequence]);
             this.emit("progress", progress);
         });
     }
@@ -64,8 +68,10 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
             throw new DownloadAlreadyStartedError();
         }
 
+        this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active;
         this.emit("start");
 
+
         const concurrencyCount = this.options.parallelDownloads ?? DEFAULT_PARALLEL_DOWNLOADS;
         await concurrency(this.downloads, concurrencyCount, async (engine) => {
             if (this._aborted) return;
@@ -78,6 +84,7 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
             this._activeEngines.delete(engine);
         });
 
+        this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished;
         this.emit("finished");
         await this._finishEnginesDownload();
         await this.close();
@@ -104,22 +111,28 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
     }
 
     public pause(): void {
+        this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Paused;
         this._activeEngines.forEach(engine => engine.pause());
     }
 
     public resume(): void {
+        this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active;
         this._activeEngines.forEach(engine => engine.resume());
     }
 
     public async close() {
         if (this._aborted) return;
         this._aborted = true;
 
+        if (this._progressStatisticsBuilder.downloadStatus !== DownloadStatus.Finished) {
+            this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Cancelled;
+        }
+
         const closePromises = Array.from(this._activeEngines)
             .map(engine => engine.close());
         await Promise.all(closePromises);
-        this.emit("closed");
 
+        this.emit("closed");
     }
 
     protected static _extractEngines<Engine>(engines: Engine[]) {

src/download/transfer-visualize/progress-statistics-builder.ts

@@ -1,9 +1,9 @@
 import BaseDownloadEngine from "../download-engine/engine/base-download-engine.js";
 import {EventEmitter} from "eventemitter3";
 import TransferStatistics from "./transfer-statistics.js";
-import DownloadEngineMultiDownload from "../download-engine/engine/download-engine-multi-download.js";
 import {createFormattedStatus, FormattedStatus} from "./format-transfer-status.js";
 import DownloadEngineFile from "../download-engine/download-file/download-engine-file.js";
+import {DownloadStatus, ProgressStatus} from "../download-engine/download-file/progress-status-file.js";
 
 export type ProgressStatusWithIndex = FormattedStatus & {
     index: number,
@@ -13,18 +13,23 @@ interface CliProgressBuilderEvents {
     progress: (progress: ProgressStatusWithIndex) => void;
 }
 
-export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload;
+export type AnyEngine = DownloadEngineFile | BaseDownloadEngine;
 export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressBuilderEvents> {
-    protected _engines: AnyEngine[] = [];
-    protected _activeTransfers: { [index: number]: number } = {};
-    protected _totalBytes = 0;
-    protected _transferredBytes = 0;
-    protected statistics = new TransferStatistics();
+    private _engines: AnyEngine[] = [];
+    private _activeTransfers: { [index: number]: number } = {};
+    private _totalBytes = 0;
+    private _transferredBytes = 0;
+    private _totalDownloadParts = 0;
+    private _activeDownloadPart = 0;
+    private _startTime = 0;
+    private statistics = new TransferStatistics();
+    public downloadStatus: DownloadStatus = null!;
 
     public get totalBytes() {
         return this._totalBytes;
     }
 
+
     public get transferredBytesWithActiveTransfers() {
         return this._transferredBytes + Object.values(this._activeTransfers)
             .reduce((acc, bytes) => acc + bytes, 0);
@@ -36,22 +41,15 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
         }
     }
 
-    protected _initEvents(engine: AnyEngine) {
+    private _initEvents(engine: AnyEngine) {
         this._engines.push(engine);
         this._totalBytes += engine.downloadSize;
         const index = this._engines.length - 1;
+        const downloadPartStart = this._totalDownloadParts;
+        this._totalDownloadParts += engine.status.totalDownloadParts;
 
         engine.on("progress", (data) => {
-            this._activeTransfers[index] = data.transferredBytes;
-            const progress = this.statistics.updateProgress(this.transferredBytesWithActiveTransfers, this.totalBytes);
-
-            this.emit("progress", {
-                ...createFormattedStatus({
-                    ...data,
-                    ...progress
-                }),
-                index
-            });
+            this._sendProgress(data, index, downloadPartStart);
         });
 
         engine.on("finished", () => {
@@ -60,6 +58,34 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
         });
     }
 
+
+    private _sendProgress(data: ProgressStatus, index: number, downloadPartStart: number) {
+        this._startTime ||= data.startTime;
+        this._activeTransfers[index] = data.transferredBytes;
+        if (downloadPartStart + data.downloadPart > this._activeDownloadPart) {
+            this._activeDownloadPart = downloadPartStart + data.downloadPart;
+        }
+
+        const progress = this.statistics.updateProgress(this.transferredBytesWithActiveTransfers, this.totalBytes);
+        const activeDownloads = Object.keys(this._activeTransfers).length;
+
+        this.emit("progress", {
+            ...createFormattedStatus({
+                ...progress,
+                downloadPart: this._activeDownloadPart,
+                totalDownloadParts: this._totalDownloadParts,
+                startTime: this._startTime,
+                fileName: data.fileName,
+                comment: data.comment,
+                transferAction: data.transferAction,
+                downloadStatus: this.downloadStatus || data.downloadStatus,
+                endTime: activeDownloads <= 1 ? data.endTime : 0,
+                downloadFlags: data.downloadFlags
+            }),
+            index
+        });
+    }
+
     static oneStatistics(engine: DownloadEngineFile) {
         const progress = engine.status;
         const statistics = TransferStatistics.oneStatistics(progress.transferredBytes, progress.totalBytes);

src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts

@@ -3,9 +3,9 @@ import DownloadEngineMultiDownload from "../../download-engine/engine/download-e
 import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./progress-bars/switch-cli-progress-style.js";
 import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js";
 import TransferCli, {CLI_LEVEL, TransferCliOptions} from "./transfer-cli.js";
-import {BaseMultiProgressBar} from "./multiProgressBars/baseMultiProgressBar.js";
+import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js";
 
-const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "fancy";
+const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto";
 type AllowedDownloadEngines = DownloadEngineNodejs | DownloadEngineMultiDownload;
 
 export type CliProgressDownloadEngineOptions = {
@@ -47,7 +47,10 @@ export default class CliAnimationWrapper {
         }
 
         cliOptions.createProgressBar = typeof this._options.cliStyle === "function" ?
-            this._options.cliStyle :
+            {
+                createStatusLine: this._options.cliStyle,
+                multiProgressBar: this._options.createMultiProgressBar ?? BaseMultiProgressBar
+            } :
             switchCliProgressStyle(this._options.cliStyle ?? DEFAULT_CLI_STYLE, {truncateName: this._options.truncateName});
 
         this._activeCLI = new TransferCli(cliOptions, this._options.cliLevel);
@@ -64,8 +67,8 @@ export default class CliAnimationWrapper {
         engine.once("start", () => {
             this._activeCLI?.start();
 
-            engine.on("progress", () => {
-                this._activeCLI?.updateStatues(engine.downloadStatues);
+            engine.on("progress", (progress) => {
+                this._activeCLI?.updateStatues(engine.downloadStatues, progress);
             });
 
             engine.on("closed", () => {

src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts

@@ -1,13 +1,16 @@
 import UpdateManager from "stdout-update";
 import sleep from "sleep-promise";
+import {CLIProgressPrintType} from "../multiProgressBars/BaseMultiProgressBar.js";
 
 export type BaseLoadingAnimationOptions = {
-    updateIntervalMs?: number;
+    updateIntervalMs?: number | null;
     loadingText?: string;
+    logType: CLIProgressPrintType
 };
 
 export const DEFAULT_LOADING_ANIMATION_OPTIONS: BaseLoadingAnimationOptions = {
-    loadingText: "Gathering information"
+    loadingText: "Gathering information",
+    logType: "update"
 };
 
 const DEFAULT_UPDATE_INTERVAL_MS = 300;
@@ -24,15 +27,23 @@ export default abstract class BaseLoadingAnimation {
     }
 
     protected _render(): void {
-        this.stdoutManager.update([this.createFrame()]);
+        const frame = this.createFrame();
+
+        if (this.options.logType === "update") {
+            this.stdoutManager.update([frame]);
+        } else {
+            console.log(frame);
+        }
     }
 
     protected abstract createFrame(): string;
 
     async start() {
         process.on("SIGINT", this._processExit);
 
-        this.stdoutManager.hook();
+        if (this.options.logType === "update") {
+            this.stdoutManager.hook();
+        }
 
         this._animationActive = true;
         while (this._animationActive) {
@@ -47,8 +58,11 @@ export default abstract class BaseLoadingAnimation {
         }
 
         this._animationActive = false;
-        this.stdoutManager.erase();
-        this.stdoutManager.unhook(false);
+
+        if (this.options.logType === "update") {
+            this.stdoutManager.erase();
+            this.stdoutManager.unhook(false);
+        }
 
         process.off("SIGINT", this._processExit);
     }

src/download/transfer-visualize/transfer-cli/multiProgressBars/baseMultiProgressBar.ts → src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts

@@ -1,23 +1,28 @@
-import {CliFormattedStatus} from "../progress-bars/base-transfer-cli-progress-bar.js";
+import {TransferCliProgressBar} from "../progress-bars/base-transfer-cli-progress-bar.js";
 import {FormattedStatus} from "../../format-transfer-status.js";
 import {DownloadStatus} from "../../../download-engine/download-file/progress-status-file.js";
 import chalk from "chalk";
 import prettyBytes from "pretty-bytes";
 
 export type MultiProgressBarOptions = {
     maxViewDownloads: number;
-    createProgressBar: (status: CliFormattedStatus) => string
+    createProgressBar: TransferCliProgressBar
     action?: string;
 };
 
+export type CLIProgressPrintType = "update" | "log";
+
 export class BaseMultiProgressBar {
+    public readonly updateIntervalMs: null | number = null;
+    public readonly printType: CLIProgressPrintType = "update";
+
     public constructor(protected options: MultiProgressBarOptions) {
     }
 
     protected createProgresses(statuses: FormattedStatus[]): string {
         return statuses.map((status) => {
             status.transferAction = this.options.action ?? status.transferAction;
-            return this.options.createProgressBar(status);
+            return this.options.createProgressBar.createStatusLine(status);
         })
             .join("\n");
     }
@@ -41,7 +46,8 @@ export class BaseMultiProgressBar {
         };
     }
 
-    createMultiProgressBar(statuses: FormattedStatus[]) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus) {
         if (statuses.length < this.options.maxViewDownloads) {
             return this.createProgresses(statuses);
         }

src/download/transfer-visualize/transfer-cli/multiProgressBars/CIMultiProgressBar.ts

@@ -0,0 +1,6 @@
+import {SummaryMultiProgressBar} from "./SummaryMultiProgressBar.js";
+
+export class CIMultiProgressBar extends SummaryMultiProgressBar {
+    public override readonly printType = "log";
+    public override readonly updateIntervalMs = parseInt(process.env.IPULL_CI_UPDATE_INTERVAL ?? "0") || 8_000;
+}