From e3138fee5eaaacd4e633e0b5fff5f6b4ed096969 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Nov 2023 20:25:18 +0100 Subject: [PATCH] Split out all file code to library --- src/convert/Convertor.tsx | 233 ++----------------- src/convert/FileTree.tsx | 61 ----- src/convert/ffmpeg.ts | 2 +- src/lib/FileTree.tsx | 279 +++++++++++++++++++++++ src/{convert => lib}/Upload.tsx | 0 src/{convert => lib}/filetree.module.css | 0 src/{convert => lib}/upload.module.css | 0 7 files changed, 296 insertions(+), 279 deletions(-) delete mode 100644 src/convert/FileTree.tsx create mode 100644 src/lib/FileTree.tsx rename src/{convert => lib}/Upload.tsx (100%) rename src/{convert => lib}/filetree.module.css (100%) rename src/{convert => lib}/upload.module.css (100%) diff --git a/src/convert/Convertor.tsx b/src/convert/Convertor.tsx index 918921a..a66e75a 100644 --- a/src/convert/Convertor.tsx +++ b/src/convert/Convertor.tsx @@ -1,5 +1,5 @@ -import {Upload} from "./Upload.js" -import {FileTree, FileTreeBranch, FileTreeLeaf} from "./FileTree.js" +import {Upload} from "../lib/Upload.js" +import {FileTree, FileTreeBranch, readFileSystemHandle, updateLeaf, convertAll} from "../lib/FileTree.js" import * as css from "./convertor.module.css" import { JSX } from "preact" import {useState} from 'preact/hooks' @@ -7,200 +7,10 @@ import {convert, getOutputFilename} from "./ffmpeg.js" const NR_WORKERS = 2 -async function fromAsync(source: Iterable | AsyncIterable): Promise { - const items:T[] = []; - for await (const item of source) { - items.push(item); - } - return items -} - -async function readFileSystemHandle(fshs: FileSystemHandle[], fileFilter: (file: File) => boolean): Promise { - const result: FileTreeBranch = new Map() - for (const fsh of fshs) { - if (fsh instanceof FileSystemFileHandle) { - const file = await fsh.getFile() - if (fileFilter(file)) { - result.set(fsh.name, {file: await fsh.getFile()}) - } - } else if (fsh instanceof FileSystemDirectoryHandle) { - result.set(fsh.name, await readFileSystemHandle( - await fromAsync(fsh.values()), fileFilter)) - } else { - throw new Error(`Unhandled case: ${fsh}`); - } - } - pruneDeadBranches(result) - return new Map( - [...result.entries()] - .sort(([a], [b]) => a.localeCompare(b, undefined, {numeric: true}))) -} - function fileFilter(file: File, extension: string): boolean { return !file.name.startsWith(".") && file.name.endsWith("." + extension) } -function pruneDeadBranches(branch: FileTreeBranch) { - for (const [name, entry] of branch.entries()) { - if (entry instanceof Map) { - pruneDeadBranches(entry) - if (entry.size === 0) { - branch.delete(name) - } - } - } -} - -function updateLeaf(files: FileTreeBranch, path: string[], update: FileTreeLeaf | ((current: FileTreeLeaf) => FileTreeLeaf)): FileTreeBranch { - const newFiles = new Map([...files]) - const [p, ...restpath] = path - const item = files.get(p) - let newItem: FileTreeLeaf | FileTreeBranch - if (restpath.length === 0) { - if (!(item && "file" in item)) { - throw new Error(`Error in path: $(path}`) - } - if ("file" in update) { - newItem = update; - } else { - newItem = update(item) - } - } else { - if (!(item instanceof Map)) { - throw new Error(`Error in path: $(path}`) - } - newItem = updateLeaf(item, restpath, update) - } - newFiles.set(p, newItem) - return newFiles -} - -function findLeaf(files: FileTreeBranch, path: string[]): FileTreeLeaf { - const [p, ...restpath] = path - const item = files.get(p) - if (restpath.length === 0) { - if (!(item && "file" in item)) { - throw new Error(`Error in path: $(path}`) - } - return item - } - if (!(item instanceof Map)) { - throw new Error(`Error in path: $(path}`) - } - return findLeaf(item, restpath) -} - -function getAllLeafPaths(files: FileTreeBranch): string[][] { - return [...files.entries()] - .flatMap(([name, entry]) => - (entry instanceof Map) - ? getAllLeafPaths(entry).map(path => [name, ...path]) - : [[name]]) -} - -async function nonEmptyFileExists( - directory: FileSystemDirectoryHandle, - path: string[] -): Promise { - let pointer = directory - for (const p of path.slice(0, -1)) { - try { - pointer = await pointer.getDirectoryHandle(p) - } catch (e) { - if ((e as DOMException).name === "NotFoundError") { - return false - } - throw e - } - } - const filename = path.slice(-1)[0] - let file: FileSystemFileHandle - try { - file = await pointer.getFileHandle(filename) - } catch (e) { - if ((e as DOMException).name === "NotFoundError") { - return false - } - throw e - } - return (await file.getFile()).size > 0 -} - -async function convertOne( - files: FileTreeBranch, - path: string[], - destination: FileSystemDirectoryHandle, - setFiles: (cb: (files: FileTreeBranch) => FileTreeBranch) => void -) { - const leaf = findLeaf(files, path) - const outfilename = getOutputFilename(leaf.file.name) - const outpath = [...path.slice(0, -1), outfilename] - if (await nonEmptyFileExists(destination, outpath)) { - setFiles(files => updateLeaf(files, path, leaf => ( - {file: leaf.file, progress: {"error": "File aready exists at the destination"}}))) - return - } - - let pointer = destination - for (const p of path.slice(0, -1)) { - //probably should catch if there is a file with this directoy name //TODO - pointer = await pointer.getDirectoryHandle(p, {create: true}) - } - const outfile = await pointer.getFileHandle(outfilename, {create: true}) - const outstream = await outfile.createWritable() - try { - await convert(leaf.file, outstream, (progress: FileTreeLeaf["progress"]) => { - setFiles(files => - updateLeaf(files, path, leaf => ({file: leaf.file, progress}))) - }) - await outstream.close() - setFiles(files => - updateLeaf(files, path, leaf => ( - {file: leaf.file, progress: "done"}))) - } catch (e) { - setFiles(files => - updateLeaf(files, path, leaf => ( - {file: leaf.file, progress: {error: `error while converting: $(e)`}}))) - await outstream.close() - await pointer.removeEntry(outfilename) - } -} - -async function convertAll(files: FileTreeBranch, setFiles: (cb: FileTreeBranch | ((files: FileTreeBranch) => FileTreeBranch)) => void) { - const destination = await window.showDirectoryPicker( - {id: "mp4save", mode: "readwrite"}) - const outputFiles = ((await readFileSystemHandle([destination], file => fileFilter(file, "mp4"))).get(destination.name) ?? new Map()) as FileTreeBranch - - const paths = getAllLeafPaths(files) - let newFiles = files - const queuedPaths: string[][] = [] - for (const path of paths) { - const outpath = [...path.slice(0, -1), getOutputFilename(path.slice(-1)[0])] - if (await nonEmptyFileExists(destination, outpath)) { - newFiles = updateLeaf( - newFiles, path, leaf => ({file: leaf.file, progress: {"error": "File aready exists at the destination"}})) - continue - } - newFiles = updateLeaf( - newFiles, path, leaf => ({file: leaf.file, progress: "queue"})) - queuedPaths.push(path) - } - setFiles(newFiles) - - const promises: Set> = new Set() - let finished: Promise[] = [] - for (const path of queuedPaths) { - while (promises.size >= NR_WORKERS) { - await Promise.any(promises) - finished.forEach(p => promises.delete(p)) - finished = [] - } - const promise = convertOne(files, path, destination, setFiles) - promises.add(promise) - promise.then(() => finished.push(promise)) - } - await Promise.all(promises) -} export function Convertor({}: {}): JSX.Element { const [files, setFiles] = useState(new Map()) @@ -210,30 +20,19 @@ export function Convertor({}: {}): JSX.Element { const newFiles = await readFileSystemHandle(fileSystemHandles, file => fileFilter(file, "MTS")) setFiles(files => new Map([...files, ...newFiles])) } - function removeFile(name: string[]) { - setFiles(files => { - const newFiles = new Map([...files]) - let pointer = newFiles - for (const dir of name.slice(0, -1)) { - if (!pointer.has(dir)) { - throw new Error(`Could not find ${dir} (${name})`) - } - const newpointer = pointer.get(dir)! - if (!(newpointer instanceof Map)) { - throw new Error(`${dir} (${name}) is a File`) - } - const copieddir = new Map([...newpointer]) - pointer.set(dir, copieddir) - pointer = copieddir - } - const [filename] = name.slice(-1) - if (!pointer.has(filename)) { - throw new Error (`Could not find ${filename} (${name})`) - } - pointer.delete(filename) - pruneDeadBranches(newFiles) - return newFiles - }) + function removeFile(path: string[]) { + setFiles(files => updateLeaf(files, path, null)) + } + + async function doConvertAll() { + setState("converting"); + await convertAll( + files, + NR_WORKERS, + convert, + getOutputFilename, + setFiles) + setState("done") } return <> @@ -246,7 +45,7 @@ export function Convertor({}: {}): JSX.Element { {state === "uploading" && } } diff --git a/src/convert/FileTree.tsx b/src/convert/FileTree.tsx deleted file mode 100644 index ec7a7e0..0000000 --- a/src/convert/FileTree.tsx +++ /dev/null @@ -1,61 +0,0 @@ -import * as css from "./filetree.module.css" -import { JSX } from "preact" - -export type FileTreeLeaf = { - file: File - progress?: "queue" | {"converting": number} | "done" | {error: string} -} -export type FileTreeBranch = Map - -type FileTreeProps = { - files: FileTreeBranch - removeFile: (name: string[]) => void -} - -function attributesForLeaf(fileTreeLeaf: FileTreeLeaf): {className: string, style: any, title?: string} { - const classes = [css.filename] - const style: any = {} - let title: undefined | string = undefined - if (fileTreeLeaf.progress === undefined) { - classes.push(css.editable) - } else if (fileTreeLeaf.progress === "queue") { - classes.push(css.inqueue) - } else if (fileTreeLeaf.progress === "done") { - classes.push(css.done) - } else if ("converting" in fileTreeLeaf.progress) { - classes.push(css.converting) - style["--convert-progress"] = fileTreeLeaf.progress.converting - style["--convert-progress-text"] = JSON.stringify(`${(fileTreeLeaf.progress.converting * 100).toFixed(1)}%`) - title = `${(fileTreeLeaf.progress.converting * 100).toFixed(1)}% done` - } else if ("error" in fileTreeLeaf.progress) { - classes.push(css.error) - style["--error-message"] = JSON.stringify(fileTreeLeaf.progress.error) - title = `Error: ${fileTreeLeaf.progress.error}` - } else { - const exhaustive: never = fileTreeLeaf.progress - throw new Error(`Exhaustive check: ${exhaustive}`) - } -return {className: classes.join(" "), style, title} -} - - -export function FileTree({files, removeFile}: FileTreeProps): JSX.Element { - return
    - {[...files.entries()].map(([name, entry]) => -
  • - {(entry instanceof Map) - ? <> -
    {name}
    - removeFile([name, ...s])} /> - - : <> -
    - {name} - removeFile([name])}> -
    - - } -
  • - )} -
-} diff --git a/src/convert/ffmpeg.ts b/src/convert/ffmpeg.ts index f70bc35..1afd6c6 100644 --- a/src/convert/ffmpeg.ts +++ b/src/convert/ffmpeg.ts @@ -1,5 +1,5 @@ import type * as LibAVTypes from '../../public/app/bundled/libavjs/dist/libav.types' -import type {FileTreeLeaf} from "./FileTree.js" +import type {FileTreeLeaf} from "../lib/FileTree.js" declare global { interface Window { diff --git a/src/lib/FileTree.tsx b/src/lib/FileTree.tsx new file mode 100644 index 0000000..930e884 --- /dev/null +++ b/src/lib/FileTree.tsx @@ -0,0 +1,279 @@ +import * as css from "./filetree.module.css" +import { JSX } from "preact" + +export type ConvertAction = ( + input: File, + output: FileSystemWritableFileStream, + onProgress: (progress: FileTreeLeaf["progress"]) => void +) => Promise + +async function fromAsync(source: Iterable | AsyncIterable): Promise { + const items:T[] = []; + for await (const item of source) { + items.push(item); + } + return items +} + +export async function readFileSystemHandle(fshs: FileSystemHandle[], fileFilter: (file: File) => boolean): Promise { + const result: FileTreeBranch = new Map() + for (const fsh of fshs) { + if (fsh instanceof FileSystemFileHandle) { + const file = await fsh.getFile() + if (fileFilter(file)) { + result.set(fsh.name, {file: await fsh.getFile()}) + } + } else if (fsh instanceof FileSystemDirectoryHandle) { + result.set(fsh.name, await readFileSystemHandle( + await fromAsync(fsh.values()), fileFilter)) + } else { + throw new Error(`Unhandled case: ${fsh}`); + } + } + pruneDeadBranches(result) + return new Map( + [...result.entries()] + .sort(([a], [b]) => a.localeCompare(b, undefined, {numeric: true}))) +} + +export type FileTreeLeaf = { + file: File + progress?: "queue" | {"converting": number} | "done" | {error: string} +} +export type FileTreeBranch = Map + +type FileTreeProps = { + files: FileTreeBranch + removeFile: (name: string[]) => void +} + +function pruneDeadBranches(branch: FileTreeBranch) { + for (const [name, entry] of branch.entries()) { + if (entry instanceof Map) { + pruneDeadBranches(entry) + if (entry.size === 0) { + branch.delete(name) + } + } + } +} + +export function updateLeaf(files: FileTreeBranch, path: string[], update: FileTreeLeaf | null | ((current: FileTreeLeaf) => (FileTreeLeaf | null))): FileTreeBranch { + const newFiles = new Map([...files]) + const [p, ...restpath] = path + const item = files.get(p) + let newItem: FileTreeLeaf | FileTreeBranch | null + if (restpath.length === 0) { + if (!(item && "file" in item)) { + throw new Error(`Error in path: $(path}`) + } + if (update === null || "file" in update) { + newItem = update; + } else { + newItem = update(item) + } + } else { + if (!(item instanceof Map)) { + throw new Error(`Error in path: $(path}`) + } + newItem = updateLeaf(item, restpath, update) + if (newItem.size === 0) { + newItem = null + } + } + if (newItem === null) { + newFiles.delete(p) + } else { + newFiles.set(p, newItem) + } + return newFiles +} + +function findLeaf(files: FileTreeBranch, path: string[]): FileTreeLeaf { + const [p, ...restpath] = path + const item = files.get(p) + if (restpath.length === 0) { + if (!(item && "file" in item)) { + throw new Error(`Error in path: $(path}`) + } + return item + } + if (!(item instanceof Map)) { + throw new Error(`Error in path: $(path}`) + } + return findLeaf(item, restpath) +} + +function getAllLeafPaths(files: FileTreeBranch): string[][] { + return [...files.entries()] + .flatMap(([name, entry]) => + (entry instanceof Map) + ? getAllLeafPaths(entry).map(path => [name, ...path]) + : [[name]]) +} + +async function nonEmptyFileExists( + directory: FileSystemDirectoryHandle, + path: string[] +): Promise { + let pointer = directory + for (const p of path.slice(0, -1)) { + try { + pointer = await pointer.getDirectoryHandle(p) + } catch (e) { + if ((e as DOMException).name === "NotFoundError") { + return false + } + throw e + } + } + const filename = path.slice(-1)[0] + let file: FileSystemFileHandle + try { + file = await pointer.getFileHandle(filename) + } catch (e) { + if ((e as DOMException).name === "NotFoundError") { + return false + } + throw e + } + return (await file.getFile()).size > 0 +} + +async function convertOne( + files: FileTreeBranch, + path: string[], + destination: FileSystemDirectoryHandle, + conversionAction: ConvertAction, + getOutputFilename: (name: string) => string, + setFiles: (cb: (files: FileTreeBranch) => FileTreeBranch) => void +) { + const leaf = findLeaf(files, path) + const outfilename = getOutputFilename(leaf.file.name) + const outpath = [...path.slice(0, -1), outfilename] + if (await nonEmptyFileExists(destination, outpath)) { + setFiles(files => updateLeaf(files, path, leaf => ( + {file: leaf.file, progress: {"error": "File aready exists at the destination"}}))) + return + } + + let pointer = destination + for (const p of path.slice(0, -1)) { + //probably should catch if there is a file with this directoy name //TODO + pointer = await pointer.getDirectoryHandle(p, {create: true}) + } + const outfile = await pointer.getFileHandle(outfilename, {create: true}) + const outstream = await outfile.createWritable() + try { + await conversionAction(leaf.file, outstream, (progress: FileTreeLeaf["progress"]) => { + setFiles(files => + updateLeaf(files, path, leaf => ({file: leaf.file, progress}))) + }) + await outstream.close() + setFiles(files => + updateLeaf(files, path, leaf => ( + {file: leaf.file, progress: "done"}))) + } catch (e) { + setFiles(files => + updateLeaf(files, path, leaf => ( + {file: leaf.file, progress: {error: `error while converting: $(e)`}}))) + await outstream.close() + await pointer.removeEntry(outfilename) + } +} + +export async function convertAll( + files: FileTreeBranch, + concurrency: number, + conversionAction: ConvertAction, + getOutputFilename: (name: string) => string, + setFiles: (cb: FileTreeBranch | ((files: FileTreeBranch) => FileTreeBranch)) => void +) { + const destination = await window.showDirectoryPicker( + {id: "mp4save", mode: "readwrite"}) + + const paths = getAllLeafPaths(files) + let newFiles = files + const queuedPaths: string[][] = [] + for (const path of paths) { + const outpath = [...path.slice(0, -1), getOutputFilename(path.slice(-1)[0])] + if (await nonEmptyFileExists(destination, outpath)) { + newFiles = updateLeaf( + newFiles, path, leaf => ({file: leaf.file, progress: {"error": "File aready exists at the destination"}})) + continue + } + newFiles = updateLeaf( + newFiles, path, leaf => ({file: leaf.file, progress: "queue"})) + queuedPaths.push(path) + } + setFiles(newFiles) + + const promises: Set> = new Set() + let finished: Promise[] = [] + for (const path of queuedPaths) { + while (promises.size >= concurrency) { + await Promise.any(promises) + finished.forEach(p => promises.delete(p)) + finished = [] + } + const promise = convertOne( + files, + path, + destination, + conversionAction, + getOutputFilename, + setFiles + ) + promises.add(promise) + promise.then(() => finished.push(promise)) + } + await Promise.all(promises) +} + +function attributesForLeaf(fileTreeLeaf: FileTreeLeaf): {className: string, style: any, title?: string} { + const classes = [css.filename] + const style: any = {} + let title: undefined | string = undefined + if (fileTreeLeaf.progress === undefined) { + classes.push(css.editable) + } else if (fileTreeLeaf.progress === "queue") { + classes.push(css.inqueue) + } else if (fileTreeLeaf.progress === "done") { + classes.push(css.done) + } else if ("converting" in fileTreeLeaf.progress) { + classes.push(css.converting) + style["--convert-progress"] = fileTreeLeaf.progress.converting + style["--convert-progress-text"] = JSON.stringify(`${(fileTreeLeaf.progress.converting * 100).toFixed(1)}%`) + title = `${(fileTreeLeaf.progress.converting * 100).toFixed(1)}% done` + } else if ("error" in fileTreeLeaf.progress) { + classes.push(css.error) + style["--error-message"] = JSON.stringify(fileTreeLeaf.progress.error) + title = `Error: ${fileTreeLeaf.progress.error}` + } else { + const exhaustive: never = fileTreeLeaf.progress + throw new Error(`Exhaustive check: ${exhaustive}`) + } +return {className: classes.join(" "), style, title} +} + + +export function FileTree({files, removeFile}: FileTreeProps): JSX.Element { + return
    + {[...files.entries()].map(([name, entry]) => +
  • + {(entry instanceof Map) + ? <> +
    {name}
    + removeFile([name, ...s])} /> + + : <> +
    + {name} + removeFile([name])}> +
    + + } +
  • + )} +
+} diff --git a/src/convert/Upload.tsx b/src/lib/Upload.tsx similarity index 100% rename from src/convert/Upload.tsx rename to src/lib/Upload.tsx diff --git a/src/convert/filetree.module.css b/src/lib/filetree.module.css similarity index 100% rename from src/convert/filetree.module.css rename to src/lib/filetree.module.css diff --git a/src/convert/upload.module.css b/src/lib/upload.module.css similarity index 100% rename from src/convert/upload.module.css rename to src/lib/upload.module.css