Fix the storage API, introduce storage().putRandom() (#34)

* Fix the storage API, introduce `storage().putRandom()`

* Link to remix

.changeset/real-beans-live.md

@@ -0,0 +1,5 @@
+---
+"superflare": patch
+---
+
+Fix the storage API, introduce `storage().putRandom()`, and provide a better way to stream file uploads with `parseMultipartFormData`

examples/remix-cms/app/routes/admin/upload.$.ts

@@ -1,22 +1,19 @@
 import { json, type ActionArgs } from "@remix-run/cloudflare";
-import { storage } from "superflare";
+import { parseMultipartFormData, storage } from "superflare";
 
 export async function action({ request }: ActionArgs) {
-  /**
-   * This is probably inefficient, but it's the only way to get the `File`
-   * which includes useful information like the file name and type.
-   *
-   * Remix offers (unstable) upload handler APIs which allow you to stream the file body to
-   * the destination _before_ it's fully loaded into memory, whiel exposing the file name and file type.
-   * This would be more efficient, but the APIs aren't fully developed, and I couldn't figure
-   * out how to convert them to a proper `File` object to pass to `storage().put()`. We can
-   * revisit this later.
-   */
-  const formData = await request.formData();
-  const file = formData.get("file") as File;
-  const object = await storage().put(file);
+  const formData = await parseMultipartFormData(
+    request,
+    async ({ stream, filename }) => {
+      const object = await storage().putRandom(stream, {
+        extension: filename?.split(".").pop(),
+      });
+
+      return object.key;
+    }
+  );
 
   return json({
-    url: storage().url(object.key),
+    url: storage().url(formData.get("file") as string),
   });
 }

packages/superflare/docs/file-storage.md

@@ -36,6 +36,69 @@ export async function action() {
 }
 ```
 
+### Handling file uploads
+
+If your application accepts file uploads, you will likely have an endpoint, like a Remix action, which handles an incoming request.
+
+The simplest approach to handling file uploads is to use the [FormData API](https://developer.mozilla.org/en-US/docs/Web/API/FormData) to parse the request body with `await request.formData()` and use the `storage().put(name, formData.get('file'))` method to store the file in your R2 bucket.
+
+However, this approach requires you to read the entire file into memory before storing it. This can be problematic if you are uploading large files:
+
+```ts
+import { storage } from "superflare";
+
+export async function action({ request }) {
+  // ⚠️ Entire file is loaded into memory!
+  const formData = await request.formData();
+  const file = formData.get("file");
+  const key = "my-file.txt";
+
+  const r2Object = await storage().put(key, file);
+}
+```
+
+Instead, a better option is to stream file uploads directly to R2. Superflare provides a `parseMultipartFormData` utility, [inspired by Remix](https://remix.run/docs/en/1.14.3/utils/parse-multipart-form-data#uploadhandler), to make parsing multipart form data requests easier:
+
+```ts
+import { storage, parseMultipartFormData } from "superflare";
+
+export async function action({ request }) {
+  const formData = await parseMultipartFormData(
+    request,
+    async ({ name, filename, stream, data }) => {
+      // This is the name of your HTML file input
+      if (name === "file") {
+        const r2Object = await storage().put(filename, stream);
+        return r2Object.key;
+      }
+
+      // Support other non-file fields
+      return data;
+    }
+  );
+
+  const url = storage.url(formData.get("file"));
+
+  // ...
+}
+```
+
+### Uploading files with random names
+
+Sometimes you may want to upload a file with a random name to prevent conflicts between other uploads. You can use the `storage().putRandom(input)` method to do this:
+
+```ts
+const r2Object = await storage().putRandom(file, {
+  // Optional extension
+  extension: filename?.split(".").pop(),
+
+  // Optional prefix
+  prefix: "avatars",
+});
+```
+
+By default, no extension or prefix will be added to the file name. You can override this by passing an `extension` or `prefix` option to the `putRandom` method. Prefixes and filenames are separated by a forward slash (`/`) similar to a folder structure.
+
 ## Serving public files
 
 By default, Superflare does not serve your bucket contents to the public. However, you can mark a disk as public by setting the `publicPath` property to a public route in your `superflare.config.ts` file:
@@ -113,3 +176,33 @@ export default defineConfig((ctx) => {
   };
 });
 ```
+
+## `storage()` API
+
+### `R2Input`
+
+The `R2Input` type is a union of the following types:
+
+```ts
+type R2Input = string | ReadableStream | ArrayBuffer | ArrayBufferView | Blob;
+```
+
+### `storage().put(key: string, input: R2Input, options?: R2PutOptions)`
+
+Put a file in your R2 bucket.
+
+### `storage().putRandom(input: R2Input, options?: R2PutOptions & { prefix?: string, extension?: string })`
+
+Put a file in your R2 bucket with a random name.
+
+### `storage().get(key: string)`
+
+Get a file from your R2 bucket.
+
+### `storage().delete(key: string)`
+
+Delete a file from your R2 bucket.
+
+### `storage().url(key: string)`
+
+Get a URL to a file in your R2 bucket. Requires that the disk has a `publicPath` property defined.

packages/superflare/index.ts

@@ -15,3 +15,4 @@ export { Listener } from "./src/listener";
 export { handleWebSockets } from "./src/websockets";
 export { Channel } from "./src/durable-objects/Channel";
 export { Schema } from "./src/schema";
+export { parseMultipartFormData } from "./src/form-data";

packages/superflare/index.types.ts

@@ -11,7 +11,7 @@ export {
 } from "./src/config";
 export { DatabaseException } from "./src/query-builder";
 export { seed } from "./src/seeder";
-export { storage, servePublicPathFromStorage } from "./src/storage";
+export { storage, servePublicPathFromStorage, R2Input } from "./src/storage";
 export { Factory } from "./src/factory";
 export { handleFetch } from "./src/fetch";
 export { handleQueue } from "./src/queue";
@@ -24,6 +24,7 @@ export { Listener } from "./src/listener";
 export { handleWebSockets } from "./src/websockets";
 export { Channel } from "./src/durable-objects/Channel";
 export { Schema } from "./src/schema";
+export { parseMultipartFormData } from "./src/form-data";
 
 /**
  * Shape of the model instance.

packages/superflare/package.json

@@ -38,6 +38,7 @@
   "license": "MIT",
   "dependencies": {
     "@clack/prompts": "^0.6.3",
+    "@web3-storage/multipart-parser": "^1.0.0",
     "bcryptjs": "^2.4.3",
     "chalk": "^4.1.2",
     "cli-table3": "^0.6.3",

packages/superflare/src/form-data.ts

@@ -0,0 +1,73 @@
+import { streamMultipart } from "@web3-storage/multipart-parser";
+
+/**
+ * Copied from Remix:
+ * @see https://github.com/remix-run/remix/blob/72c22b3deb9e84e97359b481f7f2af6cdc355877/packages/remix-server-runtime/formData.ts
+ * Copyright 2021 Remix Software Inc.
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+export type UploadHandlerPart = {
+  name: string;
+  filename?: string;
+  contentType: string;
+  data: AsyncIterable<Uint8Array>;
+  stream: ReadableStream<Uint8Array>;
+};
+
+export type UploadHandler = (
+  part: UploadHandlerPart
+) => Promise<File | string | null | undefined>;
+
+/**
+ * Allows you to handle multipart forms (file uploads) for your app.
+ */
+export async function parseMultipartFormData(
+  request: Request,
+  uploadHandler: UploadHandler
+): Promise<FormData> {
+  let contentType = request.headers.get("Content-Type") || "";
+  const [type, boundary] = contentType.split(/\s*;\s*boundary=/);
+
+  if (!request.body || !boundary || type !== "multipart/form-data") {
+    throw new TypeError("Could not parse content as FormData.");
+  }
+
+  const formData = new FormData();
+  const parts: AsyncIterable<UploadHandlerPart & { done?: true }> =
+    streamMultipart(request.body, boundary);
+
+  for await (let part of parts) {
+    if (part.done) break;
+
+    if (typeof part.filename === "string") {
+      // only pass basename as the multipart/form-data spec recommends
+      // https://datatracker.ietf.org/doc/html/rfc7578#section-4.2
+      part.filename = part.filename.split(/[/\\]/).pop();
+    }
+
+    /**
+     * Build a convenience ReadableStream for the part data.
+     */
+    const stream = new ReadableStream({
+      async pull(controller) {
+        for await (let chunk of part.data) {
+          controller.enqueue(chunk);
+        }
+        controller.close();
+      },
+    });
+
+    let value = await uploadHandler({
+      ...part,
+      stream,
+    });
+    if (typeof value !== "undefined" && value !== null) {
+      formData.append(part.name, value as any);
+    }
+  }
+
+  return formData;
+}

packages/superflare/src/storage.ts

@@ -1,5 +1,7 @@
 import { Config, StorageDiskConfig } from "./config";
 
+export type R2Input = Parameters<R2Bucket["put"]>[1];
+
 export function storage(disk?: string) {
   if (!Config.storage?.disks) {
     throw new Error(
@@ -39,19 +41,25 @@ class Storage {
     return this.disk.binding.delete(key);
   }
 
-  put(file: File) {
-    const extension = file.name.split(".").pop();
+  put(...args: Parameters<R2Bucket["put"]>) {
+    return this.disk.binding.put(...args);
+  }
+
+  /**
+   * Takes an input without a key and generates a random filename for you.
+   * Optionally pass an `{ extension: string }` option to add an extension to the filename.
+   */
+  putRandom(
+    input: R2Input,
+    options?: R2PutOptions & { extension?: string; prefix?: string }
+  ) {
     const hash = crypto.randomUUID();
-    let key = hash;
-    if (extension) {
-      key += `.${extension}`;
+    let key = options?.prefix ? `${options.prefix}/${hash}` : hash;
+    if (options?.extension) {
+      key += `.${options.extension}`;
     }
 
-    return this.disk.binding.put(key, file);
-  }
-
-  putAs(...args: Parameters<R2Bucket["put"]>) {
-    return this.disk.binding.put(...args);
+    return this.disk.binding.put(key, input, options);
   }
 }