Add multipart/form-data support

Add a new structured body type !form_multipart, which allows sending forms with binary data. This is a bit incomplete, we need to support streaming bodies in #256. Closes #243
LucasPickering · Jun 9, 2024 · fc30c64 · fc30c64
1 parent 4aa419c
commit fc30c64
Show file tree

Hide file tree

Showing 14 changed files with 495 additions and 91 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 - Structured bodies can now be defined with tags on the `body` field of a recipe, making it more convenient to construct bodies of common types. Supported types are:
   - `!json` [#242](https://github.com/LucasPickering/slumber/issues/242)
   - `!form_urlencoded` [#244](https://github.com/LucasPickering/slumber/issues/244)
+  - `!form_multipart` [#243](https://github.com/LucasPickering/slumber/issues/243)
   - [See docs](https://slumber.lucaspickering.me/book/api/request_collection/recipe_body.html) for usage instructions
 - Support multiple instances of the same query param [#245](https://github.com/LucasPickering/slumber/issues/245) (@maksimowiczm)
   - Query params can now be defined as a list of `<param>=<value>` entries

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -33,7 +33,7 @@ nom = "7.1.3"
 notify = {version = "^6.1.1", default-features = false, features = ["macos_fsevent"]}
 open = "5.1.1"
 ratatui = {version = "^0.26.0", features = ["serde", "unstable-rendered-line-info"]}
-reqwest = {version = "^0.12.4", default-features = false, features = ["rustls-tls"]}
+reqwest = {version = "^0.12.4", default-features = false, features = ["multipart", "rustls-tls"]}
 rmp-serde = "^1.1.2"
 rusqlite = {version = "^0.31.0", default-features = false, features = ["bundled", "chrono", "uuid"]}
 rusqlite_migration = "^1.2.0"
@@ -52,6 +52,7 @@ uuid = {version = "^1.4.1", default-features = false, features = ["serde", "v4"]
 [dev-dependencies]
 mockito = {version = "1.4.0", default-features = false}
 pretty_assertions = "1.4.0"
+regex = {version = "1.10.5", default-features = false}
 rstest = {version = "0.19.0", default-features = false}
 serde_test = "1.0.176"
 

diff --git a/docs/src/api/request_collection/recipe_body.md b/docs/src/api/request_collection/recipe_body.md
@@ -8,10 +8,11 @@ In addition, you can pass any [`Template`](./template.md) to render any text or
 
 The following content types have first-class support. Slumber will automatically set the `Content-Type` header to the specified value, but you can override this simply by providing your own value for the header.
 
-| Variant            | Type                                         | `Content-Type`                      | Description                                                                                               |
-| ------------------ | -------------------------------------------- | ----------------------------------- | --------------------------------------------------------------------------------------------------------- |
-| `!json`            | Any                                          | `application/json`                  | Structured JSON body; all strings are treated as templates                                                |
-| `!form_urlencoded` | [`mapping[string, Template]`](./template.md) | `application/x-www-form-urlencoded` | URL-encoded form data; [see here for more](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST |
+| Variant            | Type                                         | `Content-Type`                      | Description                                                                                                |
+| ------------------ | -------------------------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------------------------------- |
+| `!json`            | Any                                          | `application/json`                  | Structured JSON body; all strings are treated as templates                                                 |
+| `!form_urlencoded` | [`mapping[string, Template]`](./template.md) | `application/x-www-form-urlencoded` | URL-encoded form data; [see here for more](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST) |
+| `!form_multipart`  | [`mapping[string, Template]`](./template.md) | `multipart/form-data`               | Binary form data; [see here for more](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST)      |
 
 ## Examples
 
@@ -41,4 +42,19 @@ requests:
     url: "{{host}}/fishes/{{fish_id}}"
     # Content-Type header will be set automatically based on the body type
     body: !json { "name": "Alfonso" }
+
+  urlencoded_body: !request
+    method: POST
+    url: "{{host}}/fishes/{{fish_id}}"
+    # Content-Type header will be set automatically based on the body type
+    body: !form_urlencoded
+      name: Alfonso
+
+  multipart_body: !request
+    method: POST
+    url: "{{host}}/fishes/{{fish_id}}"
+    # Content-Type header will be set automatically based on the body type
+    body: !form_multipart
+      name: Alfonso
+      image: "{{chains.fish_image}}"
 ```
diff --git a/slumber.yml b/slumber.yml
@@ -29,6 +29,9 @@ chains:
       recipe: login
       trigger: !expire 12h
     selector: $.data
+  image:
+    source: !file
+      path: ./static/slumber.png
 
 .ignore:
   base: &base
@@ -81,6 +84,14 @@ requests:
     method: GET
     url: "{{host}}/image"
 
+  upload_image: !request
+    name: Upload Image
+    method: POST
+    url: "{{host}}/anything/image"
+    body: !form_multipart
+      filename: "logo.png"
+      image: "{{chains.image}}"
+
   delay: !request
     <<: *base
     name: Delay

diff --git a/src/collection/cereal.rs b/src/collection/cereal.rs
@@ -227,8 +227,12 @@ impl RecipeBody {
     const STRUCT_NAME: &'static str = "RecipeBody";
     const VARIANT_JSON: &'static str = "json";
     const VARIANT_FORM_URLENCODED: &'static str = "form_urlencoded";
-    const ALL_VARIANTS: &'static [&'static str] =
-        &[Self::VARIANT_JSON, Self::VARIANT_FORM_URLENCODED];
+    const VARIANT_FORM_MULTIPART: &'static str = "form_multipart";
+    const ALL_VARIANTS: &'static [&'static str] = &[
+        Self::VARIANT_JSON,
+        Self::VARIANT_FORM_URLENCODED,
+        Self::VARIANT_FORM_MULTIPART,
+    ];
 }
 
 /// Custom serialization for RecipeBody, so the `Raw` variant serializes as a
@@ -255,6 +259,13 @@ impl Serialize for RecipeBody {
                     Self::VARIANT_FORM_URLENCODED,
                     value,
                 ),
+            RecipeBody::FormMultipart(value) => serializer
+                .serialize_newtype_variant(
+                    Self::STRUCT_NAME,
+                    3,
+                    Self::VARIANT_FORM_MULTIPART,
+                    value,
+                ),
         }
     }
 }
@@ -316,6 +327,9 @@ impl<'de> Deserialize<'de> for RecipeBody {
                     RecipeBody::VARIANT_FORM_URLENCODED => {
                         Ok(RecipeBody::FormUrlencoded(value.newtype_variant()?))
                     }
+                    RecipeBody::VARIANT_FORM_MULTIPART => {
+                        Ok(RecipeBody::FormMultipart(value.newtype_variant()?))
+                    }
                     other => Err(A::Error::unknown_variant(
                         other,
                         RecipeBody::ALL_VARIANTS,
@@ -519,7 +533,8 @@ mod tests {
             tag: Tag::new("raw"),
             value: "{{user_id}}".into()
         })),
-        "unknown variant `raw`, expected `json` or `form_urlencoded`",
+        "unknown variant `raw`, expected one of \
+        `json`, `form_urlencoded`, `form_multipart`",
     )]
     #[case::form_urlencoded_wrong_type(
         serde_yaml::Value::Tagged(Box::new(TaggedValue{

diff --git a/src/collection/insomnia.rs b/src/collection/insomnia.rs
@@ -3,9 +3,9 @@
 
 use crate::{
     collection::{
-        self, cereal::deserialize_from_str, Collection, Folder, JsonBody,
-        Method, Profile, ProfileId, Recipe, RecipeBody, RecipeId, RecipeNode,
-        RecipeTree,
+        self, cereal::deserialize_from_str, Chain, ChainId, ChainSource,
+        Collection, Folder, JsonBody, Method, Profile, ProfileId, Recipe,
+        RecipeBody, RecipeId, RecipeNode, RecipeTree,
     },
     template::Template,
 };
@@ -16,7 +16,7 @@ use mime::Mime;
 use reqwest::header;
 use serde::{Deserialize, Deserializer};
 use std::{collections::HashMap, fs::File, path::Path};
-use tracing::{info, warn};
+use tracing::{debug, info, warn};
 
 impl Collection {
     /// Convert an Insomnia exported collection into the slumber format. This
@@ -60,15 +60,14 @@ impl Collection {
 
         // Convert everything we care about
         let profiles = build_profiles(&workspace_id, environments);
+        let chains = build_chains(&requests);
         let recipes =
             build_recipe_tree(&workspace_id, request_groups, requests)?;
 
         Ok(Collection {
             profiles,
             recipes,
-            // Parse templates into chains:
-            // https://github.com/LucasPickering/slumber/issues/164
-            chains: IndexMap::new(),
+            chains,
             _ignore: serde::de::IgnoredAny,
         })
     }
@@ -81,9 +80,14 @@ struct Insomnia {
 
 /// Group the resources by type so they're easier to access
 struct Grouped {
+    /// Root workspace ID. Useful to check which resources are top-level and
+    /// which aren't.
     workspace_id: String,
+    /// Profiles
     environments: Vec<Environment>,
+    /// Folders
     request_groups: Vec<RequestGroup>,
+    /// Recipes
     requests: Vec<Request>,
 }
 
@@ -201,11 +205,28 @@ impl Body {
     }
 }
 
+/// One parameter in a form (urlencoded or multipart)
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct FormParam {
+    id: String,
     name: String,
     value: String,
+    /// Variant of param. This is omitted by Insomnia for simple text params,
+    /// so we need to fill with default
+    #[serde(default, rename = "type")]
+    kind: FormParamKind,
+    /// Path of linked file, for file params only
+    file_name: Option<String>,
+}
+
+/// The variant of a form parameter
+#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+enum FormParamKind {
+    #[default]
+    String,
+    File,
 }
 
 impl Grouped {
@@ -364,10 +385,11 @@ impl TryFrom<Body> for RecipeBody {
             RecipeBody::Json(json.map(Template::dangerous))
         } else if body.mime_type == mime::APPLICATION_WWW_FORM_URLENCODED {
             RecipeBody::FormUrlencoded(
-                body.params
-                    .into_iter()
-                    .map(|param| (param.name, Template::dangerous(param.value)))
-                    .collect(),
+                body.params.into_iter().map(FormParam::into).collect(),
+            )
+        } else if body.mime_type == mime::MULTIPART_FORM_DATA {
+            RecipeBody::FormMultipart(
+                body.params.into_iter().map(FormParam::into).collect(),
             )
         } else {
             RecipeBody::Raw(body.try_text()?)
@@ -376,6 +398,27 @@ impl TryFrom<Body> for RecipeBody {
     }
 }
 
+/// Convert an Insomnia form parameter into a corresponding map entry, to be
+/// used in a structured body
+impl From<FormParam> for (String, Template) {
+    fn from(param: FormParam) -> Self {
+        match param.kind {
+            // Simple string, map to a raw template
+            FormParamKind::String => {
+                (param.name, Template::dangerous(param.value))
+            }
+            // We'll map this to a chain that loads the file. The ID of the
+            // chain is the ID of this param. We're banking on that chain being
+            // created elsewhere. It's a bit spaghetti but otherwise we'd need
+            // mutable access to the entire collection, which I think would end
+            // up with even more spaghetti
+            FormParamKind::File => {
+                (param.name, Template::from_chain(&param.id.into()))
+            }
+        }
+    }
+}
+
 /// Convert authentication type. If the type is unknown, return is as `Err`
 impl TryFrom<Authentication> for collection::Authentication {
     type Error = String;
@@ -425,6 +468,7 @@ fn build_profiles(
     environments
         .into_iter()
         .map(|environment| {
+            debug!("Generating profile for environment `{}`", environment.id);
             let id: ProfileId = environment.id.into();
             // Start with base data so we can overwrite it
             let data = base_data
@@ -444,6 +488,52 @@ fn build_profiles(
         .collect()
 }
 
+/// Build up all the chains we need to represent the Insomnia collection.
+/// Chains don't map 1:1 with any Insomnia resource. They generally are an
+/// explicit representation of some implicit Insomnia behavior, so we have to
+/// crawl over the Insomnia collection to find where chains need to exist. For
+/// each generated chain, we'll need to pick a consistent ID so the consumer can
+/// link to the same chain.
+fn build_chains(requests: &[Request]) -> IndexMap<ChainId, Chain> {
+    let mut chains = IndexMap::new();
+
+    for request in requests {
+        debug!("Generating chains for request `{}`", request.id);
+
+        // Any multipart form param that references a file needs a chain
+        for param in request.body.iter().flat_map(|body| &body.params) {
+            debug!("Generating chains for form parameter `{}`", param.id);
+
+            if let FormParamKind::File = param.kind {
+                let id: ChainId = param.id.as_str().into();
+                let Some(path) = &param.file_name else {
+                    warn!(
+                        "Form param `{}` is of type `file` \
+                        but missing `file_name` field",
+                        param.id
+                    );
+                    continue;
+                };
+                chains.insert(
+                    id.clone(),
+                    Chain {
+                        id,
+                        source: ChainSource::File {
+                            path: Template::dangerous(path.to_owned()),
+                        },
+                        sensitive: false,
+                        selector: None,
+                        content_type: None,
+                        trim: Default::default(),
+                    },
+                );
+            }
+        }
+    }
+
+    chains
+}
+
 /// Expand the flat list of Insomnia resources into a recipe tree
 fn build_recipe_tree(
     workspace_id: &str,