Init commit

Signed-off-by: lloydmeta <[email protected]>

.github/workflows/ci.yaml

@@ -0,0 +1,77 @@
+on: [ push, pull_request ]
+
+name: Continuous integration
+
+jobs:
+  check-all:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --all
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  terraform_format:
+    name: Terraform fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v3
+      - run: terraform fmt -recursive -check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all -- -D warnings

.gitignore

@@ -0,0 +1,13 @@
+/target
+
+# TF related
+terraform.tfstate
+errored.tfstate
+terraform.tfstate.backup
+.terraform
+.terraform.lock.hcl
+terraform/prod/main.tf
+terraform/prod/terraform.tfvars
+
+# Localstack
+terraform/localdev/volume

Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+resolver = "2"
+
+members = ["shared", "server"]
+
+exclude = ["client"]

Makefile

@@ -0,0 +1,37 @@
+start_dev_env:
+	cd terraform/localdev && docker-compose up &
+
+provision_dev_env:
+	cd terraform/localdev && tflocal apply -auto-approve
+
+clean_dev_env:
+	rm -rf terraform/localdev/volume
+
+stop_dev_env:
+	cd terraform/localdev && docker-compose down
+
+begin_dev:
+	source dev.env && cd server && RUST_BACKTRACE=full cargo lambda watch
+
+prod_workspace:
+	@cd terraform/prod && terraform workspace select miniaturs
+
+provision_prod: prod_workspace
+	@cd terraform/prod && terraform apply -auto-approve
+
+plan_prod: prod_workspace
+	@cd terraform/prod && terraform plan
+
+signature_for_dev:
+	@echo "http://localhost:9000/$$(source dev.env && cd client && cargo run -- $(TO_SIGN))"
+
+signature_for_localstack:
+	@echo "$$(cd terraform/localdev && tflocal output --raw lambda_function_url)$$(export MINIATURS_SHARED_SECRET=$$(cd terraform/localdev && tflocal output --raw miniaturs_shared_secret) && cd client && cargo run -- $(TO_SIGN))"
+
+signature_for_prod:
+	@echo "$$(cd terraform/prod && terraform output --raw miniaturs_deployed_url)/$$(export MINIATURS_SHARED_SECRET=$$(cd terraform/prod && terraform output --raw miniaturs_shared_secret) && cd client && cargo run -- $(TO_SIGN))"
+
+format:
+	@cd terraform && terraform fmt -recursive
+	@cargo fmt --all
+	@cd client && cargo fmt

README.md

@@ -0,0 +1,124 @@
+# Miniaturs 
+[![Continuous integration](https://github.com/lloydmeta/miniaturs/actions/workflows/ci.yaml/badge.svg)](https://github.com/lloydmeta/miniaturs/actions/workflows/ci.yaml)
+
+HTTP image resizer
+
+## Goals
+
+* Secure 
+* Fast: 
+  * Startup should be low 2 digit ms (e.g. avoid "oh, it's a lambda")
+  * Processing should be quick
+* Cheap: 
+  * Pay as little as possible, and do as little as possible
+  * Being fast can help avoid paying
+* Scalable: can handle lots of requests
+* Thumbor-ish 
+* A good net-citizen (don't make requests to 3rd parties if we have it in cache)
+* Debuggable
+
+To fulfil the above:
+
+* Runs in a lambda
+* Rust ⚡️
+* Caching in layers: CDN with S3 for images
+* Serverless, but built on HTTP-framework 
+
+A example TF in `terraform/prod` is provided to show how to deploy something that sits
+at a subdomain, using Cloudflare as our (free!) CDN + WAF.
+
+## Flow
+
+1. Layer 1 validations (is the request well formed)
+2. Ensure trusted request (e.g. check hash)
+3. Layer 2 validations (no I/O)
+  1. Are the image processing options supported?
+    1. Resize-to target size check (e.g. is it too big?) PENDING
+  2. Is the remote image path pointing to a supported source? PENDING
+  3. Is the remote image extension supported?
+4. Determine if we can return a cached result
+  1. Is there a cached result in the storage bucket?
+    1. If yes, return it as the result
+    2. Else continue
+5. Image retrieval:
+  1. Is the remote image already cached in our source bucket?
+    1. If yes, retrieve it
+    2. If not, issue a HEAD request to get image size PENDING
+      1. If the image size does not exceed configured max, retrieve it 
+      2. Else return an error
+    3. Is the actual downloadeded image too big? PENDING
+      1. If yes, return an error
+6. Image processing:
+  1. Is the image in a supported format for our processor?
+    1. If yes, process
+    2. Else return an error
+7. Cache resulting image in our bucket
+8. Return the resulting image
+
+## Development
+
+### Rust
+
+Assuming we have the [Rust toolbelt installed](https://doc.rust-lang.org/cargo/getting-started/installation.html#install-rust-and-cargo), the main thing we need is `cargo-lambda`
+
+```sh
+❯ brew tap cargo-lambda/cargo-lambda
+```
+
+### AWS
+
+* `brew install awscli` to install the CLI
+* Log into your app
+
+Ensure 
+
+* `aws configure sso` is done
+* `.aws/config` has the right profile config, with a `[profile ${PROFILE_NAME}]` line, where `PROFILE_NAME` matches what is in `main.tf`
+
+
+#### Login for Terraform
+
+`aws sso login --profile ${PROFILE_NAME}`
+
+### Cloudflare
+
+* Ensure `CLOUDFLARE_API_TOKEN` is defined in the env (needed for Cloudflare provider and cache busting). It'll need the privileges for updating DNS and cache settings
+
+## Deploying
+
+### Terraform
+
+* Use tfenv: https://formulae.brew.sh/formula/tfenv
+* Check what version is needed an install using ^
+
+* For local dev, `localstack` is used (see terraform/localdev/docker-compose.yaml), and `tflocal` is used (https://formulae.brew.sh/formula/terraform-local)
+  * `docker-compose` through official docker _or_ Rancher is supported, but [enabling admin access](https://github.com/rancher-sandbox/rancher-desktop/issues/2534#issuecomment-1909912585) is needed for running tests with Rancher
+
+### Per env
+
+Use `Makefile`  targets
+
+* For local def:
+    * `make start_dev_env provision_dev_env`
+    * `make begin_dev`
+    * `TO_SIGN="200x-100/https://beachape.com/images/octopress_with_container.png" make signature_for_localstack` to get a signed path for devenv
+    * `TO_SIGN="200x-100/https://beachape.com/images/octopress_with_container.png" make signature_for_dev` to get a signed path for dev
+* For prod:
+    * Copy + customise:
+      * `main.tf.example` to `main.tf` 
+      * `terraform.tfvars.example` to `terraform.tfvars`
+    * `make plan_prod` to see changes
+    * `make provision_prod` to apply changes
+    * `TO_SIGN="200x-100/https://beachape.com/images/octopress_with_container.png" make signature_for_prod` to get a signed path
+
+## To explore
+
+* img resizing 
+  * https://imgproxy.net/blog/almost-free-image-processing-with-imgproxy-and-aws-lambda/
+  * https://zenn.dev/devneko/articles/0a6fb5c9ea5689
+  * https://crates.io/crates/image
+* a [metadata endpoint](https://thumbor.readthedocs.io/en/stable/usage.html#metadata-endpoint)
+* [logs, tracing](https://github.com/tokio-rs/tracing?tab=readme-ov-file#in-applications)
+* improve image resizing
+  * Encapsulate + test
+  * Do in another thread?

client/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "miniaturs"
+version = "0.1.0"
+edition = "2021"
+description = "Client for miniaturs server"
+license = "MIT"
+documentation = "https://docs.rs/miniaturs"
+keywords = ["image", "manipulation", "client"]
+
+# Starting in Rust 1.62 you can use `cargo add` to add dependencies 
+# to your project.
+#
+# If you're using an older Rust version,
+# download cargo-edit(https://github.com/killercup/cargo-edit#installation) 
+# to install the `add` subcommand.
+#
+# Running `cargo add DEPENDENCY_NAME` will
+# add the latest version of a dependency to the list,
+# and it will keep the alphabetic ordering for you.
+
+[dependencies]
+
+anyhow = "1.0"
+miniaturs_shared = { path = "../shared", version = "0.1.0" }

client/src/main.rs

@@ -0,0 +1,18 @@
+use std::{env, error::Error};
+
+use anyhow::Context;
+use miniaturs_shared::signature::*;
+
+const SHARED_SECRET_ENV_KEY: &'static str = "MINIATURS_SHARED_SECRET";
+fn main() -> Result<(), Box<dyn Error>> {
+    let shared_secret = env::var(SHARED_SECRET_ENV_KEY)
+        .context("Expected {SHARED_SECRET_ENV_KEY} to be defined")?;
+
+    let args: Vec<_> = env::args().collect();
+
+    let to_sign = &args.get(1).context("Expected an argument to sign")?;
+    let signed = make_url_safe_base64_hash(&shared_secret, to_sign).context("Failed to sign")?;
+    println!("{signed}/{to_sign}");
+
+    Ok(())
+}

dev.env

@@ -0,0 +1,8 @@
+export MINIATURS_SHARED_SECRET="doyouwanttoknowasecretdoyoupromisenottotellwhoaohoh"
+export AWS_DEFAULT_REGION="us-east-1"
+export AWS_SECRET_ACCESS_KEY="mocksecretaccesskey"
+export AWS_ACCESS_KEY_ID="mockaccesskeyid"
+export AWS_ENDPOINT_URL="http://localhost:4566"
+export REQUIRE_PATH_STYLE_S3=true
+export PROCESSED_IMAGES_BUCKET=$(cd terraform/localdev && tflocal output --raw unprocessed_image_bucket_name)
+export UNPROCESSED_IMAGES_BUCKET=$(cd terraform/localdev && tflocal output --raw processed_image_bucket_name)

server/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "miniaturs_server"
+version = "0.1.0"
+edition = "2021"
+
+# Starting in Rust 1.62 you can use `cargo add` to add dependencies 
+# to your project.
+#
+# If you're using an older Rust version,
+# download cargo-edit(https://github.com/killercup/cargo-edit#installation) 
+# to install the `add` subcommand.
+#
+# Running `cargo add DEPENDENCY_NAME` will
+# add the latest version of a dependency to the list,
+# and it will keep the alphabetic ordering for you.
+
+[dependencies]
+anyhow = "1.0"
+lambda_http = "0.13.0"
+reqwest = { version = "0.12", default-features = false, features = [
+    "json",
+    "rustls-tls",
+] }
+axum = "0.7"
+serde = "1.0"
+serde_json = "1.0"
+tokio = { version = "1", features = ["macros"] }
+image = { version = "0.25", features = ["rayon"] }
+
+miniaturs_shared = { path = "../shared" }
+aws-sdk-s3 = "1.57"
+aws-config = "1.5"
+sha256 = "1.5"
+http-body-util = "0.1"
+bytes = "1.7"
+tower-http = { version = "0.6.1", features = ["catch-panic"] }
+
+[dev-dependencies]
+testcontainers = { version = "0.23" }
+testcontainers-modules = { version = "0.11", features = ["localstack"] }

server/src/api/mod.rs

@@ -0,0 +1,3 @@
+pub mod requests;
+pub mod responses;
+pub mod routing;