use encryption for file upload

.env.default

@@ -13,8 +13,13 @@ DOCKER_COMPOSE_APP_MW_DEFAULT_LOCALE=en
 DOCKER_COMPOSE_APP_MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS=30
 DOCKER_COMPOSE_APP_MW_FEATURE_BRAINSTORMING_TEASER=true
 DOCKER_COMPOSE_APP_MW_FEATURE_STORAGE_PROVIDER=local
+DOCKER_COMPOSE_APP_MW_FEATURE_IDEA_FILE_UPLOAD=true
 DOCKER_COMPOSE_APP_OBJECT_STORAGE_USER=
 DOCKER_COMPOSE_APP_OBJECT_STORAGE_PASSWORD=
+# please generate a secure key before, e.g. by using the elixir console inside the container:
+# iex
+# iex> 32 |> :crypto.strong_rand_bytes() |> Base.encode64()
+DOCKER_COMPOSE_APP_VAULT_ENCRYPTION_KEY_BASE64=
 # This is an example secret key base that can be use in development
 # NOTE: There are multiple commands you can use to generate a secret key base. Pick one command you like, e.g. `date +%s | sha256sum | base64 | head -c 64 ; echo`
 # !!ATTENTION: DO NOT USE THIS FOR PRODUCTION!!

.env.prod.default

@@ -13,9 +13,14 @@ DOCKER_COMPOSE_APP_PROD_DATABASE_USER_PASSWORD=
 DOCKER_COMPOSE_APP_PROD_MW_DEFAULT_LOCALE=en
 DOCKER_COMPOSE_APP_PROD_MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS=30
 DOCKER_COMPOSE_APP_PROD_MW_FEATURE_BRAINSTORMING_TEASER=true
+DOCKER_COMPOSE_APP_PROD_MW_FEATURE_IDEA_FILE_UPLOAD=true
 DOCKER_COMPOSE_APP_PROD_MW_FEATURE_STORAGE_PROVIDER=local
 DOCKER_COMPOSE_APP_PROD_OBJECT_STORAGE_USER=
 DOCKER_COMPOSE_APP_PROD_OBJECT_STORAGE_PASSWORD=
+# please generate a secure key before, e.g. by using the elixir console inside the container:
+# iex
+# iex> 32 |> :crypto.strong_rand_bytes() |> Base.encode64()
+DOCKER_COMPOSE_APP_PROD_VAULT_ENCRYPTION_KEY_BASE64=
 # Please generate a new secret key base for production
 # NOTE: There are multiple commands you can use to generate a secret key base. Pick one command you like, e.g.:
 # - `date +%s | sha256sum | base64 | head -c 64 ; echo`
@@ -30,4 +35,7 @@ DOCKER_COMPOSE_APP_PROD_PORT_TARGET=4000
 DOCKER_COMPOSE_POSTGRES_PROD_DB=
 DOCKER_COMPOSE_POSTGRES_PROD_PASSWORD=
 DOCKER_COMPOSE_POSTGRES_PROD_PORT=
-DOCKER_COMPOSE_POSTGRES_PROD_USER=
+DOCKER_COMPOSE_POSTGRES_PROD_USER=
+
+DOCKER_COMPOSE_MINIO_PROD_USER=
+DOCKER_COMPOSE_MINIO_PROD_PASSWORD=

.github/workflows/on_push_branch__execute_ci_cd.yml

@@ -70,6 +70,10 @@ jobs:
       TEST_DATABASE_USER: postgres
       TEST_DATABASE_USER_PASSWORD: postgres
       MIX_ENV: "test"
+      OBJECT_STORAGE_BUCKET: "mindwendel"
+      # The key for encrypting file contents used only in tests
+      # Do not use this key in any kind of prod related environments!
+      VAULT_ENCRYPTION_KEY_BASE64: "gI6L07o3RTppqy+cfAxO4C8G8psYHWn2NYPbUymYI1o="
 
     steps:
       # Downloads a copy of the code in your repository before running CI tests

README.md

@@ -28,7 +28,7 @@ Create a challenge. Ready? Brainstorm. mindwendel helps you to easily brainstorm
 - Easily create and upvote ideas, with live updates from your companions
 - Cluster or filter your ideas with custom labels
 - Preview of links to ease URL sharing
-- Add file attachments
+- Add automatically encrypted file attachments which are uploaded to an S3 compatible storage backend
 - Add lanes, use drag & drop to order ideas
 - Export your generated ideas to html or csv (currently comma separated)
 - German & English Translation files

config/dev.exs

@@ -32,6 +32,8 @@ config :mindwendel, MindwendelWeb.Endpoint,
     esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]}
   ]
 
+config :mindwendel, :s3_storage_provider, Mindwendel.Services.S3ObjectStorageService
+
 # ## SSL Support
 #
 # In order to use HTTPS in development, a self-signed

config/prod.exs

@@ -21,6 +21,8 @@ unless secret_key_base do
   )
 end
 
+config :mindwendel, :s3_storage_provider, Mindwendel.Services.S3ObjectStorageService
+
 config :mindwendel, MindwendelWeb.Endpoint,
   # This configuration ensures / enforces ssl requests sent to this mindwendel instance.
   # See https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-compile-time-configuration

config/runtime.exs

@@ -163,6 +163,11 @@ config :mindwendel, :options,
       ["", "true"],
       String.trim(System.get_env("MW_FEATURE_BRAINSTORMING_TEASER") || "")
     ),
+  feature_file_upload:
+    Enum.member?(
+      ["", "true"],
+      String.trim(System.get_env("MW_FEATURE_IDEA_FILE_UPLOAD") || "")
+    ),
   feature_brainstorming_removal_after_days: delete_brainstormings_after_days,
   # use a strict csp everywhere except in development. we need to relax the setting a bit for webpack
   csp_relax: config_env() == :dev
@@ -180,33 +185,26 @@ if config_env() == :prod || config_env() == :dev do
     queues: [default: 1]
 end
 
-# Waffle config for uploading files
-# By default, the local file system is used. For production environments, an S3 storage provider is recommended.
-storage_provider = String.trim(System.get_env("MW_FEATURE_STORAGE_PROVIDER") || "local")
+config :mindwendel, max_upload_length: System.get_env("MW_FILE_UPLOAD_MAX_FILE_SIZE", "2666666")
 
-if storage_provider == "local" do
-  config :waffle,
-    storage: Waffle.Storage.Local,
-    storage_dir_prefix: "priv/static",
-    storage_dir: "uploads"
-end
-
-# Examples for waffle storage configurations for S3, see https://hexdocs.pm/waffle/Waffle.Storage.S3.html
-if storage_provider == "s3" do
-  config :waffle,
-    storage: Waffle.Storage.S3,
-    bucket: System.fetch_env!("OBJECT_STORAGE_BUCKET"),
-    asset_host: System.fetch_env!("OBJECT_STORAGE_ASSET_HOST")
+# configure cloak:
+config :mindwendel, Mindwendel.Services.Vault,
+  ciphers: [
+    default:
+      {Cloak.Ciphers.AES.GCM,
+       tag: "AES.GCM.V1",
+       key: Base.decode64!(System.fetch_env!("VAULT_ENCRYPTION_KEY_BASE64")),
+       iv_length: 12}
+  ]
 
-  config(:ex_aws,
-    json_codec: Jason,
+# check all object storage system envs at once:
+if config_env() == :prod || config_env() == :dev do
+  config(:ex_aws, :s3,
+    scheme: System.fetch_env!("OBJECT_STORAGE_SCHEME"),
+    host: System.fetch_env!("OBJECT_STORAGE_HOST"),
+    port: System.fetch_env!("OBJECT_STORAGE_PORT"),
+    region: System.fetch_env!("OBJECT_STORAGE_REGION"),
     access_key_id: System.fetch_env!("OBJECT_STORAGE_USER"),
-    secret_access_key: System.fetch_env!("OBJECT_STORAGE_PASSWORD"),
-    s3: [
-      scheme: System.fetch_env!("OBJECT_STORAGE_SCHEME"),
-      host: System.fetch_env!("OBJECT_STORAGE_HOST"),
-      port: System.fetch_env!("OBJECT_STORAGE_PORT"),
-      region: System.fetch_env!("OBJECT_STORAGE_REGION")
-    ]
+    secret_access_key: System.fetch_env!("OBJECT_STORAGE_PASSWORD")
   )
 end

config/test.exs

@@ -13,6 +13,8 @@ config :mindwendel, Mindwendel.Repo,
   show_sensitive_data_on_connection_error: true,
   pool: Ecto.Adapters.SQL.Sandbox
 
+config :mindwendel, :s3_storage_provider, Mindwendel.Services.S3ObjectStorageLocalSandboxService
+
 # We don't run a server during test. If one is required,
 # you can enable the server option below.
 config :mindwendel, MindwendelWeb.Endpoint,

docker-compose-prod.yml

@@ -22,7 +22,17 @@ services:
       MW_DEFAULT_LOCALE: ${DOCKER_COMPOSE_APP_PROD_MW_DEFAULT_LOCALE:-en}
       MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS: ${DOCKER_COMPOSE_APP_PROD_MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS:-30}
       MW_FEATURE_BRAINSTORMING_TEASER: ${DOCKER_COMPOSE_APP_PROD_MW_FEATURE_BRAINSTORMING_TEASER:-true}
-      MW_FEATURE_STORAGE_PROVIDER: ${DOCKER_COMPOSE_APP_PROD_MW_FEATURE_STORAGE_PROVIDER:-local}
+      MW_FEATURE_IDEA_FILE_UPLOAD: ${DOCKER_COMPOSE_APP_PROD_MW_FEATURE_IDEA_FILE_UPLOAD:-true}
+
+      # Variables for s3 file storage
+      OBJECT_STORAGE_BUCKET: mindwendel
+      OBJECT_STORAGE_SCHEME: "https://"
+      OBJECT_STORAGE_HOST: minio
+      OBJECT_STORAGE_PORT: 9000
+      OBJECT_STORAGE_REGION: local
+      OBJECT_STORAGE_USER: ${DOCKER_COMPOSE_APP_PROD_OBJECT_STORAGE_USER}
+      OBJECT_STORAGE_PASSWORD: $DOCKER_COMPOSE_APP_PROD_OBJECT_STORAGE_PASSWORD}
+      VAULT_ENCRYPTION_KEY_BASE64: ${DOCKER_COMPOSE_APP_PROD_VAULT_ENCRYPTION_KEY_BASE64}
 
       # Add a secret key base for mindwendel for encrypting the use session
       # NOTE: There are multiple commands you can use to generate a secret key base. Pick one command you like.
@@ -44,6 +54,19 @@ services:
     depends_on:
       - postgres_prod
 
+  minio:
+    image: minio/minio
+    container_name: minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: ${DOCKER_COMPOSE_MINIO_PROD_USER}
+      MINIO_ROOT_PASSWORD: ${DOCKER_COMPOSE_MINIO_PROD_PASSWORD}
+    volumes:
+      - ~/minio/data:/data
+    command: server /data --console-address ":9001"
+
   # If you do not have another postgres database service in this docker-compose, you can add this postgres service.
   # Note: Please use other credentials when using this in production.
   postgres_prod:

docker-compose.yml

@@ -14,6 +14,7 @@ services:
       MW_DEFAULT_LOCALE: ${DOCKER_COMPOSE_APP_MW_DEFAULT_LOCALE:-en}
       MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS: ${DOCKER_COMPOSE_APP_MW_FEATURE_BRAINSTORMING_REMOVAL_AFTER_DAYS:-30}
       MW_FEATURE_BRAINSTORMING_TEASER: ${DOCKER_COMPOSE_APP_MW_FEATURE_BRAINSTORMING_TEASER:-true}
+      MW_FEATURE_IDEA_FILE_UPLOAD: ${DOCKER_COMPOSE_APP_MW_FEATURE_IDEA_FILE_UPLOAD:-true}
       # This is an example secret key base that can be use in development
       # NOTE: There are multiple commands you can use to generate a secret key base. Pick one command you like, e.g.:
       # - `date +%s | sha256sum | base64 | head -c 64 ; echo`
@@ -28,19 +29,17 @@ services:
       URL_PORT: ${DOCKER_COMPOSE_APP_URL_PORT:-4000}
       URL_SCHEME: ${DOCKER_COMPOSE_APP_URL_SCHEME:-http}
       MW_ENDPOINT_HTTP_PORT: ${DOCKER_COMPOSE_APP_MW_ENDPOINT_HTTP_PORT:-4000}
-      MW_FEATURE_STORAGE_PROVIDER: ${DOCKER_COMPOSE_APP_MW_FEATURE_STORAGE_PROVIDER:-local}
-      # In case s3 is used as storage provider
       # Attention: Please note that you have to add minio in etc/hosts to make the links of signed urls work:
       # Example: 127.0.0.1 minio
-      #
-      # OBJECT_STORAGE_ASSET_HOST: "http://localhost:9000/mindwendel"
-      # OBJECT_STORAGE_BUCKET: mindwendel
-      # OBJECT_STORAGE_SCHEME: "http://"
-      # OBJECT_STORAGE_HOST: minio
-      # OBJECT_STORAGE_PORT: 9000
-      # OBJECT_STORAGE_REGION: local
-      # OBJECT_STORAGE_USER: ${DOCKER_COMPOSE_APP_OBJECT_STORAGE_USER}
-      # OBJECT_STORAGE_PASSWORD: ${DOCKER_COMPOSE_APP_OBJECT_STORAGE_PASSWORD}
+
+      OBJECT_STORAGE_BUCKET: mindwendel
+      OBJECT_STORAGE_SCHEME: "http://"
+      OBJECT_STORAGE_HOST: minio
+      OBJECT_STORAGE_PORT: 9000
+      OBJECT_STORAGE_REGION: local
+      OBJECT_STORAGE_USER: ${DOCKER_COMPOSE_APP_OBJECT_STORAGE_USER}
+      OBJECT_STORAGE_PASSWORD: ${DOCKER_COMPOSE_APP_OBJECT_STORAGE_PASSWORD}
+      VAULT_ENCRYPTION_KEY_BASE64: ${DOCKER_COMPOSE_APP_VAULT_ENCRYPTION_KEY_BASE64}
     ports:
       - "${DOCKER_COMPOSE_APP_PORT_PUBLISHED:-4000}:${DOCKER_COMPOSE_APP_PORT_TARGET:-4000}"
     depends_on:
@@ -52,18 +51,18 @@ services:
       - /app/deps/
       - /app/assets/node_modules
 
-  # minio:
-  #   image: minio/minio
-  #   container_name: minio
-  #   ports:
-  #     - "9000:9000"
-  #     - "9001:9001"
-  #   environment:
-  #     MINIO_ROOT_USER: ROOTNAME
-  #     MINIO_ROOT_PASSWORD: CHANGEME123
-  #   volumes:
-  #     - ~/minio/data:/data
-  #   command: server /data --console-address ":9001"
+  minio:
+    image: minio/minio
+    container_name: minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: ${DOCKER_COMPOSE_MINIO_USER:-ROOTNAME}
+      MINIO_ROOT_PASSWORD: ${DOCKER_COMPOSE_MINIO_PASSWORD:-CHANGEME123}
+    volumes:
+      - ~/minio/data:/data
+    command: server /data --console-address ":9001"
 
   postgres:
     image: postgres:15

docs/installing_mindwendel.md

@@ -7,6 +7,17 @@ Below, we provide detailed instructions on how to install and run mindwendel in
 - [Running on Docker-Compose](#running-on-docker-compose) (RECOMMENDED)
 - [Running on Docker](#running-on-docker)
 
+## Setup of .env secrets and variables
+
+Please copy the .env.default file to .env and set the secrets!
+
+https://hexdocs.pm/cloak/install.html
+
+```
+iex
+iex> 32 |> :crypto.strong_rand_bytes() |> Base.encode64()
+```
+
 ## Running on Docker-Compose
 
 When you use [docker compose](https://docs.docker.com/compose/), you will be using one or several `docker-compose.yml` files.

lib/mindwendel/application.ex

@@ -33,6 +33,7 @@ defmodule Mindwendel.Application do
       {Phoenix.PubSub, name: Mindwendel.PubSub},
       # Start the Endpoint (http/https)
       MindwendelWeb.Endpoint,
+      Mindwendel.Services.Vault,
       # Start a worker by calling: Mindwendel.Worker.start_link(arg)
       # {Mindwendel.Worker, arg}
       {Oban, oban_config()}

lib/mindwendel/attachments.ex

@@ -2,28 +2,21 @@ defmodule Mindwendel.Attachments do
   import Ecto.Query, warn: false
   alias Mindwendel.Repo
   alias Mindwendel.Attachments.File
+  alias Mindwendel.Services.StorageService
 
   require Logger
 
   @doc """
   Gets a single attached_file
 
-  Raises `Ecto.NoResultsError` if the Brainstorming does not exist.
-
   ## Examples
 
-      iex> get_attached_file!("0323906b-b496-4778-ae67-1dd779d3de3c")
-      %Brainstorming{ ... }
-
-      iex> get_attached_file!("0323906b-b496-4778-ae67-1dd779d3de3c")
-      ** (Ecto.NoResultsError)
-
-      iex> get_attached_file!("not_a_valid_uuid_string")
-      ** (Ecto.Query.CastError)
+      iex> get_attached_file("0323906b-b496-4778-ae67-1dd779d3de3c")
+      %File{ ... }
 
   """
-  def get_attached_file!(id) do
-    Repo.get!(File, id)
+  def get_attached_file(id) do
+    Repo.get(File, id)
   end
 
   @doc """
@@ -53,9 +46,10 @@ defmodule Mindwendel.Attachments do
   """
   def delete_attached_file(%File{} = attached_file) do
     if attached_file.path do
-      :ok = Mindwendel.Attachment.delete(attached_file.path)
+      case StorageService.delete_file(attached_file.path) do
+        {:ok} -> Repo.delete(attached_file)
+        {:error, message} -> {:error, message}
+      end
     end
-
-    Repo.delete(attached_file)
   end
 end

lib/mindwendel/attachments/file.ex

@@ -2,6 +2,7 @@ defmodule Mindwendel.Attachments.File do
   use Mindwendel.Schema
   import Ecto.Changeset
   alias Mindwendel.Brainstormings.Idea
+  alias Mindwendel.Services.StorageService
 
   schema "idea_files" do
     field :name, :string
@@ -18,15 +19,25 @@ defmodule Mindwendel.Attachments.File do
   def changeset(attachment, attrs) do
     attachment
     |> cast(attrs, [:path, :name, :file_type])
-    |> maybe_store_from_path_tmp(attrs)
+    |> maybe_store_from_path_tmp()
   end
 
-  defp maybe_store_from_path_tmp(changeset, attrs) do
-    if attrs[:path] do
-      {:ok, final_path} = Mindwendel.Attachment.store(attrs[:path])
+  defp maybe_store_from_path_tmp(changeset) do
+    upload_feature_flag = Application.fetch_env!(:mindwendel, :options)[:feature_file_upload]
+
+    if upload_feature_flag and get_change(changeset, :path) do
+      object_filename = Path.basename(get_change(changeset, :path))
+
+      {:ok, encrypted_file_path} =
+        StorageService.store_file(
+          object_filename,
+          get_change(changeset, :path),
+          get_change(changeset, :file_type)
+        )
+
       # clear old tmp file
-      File.rm(attrs[:path])
-      changeset |> put_change(:path, final_path)
+      File.rm(get_change(changeset, :path))
+      changeset |> put_change(:path, encrypted_file_path)
     else
       changeset
     end

lib/mindwendel/brainstormings/idea.ex

@@ -15,7 +15,7 @@ defmodule Mindwendel.Brainstormings.Idea do
   alias Mindwendel.Accounts.User
 
   @label_values [:label_1, :label_2, :label_3, :label_4, :label_5]
-  @max_file_attachments 4
+  @max_file_attachments 2
 
   schema "ideas" do
     field :body, :string
@@ -77,7 +77,10 @@ defmodule Mindwendel.Brainstormings.Idea do
   end
 
   defp maybe_put_attachments(changeset, idea, attrs) do
-    if attrs["tmp_attachments"] != nil and Enum.empty?(changeset.errors) do
+    upload_feature_flag = Application.fetch_env!(:mindwendel, :options)[:feature_file_upload]
+
+    if upload_feature_flag and
+         attrs["tmp_attachments"] != nil and Enum.empty?(changeset.errors) do
       new_files =
         Enum.map(attrs["tmp_attachments"], fn change ->
           Attachments.change_attached_file(%File{}, change)