From 54ab7b52cbde50367c8cf767f8d31e8771b2ae88 Mon Sep 17 00:00:00 2001 From: Robert Kowalski Date: Wed, 2 Feb 2022 13:46:34 +0100 Subject: [PATCH] Load database repo dynamically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When creating an elixir release, the database adapter is hardcoded into `sys.config`. With the hardcoded database, the docker image is strictly set to Postgres. This change allows to set the database url to a MySQL host by environment variable, so the current docker image can connect to MySQL and use the appropiate Ecto Adapter. Co-authored-by: Björn Brauer Co-authored-by: Theodor Fiedler Co-authored-by: Philipp Hinrichsen Co-authored-by: Markus Wolf --- .github/workflows/main.yml | 39 ++++++++- README.md | 9 +- config/config.exs | 2 +- config/dev.exs | 10 ++- config/prod.secret.exs | 15 +++- config/test.exs | 29 +++---- lib/application.ex | 22 ++++- lib/database/migrate.ex | 8 +- lib/database/repo.ex | 85 ++++++++++--------- lib/database/repo_callbacks.txt | 40 +++++++++ lib/database/repo_mysql.ex | 19 +++++ lib/database/repo_postgres.ex | 48 +++++++++++ lib/mix/bors.cleanup.ex | 2 +- ...2140_change_users_login_type_to_citext.exs | 14 ++- test/support/channel_case.ex | 4 +- test/support/conn_case.ex | 4 +- test/support/model_case.ex | 4 +- test/support/worker_test_case.ex | 4 +- test/test_helper.exs | 2 +- 19 files changed, 278 insertions(+), 82 deletions(-) create mode 100644 lib/database/repo_callbacks.txt create mode 100644 lib/database/repo_mysql.ex create mode 100644 lib/database/repo_postgres.ex diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6879a8da5..1d4ccf894 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,10 +29,45 @@ jobs: elixir-version: ${{ matrix.elixir }} - run: mix deps.get env: - BORS_TEST_DATABASE: ${{ matrix.database }} + BORS_DATABASE: ${{ matrix.database }} + + - name: Create Mysql Repo + if: matrix.database == 'mysql' + run: | + echo "running mysql database creation" + mix ecto.create -r BorsNG.Database.RepoMysql + env: + BORS_DATABASE: ${{ matrix.database }} + MIX_ENV: test + - name: Migrate Mysql Repo + if: matrix.database == 'mysql' + run: | + echo "running mysql migration" + mix ecto.migrate -r BorsNG.Database.RepoMysql + env: + BORS_DATABASE: ${{ matrix.database }} + MIX_ENV: test + + - name: Create Postgres Repo + if: matrix.database == 'postgresql' + run: | + echo "running postgres database creation" + mix ecto.create -r BorsNG.Database.RepoPostgres + env: + BORS_DATABASE: ${{ matrix.database }} + MIX_ENV: test + - name: Migrate Postgres Repo + if: matrix.database == 'postgresql' + run: | + echo "running postgres migration" + mix ecto.migrate -r BorsNG.Database.RepoPostgres + env: + BORS_DATABASE: ${{ matrix.database }} + MIX_ENV: test + - run: mix test env: - BORS_TEST_DATABASE: ${{ matrix.database }} + BORS_DATABASE: ${{ matrix.database }} services: postgresql: image: postgres:13 diff --git a/README.md b/README.md index ecb3f1f00..2070b52dc 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,10 @@ You can then run it using `mix`: $ mix ecto.migrate $ mix phx.server +If you want to use MySQL, add the `-r` flag: + + $ mix ecto.create -r BorsNG.Database.RepoMysql + And it'll run with the GitHub API mocked-out. To run tests, run: @@ -358,10 +362,11 @@ All the same recommendations apply, with some extra notes: - `ELIXIR_VERSION` can be set as a build-time argument. Its default value is defined in the [Dockerfile](Dockerfile). - `ALLOW_PRIVATE_REPOS` must be set at both build and run times to take effect. It is set to ` true` by default. -- `DATABASE_URL` *must* contain the database port, as it will be used at container startup to wait until the database is reachable. [The format is documented here](https://hexdocs.pm/ecto/Ecto.Repo.html#module-urls). +- `DATABASE_URL` _must_ contain the database port, as it will be used at container startup to wait until the database is reachable. [The format is documented here](https://hexdocs.pm/ecto/Ecto.Repo.html#module-urls). For using MySQL in the docker image, use a mysql scheme url: `-e DATABASE_URL="mysql://root:@db:3306/bors_ng"` in conjunction with `BORS_DATABASE=mysql` - `DATABASE_TIMEOUT` may be set higher than the default of `15_000`(ms). This may be necessary with repositories with a very large amount of members. - `DATABASE_PREPARE_MODE` can be set to to `unnamed` to disable prepared statements, [which is necessary when using a transaction/statement pooler, like pgbouncer](https://github.com/elixir-ecto/postgrex#pgbouncer). It is set to `named` by default. -- The database schema will be automatically created and migrated at container startup, unless the ` DATABASE_AUTO_MIGRATE` env. var. +- `BORS_DATABASE` can be set to `mysql` to switch the Docker container to MySQL +- The database schema will be automatically created and migrated at container startup, unless the ` DATABASE_AUTO_MIGRATE` env. var. is set to `false`. Make that change if the database state is managed externally, or if you are using a database that cannot safely handle concurrent schema changes (such as older MariaDB/MySQL versions). - Database migrations can be manually applied from a container using the `migrate` release command. Example: diff --git a/config/config.exs b/config/config.exs index 9ce135a0f..b4f0adcae 100644 --- a/config/config.exs +++ b/config/config.exs @@ -21,7 +21,7 @@ case config_env() do end config :bors, - ecto_repos: [BorsNG.Database.Repo], + ecto_repos: [BorsNG.Database.RepoPostgres], api_github_root: {:system, :string, "GITHUB_URL_ROOT_API", "https://api.github.com"}, html_github_root: {:system, :string, "GITHUB_URL_ROOT_HTML", "https://github.com"}, api_github_timeout: {:system, :integer, "GITHUB_API_TIMEOUT", 100_000}, diff --git a/config/dev.exs b/config/dev.exs index c90640cb9..d649d26fa 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -38,10 +38,16 @@ config :bors, BorsNG.Endpoint, # in production as building large stacktraces may be expensive. config :phoenix, :stacktrace_depth, 20 -config :bors, BorsNG.Database.Repo, +config :bors, BorsNG.Database.RepoPostgres, adapter: Ecto.Adapters.Postgres, url: {:system, "DATABASE_URL", "postgresql://postgres:Postgres1234@localhost/bors_dev"}, - pool_size: 10 + pool_size: 10, + priv: "priv/repo" + +config :bors, BorsNG.Database.RepoMysql, + adapter: Ecto.Adapters.MyXQL, + url: {:system, "DATABASE_URL", "mysql://root@localhost:3306/bors_dev"}, + priv: "priv/repo" # On developer boxes, we do not actually talk to GitHub. # Use the mock instance. diff --git a/config/prod.secret.exs b/config/prod.secret.exs index 0f81373ec..2a88dba17 100644 --- a/config/prod.secret.exs +++ b/config/prod.secret.exs @@ -1,13 +1,24 @@ import Config -config :bors, BorsNG.Database.Repo, +config :bors, BorsNG.Database.RepoPostgres, adapter: Ecto.Adapters.Postgres, url: {:system, "DATABASE_URL"}, timeout: {:system, :integer, "DATABASE_TIMEOUT", 15_000}, pool_size: {:system, :integer, "POOL_SIZE", 10}, loggers: [{Ecto.LogEntry, :log, []}], ssl: {:system, :boolean, "DATABASE_USE_SSL", true}, - prepare: {:system, :atom, "DATABASE_PREPARE_MODE", :named} + prepare: {:system, :atom, "DATABASE_PREPARE_MODE", :named}, + priv: "priv/repo" + +config :bors, BorsNG.Database.RepoMysql, + adapter: Ecto.Adapters.MyXQL, + url: {:system, "DATABASE_URL"}, + ssl: {:system, :boolean, "DATABASE_USE_SSL", true}, + timeout: {:system, :integer, "DATABASE_TIMEOUT", 15_000}, + pool_size: {:system, :integer, "POOL_SIZE", 10}, + loggers: [{Ecto.LogEntry, :log, []}], + ssl: {:system, :boolean, "DATABASE_USE_SSL", true}, + priv: "priv/repo" config :bors, BorsNG.Endpoint, http: [port: {:system, "PORT"}], diff --git a/config/test.exs b/config/test.exs index 6a5330dbc..e9c343b40 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,22 +1,19 @@ import Config -case System.get_env("BORS_TEST_DATABASE") do - "mysql" -> - config :bors, BorsNG.Database.Repo, - adapter: Ecto.Adapters.MyXQL, - username: "root", - password: "", - database: "bors_test", - hostname: {:system, "MYSQL_HOST", "localhost"}, - pool: Ecto.Adapters.SQL.Sandbox +config :bors, BorsNG.Database.RepoMysql, + adapter: Ecto.Adapters.MyXQL, + username: "root", + password: "", + database: "bors_test", + hostname: {:system, "MYSQL_HOST", "localhost"}, + pool: Ecto.Adapters.SQL.Sandbox, + priv: "priv/repo" - _ -> - config :bors, BorsNG.Database.Repo, - adapter: Ecto.Adapters.Postgres, - url: - {:system, "DATABASE_URL_TEST", "postgresql://postgres:Postgres1234@localhost/bors_test"}, - pool: Ecto.Adapters.SQL.Sandbox -end +config :bors, BorsNG.Database.RepoPostgres, + adapter: Ecto.Adapters.Postgres, + url: {:system, "DATABASE_URL_TEST", "postgresql://postgres:Postgres1234@localhost/bors_test"}, + pool: Ecto.Adapters.SQL.Sandbox, + priv: "priv/repo" # We don't run a server during test. If one is required, # you can enable the server option below. diff --git a/lib/application.ex b/lib/application.ex index 2db4d8f01..cf1ff7bd8 100644 --- a/lib/application.ex +++ b/lib/application.ex @@ -5,15 +5,33 @@ defmodule BorsNG.Application do use Application + def set_repo do + repo_module = + case System.get_env("BORS_DATABASE", "postgresql") do + "mysql" -> BorsNG.Database.RepoMysql + _ -> BorsNG.Database.RepoPostgres + end + + :persistent_term.put(:db_repo, repo_module) + end + + def fetch_repo do + :persistent_term.get(:db_repo) + end + # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do # Define workers and child supervisors to be supervised + set_repo() + + repo = fetch_repo() + children = [ %{ type: :supervisor, - id: BorsNG.Database.Repo, - start: {BorsNG.Database.Repo, :start_link, []} + id: repo, + start: {repo, :start_link, []} }, %{ type: :worker, diff --git a/lib/database/migrate.ex b/lib/database/migrate.ex index c1e3af0b6..41bc54d0a 100644 --- a/lib/database/migrate.ex +++ b/lib/database/migrate.ex @@ -11,12 +11,13 @@ defmodule BorsNG.Database.Migrate do continue running normally afterwards. """ - def repos, do: Application.get_env(:bors, :ecto_repos, []) + def repos, do: [BorsNG.Application.fetch_repo()] @start_apps [ :crypto, :ssl, :postgrex, + :myxql, :ecto, :confex ] @@ -33,6 +34,8 @@ defmodule BorsNG.Database.Migrate do # which won't start because the database isn't set up yet, # we start the Ecto Repo directly. Application.load(:bors) + BorsNG.Application.set_repo() + Enum.each(@start_apps, &Application.ensure_all_started/1) Enum.each(repos(), & &1.start_link(pool_size: 3)) @@ -94,7 +97,6 @@ defmodule BorsNG.Database.Migrate do def priv_path_for(repo, filename) do app = Keyword.get(repo.config, :otp_app) - repo_underscore = repo |> Module.split() |> List.last() |> Macro.underscore() - Path.join([priv_dir(app), repo_underscore, filename]) + Path.join([priv_dir(app), "repo", filename]) end end diff --git a/lib/database/repo.ex b/lib/database/repo.ex index 0f829169d..3e5e22311 100644 --- a/lib/database/repo.ex +++ b/lib/database/repo.ex @@ -1,48 +1,55 @@ -defmodule BorsNG.Database.Repo do +defmodule DynamicEctoRepoWrapper do @moduledoc """ - An ecto data repository; - this process interacts with your persistent database. - - Do not confuse this with a GitHub repo. - We call those `Project`s internally. + Exposes a macro to define Ecto.Repo functions dynamically. """ + defmacro create_ecto_repo_callback(args, name) do + quote bind_quoted: [args: args, name: name] do + def unquote(name)(unquote_splicing(args)) do + repo = :persistent_term.get(:db_repo) + apply(repo, unquote(name), unquote(args)) + end + end + end - use Ecto.Repo, - otp_app: :bors, - adapter: Application.get_env(:bors, BorsNG.Database.Repo)[:adapter] + def create_ecto_repo_callback_args(_, 0) do + [] + end - def init(_, config) do - # Backwards compatibility hack: if POSTGRES_HOST is set, and the database URL is left at default, - # use the older configuration. - config = Confex.Resolver.resolve!(config) - no_host = is_nil(System.get_env("POSTGRES_HOST")) + def create_ecto_repo_callback_args(module, arity) do + Enum.map(1..arity, &Macro.var(:"arg#{&1}", module)) + end +end - config = - case config[:url] do - _ when no_host -> - config +defmodule BorsNG.Database.Repo do + @moduledoc """ + This is an Ecto.Repo wrapper that defines all callback functions. + """ + import DynamicEctoRepoWrapper - "postgresql://postgres:Postgres1234@localhost/bors_dev" -> - [ - adapter: Ecto.Adapters.Postgres, - username: "postgres", - password: "Postgres1234", - database: "bors_dev", - hostname: {:system, "POSTGRES_HOST", "localhost"}, - pool_size: 10 - ] + @ecto_repo_callbacks Path.join([__DIR__, "repo_callbacks.txt"]) + |> File.read!() + |> String.trim() + |> String.split("\n") + |> Enum.map(fn line -> + [line, arity] = line |> String.split("/") |> Enum.map(&String.trim(&1)) + {String.to_atom(line), String.to_integer(arity)} + end) - "postgresql://postgres:Postgres1234@localhost/bors_test" -> - [ - adapter: Ecto.Adapters.Postgres, - username: "postgres", - password: "Postgres1234", - database: "bors_test", - hostname: {:system, "POSTGRES_HOST", "localhost"}, - pool_size: 10 - ] - end + @ecto_repo_callbacks + |> Enum.flat_map(fn + {callback, arity} when arity >= 1 -> + is_defined = Enum.any?(@ecto_repo_callbacks, fn fun -> fun == {callback, arity - 1} end) - {:ok, config} - end + if is_defined, + do: [{callback, arity}], + else: [{callback, arity}, {callback, arity - 1}] + + {callback, arity} -> + [{callback, arity}] + end) + |> Enum.map(fn {callback, arity} -> + __MODULE__ + |> create_ecto_repo_callback_args(arity) + |> create_ecto_repo_callback(callback) + end) end diff --git a/lib/database/repo_callbacks.txt b/lib/database/repo_callbacks.txt new file mode 100644 index 000000000..35977ec99 --- /dev/null +++ b/lib/database/repo_callbacks.txt @@ -0,0 +1,40 @@ +__adapter__/0 +aggregate/3 +aggregate/4 +all/2 +checked_out?/0 +checkout/2 +config/0 +default_options/1 +delete/2 +delete!/2 +delete_all/2 +exists?/2 +get/3 +get!/3 +get_by/3 +get_by!/3 +get_dynamic_repo/0 +in_transaction?/0 +init/2 +insert/2 +insert!/2 +insert_all/3 +insert_or_update/2 +insert_or_update!/2 +load/2 +one/2 +one!/2 +preload/3 +prepare_query/3 +put_dynamic_repo/1 +reload/2 +reload!/2 +rollback/1 +start_link/1 +stop/1 +stream/2 +transaction/2 +update/2 +update!/2 +update_all/3 diff --git a/lib/database/repo_mysql.ex b/lib/database/repo_mysql.ex new file mode 100644 index 000000000..a46aaaee8 --- /dev/null +++ b/lib/database/repo_mysql.ex @@ -0,0 +1,19 @@ +defmodule BorsNG.Database.RepoMysql do + @moduledoc """ + An ecto data repository; + this process interacts with your persistent database. + + Do not confuse this with a GitHub repo. + We call those `Project`s internally. + """ + + use Ecto.Repo, + otp_app: :bors, + adapter: Ecto.Adapters.MyXQL + + def init(_, config) do + config = Confex.Resolver.resolve!(config) + + {:ok, config} + end +end diff --git a/lib/database/repo_postgres.ex b/lib/database/repo_postgres.ex new file mode 100644 index 000000000..c279eee37 --- /dev/null +++ b/lib/database/repo_postgres.ex @@ -0,0 +1,48 @@ +defmodule BorsNG.Database.RepoPostgres do + @moduledoc """ + An ecto data repository; + this process interacts with your persistent database. + + Do not confuse this with a GitHub repo. + We call those `Project`s internally. + """ + + use Ecto.Repo, + otp_app: :bors, + adapter: Ecto.Adapters.Postgres + + def init(_, config) do + # Backwards compatibility hack: if POSTGRES_HOST is set, and the database URL is left at default, + # use the older configuration. + config = Confex.Resolver.resolve!(config) + no_host = is_nil(System.get_env("POSTGRES_HOST")) + + config = + case config[:url] do + _ when no_host -> + config + + "postgresql://postgres:Postgres1234@localhost/bors_dev" -> + [ + adapter: Ecto.Adapters.Postgres, + username: "postgres", + password: "Postgres1234", + database: "bors_dev", + hostname: {:system, "POSTGRES_HOST", "localhost"}, + pool_size: 10 + ] + + "postgresql://postgres:Postgres1234@localhost/bors_test" -> + [ + adapter: Ecto.Adapters.Postgres, + username: "postgres", + password: "Postgres1234", + database: "bors_test", + hostname: {:system, "POSTGRES_HOST", "localhost"}, + pool_size: 10 + ] + end + + {:ok, config} + end +end diff --git a/lib/mix/bors.cleanup.ex b/lib/mix/bors.cleanup.ex index 3cf69bb69..d38e67105 100644 --- a/lib/mix/bors.cleanup.ex +++ b/lib/mix/bors.cleanup.ex @@ -67,5 +67,5 @@ defmodule Mix.Tasks.Bors.Cleanup do end end - def repos, do: Application.get_env(:bors, :ecto_repos, []) + def repos, do: [BorsNG.Application.fetch_repo()] end diff --git a/priv/repo/migrations/20170823012140_change_users_login_type_to_citext.exs b/priv/repo/migrations/20170823012140_change_users_login_type_to_citext.exs index 5a431a5a5..57d3a66f7 100644 --- a/priv/repo/migrations/20170823012140_change_users_login_type_to_citext.exs +++ b/priv/repo/migrations/20170823012140_change_users_login_type_to_citext.exs @@ -1,10 +1,18 @@ defmodule BorsNG.Database.Repo.Migrations.ChangeUsersLoginTypeToCitext do use Ecto.Migration - @adapter Confex.fetch_env!(:bors, BorsNG.Database.Repo)[:adapter] + defp fetch_adapter do + repo_module = + case System.get_env("BORS_DATABASE", "postgresql") do + "mysql" -> BorsNG.Database.RepoMysql + _ -> BorsNG.Database.RepoPostgres + end + + Confex.fetch_env!(:bors, repo_module)[:adapter] + end def up do - case @adapter do + case fetch_adapter() do Ecto.Adapters.Postgres -> execute "CREATE EXTENSION IF NOT EXISTS citext" alter table(:users) do @@ -16,7 +24,7 @@ defmodule BorsNG.Database.Repo.Migrations.ChangeUsersLoginTypeToCitext do end def down do - case @adapter do + case fetch_adapter() do Ecto.Adapters.Postgres -> alter table(:users) do modify :login, :string diff --git a/test/support/channel_case.ex b/test/support/channel_case.ex index fc8f40538..3242a54a6 100644 --- a/test/support/channel_case.ex +++ b/test/support/channel_case.ex @@ -31,10 +31,10 @@ defmodule BorsNG.ChannelCase do end setup tags do - :ok = Ecto.Adapters.SQL.Sandbox.checkout(BorsNG.Database.Repo) + :ok = Ecto.Adapters.SQL.Sandbox.checkout(:persistent_term.get(:db_repo)) unless tags[:async] do - Ecto.Adapters.SQL.Sandbox.mode(BorsNG.Database.Repo, {:shared, self()}) + Ecto.Adapters.SQL.Sandbox.mode(:persistent_term.get(:db_repo), {:shared, self()}) end :ok diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index 828e7d006..f4ece34c8 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -34,10 +34,10 @@ defmodule BorsNG.ConnCase do end setup tags do - :ok = Ecto.Adapters.SQL.Sandbox.checkout(BorsNG.Database.Repo) + :ok = Ecto.Adapters.SQL.Sandbox.checkout(:persistent_term.get(:db_repo)) unless tags[:async] do - Ecto.Adapters.SQL.Sandbox.mode(BorsNG.Database.Repo, {:shared, self()}) + Ecto.Adapters.SQL.Sandbox.mode(:persistent_term.get(:db_repo), {:shared, self()}) end {:ok, conn: Phoenix.ConnTest.build_conn()} diff --git a/test/support/model_case.ex b/test/support/model_case.ex index 3da0701e2..f3433863c 100644 --- a/test/support/model_case.ex +++ b/test/support/model_case.ex @@ -25,10 +25,10 @@ defmodule BorsNG.Database.ModelCase do end setup tags do - :ok = Ecto.Adapters.SQL.Sandbox.checkout(BorsNG.Database.Repo) + :ok = Ecto.Adapters.SQL.Sandbox.checkout(:persistent_term.get(:db_repo)) unless tags[:async] do - Ecto.Adapters.SQL.Sandbox.mode(BorsNG.Database.Repo, {:shared, self()}) + Ecto.Adapters.SQL.Sandbox.mode(:persistent_term.get(:db_repo), {:shared, self()}) end :ok diff --git a/test/support/worker_test_case.ex b/test/support/worker_test_case.ex index 483dc7317..42b8350bd 100644 --- a/test/support/worker_test_case.ex +++ b/test/support/worker_test_case.ex @@ -21,10 +21,10 @@ defmodule BorsNG.Worker.TestCase do end setup tags do - :ok = Ecto.Adapters.SQL.Sandbox.checkout(BorsNG.Database.Repo) + :ok = Ecto.Adapters.SQL.Sandbox.checkout(:persistent_term.get(:db_repo)) unless tags[:async] do - Ecto.Adapters.SQL.Sandbox.mode(BorsNG.Database.Repo, {:shared, self()}) + Ecto.Adapters.SQL.Sandbox.mode(:persistent_term.get(:db_repo), {:shared, self()}) end :ok diff --git a/test/test_helper.exs b/test/test_helper.exs index 8e182566a..d0a2c5100 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,3 +1,3 @@ ExUnit.start() -Ecto.Adapters.SQL.Sandbox.mode(BorsNG.Database.Repo, :manual) +Ecto.Adapters.SQL.Sandbox.mode(:persistent_term.get(:db_repo), :manual)