diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e8bede..5f20a06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,159 +36,137 @@ jobs: elixir: 1.11 otp: 24 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: erlef/setup-beam@v1 env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true with: otp-version: ${{matrix.pair.otp}} elixir-version: ${{matrix.pair.elixir}} - - name: Run PostgreSQL 11 - run: | - docker run --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=ecto_psql_extras_test \ - --env POSTGRES_PASSWORD=postgres \ - -d -p 5432:5432 postgres:11.18-alpine \ - postgres -c shared_preload_libraries=pg_stat_statements - sleep 5 - name: Run PostgreSQL 12 run: | docker run --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=ecto_psql_extras_test \ - --env POSTGRES_PASSWORD=postgres \ - -d -p 5433:5432 postgres:12.13-alpine \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5432:5432 postgres:12.20-alpine \ postgres -c shared_preload_libraries=pg_stat_statements - sleep 5 - name: Run PostgreSQL 13 run: | docker run --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=ecto_psql_extras_test \ - --env POSTGRES_PASSWORD=postgres \ - -d -p 5434:5432 postgres:13.9-alpine \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5433:5432 postgres:13.16-alpine \ postgres -c shared_preload_libraries=pg_stat_statements - sleep 5 - name: Run PostgreSQL 14 run: | docker run --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=ecto_psql_extras_test \ - --env POSTGRES_PASSWORD=postgres \ - -d -p 5435:5432 postgres:14.6-alpine \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5434:5432 postgres:14.13-alpine \ postgres -c shared_preload_libraries=pg_stat_statements - sleep 5 - name: Run PostgreSQL 15 run: | docker run --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=ecto_psql_extras_test \ - --env POSTGRES_PASSWORD=postgres \ - -d -p 5436:5432 postgres:15.1-alpine \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5435:5432 postgres:15.8-alpine \ + postgres -c shared_preload_libraries=pg_stat_statements + sleep 15 + - name: Run PostgreSQL 16 + run: | + docker run --env POSTGRES_USER=postgres \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5436:5432 postgres:16.4-alpine \ + postgres -c shared_preload_libraries=pg_stat_statements + sleep 15 + - name: Run PostgreSQL 17 + run: | + docker run --env POSTGRES_USER=postgres \ + --env POSTGRES_DB=ecto-psql-extras-test \ + --env POSTGRES_PASSWORD=secret \ + -d -p 5437:5432 postgres:17.0-alpine \ postgres -c shared_preload_libraries=pg_stat_statements - sleep 5 + sleep 15 - name: Install Dependencies run: | mix local.hex --force mix local.rebar --force mix deps.get --only test - - name: Run tests for PG 11 - env: - PG_VERSION: 11 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres - run: | - sleep 4 # wait pg - mix test --include distribution - name: Run tests for PG 12 env: PG_VERSION: 12 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 13 env: PG_VERSION: 13 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 14 env: PG_VERSION: 14 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 15 env: PG_VERSION: 15 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - - name: Install latest compatible postgrex + - name: Run tests for PG 16 + env: + PG_VERSION: 16 run: | - mix deps.unlock --all - mix deps.update postgrex - - name: Run tests for PG 11/latest compatible postgrex + sleep 4 # wait pg + mix test --include distribution + - name: Run tests for PG 17 env: - PG_VERSION: 11 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + PG_VERSION: 17 run: | sleep 4 # wait pg mix test --include distribution + - name: Install latest compatible postgrex + run: | + mix deps.unlock --all + mix deps.update postgrex - name: Run tests for PG 12/latest compatible postgrex env: PG_VERSION: 12 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 13/latest compatible postgrex env: PG_VERSION: 13 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 14/latest compatible postgrex env: PG_VERSION: 14 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution - name: Run tests for PG 15/latest compatible postgrex env: PG_VERSION: 15 - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres run: | sleep 4 # wait pg mix test --include distribution + - name: Run tests for PG 16/latest compatible postgrex + env: + PG_VERSION: 16 + run: | + sleep 4 # wait pg + mix test --include distribution + - name: Run tests for PG 17/latest compatible postgrex + env: + PG_VERSION: 17 + run: | + sleep 4 # wait pg + mix test --include distribution + diff --git a/docker-compose.yml.sample b/docker-compose.yml.sample index 827b64e..bae0aa7 100644 --- a/docker-compose.yml.sample +++ b/docker-compose.yml.sample @@ -1,48 +1,56 @@ -version: '3' - services: - postgres11: - image: postgres:11.18-alpine + postgres12: + image: postgres:12.20-alpine command: postgres -c shared_preload_libraries=pg_stat_statements environment: POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret ports: - '5432:5432' - postgres12: - image: postgres:12.13-alpine + postgres13: + image: postgres:13.16-alpine command: postgres -c shared_preload_libraries=pg_stat_statements environment: POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret ports: - '5433:5432' - postgres13: - image: postgres:13.3-alpine + postgres14: + image: postgres:14.13-alpine command: postgres -c shared_preload_libraries=pg_stat_statements environment: POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret ports: - '5434:5432' - postgres14: - image: postgres:14.6-alpine + postgres15: + image: postgres:15.8-alpine command: postgres -c shared_preload_libraries=pg_stat_statements environment: POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret ports: - '5435:5432' - postgres15: - image: postgres:15.1-alpine + postgres16: + image: postgres:16.4-alpine command: postgres -c shared_preload_libraries=pg_stat_statements environment: POSTGRES_USER: postgres - POSTGRES_DB: ecto_psql_extras_test - POSTGRES_PASSWORD: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret ports: - '5436:5432' + postgres17: + image: postgres:17.0-alpine + command: postgres -c shared_preload_libraries=pg_stat_statements + environment: + POSTGRES_USER: postgres + POSTGRES_DB: ecto-psql-extras-test + POSTGRES_PASSWORD: secret + ports: + - '5437:5432' + diff --git a/lib/ecto_psql_extras.ex b/lib/ecto_psql_extras.ex index 24ed056..fb69004 100644 --- a/lib/ecto_psql_extras.ex +++ b/lib/ecto_psql_extras.ex @@ -67,6 +67,9 @@ defmodule EctoPSQLExtras do vsn when vsn < {1, 8, 0} -> %{calls: EctoPSQLExtras.CallsLegacy, outliers: EctoPSQLExtras.OutliersLegacy} + vsn when vsn >= {1, 11, 0} -> + %{calls: EctoPSQLExtras.Calls17, outliers: EctoPSQLExtras.Outliers17} + _vsn -> %{calls: EctoPSQLExtras.Calls, outliers: EctoPSQLExtras.Outliers} end @@ -133,11 +136,12 @@ defmodule EctoPSQLExtras do query_module = Map.fetch!(queries(repo), name) opts = prepare_opts(opts, query_module.info[:default_args]) - result = query!( - repo, - query_module.query(Keyword.fetch!(opts, :args)), - Keyword.get(opts, :query_opts, @default_query_opts) - ) + result = + query!( + repo, + query_module.query(Keyword.fetch!(opts, :args)), + Keyword.get(opts, :query_opts, @default_query_opts) + ) format( Keyword.fetch!(opts, :format), @@ -432,15 +436,21 @@ defmodule EctoPSQLExtras do defp memory_unit(:KB), do: 1024 defp prepare_opts(format, default_args) when is_atom(format) do - IO.warn "This API is deprecated. Please pass format value as a keyword list: `format: :raw`", - Macro.Env.stacktrace(__ENV__) + IO.warn( + "This API is deprecated. Please pass format value as a keyword list: `format: :raw`", + Macro.Env.stacktrace(__ENV__) + ) + prepare_opts([format: format], default_args) end defp prepare_opts(opts, default_args) do - Keyword.merge([ - format: Keyword.get(opts, :format, :ascii), - args: Keyword.merge(default_args || [], opts[:args] || []) - ], opts) + Keyword.merge( + [ + format: Keyword.get(opts, :format, :ascii), + args: Keyword.merge(default_args || [], opts[:args] || []) + ], + opts + ) end end diff --git a/lib/queries/calls_17.ex b/lib/queries/calls_17.ex new file mode 100644 index 0000000..120dfca --- /dev/null +++ b/lib/queries/calls_17.ex @@ -0,0 +1,36 @@ +defmodule EctoPSQLExtras.Calls17 do + @behaviour EctoPSQLExtras + + def info do + %{ + title: "Queries that have the highest frequency of execution", + index: 21, + order_by: [calls: :desc], + default_args: [limit: 10], + columns: [ + %{name: :query, type: :string}, + %{name: :exec_time, type: :interval}, + %{name: :prop_exec_time, type: :percent}, + %{name: :calls, type: :integer}, + %{name: :sync_io_time, type: :interval} + ] + } + end + + def query(args \\ []) do + """ + /* ECTO_PSQL_EXTRAS: Queries that have the highest frequency of execution */ + + SELECT query AS query, + interval '1 millisecond' * total_exec_time AS exec_time, + (total_exec_time/sum(total_exec_time) OVER()) AS exec_time_ratio, + calls, + interval '1 millisecond' * (shared_blk_read_time + shared_blk_write_time) AS sync_io_time + FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) + AND query NOT LIKE '/* ECTO_PSQL_EXTRAS:%' + ORDER BY calls DESC + LIMIT <%= limit %>; + """ + |> EEx.eval_string(args) + end +end diff --git a/lib/queries/outliers_17.ex b/lib/queries/outliers_17.ex new file mode 100644 index 0000000..3b8c06e --- /dev/null +++ b/lib/queries/outliers_17.ex @@ -0,0 +1,36 @@ +defmodule EctoPSQLExtras.Outliers17 do + @behaviour EctoPSQLExtras + + def info do + %{ + title: "Queries that have longest execution time in aggregate", + index: 8, + order_by: [exec_time: :desc], + default_args: [limit: 10], + columns: [ + %{name: :query, type: :string}, + %{name: :exec_time, type: :interval}, + %{name: :prop_exec_time, type: :percent}, + %{name: :calls, type: :integer}, + %{name: :sync_io_time, type: :interval} + ] + } + end + + def query(args \\ []) do + """ + /* ECTO_PSQL_EXTRAS: Queries that have longest execution time in aggregate */ + + SELECT query AS query, + interval '1 millisecond' * total_exec_time AS exec_time, + (total_exec_time/sum(total_exec_time) OVER()) AS prop_exec_time, + calls, + interval '1 millisecond' * (shared_blk_read_time + shared_blk_write_time) AS sync_io_time + FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) + AND query NOT LIKE '/* ECTO_PSQL_EXTRAS:%' + ORDER BY total_exec_time DESC + LIMIT <%= limit %>; + """ + |> EEx.eval_string(args) + end +end diff --git a/mix.exs b/mix.exs index c600966..56eab27 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule EctoPSQLExtras.Mixfile do use Mix.Project @github_url "https://github.com/pawurb/ecto_psql_extras" - @version "0.8.1" + @version "0.8.2" def project do [ diff --git a/test/ecto_psql_extras_test.exs b/test/ecto_psql_extras_test.exs index c59cc68..c7118f4 100644 --- a/test/ecto_psql_extras_test.exs +++ b/test/ecto_psql_extras_test.exs @@ -36,7 +36,7 @@ defmodule EctoPSQLExtrasTest do end defmodule Repo do - def query!(_, _, _), do: %{rows: Process.get(Repo) || raise "no query result"} + def query!(_, _, _), do: %{rows: Process.get(Repo) || raise("no query result")} end describe "pg_stat_statements queries" do @@ -65,16 +65,18 @@ defmodule EctoPSQLExtrasTest do assert {:outliers, EctoPSQLExtras.OutliersLegacy} in queries end - test "includes recent queries on later versions" do + test "includes standard queries" do Process.put(Repo, [["1.9.0"]]) queries = queries(Repo) assert {:calls, EctoPSQLExtras.Calls} in queries assert {:outliers, EctoPSQLExtras.Outliers} in queries + end + test "includes recent queries on newest pg versions" do Process.put(Repo, [["2.1.0"]]) queries = queries(Repo) - assert {:calls, EctoPSQLExtras.Calls} in queries - assert {:outliers, EctoPSQLExtras.Outliers} in queries + assert {:calls, EctoPSQLExtras.Calls17} in queries + assert {:outliers, EctoPSQLExtras.Outliers17} in queries end end @@ -109,78 +111,101 @@ defmodule EctoPSQLExtrasTest do describe "database interaction" do setup do start_supervised!(EctoPSQLExtras.TestRepo) - EctoPSQLExtras.TestRepo.query!("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;", [], log: false) + + EctoPSQLExtras.TestRepo.query!("CREATE EXTENSION IF NOT EXISTS pg_stat_statements;", [], + log: false + ) + :ok end test "run queries by param" do - for query <- Enum.reduce((queries(TestRepo) |> Map.to_list), [], fn(el, acc) -> - case elem(el, 0) in @skip_queries do - true -> - acc - false -> - [elem(el, 0) | acc] - end - end) do - assert(length( - EctoPSQLExtras.query( - query, - TestRepo, - [format: :raw] - ).columns - ) > 0) + for query <- + Enum.reduce(queries(TestRepo) |> Map.to_list(), [], fn el, acc -> + case elem(el, 0) in @skip_queries do + true -> + acc + + false -> + [elem(el, 0) | acc] + end + end) do + assert( + length( + EctoPSQLExtras.query( + query, + TestRepo, + format: :raw + ).columns + ) > 0 + ) end end test "provide custom param" do - assert(length( - EctoPSQLExtras.long_running_queries( - TestRepo, - [format: :raw, args: [threshold: ~c"1 second"]] - ).columns - ) > 0) - - assert(length( - EctoPSQLExtras.query( - :long_running_queries, - TestRepo, - [format: :raw, args: [threshold: ~c"200 milliseconds"]] - ).columns - ) > 0) + assert( + length( + EctoPSQLExtras.long_running_queries( + TestRepo, + format: :raw, + args: [threshold: ~c"1 second"] + ).columns + ) > 0 + ) + + assert( + length( + EctoPSQLExtras.query( + :long_running_queries, + TestRepo, + format: :raw, + args: [threshold: ~c"200 milliseconds"] + ).columns + ) > 0 + ) end test "test legacy API" do - warning = capture_io(:stderr, fn -> - columns = EctoPSQLExtras.long_running_queries(TestRepo, :raw).columns - assert length(columns) > 0 - end) - - assert warning =~ "This API is deprecated. Please pass format value as a keyword list: `format: :raw`" + warning = + capture_io(:stderr, fn -> + columns = EctoPSQLExtras.long_running_queries(TestRepo, :raw).columns + assert length(columns) > 0 + end) + + assert warning =~ + "This API is deprecated. Please pass format value as a keyword list: `format: :raw`" end test "test query_opts allows for logging" do - logs = capture_log(fn -> - EctoPSQLExtras.long_running_queries(TestRepo, format: :raw, query_opts: [log: true]) - end) - assert logs =~ "ECTO_PSQL_EXTRAS: All queries longer than the threshold by descending duration" + logs = + capture_log(fn -> + EctoPSQLExtras.long_running_queries(TestRepo, format: :raw, query_opts: [log: true]) + end) + + assert logs =~ + "ECTO_PSQL_EXTRAS: All queries longer than the threshold by descending duration" end test "run queries by method" do - for query <- Enum.reduce((queries(TestRepo) |> Map.to_list), [], fn(el, acc) -> - case elem(el, 0) in @skip_queries do - true -> - acc - false -> - [elem(el, 0) | acc] - end - end) do - assert(length( - apply( - EctoPSQLExtras, - query, - [TestRepo, [format: :raw]] - ).columns - ) > 0) + for query <- + Enum.reduce(queries(TestRepo) |> Map.to_list(), [], fn el, acc -> + case elem(el, 0) in @skip_queries do + true -> + acc + + false -> + [elem(el, 0) | acc] + end + end) do + assert( + length( + apply( + EctoPSQLExtras, + query, + [TestRepo, [format: :raw]] + ).columns + ) > 0 + ) end end end diff --git a/test/support/test_repo.exs b/test/support/test_repo.exs index 231f66c..38afb7f 100644 --- a/test/support/test_repo.exs +++ b/test/support/test_repo.exs @@ -2,11 +2,12 @@ defmodule EctoPSQLExtras.TestRepo do use Ecto.Repo, otp_app: :ecto_psql_extras, adapter: Ecto.Adapters.Postgres @ports_mapping %{ - "11" => "5432", - "12" => "5433", - "13" => "5434", - "14" => "5435", - "15" => "5436" + "12" => "5432", + "13" => "5433", + "14" => "5434", + "15" => "5435", + "16" => "5436", + "17" => "5437" } def init(type, opts) do @@ -22,9 +23,9 @@ defmodule EctoPSQLExtras.TestRepo do postgres_url else user = System.get_env("POSTGRES_USER") || "postgres" - password = System.get_env("POSTGRES_USER") || "postgres" + password = System.get_env("POSTGRES_USER") || "secret" host = System.get_env("POSTGRES_HOST") || "localhost" - db_name = System.get_env("POSTGRES_DB") || "ecto_psql_extras_test" + db_name = System.get_env("POSTGRES_DB") || "ecto-psql-extras-test" port = Map.get(@ports_mapping, System.get_env("PG_VERSION"), "5432")