Skip to content

Commit

Permalink
Add rate limiter
Browse files Browse the repository at this point in the history
  • Loading branch information
Adrien Moreau committed Dec 1, 2021
1 parent 5e20f1c commit 9e84826
Show file tree
Hide file tree
Showing 8 changed files with 151 additions and 109 deletions.
14 changes: 8 additions & 6 deletions .github/workflows/elixir.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,16 @@ jobs:
build:

runs-on: ubuntu-latest

strategy:
matrix:
otp: ['24.1']
elixir: ['1.12.3']
steps:
- uses: actions/checkout@v2
- name: Setup elixir
uses: actions/setup-elixir@v1
- uses: erlef/setup-beam@v1
with:
elixir-version: 1.10.3
otp-version: 22.2
otp-version: ${{matrix.otp}}
elixir-version: ${{matrix.elixir}}
- name: deps cache
uses: actions/cache@v1
with:
Expand All @@ -32,7 +34,7 @@ jobs:
key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
restore-keys: |
${{ runner.os }}-build-
- name: dializer cache
- name: dialyzer cache
uses: actions/cache@v1
with:
path: deps
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,6 @@ erl_crash.dump
http_client-*.tar

.elixir_ls/

*.plt
*.xml
24 changes: 17 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,21 +1,31 @@
# HttpClient

**TODO: Add description**
Httpoison boosted with telemetry, mox and ex_rated.

```elixir
HttpClient.get("http://mydomain.com")
```

## Installation

If [available in Hex](https://hex.pm/docs/publish), the package can be installed
by adding `http_client` to your list of dependencies in `mix.exs`:
The package can be installed by adding `http_client` to your list of dependencies
in `mix.exs`:

```elixir
def deps do
[
{:http_client, "~> 0.1.0"}
{:http_client, "~> 0.2.3"}
]
end
```

Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
be found at [https://hexdocs.pm/http_client](https://hexdocs.pm/http_client).
The docs can be found at [https://hexdocs.pm/http_client](https://hexdocs.pm/http_client).

## Configuration

```elixir
# Will rate limit calls to mydomain.com to 5 per seconds
# HttpClient.get("http://mydomain.com/any_path")
config :http_client, :rate_limits, "mydomain.com": [{:timer.seconds(1), 5}]
```

90 changes: 20 additions & 70 deletions lib/http_client.ex
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
defmodule HttpClient do
@moduledoc false
alias HttpClient.Instrumenter
alias HttpClient.RateLimiter

@callback get(String.t()) :: {:ok, any()} | {:error, any()}
@callback get(String.t(), list()) :: {:ok, any()} | {:error, any()}
@callback get(String.t(), list(), Keyword.t()) :: {:ok, any()} | {:error, any()}
Expand All @@ -17,87 +19,35 @@ defmodule HttpClient do
@callback delete(String.t(), list()) :: {:ok, any()} | {:error, any()}
@callback delete(String.t(), list(), Keyword.t()) :: {:ok, any()} | {:error, any()}

def get(url) do
:timer.tc(fn ->
impl().get(url)
end)
|> Instrumenter.instrument(url)
end
def get(url), do: request(:get, [url])

def get(url, headers) do
:timer.tc(fn ->
impl().get(url, headers)
end)
|> Instrumenter.instrument(url)
end
def get(url, headers), do: request(:get, [url, headers])

def get(url, headers, opts) do
:timer.tc(fn ->
impl().get(url, headers, opts)
end)
|> Instrumenter.instrument(url)
end
def get(url, headers, options), do: request(:get, [url, headers, options])

def post(url, body) do
:timer.tc(fn ->
impl().post(url, body)
end)
|> Instrumenter.instrument(url)
end
def post(url, body), do: request(:post, [url, body])

def post(url, body, headers) do
:timer.tc(fn ->
impl().post(url, body, headers)
end)
|> Instrumenter.instrument(url)
end
def post(url, body, headers), do: request(:post, [url, body, headers])

def post(url, body, headers, opts) do
:timer.tc(fn ->
impl().post(url, body, headers, opts)
end)
|> Instrumenter.instrument(url)
end
def post(url, body, headers, options), do: request(:post, [url, body, headers, options])

def put(url, body) do
:timer.tc(fn ->
impl().put(url, body)
end)
|> Instrumenter.instrument(url)
end
def put(url, body), do: request(:put, [url, body])

def put(url, body, headers) do
:timer.tc(fn ->
impl().put(url, body, headers)
end)
|> Instrumenter.instrument(url)
end
def put(url, body, headers), do: request(:put, [url, body, headers])

def put(url, body, headers, opts) do
:timer.tc(fn ->
impl().put(url, body, headers, opts)
end)
|> Instrumenter.instrument(url)
end
def put(url, body, headers, options), do: request(:put, [url, body, headers, options])

def delete(url) do
:timer.tc(fn ->
impl().delete(url)
end)
|> Instrumenter.instrument(url)
end
def delete(url), do: request(:delete, [url])

def delete(url, headers) do
:timer.tc(fn ->
impl().delete(url, headers)
end)
|> Instrumenter.instrument(url)
end
def delete(url, headers), do: request(:delete, [url, headers])

def delete(url, headers, options), do: request(:delete, [url, headers, options])

defp request(method, [url | _] = args) do
:ok = RateLimiter.rate_limit(url)

def delete(url, headers, options) do
:timer.tc(fn ->
impl().delete(url, headers, options)
end)
fn -> apply(impl(), method, args) end
|> :timer.tc()
|> Instrumenter.instrument(url)
end

Expand Down
50 changes: 50 additions & 0 deletions lib/http_client/rate_limiter.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
defmodule HttpClient.RateLimiter do
@moduledoc """
Module used to rate limit calls to certain host
In `config.exs` :
config :http_client, :rate_limits,
"google.com": [{:timer.seconds(1), 5}, {:timer.hours(24), 500_000}]
"""

@spec rate_limit(String.t()) :: :ok
def rate_limit(url) do
case retrieve_rate_limit_config(url) do
{:ok, {host, rate_limits}} ->
:ok = enforce_rate_limits(host, rate_limits)

:error ->
:ok
end
end

@spec enforce_rate_limits(atom(), list({non_neg_integer(), non_neg_integer()})) :: :ok
defp enforce_rate_limits(_host, []), do: :ok

defp enforce_rate_limits(host, [{scale, limit} | remaining_limits] = rate_limits) do
bucket = {host, scale}
{_, _, ms_to_next_bucket, _, _} = ExRated.inspect_bucket(bucket, scale, limit)

case ExRated.check_rate(bucket, scale, limit) do
{:ok, _} ->
enforce_rate_limits(host, remaining_limits)

_ ->
Process.sleep(ms_to_next_bucket)
enforce_rate_limits(host, rate_limits)
end
end

@spec retrieve_rate_limit_config(String.t()) ::
{:ok, {atom(), list({non_neg_integer(), non_neg_integer()})}} | :error
defp retrieve_rate_limit_config(url) do
%URI{host: host} = URI.parse(url)
host_atom = String.to_existing_atom(host)
{:ok, rate_limits} = Application.fetch_env(:http_client, :rate_limits)
{:ok, value} = Keyword.fetch(rate_limits, host_atom)
{:ok, {host_atom, value}}
rescue
_ -> :error
end
end
11 changes: 4 additions & 7 deletions mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule HttpClient.MixProject do
def project do
[
app: :http_client,
version: "0.2.2",
version: "0.2.3",
elixir: "~> 1.10",
start_permanent: Mix.env() == :prod,
test_coverage: [tool: ExCoveralls],
Expand All @@ -20,18 +20,15 @@ defmodule HttpClient.MixProject do
]
end

# Run "mix help compile.app" to learn about applications.
def application do
[
extra_applications: [:logger]
]
[extra_applications: [:logger]]
end

# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:httpoison, "~> 1.5"},
{:telemetry, "~> 0.4.0"},
{:ex_rated, "~> 2.0"},
{:mox, "~> 0.5", only: :test},
{:excoveralls, "~> 0.12", only: [:test]},
{:ex_unit_sonarqube, "~> 0.1.2", only: [:dev, :test]},
Expand All @@ -44,7 +41,7 @@ defmodule HttpClient.MixProject do
end

defp description do
"Httpoison boosted with telemetry and mox."
"Httpoison boosted with telemetry, mox and ex_rated."
end

defp package do
Expand Down
Loading

0 comments on commit 9e84826

Please sign in to comment.