Introduce Steps

* Add exponent retry
* Dynamically start Finch pool with the given proxy configs
* Support multiple content-encoding headers
* Add follow_redirects step

Author: anronin

.github/workflows/main.yml

@@ -3,49 +3,37 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  format:
-    name: Format and compile with warnings as errors
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install OTP and Elixir
-        uses: erlef/setup-beam@v1
-        with:
-          otp-version: 27.x
-          elixir-version: 1.18.x
-
-      - name: Install dependencies
-        run: mix deps.get
-
-      - name: Run "mix format"
-        run: mix format --check-formatted
-
-      - name: Compile with --warnings-as-errors
-        run: mix compile --warnings-as-errors
-
   test:
-    name: Test
     runs-on: ubuntu-latest
+    env:
+      MIX_ENV: test
     strategy:
       fail-fast: false
       matrix:
         include:
-          - erlang: 26.x
-            elixir: 1.16.x
-          - erlang: 27.x
-            elixir: 1.19.x
+          - pair:
+              elixir: 1.16.x
+              otp: 26.x
+          - pair:
+              elixir: 1.19.x
+              otp: 27.x
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Install OTP and Elixir
-        uses: erlef/setup-elixir@v1
+        uses: erlef/setup-beam@v1
         with:
-          otp-version: ${{matrix.erlang}}
+          otp-version: ${{matrix.otp}}
           elixir-version: ${{matrix.elixir}}
 
       - name: Install dependencies
         run: mix deps.get
 
-      - name: Run tests
-        run: mix test --trace
+      - name: Run "mix format"
+        run: mix format --check-formatted
+
+      - name: Check unused dependencies
+        run: mix deps.unlock --check-unused
+
+      - name: Compile with --warnings-as-errors
+        run: mix compile --warnings-as-errors

lib/http_client/adapter.ex

@@ -16,7 +16,7 @@ defmodule HTTPClient.Adapter do
   """
 
   alias HTTPClient.Adapters.{Finch, HTTPoison}
-  alias HTTPClient.{Error, Response, Telemetry}
+  alias HTTPClient.{Request, Steps}
   alias NimbleOptions.ValidationError
 
   @typedoc """
@@ -123,50 +123,39 @@ defmodule HTTPClient.Adapter do
 
   @doc false
   def request(adapter, method, url, body, headers, options) do
-    perform(adapter, :request, [method, url, body, headers, options])
+    perform(adapter, method, url, body: body, headers: headers, options: options)
   end
 
   @doc false
   def get(adapter, url, headers, options) do
-    perform(adapter, :get, [url, headers, options])
+    perform(adapter, :get, url, headers: headers, options: options)
   end
 
   @doc false
   def post(adapter, url, body, headers, options) do
-    perform(adapter, :post, [url, body, headers, options])
+    perform(adapter, :post, url, body: body, headers: headers, options: options)
   end
 
   @doc false
   def put(adapter, url, body, headers, options) do
-    perform(adapter, :put, [url, body, headers, options])
+    perform(adapter, :put, url, body: body, headers: headers, options: options)
   end
 
   @doc false
   def patch(adapter, url, body, headers, options) do
-    perform(adapter, :patch, [url, body, headers, options])
+    perform(adapter, :patch, url, body: body, headers: headers, options: options)
   end
 
   @doc false
   def delete(adapter, url, headers, options) do
-    perform(adapter, :delete, [url, headers, options])
+    perform(adapter, :delete, url, headers: headers, options: options)
   end
 
-  defp perform(adapter, method, args) do
-    metadata = %{adapter: adapter, args: args, method: method}
-    start_time = Telemetry.start(:request, metadata)
-
-    case apply(adapter, method, args) do
-      {:ok, %Response{status: status, headers: headers} = response} ->
-        metadata = Map.put(metadata, :status_code, status)
-        Telemetry.stop(:request, start_time, metadata)
-        headers = Enum.map(headers, fn {key, value} -> {String.downcase(key), value} end)
-        {:ok, %{response | headers: headers}}
-
-      {:error, %Error{reason: reason}} = error_response ->
-        metadata = Map.put(metadata, :error, reason)
-        Telemetry.stop(:request, start_time, metadata)
-        error_response
-    end
+  defp perform(adapter, method, url, options) do
+    adapter
+    |> Request.build(method, url, options)
+    |> Steps.put_default_steps()
+    |> Request.run()
   end
 
   defp adapter_mod(:finch), do: HTTPClient.Adapters.Finch

lib/http_client/adapters/finch.ex

@@ -3,118 +3,97 @@ defmodule HTTPClient.Adapters.Finch do
   Implementation of `HTTPClient.Adapter` behaviour using Finch HTTP client.
   """
 
-  alias HTTPClient.{Error, Response}
+  alias HTTPClient.{Request, Response}
 
   @type method() :: Finch.Request.method()
   @type url() :: Finch.Request.url()
   @type headers() :: Finch.Request.headers()
   @type body() :: Finch.Request.body()
   @type options() :: keyword()
 
-  @behaviour HTTPClient.Adapter
+  @doc """
+  Performs the request using `Finch`.
+  """
+  def perform_request(request) do
+    options = prepare_options(request.options)
 
-  @delay 1000
+    request.method
+    |> Finch.build(request.url, request.headers, request.body)
+    |> Finch.request(request.private.finch_name, options)
+    |> case do
+      {:ok, %{status: status, body: body, headers: headers}} ->
+        {request,
+         Response.new(status: status, body: body, headers: headers, request_url: request.url)}
 
-  @impl true
-  def request(method, url, body, headers, options) do
-    perform_request(method, url, headers, body, options)
+      {:error, exception} ->
+        {request, exception}
+    end
   end
 
-  @impl true
-  def get(url, headers, options) do
-    perform_request(:get, url, headers, nil, options)
+  @doc false
+  def proxy(request) do
+    Request.put_private(request, :finch_name, get_client())
   end
 
-  @impl true
-  def post(url, body, headers, options) do
-    perform_request(:post, url, headers, body, options)
+  defp prepare_options(options) do
+    Enum.map(options, &normalize_option/1)
   end
 
-  @impl true
-  def put(url, body, headers, options) do
-    perform_request(:put, url, headers, body, options)
-  end
+  defp normalize_option({:timeout, value}), do: {:pool_timeout, value}
+  defp normalize_option({:recv_timeout, value}), do: {:receive_timeout, value}
+  defp normalize_option({key, value}), do: {key, value}
 
-  @impl true
-  def patch(url, body, headers, options) do
-    perform_request(:patch, url, headers, body, options)
+  defp get_client() do
+    :http_client
+    |> Application.get_env(:proxy)
+    |> get_client_name()
   end
 
-  @impl true
-  def delete(url, headers, options) do
-    perform_request(:delete, url, headers, nil, options)
+  defp get_client_name(nil), do: HTTPClient.Finch
+
+  defp get_client_name(proxies) when is_list(proxies) do
+    proxies
+    |> Enum.random()
+    |> get_client_name()
   end
 
-  defp perform_request(method, url, headers, body, options, attempt \\ 0) do
-    {params, options} = Keyword.pop(options, :params)
-    {basic_auth, options} = Keyword.pop(options, :basic_auth)
+  defp get_client_name(proxy) when is_map(proxy) do
+    name = custom_pool_name(proxy)
 
-    url = build_request_url(url, params)
-    headers = add_basic_auth_header(headers, basic_auth)
-    options = prepare_options(options)
+    pools = %{
+      default: [
+        conn_opts: [proxy: compose_proxy(proxy), proxy_headers: compose_proxy_headers(proxy)]
+      ]
+    }
 
-    method
-    |> Finch.build(url, headers, body)
-    |> Finch.request(get_client(), options)
-    |> case do
-      {:ok, %{status: status, body: body, headers: headers}} ->
-        {:ok, %Response{status: status, body: body, headers: headers, request_url: url}}
-
-      {:error,
-       %Mint.HTTPError{
-         reason: {:proxy, _}
-       }} ->
-        case attempt < 5 do
-          true ->
-            Process.sleep(attempt * @delay)
-            perform_request(method, url, headers, body, options, attempt + 1)
-
-          false ->
-            {:error, %Error{reason: :proxy_error}}
-        end
-
-      {:error, error} ->
-        {:error, %Error{reason: error.reason}}
-    end
-  end
-
-  defp build_request_url(url, nil), do: url
+    child_spec = {Finch, name: name, pools: pools}
 
-  defp build_request_url(url, params) do
-    cond do
-      Enum.count(params) === 0 -> url
-      URI.parse(url).query -> url <> "&" <> URI.encode_query(params)
-      true -> url <> "?" <> URI.encode_query(params)
+    case DynamicSupervisor.start_child(HTTPClient.FinchSupervisor, child_spec) do
+      {:ok, _} -> name
+      {:error, {:already_started, _}} -> name
     end
   end
 
-  defp add_basic_auth_header(headers, {username, password}) do
-    credentials = Base.encode64("#{username}:#{password}")
-    [{"Authorization", "Basic " <> credentials} | headers || []]
+  defp compose_proxy_headers(%{opts: opts}) do
+    Keyword.get(opts, :proxy_headers, [])
   end
 
-  defp add_basic_auth_header(headers, _basic_auth), do: headers
+  defp compose_proxy_headers(_), do: []
 
-  defp prepare_options(options) do
-    Enum.map(options, &normalize_option/1)
+  defp compose_proxy(proxy) do
+    {proxy.scheme, proxy.address, to_integer(proxy.port), proxy.opts}
   end
 
-  defp normalize_option({:timeout, value}), do: {:pool_timeout, value}
-  defp normalize_option({:recv_timeout, value}), do: {:receive_timeout, value}
-  defp normalize_option({key, value}), do: {key, value}
-
-  defp get_client do
-    case Application.get_env(:http_client, :proxy, nil) do
-      nil -> FinchHTTPClient
-      proxies -> get_client_with_proxy(proxies)
-    end
-  end
+  defp to_integer(term) when is_integer(term), do: term
+  defp to_integer(term) when is_binary(term), do: String.to_integer(term)
 
-  defp get_client_with_proxy(proxy) when is_map(proxy) do
-    FinchHTTPClientWithProxy_0
-  end
+  defp custom_pool_name(opts) do
+    name =
+      opts
+      |> :erlang.term_to_binary()
+      |> :erlang.md5()
+      |> Base.url_encode64(padding: false)
 
-  defp get_client_with_proxy(proxies) when is_list(proxies) do
-    :"FinchHTTPClientWithProxy_#{Enum.random(0..(length(proxies) - 1))}"
+    Module.concat(HTTPClient.FinchSupervisor, "Pool_#{name}")
   end
 end