Add request_to_file to HTTP contract for streaming downloads to disk (#169)

ericmj · web-flow · commit c6a399571c3b · 2026-03-09T22:33:14.000+01:00
Add a new `request_to_file` callback to `hex_http` that streams the HTTP
response body directly to a file, avoiding loading the full body into memory.

Add `get_tarball_to_file/4` and `get_docs_to_file/4` to `hex_repo` that use
the new callback to download tarballs and docs directly to disk.
diff --git a/src/hex_http.erl b/src/hex_http.erl
@@ -1,7 +1,7 @@
 %% @doc
 %% HTTP contract.
 -module(hex_http).
--export([request/5]).
+-export([request/5, request_to_file/6]).
 -ifdef(TEST).
 -export([user_agent/1]).
 -endif.
@@ -21,26 +21,31 @@
     {ok, {status(), headers(), binary()}}
     | {error, term()}.
 
+-callback request_to_file(
+    method(), URI :: binary(), headers(), body(), file:name_all(), adapter_config()
+) ->
+    {ok, {status(), headers()}} | {error, term()}.
+
 -spec request(hex_core:config(), method(), URI :: binary(), headers(), body()) ->
     {ok, {status(), headers(), binary()}} | {error, term()}.
 request(Config, Method, URI, Headers, Body) when is_binary(URI) and is_map(Headers) ->
-    {Adapter, AdapterConfig} =
-        case maps:get(http_adapter, Config, {hex_http_httpc, #{}}) of
-            {Adapter0, AdapterConfig0} ->
-                {Adapter0, AdapterConfig0};
-            %% TODO: remove in v0.9
-            Adapter0 when is_atom(Adapter0) ->
-                AdapterConfig0 = maps:get(http_adapter_config, Config, #{}),
-                io:format(
-                    "[hex_http] setting #{http_adapter => Module, http_adapter_config => Map} "
-                    "is deprecated in favour of #{http_adapter => {Module, Map}}~n"
-                ),
-                {Adapter0, AdapterConfig0}
-        end,
+    {Adapter, AdapterConfig} = adapter(Config),
     UserAgentFragment = maps:get(http_user_agent_fragment, Config),
     Headers2 = put_new(<<"user-agent">>, user_agent(UserAgentFragment), Headers),
     Adapter:request(Method, URI, Headers2, Body, AdapterConfig).
 
+-spec request_to_file(
+    hex_core:config(), method(), URI :: binary(), headers(), body(), file:name_all()
+) ->
+    {ok, {status(), headers()}} | {error, term()}.
+request_to_file(Config, Method, URI, Headers, Body, Filename) when
+    is_binary(URI) and is_map(Headers)
+->
+    {Adapter, AdapterConfig} = adapter(Config),
+    UserAgentFragment = maps:get(http_user_agent_fragment, Config),
+    Headers2 = put_new(<<"user-agent">>, user_agent(UserAgentFragment), Headers),
+    Adapter:request_to_file(Method, URI, Headers2, Body, Filename, AdapterConfig).
+
 %% @private
 user_agent(UserAgentFragment) ->
     OTPRelease = erlang:system_info(otp_release),
@@ -52,6 +57,21 @@ user_agent(UserAgentFragment) ->
 %% Internal functions
 %%====================================================================
 
+%% @private
+adapter(Config) ->
+    case maps:get(http_adapter, Config, {hex_http_httpc, #{}}) of
+        {Adapter, AdapterConfig} ->
+            {Adapter, AdapterConfig};
+        %% TODO: remove in v0.9
+        Adapter when is_atom(Adapter) ->
+            AdapterConfig = maps:get(http_adapter_config, Config, #{}),
+            io:format(
+                "[hex_http] setting #{http_adapter => Module, http_adapter_config => Map} "
+                "is deprecated in favour of #{http_adapter => {Module, Map}}~n"
+            ),
+            {Adapter, AdapterConfig}
+    end.
+
 %% @private
 put_new(Key, Value, Map) ->
     case maps:find(Key, Map) of
diff --git a/src/hex_http_httpc.erl b/src/hex_http_httpc.erl
@@ -12,53 +12,15 @@
 
 -module(hex_http_httpc).
 -behaviour(hex_http).
--export([request/5]).
+-export([request/5, request_to_file/6]).
 
 %%====================================================================
 %% API functions
 %%====================================================================
 
 request(Method, URI, ReqHeaders, Body, AdapterConfig) when is_binary(URI) ->
     Profile = maps:get(profile, AdapterConfig, default),
-    HTTPOptions0 = maps:get(http_options, AdapterConfig, []),
-
-    HTTPS =
-        case URI of
-            <<"https", _/binary>> -> true;
-            _ -> false
-        end,
-    SSLOpts0 = proplists:get_value(ssl, HTTPOptions0),
-
-    HTTPOptions =
-        if
-            HTTPS == true andalso SSLOpts0 == undefined ->
-                %% Add safe defaults if possible.
-                try
-                    [
-                        {ssl, [
-                            {verify, verify_peer},
-                            {cacerts, public_key:cacerts_get()},
-                            {depth, 3},
-                            {customize_hostname_check, [
-                                {match_fun, public_key:pkix_verify_hostname_match_fun(https)}
-                            ]}
-                        ]}
-                        | HTTPOptions0
-                    ]
-                catch
-                    _:_ ->
-                        io:format(
-                            "[hex_http_httpc] using default ssl options which are insecure.~n"
-                            "Configure your adapter with: "
-                            "{hex_http_httpc, #{http_options => [{ssl, SslOpts}]}}~n"
-                            "or upgrade Erlang/OTP to OTP-25 or later.~n"
-                        ),
-                        HTTPOptions0
-                end;
-            true ->
-                HTTPOptions0
-        end,
-
+    HTTPOptions = http_options(URI, AdapterConfig),
     Request = build_request(URI, ReqHeaders, Body),
     case httpc:request(Method, Request, HTTPOptions, [{body_format, binary}], Profile) of
         {ok, {{_, StatusCode, _}, RespHeaders, RespBody}} ->
@@ -68,10 +30,71 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) when is_binary(URI) ->
             {error, Reason}
     end.
 
+request_to_file(Method, URI, ReqHeaders, Body, Filename, AdapterConfig) when is_binary(URI) ->
+    Profile = maps:get(profile, AdapterConfig, default),
+    HTTPOptions = http_options(URI, AdapterConfig),
+    Request = build_request(URI, ReqHeaders, Body),
+    case
+        httpc:request(
+            Method,
+            Request,
+            HTTPOptions,
+            [{stream, unicode:characters_to_list(Filename)}],
+            Profile
+        )
+    of
+        {ok, saved_to_file} ->
+            {ok, {200, #{}}};
+        {ok, {{_, StatusCode, _}, RespHeaders, _RespBody}} ->
+            RespHeaders2 = load_headers(RespHeaders),
+            {ok, {StatusCode, RespHeaders2}};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
 %%====================================================================
 %% Internal functions
 %%====================================================================
 
+%% @private
+http_options(URI, AdapterConfig) ->
+    HTTPOptions0 = maps:get(http_options, AdapterConfig, []),
+
+    HTTPS =
+        case URI of
+            <<"https", _/binary>> -> true;
+            _ -> false
+        end,
+    SSLOpts0 = proplists:get_value(ssl, HTTPOptions0),
+
+    if
+        HTTPS == true andalso SSLOpts0 == undefined ->
+            try
+                [
+                    {ssl, [
+                        {verify, verify_peer},
+                        {cacerts, public_key:cacerts_get()},
+                        {depth, 3},
+                        {customize_hostname_check, [
+                            {match_fun, public_key:pkix_verify_hostname_match_fun(https)}
+                        ]}
+                    ]}
+                    | HTTPOptions0
+                ]
+            catch
+                _:_ ->
+                    io:format(
+                        "[hex_http_httpc] using default ssl options which are insecure.~n"
+                        "Configure your adapter with: "
+                        "{hex_http_httpc, #{http_options => [{ssl, SslOpts}]}}~n"
+                        "or upgrade Erlang/OTP to OTP-25 or later.~n"
+                    ),
+                    HTTPOptions0
+            end;
+        true ->
+            HTTPOptions0
+    end.
+
 %% @private
 build_request(URI, ReqHeaders, Body) ->
     build_request2(binary_to_list(URI), dump_headers(ReqHeaders), Body).
diff --git a/src/hex_repo.erl b/src/hex_repo.erl
@@ -6,7 +6,9 @@
     get_versions/1,
     get_package/2,
     get_tarball/3,
+    get_tarball_to_file/4,
     get_docs/3,
+    get_docs_to_file/4,
     get_public_key/1,
     get_hex_installs/1
 ]).
@@ -91,7 +93,7 @@ get_package(Config, Name) when is_binary(Name) and is_map(Config) ->
 %%
 %% ```
 %% > {ok, {200, _, Tarball}} = hex_repo:get_tarball(hex_core:default_config(), <<"package1">>, <<"1.0.0">>),
-%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack(Tarball, memory).
+%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack(Tarball, "/tmp/package").
 %% '''
 %% @end
 get_tarball(Config, Name, Version) ->
@@ -104,15 +106,35 @@ get_tarball(Config, Name, Version) ->
             Other
     end.
 
+%% @doc
+%% Gets tarball from the repository and writes it to a file.
+%%
+%% Examples:
+%%
+%% ```
+%% > {ok, {200, _}} = hex_repo:get_tarball_to_file(hex_core:default_config(), <<"package1">>, <<"1.0.0">>, "/tmp/package.tar"),
+%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack({file, "/tmp/package.tar"}, "/tmp/package").
+%% '''
+%% @end
+get_tarball_to_file(Config, Name, Version, Filename) ->
+    ReqHeaders = make_headers(Config),
+
+    case get_to_file(Config, tarball_url(Config, Name, Version), ReqHeaders, Filename) of
+        {ok, {200, RespHeaders}} ->
+            {ok, {200, RespHeaders}};
+        Other ->
+            Other
+    end.
+
 %% @doc
 %% Gets docs tarball from the repository.
 %%
 %% Examples:
 %%
 %% ```
 %% > {ok, {200, _, Docs}} = hex_repo:get_docs(hex_core:default_config(), <<"package1">>, <<"1.0.0">>),
-%% > hex_tarball:unpack_docs(Docs, memory)
-%% {ok, [{"index.html", <<"<!doctype>">>}, ...]}
+%% > hex_tarball:unpack_docs(Docs, "/tmp/docs")
+%% ok
 %% '''
 get_docs(Config, Name, Version) ->
     ReqHeaders = make_headers(Config),
@@ -124,6 +146,25 @@ get_docs(Config, Name, Version) ->
             Other
     end.
 
+%% @doc
+%% Gets docs tarball from the repository and writes it to a file.
+%%
+%% Examples:
+%%
+%% ```
+%% > {ok, {200, _}} = hex_repo:get_docs_to_file(hex_core:default_config(), <<"package1">>, <<"1.0.0">>, "/tmp/docs.tar.gz"),
+%% > ok = hex_tarball:unpack_docs({file, "/tmp/docs.tar.gz"}, "/tmp/docs").
+%% '''
+get_docs_to_file(Config, Name, Version, Filename) ->
+    ReqHeaders = make_headers(Config),
+
+    case get_to_file(Config, docs_url(Config, Name, Version), ReqHeaders, Filename) of
+        {ok, {200, RespHeaders}} ->
+            {ok, {200, RespHeaders}};
+        Other ->
+            Other
+    end.
+
 %% @doc
 %% Gets the public key from the repository.
 %%
@@ -173,6 +214,10 @@ get_hex_installs(Config) ->
 get(Config, URI, Headers) ->
     hex_http:request(Config, get, URI, Headers, undefined).
 
+%% @private
+get_to_file(Config, URI, Headers, Filename) ->
+    hex_http:request_to_file(Config, get, URI, Headers, undefined, Filename).
+
 %% @private
 get_protobuf(Config, Path, Decoder) ->
     PublicKey = maps:get(repo_public_key, Config),
diff --git a/test/hex_repo_SUITE.erl b/test/hex_repo_SUITE.erl
@@ -24,7 +24,9 @@ all() ->
         get_versions_test,
         get_package_test,
         get_tarball_test,
+        get_tarball_to_file_test,
         get_docs_test,
+        get_docs_to_file_test,
         get_hex_installs_test,
         get_public_key_test,
         repo_org_not_set
@@ -69,6 +71,18 @@ get_tarball_test(_Config) ->
     {ok, {403, _, _}} = hex_repo:get_tarball(?CONFIG, <<"ecto">>, <<"9.9.9">>),
     ok.
 
+get_tarball_to_file_test(Config) ->
+    PrivDir = proplists:get_value(priv_dir, Config),
+    Filename = filename:join(PrivDir, "ecto-1.0.0.tar"),
+    {ok, {200, #{<<"etag">> := _}}} = hex_repo:get_tarball_to_file(
+        ?CONFIG, <<"ecto">>, <<"1.0.0">>, Filename
+    ),
+    {ok, Tarball} = file:read_file(Filename),
+    {ok, _} = hex_tarball:unpack(Tarball, memory),
+
+    {ok, {403, _}} = hex_repo:get_tarball_to_file(?CONFIG, <<"ecto">>, <<"9.9.9">>, Filename),
+    ok.
+
 get_docs_test(_Config) ->
     {ok, {200, #{<<"etag">> := ETag}, Docs}} = hex_repo:get_docs(?CONFIG, <<"ecto">>, <<"1.0.0">>),
     {ok, _} = hex_tarball:unpack_docs(Docs, memory),
@@ -80,6 +94,18 @@ get_docs_test(_Config) ->
     {ok, {403, _, _}} = hex_repo:get_docs(?CONFIG, <<"ecto">>, <<"9.9.9">>),
     ok.
 
+get_docs_to_file_test(Config) ->
+    PrivDir = proplists:get_value(priv_dir, Config),
+    Filename = filename:join(PrivDir, "ecto-1.0.0-docs.tar.gz"),
+    {ok, {200, #{<<"etag">> := _}}} = hex_repo:get_docs_to_file(
+        ?CONFIG, <<"ecto">>, <<"1.0.0">>, Filename
+    ),
+    {ok, Docs} = file:read_file(Filename),
+    {ok, _} = hex_tarball:unpack_docs(Docs, memory),
+
+    {ok, {403, _}} = hex_repo:get_docs_to_file(?CONFIG, <<"ecto">>, <<"9.9.9">>, Filename),
+    ok.
+
 get_hex_installs_test(_Config) ->
     {ok, {200, _, CSV}} = hex_repo:get_hex_installs(?CONFIG),
     ?assert(is_binary(CSV)),
diff --git a/test/support/hex_http_test.erl b/test/support/hex_http_test.erl
@@ -1,6 +1,6 @@
 -module(hex_http_test).
 -behaviour(hex_http).
--export([request/5]).
+-export([request/5, request_to_file/6]).
 -define(TEST_REPO_URL, "https://repo.test").
 -define(TEST_API_URL, "https://api.test").
 -define(PRIVATE_KEY, ct:get_config({ssl_certs, test_priv})).
@@ -13,6 +13,17 @@
 request(Method, URI, Headers, Body, _Options) when is_binary(URI) and is_map(Headers) ->
     fixture(Method, URI, Headers, Body).
 
+request_to_file(Method, URI, Headers, Body, Filename, Options) ->
+    case request(Method, URI, Headers, Body, Options) of
+        {ok, {200, RespHeaders, RespBody}} ->
+            ok = file:write_file(Filename, RespBody),
+            {ok, {200, RespHeaders}};
+        {ok, {StatusCode, RespHeaders, _RespBody}} ->
+            {ok, {StatusCode, RespHeaders}};
+        {error, _} = Error ->
+            Error
+    end.
+
 %%====================================================================
 %% Internal functions
 %%====================================================================
