diff --git a/src/hex_http.erl b/src/hex_http.erl index c073481..ed95b46 100644 --- a/src/hex_http.erl +++ b/src/hex_http.erl @@ -1,7 +1,7 @@ %% @doc %% HTTP contract. -module(hex_http). --export([request/5]). +-export([request/5, request_to_file/6]). -ifdef(TEST). -export([user_agent/1]). -endif. @@ -21,26 +21,31 @@ {ok, {status(), headers(), binary()}} | {error, term()}. +-callback request_to_file( + method(), URI :: binary(), headers(), body(), file:name_all(), adapter_config() +) -> + {ok, {status(), headers()}} | {error, term()}. + -spec request(hex_core:config(), method(), URI :: binary(), headers(), body()) -> {ok, {status(), headers(), binary()}} | {error, term()}. request(Config, Method, URI, Headers, Body) when is_binary(URI) and is_map(Headers) -> - {Adapter, AdapterConfig} = - case maps:get(http_adapter, Config, {hex_http_httpc, #{}}) of - {Adapter0, AdapterConfig0} -> - {Adapter0, AdapterConfig0}; - %% TODO: remove in v0.9 - Adapter0 when is_atom(Adapter0) -> - AdapterConfig0 = maps:get(http_adapter_config, Config, #{}), - io:format( - "[hex_http] setting #{http_adapter => Module, http_adapter_config => Map} " - "is deprecated in favour of #{http_adapter => {Module, Map}}~n" - ), - {Adapter0, AdapterConfig0} - end, + {Adapter, AdapterConfig} = adapter(Config), UserAgentFragment = maps:get(http_user_agent_fragment, Config), Headers2 = put_new(<<"user-agent">>, user_agent(UserAgentFragment), Headers), Adapter:request(Method, URI, Headers2, Body, AdapterConfig). +-spec request_to_file( + hex_core:config(), method(), URI :: binary(), headers(), body(), file:name_all() +) -> + {ok, {status(), headers()}} | {error, term()}. +request_to_file(Config, Method, URI, Headers, Body, Filename) when + is_binary(URI) and is_map(Headers) +-> + {Adapter, AdapterConfig} = adapter(Config), + UserAgentFragment = maps:get(http_user_agent_fragment, Config), + Headers2 = put_new(<<"user-agent">>, user_agent(UserAgentFragment), Headers), + Adapter:request_to_file(Method, URI, Headers2, Body, Filename, AdapterConfig). + %% @private user_agent(UserAgentFragment) -> OTPRelease = erlang:system_info(otp_release), @@ -52,6 +57,21 @@ user_agent(UserAgentFragment) -> %% Internal functions %%==================================================================== +%% @private +adapter(Config) -> + case maps:get(http_adapter, Config, {hex_http_httpc, #{}}) of + {Adapter, AdapterConfig} -> + {Adapter, AdapterConfig}; + %% TODO: remove in v0.9 + Adapter when is_atom(Adapter) -> + AdapterConfig = maps:get(http_adapter_config, Config, #{}), + io:format( + "[hex_http] setting #{http_adapter => Module, http_adapter_config => Map} " + "is deprecated in favour of #{http_adapter => {Module, Map}}~n" + ), + {Adapter, AdapterConfig} + end. + %% @private put_new(Key, Value, Map) -> case maps:find(Key, Map) of diff --git a/src/hex_http_httpc.erl b/src/hex_http_httpc.erl index bb44e81..ae9cf62 100644 --- a/src/hex_http_httpc.erl +++ b/src/hex_http_httpc.erl @@ -12,7 +12,7 @@ -module(hex_http_httpc). -behaviour(hex_http). --export([request/5]). +-export([request/5, request_to_file/6]). %%==================================================================== %% API functions @@ -20,45 +20,7 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) when is_binary(URI) -> Profile = maps:get(profile, AdapterConfig, default), - HTTPOptions0 = maps:get(http_options, AdapterConfig, []), - - HTTPS = - case URI of - <<"https", _/binary>> -> true; - _ -> false - end, - SSLOpts0 = proplists:get_value(ssl, HTTPOptions0), - - HTTPOptions = - if - HTTPS == true andalso SSLOpts0 == undefined -> - %% Add safe defaults if possible. - try - [ - {ssl, [ - {verify, verify_peer}, - {cacerts, public_key:cacerts_get()}, - {depth, 3}, - {customize_hostname_check, [ - {match_fun, public_key:pkix_verify_hostname_match_fun(https)} - ]} - ]} - | HTTPOptions0 - ] - catch - _:_ -> - io:format( - "[hex_http_httpc] using default ssl options which are insecure.~n" - "Configure your adapter with: " - "{hex_http_httpc, #{http_options => [{ssl, SslOpts}]}}~n" - "or upgrade Erlang/OTP to OTP-25 or later.~n" - ), - HTTPOptions0 - end; - true -> - HTTPOptions0 - end, - + HTTPOptions = http_options(URI, AdapterConfig), Request = build_request(URI, ReqHeaders, Body), case httpc:request(Method, Request, HTTPOptions, [{body_format, binary}], Profile) of {ok, {{_, StatusCode, _}, RespHeaders, RespBody}} -> @@ -68,10 +30,71 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) when is_binary(URI) -> {error, Reason} end. +request_to_file(Method, URI, ReqHeaders, Body, Filename, AdapterConfig) when is_binary(URI) -> + Profile = maps:get(profile, AdapterConfig, default), + HTTPOptions = http_options(URI, AdapterConfig), + Request = build_request(URI, ReqHeaders, Body), + case + httpc:request( + Method, + Request, + HTTPOptions, + [{stream, unicode:characters_to_list(Filename)}], + Profile + ) + of + {ok, saved_to_file} -> + {ok, {200, #{}}}; + {ok, {{_, StatusCode, _}, RespHeaders, _RespBody}} -> + RespHeaders2 = load_headers(RespHeaders), + {ok, {StatusCode, RespHeaders2}}; + {error, Reason} -> + {error, Reason} + end. + %%==================================================================== %% Internal functions %%==================================================================== +%% @private +http_options(URI, AdapterConfig) -> + HTTPOptions0 = maps:get(http_options, AdapterConfig, []), + + HTTPS = + case URI of + <<"https", _/binary>> -> true; + _ -> false + end, + SSLOpts0 = proplists:get_value(ssl, HTTPOptions0), + + if + HTTPS == true andalso SSLOpts0 == undefined -> + try + [ + {ssl, [ + {verify, verify_peer}, + {cacerts, public_key:cacerts_get()}, + {depth, 3}, + {customize_hostname_check, [ + {match_fun, public_key:pkix_verify_hostname_match_fun(https)} + ]} + ]} + | HTTPOptions0 + ] + catch + _:_ -> + io:format( + "[hex_http_httpc] using default ssl options which are insecure.~n" + "Configure your adapter with: " + "{hex_http_httpc, #{http_options => [{ssl, SslOpts}]}}~n" + "or upgrade Erlang/OTP to OTP-25 or later.~n" + ), + HTTPOptions0 + end; + true -> + HTTPOptions0 + end. + %% @private build_request(URI, ReqHeaders, Body) -> build_request2(binary_to_list(URI), dump_headers(ReqHeaders), Body). diff --git a/src/hex_repo.erl b/src/hex_repo.erl index 08728b7..e68e312 100644 --- a/src/hex_repo.erl +++ b/src/hex_repo.erl @@ -6,7 +6,9 @@ get_versions/1, get_package/2, get_tarball/3, + get_tarball_to_file/4, get_docs/3, + get_docs_to_file/4, get_public_key/1, get_hex_installs/1 ]). @@ -91,7 +93,7 @@ get_package(Config, Name) when is_binary(Name) and is_map(Config) -> %% %% ``` %% > {ok, {200, _, Tarball}} = hex_repo:get_tarball(hex_core:default_config(), <<"package1">>, <<"1.0.0">>), -%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack(Tarball, memory). +%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack(Tarball, "/tmp/package"). %% ''' %% @end get_tarball(Config, Name, Version) -> @@ -104,6 +106,26 @@ get_tarball(Config, Name, Version) -> Other end. +%% @doc +%% Gets tarball from the repository and writes it to a file. +%% +%% Examples: +%% +%% ``` +%% > {ok, {200, _}} = hex_repo:get_tarball_to_file(hex_core:default_config(), <<"package1">>, <<"1.0.0">>, "/tmp/package.tar"), +%% > {ok, #{metadata := Metadata}} = hex_tarball:unpack({file, "/tmp/package.tar"}, "/tmp/package"). +%% ''' +%% @end +get_tarball_to_file(Config, Name, Version, Filename) -> + ReqHeaders = make_headers(Config), + + case get_to_file(Config, tarball_url(Config, Name, Version), ReqHeaders, Filename) of + {ok, {200, RespHeaders}} -> + {ok, {200, RespHeaders}}; + Other -> + Other + end. + %% @doc %% Gets docs tarball from the repository. %% @@ -111,8 +133,8 @@ get_tarball(Config, Name, Version) -> %% %% ``` %% > {ok, {200, _, Docs}} = hex_repo:get_docs(hex_core:default_config(), <<"package1">>, <<"1.0.0">>), -%% > hex_tarball:unpack_docs(Docs, memory) -%% {ok, [{"index.html", <<"">>}, ...]} +%% > hex_tarball:unpack_docs(Docs, "/tmp/docs") +%% ok %% ''' get_docs(Config, Name, Version) -> ReqHeaders = make_headers(Config), @@ -124,6 +146,25 @@ get_docs(Config, Name, Version) -> Other end. +%% @doc +%% Gets docs tarball from the repository and writes it to a file. +%% +%% Examples: +%% +%% ``` +%% > {ok, {200, _}} = hex_repo:get_docs_to_file(hex_core:default_config(), <<"package1">>, <<"1.0.0">>, "/tmp/docs.tar.gz"), +%% > ok = hex_tarball:unpack_docs({file, "/tmp/docs.tar.gz"}, "/tmp/docs"). +%% ''' +get_docs_to_file(Config, Name, Version, Filename) -> + ReqHeaders = make_headers(Config), + + case get_to_file(Config, docs_url(Config, Name, Version), ReqHeaders, Filename) of + {ok, {200, RespHeaders}} -> + {ok, {200, RespHeaders}}; + Other -> + Other + end. + %% @doc %% Gets the public key from the repository. %% @@ -173,6 +214,10 @@ get_hex_installs(Config) -> get(Config, URI, Headers) -> hex_http:request(Config, get, URI, Headers, undefined). +%% @private +get_to_file(Config, URI, Headers, Filename) -> + hex_http:request_to_file(Config, get, URI, Headers, undefined, Filename). + %% @private get_protobuf(Config, Path, Decoder) -> PublicKey = maps:get(repo_public_key, Config), diff --git a/test/hex_repo_SUITE.erl b/test/hex_repo_SUITE.erl index 7ab4a4f..6ab9556 100644 --- a/test/hex_repo_SUITE.erl +++ b/test/hex_repo_SUITE.erl @@ -24,7 +24,9 @@ all() -> get_versions_test, get_package_test, get_tarball_test, + get_tarball_to_file_test, get_docs_test, + get_docs_to_file_test, get_hex_installs_test, get_public_key_test, repo_org_not_set @@ -69,6 +71,18 @@ get_tarball_test(_Config) -> {ok, {403, _, _}} = hex_repo:get_tarball(?CONFIG, <<"ecto">>, <<"9.9.9">>), ok. +get_tarball_to_file_test(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Filename = filename:join(PrivDir, "ecto-1.0.0.tar"), + {ok, {200, #{<<"etag">> := _}}} = hex_repo:get_tarball_to_file( + ?CONFIG, <<"ecto">>, <<"1.0.0">>, Filename + ), + {ok, Tarball} = file:read_file(Filename), + {ok, _} = hex_tarball:unpack(Tarball, memory), + + {ok, {403, _}} = hex_repo:get_tarball_to_file(?CONFIG, <<"ecto">>, <<"9.9.9">>, Filename), + ok. + get_docs_test(_Config) -> {ok, {200, #{<<"etag">> := ETag}, Docs}} = hex_repo:get_docs(?CONFIG, <<"ecto">>, <<"1.0.0">>), {ok, _} = hex_tarball:unpack_docs(Docs, memory), @@ -80,6 +94,18 @@ get_docs_test(_Config) -> {ok, {403, _, _}} = hex_repo:get_docs(?CONFIG, <<"ecto">>, <<"9.9.9">>), ok. +get_docs_to_file_test(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + Filename = filename:join(PrivDir, "ecto-1.0.0-docs.tar.gz"), + {ok, {200, #{<<"etag">> := _}}} = hex_repo:get_docs_to_file( + ?CONFIG, <<"ecto">>, <<"1.0.0">>, Filename + ), + {ok, Docs} = file:read_file(Filename), + {ok, _} = hex_tarball:unpack_docs(Docs, memory), + + {ok, {403, _}} = hex_repo:get_docs_to_file(?CONFIG, <<"ecto">>, <<"9.9.9">>, Filename), + ok. + get_hex_installs_test(_Config) -> {ok, {200, _, CSV}} = hex_repo:get_hex_installs(?CONFIG), ?assert(is_binary(CSV)), diff --git a/test/support/hex_http_test.erl b/test/support/hex_http_test.erl index 8de42bf..ecd6c1d 100644 --- a/test/support/hex_http_test.erl +++ b/test/support/hex_http_test.erl @@ -1,6 +1,6 @@ -module(hex_http_test). -behaviour(hex_http). --export([request/5]). +-export([request/5, request_to_file/6]). -define(TEST_REPO_URL, "https://repo.test"). -define(TEST_API_URL, "https://api.test"). -define(PRIVATE_KEY, ct:get_config({ssl_certs, test_priv})). @@ -13,6 +13,17 @@ request(Method, URI, Headers, Body, _Options) when is_binary(URI) and is_map(Headers) -> fixture(Method, URI, Headers, Body). +request_to_file(Method, URI, Headers, Body, Filename, Options) -> + case request(Method, URI, Headers, Body, Options) of + {ok, {200, RespHeaders, RespBody}} -> + ok = file:write_file(Filename, RespBody), + {ok, {200, RespHeaders}}; + {ok, {StatusCode, RespHeaders, _RespBody}} -> + {ok, {StatusCode, RespHeaders}}; + {error, _} = Error -> + Error + end. + %%==================================================================== %% Internal functions %%====================================================================