From 5de97a18f900c3d0fe15ae51cbcf96aec4b4518c Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Mon, 1 Jun 2026 23:36:34 +0200 Subject: [PATCH 1/2] test(storage): cover GCS HTTPS OpenDAL TLS regression --- quickwit/Cargo.lock | 3 + quickwit/quickwit-storage/Cargo.toml | 6 + .../src/opendal_storage/base.rs | 22 ++- .../opendal_storage/google_cloud_storage.rs | 173 +++++++++++++++++- .../tests/test_data/regenerate-gcs-certs.sh | 60 ++++++ 5 files changed, 261 insertions(+), 3 deletions(-) create mode 100755 quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 29e7757952a..e7a320945f7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -9482,6 +9482,8 @@ dependencies = [ "quickwit-proto", "regex", "reqwest 0.12.28", + "reqwest 0.13.3", + "rustls 0.23.40", "serde", "serde_json", "stable_deref_trait", @@ -9489,6 +9491,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls 0.26.4", "tokio-stream", "tokio-util", "tracing", diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 8fe4701f631..f777c25a7ce 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -62,7 +62,13 @@ reqwest = { workspace = true, optional = true } http = { workspace = true } mockall = { workspace = true } proptest = { workspace = true } +# Match OpenDAL's internal reqwest major. `default-features = false` is +# intentional: the HTTPS regression test should get TLS only from OpenDAL's +# `reqwest-rustls-tls` feature, not from this dev-dependency. +reqwest-013 = { package = "reqwest", version = "0.13", default-features = false } +rustls = { workspace = true } tokio = { workspace = true } +tokio-rustls = { workspace = true } tracing-subscriber = { workspace = true } aws-sdk-s3 = { workspace = true } diff --git a/quickwit/quickwit-storage/src/opendal_storage/base.rs b/quickwit/quickwit-storage/src/opendal_storage/base.rs index a6933db9178..d0fac5e9e7a 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/base.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/base.rs @@ -60,12 +60,30 @@ impl OpendalStorage { cfg: opendal::services::Gcs, ) -> Result { let op = Operator::new(cfg)?.finish(); - Ok(Self { + Ok(Self::from_operator(uri, op)) + } + + fn from_operator(uri: Uri, op: Operator) -> Self { + Self { uri, op, // limits are the same as on S3 multipart_policy: MultiPartPolicy::default(), - }) + } + } + + #[cfg(test)] + // Lets local HTTPS tests trust a private CA without changing global trust, + // while still using Quickwit's GCS storage construction and read path. + pub(super) fn new_google_cloud_storage_with_http_client_for_test( + uri: Uri, + cfg: opendal::services::Gcs, + http_client: opendal::raw::HttpClient, + ) -> Result { + let op = Operator::new(cfg)? + .layer(opendal::layers::HttpClientLayer::new(http_client)) + .finish(); + Ok(Self::from_operator(uri, op)) } #[cfg(feature = "integration-testsuite")] diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index 58971a432b7..23397e90a7c 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -113,9 +113,57 @@ fn parse_google_uri(uri: &Uri) -> Option<(String, PathBuf)> { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::path::Path; + use std::sync::Arc; + + use base64::Engine; + use opendal::raw::HttpClient; use quickwit_common::uri::Uri; + use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivateSec1KeyDer}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio::task::JoinHandle; + + use super::{OpendalStorage, parse_google_uri}; + use crate::Storage; + + // Test-only CA and server certificate for 127.0.0.1/localhost, valid from + // 2020 to 3020. The client trusts only this CA, so the test never depends + // on the host root store. + // Regenerate with `quickwit-storage/tests/test_data/regenerate-gcs-certs.sh`. + const TEST_CA_CERT_DER_BASE64: &str = concat!( + "MIIBizCCATGgAwIBAgICEAEwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", + "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowGzEZ", + "MBcGA1UEAwwQUXVpY2t3aXQgVGVzdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEH", + "A0IABH+1ZvivhT0E5FydtoMGBkyenql8XPyFTPBhTfHycTjfTWJiETjILGadPLKY", + "OZJky8ThPZUpKAux5M4SaazdX1WjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0P", + "AQH/BAQDAgEGMB0GA1UdDgQWBBQmOMvIHAegmBHwvdVGyguC/57/4zAfBgNVHSME", + "GDAWgBQmOMvIHAegmBHwvdVGyguC/57/4zAKBggqhkjOPQQDAgNIADBFAiEAnE7M", + "lcB35MOr+7WKDAhu/c6ZrpgRz+chqqfc3g5YTOECIEDmoPkOigkulNON67opCPaT", + "y+MQhMA9KDEzE3t/CY9V", + ); + + const TEST_SERVER_CERT_DER_BASE64: &str = concat!( + "MIIBszCCAVqgAwIBAgICEAIwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", + "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowFDES", + "MBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEowPj", + "3vpXPAkf04MNeGhaDBvtwMsmeipV57lSWx5K2FwXH7JDmt74k4HmQFB6JESy6FbM", + "tAVhivr7kG5dWKK/sqOBkjCBjzAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIH", + "gDATBgNVHSUEDDAKBggrBgEFBQcDATAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A", + "AAEwHQYDVR0OBBYEFES7BP5uQpa3+PktDVlgc9zYGIqDMB8GA1UdIwQYMBaAFCY4", + "y8gcB6CYEfC91UbKC4L/nv/jMAoGCCqGSM49BAMCA0cAMEQCIGtIKEWRn7ec82TY", + "s1jrUoKWnhzRDbZTUtvXORk190rHAiAosxVgu45TjDyuROKU39TxJ1z+JObhNGk8", + "J6PkuOTFqg==", + ); + + const TEST_SERVER_KEY_DER_BASE64: &str = concat!( + "MHcCAQEEIIql19flBaZJE16Ivs8GjdJHedhuU5YFZgvIn4WaOs6HoAoGCCqGSM49", + "AwEHoUQDQgAEowPj3vpXPAkf04MNeGhaDBvtwMsmeipV57lSWx5K2FwXH7JDmt74", + "k4HmQFB6JESy6FbMtAVhivr7kG5dWKK/sg==", + ); - use super::parse_google_uri; + type LocalHttpsGcsServer = (String, JoinHandle>); #[test] fn test_parse_google_uri() { @@ -134,4 +182,127 @@ mod tests { assert_eq!(bucket, "test-bucket"); assert_eq!(prefix.to_str().unwrap(), "indexes"); } + + #[tokio::test] + async fn test_gcs_storage_get_slice_over_https_with_verified_tls() -> anyhow::Result<()> { + // Nextest runs tests in separate processes, so this test must not rely + // on another rustls user having already selected a process-wide provider. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let (endpoint, server_task) = start_local_https_gcs_server().await?; + let ca_cert_der = decode_test_der(TEST_CA_CERT_DER_BASE64)?; + let ca_cert = reqwest_013::Certificate::from_der(&ca_cert_der)?; + let reqwest_client = reqwest_013::Client::builder() + .no_proxy() + .tls_certs_only([ca_cert]) + .build()?; + + let cfg = opendal::services::Gcs::default() + .bucket("quickwit-test-bucket") + .endpoint(&endpoint) + .allow_anonymous() + .disable_config_load() + .disable_vm_metadata(); + let storage = OpendalStorage::new_google_cloud_storage_with_http_client_for_test( + Uri::for_test("gs://quickwit-test-bucket"), + cfg, + HttpClient::with(reqwest_client), + )?; + + let bytes = storage.get_slice(Path::new("hello.txt"), 0..2).await?; + assert_eq!(bytes.as_slice(), b"ok"); + server_task.await??; + Ok(()) + } + + async fn start_local_https_gcs_server() -> anyhow::Result { + let cert_chain = vec![CertificateDer::from(decode_test_der( + TEST_SERVER_CERT_DER_BASE64, + )?)]; + let private_key = PrivateKeyDer::Sec1(PrivateSec1KeyDer::from(decode_test_der( + TEST_SERVER_KEY_DER_BASE64, + )?)); + let tls_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert_chain, private_key)?; + let tls_acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(tls_config)); + let listener = TcpListener::bind(("127.0.0.1", 0)).await?; + let endpoint = format!("https://127.0.0.1:{}", listener.local_addr()?.port()); + + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await?; + let mut stream = tls_acceptor.accept(stream).await?; + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + loop { + let bytes_read = stream.read(&mut buffer).await?; + if bytes_read == 0 { + break; + } + request.extend_from_slice(&buffer[..bytes_read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request); + let (header_block, _) = request + .split_once("\r\n\r\n") + .expect("request must contain HTTP header terminator"); + let mut header_lines = header_block.lines(); + let request_line = header_lines.next().expect("request line must be present"); + let mut request_line_parts = request_line.split_whitespace(); + let method = request_line_parts.next(); + let target = request_line_parts.next(); + let version = request_line_parts.next(); + assert_eq!( + method, + Some("GET"), + "unexpected request line: {request_line}" + ); + assert_eq!( + target, + Some("/storage/v1/b/quickwit-test-bucket/o/hello.txt?alt=media"), + "unexpected GCS request target: {request_line}" + ); + assert!( + matches!(version, Some(version) if version.starts_with("HTTP/")), + "unexpected HTTP version in request line: {request_line}" + ); + assert_eq!( + request_line_parts.next(), + None, + "unexpected extra request line segment: {request_line}" + ); + + let headers: BTreeMap = header_lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) + .collect(); + assert_eq!( + headers.get("range").map(String::as_str), + Some("bytes=0-1"), + "expected range read request header: {request}" + ); + + stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\n\ + Content-Length: 2\r\n\ + Content-Range: bytes 0-1/2\r\n\ + Accept-Ranges: bytes\r\n\ + Connection: close\r\n\ + \r\n\ + ok", + ) + .await?; + stream.shutdown().await?; + Ok(()) + }); + + Ok((endpoint, server_task)) + } + + fn decode_test_der(base64_der: &str) -> anyhow::Result> { + Ok(base64::engine::general_purpose::STANDARD.decode(base64_der)?) + } } diff --git a/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh new file mode 100755 index 00000000000..ea8c200570c --- /dev/null +++ b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Regenerate the DER-encoded test CA, server certificate, and server private +# key embedded in `src/opendal_storage/google_cloud_storage.rs`. + +set -euo pipefail + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/ca.conf" <<'EOF' +[req] +distinguished_name=dn +[dn] +[v3_ca] +basicConstraints=critical,CA:true +keyUsage=critical,keyCertSign,cRLSign +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always +EOF + +openssl req -x509 -newkey ec \ + -pkeyopt ec_paramgen_curve:prime256v1 \ + -nodes -subj "/CN=Quickwit Test CA" \ + -set_serial 0x1001 \ + -not_before 20200101000000Z \ + -not_after 30200101000000Z \ + -extensions v3_ca -config "$tmp_dir/ca.conf" \ + -keyout "$tmp_dir/ca.key" -out "$tmp_dir/ca.crt" 2>/dev/null + +cat > "$tmp_dir/server.conf" <<'EOF' +[req] +distinguished_name=dn +[dn] +[v3_server] +basicConstraints=critical,CA:false +keyUsage=critical,digitalSignature +extendedKeyUsage=serverAuth +subjectAltName=DNS:localhost,IP:127.0.0.1 +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +EOF + +openssl req -x509 -newkey ec \ + -pkeyopt ec_paramgen_curve:prime256v1 \ + -nodes -subj "/CN=localhost" \ + -CA "$tmp_dir/ca.crt" -CAkey "$tmp_dir/ca.key" \ + -set_serial 0x1002 \ + -not_before 20200101000000Z \ + -not_after 30200101000000Z \ + -extensions v3_server -config "$tmp_dir/server.conf" \ + -keyout "$tmp_dir/server.key" -out "$tmp_dir/server.crt" 2>/dev/null + +printf "CA_DER=" +openssl x509 -in "$tmp_dir/ca.crt" -outform der | base64 | tr -d "\n" +printf "\nSERVER_DER=" +openssl x509 -in "$tmp_dir/server.crt" -outform der | base64 | tr -d "\n" +printf "\nSERVER_KEY_DER=" +openssl ec -in "$tmp_dir/server.key" -outform der 2>/dev/null | base64 | tr -d "\n" +printf "\n" From 4eae68ff26cfdbbb742943f85dd6bcf5a63164f1 Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Tue, 2 Jun 2026 00:53:56 +0200 Subject: [PATCH 2/2] test(storage): make GCS cert regeneration portable --- .../tests/test_data/regenerate-gcs-certs.sh | 79 +++++++++++++------ 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh index ea8c200570c..7f64a3ab402 100755 --- a/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh +++ b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh @@ -8,30 +8,47 @@ set -euo pipefail tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' EXIT -cat > "$tmp_dir/ca.conf" <<'EOF' +cert_not_before=20200101000000Z +cert_not_after=30200101000000Z + +mkdir "$tmp_dir/certs" +: > "$tmp_dir/index.txt" +export OPENSSL_TEST_DIR="$tmp_dir" + +run_quietly() { + local stderr_file="$tmp_dir/openssl-stderr.log" + if ! "$@" >/dev/null 2>"$stderr_file"; then + cat "$stderr_file" >&2 + return 1 + fi +} + +cat > "$tmp_dir/openssl.conf" <<'EOF' +[ca] +default_ca=test_ca + +[test_ca] +dir=$ENV::OPENSSL_TEST_DIR +database=$dir/index.txt +new_certs_dir=$dir/certs +serial=$dir/serial +default_md=sha256 +policy=test_policy + +[test_policy] +commonName=supplied + [req] distinguished_name=dn + [dn] + [v3_ca] basicConstraints=critical,CA:true keyUsage=critical,keyCertSign,cRLSign subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always -EOF - -openssl req -x509 -newkey ec \ - -pkeyopt ec_paramgen_curve:prime256v1 \ - -nodes -subj "/CN=Quickwit Test CA" \ - -set_serial 0x1001 \ - -not_before 20200101000000Z \ - -not_after 30200101000000Z \ - -extensions v3_ca -config "$tmp_dir/ca.conf" \ - -keyout "$tmp_dir/ca.key" -out "$tmp_dir/ca.crt" 2>/dev/null -cat > "$tmp_dir/server.conf" <<'EOF' -[req] -distinguished_name=dn -[dn] [v3_server] basicConstraints=critical,CA:false keyUsage=critical,digitalSignature @@ -41,15 +58,29 @@ subjectKeyIdentifier=hash authorityKeyIdentifier=keyid,issuer EOF -openssl req -x509 -newkey ec \ - -pkeyopt ec_paramgen_curve:prime256v1 \ - -nodes -subj "/CN=localhost" \ - -CA "$tmp_dir/ca.crt" -CAkey "$tmp_dir/ca.key" \ - -set_serial 0x1002 \ - -not_before 20200101000000Z \ - -not_after 30200101000000Z \ - -extensions v3_server -config "$tmp_dir/server.conf" \ - -keyout "$tmp_dir/server.key" -out "$tmp_dir/server.crt" 2>/dev/null +openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/ca.key" +openssl req -new -key "$tmp_dir/ca.key" -subj "/CN=Quickwit Test CA" \ + -out "$tmp_dir/ca.csr" -config "$tmp_dir/openssl.conf" + +printf "1001\n" > "$tmp_dir/serial" +run_quietly openssl ca -batch -selfsign -config "$tmp_dir/openssl.conf" \ + -in "$tmp_dir/ca.csr" -keyfile "$tmp_dir/ca.key" \ + -out "$tmp_dir/ca.crt" \ + -extensions v3_ca \ + -startdate "$cert_not_before" -enddate "$cert_not_after" \ + -notext + +openssl ecparam -name prime256v1 -genkey -noout -out "$tmp_dir/server.key" +openssl req -new -key "$tmp_dir/server.key" -subj "/CN=localhost" \ + -out "$tmp_dir/server.csr" -config "$tmp_dir/openssl.conf" + +printf "1002\n" > "$tmp_dir/serial" +run_quietly openssl ca -batch -config "$tmp_dir/openssl.conf" \ + -cert "$tmp_dir/ca.crt" -keyfile "$tmp_dir/ca.key" \ + -in "$tmp_dir/server.csr" -out "$tmp_dir/server.crt" \ + -extensions v3_server \ + -startdate "$cert_not_before" -enddate "$cert_not_after" \ + -notext printf "CA_DER=" openssl x509 -in "$tmp_dir/ca.crt" -outform der | base64 | tr -d "\n"