From f7329bb93ad022a25911e9813d63462ab6a19a01 Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Sat, 30 May 2026 14:38:29 +0200 Subject: [PATCH 1/5] test(storage): cover GCS HTTPS OpenDAL TLS --- quickwit/Cargo.lock | 3 + quickwit/quickwit-storage/Cargo.toml | 6 + .../src/opendal_storage/base.rs | 22 ++- .../opendal_storage/google_cloud_storage.rs | 170 +++++++++++++++++- 4 files changed, 198 insertions(+), 3 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 29e7757952a..e7a320945f7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -9482,6 +9482,8 @@ dependencies = [ "quickwit-proto", "regex", "reqwest 0.12.28", + "reqwest 0.13.3", + "rustls 0.23.40", "serde", "serde_json", "stable_deref_trait", @@ -9489,6 +9491,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls 0.26.4", "tokio-stream", "tokio-util", "tracing", diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 8fe4701f631..f777c25a7ce 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -62,7 +62,13 @@ reqwest = { workspace = true, optional = true } http = { workspace = true } mockall = { workspace = true } proptest = { workspace = true } +# Match OpenDAL's internal reqwest major. `default-features = false` is +# intentional: the HTTPS regression test should get TLS only from OpenDAL's +# `reqwest-rustls-tls` feature, not from this dev-dependency. +reqwest-013 = { package = "reqwest", version = "0.13", default-features = false } +rustls = { workspace = true } tokio = { workspace = true } +tokio-rustls = { workspace = true } tracing-subscriber = { workspace = true } aws-sdk-s3 = { workspace = true } diff --git a/quickwit/quickwit-storage/src/opendal_storage/base.rs b/quickwit/quickwit-storage/src/opendal_storage/base.rs index a6933db9178..d0fac5e9e7a 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/base.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/base.rs @@ -60,12 +60,30 @@ impl OpendalStorage { cfg: opendal::services::Gcs, ) -> Result { let op = Operator::new(cfg)?.finish(); - Ok(Self { + Ok(Self::from_operator(uri, op)) + } + + fn from_operator(uri: Uri, op: Operator) -> Self { + Self { uri, op, // limits are the same as on S3 multipart_policy: MultiPartPolicy::default(), - }) + } + } + + #[cfg(test)] + // Lets local HTTPS tests trust a private CA without changing global trust, + // while still using Quickwit's GCS storage construction and read path. + pub(super) fn new_google_cloud_storage_with_http_client_for_test( + uri: Uri, + cfg: opendal::services::Gcs, + http_client: opendal::raw::HttpClient, + ) -> Result { + let op = Operator::new(cfg)? + .layer(opendal::layers::HttpClientLayer::new(http_client)) + .finish(); + Ok(Self::from_operator(uri, op)) } #[cfg(feature = "integration-testsuite")] diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index 58971a432b7..39a1d9fa3e8 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -113,9 +113,58 @@ fn parse_google_uri(uri: &Uri) -> Option<(String, PathBuf)> { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::path::Path; + use std::sync::Arc; + + use base64::Engine; + use opendal::raw::HttpClient; use quickwit_common::uri::Uri; + use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivateSec1KeyDer}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio::task::JoinHandle; + + use super::{OpendalStorage, parse_google_uri}; + use crate::Storage; + + // Test-only CA and server certificate for 127.0.0.1/localhost, valid from + // 2020 to 3020. The client trusts only this CA, so the test never depends + // on the host root store. + // Regenerate with: + // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` + const TEST_CA_CERT_DER_BASE64: &str = concat!( + "MIIBizCCATGgAwIBAgICEAEwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", + "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowGzEZ", + "MBcGA1UEAwwQUXVpY2t3aXQgVGVzdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEH", + "A0IABH+1ZvivhT0E5FydtoMGBkyenql8XPyFTPBhTfHycTjfTWJiETjILGadPLKY", + "OZJky8ThPZUpKAux5M4SaazdX1WjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0P", + "AQH/BAQDAgEGMB0GA1UdDgQWBBQmOMvIHAegmBHwvdVGyguC/57/4zAfBgNVHSME", + "GDAWgBQmOMvIHAegmBHwvdVGyguC/57/4zAKBggqhkjOPQQDAgNIADBFAiEAnE7M", + "lcB35MOr+7WKDAhu/c6ZrpgRz+chqqfc3g5YTOECIEDmoPkOigkulNON67opCPaT", + "y+MQhMA9KDEzE3t/CY9V", + ); + + const TEST_SERVER_CERT_DER_BASE64: &str = concat!( + "MIIBszCCAVqgAwIBAgICEAIwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", + "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowFDES", + "MBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEowPj", + "3vpXPAkf04MNeGhaDBvtwMsmeipV57lSWx5K2FwXH7JDmt74k4HmQFB6JESy6FbM", + "tAVhivr7kG5dWKK/sqOBkjCBjzAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIH", + "gDATBgNVHSUEDDAKBggrBgEFBQcDATAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A", + "AAEwHQYDVR0OBBYEFES7BP5uQpa3+PktDVlgc9zYGIqDMB8GA1UdIwQYMBaAFCY4", + "y8gcB6CYEfC91UbKC4L/nv/jMAoGCCqGSM49BAMCA0cAMEQCIGtIKEWRn7ec82TY", + "s1jrUoKWnhzRDbZTUtvXORk190rHAiAosxVgu45TjDyuROKU39TxJ1z+JObhNGk8", + "J6PkuOTFqg==", + ); + + const TEST_SERVER_KEY_DER_BASE64: &str = concat!( + "MHcCAQEEIIql19flBaZJE16Ivs8GjdJHedhuU5YFZgvIn4WaOs6HoAoGCCqGSM49", + "AwEHoUQDQgAEowPj3vpXPAkf04MNeGhaDBvtwMsmeipV57lSWx5K2FwXH7JDmt74", + "k4HmQFB6JESy6FbMtAVhivr7kG5dWKK/sg==", + ); - use super::parse_google_uri; + type LocalHttpsGcsServer = (String, JoinHandle>); #[test] fn test_parse_google_uri() { @@ -134,4 +183,123 @@ mod tests { assert_eq!(bucket, "test-bucket"); assert_eq!(prefix.to_str().unwrap(), "indexes"); } + + #[tokio::test] + async fn test_gcs_storage_get_slice_over_https_with_verified_tls() -> anyhow::Result<()> { + let (endpoint, server_task) = start_local_https_gcs_server().await?; + let ca_cert_der = decode_test_der(TEST_CA_CERT_DER_BASE64)?; + let ca_cert = reqwest_013::Certificate::from_der(&ca_cert_der)?; + let reqwest_client = reqwest_013::Client::builder() + .no_proxy() + .tls_certs_only([ca_cert]) + .build()?; + + let cfg = opendal::services::Gcs::default() + .bucket("quickwit-test-bucket") + .endpoint(&endpoint) + .allow_anonymous() + .disable_config_load() + .disable_vm_metadata(); + let storage = OpendalStorage::new_google_cloud_storage_with_http_client_for_test( + Uri::for_test("gs://quickwit-test-bucket"), + cfg, + HttpClient::with(reqwest_client), + )?; + + let bytes = storage.get_slice(Path::new("hello.txt"), 0..2).await?; + assert_eq!(bytes.as_slice(), b"ok"); + server_task.await??; + Ok(()) + } + + async fn start_local_https_gcs_server() -> anyhow::Result { + let cert_chain = vec![CertificateDer::from(decode_test_der( + TEST_SERVER_CERT_DER_BASE64, + )?)]; + let private_key = PrivateKeyDer::Sec1(PrivateSec1KeyDer::from(decode_test_der( + TEST_SERVER_KEY_DER_BASE64, + )?)); + let tls_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert_chain, private_key)?; + let tls_acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(tls_config)); + let listener = TcpListener::bind(("127.0.0.1", 0)).await?; + let endpoint = format!("https://127.0.0.1:{}", listener.local_addr()?.port()); + + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await?; + let mut stream = tls_acceptor.accept(stream).await?; + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + loop { + let bytes_read = stream.read(&mut buffer).await?; + if bytes_read == 0 { + break; + } + request.extend_from_slice(&buffer[..bytes_read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request); + let (header_block, _) = request + .split_once("\r\n\r\n") + .expect("request must contain HTTP header terminator"); + let mut header_lines = header_block.lines(); + let request_line = header_lines.next().expect("request line must be present"); + let mut request_line_parts = request_line.split_whitespace(); + let method = request_line_parts.next(); + let target = request_line_parts.next(); + let version = request_line_parts.next(); + assert_eq!( + method, + Some("GET"), + "unexpected request line: {request_line}" + ); + assert_eq!( + target, + Some("/storage/v1/b/quickwit-test-bucket/o/hello.txt?alt=media"), + "unexpected GCS request target: {request_line}" + ); + assert!( + version.is_some_and(|version| version.starts_with("HTTP/")), + "unexpected HTTP version in request line: {request_line}" + ); + assert_eq!( + request_line_parts.next(), + None, + "unexpected extra request line segment: {request_line}" + ); + + let headers: BTreeMap = header_lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) + .collect(); + assert_eq!( + headers.get("range").map(String::as_str), + Some("bytes=0-1"), + "expected range read request header: {request}" + ); + + stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\n\ + Content-Length: 2\r\n\ + Content-Range: bytes 0-1/2\r\n\ + Accept-Ranges: bytes\r\n\ + Connection: close\r\n\ + \r\n\ + ok", + ) + .await?; + stream.shutdown().await?; + Ok(()) + }); + + Ok((endpoint, server_task)) + } + + fn decode_test_der(base64_der: &str) -> anyhow::Result> { + Ok(base64::engine::general_purpose::STANDARD.decode(base64_der)?) + } } From dc9056b331f05efb499a7cb2b77deb775bdeadf2 Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Mon, 1 Jun 2026 19:38:57 +0200 Subject: [PATCH 2/5] fix(storage): satisfy GCS TLS test lints --- .../opendal_storage/google_cloud_storage.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index 39a1d9fa3e8..c831be35ebc 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -132,7 +132,23 @@ mod tests { // 2020 to 3020. The client trusts only this CA, so the test never depends // on the host root store. // Regenerate with: - // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` + // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey + // ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial + // 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config + // <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" + // "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" + // "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out + // "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 + // -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 + // -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf + // "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" + // "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" + // "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" + // "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" + // -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der + // | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der + // | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform + // der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` const TEST_CA_CERT_DER_BASE64: &str = concat!( "MIIBizCCATGgAwIBAgICEAEwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowGzEZ", @@ -262,7 +278,7 @@ mod tests { "unexpected GCS request target: {request_line}" ); assert!( - version.is_some_and(|version| version.starts_with("HTTP/")), + matches!(version, Some(version) if version.starts_with("HTTP/")), "unexpected HTTP version in request line: {request_line}" ); assert_eq!( From bd72fa86464c03b67d4b0b3c27e4d8c6a08a4b75 Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Mon, 1 Jun 2026 19:45:19 +0200 Subject: [PATCH 3/5] test(storage): keep GCS cert regeneration command copy-pastable --- .../opendal_storage/google_cloud_storage.rs | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index c831be35ebc..085ad310267 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -131,24 +131,10 @@ mod tests { // Test-only CA and server certificate for 127.0.0.1/localhost, valid from // 2020 to 3020. The client trusts only this CA, so the test never depends // on the host root store. - // Regenerate with: - // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey - // ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial - // 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config - // <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" - // "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" - // "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out - // "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 - // -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 - // -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf - // "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" - // "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" - // "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" - // "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" - // -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der - // | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der - // | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform - // der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` + // Regenerate with this copy-pastable command. The rustfmt skip below keeps + // the command intact. + #[rustfmt::skip] + // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` const TEST_CA_CERT_DER_BASE64: &str = concat!( "MIIBizCCATGgAwIBAgICEAEwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowGzEZ", From 6ef69b936677e142a4c0332e48d790bccc674736 Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Mon, 1 Jun 2026 22:18:35 +0200 Subject: [PATCH 4/5] test(storage): add GCS cert regeneration helper --- LICENSE-3rdparty.csv | 9 +++ .../opendal_storage/google_cloud_storage.rs | 5 +- .../tests/test_data/regenerate-gcs-certs.sh | 60 +++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100755 quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index c254b498788..a4d9ac6fb76 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -165,6 +165,7 @@ cobs,https://github.com/jamesmunns/cobs.rs,MIT OR Apache-2.0,"Allen Welkie <>, J codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zabarauskas colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham +combine,https://github.com/Marwes/combine,MIT,Markus Westerlind comfy-table,https://github.com/nukesor/comfy-table,MIT,Arne Beer community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " @@ -419,6 +420,10 @@ jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jiff-tzdb,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jiff-tzdb-platform,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant +jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team +jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors +jni-sys,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,"Steven Fackler , Robert Bragg " +jni-sys-macros,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,Robert Bragg jobserver,https://github.com/rust-lang/jobserver-rs,MIT OR Apache-2.0,Alex Crichton js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers json_comments,https://github.com/tmccombs/json-comments-rs,Apache-2.0,Thayne McCombs @@ -706,6 +711,8 @@ rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Auth rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors +rustls-platform-verifier,https://github.com/rustls/rustls-platform-verifier,MIT OR Apache-2.0,The rustls-platform-verifier Authors +rustls-platform-verifier-android,https://github.com/rustls/rustls-platform-verifier,MIT OR Apache-2.0,The rustls-platform-verifier-android Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors rustop,https://chiselapp.com/user/fifr/repository/rustop,MIT,Frank Fischer rustversion,https://github.com/dtolnay/rustversion,MIT OR Apache-2.0,David Tolnay @@ -765,6 +772,7 @@ signal-hook,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vor signal-hook-registry,https://github.com/vorner/signal-hook,MIT OR Apache-2.0,"Michal 'vorner' Vaner , Masaki Hara " signature,https://github.com/RustCrypto/traits/tree/master/signature,Apache-2.0 OR MIT,RustCrypto Developers simd-adler32,https://github.com/mcountryman/simd-adler32,MIT,Marvin Countryman +simd_cesu8,https://github.com/seancroach/simd_cesu8,Apache-2.0 OR MIT,Sean C. Roach simdutf8,https://github.com/rusticstuff/simdutf8,MIT OR Apache-2.0,Hans Kratz simple_asn1,https://github.com/acw/simple_asn1,ISC,Adam Wick siphasher,https://github.com/jedisct1/rust-siphash,MIT OR Apache-2.0,Frank Denis @@ -939,6 +947,7 @@ wasmparser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasmp wasmtimer,https://github.com/whizsid/wasmtimer-rs,MIT,"WhizSid , Pierre Krieger " web-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors +webpki-root-certs,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-root-certs Authors webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index 085ad310267..bce3a5d1e10 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -131,10 +131,7 @@ mod tests { // Test-only CA and server certificate for 127.0.0.1/localhost, valid from // 2020 to 3020. The client trusts only this CA, so the test never depends // on the host root store. - // Regenerate with this copy-pastable command. The rustfmt skip below keeps - // the command intact. - #[rustfmt::skip] - // `bash -c 'set -euo pipefail; d=$(mktemp -d); trap "rm -rf $d" EXIT; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=Quickwit Test CA" -set_serial 0x1001 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_ca -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_ca]" "basicConstraints=critical,CA:true" "keyUsage=critical,keyCertSign,cRLSign" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid:always") -keyout "$d/ca.key" -out "$d/ca.crt" 2>/dev/null; openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=localhost" -CA "$d/ca.crt" -CAkey "$d/ca.key" -set_serial 0x1002 -not_before 20200101000000Z -not_after 30200101000000Z -extensions v3_server -config <(printf "%s\n" "[req]" "distinguished_name=dn" "[dn]" "[v3_server]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=serverAuth" "subjectAltName=DNS:localhost,IP:127.0.0.1" "subjectKeyIdentifier=hash" "authorityKeyIdentifier=keyid,issuer") -keyout "$d/server.key" -out "$d/server.crt" 2>/dev/null; printf "CA_DER="; openssl x509 -in "$d/ca.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_DER="; openssl x509 -in "$d/server.crt" -outform der | base64 | tr -d "\n"; printf "\nSERVER_KEY_DER="; openssl ec -in "$d/server.key" -outform der 2>/dev/null | base64 | tr -d "\n"; printf "\n"'` + // Regenerate with `quickwit-storage/tests/test_data/regenerate-gcs-certs.sh`. const TEST_CA_CERT_DER_BASE64: &str = concat!( "MIIBizCCATGgAwIBAgICEAEwCgYIKoZIzj0EAwIwGzEZMBcGA1UEAwwQUXVpY2t3", "aXQgVGVzdCBDQTAgFw0yMDAxMDEwMDAwMDBaGA8zMDIwMDEwMTAwMDAwMFowGzEZ", diff --git a/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh new file mode 100755 index 00000000000..ea8c200570c --- /dev/null +++ b/quickwit/quickwit-storage/tests/test_data/regenerate-gcs-certs.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Regenerate the DER-encoded test CA, server certificate, and server private +# key embedded in `src/opendal_storage/google_cloud_storage.rs`. + +set -euo pipefail + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/ca.conf" <<'EOF' +[req] +distinguished_name=dn +[dn] +[v3_ca] +basicConstraints=critical,CA:true +keyUsage=critical,keyCertSign,cRLSign +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always +EOF + +openssl req -x509 -newkey ec \ + -pkeyopt ec_paramgen_curve:prime256v1 \ + -nodes -subj "/CN=Quickwit Test CA" \ + -set_serial 0x1001 \ + -not_before 20200101000000Z \ + -not_after 30200101000000Z \ + -extensions v3_ca -config "$tmp_dir/ca.conf" \ + -keyout "$tmp_dir/ca.key" -out "$tmp_dir/ca.crt" 2>/dev/null + +cat > "$tmp_dir/server.conf" <<'EOF' +[req] +distinguished_name=dn +[dn] +[v3_server] +basicConstraints=critical,CA:false +keyUsage=critical,digitalSignature +extendedKeyUsage=serverAuth +subjectAltName=DNS:localhost,IP:127.0.0.1 +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +EOF + +openssl req -x509 -newkey ec \ + -pkeyopt ec_paramgen_curve:prime256v1 \ + -nodes -subj "/CN=localhost" \ + -CA "$tmp_dir/ca.crt" -CAkey "$tmp_dir/ca.key" \ + -set_serial 0x1002 \ + -not_before 20200101000000Z \ + -not_after 30200101000000Z \ + -extensions v3_server -config "$tmp_dir/server.conf" \ + -keyout "$tmp_dir/server.key" -out "$tmp_dir/server.crt" 2>/dev/null + +printf "CA_DER=" +openssl x509 -in "$tmp_dir/ca.crt" -outform der | base64 | tr -d "\n" +printf "\nSERVER_DER=" +openssl x509 -in "$tmp_dir/server.crt" -outform der | base64 | tr -d "\n" +printf "\nSERVER_KEY_DER=" +openssl ec -in "$tmp_dir/server.key" -outform der 2>/dev/null | base64 | tr -d "\n" +printf "\n" From 649c830792c5e973337240209f9d00eabef8da4a Mon Sep 17 00:00:00 2001 From: Krzysztof Palka Date: Mon, 1 Jun 2026 22:49:21 +0200 Subject: [PATCH 5/5] test(storage): initialize rustls provider for GCS HTTPS test --- .../src/opendal_storage/google_cloud_storage.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs index bce3a5d1e10..23397e90a7c 100644 --- a/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs +++ b/quickwit/quickwit-storage/src/opendal_storage/google_cloud_storage.rs @@ -185,6 +185,10 @@ mod tests { #[tokio::test] async fn test_gcs_storage_get_slice_over_https_with_verified_tls() -> anyhow::Result<()> { + // Nextest runs tests in separate processes, so this test must not rely + // on another rustls user having already selected a process-wide provider. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let (endpoint, server_task) = start_local_https_gcs_server().await?; let ca_cert_der = decode_test_der(TEST_CA_CERT_DER_BASE64)?; let ca_cert = reqwest_013::Certificate::from_der(&ca_cert_der)?;