From 4bbb9bca3d4c5e7dfae69a652e6414ee24950472 Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Mon, 20 Jul 2026 09:26:40 +0200 Subject: [PATCH] use macro with cache to load env --- quickwit/quickwit-cli/src/main.rs | 8 ++++---- quickwit/quickwit-common/src/env.rs | 20 +++++++++---------- quickwit/quickwit-common/src/runtimes.rs | 2 +- quickwit/quickwit-common/src/shared_consts.rs | 3 ++- .../quickwit-config/src/storage_config.rs | 14 ++++--------- .../src/actors/indexing_service.rs | 2 +- .../quickwit-ingest/src/ingest_v2/router.rs | 3 ++- .../src/metastore/postgres/migrator.rs | 16 +++++++++++---- quickwit/quickwit-proto/src/types/split_id.rs | 7 +++---- .../src/datafusion_api/setup.rs | 2 +- quickwit/quickwit-serve/src/lib.rs | 7 ++++--- .../object_storage/s3_compatible_storage.rs | 2 +- .../src/otlp/config.rs | 10 +++++++--- 13 files changed, 52 insertions(+), 44 deletions(-) diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index efcb1ea1d6f..60de3839d26 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -34,10 +34,10 @@ const QW_ENABLE_TOKIO_CONSOLE_ENV_KEY: &str = "QW_ENABLE_TOKIO_CONSOLE"; /// The main tokio runtime takes num_cores / 3 threads by default, and can be overridden by the /// QW_RUNTIME_NUM_THREADS environment variable. fn get_main_runtime_num_threads() -> usize { - let default_num_runtime_threads: usize = quickwit_common::num_cpus().div_ceil(3); - quickwit_common::get_from_env( + quickwit_common::get_from_env_cached!( + usize, "QW_TOKIO_RUNTIME_NUM_THREADS", - default_num_runtime_threads, + quickwit_common::num_cpus().div_ceil(3), false, ) } @@ -92,7 +92,7 @@ fn init_telemetry( )> { #[cfg(feature = "tokio-console")] { - if quickwit_common::get_bool_from_env(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) { + if quickwit_common::get_bool_from_env_cached!(QW_ENABLE_TOKIO_CONSOLE_ENV_KEY, false) { let telemetry_handle = quickwit_telemetry_exporters::init_meter_provider_only(service_version)?; console_subscriber::init(); diff --git a/quickwit/quickwit-common/src/env.rs b/quickwit/quickwit-common/src/env.rs index 23aa8cd256d..449e8d57d56 100644 --- a/quickwit/quickwit-common/src/env.rs +++ b/quickwit/quickwit-common/src/env.rs @@ -78,14 +78,14 @@ pub fn parse_bool_lenient(bool_str: &str) -> Option { } /// Reads and parses an environment variable exactly once, caching the result for the lifetime of -/// the process via a per-call-site [`std::sync::LazyLock`]. +/// the process via a per-call-site [`std::sync::OnceLock`]. /// /// Prefer this over [`get_from_env`](crate::get_from_env) on paths that may run repeatedly (e.g. /// constructors that are re-invoked): it avoids re-reading and, more importantly, re-logging the /// same variable on every call. /// /// The type is required because the backing `static` needs a concrete type. The key and default -/// must be `const` expressions or literals — a `static` initializer cannot capture locals. +/// are evaluated only on the first invocation and may reference local values. /// /// ```no_run /// # use quickwit_common::get_from_env_cached; @@ -94,12 +94,13 @@ pub fn parse_bool_lenient(bool_str: &str) -> Option { #[macro_export] macro_rules! get_from_env_cached { ($ty:ty, $key:expr, $default:expr, $sensitive:expr $(,)?) => {{ - static CACHED: ::std::sync::LazyLock<$ty> = - ::std::sync::LazyLock::new(|| $crate::get_from_env::<$ty>($key, $default, $sensitive)); - // `LazyLock` derefs to `T`; clone so callers receive an owned value, matching - // `get_from_env`'s return type (a no-op copy for the common `Copy` cases). + static CACHED: ::std::sync::OnceLock<$ty> = ::std::sync::OnceLock::new(); + // Clone so callers receive an owned value, matching `get_from_env`'s return type (a no-op + // copy for the common `Copy` cases). #[allow(clippy::clone_on_copy)] - let value = (*CACHED).clone(); + let value = CACHED + .get_or_init(|| $crate::get_from_env::<$ty>($key, $default, $sensitive)) + .clone(); value }}; } @@ -115,9 +116,8 @@ macro_rules! get_from_env_cached { #[macro_export] macro_rules! get_bool_from_env_cached { ($key:expr, $default:expr $(,)?) => {{ - static CACHED: ::std::sync::LazyLock = - ::std::sync::LazyLock::new(|| $crate::get_bool_from_env($key, $default)); - *CACHED + static CACHED: ::std::sync::OnceLock = ::std::sync::OnceLock::new(); + *CACHED.get_or_init(|| $crate::get_bool_from_env($key, $default)) }}; } diff --git a/quickwit/quickwit-common/src/runtimes.rs b/quickwit/quickwit-common/src/runtimes.rs index 096d8ffc572..52a7912d9c4 100644 --- a/quickwit/quickwit-common/src/runtimes.rs +++ b/quickwit/quickwit-common/src/runtimes.rs @@ -121,7 +121,7 @@ impl Default for RuntimesConfig { fn start_runtimes(config: RuntimesConfig) -> HashMap { let mut runtimes = HashMap::with_capacity(2); - let disable_lifo_slot = crate::get_bool_from_env("QW_DISABLE_TOKIO_LIFO_SLOT", true); + let disable_lifo_slot = crate::get_bool_from_env_cached!("QW_DISABLE_TOKIO_LIFO_SLOT", true); let mut blocking_runtime_builder = tokio::runtime::Builder::new_multi_thread(); if disable_lifo_slot { diff --git a/quickwit/quickwit-common/src/shared_consts.rs b/quickwit/quickwit-common/src/shared_consts.rs index 95b3a531aaf..e9f60a9d108 100644 --- a/quickwit/quickwit-common/src/shared_consts.rs +++ b/quickwit/quickwit-common/src/shared_consts.rs @@ -36,7 +36,8 @@ pub fn split_deletion_grace_period() -> Duration { const DEFAULT_DELETION_GRACE_PERIOD: Duration = Duration::from_secs(60 * 32); // 32 min static SPLIT_DELETION_GRACE_PERIOD_SECS_LOCK: LazyLock = LazyLock::new(|| { - let deletion_grace_period_secs: u64 = crate::get_from_env( + let deletion_grace_period_secs: u64 = crate::get_from_env_cached!( + u64, "QW_SPLIT_DELETION_GRACE_PERIOD_SECS", DEFAULT_DELETION_GRACE_PERIOD.as_secs(), false, diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index d04cd93aaa0..d30db9fee12 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -13,12 +13,10 @@ // limitations under the License. use std::ops::Deref; -use std::sync::OnceLock; use std::{env, fmt}; use anyhow::ensure; use itertools::Itertools; -use quickwit_common::get_bool_from_env; use serde::{Deserialize, Serialize}; use serde_with::{EnumMap, serde_as}; @@ -406,14 +404,10 @@ impl S3StorageConfig { } pub fn force_path_style_access(&self) -> Option { - static FORCE_PATH_STYLE: OnceLock> = OnceLock::new(); - *FORCE_PATH_STYLE.get_or_init(|| { - let force_path_style_access = get_bool_from_env( - "QW_S3_FORCE_PATH_STYLE_ACCESS", - self.force_path_style_access, - ); - Some(force_path_style_access) - }) + Some(quickwit_common::get_bool_from_env_cached!( + "QW_S3_FORCE_PATH_STYLE_ACCESS", + self.force_path_style_access, + )) } } diff --git a/quickwit/quickwit-indexing/src/actors/indexing_service.rs b/quickwit/quickwit-indexing/src/actors/indexing_service.rs index 09434b2a6f7..c6ee35fd65b 100644 --- a/quickwit/quickwit-indexing/src/actors/indexing_service.rs +++ b/quickwit/quickwit-indexing/src/actors/indexing_service.rs @@ -743,7 +743,7 @@ impl IndexingService { merge_scheduler_service, max_concurrent_split_uploads: self.max_concurrent_split_uploads, event_broker: self.event_broker.clone(), - skip_initial_seed: quickwit_common::get_bool_from_env( + skip_initial_seed: quickwit_common::get_bool_from_env_cached!( super::parquet_pipeline::PARQUET_MERGE_SKIP_INITIAL_SEED_ENV_KEY, false, ), diff --git a/quickwit/quickwit-ingest/src/ingest_v2/router.rs b/quickwit/quickwit-ingest/src/ingest_v2/router.rs index ecf5f707642..79ef64cf946 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/router.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/router.rs @@ -67,7 +67,8 @@ fn ingest_request_timeout() -> Duration { Duration::from_secs(35) }; static TIMEOUT: LazyLock = LazyLock::new(|| { - let duration_ms = quickwit_common::get_from_env( + let duration_ms = quickwit_common::get_from_env_cached!( + u64, "QW_INGEST_REQUEST_TIMEOUT_MS", DEFAULT_INGEST_REQUEST_TIMEOUT.as_millis() as u64, false, diff --git a/quickwit/quickwit-metastore/src/metastore/postgres/migrator.rs b/quickwit/quickwit-metastore/src/metastore/postgres/migrator.rs index 342973483ef..ef3bef7f91d 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgres/migrator.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgres/migrator.rs @@ -14,7 +14,6 @@ use std::collections::BTreeMap; -use quickwit_common::get_bool_from_env; use quickwit_metrics::{counter, labels}; use quickwit_proto::metastore::{MetastoreError, MetastoreResult}; use sqlx::migrate::{Migrate, Migrator}; @@ -63,9 +62,18 @@ impl Migrations { pub(super) fn from_env(connection_pool: TrackedPool) -> Self { Self { connection_pool, - skip_migrations: get_bool_from_env(QW_POSTGRES_SKIP_MIGRATIONS_ENV_KEY, false), - skip_locking: get_bool_from_env(QW_POSTGRES_SKIP_MIGRATION_LOCKING_ENV_KEY, false), - skip_deferred: get_bool_from_env(QW_POSTGRES_SKIP_DEFERRED_MIGRATIONS_ENV_KEY, false), + skip_migrations: quickwit_common::get_bool_from_env_cached!( + QW_POSTGRES_SKIP_MIGRATIONS_ENV_KEY, + false + ), + skip_locking: quickwit_common::get_bool_from_env_cached!( + QW_POSTGRES_SKIP_MIGRATION_LOCKING_ENV_KEY, + false + ), + skip_deferred: quickwit_common::get_bool_from_env_cached!( + QW_POSTGRES_SKIP_DEFERRED_MIGRATIONS_ENV_KEY, + false + ), } } diff --git a/quickwit/quickwit-proto/src/types/split_id.rs b/quickwit/quickwit-proto/src/types/split_id.rs index af8942d17fb..385181f6c05 100644 --- a/quickwit/quickwit-proto/src/types/split_id.rs +++ b/quickwit/quickwit-proto/src/types/split_id.rs @@ -50,10 +50,9 @@ impl SplitId { /// Layout with prefix enabled: `<4 random chars>_<22 remaining ULID chars>` /// (27 chars total vs the usual 26). pub fn new() -> Self { - static RANDOM_PREFIX_ENABLED: std::sync::LazyLock = std::sync::LazyLock::new(|| { - quickwit_common::get_bool_from_env("QW_RANDOM_SPLIT_PREFIX", false) - }); - Self::from_ulid(ulid::Ulid::new(), *RANDOM_PREFIX_ENABLED) + let random_prefix_enabled = + quickwit_common::get_bool_from_env_cached!("QW_RANDOM_SPLIT_PREFIX", false); + Self::from_ulid(ulid::Ulid::new(), random_prefix_enabled) } /// Constructs a `SplitId` from a `Ulid`, optionally prepending a random prefix. diff --git a/quickwit/quickwit-serve/src/datafusion_api/setup.rs b/quickwit/quickwit-serve/src/datafusion_api/setup.rs index aad5e45a57b..375135e969d 100644 --- a/quickwit/quickwit-serve/src/datafusion_api/setup.rs +++ b/quickwit/quickwit-serve/src/datafusion_api/setup.rs @@ -72,7 +72,7 @@ pub(crate) fn build_datafusion_session_builder( if !node_config.is_service_enabled(QuickwitService::Searcher) { return Ok(None); } - if !quickwit_common::get_bool_from_env("QW_ENABLE_DATAFUSION_ENDPOINT", false) { + if !quickwit_common::get_bool_from_env_cached!("QW_ENABLE_DATAFUSION_ENDPOINT", false) { return Ok(None); } diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 89c5914027c..2b5298ebf07 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -67,13 +67,13 @@ use quickwit_common::pubsub::{EventBroker, EventSubscriptionHandle}; use quickwit_common::rate_limiter::RateLimiterSettings; use quickwit_common::retry::RetryParams; use quickwit_common::runtimes::RuntimesConfig; +use quickwit_common::spawn_named_task; use quickwit_common::tower::{ BalanceChannel, BoxFutureInfaillible, BufferLayer, Change, CircuitBreakerEvaluator, ConstantRate, EstimateRateLayer, EventListenerLayer, GrpcMetricsLayer, LoadShedLayer, RateLimitLayer, RetryLayer, RetryPolicy, SmaRateEstimator, TimeoutLayer, }; use quickwit_common::uri::Uri; -use quickwit_common::{get_bool_from_env, spawn_named_task}; use quickwit_compaction::planner::CompactionPlanner; use quickwit_compaction::{ CompactorSupervisor, notify_compactor_decommission, start_compactor_service, @@ -159,7 +159,8 @@ const DEFAULT_METASTORE_CLIENT_MAX_CONCURRENCY: usize = 6; const DISABLE_DELETE_TASK_SERVICE_ENV_KEY: &str = "QW_DISABLE_DELETE_TASK_SERVICE"; fn get_metastore_client_max_concurrency() -> usize { - quickwit_common::get_from_env( + quickwit_common::get_from_env_cached!( + usize, METASTORE_CLIENT_MAX_CONCURRENCY_ENV_KEY, DEFAULT_METASTORE_CLIENT_MAX_CONCURRENCY, false, @@ -882,7 +883,7 @@ pub async fn serve_quickwit( search_job_placer, storage_resolver.clone(), event_broker.clone(), - !get_bool_from_env(DISABLE_DELETE_TASK_SERVICE_ENV_KEY, false), + !quickwit_common::get_bool_from_env_cached!(DISABLE_DELETE_TASK_SERVICE_ENV_KEY, false), compaction_planner_handle_opt, ) .await diff --git a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs index d015bf799d6..091c0f432e5 100644 --- a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs @@ -66,7 +66,7 @@ use crate::{ /// (R2, SeaweedFs...) return errors when too many concurrent requests are emitted. static REQUEST_SEMAPHORE: LazyLock = LazyLock::new(|| { let num_permits: usize = - quickwit_common::get_from_env("QW_S3_MAX_CONCURRENCY", 10_000usize, false); + quickwit_common::get_from_env_cached!(usize, "QW_S3_MAX_CONCURRENCY", 10_000usize, false); Semaphore::new(num_permits) }); diff --git a/quickwit/quickwit-telemetry-exporters/src/otlp/config.rs b/quickwit/quickwit-telemetry-exporters/src/otlp/config.rs index 48fe64aaee8..de89f6bba81 100644 --- a/quickwit/quickwit-telemetry-exporters/src/otlp/config.rs +++ b/quickwit/quickwit-telemetry-exporters/src/otlp/config.rs @@ -17,7 +17,7 @@ use std::str::FromStr; use anyhow::Context; use opentelemetry::KeyValue; use opentelemetry_sdk::Resource; -use quickwit_common::{get_bool_from_env, get_from_env, get_from_env_opt}; +use quickwit_common::get_from_env_opt; pub const QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY: &str = "QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER"; @@ -66,8 +66,12 @@ pub(crate) struct OtlpExporterConfig { impl OtlpExporterConfig { pub(crate) fn load_from_env() -> Self { OtlpExporterConfig { - enabled: get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false), - default_protocol: get_from_env( + enabled: quickwit_common::get_bool_from_env_cached!( + QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, + false + ), + default_protocol: quickwit_common::get_from_env_cached!( + String, OTEL_EXPORTER_OTLP_PROTOCOL_ENV_KEY, "grpc".to_string(), false,