refactor: migrate metrics from prometheus crate to metrics-rs facade

LICENSE-3rdparty.csv

@@ -174,6 +174,7 @@ crc-catalog,https://github.com/akhilles/crc-catalog,MIT OR Apache-2.0,Akhil Vela
 crc-fast,https://github.com/awesomized/crc-fast-rust,MIT OR Apache-2.0,Don MacAskill
 crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs <srijs@airpost.net>, Alex Crichton <alex@alexcrichton.com>"
 criterion-plot,https://github.com/criterion-rs/criterion.rs,Apache-2.0 OR MIT,"Jorge Aparicio <japaricious@gmail.com>, Brook Heisler <brookheisler@gmail.com>"
+critical-section,https://github.com/rust-embedded/critical-section,MIT OR Apache-2.0,The critical-section Authors
 cron,https://github.com/zslayton/cron,MIT OR Apache-2.0,Zack Slayton <zack.slayton@gmail.com>
 crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors
 crossbeam-deque,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-deque Authors
@@ -404,6 +405,8 @@ memchr,https://github.com/BurntSushi/memchr,Unlicense OR MIT,"Andrew Gallant <ja
 memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert <dan@danburkert.com>, Yevhenii Reizner <razrfalcon@gmail.com>, The Contributors"
 metrics,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence <toby@nuclearfurnace.com>
 metrics-exporter-dogstatsd,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence <toby@nuclearfurnace.com>
+metrics-exporter-otel,https://github.com/palindrom615/metrics,MIT,Whoemoon Jang <palindrom615@gmail.com>
+metrics-exporter-prometheus,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence <toby@nuclearfurnace.com>
 metrics-util,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence <toby@nuclearfurnace.com>
 mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur <sean@seanmonstar.com>
 mime_guess,https://github.com/abonander/mime_guess,MIT,Austin Bonander <austin.bonander@gmail.com>

quickwit/Cargo.toml

@@ -110,6 +110,7 @@ colored = "3.0"
 console-subscriber = "0.5"
 criterion = { version = "0.8", features = ["async_tokio"] }
 cron = "0.16"
+dashmap = "6"
 dialoguer = { version = "0.12", default-features = false }
 dotenvy = "0.15"
 dyn-clone = "1.0"
@@ -149,6 +150,7 @@ hyper-util = { version = "0.1", default-features = false, features = [
 ] }
 indexmap = { version = "2.12", features = ["serde"] }
 indicatif = "0.18"
+inventory = "0.3"
 itertools = "0.14"
 lambda_runtime = "0.13"
 json_comments = "0.2"
@@ -158,6 +160,9 @@ matches = "0.1"
 md5 = "0.8"
 metrics = "0.24"
 metrics-exporter-dogstatsd = "0.9"
+metrics-exporter-otel = "0.3"
+metrics-exporter-prometheus = { version = "0.18.1", default-features = false }
+metrics-util = "0.20"
 mime_guess = "2.0"
 mini-moka = "0.10.3"
 mockall = "0.14"
@@ -170,7 +175,7 @@ openssl = { version = "0.10", default-features = false }
 openssl-probe = "0.1"
 opentelemetry = "0.31"
 opentelemetry-appender-tracing = "0.31"
-opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
+opentelemetry_sdk = { version = "0.31", features = ["rt-tokio", "metrics"] }
 opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "http-json"] }
 ouroboros = "0.18"
 parquet = { version = "58", default-features = false, features = ["arrow", "snap", "variant_experimental", "zstd"] }

quickwit/quickwit-cli/Cargo.toml

@@ -57,6 +57,9 @@ tracing-subscriber = { workspace = true }
 
 metrics = { workspace = true }
 metrics-exporter-dogstatsd = { workspace = true }
+metrics-exporter-otel = { workspace = true }
+metrics-exporter-prometheus = { workspace = true }
+metrics-util = { workspace = true }
 
 quickwit-actors = { workspace = true }
 quickwit-cluster = { workspace = true }

quickwit/quickwit-cli/src/lib.rs

@@ -395,7 +395,6 @@ pub mod busy_detector {
             let delta = now - time.load(Ordering::Relaxed);
             CLI_METRICS
                 .thread_unpark_duration_microseconds
-                .with_label_values([])
                 .observe(delta as f64);
             if delta > ALLOWED_DELAY_MICROS {
                 emit_debug(delta, now);

quickwit/quickwit-cli/src/logger.rs

@@ -20,6 +20,8 @@ use anyhow::Context;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{KeyValue, global};
 use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
+#[cfg(not(test))]
+use opentelemetry_otlp::MetricExporter;
 use opentelemetry_otlp::{
     LogExporter, Protocol as OtlpWireProtocol, SpanExporter, WithExportConfig,
 };
@@ -81,6 +83,22 @@ impl OtlpProtocol {
         }
         .context("failed to initialize OTLP traces exporter")
     }
+
+    #[cfg(not(test))]
+    fn metric_exporter(&self) -> anyhow::Result<MetricExporter> {
+        match self {
+            OtlpProtocol::Grpc => MetricExporter::builder().with_tonic().build(),
+            OtlpProtocol::HttpProtobuf => MetricExporter::builder()
+                .with_http()
+                .with_protocol(OtlpWireProtocol::HttpBinary)
+                .build(),
+            OtlpProtocol::HttpJson => MetricExporter::builder()
+                .with_http()
+                .with_protocol(OtlpWireProtocol::HttpJson)
+                .build(),
+        }
+        .context("failed to initialize OTLP metrics exporter")
+    }
 }
 
 impl FromStr for OtlpProtocol {
@@ -238,11 +256,65 @@ pub fn setup_logging_and_tracing(
     ))
 }
 
-/// Set up DogStatsD metrics exporter and invariant recorder.
+/// Set up metrics recorders: Prometheus (always), OTLP (opt-in), DogStatsD.
+///
+/// Routes metrics by prefix:
+/// - `quickwit_*` → Prometheus + OTLP
+/// - `pomsky.*`   → DogStatsD
 #[cfg(not(test))]
 pub fn setup_metrics(build_info: &BuildInfo) -> anyhow::Result<()> {
-    // Reading both `CLOUDPREM_*` and `CP_*` env vars for backward compatibility. The former is
-    // deprecated and can be removed after 2026-04-01.
+    use quickwit_common::metrics::HistogramConfig;
+
+    let mut prom_builder = metrics_exporter_prometheus::PrometheusBuilder::new();
+    for config in quickwit_common::inventory::iter::<HistogramConfig>() {
+        prom_builder = prom_builder
+            .set_buckets_for_metric(
+                metrics_exporter_prometheus::Matcher::Full(config.full_name()),
+                &(config.buckets_fn)(),
+            )
+            .context("failed to set histogram buckets")?;
+    }
+    let prom_recorder = prom_builder.build_recorder();
+    let prom_handle = prom_recorder.handle();
+
+    quickwit_common::metrics::set_prom_handle(prom_handle);
+
+    let mut quickwit_fanout =
+        metrics_util::layers::FanoutBuilder::default().add_recorder(prom_recorder);
+
+    if get_bool_from_env(QW_ENABLE_OPENTELEMETRY_OTLP_EXPORTER_ENV_KEY, false) {
+        let global_protocol_str =
+            get_from_env("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc".to_string(), false);
+        let global_protocol = OtlpProtocol::from_str(&global_protocol_str)?;
+
+        let metrics_protocol_str =
+            get_from_env_opt::<String>("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", false);
+        let metrics_protocol = metrics_protocol_str
+            .as_deref()
+            .map(OtlpProtocol::from_str)
+            .transpose()?
+            .unwrap_or(global_protocol);
+
+        let metric_exporter = metrics_protocol.metric_exporter()?;
+        let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter).build();
+        let resource = opentelemetry_sdk::Resource::builder()
+            .with_service_name("quickwit")
+            .with_attribute(KeyValue::new("service.version", build_info.version.clone()))
+            .build();
+        let meter_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
+            .with_reader(reader)
+            .with_resource(resource)
+            .build();
+        let meter = opentelemetry::metrics::MeterProvider::meter(&meter_provider, "quickwit");
+        let otel_recorder = metrics_exporter_otel::OpenTelemetryRecorder::new(meter);
+        for config in quickwit_common::inventory::iter::<HistogramConfig>() {
+            let key_name = metrics::KeyName::from(config.full_name());
+            otel_recorder.set_histogram_bounds(&key_name, (config.buckets_fn)());
+        }
+
+        quickwit_fanout = quickwit_fanout.add_recorder(otel_recorder);
+    }
+
     let host: String = quickwit_common::get_from_env_opt("CLOUDPREM_DOGSTATSD_SERVER_HOST", false)
         .unwrap_or_else(|| {
             quickwit_common::get_from_env(
@@ -270,13 +342,36 @@ pub fn setup_metrics(build_info: &BuildInfo) -> anyhow::Result<()> {
             global_labels.push(::metrics::Label::new(label_key, label_val));
         }
     }
-    metrics_exporter_dogstatsd::DogStatsDBuilder::default()
+    let dogstatsd_recorder = metrics_exporter_dogstatsd::DogStatsDBuilder::default()
         .set_global_prefix("cloudprem")
         .with_global_labels(global_labels)
         .with_remote_address(addr)
         .context("failed to parse DogStatsD server address")?
-        .install()
-        .context("failed to register DogStatsD exporter")?;
+        .build()
+        .context("failed to build DogStatsD recorder")?;
+
+    let quickwit_recorder = quickwit_fanout.build();
+
+    let mut router = metrics_util::layers::RouterBuilder::from_recorder(metrics::NoopRecorder);
+    router
+        .add_route(
+            metrics_util::MetricKindMask::ALL,
+            "quickwit_",
+            quickwit_recorder,
+        )
+        .add_route(
+            metrics_util::MetricKindMask::ALL,
+            "pomsky.",
+            dogstatsd_recorder,
+        );
+
+    let recorder = router.build();
+    metrics::set_global_recorder(recorder)
+        .map_err(|_| anyhow::anyhow!("failed to set global metrics recorder"))?;
+    for config in quickwit_common::inventory::iter::<HistogramConfig>() {
+        metrics::describe_histogram!(config.full_name(), config.help);
+    }
+
     quickwit_dst::invariants::set_invariant_recorder(invariant_recorder);
     Ok(())
 }

quickwit/quickwit-cli/src/metrics.rs

@@ -14,26 +14,28 @@
 
 use std::sync::LazyLock;
 
-use quickwit_common::metrics::{HistogramVec, new_histogram_vec};
+use quickwit_common::metrics::Histogram;
+
+pub static THREAD_UNPARK_DURATION_MICROSECONDS: LazyLock<Histogram> = LazyLock::new(|| {
+    quickwit_common::define_histogram! {
+        name: "thread_unpark_duration_microseconds",
+        help: "Duration for which a thread of the main tokio runtime is unparked.",
+        subsystem: "cli",
+        buckets: quickwit_common::metrics::exponential_buckets(5.0, 5.0, 5).unwrap(),
+    }
+});
 
 pub struct CliMetrics {
-    pub thread_unpark_duration_microseconds: HistogramVec<0>,
+    pub thread_unpark_duration_microseconds: Histogram,
 }
 
 impl Default for CliMetrics {
     fn default() -> Self {
         CliMetrics {
-            thread_unpark_duration_microseconds: new_histogram_vec(
-                "thread_unpark_duration_microseconds",
-                "Duration for which a thread of the main tokio runtime is unparked.",
-                "cli",
-                &[],
-                [],
-                quickwit_common::metrics::exponential_buckets(5.0, 5.0, 5).unwrap(),
-            ),
+            thread_unpark_duration_microseconds: THREAD_UNPARK_DURATION_MICROSECONDS.clone(),
         }
     }
 }
 
-/// Serve counters exposes a bunch a set of metrics about the request received to quickwit.
+/// CLI counters exposes a set of metrics about the main tokio runtime.
 pub static CLI_METRICS: LazyLock<CliMetrics> = LazyLock::new(CliMetrics::default);

quickwit/quickwit-common/Cargo.toml

@@ -17,6 +17,7 @@ async-trait = { workspace = true }
 backtrace = { workspace = true, optional = true }
 bytesize = { workspace = true }
 coarsetime = { workspace = true }
+dashmap = { workspace = true }
 dyn-clone = { workspace = true }
 env_logger = { workspace = true }
 fnv = { workspace = true }
@@ -27,7 +28,10 @@ hostname = { workspace = true }
 http = { workspace = true }
 hyper = { workspace = true }
 hyper-util = { workspace = true, optional = true }
+inventory = { workspace = true }
 itertools = { workspace = true }
+metrics = { workspace = true }
+metrics-exporter-prometheus = { workspace = true }
 pin-project = { workspace = true }
 pnet = { workspace = true }
 prometheus = { workspace = true }
@@ -62,6 +66,7 @@ jemalloc-profiled = [
 
 [dev-dependencies]
 hyper-util = { workspace = true }
+metrics-util = { workspace = true }
 proptest = { workspace = true }
 serde_json = { workspace = true }
 serial_test = { workspace = true }

quickwit/quickwit-common/src/io.rs

@@ -34,10 +34,9 @@ use async_speed_limit::clock::StandardClock;
 use async_speed_limit::limiter::Consume;
 use bytesize::ByteSize;
 use pin_project::pin_project;
-use prometheus::IntCounter;
 use tokio::io::AsyncWrite;
 
-use crate::metrics::{IntCounterVec, new_counter_vec};
+use crate::metrics::{IntCounter, IntCounterVec, new_counter_vec};
 use crate::{KillSwitch, Progress, ProtectedZoneGuard};
 
 // Max 1MB at a time.

quickwit/quickwit-common/src/lib.rs

@@ -62,6 +62,8 @@ use std::str::FromStr;
 
 pub use coolid::new_coolid;
 pub use cpus::num_cpus;
+#[doc(hidden)]
+pub use inventory;
 pub use kill_switch::KillSwitch;
 pub use path_hasher::PathHasher;
 pub use progress::{Progress, ProtectedZoneGuard};