fix(tools): parse ms-suffixed Prometheus scrape interval (#447)

Author: milindsrivastava1997

asap-tools/experiments/experiment_only_ingest_path.py

@@ -227,7 +227,9 @@ def main(cfg: DictConfig):
         node_offset=args.node_offset,
     )
 
-    prometheus_scrape_interval = config.get_prometheus_scrape_interval(cfg.prometheus)
+    prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms(
+        cfg.prometheus
+    )
 
     # Start fake exporter if configured
     if config.check_exporter_and_queries_exist("fake_exporter", cfg.experiment_params):
@@ -288,16 +290,16 @@ def main(cfg: DictConfig):
 
         prometheus_service.wait_until_ready()
 
-        label_discovery_wait = prometheus_scrape_interval * 2
+        label_discovery_wait_ms = prometheus_scrape_interval_ms * 2
         print(
-            f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data "
+            f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data "
             f"before running controller label inference..."
         )
-        time.sleep(label_discovery_wait)
+        time.sleep(label_discovery_wait_ms / 1000)
 
         controller_service.start(
             controller_input_file=controller_input_config,
-            prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000,
+            prometheus_scrape_interval_ms=prometheus_scrape_interval_ms,
             streaming_engine=args.streaming_engine,
             controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR,
             punting=args.controller_punting,

asap-tools/experiments/experiment_run_e2e.py

@@ -324,7 +324,7 @@ def main(cfg: DictConfig):
             prometheus_config_output_dir,
             node_offset=args.node_offset,
         )
-        prometheus_scrape_interval = config.get_prometheus_scrape_interval(
+        prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms(
             cfg.prometheus
         )
 
@@ -352,12 +352,12 @@ def main(cfg: DictConfig):
             prometheus_service.wait_until_ready()
 
             # Wait for two scrape intervals so Prometheus has series to return.
-            label_discovery_wait = prometheus_scrape_interval * 2
+            label_discovery_wait_ms = prometheus_scrape_interval_ms * 2
             print(
-                f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data "
+                f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data "
                 f"before running controller label inference..."
             )
-            time.sleep(label_discovery_wait)
+            time.sleep(label_discovery_wait_ms / 1000)
 
             prometheus_url = (
                 f"http://localhost:{prometheus_service.get_query_endpoint_port()}"
@@ -372,7 +372,7 @@ def main(cfg: DictConfig):
                     url=prometheus_url,
                     database=None,
                 ),
-                prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000,
+                prometheus_scrape_interval_ms=prometheus_scrape_interval_ms,
             )
             sync.rsync_controller_config_remote_to_local(
                 provider,
@@ -516,7 +516,7 @@ def main(cfg: DictConfig):
                     experiment_output_dir=experiment_output_dir,
                     local_experiment_dir=local_experiment_dir,
                     flink_output_format=args.flink_output_format,
-                    prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000,
+                    prometheus_scrape_interval_ms=prometheus_scrape_interval_ms,
                     log_level=args.log_level,
                     profile_query_engine=args.profile_query_engine,
                     manual=args.manual_query_engine,

asap-tools/experiments/experiment_run_grafana_demo.py

@@ -273,7 +273,9 @@ def main(cfg: DictConfig):
         prometheus_config_output_dir,
         node_offset=args.node_offset,
     )
-    prometheus_scrape_interval = config.get_prometheus_scrape_interval(cfg.prometheus)
+    prometheus_scrape_interval_ms = config.get_prometheus_scrape_interval_ms(
+        cfg.prometheus
+    )
 
     if config.check_exporter_and_queries_exist("fake_exporter", cfg.experiment_params):
         # this DOES NOT block
@@ -341,16 +343,16 @@ def main(cfg: DictConfig):
 
         prometheus_service.wait_until_ready()
 
-        label_discovery_wait = prometheus_scrape_interval * 2
+        label_discovery_wait_ms = prometheus_scrape_interval_ms * 2
         print(
-            f"Waiting {label_discovery_wait}s for Prometheus to scrape initial data "
+            f"Waiting {label_discovery_wait_ms / 1000}s for Prometheus to scrape initial data "
             f"before running controller label inference..."
         )
-        time.sleep(label_discovery_wait)
+        time.sleep(label_discovery_wait_ms / 1000)
 
         controller_service.start(
             controller_input_file=controller_input_config,
-            prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000,
+            prometheus_scrape_interval_ms=prometheus_scrape_interval_ms,
             streaming_engine=args.streaming_engine,
             controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR,
             punting=args.controller_punting,
@@ -467,7 +469,7 @@ def main(cfg: DictConfig):
                 experiment_output_dir=experiment_output_dir,
                 local_experiment_dir=local_experiment_dir,
                 flink_output_format=args.flink_output_format,
-                prometheus_scrape_interval_ms=prometheus_scrape_interval * 1000,
+                prometheus_scrape_interval_ms=prometheus_scrape_interval_ms,
                 log_level=args.log_level,
                 profile_query_engine=args.profile_query_engine,
                 manual=args.manual_query_engine,

asap-tools/experiments/experiment_utils/config.py

@@ -478,20 +478,18 @@ def read_workloads_config(experiment_params: DictConfig):
     return workloads_config
 
 
-def get_prometheus_scrape_interval(prometheus_config):
-    """Extract scrape interval from Prometheus configuration."""
-    prometheus_scrape_interval_string = prometheus_config.scrape_interval
-    # convert to seconds
-    if prometheus_scrape_interval_string.endswith("s"):
-        prometheus_scrape_interval = int(prometheus_scrape_interval_string[:-1])
-    elif prometheus_scrape_interval_string.endswith("m"):
-        prometheus_scrape_interval = int(prometheus_scrape_interval_string[:-1]) * 60
+def get_prometheus_scrape_interval_ms(prometheus_config):
+    """Extract scrape interval from Prometheus configuration, returned in milliseconds."""
+    s = prometheus_config.scrape_interval
+    # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute
+    if s.endswith("ms"):
+        return int(s[:-2])
+    elif s.endswith("s"):
+        return int(s[:-1]) * 1000
+    elif s.endswith("m"):
+        return int(s[:-1]) * 60 * 1000
     else:
-        raise ValueError(
-            f"Invalid scrape interval string: {prometheus_scrape_interval_string}"
-        )
-
-    return prometheus_scrape_interval
+        raise ValueError(f"Invalid scrape interval string: {s}")
 
 
 class Args: