chore(logging): Improve datadog logs semantic meaning warning

changelog.d/datadog_logs_sink_reserved_attr_warn_clarity.enhancement.md

@@ -0,0 +1,4 @@
+Improved the warning log emitted by the `datadog_logs` sink when a field with a Datadog reserved attribute semantic meaning needs to be relocated but the destination path already exists. The log now includes `source_path`, `destination_path`, and `renamed_existing_to` fields to make the conflict easier to diagnose;
+additionally, it will now also increment a new counter `datadog_logs_reserved_attribute_conflicts_total`.
+
+authors: gwenaskell

lib/vector-common/src/internal_event/metric_name.rs

@@ -100,6 +100,7 @@ pub enum CounterName {
     MemoryEnrichmentTableReadsTotal,
     MemoryEnrichmentTableTtlExpirations,
     ComponentCpuUsageNsTotal,
+    DatadogLogsReservedAttributeConflictsTotal,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)]
@@ -366,6 +367,9 @@ impl CounterName {
             Self::MemoryEnrichmentTableReadsTotal => "memory_enrichment_table_reads_total",
             Self::MemoryEnrichmentTableTtlExpirations => "memory_enrichment_table_ttl_expirations",
             Self::ComponentCpuUsageNsTotal => "component_cpu_usage_ns_total",
+            Self::DatadogLogsReservedAttributeConflictsTotal => {
+                "datadog_logs_reserved_attribute_conflicts_total"
+            }
         }
     }
 }

src/internal_events/datadog_logs.rs

@@ -0,0 +1,30 @@
+use vector_lib::{
+    NamedInternalEvent, counter,
+    internal_event::{CounterName, InternalEvent},
+};
+use vrl::path::OwnedTargetPath;
+
+#[derive(Debug, NamedInternalEvent)]
+pub struct DatadogLogsReservedAttributeConflict<'a> {
+    pub meaning: &'static str,
+    pub source_path: &'a OwnedTargetPath,
+    pub destination_path: &'a str,
+    pub renamed_existing_to: &'a str,
+}
+
+impl InternalEvent for DatadogLogsReservedAttributeConflict<'_> {
+    fn emit(self) {
+        warn!(
+            message = "Relocated a field with semantic meaning to a Datadog reserved attribute, but the destination path already exists. The existing field was renamed to not overwrite.",
+            meaning = self.meaning,
+            source_path = %self.source_path,
+            destination_path = self.destination_path,
+            renamed_existing_to = self.renamed_existing_to,
+        );
+        counter!(
+            CounterName::DatadogLogsReservedAttributeConflictsTotal,
+            "meaning" => self.meaning,
+        )
+        .increment(1);
+    }
+}

src/internal_events/mod.rs

@@ -32,6 +32,8 @@ mod common;
 mod conditions;
 #[cfg(feature = "sources-datadog_agent")]
 mod datadog_agent;
+#[cfg(feature = "sinks-datadog_logs")]
+mod datadog_logs;
 #[cfg(feature = "sinks-datadog_metrics")]
 mod datadog_metrics;
 #[cfg(feature = "sinks-datadog_traces")]
@@ -186,6 +188,8 @@ pub(crate) use self::aws_kinesis_firehose::*;
 pub(crate) use self::aws_sqs::*;
 #[cfg(feature = "sources-datadog_agent")]
 pub(crate) use self::datadog_agent::*;
+#[cfg(feature = "sinks-datadog_logs")]
+pub(crate) use self::datadog_logs::*;
 #[cfg(feature = "sinks-datadog_metrics")]
 pub(crate) use self::datadog_metrics::*;
 #[cfg(feature = "sinks-datadog_traces")]

src/sinks/datadog/logs/sink.rs

@@ -13,6 +13,7 @@ use vrl::path::{OwnedSegment, OwnedTargetPath, PathPrefix};
 use super::{config::MAX_PAYLOAD_BYTES, service::LogApiRequest};
 use crate::{
     common::datadog::{DD_RESERVED_SEMANTIC_ATTRS, DDTAGS, MESSAGE, is_reserved_attribute},
+    internal_events::DatadogLogsReservedAttributeConflict,
     sinks::{
         prelude::*,
         util::{Compressor, http::HttpJsonBatchSizer},
@@ -184,7 +185,7 @@ pub fn position_reserved_attr_event_root(
     log: &mut LogEvent,
     current_path: &OwnedTargetPath,
     expected_field_name: &str,
-    meaning: &str,
+    meaning: &'static str,
 ) {
     // the path that DD archives expects this reserved attribute to be in.
     let desired_path = event_path!(expected_field_name);
@@ -196,11 +197,12 @@ pub fn position_reserved_attr_event_root(
         if log.contains(desired_path) {
             let rename_attr = format!("_RESERVED_{meaning}");
             let rename_path = event_path!(rename_attr.as_str());
-            warn!(
-                message = "Semantic meaning is defined, but the event path already exists. Renaming to not overwrite.",
-                meaning = meaning,
-                renamed = &rename_attr,
-            );
+            emit!(DatadogLogsReservedAttributeConflict {
+                meaning,
+                source_path: current_path,
+                destination_path: expected_field_name,
+                renamed_existing_to: &rename_attr,
+            });
             log.rename_key(desired_path, rename_path);
         }
 

website/cue/reference/components/sources/internal_metrics.cue

@@ -562,6 +562,12 @@ components: sources: internal_metrics: {
 			default_namespace: "vector"
 			tags: _component_tags & {output: _output}
 		}
+		datadog_logs_reserved_attribute_conflicts_total: {
+			description:       "The total number of conflicts encountered when relocating fields with semantic meaning to a Datadog reserved attribute."
+			type:              "counter"
+			default_namespace: "vector"
+			tags: _component_tags & {meaning: _meaning}
+		}
 		internal_metrics_cardinality: {
 			description:       "The total number of metrics emitted from the internal metrics registry."
 			type:              "gauge"
@@ -1258,6 +1264,20 @@ components: sources: internal_metrics: {
 			required:    true
 			examples: [_values.local_host]
 		}
+		_meaning: {
+			description: "The semantic meaning."
+			required:    true
+			enum: {
+				service:   "The service typically represents the application that generated the event."
+				message:   "The main text message of the event."
+				timestamp: "The main timestamp of the event."
+				host:      "The hostname of the machine where the event was generated."
+				tags:      "The tags of an event, generally a key-value paired list."
+				source:    "The source of the event."
+				severity:  "The severity of the event."
+				trace_id:  "The Id of the trace associated to the event."
+			}
+		}
 		_mode: {
 			description: "The connection mode used by the component."
 			required:    false