From e2e4a66b9d7a3c2863a8b9b8398f395061cc7970 Mon Sep 17 00:00:00 2001
From: Charles Yu <charles.yu@datadoghq.com>
Date: Fri, 9 Jan 2026 14:36:09 -0500
Subject: [PATCH] [DJM-0000] Debug Databricks

---
 .../spark/AbstractDatadogSparkListener.java    |  4 ++++
 .../main/java/datadog/trace/core/DDSpan.java   | 18 +++++++++---------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java b/dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
index 5b4e1501826..7b89f7f0829 100644
--- a/dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
+++ b/dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
@@ -399,6 +399,7 @@ private void addDatabricksSpecificTags(
 
       // ids to link those spans to databricks job/task traces
       builder.withTag("databricks_job_id", databricksJobId);
+      log.error("[CHARLES] addDatabricksSpecificTags databricksJobId: '{}'", databricksJobId);
       builder.withTag("databricks_job_run_id", databricksJobRunId);
       builder.withTag("databricks_task_run_id", databricksTaskRunId);
 
@@ -1085,6 +1086,7 @@ private long stageSpanKey(int stageId, int attemptId) {
   @SuppressForbidden // split with one-char String use a fast-path without regex usage
   private static String getDatabricksJobId(Properties properties) {
     String jobId = properties.getProperty("spark.databricks.job.id");
+    log.error("[CHARLES] getDatabricksJobId spark.databricks.job.id: '{}'", jobId);
     if (jobId != null) {
       return jobId;
     }
@@ -1092,6 +1094,7 @@ private static String getDatabricksJobId(Properties properties) {
     // First fallback, use spark.jobGroup.id with the pattern
     // <scheduler_id>_job-<job_id>-run-<task_run_id>-action-<action_id>
     String jobGroupId = properties.getProperty("spark.jobGroup.id");
+    log.error("[CHARLES] getDatabricksJobId spark.jobGroup.id: '{}'", jobGroupId);
     if (jobGroupId != null) {
       int startIndex = jobGroupId.indexOf("job-");
       int endIndex = jobGroupId.indexOf("-run", startIndex);
@@ -1103,6 +1106,7 @@ private static String getDatabricksJobId(Properties properties) {
     // Second fallback, use spark.databricks.workload.id with pattern
     // <org_id>-<job_id>-<task_run_id>
     String workloadId = properties.getProperty("spark.databricks.workload.id");
+    log.error("[CHARLES] getDatabricksJobId spark.databricks.workload.id: '{}'", workloadId);
     if (workloadId != null) {
       String[] parts = workloadId.split("-");
       if (parts.length > 1) {
diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java
index 1b321f4536a..46af9dea43b 100644
--- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java
+++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java
@@ -59,7 +59,7 @@ static DDSpan create(
       @Nonnull DDSpanContext context,
       final List<AgentSpanLink> links) {
     final DDSpan span = new DDSpan(instrumentationName, timestampMicro, context, links);
-    log.debug("Started span: {}", span);
+    log.error("Started span: {}", span);
     context.getTraceCollector().registerSpan(span);
     return span;
   }
@@ -160,9 +160,9 @@ private void finishAndAddToTrace(final long durationNano) {
       }
       this.metrics.onSpanFinished();
       TraceCollector.PublishState publishState = context.getTraceCollector().onPublish(this);
-      log.debug("Finished span ({}): {}", publishState, this);
+      log.error("Finished span ({}): {}", publishState, this);
     } else {
-      log.debug("Already finished: {}", this);
+      log.error("Already finished: {}", this);
     }
   }
 
@@ -224,7 +224,7 @@ public void finishWithEndToEnd() {
       try {
         e2eStart = null != value ? MILLISECONDS.toNanos(Long.parseLong(value)) : 0;
       } catch (RuntimeException e) {
-        log.debug("Ignoring invalid end-to-end start time {}", value, e);
+        log.error("Ignoring invalid end-to-end start time {}", value, e);
         e2eStart = 0;
       }
     } else {
@@ -254,10 +254,10 @@ public final boolean phasedFinish() {
     }
     // Flip the negative bit of the result to allow verifying that publish() is only called once.
     if (DURATION_NANO_UPDATER.compareAndSet(this, 0, Math.max(1, durationNano) | Long.MIN_VALUE)) {
-      log.debug("Finished span (PHASED): {}", this);
+      log.error("Finished span (PHASED): {}", this);
       return true;
     } else {
-      log.debug("Already finished: {}", this);
+      log.error("Already finished: {}", this);
       return false;
     }
   }
@@ -266,13 +266,13 @@ public final boolean phasedFinish() {
   public final void publish() {
     long durationNano = this.durationNano;
     if (durationNano == 0) {
-      log.debug("Can't publish unfinished span: {}", this);
+      log.error("Can't publish unfinished span: {}", this);
     } else if (durationNano > 0) {
-      log.debug("Already published: {}", this);
+      log.error("Already published: {}", this);
     } else if (DURATION_NANO_UPDATER.compareAndSet(
         this, durationNano, durationNano & Long.MAX_VALUE)) {
       TraceCollector.PublishState publishState = context.getTraceCollector().onPublish(this);
-      log.debug("Published span ({}): {}", publishState, this);
+      log.error("Published span ({}): {}", publishState, this);
     }
   }