Adds new metric tag to Manager to denote primary

core/src/main/java/org/apache/accumulo/core/metrics/MetricsInfo.java

@@ -109,12 +109,22 @@ static Collection<Tag> serviceTags(final String instanceName, final String appli
 
   void addMetricsProducers(MetricsProducer... producer);
 
+  /**
+   * @return true if {@code #init(Collection)} has been called, false otherwise
+   */
+  boolean isInitialized();
+
   /**
    * Initialize the metrics system. This sets the list of common tags that are emitted with the
    * metrics.
    */
   void init(Collection<Tag> commonTags);
 
+  /**
+   * Clears all Meters and re-registers all MetricsProducers
+   */
+  void reinit(Collection<Tag> replacementTags);
+
   /**
    * Close the underlying registry and release resources. The registry will not accept new meters
    * and will stop publishing metrics.

core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java

@@ -763,6 +763,10 @@ public void setMeterRegistry(MeterRegistry r) {
     }
   }
 
+  public void clearMeterRegistry() {
+    registry.set(null);
+  }
+
   /**
    * Called by MetricsInfoImpl.init on the server side if metrics are disabled. ClientContext calls
    * {@code #getClientThreadPools(AccumuloConfiguration, UncaughtExceptionHandler)} above.

server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java

@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.accumulo.core.classloader.ClassLoaderUtil;
 import org.apache.accumulo.core.conf.Property;
@@ -55,8 +56,6 @@ public class MetricsInfoImpl implements MetricsInfo {
 
   private final ServerContext context;
 
-  private List<Tag> commonTags = null;
-
   // JvmGcMetrics are declared with AutoCloseable - keep reference to use with close()
   private JvmGcMetrics jvmGcMetrics;
   // Log4j2Metrics and LogbackMetrics are declared with AutoCloseable - keep reference to use with
@@ -65,7 +64,9 @@ public class MetricsInfoImpl implements MetricsInfo {
 
   private final boolean metricsEnabled;
 
-  private final List<MetricsProducer> producers = new ArrayList<>();
+  private List<Tag> commonTags = null;
+  private final List<MetricsProducer> initializedProducers = new ArrayList<>();
+  private final AtomicReference<String> configuration = new AtomicReference<>();
 
   public MetricsInfoImpl(final ServerContext context) {
     this.context = context;
@@ -103,13 +104,17 @@ public synchronized void addMetricsProducers(MetricsProducer... producer) {
       return;
     }
 
-    if (commonTags == null) {
-      producers.addAll(Arrays.asList(producer));
-    } else {
+    initializedProducers.addAll(Arrays.asList(producer));
+    if (isInitialized()) {
       Arrays.stream(producer).forEach(p -> p.registerMetrics(Metrics.globalRegistry));
     }
   }
 
+  @Override
+  public synchronized boolean isInitialized() {
+    return configuration.get() != null;
+  }
+
   @Override
   public synchronized void init(Collection<Tag> tags) {
     Objects.requireNonNull(tags);
@@ -120,7 +125,7 @@ public synchronized void init(Collection<Tag> tags) {
       return;
     }
 
-    if (commonTags != null) {
+    if (isInitialized()) {
       LOG.warn("metrics registry has already been initialized");
       return;
     }
@@ -141,10 +146,7 @@ public synchronized void init(Collection<Tag> tags) {
       }
     }
 
-    commonTags = List.copyOf(tags);
-
-    LOG.info("Metrics initialization. common tags: {}", commonTags);
-
+    commonTags = new ArrayList<>(tags);
     Metrics.globalRegistry.config().commonTags(commonTags);
 
     boolean jvmMetricsEnabled =
@@ -197,8 +199,30 @@ public synchronized void init(Collection<Tag> tags) {
             Property.GENERAL_MICROMETER_LOG_METRICS.getKey());
     }
 
-    LOG.info("Metrics initialization. Register producers: {}", producers);
-    producers.forEach(p -> p.registerMetrics(Metrics.globalRegistry));
+    initializedProducers.forEach(p -> p.registerMetrics(Metrics.globalRegistry));
+    configuration.set("Producers: " + initializedProducers + ", common tags: " + commonTags);
+    LOG.info("Metrics initialized. " + configuration.get());
+  }
+
+  @Override
+  public void reinit(Collection<Tag> replacementTags) {
+    LOG.info("Reinitializing Metrics");
+    List<Tag> removals = new ArrayList<>();
+    for (Tag r : replacementTags) {
+      for (Tag t : commonTags) {
+        if (t.getKey().equals(r.getKey())) {
+          removals.add(t);
+        }
+      }
+    }
+    commonTags.removeAll(removals);
+    commonTags.addAll(replacementTags);
+    Metrics.globalRegistry.getRegistries().forEach(r -> r.close());
+    Metrics.globalRegistry.getRegistries().forEach(r -> Metrics.globalRegistry.remove(r));
+    Metrics.globalRegistry.clear();
+    configuration.set(null);
+    ThreadPools.getServerThreadPools().clearMeterRegistry();
+    init(commonTags);
   }
 
   @VisibleForTesting
@@ -239,6 +263,10 @@ public synchronized void close() {
 
   @Override
   public synchronized String toString() {
-    return "MetricsCommonTags{tags=" + commonTags + '}';
+    String msg = configuration.get();
+    if (msg == null) {
+      return "MetricsInfo not initialized yet.";
+    }
+    return msg;
   }
 }

server/manager/src/main/java/org/apache/accumulo/manager/Manager.java

@@ -170,6 +170,7 @@
 
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.context.Scope;
 
@@ -185,6 +186,9 @@ public class Manager extends AbstractServer
 
   static final Logger log = LoggerFactory.getLogger(Manager.class);
 
+  // visible for testing
+  public static final String PRIMARY_TAG_KEY = "manager.primary";
+
   // When in safe mode totalAssignedOrHosted() is called every 10s
   // which logs 3 messages about assigned tablets, 1 message
   // per TabletGroupWatcher. This DeduplicatingLogger slows
@@ -944,14 +948,19 @@ private void checkForHeldServer(SortedMap<TServerInstance,TabletServerStatus> ts
   // This is called after getting the assistant manager lock
   private void setupAssistantMetrics(MetricsProducer... producers) {
     MetricsInfo metricsInfo = getContext().getMetricsInfo();
-    metricsInfo.addMetricsProducers(producers);
-    metricsInfo.init(MetricsInfo.serviceTags(getContext().getInstanceName(), getApplicationName(),
+    List<Tag> tags = new ArrayList<>();
+    tags.addAll(MetricsInfo.serviceTags(getContext().getInstanceName(), getApplicationName(),
         getAdvertiseAddress(), getResourceGroup()));
+    // Add the primary tag
+    tags.add(Tag.of(PRIMARY_TAG_KEY, "false"));
+    metricsInfo.init(tags);
+    metricsInfo.addMetricsProducers(producers);
   }
 
   // This is called after getting the primary manager lock
   private void setupPrimaryMetrics() {
     MetricsInfo metricsInfo = getContext().getMetricsInfo();
+    metricsInfo.reinit(List.of(Tag.of(PRIMARY_TAG_KEY, "true")));
     metricsInfo.addMetricsProducers(balanceManager.getMetrics());
     // ensure all tablet group watchers are setup
     Preconditions.checkState(watchers.size() == DataLevel.values().length);