[fm] mark ereports included in a sitrep as "seen"

dev-tools/omdb/src/bin/omdb/db/ereport.rs

@@ -259,6 +259,7 @@ async fn cmd_db_ereport_info(
     const SLED_ID: &str = "  sled ID";
     const PART_NUMBER: &str = "  part number";
     const SERIAL_NUMBER: &str = "  serial number";
+    const MARKED_SEEN_IN: &str = "marked seen in sitrep";
     const WIDTH: usize = const_max_len(&[
         CLASS,
         TIME_COLLECTED,
@@ -268,6 +269,7 @@ async fn cmd_db_ereport_info(
         SLED_ID,
         PART_NUMBER,
         SERIAL_NUMBER,
+        MARKED_SEEN_IN,
     ]);
     let db::model::Ereport {
         ena: DbEna(ena),
@@ -280,6 +282,7 @@ async fn cmd_db_ereport_info(
         ref class,
         ref report,
         reporter,
+        marked_seen_in,
     } = ereport;
     println!("\n{:=<80}", "== EREPORT METADATA ");
     println!("    {ENA:>WIDTH$}: {ena}");
@@ -319,6 +322,7 @@ async fn cmd_db_ereport_info(
         "    {SERIAL_NUMBER:>WIDTH$}: {}",
         serial_number.as_deref().unwrap_or("<unknown>")
     );
+    println!("    {MARKED_SEEN_IN:>WIDTH$}: {marked_seen_in:?}",);
 
     println!("\n{:=<80}", "== EREPORT ");
     serde_json::to_writer_pretty(std::io::stdout(), &report)

dev-tools/omdb/src/bin/omdb/db/sitrep.rs

@@ -238,7 +238,7 @@ async fn cmd_db_sitrep_show(
         }
     };
 
-    let fm::Sitrep { metadata, cases } = sitrep;
+    let fm::Sitrep { metadata, cases, ereports_by_id } = sitrep;
     let fm::SitrepMetadata {
         id,
         creator_id,
@@ -259,6 +259,7 @@ async fn cmd_db_sitrep_show(
     const INV_COLLECTION_ID: &'static str = "inventory collection ID";
     const INV_STARTED_AT: &'static str = "  started at";
     const INV_FINISHED_AT: &'static str = "  finished at";
+    const TOTAL_EREPORTS: &'static str = "ereports in this sitrep";
 
     const WIDTH: usize = const_max_len(&[
         ID,
@@ -272,6 +273,7 @@ async fn cmd_db_sitrep_show(
         INV_COLLECTION_ID,
         INV_STARTED_AT,
         INV_FINISHED_AT,
+        TOTAL_EREPORTS,
     ]);
 
     println!("\n{:=<80}", "== FAULT MANAGEMENT SITUATION REPORT ");
@@ -344,6 +346,9 @@ async fn cmd_db_sitrep_show(
             )
         }
     }
+    println!("    {TOTAL_EREPORTS}: {}", ereports_by_id.len());
+    // TODO(eliza): perhaps display a table summarizing those ereports? possibly
+    // behind a verbose flag?
 
     if !cases.is_empty() {
         println!("\n{:-<80}\n", "== CASES");

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -60,6 +60,7 @@ use nexus_types::internal_api::background::BlueprintRendezvousStatus;
 use nexus_types::internal_api::background::DatasetsRendezvousStats;
 use nexus_types::internal_api::background::EreporterStatus;
 use nexus_types::internal_api::background::FmAlertStats;
+use nexus_types::internal_api::background::FmEreportMarkingStats;
 use nexus_types::internal_api::background::FmRendezvousStatus;
 use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
@@ -3499,9 +3500,10 @@ fn print_task_fm_rendezvous(details: &serde_json::Value) {
         Ok(FmRendezvousStatus::NoSitrep) => {
             println!("    no FM situation report loaded");
         }
-        Ok(FmRendezvousStatus::Executed { sitrep_id, alerts }) => {
+        Ok(FmRendezvousStatus::Executed { sitrep_id, alerts, marking }) => {
             println!("    current sitrep: {sitrep_id}");
             display_fm_alert_stats(&alerts);
+            display_fm_ereport_marking_stats(&marking);
         }
     }
 }
@@ -3545,6 +3547,59 @@ fn display_fm_alert_stats(stats: &FmAlertStats) {
     }
 }
 
+fn display_fm_ereport_marking_stats(stats: &FmEreportMarkingStats) {
+    let FmEreportMarkingStats {
+        batch_size,
+        batches,
+        total_ereports_in_sitrep,
+        ereports_not_marked_in_sitrep,
+        ereports_marked_seen,
+        errors,
+    } = stats;
+    pub const IN_SITREP: &str = "total ereports in sitrep:";
+    pub const NOT_ALREADY_MARKED: &str =
+        "not marked when the sitrep was loaded:";
+    pub const MARKED_SEEN: &str = "  marked seen by this activation:";
+    pub const ALREADY_MARKED: &str = "  already marked seen:";
+    pub const BATCH_SIZE: &str = "batch size:";
+    pub const BATCHES: &str = "batches:";
+    pub const ERRORS: &str = "errors:";
+    pub const WIDTH: usize = const_max_len(&[
+        IN_SITREP,
+        MARKED_SEEN,
+        ALREADY_MARKED,
+        ERRORS,
+        BATCH_SIZE,
+        BATCHES,
+    ]) + 1;
+    pub const NUM_WIDTH: usize = 4;
+    println!("    {IN_SITREP:<WIDTH$}{total_ereports_in_sitrep:>NUM_WIDTH$}");
+    println!(
+        "    {NOT_ALREADY_MARKED:<WIDTH$}{ereports_not_marked_in_sitrep:>NUM_WIDTH$}"
+    );
+    println!("    {MARKED_SEEN:<WIDTH$}{ereports_marked_seen:>NUM_WIDTH$}");
+    // This subtraction really shouldn't underflow, since
+    // `ereports_marked_seen`, which is the sum of records updated by the
+    // queries marking ereports as seen, will always be less than or equal to
+    // `ereports_not_marked_in_sitrep` which is the number of ereport IDs passed
+    // as *inputs* to those queries. But, since OMDB needs to basically work
+    // even in the face of Nexus bugs, we'll saturate here instead of panicking,
+    // just in case.
+    let already_marked =
+        ereports_not_marked_in_sitrep.saturating_sub(*ereports_marked_seen);
+    println!("    {ALREADY_MARKED:<WIDTH$}{already_marked:>NUM_WIDTH$}");
+    println!("    {BATCH_SIZE:<WIDTH$}{batch_size:>NUM_WIDTH$}");
+    println!("    {BATCHES:<WIDTH$}{batches:>NUM_WIDTH$}");
+    println!(
+        "{} {ERRORS:<WIDTH$}{:>NUM_WIDTH$}",
+        warn_if_nonzero(errors.len()),
+        errors.len()
+    );
+    for error in errors {
+        println!("      > {error}");
+    }
+}
+
 fn print_task_trust_quorum_manager(details: &serde_json::Value) {
     let status = match serde_json::from_value::<TrustQuorumManagerStatus>(
         details.clone(),

nexus/db-model/src/ereport.rs

@@ -16,7 +16,9 @@ use diesel::sql_types;
 use nexus_db_schema::schema::ereport;
 use nexus_types::fm::ereport::{self as types, Ena, EreportId};
 use omicron_common::api::external::Error;
-use omicron_uuid_kinds::{EreporterRestartKind, OmicronZoneKind, SledKind};
+use omicron_uuid_kinds::{
+    EreporterRestartKind, OmicronZoneKind, SitrepKind, SledKind,
+};
 use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 
@@ -97,6 +99,25 @@ pub struct Ereport {
 
     #[diesel(embed)]
     pub reporter: Reporter,
+
+    /// The sitrep ID of the sitrep which was being executed by `fm_rendezvous`
+    /// when this ereport was marked as "seen".
+    ///
+    /// If this is `Some`, the ereport has *definitely* been seen by at least
+    /// one committed sitrep at some point in time. If it is None`, the
+    /// ereport may or may not have been included in a sitrep, and you will
+    /// have to actually check the sitrep to find out.
+    ///
+    /// When this is `Some`, the value is the ID of the sitrep which the
+    /// `fm_rendezvous` task was executing when this ereport was marked as seen.
+    /// because execution may lag arbitrarily behind the generation of new
+    /// sitreps, this does *not* indicate that this was the *first* sitrep in
+    /// which this ereport was seen (which is why this is called "marked seen
+    /// in" rather than "first seen in" or similar) --- in general, this field
+    /// should basically just be treated as a `bool` and the actual value of the
+    /// sitrep ID is included only to provide *some* record for human-readable
+    /// debugging purposes.
+    pub marked_seen_in: Option<DbTypedUuid<SitrepKind>>,
 }
 
 #[derive(Copy, Clone, Debug, Insertable, Queryable, Selectable)]
@@ -175,13 +196,19 @@ impl Ereport {
             class,
             report,
             reporter: reporter.into(),
+            marked_seen_in: None,
         }
     }
 }
 
 impl From<types::Ereport> for Ereport {
-    fn from(types::Ereport { data, reporter }: types::Ereport) -> Self {
-        Self::new(data, reporter)
+    fn from(
+        types::Ereport { data, reporter, marked_seen_in }: types::Ereport,
+    ) -> Self {
+        Self {
+            marked_seen_in: marked_seen_in.map(Into::into),
+            ..Self::new(data, reporter)
+        }
     }
 }
 
@@ -197,6 +224,7 @@ impl TryFrom<Ereport> for types::Ereport {
             class,
             report,
             reporter,
+            marked_seen_in,
             ..
         } = ereport;
         let reporter = reporter.try_into().map_err(|e: Error| {
@@ -215,6 +243,7 @@ impl TryFrom<Ereport> for types::Ereport {
                 report,
             },
             reporter,
+            marked_seen_in: marked_seen_in.map(Into::into),
         })
     }
 }

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(245, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(246, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(246, "ereport-marked-seen"),
         KnownVersion::new(245, "rename-default-igw-ip-pool"),
         KnownVersion::new(244, "ereporter-restart-order-is-bad-actually"),
         KnownVersion::new(243, "ereporter-restart-order"),