[uss_qualifier] Tolerate mock_uss clock mismatch in NetRID

.basedpyright/baseline.json

@@ -7013,14 +7013,6 @@
                     "lineCount": 1
                 }
             },
-            {
-                "code": "reportArgumentType",
-                "range": {
-                    "startColumn": 47,
-                    "endColumn": 66,
-                    "lineCount": 1
-                }
-            },
             {
                 "code": "reportOptionalMemberAccess",
                 "range": {

monitoring/mock_uss/routes.py

@@ -1,5 +1,6 @@
 import traceback
 
+import arrow
 import flask
 from werkzeug.exceptions import HTTPException
 
@@ -17,6 +18,11 @@ def status():
     )
 
 
+@webapp.route("/clock")
+def get_clock() -> str:
+    return arrow.utcnow().isoformat()
+
+
 @webapp.route("/favicon.ico")
 def favicon():
     flask.abort(404)

monitoring/monitorlib/fetch/__init__.py

@@ -398,6 +398,7 @@ class QueryType(StrEnum):
     )
 
     # InterUSS mock_uss
+    InterUSSMockUSSGetClock = "interuss.mock_uss.clock"
     InterUSSMockUSSGetLogs = "interuss.mock_uss.logging.interaction_logs"
     InterUSSMockUSSGetLocality = "interuss.mock_uss.locality.locality_get"
     InterUSSMockUSSSetLocality = "interuss.mock_uss.locality.locality_set"

monitoring/uss_qualifier/resources/interuss/mock_uss/client.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+import arrow
 from implicitdict import ImplicitDict, Optional, StringBasedDateTime
 
 from monitoring.monitorlib import fetch
@@ -90,6 +93,22 @@ def set_locality(self, locality_code: LocalityCode) -> fetch.Query:
             json=PutLocalityRequest(locality_code=locality_code),
         )
 
+    def get_clock(self) -> tuple[datetime | None, fetch.Query]:
+        query = fetch.query_and_describe(
+            self.session,
+            "GET",
+            "/clock",
+            participant_id=self.participant_id,
+            query_type=QueryType.InterUSSMockUSSGetClock,
+        )
+        try:
+            result = (
+                arrow.get(query.response.body).datetime if query.response.body else None
+            )
+        except arrow.ParserError:
+            result = None
+        return result, query
+
     # TODO: Add other methods to interact with the mock USS in other ways (like starting/stopping message signing data collection)
 
     def get_interactions(

monitoring/uss_qualifier/scenarios/astm/netrid/common/dp_behavior.py

@@ -133,7 +133,20 @@ def run(self, context: ExecutionContext):
         self.begin_test_case("Display Provider Behavior")
 
         for obs in self._observers:
-            test_step_start_time = arrow.utcnow().datetime
+            self.begin_test_step("Note remote clock")
+            test_case_start_time, query = self._mock_uss.get_clock()
+            self.record_query(query)
+            with self.check(
+                "mock_uss clock time retrievable", self._mock_uss.participant_id
+            ) as check:
+                if test_case_start_time is None:
+                    check.record_failed(
+                        "mock_uss clock time was not retrievable",
+                        f"mock_uss responded {query.response.status_code} without a valid clock time",
+                        queries=query,
+                    )
+            self.end_test_step()
+
             self.begin_test_step("Query acceptable diagonal area")
             # Query the DP for the exact area of the ISA
             self._step_query_ok_diagonal(obs)
@@ -148,9 +161,10 @@ def run(self, context: ExecutionContext):
             self._step_query_too_big_diagonal(obs)
             self.end_test_step()
 
-            self.begin_test_step("Verify query to SP")
-            self._step_validate_queries_to_sp(obs, test_step_start_time)
-            self.end_test_step()
+            if test_case_start_time:
+                self.begin_test_step("Verify query to SP")
+                self._step_validate_queries_to_sp(obs, test_case_start_time)
+                self.end_test_step()
 
         self.end_test_case()
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py

@@ -128,9 +128,20 @@ def _test_env_reqs(self):
                 "DSS instance is publicly addressable", [dss.participant_id]
             ) as check:
                 parsed_url = urlparse(dss.base_url)
-                ip_addr = socket.gethostbyname(parsed_url.hostname)
+                try:
+                    if not parsed_url.hostname:
+                        raise ValueError(
+                            f"Invalid hostname from urlparse: {parsed_url.hostname}"
+                        )
+                    ip_addr = socket.gethostbyname(parsed_url.hostname)
+                except (socket.gaierror, ValueError) as e:
+                    ip_addr = None
+                    check.record_failed(
+                        summary=f"DSS host {parsed_url.netloc} could not be checked for public addressability",
+                        details=f"DSS (URL: {dss.base_url}, netloc: {parsed_url.netloc}), could not resolve to an IP because {str(e)}",
+                    )
 
-                if ipaddress.ip_address(ip_addr).is_private:
+                if ip_addr and ipaddress.ip_address(ip_addr).is_private:
                     if self._allow_private_addresses:
                         check.skip()
                     else:

monitoring/uss_qualifier/scenarios/astm/netrid/common/sp_notification_behavior.py

@@ -2,7 +2,6 @@
 from datetime import datetime, timedelta
 from typing import TypeVar
 
-import arrow
 from s2sphere import LatLngRect
 
 from monitoring.monitorlib.clients.mock_uss.interactions import (
@@ -38,7 +37,10 @@
     get_mock_uss_interactions,
     query_type_filter,
 )
-from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.scenarios.scenario import (
+    GenericTestScenario,
+    ScenarioDidNotStopError,
+)
 from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 TOperationResult = TypeVar("TOperationResult")
@@ -101,7 +103,18 @@ def run(self, context: ExecutionContext):
         self.end_test_step()
 
         self.begin_test_step("Injection")
-        injection_time = arrow.utcnow().datetime
+        injection_time, query = self._mock_uss.get_clock()
+        self.record_query(query)
+        with self.check(
+            "mock_uss clock time retrievable", self._mock_uss.participant_id
+        ) as check:
+            if injection_time is None:
+                check.record_failed(
+                    "mock_uss clock time was not retrievable",
+                    f"mock_uss responded {query.response.status_code} without a valid clock time",
+                    queries=query,
+                )
+                raise ScenarioDidNotStopError(check)
         self._inject_flights()
         self.end_test_step()
 

monitoring/uss_qualifier/scenarios/astm/netrid/v19/dp_behavior.md

@@ -66,6 +66,14 @@ Prior to ISA creation, the Display Providers of one or more observers may have e
 
 ## Display Provider Behavior test case
 
+### Note remote clock test step
+
+To check whether the mock_uss, acting as Service Provider, received a valid request from the Display Provider due to the following test steps, we must known the earliest time such a request could have been made, according to mock_uss.  In this step, we retrieve mock_uss's clock time for "now" so we can only ask for interactions after "now" later in this case.
+
+#### ⚠️ mock_uss clock time retrievable check
+
+If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
+
 ### Query acceptable diagonal area test step
 
 This test step queries the Display Provider for the exact area of the ISA.

monitoring/uss_qualifier/scenarios/astm/netrid/v19/sp_notification_behavior.md

@@ -52,6 +52,10 @@ start and end time missing, provided all the required parameters are valid.
 
 In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future.  Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps.
 
+#### 🛑 mock_uss clock time retrievable check
+
+We need to know mock_uss's clock time to later request observed interactions after the injection time.  If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
+
 ### Validate Mock USS received notification test step
 
 This test step verifies that the mock_uss for which a subscription was registered before flight injection properly received a notification from each Service Provider

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dp_behavior.md

@@ -66,6 +66,14 @@ Prior to ISA creation, the Display Providers of one or more observers may have e
 
 ## Display Provider Behavior test case
 
+### Note remote clock test step
+
+To check whether the mock_uss, acting as Service Provider, received a valid request from the Display Provider due to the following test steps, we must known the earliest time such a request could have been made, according to mock_uss.  In this step, we retrieve mock_uss's clock time for "now" so we can only ask for interactions after "now" later in this case.
+
+#### ⚠️ mock_uss clock time retrievable check
+
+If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
+
 ### Query acceptable diagonal area test step
 
 This test step queries the Display Provider for the exact area of the ISA.

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/sp_notification_behavior.md

@@ -52,6 +52,10 @@ start and end time missing, provided all the required parameters are valid.
 
 In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future.  Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps.
 
+#### 🛑 mock_uss clock time retrievable check
+
+We need to know mock_uss's clock time to later request observed interactions after the injection time.  If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
+
 ### Validate Mock USS received notification test step
 
 This test step verifies that the mock_uss for which a subscription was registered before flight injection properly received a notification from each Service Provider

monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md

@@ -8,11 +8,11 @@
 2. Scenario: [ASTM NetRID Display Provider behavior](../../../scenarios/astm/netrid/v19/dp_behavior.md) ([`scenarios.astm.netrid.v19.DisplayProviderBehavior`](../../../scenarios/astm/netrid/v19/dp_behavior.py))
 3. Scenario: [ASTM NetRID networked UAS disconnection](../../../scenarios/astm/netrid/v19/networked_uas_disconnect.md) ([`scenarios.astm.netrid.v19.NetworkedUASDisconnect`](../../../scenarios/astm/netrid/v19/networked_uas_disconnect.py))
 4. Scenario: [ASTM NetRID Service Provider operator notification on missing fields](../../../scenarios/astm/netrid/v19/sp_operator_notify_missing_fields.md) ([`scenarios.astm.netrid.v19.SpOperatorNotifyMissingFields`](../../../scenarios/astm/netrid/v19/sp_operator_notify_missing_fields.py))
-5. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py)
+5. Scenario: [ASTM NetRID Service Provider notification behavior](../../../scenarios/astm/netrid/v19/sp_notification_behavior.md) ([`scenarios.astm.netrid.v19.ServiceProviderNotificationBehavior`](../../../scenarios/astm/netrid/v19/sp_notification_behavior.py))
+6. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py))
+7. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py))
+8. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py)
     1. Suite: [DSS testing for ASTM NetRID F3411-19](f3411_19/dss_probing.md) ([`suites.astm.netrid.f3411_19.dss_probing`](f3411_19/dss_probing.yaml))
-6. Scenario: [ASTM NetRID Service Provider notification behavior](../../../scenarios/astm/netrid/v19/sp_notification_behavior.md) ([`scenarios.astm.netrid.v19.ServiceProviderNotificationBehavior`](../../../scenarios/astm/netrid/v19/sp_notification_behavior.py))
-7. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py))
-8. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py))
 9. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py))
 
 ## [Checked requirements](../../README.md#checked-requirements)

monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml

@@ -48,6 +48,34 @@ actions:
         service_providers: service_providers
         evaluation_configuration: evaluation_configuration
     on_failure: Continue
+  - test_scenario:
+      scenario_type: scenarios.astm.netrid.v19.ServiceProviderNotificationBehavior
+      resources:
+        flights_data: flights_data
+        service_providers: service_providers
+        mock_uss: mock_uss_dp
+        id_generator: id_generator
+        dss_pool: dss_instances
+        uss_identification: uss_identification
+    on_failure: Continue
+  - test_scenario:
+      scenario_type: scenarios.astm.netrid.v19.NominalBehavior
+      resources:
+        flights_data: flights_data
+        service_providers: service_providers
+        observers: observers
+        evaluation_configuration: evaluation_configuration
+        dss_pool: dss_instances
+    on_failure: Continue
+  - test_scenario:
+      scenario_type: scenarios.astm.netrid.v19.Misbehavior
+      resources:
+        flights_data: flights_data
+        service_providers: service_providers
+        observers: observers
+        evaluation_configuration: evaluation_configuration
+        dss_pool: dss_instances
+      on_failure: Continue
   - action_generator:
       generator_type: action_generators.astm.f3411.ForEachDSS
       resources:
@@ -77,34 +105,6 @@ actions:
         dss_instances_source: dss_instances
         dss_instance_id: dss
       on_failure: Continue
-  - test_scenario:
-      scenario_type: scenarios.astm.netrid.v19.ServiceProviderNotificationBehavior
-      resources:
-        flights_data: flights_data
-        service_providers: service_providers
-        mock_uss: mock_uss_dp
-        id_generator: id_generator
-        dss_pool: dss_instances
-        uss_identification: uss_identification
-    on_failure: Continue
-  - test_scenario:
-      scenario_type: scenarios.astm.netrid.v19.NominalBehavior
-      resources:
-        flights_data: flights_data
-        service_providers: service_providers
-        observers: observers
-        evaluation_configuration: evaluation_configuration
-        dss_pool: dss_instances
-    on_failure: Continue
-  - test_scenario:
-      scenario_type: scenarios.astm.netrid.v19.Misbehavior
-      resources:
-        flights_data: flights_data
-        service_providers: service_providers
-        observers: observers
-        evaluation_configuration: evaluation_configuration
-        dss_pool: dss_instances
-      on_failure: Continue
   - test_scenario:
       scenario_type: scenarios.astm.netrid.v19.AggregateChecks
       resources:

monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md

@@ -9,11 +9,11 @@
 3. Scenario: [ASTM NetRID Display Provider behavior](../../../scenarios/astm/netrid/v22a/dp_behavior.md) ([`scenarios.astm.netrid.v22a.DisplayProviderBehavior`](../../../scenarios/astm/netrid/v22a/dp_behavior.py))
 4. Scenario: [ASTM NetRID networked UAS disconnection](../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.md) ([`scenarios.astm.netrid.v22a.NetworkedUASDisconnect`](../../../scenarios/astm/netrid/v22a/networked_uas_disconnect.py))
 5. Scenario: [ASTM NetRID Service Provider operator notification on missing fields](../../../scenarios/astm/netrid/v22a/sp_operator_notify_missing_fields.md) ([`scenarios.astm.netrid.v22a.SpOperatorNotifyMissingFields`](../../../scenarios/astm/netrid/v22a/sp_operator_notify_missing_fields.py))
-6. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py)
+6. Scenario: [ASTM NetRID Service Provider notification behavior](../../../scenarios/astm/netrid/v22a/sp_notification_behavior.md) ([`scenarios.astm.netrid.v22a.ServiceProviderNotificationBehavior`](../../../scenarios/astm/netrid/v22a/sp_notification_behavior.py))
+7. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py))
+8. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v22a/misbehavior.md) ([`scenarios.astm.netrid.v22a.Misbehavior`](../../../scenarios/astm/netrid/v22a/misbehavior.py))
+9. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py)
     1. Suite: [DSS testing for ASTM NetRID F3411-22a](f3411_22a/dss_probing.md) ([`suites.astm.netrid.f3411_22a.dss_probing`](f3411_22a/dss_probing.yaml))
-7. Scenario: [ASTM NetRID Service Provider notification behavior](../../../scenarios/astm/netrid/v22a/sp_notification_behavior.md) ([`scenarios.astm.netrid.v22a.ServiceProviderNotificationBehavior`](../../../scenarios/astm/netrid/v22a/sp_notification_behavior.py))
-8. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py))
-9. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v22a/misbehavior.md) ([`scenarios.astm.netrid.v22a.Misbehavior`](../../../scenarios/astm/netrid/v22a/misbehavior.py))
 10. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py))
 
 ## [Checked requirements](../../README.md#checked-requirements)