From 381fc1ea5b20234835fdaccbaef4bdb2d63ba949 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Wed, 27 May 2026 23:49:22 +0000 Subject: [PATCH] Fix checks assuming perfect clock sync --- .../get_op_data_validation.md | 16 +++++ .../get_op_data_validation.py | 58 +++++++++++++++++-- .../receive_notifications_for_awareness.md | 8 +++ .../receive_notifications_for_awareness.py | 34 +++++++++-- .../flight_planning/injection_evaluation.py | 9 ++- 5 files changed, 115 insertions(+), 10 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index c9bcf3acbc..4cb77c86ae 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -30,6 +30,10 @@ DSSInstanceResource that provides access to a DSS instance where flight creation ### mock_uss plans flight 2 test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Plan successfully](../../../flight_planning/plan_flight_intent.md) Flight 2 should be successfully planned by mock_uss. @@ -38,6 +42,10 @@ Flight 2 should be successfully planned by mock_uss. ### tested_uss plans flight 1 test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Plan successfully](../../../flight_planning/plan_flight_intent.md) The test driver instructs tested_uss to attempt to plan flight 1. tested_uss checks if any conflicts with flight 2 @@ -67,6 +75,10 @@ In this test case, mock_uss is manipulated to share invalid operational intent d ### mock_uss plans flight 2, sharing invalid operational intent data test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Plan successfully](../../../flight_planning/plan_flight_intent.md) Flight 2 should be successfully planned by the mock_uss. @@ -77,6 +89,10 @@ The mock_uss is instructed to share invalid data with other USS, for negative te ### tested_uss attempts to plan flight 1, expect failure test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Plan unsuccessfully](test_steps/plan_flight_intent_expect_failed.md) The test driver instructs tested_uss to plan flight 1. tested_uss should (per SCD0035) check if any conflicts with flight 2 diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 74eda03715..0a9237942a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -1,4 +1,3 @@ -import arrow from uas_standards.astm.f3548.v21.api import EntityID from uas_standards.astm.f3548.v21.constants import Scope @@ -52,6 +51,7 @@ ) from monitoring.uss_qualifier.scenarios.scenario import ( ScenarioCannotContinueError, + ScenarioDidNotStopError, TestScenario, ) from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -148,13 +148,25 @@ def _plan_successfully_test_case(self): flight_2 = self.flight_2.resolve(self.time_context.evaluate_now()) self.begin_test_step("mock_uss plans flight 2") + t0, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t0 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_2_planning_time = Time(t0) with OpIntentValidator( self, self.mock_uss_client, self.dss, flight_2.basic_information.area.bounding_volume.to_f3548v21(), ) as validator: - flight_2_planning_time = Time(arrow.utcnow().datetime) _, self.flight_2_id, as_planned = plan_flight( self, self.mock_uss_client, @@ -170,13 +182,25 @@ def _plan_successfully_test_case(self): flight_1 = self.flight_1.resolve(self.time_context.evaluate_now()) self.begin_test_step("tested_uss plans flight 1") + t1, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t1 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_1_planning_time = Time(t1) with OpIntentValidator( self, self.tested_uss_client, self.dss, flight_1.basic_information.area.bounding_volume.to_f3548v21(), ) as validator: - flight_1_planning_time = Time(arrow.utcnow().datetime) plan_res, self.flight_1_id, as_planned = plan_flight( self, self.tested_uss_client, @@ -239,13 +263,25 @@ def _plan_unsuccessfully_test_case(self): self.begin_test_step( "mock_uss plans flight 2, sharing invalid operational intent data" ) + t2, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t2 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_2_planning_time = Time(t2) with OpIntentValidator( self, self.mock_uss_client, self.dss, flight_info.basic_information.area.bounding_volume.to_f3548v21(), ) as validator: - flight_2_planning_time = Time(arrow.utcnow().datetime) _, self.flight_2_id, as_planned = plan_flight( self, self.mock_uss_client, @@ -264,13 +300,25 @@ def _plan_unsuccessfully_test_case(self): flight_1 = self.flight_1.resolve(self.time_context.evaluate_now()) self.begin_test_step("tested_uss attempts to plan flight 1, expect failure") + t3, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t3 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_1_planning_time = Time(t3) with OpIntentValidator( self, self.tested_uss_client, self.dss, flight_1.basic_information.area.bounding_volume.to_f3548v21(), ) as validator: - flight_1_planning_time = Time(arrow.utcnow().datetime) _, self.flight_1_id, _ = submit_flight( self, "Plan should fail", diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md index 5cee80e703..55a34bf99b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md @@ -89,6 +89,10 @@ Notifications of relevant flights will be sent to tested_uss using this subscrip ### Mock_uss plans Flight 2 test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Plan Flight 2](../../../../flight_planning/plan_flight_intent.md) The test driver successfully plans Flight 2 via the mock uss, as there is no conflict with Flight 1. @@ -106,6 +110,10 @@ This test case verifies that relevant notifications for modified operational int ### Mock_uss modifies planned Flight 2 test step +#### 🛑 mock_uss clock time retrievable check + +We need to know mock_uss's clock time to later request observed interactions after the planning time. If mock_uss's current time isn't retrievable, the mock_uss provider's mock_uss does not meet **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + #### [Modify Flight 2](../../../../flight_planning/modify_planned_flight_intent.md) The test driver successfully modifies planned Flight 2 via the mock uss, as there is still no conflict with Flight 1. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py index 9ce0ac0ca6..d26cc70882 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py @@ -1,4 +1,3 @@ -import arrow from uas_standards.astm.f3548.v21.api import OperationalIntentReference from uas_standards.astm.f3548.v21.constants import Scope @@ -35,7 +34,10 @@ modify_planned_flight, plan_flight, ) -from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.scenario import ( + ScenarioDidNotStopError, + TestScenario, +) from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -185,13 +187,25 @@ def _receive_notification_successfully_when_activated_test_case(self): self.end_test_step() self.begin_test_step("Mock_uss plans Flight 2") + t0, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t0 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_2_planning_time = t0 with OpIntentValidator( self, self.mock_uss_client, self.dss, resolved_extents, ) as validator: - flight_2_planning_time = arrow.utcnow().datetime _, self.flight_2_id, as_planned = plan_flight( self, self.mock_uss_client, @@ -222,6 +236,19 @@ def _receive_notification_successfully_when_activated_modified_test_case(self): ) self.begin_test_step("Mock_uss modifies planned Flight 2") + t1, query = self.mock_uss.get_clock() + self.record_query(query) + with self.check( + "mock_uss clock time retrievable", self.mock_uss.participant_id + ) as check: + if t1 is None: + check.record_failed( + "mock_uss clock time was not retrievable", + f"mock_uss responded {query.response.status_code} without a valid clock time; is mock_uss running the latest version of `monitoring`?", + queries=query, + ) + raise ScenarioDidNotStopError(check) + flight_2_modif_time = t1 with OpIntentValidator( self, self.mock_uss_client, @@ -229,7 +256,6 @@ def _receive_notification_successfully_when_activated_modified_test_case(self): flight_2_planned_modified.basic_information.area.bounding_volume.to_f3548v21(), self.flight_2_oi_ref, ) as validator: - flight_2_modif_time = arrow.utcnow().datetime _, as_planned = modify_planned_flight( self, self.mock_uss_client, diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/injection_evaluation.py b/monitoring/uss_qualifier/scenarios/flight_planning/injection_evaluation.py index 28f125e376..c921a4fbbd 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/injection_evaluation.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/injection_evaluation.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Iterable +from datetime import timedelta from numbers import Number from types import NoneType from typing import Any @@ -6,6 +7,9 @@ import arrow import bc_jsonpath_ng from implicitdict import StringBasedDateTime +from uas_standards.astm.f3548.v21.constants import ( + TimeSyncMaxDifferentialSeconds, +) from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo from monitoring.monitorlib.dicts import JSONAddress, JSONPath @@ -79,7 +83,10 @@ def times_not_later_than_specified_or_now( return now = arrow.utcnow().datetime latest = now if as_requested.datetime < now else as_requested.datetime - if as_planned.datetime > latest: + # Different servers can have different clocks. Use max time skew from ASTM F3548-21. + if as_planned.datetime > latest + timedelta( + seconds=2 * TimeSyncMaxDifferentialSeconds + ): check.record_failed( f"Planned time {as_planned} is too late", details=f"Requested time no later than {as_requested} (or now, at {now}) for {address}, but planned time was {as_planned} which is later than the latest time allowed of {latest}",