diff --git a/.gitignore b/.gitignore index 3f8ff926..a48918e9 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ projects/intel_x86/linux/gcc/aether-client-cpp /build-windows*/ *.log + +.artifacts/ + diff --git a/CMakeLists.txt b/CMakeLists.txt index a9ceadd8..52315ce6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ option(AE_BUILD_TESTS "Build tests" ${AE_ROOT_PORJECT}) option(AE_BUILD_ANDROID_SMOKE "Build Android NDK smoke shared library and runner" Off) option(AE_ADDRESS_SANITIZE "Enable address sanitizer" Off) option(AE_NO_STRIP_ALL "Do not apply --strip_all, useful for bloaty and similar tools " Off) +option(AE_ENABLE_PING_TEST_FAULTS "Enable test-only ping request/response fault injection" Off) set(UTM_ID "0" CACHE STRING "User Tracking Measurement ID, must be a uint32 value") set(USER_CONFIG "" CACHE PATH "Path to user provided configuration header file") @@ -81,6 +82,7 @@ message(STATUS "Aether build options: AE_BUILD_ANDROID_SMOKE=${AE_BUILD_ANDROID_SMOKE} AE_ADDRESS_SANITIZE=${AE_ADDRESS_SANITIZE} AE_NO_STRIP_ALL=${AE_NO_STRIP_ALL} + AE_ENABLE_PING_TEST_FAULTS=${AE_ENABLE_PING_TEST_FAULTS} UTM_ID=${UTM_ID} USER_CONFIG=${USER_CONFIG} FS_INIT=${FS_INIT} @@ -268,6 +270,9 @@ endif() if (AE_FILTRATION) target_compile_definitions(${TARGET_NAME} PUBLIC "AE_FILTRATION=1") endif() +if (AE_ENABLE_PING_TEST_FAULTS) + target_compile_definitions(${TARGET_NAME} PUBLIC "AE_ENABLE_PING_TEST_FAULTS=1") +endif() # for debug purposes only, set registration server ip address if(NOT "${AE_REG_CLOUD_ADDR}" STREQUAL "") @@ -320,7 +325,7 @@ target_compile_options(${TARGET_NAME} PRIVATE target_compile_options(${TARGET_NAME} PUBLIC $<$: /wd4100 /wd4101 /wd4127 /wd4244 /wd4324 - /wd4456 /wd4459 /wd4714 + /wd4456 /wd4459 /wd4702 /wd4714 > ) @@ -386,6 +391,10 @@ if(AE_BUILD_EXAMPLES) add_subdirectory(examples/capi/oddity) add_subdirectory(examples/benches/send_message_delays) add_subdirectory(examples/benches/send_messages_bandwidth) + add_subdirectory(examples/benches/aether_uap_delivery_timing_bench) + add_subdirectory(examples/aether_uap_peer_deadline_test) + add_subdirectory(examples/aether_uap_ping_retry_window_test) + add_subdirectory(examples/aether_uap_1s_timing_characterization) endif() if(AE_BUILD_TESTS) diff --git a/aether/CMakeLists.txt b/aether/CMakeLists.txt index 2e0da7ba..aeef184b 100644 --- a/aether/CMakeLists.txt +++ b/aether/CMakeLists.txt @@ -71,7 +71,9 @@ list(APPEND aether_srcs "ae_actions/ping.cpp" "ae_actions/check_access_for_send_message.cpp" "ae_actions/telemetry.cpp" - "ae_actions/select_client.cpp") + "ae_actions/select_client.cpp" + "ae_actions/query_peer_receive_schedule.cpp" + "ae_actions/announce_next_ping_unknown.cpp") list(APPEND aether_srcs "registration/api/client_reg_api_safe.cpp" diff --git a/aether/actions/action_pool.h b/aether/actions/action_pool.h index 9b047ca8..1a0e4f9d 100644 --- a/aether/actions/action_pool.h +++ b/aether/actions/action_pool.h @@ -18,6 +18,7 @@ #define AETHER_ACTIONS_ACTION_POLL_H_ #include +#include #include #include "aether/warning_disable.h" @@ -59,11 +60,12 @@ class ActionPool : public etl::pool { private: void Destroy(T* p) { - ts_ = ac_.scheduler().Task([&, p]() { base_t::template destroy(p); }); + destroy_tasks_.push_back( + ac_.scheduler().Task([this, p]() { base_t::template destroy(p); })); } AC ac_; - TaskSubscription ts_; + std::vector destroy_tasks_; }; template @@ -96,11 +98,12 @@ class ActionPool, Capacity> private: void Destroy(Action* p) { - ts_ = ac_.scheduler().Task([&, p]() { base_t::destroy(p); }); + destroy_tasks_.push_back( + ac_.scheduler().Task([this, p]() { base_t::destroy(p); })); } AC ac_; - TaskSubscription ts_; + std::vector destroy_tasks_; }; } // namespace ae diff --git a/aether/ae_actions/announce_next_ping_unknown.cpp b/aether/ae_actions/announce_next_ping_unknown.cpp new file mode 100644 index 00000000..aaf18edc --- /dev/null +++ b/aether/ae_actions/announce_next_ping_unknown.cpp @@ -0,0 +1,90 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/ae_actions/announce_next_ping_unknown.h" + +#include + +#include "aether/client.h" +#include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/config.h" + +namespace ae { + +AnnounceNextPingUnknown::AnnounceNextPingUnknown(AeContext const& ae_context, + Client& client) + : ae_context_{ae_context}, client_{&client} { + start_sub_ = ae_context_.scheduler().Task([this]() { Start(); }); + if (!start_sub_) { + assert(false && "Task allocation failed"); + Fail(static_cast(AnnounceNextPingUnknownError::kAnnounceFailed)); + } +} + +AnnounceNextPingUnknown::~AnnounceNextPingUnknown() { finished_ = true; } + +AnnounceNextPingUnknown::ResultEvent::Subscriber +AnnounceNextPingUnknown::result_event() noexcept { + return EventSubscriber{result_event_}; +} + +void AnnounceNextPingUnknown::Start() { + if (finished_ || client_ == nullptr) { + return; + } +#if AE_ENABLE_PING + (void)client_->cloud_connection(); + auto* pings = client_->ping_cloud_servers(); + if (pings == nullptr) { + Fail(static_cast(AnnounceNextPingUnknownError::kNoPingManager)); + return; + } + announce_sub_ = pings->announce_event().Subscribe( + [this](Result const& res) { + if (!res) { + Fail(res.error() == 0 + ? static_cast( + AnnounceNextPingUnknownError::kAnnounceFailed) + : res.error()); + return; + } + CompleteOk(); + }); + pings->BeginAnnounceUnknown(); +#else + Fail(static_cast(AnnounceNextPingUnknownError::kPingDisabled)); +#endif +} + +void AnnounceNextPingUnknown::CompleteOk() { + if (finished_) { + return; + } + finished_ = true; + result_event_.Emit(Ok{std::monostate{}}); + Finish(); +} + +void AnnounceNextPingUnknown::Fail(int code) { + if (finished_) { + return; + } + finished_ = true; + result_event_.Emit(Error{code}); + Finish(); +} + +} // namespace ae diff --git a/aether/ae_actions/announce_next_ping_unknown.h b/aether/ae_actions/announce_next_ping_unknown.h new file mode 100644 index 00000000..58b40262 --- /dev/null +++ b/aether/ae_actions/announce_next_ping_unknown.h @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_AE_ACTIONS_ANNOUNCE_NEXT_PING_UNKNOWN_H_ +#define AETHER_AE_ACTIONS_ANNOUNCE_NEXT_PING_UNKNOWN_H_ + +#include "aether/config.h" + +#include + +#include "aether-miscpp/types/result.h" + +#include "aether/ae_context.h" +#include "aether/actions/action.h" +#include "aether/events/event_subscription.h" +#include "aether/events/events.h" +#include "aether/tasks/details/task_subsctiption.h" + +namespace ae { +class Client; + +enum class AnnounceNextPingUnknownError : int { + kPingDisabled = 1, + kNoPingManager = 2, + kAnnounceFailed = 3, +}; + +class AnnounceNextPingUnknown final : public Action { + public: + using ResultEvent = Event)>; + + AnnounceNextPingUnknown(AeContext const& ae_context, Client& client); + ~AnnounceNextPingUnknown() override; + + AE_CLASS_NO_COPY_MOVE(AnnounceNextPingUnknown) + + ResultEvent::Subscriber result_event() noexcept; + + private: + void Start(); + void CompleteOk(); + void Fail(int code); + + AeContext ae_context_; + Client* client_{nullptr}; + ResultEvent result_event_; + Subscription announce_sub_; + TaskSubscription start_sub_; + bool finished_{false}; +}; + +} // namespace ae + +#endif // AETHER_AE_ACTIONS_ANNOUNCE_NEXT_PING_UNKNOWN_H_ diff --git a/aether/ae_actions/ping.cpp b/aether/ae_actions/ping.cpp index bef22008..8943aa12 100644 --- a/aether/ae_actions/ping.cpp +++ b/aether/ae_actions/ping.cpp @@ -25,7 +25,11 @@ # include "aether/server.h" # include "aether/cloud_connections/cloud_server_connection.h" +# include "aether/cloud_connections/ping_schedule_guard.h" # include "aether/work_cloud_api/work_server_api/authorized_api.h" +# if AE_ENABLE_PING_TEST_FAULTS +# include "aether/ae_actions/ping_test_faults.h" +# endif # include "aether/ae_actions/ae_actions_tele.h" @@ -89,16 +93,39 @@ void Ping::Start(TimePoint current_time) { } state_ = RequestState::kPending; +#if AE_ENABLE_PING_TEST_FAULTS + if (test_fault_mode_ == + static_cast(PingFaultMode::kDropRequest)) { + request_start_ = current_time; + timeout_sub_ = ae_context_.scheduler().DelayedTask( + [this]() { PingResponseTimeout(RequestId{}); }, + current_time + timeout_); + if (state_ == RequestState::kPending && !timeout_sub_) { + AE_TELE_ERROR( + kPingTimeoutError, + "Ping timeout task allocation failed server id {} request {}", + server_id_, RequestId{}); + state_ = RequestState::kFinished; + ResetRequestSubscriptions(); + result_event_.Emit(PingResult{Error{5}}); + } + return; + } +#endif + auto& write_action = cc->AuthorizedApiCall( SubApi{[this, current_time](ApiContext& auth_api) { - auto next_ping_hint_ms = static_cast( - std::chrono::duration_cast( - next_ping_hint_) - .count()); - auto rx_window_ms = static_cast( - std::chrono::duration_cast(rx_window_) - .count()); - + auto next_ping_hint_ms = + next_ping_hint_.count() == 0 + ? std::int64_t{0} + : FloorDurationToPositiveInt64Ms(next_ping_hint_); + auto rx_window_ms = CeilDurationToSaturatedInt64Ms(rx_window_); + + // ping() is the full schedule contract: nextConnectMsDuration and + // rxWindowMs. Do not follow with set_next_read_delay(interval). +#if AE_ENABLE_PING_TEST_FAULTS + PingTestFaults::Instance().OnAuthPing(); +#endif auto pong_promise = auth_api->ping(next_ping_hint_ms, rx_window_ms); auto req_id = pong_promise.request_id(); @@ -110,6 +137,13 @@ void Ping::Start(TimePoint current_time) { auto wait_result_sub = pong_promise.Subscribe([this, req_id](auto&& res) { +#if AE_ENABLE_PING_TEST_FAULTS + PingTestFaults::Instance().OnProtocolResponse(); + if (test_fault_mode_ == + static_cast(PingFaultMode::kIgnoreResponse)) { + return; + } +#endif if (res) { PingResponse(req_id); } else { diff --git a/aether/ae_actions/ping.h b/aether/ae_actions/ping.h index 411deec6..4f43483c 100644 --- a/aether/ae_actions/ping.h +++ b/aether/ae_actions/ping.h @@ -21,6 +21,7 @@ #if AE_ENABLE_PING +# include # include # include "aether-miscpp/types/result.h" @@ -55,6 +56,10 @@ class Ping { void Start(TimePoint current_time); +#if AE_ENABLE_PING_TEST_FAULTS + void ApplyTestFault(std::uint8_t mode) noexcept { test_fault_mode_ = mode; } +#endif + private: void PingResponse(RequestId request_id); void PingResponseError(RequestId request_id, std::int32_t error_code); @@ -82,6 +87,9 @@ class Ping { ResultEvent result_event_; RequestState state_{RequestState::kCreated}; +#if AE_ENABLE_PING_TEST_FAULTS + std::uint8_t test_fault_mode_{0}; +#endif }; } // namespace ae #endif // AE_ENABLE_PING diff --git a/aether/ae_actions/ping_test_faults.h b/aether/ae_actions/ping_test_faults.h new file mode 100644 index 00000000..c317661a --- /dev/null +++ b/aether/ae_actions/ping_test_faults.h @@ -0,0 +1,250 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_AE_ACTIONS_PING_TEST_FAULTS_H_ +#define AETHER_AE_ACTIONS_PING_TEST_FAULTS_H_ + +#include "aether/config.h" + +#if AE_ENABLE_PING_TEST_FAULTS + +# include +# include +# include +# include + +# include "aether/clock.h" +# include "aether/types/server_id.h" + +namespace ae { + +enum class PingFaultHarnessState : std::uint8_t { + kIdle = 0, + kArmed = 1, + kMatched = 2, + kDropped = 3, + kConsumed = 4, +}; + +enum class PingFaultTraceKind : std::uint8_t { + kCleared = 0, + kArmed = 1, + kBound = 2, + kMatched = 3, + kDropped = 4, +}; + +enum class PingFaultMode : std::uint8_t { + kNone = 0, + kDropRequest = 1, + kIgnoreResponse = 2, +}; + +struct PingFaultTraceEvent { + PingFaultTraceKind kind{PingFaultTraceKind::kCleared}; + ServerId server_id{}; + std::uint64_t logical_cycle_id{0}; + std::uint32_t physical_attempt_index{0}; + PingFaultMode mode{PingFaultMode::kNone}; + PingFaultHarnessState harness_state{PingFaultHarnessState::kIdle}; + std::int64_t steady_us{0}; +}; + +using PingFaultTraceHook = void (*)(PingFaultTraceEvent const&); + +struct PingFaultContext { + ServerId server_id{}; + std::uint64_t logical_cycle_id{0}; + std::uint32_t physical_attempt_index{0}; + TimePoint planned_send_at{}; + TimePoint actual_send_at{}; +}; + +struct PingFaultDecision { + PingFaultMode mode{PingFaultMode::kNone}; + Duration timeout_override{}; +}; + +struct PingFaultPlan { + ServerId server_id{}; + std::uint64_t logical_cycle_id{0}; // 0 = next new cycle on this server + std::uint32_t physical_attempt_index{1}; + PingFaultMode mode{PingFaultMode::kNone}; + Duration timeout_override{}; + bool consumed{false}; + // Test-only: delay this attempt until nominal_ping_at + offset. + // Offset is send-time in the client clock, not a production schedule change. + bool hold_enabled{false}; + std::int64_t retry_hold_offset_us{0}; +}; + +class PingTestFaults { + public: + static PingTestFaults& Instance() noexcept { + static PingTestFaults inst; + return inst; + } + + static void SetTraceHook(PingFaultTraceHook hook) noexcept { + trace_hook_ = hook; + } + + PingFaultHarnessState harness_state() const noexcept { + return harness_state_; + } + + bool HasUnconsumedPlan() const noexcept { + for (auto const& plan : plans_) { + if (!plan.consumed) { + return true; + } + } + return false; + } + + void Clear() noexcept { + plans_.clear(); + auth_ping_calls_ = 0; + protocol_responses_ = 0; + harness_state_ = PingFaultHarnessState::kIdle; + EmitTrace(PingFaultTraceKind::kCleared, PingFaultPlan{}, 0, 0); + } + + void Arm(PingFaultPlan plan) { + plan.consumed = false; + plans_.push_back(plan); + harness_state_ = PingFaultHarnessState::kArmed; + EmitTrace(PingFaultTraceKind::kArmed, plan, plan.logical_cycle_id, + plan.physical_attempt_index); + } + + void BindNextCycle(ServerId server_id, std::uint64_t cycle_id) noexcept { + if (cycle_id == 0) { + return; + } + for (auto& plan : plans_) { + if (!plan.consumed && plan.server_id == server_id && + plan.logical_cycle_id == 0) { + plan.logical_cycle_id = cycle_id; + EmitTrace(PingFaultTraceKind::kBound, plan, cycle_id, + plan.physical_attempt_index); + } + } + } + + std::optional RetryHoldOffsetUs( + ServerId server_id, std::uint64_t cycle_id, + std::uint32_t physical_attempt_index) const noexcept { + for (auto const& plan : plans_) { + if (plan.consumed || !plan.hold_enabled) { + continue; + } + if (plan.server_id != server_id) { + continue; + } + if (plan.logical_cycle_id != 0 && plan.logical_cycle_id != cycle_id) { + continue; + } + if (plan.physical_attempt_index != physical_attempt_index) { + continue; + } + return plan.retry_hold_offset_us; + } + return std::nullopt; + } + + PingFaultDecision Consume(PingFaultContext const& ctx) noexcept { + for (auto& plan : plans_) { + if (plan.consumed) { + continue; + } + if (plan.server_id != ctx.server_id) { + continue; + } + if (plan.logical_cycle_id != 0 && + plan.logical_cycle_id != ctx.logical_cycle_id) { + continue; + } + if (plan.physical_attempt_index != ctx.physical_attempt_index) { + continue; + } + harness_state_ = PingFaultHarnessState::kMatched; + EmitTrace(PingFaultTraceKind::kMatched, plan, ctx.logical_cycle_id, + ctx.physical_attempt_index); + plan.consumed = true; + (void)ctx.planned_send_at; + (void)ctx.actual_send_at; + auto const decision = + PingFaultDecision{plan.mode, plan.timeout_override}; + if (plan.mode == PingFaultMode::kDropRequest) { + harness_state_ = PingFaultHarnessState::kDropped; + EmitTrace(PingFaultTraceKind::kDropped, plan, ctx.logical_cycle_id, + ctx.physical_attempt_index); + } + harness_state_ = PingFaultHarnessState::kConsumed; + return decision; + } + return PingFaultDecision{}; + } + + void OnAuthPing() noexcept { ++auth_ping_calls_; } + void OnProtocolResponse() noexcept { ++protocol_responses_; } + + std::uint32_t auth_ping_calls() const noexcept { return auth_ping_calls_; } + std::uint32_t protocol_responses() const noexcept { + return protocol_responses_; + } + + private: + static inline PingFaultTraceHook trace_hook_{nullptr}; + + static std::int64_t TraceSteadyUs() noexcept { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + + static void EmitTrace(PingFaultTraceKind kind, PingFaultPlan const& plan, + std::uint64_t logical_cycle_id, + std::uint32_t physical_attempt_index) noexcept { + if (trace_hook_ == nullptr) { + return; + } + PingFaultTraceEvent event{}; + event.kind = kind; + event.server_id = plan.server_id; + event.logical_cycle_id = logical_cycle_id; + event.physical_attempt_index = physical_attempt_index; + event.mode = plan.mode; + event.harness_state = PingTestFaults::Instance().harness_state_; + event.steady_us = TraceSteadyUs(); + trace_hook_(event); + } + + std::vector plans_{}; + std::uint32_t auth_ping_calls_{0}; + std::uint32_t protocol_responses_{0}; + PingFaultHarnessState harness_state_{PingFaultHarnessState::kIdle}; +}; + +inline void SetPingFaultTraceHook(PingFaultTraceHook hook) noexcept { + PingTestFaults::SetTraceHook(hook); +} + +} // namespace ae + +#endif // AE_ENABLE_PING_TEST_FAULTS +#endif // AETHER_AE_ACTIONS_PING_TEST_FAULTS_H_ diff --git a/aether/ae_actions/query_peer_receive_schedule.cpp b/aether/ae_actions/query_peer_receive_schedule.cpp new file mode 100644 index 00000000..54d8b29c --- /dev/null +++ b/aether/ae_actions/query_peer_receive_schedule.cpp @@ -0,0 +1,276 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/ae_actions/query_peer_receive_schedule.h" + +#include "aether/channels/channel.h" +#include "aether/client.h" +#include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/config.h" +#include "aether/connection_manager/client_cloud_manager.h" +#include "aether/server.h" +#include "aether/server_connections/client_server_connection.h" +#include "aether/tele.h" +#include "aether/work_cloud_api/work_server_api/authorized_api.h" + +namespace ae { + +QueryPeerReceiveSchedule::QueryPeerReceiveSchedule(AeContext const& ae_context, + Client& client, + Uid peer_uid) + : ae_context_{ae_context}, client_{&client}, peer_uid_{peer_uid} { + auto& get_cloud = client_->cloud_manager()->GetCloud(peer_uid_); + get_cloud_sub_ = get_cloud.result_event().Subscribe( + [this](Result result) { OnCloud(std::move(result)); }); +} + +QueryPeerReceiveSchedule::~QueryPeerReceiveSchedule() { + finished_ = true; + query_state_.Cancel(); + timing_subs_.clear(); +} + +QueryPeerReceiveSchedule::ResultEvent::Subscriber +QueryPeerReceiveSchedule::result_event() noexcept { + return EventSubscriber{result_event_}; +} + +std::vector const& +QueryPeerReceiveSchedule::server_diagnostics() const noexcept { + return diagnostics_; +} + +PeerTimingQueryCoverage QueryPeerReceiveSchedule::coverage() const noexcept { + return query_state_.QueryCoverage(); +} + +Duration QueryPeerReceiveSchedule::OneWayEstimateFor( + CloudServerConnection* sc) const { + auto const fallback = FallbackOneWayPingEstimate(); + if (sc == nullptr) { + return fallback; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return fallback; + } + auto channel = conn->server_connection().current_channel(); + if (!channel) { + return fallback; + } + auto const& stats = channel->channel_statistics().response_time_statistics(); + return OneWayPingEstimate(stats.empty(), + stats.empty() ? fallback : stats.min()); +} + +void QueryPeerReceiveSchedule::OnCloud(Result result) { + if (!result) { + Fail(static_cast(QueryPeerReceiveScheduleError::kGetCloudFailed)); + return; + } + auto cloud = std::move(result).value(); + dest_cloud_ = std::make_unique( + ae_context_, cloud.Load(), + client_->server_connection_manager().GetServerConnectionFactory(), + AE_CLOUD_MAX_SERVER_CONNECTIONS); + StartQuery(); +} + +void QueryPeerReceiveSchedule::SnapshotExpectedServers() { + std::vector expected; + PeerTimingQueryCoverage cov; + if (dest_cloud_ != nullptr) { + auto const& selected = dest_cloud_->selected_servers(); + cov.selected_server_count = selected.size(); + expected.reserve(selected.size()); + for (auto* sc : selected) { + if (sc == nullptr) { + continue; + } + if (sc->quarantine()) { + ++cov.quarantined_skipped_count; + continue; + } + if (!sc->server()) { + continue; + } + expected.push_back(sc->server_id()); + } + cov.queried_server_count = expected.size(); + } + query_state_.Begin(std::move(expected), false, cov); +} + +void QueryPeerReceiveSchedule::StartQuery() { + SnapshotExpectedServers(); + timing_subs_.clear(); + diagnostics_.clear(); + if (query_state_.expected_server_ids.empty()) { + Fail(static_cast( + QueryPeerReceiveScheduleError::kNoWorkServerAvailable)); + return; + } + + cloud_request_.emplace( + ae_context_, + ApiRequestHandler{[this](ApiContext& auth_api, + CloudServerConnection* sc, + CloudRequest* request) { + if (finished_ || query_state_.cancelled) { + return; + } + if (sc == nullptr || !sc->server() || sc->quarantine()) { + return; + } + auto const server_id = sc->server_id(); + auto existing = query_state_.attempts.find(server_id); + if (existing != query_state_.attempts.end() && + existing->second.status == ServerTimingAttemptStatus::kSuccess) { + return; + } + + auto const qsend = Now(); + auto const one_way = OneWayEstimateFor(sc); + auto const send_generation = + query_state_.RegisterSend(server_id, qsend, one_way); + + timing_subs_[server_id] = + auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, send_generation](auto const& res) { + OnServerTiming(sc, send_generation, res); + }); + static_cast(request); + }}, + *dest_cloud_, RequestPolicy::All{}); + + exhausted_sub_ = cloud_request_->attempt_exhausted_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + query_state_.MarkTerminalError(sc->server_id()); + MaybeComplete(); + }); + + cloud_request_sub_ = + cloud_request_->result_event().Subscribe([this](bool ok) { + if (finished_) { + return; + } + if (ok) { + return; + } + for (auto const id : query_state_.expected_server_ids) { + auto it = query_state_.attempts.find(id); + if (it == query_state_.attempts.end() || + (it->second.status != ServerTimingAttemptStatus::kSuccess && + it->second.status != + ServerTimingAttemptStatus::kTerminalError)) { + query_state_.MarkTerminalError(id); + } + } + MaybeComplete(); + if (!finished_) { + Fail(static_cast( + QueryPeerReceiveScheduleError::kGetClientTimingFailed)); + } + }); +} + +void QueryPeerReceiveSchedule::OnServerTiming( + CloudServerConnection* sc, std::uint64_t send_generation, + Result const& res) { + if (finished_ || sc == nullptr) { + return; + } + auto const server_id = sc->server_id(); + if (!res) { + if (!query_state_.ApplyTransientError(server_id, send_generation)) { + return; + } + bool exhausted = false; + if (cloud_request_.has_value()) { + exhausted = cloud_request_->FailAttempt(sc); + } + if (exhausted) { + query_state_.ApplyTerminalError(server_id, send_generation); + } + MaybeComplete(); + return; + } + if (!query_state_.ApplyTiming(server_id, send_generation, res.value())) { + return; + } + auto const& timing = res.value(); + AE_TELED_DEBUG( + "get_client_timing server {} next_delta_ms {} last_connect_delta_ms {}", + server_id, timing.next_ping_delta_ms, timing.last_connect_delta_ms); + if (cloud_request_.has_value()) { + cloud_request_->SucceedAttempt(sc); + } + MaybeComplete(); +} + +void QueryPeerReceiveSchedule::MaybeComplete() { + if (finished_ || query_state_.cancelled) { + return; + } + if (!query_state_.ReadyToComplete()) { + return; + } + auto aggregated = query_state_.TryAggregate(); + if (aggregated.has_value()) { + Complete(*aggregated); + return; + } + Fail(static_cast(QueryPeerReceiveScheduleError::kGetClientTimingFailed)); +} + +void QueryPeerReceiveSchedule::Complete(PeerReceiveSchedule const& schedule) { + if (finished_) { + return; + } + finished_ = true; + query_state_.completed = true; + ++query_state_.user_callback_count; + diagnostics_ = query_state_.Diagnostics(); + timing_subs_.clear(); + exhausted_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Succeeded(); + } + result_event_.Emit(Ok{schedule}); + Finish(); +} + +void QueryPeerReceiveSchedule::Fail(int code) { + if (finished_) { + return; + } + finished_ = true; + query_state_.completed = true; + ++query_state_.user_callback_count; + diagnostics_ = query_state_.Diagnostics(); + timing_subs_.clear(); + exhausted_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Failed(); + } + result_event_.Emit(Error{code}); + Finish(); +} + +} // namespace ae diff --git a/aether/ae_actions/query_peer_receive_schedule.h b/aether/ae_actions/query_peer_receive_schedule.h new file mode 100644 index 00000000..04543588 --- /dev/null +++ b/aether/ae_actions/query_peer_receive_schedule.h @@ -0,0 +1,600 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_AE_ACTIONS_QUERY_PEER_RECEIVE_SCHEDULE_H_ +#define AETHER_AE_ACTIONS_QUERY_PEER_RECEIVE_SCHEDULE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "aether-miscpp/types/result.h" + +#include "aether/ae_context.h" +#include "aether/actions/action.h" +#include "aether/clock.h" +#include "aether/cloud.h" +#include "aether/cloud_connections/cloud_request.h" +#include "aether/events/event_subscription.h" +#include "aether/events/events.h" +#include "aether/receive_schedule.h" +#include "aether/types/server_id.h" +#include "aether/types/uid.h" +#include "aether/work_cloud_api/client_timing.h" + +namespace ae { +class Client; +class CloudServerConnections; +class CloudServerConnection; + +inline Duration FallbackOneWayPingEstimate() noexcept { + return std::chrono::duration_cast(std::chrono::milliseconds{100}); +} + +inline Duration OneWayPingEstimate(bool stats_empty, + Duration min_rtt) noexcept { + if (stats_empty) { + return FallbackOneWayPingEstimate(); + } + return min_rtt / 2; +} + +inline TimePoint TimePointOffsetByMs(TimePoint anchor, + std::int64_t delta_ms) noexcept { + if (delta_ms == 0) { + return anchor; + } + using ClockDuration = typename TimePoint::duration; + using Rep = typename ClockDuration::rep; + auto const max_safe_ms = std::chrono::duration_cast( + ClockDuration{std::numeric_limits::max() / 4}) + .count(); + if (max_safe_ms > 0) { + if (delta_ms > max_safe_ms) { + return TimePoint::max(); + } + if (delta_ms < -max_safe_ms) { + return TimePoint::min(); + } + } + auto const offset = + std::chrono::duration_cast(std::chrono::milliseconds{ + delta_ms}); + auto const base = anchor.time_since_epoch().count(); + auto const add = offset.count(); + if (add > 0) { + if (base > std::numeric_limits::max() - add) { + return TimePoint::max(); + } + } else if (add < 0) { + if (base < std::numeric_limits::min() - add) { + return TimePoint::min(); + } + } + return TimePoint{ClockDuration{static_cast(base + add)}}; +} + +struct ConvertedServerTiming { + ServerId server_id{}; + TimePoint last_online{}; + std::optional next_ping_deadline{}; + PeerScheduleState state{PeerScheduleState::kUnknown}; +}; + +inline ConvertedServerTiming ConvertClientTiming( + TimePoint qsend, Duration one_way, ClientTiming const& timing, + ServerId server_id = {}) noexcept { + auto const qserver = qsend + one_way; + ConvertedServerTiming out; + out.server_id = server_id; + out.last_online = + TimePointOffsetByMs(qserver, timing.last_connect_delta_ms); + if (timing.next_ping_delta_ms > 0) { + out.next_ping_deadline = + TimePointOffsetByMs(qserver, timing.next_ping_delta_ms); + out.state = PeerScheduleState::kExpected; + } else if (timing.next_ping_delta_ms < 0) { + out.next_ping_deadline = + TimePointOffsetByMs(qserver, timing.next_ping_delta_ms); + out.state = PeerScheduleState::kMissedDeadline; + } else { + out.next_ping_deadline = std::nullopt; + out.state = PeerScheduleState::kUnknown; + } + return out; +} + +struct PeerTimingQueryCoverage { + std::size_t selected_server_count{0}; + std::size_t queried_server_count{0}; + std::size_t successful_server_count{0}; + std::size_t failed_server_count{0}; + std::size_t quarantined_skipped_count{0}; +}; + +struct SelectedServerSnapshotItem { + ServerId id{}; + bool quarantine{false}; + bool has_descriptor{true}; +}; + +inline PeerTimingQueryCoverage BuildPeerTimingQuerySet( + std::vector const& selected, + std::vector& query_set) noexcept { + PeerTimingQueryCoverage cov; + cov.selected_server_count = selected.size(); + query_set.clear(); + query_set.reserve(selected.size()); + for (auto const& item : selected) { + if (item.quarantine) { + ++cov.quarantined_skipped_count; + continue; + } + if (!item.has_descriptor) { + continue; + } + query_set.push_back(item.id); + } + cov.queried_server_count = query_set.size(); + return cov; +} + +// Conservative aggregation for a query with a known expected-server snapshot. +// MissedDeadline is returned only when every expected server succeeded with a +// negative nextPingDelta. Partial failure, retry, or an unqueried expected +// server yields Unknown (or a query error when there are no successes). +struct PeerTimingAggregateContext { + std::size_t expected_server_count{0}; + std::size_t success_count{0}; + std::size_t terminal_error_count{0}; + std::size_t unresolved_count{0}; + bool snapshot_incomplete{false}; + std::vector successes; +}; + +inline std::optional AggregatePeerTimings( + PeerTimingAggregateContext const& ctx) noexcept { + if (ctx.success_count == 0 || ctx.successes.empty()) { + return std::nullopt; + } + + PeerReceiveSchedule out{}; + out.last_online = ctx.successes.front().last_online; + + bool any_future = false; + bool any_unknown = false; + bool any_missed = false; + TimePoint latest_future{}; + TimePoint latest_missed{}; + + for (auto const& sample : ctx.successes) { + if (sample.last_online > out.last_online) { + out.last_online = sample.last_online; + } + if (sample.state == PeerScheduleState::kExpected && + sample.next_ping_deadline.has_value()) { + if (!any_future || *sample.next_ping_deadline > latest_future) { + latest_future = *sample.next_ping_deadline; + } + any_future = true; + } else if (sample.state == PeerScheduleState::kUnknown) { + any_unknown = true; + } else if (sample.state == PeerScheduleState::kMissedDeadline && + sample.next_ping_deadline.has_value()) { + if (!any_missed || *sample.next_ping_deadline > latest_missed) { + latest_missed = *sample.next_ping_deadline; + } + any_missed = true; + } + } + + if (any_future) { + out.state = PeerScheduleState::kExpected; + out.next_ping_deadline = latest_future; + return out; + } + if (any_unknown) { + out.state = PeerScheduleState::kUnknown; + out.next_ping_deadline = std::nullopt; + return out; + } + + bool const incomplete = + ctx.snapshot_incomplete || ctx.unresolved_count > 0 || + ctx.terminal_error_count > 0 || ctx.expected_server_count == 0 || + ctx.success_count < ctx.expected_server_count; + if (incomplete) { + out.state = PeerScheduleState::kUnknown; + out.next_ping_deadline = std::nullopt; + return out; + } + + out.state = PeerScheduleState::kMissedDeadline; + out.next_ping_deadline = latest_missed; + return out; +} + +// All-success complete set: expected_count == samples.size(). +inline std::optional AggregatePeerTimings( + std::vector const& samples) noexcept { + PeerTimingAggregateContext ctx; + ctx.expected_server_count = samples.size(); + ctx.success_count = samples.size(); + ctx.successes = samples; + return AggregatePeerTimings(ctx); +} + +enum class ServerTimingAttemptStatus { + kPending, + kInFlight, + kRetrying, + kSuccess, + kTerminalError, +}; + +struct ServerTimingAttempt { + std::uint64_t send_generation{0}; + TimePoint qsend{}; + Duration one_way{}; + ServerTimingAttemptStatus status{ServerTimingAttemptStatus::kPending}; + ConvertedServerTiming converted{}; + ClientTiming raw{}; + bool has_raw{false}; +}; + +struct ServerTimingDiagnostic { + ServerId server_id{}; + ServerTimingAttemptStatus status{ServerTimingAttemptStatus::kPending}; + ClientTiming raw{}; + bool has_raw{false}; + ConvertedServerTiming converted{}; + TimePoint qsend{}; + Duration one_way{}; +}; + +// Pure helper for unit tests: generation, stale ignore, per-server isolation. +struct PeerTimingQueryState { + std::uint64_t query_generation{0}; + bool cancelled{false}; + bool completed{false}; + int user_callback_count{0}; + bool snapshot_incomplete{false}; + PeerTimingQueryCoverage coverage{}; + std::vector expected_server_ids; + std::map attempts; + + std::uint64_t Begin(std::vector expected = {}, + bool incomplete = false, + PeerTimingQueryCoverage cov = {}) { + ++query_generation; + cancelled = false; + completed = false; + user_callback_count = 0; + snapshot_incomplete = incomplete; + coverage = cov; + expected_server_ids = std::move(expected); + if (coverage.queried_server_count == 0) { + coverage.queried_server_count = expected_server_ids.size(); + } + attempts.clear(); + for (auto const id : expected_server_ids) { + attempts[id].status = ServerTimingAttemptStatus::kPending; + } + return query_generation; + } + + bool IsCurrentQuery(std::uint64_t generation) const noexcept { + return !cancelled && generation == query_generation; + } + + std::uint64_t RegisterSend(ServerId server_id, TimePoint qsend, + Duration one_way) { + auto& attempt = attempts[server_id]; + if (attempt.status == ServerTimingAttemptStatus::kSuccess) { + return attempt.send_generation; + } + ++attempt.send_generation; + attempt.qsend = qsend; + attempt.one_way = one_way; + attempt.status = ServerTimingAttemptStatus::kInFlight; + attempt.converted = {}; + attempt.raw = {}; + attempt.has_raw = false; + return attempt.send_generation; + } + + bool ApplyTiming(ServerId server_id, std::uint64_t send_generation, + ClientTiming const& timing) { + if (cancelled || completed) { + return false; + } + auto it = attempts.find(server_id); + if (it == attempts.end() || it->second.send_generation != send_generation) { + return false; + } + if (it->second.status == ServerTimingAttemptStatus::kSuccess) { + return false; + } + it->second.status = ServerTimingAttemptStatus::kSuccess; + it->second.raw = timing; + it->second.has_raw = true; + it->second.converted = ConvertClientTiming( + it->second.qsend, it->second.one_way, timing, server_id); + return true; + } + + bool ApplyTransientError(ServerId server_id, std::uint64_t send_generation) { + if (cancelled || completed) { + return false; + } + auto it = attempts.find(server_id); + if (it == attempts.end() || it->second.send_generation != send_generation) { + return false; + } + if (it->second.status == ServerTimingAttemptStatus::kSuccess || + it->second.status == ServerTimingAttemptStatus::kTerminalError) { + return false; + } + it->second.status = ServerTimingAttemptStatus::kRetrying; + return true; + } + + bool ApplyTerminalError(ServerId server_id, std::uint64_t send_generation) { + if (cancelled || completed) { + return false; + } + auto it = attempts.find(server_id); + if (it == attempts.end() || it->second.send_generation != send_generation) { + return false; + } + if (it->second.status == ServerTimingAttemptStatus::kSuccess) { + return false; + } + it->second.status = ServerTimingAttemptStatus::kTerminalError; + return true; + } + + bool ApplyError(ServerId server_id, std::uint64_t send_generation) { + return ApplyTerminalError(server_id, send_generation); + } + + bool MarkTerminalError(ServerId server_id) { + if (cancelled || completed) { + return false; + } + auto& attempt = attempts[server_id]; + if (attempt.status == ServerTimingAttemptStatus::kSuccess) { + return false; + } + attempt.status = ServerTimingAttemptStatus::kTerminalError; + return true; + } + + bool ReadyToComplete() const { + if (cancelled || completed) { + return false; + } + auto ids = expected_server_ids; + if (ids.empty()) { + ids.reserve(attempts.size()); + for (auto const& [id, _] : attempts) { + ids.push_back(id); + } + } + if (ids.empty()) { + return false; + } + for (auto const id : ids) { + auto it = attempts.find(id); + if (it == attempts.end()) { + return false; + } + auto const status = it->second.status; + if (status != ServerTimingAttemptStatus::kSuccess && + status != ServerTimingAttemptStatus::kTerminalError) { + return false; + } + } + return true; + } + + std::optional TryAggregate() const { + PeerTimingAggregateContext ctx; + ctx.snapshot_incomplete = snapshot_incomplete; + auto ids = expected_server_ids; + if (ids.empty()) { + ids.reserve(attempts.size()); + for (auto const& [id, _] : attempts) { + ids.push_back(id); + } + } + ctx.expected_server_count = ids.size(); + ctx.successes.reserve(ids.size()); + for (auto const id : ids) { + auto it = attempts.find(id); + if (it == attempts.end()) { + ++ctx.unresolved_count; + continue; + } + switch (it->second.status) { + case ServerTimingAttemptStatus::kSuccess: + ++ctx.success_count; + ctx.successes.push_back(it->second.converted); + break; + case ServerTimingAttemptStatus::kTerminalError: + ++ctx.terminal_error_count; + break; + default: + ++ctx.unresolved_count; + break; + } + } + return AggregatePeerTimings(ctx); + } + + std::vector Diagnostics() const { + std::vector out; + out.reserve(attempts.size()); + for (auto const& [id, attempt] : attempts) { + ServerTimingDiagnostic d; + d.server_id = id; + d.status = attempt.status; + d.raw = attempt.raw; + d.has_raw = attempt.has_raw; + d.converted = attempt.converted; + d.qsend = attempt.qsend; + d.one_way = attempt.one_way; + out.push_back(d); + } + return out; + } + + PeerTimingQueryCoverage QueryCoverage() const { + auto c = coverage; + c.queried_server_count = expected_server_ids.size(); + c.successful_server_count = 0; + c.failed_server_count = 0; + for (auto const id : expected_server_ids) { + auto it = attempts.find(id); + if (it == attempts.end()) { + continue; + } + if (it->second.status == ServerTimingAttemptStatus::kSuccess) { + ++c.successful_server_count; + } else if (it->second.status == + ServerTimingAttemptStatus::kTerminalError) { + ++c.failed_server_count; + } + } + return c; + } + + void Cancel() { cancelled = true; } +}; + +// In-memory query orchestration without Client/Cloud. Models completion, +// retry, and destruction for lifecycle tests. +struct PeerTimingQueryOrchestrator { + PeerTimingQueryState state; + int callback_count{0}; + std::optional last_schedule; + std::optional last_error; + + void Start(std::vector ids, bool incomplete = false) { + state.Begin(std::move(ids), incomplete); + callback_count = 0; + last_schedule.reset(); + last_error.reset(); + } + + std::uint64_t Send(ServerId id, TimePoint qsend, Duration one_way) { + return state.RegisterSend(id, qsend, one_way); + } + + void OnSuccess(ServerId id, std::uint64_t gen, ClientTiming const& timing) { + if (!state.ApplyTiming(id, gen, timing)) { + return; + } + TryFinish(); + } + + void OnTransient(ServerId id, std::uint64_t gen) { + if (!state.ApplyTransientError(id, gen)) { + return; + } + TryFinish(); + } + + void OnTerminal(ServerId id, std::uint64_t gen) { + if (!state.ApplyTerminalError(id, gen)) { + return; + } + TryFinish(); + } + + void Destroy() { state.Cancel(); } + + void TryFinish() { + if (!state.ReadyToComplete() || state.completed) { + return; + } + auto aggregated = state.TryAggregate(); + state.completed = true; + ++state.user_callback_count; + ++callback_count; + if (aggregated.has_value()) { + last_schedule = *aggregated; + } else { + last_error = static_cast(3); + } + } +}; + +enum class QueryPeerReceiveScheduleError : int { + kGetCloudFailed = 1, + kNoWorkServerAvailable = 2, + kGetClientTimingFailed = 3, +}; + +class QueryPeerReceiveSchedule final : public Action { + public: + using ResultEvent = Event)>; + + QueryPeerReceiveSchedule(AeContext const& ae_context, Client& client, + Uid peer_uid); + ~QueryPeerReceiveSchedule() override; + + AE_CLASS_NO_COPY_MOVE(QueryPeerReceiveSchedule) + + ResultEvent::Subscriber result_event() noexcept; + // Per-server timing captured for this query. Not part of PeerReceiveSchedule. + std::vector const& server_diagnostics() const noexcept; + PeerTimingQueryCoverage coverage() const noexcept; + Uid peer_uid() const noexcept { return peer_uid_; } + + private: + Duration OneWayEstimateFor(CloudServerConnection* sc) const; + void OnCloud(Result result); + void StartQuery(); + void SnapshotExpectedServers(); + void OnServerTiming(CloudServerConnection* sc, std::uint64_t send_generation, + Result const& res); + void MaybeComplete(); + void Complete(PeerReceiveSchedule const& schedule); + void Fail(int code); + + AeContext ae_context_; + Client* client_{nullptr}; + Uid peer_uid_{}; + ResultEvent result_event_; + Subscription get_cloud_sub_; + Subscription cloud_request_sub_; + Subscription exhausted_sub_; + std::unique_ptr dest_cloud_; + std::optional cloud_request_; + std::map timing_subs_; + PeerTimingQueryState query_state_{}; + std::vector diagnostics_; + bool finished_{false}; +}; + +} // namespace ae + +#endif // AETHER_AE_ACTIONS_QUERY_PEER_RECEIVE_SCHEDULE_H_ diff --git a/aether/api_protocol/protocol_context.cpp b/aether/api_protocol/protocol_context.cpp index 37d2f9b5..a7af97fa 100644 --- a/aether/api_protocol/protocol_context.cpp +++ b/aether/api_protocol/protocol_context.cpp @@ -33,6 +33,18 @@ ProtocolContext::~ProtocolContext() { } } +void ProtocolContext::set_inbound_server_response_hook( + InboundServerResponseHook hook, void* user) noexcept { + inbound_server_response_hook_ = hook; + inbound_server_response_user_ = user; +} + +void ProtocolContext::NotifyInboundServerResponse() noexcept { + if (inbound_server_response_hook_ != nullptr) { + inbound_server_response_hook_(inbound_server_response_user_); + } +} + void ProtocolContext::SetSendResultResponse(RequestId request_id) { auto entry = TakePending(request_id); if (entry.response == nullptr) { @@ -42,6 +54,10 @@ void ProtocolContext::SetSendResultResponse(RequestId request_id) { auto* p = parser(); assert(p != nullptr && "Parser shouldn't be null"); + // Matched inbound result from the server (not a local eviction/timeout). + if (entry.response != nullptr) { + NotifyInboundServerResponse(); + } entry.response->OnResult(*p); DestroyPending(entry); } @@ -55,6 +71,10 @@ void ProtocolContext::SetSendErrorResponse(RequestId req_id, parser()->Cancel(); } + // Matched inbound error response from the server (not OnEvicted). + if (entry.response != nullptr) { + NotifyInboundServerResponse(); + } entry.response->OnError(error_type, static_cast(error_code)); DestroyPending(entry); } diff --git a/aether/api_protocol/protocol_context.h b/aether/api_protocol/protocol_context.h index e4d5741e..28226d92 100644 --- a/aether/api_protocol/protocol_context.h +++ b/aether/api_protocol/protocol_context.h @@ -70,6 +70,12 @@ class ProtocolContext { ProtocolContext(); ~ProtocolContext(); + // Invoked when an inbound server result/error response is matched to a + // pending request. Not called for local OnEvicted / synthetic failures. + using InboundServerResponseHook = void (*)(void* user); + void set_inbound_server_response_hook(InboundServerResponseHook hook, + void* user) noexcept; + template Entry& CreatePendingResponse(RequestId request_id) { static_assert(sizeof(Entry) <= kPendingResponseMaxSize, @@ -124,6 +130,7 @@ class ProtocolContext { void PreparePendingResponseSlot(RequestId request_id); PendingEntry TakePending(RequestId request_id); PendingEntry TakeOldestPending(); + void NotifyInboundServerResponse() noexcept; PendingResponsePool pending_response_pool_; PendingList pending_responses_; @@ -131,6 +138,9 @@ class ProtocolContext { PacketStackStack packet_stacks_; ParserStack parsers_; PackerStack packers_; + + InboundServerResponseHook inbound_server_response_hook_{nullptr}; + void* inbound_server_response_user_{nullptr}; }; } // namespace ae diff --git a/aether/channels/channel.cpp b/aether/channels/channel.cpp index 6c3f6a4f..4598b350 100644 --- a/aether/channels/channel.cpp +++ b/aether/channels/channel.cpp @@ -24,8 +24,6 @@ namespace ae { Channel::Channel(ObjProp prop) : Obj{prop}, channel_statistics_{ChannelStatistics::ptr::Create(domain)} { - channel_statistics_->AddResponseTime( - std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}); channel_statistics_->AddConnectionTime( std::chrono::milliseconds{AE_DEFAULT_CONNECTION_TIMEOUT_MS}); } @@ -43,7 +41,11 @@ Duration Channel::TransportBuildTimeout() const { } Duration Channel::ResponseTimeout() const { - return channel_statistics_->response_time_statistics().percentile<99>(); + auto const& stats = channel_statistics_->response_time_statistics(); + if (stats.empty()) { + return Channel::kInitialResponseEstimate; + } + return stats.percentile<99>(); } } // namespace ae diff --git a/aether/channels/channel.h b/aether/channels/channel.h index 107b23f4..db2aefb3 100644 --- a/aether/channels/channel.h +++ b/aether/channels/channel.h @@ -17,6 +17,8 @@ #ifndef AETHER_CHANNELS_CHANNEL_H_ #define AETHER_CHANNELS_CHANNEL_H_ +#include + #include "aether/memory.h" #include "aether/obj/obj.h" #include "aether/executors/executors.h" @@ -37,6 +39,10 @@ class Channel : public Obj { explicit Channel(ObjProp prop); + // Used when response RTT samples are empty (no synthetic seed). + static constexpr Duration kInitialResponseEstimate = + std::chrono::duration_cast(std::chrono::milliseconds{200}); + AE_OBJECT_REFLECT(AE_MMBRS(transport_properties_, channel_statistics_)) /** diff --git a/aether/client.cpp b/aether/client.cpp index cd31717d..a920373c 100644 --- a/aether/client.cpp +++ b/aether/client.cpp @@ -18,9 +18,12 @@ #include +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/ae_actions/announce_next_ping_unknown.h" #include "aether/ae_actions/telemetry.h" #include "aether/aether.h" +#include "aether/config.h" namespace ae { @@ -49,8 +52,15 @@ ServerKeys* Client::server_state(ServerId server_id) { Cloud::ptr const& Client::cloud() const { return cloud_; } -ClientCloudManager::ptr const& Client::cloud_manager() const { - assert(client_cloud_manager_.is_valid()); +ClientCloudManager::ptr const& Client::cloud_manager() { + // Lazy: ClientCloudManager::Init listens for cloud updates and therefore + // calls cloud_connection(), which would start pings before SetReceiveSchedule + // can run (SetReceiveSchedule must run after SetConfig, before first ping). + if (!client_cloud_manager_) { + client_cloud_manager_ = ClientCloudManager::ptr::Create( + CreateWith{domain}.with_flags(ObjFlags::kUnloadedByDefault), + Aether::ptr{aether_}, Client::ptr::MakeFromThis(this)); + } return client_cloud_manager_; } @@ -79,9 +89,11 @@ CloudServerConnections& Client::cloud_connection() { // also create telemetry telemetry_ = std::make_unique(*aether_, *cloud_connection_); #endif - + // Cloud-config push subscription needs a live connection; start it here + // (not in ClientCloudManager construction) so SetReceiveSchedule can run + // after SetConfig and before the first cloud_connection(). client_cloud_manager_.WithLoaded( - [&](auto const& ccm) { ccm->StartListenForCloudUpdate(); }); + [&](auto const& ccm) { ccm->StartCloudUpdateListener(); }); } return *cloud_connection_; @@ -116,10 +128,75 @@ void Client::SetConfig(std::string client_id, Uid parent_uid, Uid uid, connectivity_policy_ = ClientConnectivityPolicy::ptr::Create( CreateWith{domain}.with_flags(ObjFlags::kUnloadedByDefault)); + // client_cloud_manager_ is created lazily in cloud_manager() so that + // SetReceiveSchedule can configure RX timings before pings start. +} + + +Result Client::SetReceiveSchedule(ReceiveSchedule schedule) { + if (cloud_connection_) { + return Error{static_cast(SetReceiveScheduleError::kPingAlreadyStarted)}; + } +#if AE_ENABLE_PING + if (ping_cloud_servers_) { + return Error{static_cast(SetReceiveScheduleError::kPingAlreadyStarted)}; + } +#endif + if (!connectivity_policy_.is_valid()) { + return Error{static_cast(SetReceiveScheduleError::kPingAlreadyStarted)}; + } + // Keep the policy loaded: it is created with kUnloadedByDefault, and a bare + // Load()/configure would be lost when the temporary Ptr releases. + connectivity_policy_keep_alive_ = connectivity_policy_.Load(); + connectivity_policy_keep_alive_->ConfigureRxTimings().ForAllPriorities( + RxTimingConf{.interval = schedule.ping_interval, + .rx_window = schedule.receive_window}); + connectivity_policy_keep_alive_->set_ping_retry_count( + schedule.ping_retry_count); + // Persist so a later Load() in cloud_connection() cannot revive defaults. + connectivity_policy_.Save(); + return Ok{std::monostate{}}; +} + +::ae::QueryPeerReceiveSchedule& Client::QueryPeerReceiveSchedule(Uid peer_uid) { + if (query_peer_receive_schedule_ && + !query_peer_receive_schedule_->is_finished() && + query_peer_receive_schedule_->peer_uid() == peer_uid) { + return *query_peer_receive_schedule_; + } + query_peer_receive_schedule_ = + std::make_unique<::ae::QueryPeerReceiveSchedule>( + AeContext{*aether_}, *this, peer_uid); + return *query_peer_receive_schedule_; +} + +::ae::AnnounceNextPingUnknown& Client::AnnounceNextPingUnknown() { + announce_next_ping_unknown_ = std::make_unique<::ae::AnnounceNextPingUnknown>( + AeContext{*aether_}, *this); + return *announce_next_ping_unknown_; +} + +#if AE_ENABLE_PING +PingCloudServers* Client::ping_cloud_servers() noexcept { + return ping_cloud_servers_.get(); +} +#endif + +std::optional Client::last_online_time() const noexcept { + return last_online_time_; +} + +std::optional Client::expected_ping_response_time() const noexcept { +#if AE_ENABLE_PING + if (ping_cloud_servers_ != nullptr) { + return ping_cloud_servers_->expected_ping_response_time(); + } +#endif + return std::nullopt; +} - client_cloud_manager_ = ClientCloudManager::ptr::Create( - CreateWith{domain}.with_flags(ObjFlags::kUnloadedByDefault), - Aether::ptr{aether_}, Client::ptr::MakeFromThis(this)); +void Client::MarkServerResponseReceived(TimePoint when) noexcept { + UpdateMonotonicLastOnlineTime(last_online_time_, when); } void Client::SendTelemetry() { diff --git a/aether/client.h b/aether/client.h index fe8206c5..f96ed26d 100644 --- a/aether/client.h +++ b/aether/client.h @@ -19,8 +19,13 @@ #include #include +#include #include +#include +#include "aether-miscpp/types/result.h" + +#include "aether/clock.h" #include "aether/client_connectivity_policy.h" #include "aether/cloud.h" #include "aether/memory.h" @@ -35,10 +40,14 @@ #include "aether/connection_manager/server_connection_manager.h" #include "aether/client_messages/p2p_message_stream_manager.h" +#include "aether/ptr/ptr.h" +#include "aether/receive_schedule.h" namespace ae { class Aether; class Telemetry; +class QueryPeerReceiveSchedule; +class AnnounceNextPingUnknown; class Client : public Obj { AE_OBJECT(Client, Obj, 0) @@ -59,7 +68,7 @@ class Client : public Obj { Uid const& ephemeral_uid() const; ServerKeys* server_state(ServerId server_id); Cloud::ptr const& cloud() const; - ClientCloudManager::ptr const& cloud_manager() const; + ClientCloudManager::ptr const& cloud_manager(); ServerConnectionManager& server_connection_manager(); CloudServerConnections& cloud_connection(); ClientConnectivityPolicy::ptr const& connectivity_policy(); @@ -68,12 +77,38 @@ class Client : public Obj { void SetConfig(std::string client_id, Uid parent_uid, Uid uid, Uid ephemeral_uid, Key master_key, Cloud::ptr c); + // Must run after SetConfig and before first cloud_connection()/ping start. + Result SetReceiveSchedule(ReceiveSchedule schedule); + + // Stores action on Client (replaced each call); returns a live reference. + ::ae::QueryPeerReceiveSchedule& QueryPeerReceiveSchedule(Uid peer_uid); + ::ae::AnnounceNextPingUnknown& AnnounceNextPingUnknown(); + +#if AE_ENABLE_PING + PingCloudServers* ping_cloud_servers() noexcept; +#endif + + // Most recent local time at which a valid response from any cloud server was + // actually received. Not updated on send, connect, timeout, or eviction. + std::optional last_online_time() const noexcept; + + // Expected local receive time of the scheduled ping response. + // Tn is the request-arrival contract deadline; response return time is + // estimated as Tn + p99_RTT/2. nullopt when ping scheduling is disabled or + // no contract deadline exists yet. + std::optional expected_ping_response_time() const noexcept; + AE_OBJECT_REFLECT(AE_MMBRS(aether_, client_id_, parent_uid_, uid_, ephemeral_uid_, master_key_, cloud_, server_keys_, connectivity_policy_, client_cloud_manager_)) void SendTelemetry(); private: + friend class ClientServerConnection; + + // Central inbound-response hook used by cloud ProtocolContext matching. + void MarkServerResponseReceived(TimePoint when) noexcept; + ObjPtr aether_; // configuration std::string client_id_; // User-defined client id @@ -87,6 +122,8 @@ class Client : public Obj { std::map server_keys_; ClientConnectivityPolicy::ptr connectivity_policy_; + // Keeps configured receive-schedule timings resident until/after ping start. + Ptr connectivity_policy_keep_alive_; ClientCloudManager::ptr client_cloud_manager_; std::unique_ptr server_connection_manager_; std::unique_ptr cloud_connection_; @@ -95,9 +132,14 @@ class Client : public Obj { #if AE_ENABLE_PING std::unique_ptr ping_cloud_servers_; #endif + std::unique_ptr<::ae::QueryPeerReceiveSchedule> query_peer_receive_schedule_; + std::unique_ptr<::ae::AnnounceNextPingUnknown> announce_next_ping_unknown_; #if AE_TELE_ENABLED && AE_TELE_LOG_TO_STATISTICS std::unique_ptr telemetry_; #endif + + // Runtime-only connectivity timestamps (not serialized). + std::optional last_online_time_; }; } // namespace ae diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index d038af72..f35fbc24 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -25,8 +25,10 @@ #include "aether/config.h" #include "aether/events/events.h" #include "aether/obj/obj.h" +#include "aether/obj/version_iterator.h" #include "aether/cloud_connections/request_policy.h" +#include "aether/receive_schedule.h" namespace ae { @@ -64,7 +66,7 @@ struct ConnectivityStatus { }; class ClientConnectivityPolicy : public Obj { - AE_OBJECT(ClientConnectivityPolicy, Obj, 0) + AE_OBJECT(ClientConnectivityPolicy, Obj, 1) public: class RxTimingConfig { @@ -108,10 +110,19 @@ class ClientConnectivityPolicy : public Obj { AE_CLASS_NO_COPY_MOVE(ClientConnectivityPolicy); - AE_OBJECT_REFLECT(AE_MMBRS(rx_targets_, rx_timings_)) + AE_OBJECT_REFLECT(AE_MMBRS(rx_targets_, rx_timings_, ping_retry_count_)) template - void Load(CurrentVersion, Dnv& dnv) { + void Load(Version<0>, Dnv& dnv) { dnv(base_, rx_targets_, rx_timings_); + ping_retry_count_ = kDefaultPingRetryCount; + ResetRuntimeState(); + } + template + void Load(CurrentVersion, Dnv& dnv) { + dnv(base_, rx_targets_, rx_timings_, ping_retry_count_); + if (ping_retry_count_ > kMaxPingRetryCount) { + ping_retry_count_ = kMaxPingRetryCount; + } ResetRuntimeState(); } @@ -125,6 +136,11 @@ class ClientConnectivityPolicy : public Obj { const noexcept { return rx_timings_; } + std::uint8_t ping_retry_count() const noexcept { return ping_retry_count_; } + void set_ping_retry_count(std::uint8_t count) noexcept { + ping_retry_count_ = + count > kMaxPingRetryCount ? kMaxPingRetryCount : count; + } Event::Subscriber suspend_allowed_event() noexcept { return EventSubscriber{suspend_allowed_event_}; } @@ -142,6 +158,7 @@ class ClientConnectivityPolicy : public Obj { RequestPolicy::Variant rx_targets_; std::array rx_timings_; + std::uint8_t ping_retry_count_{kDefaultPingRetryCount}; bool can_suspend_{true}; std::uint8_t suspend_block_count_{}; diff --git a/aether/client_messages/p2p_message_stream.cpp b/aether/client_messages/p2p_message_stream.cpp index c202645f..609920df 100644 --- a/aether/client_messages/p2p_message_stream.cpp +++ b/aether/client_messages/p2p_message_stream.cpp @@ -17,6 +17,7 @@ #include "aether/client_messages/p2p_message_stream.h" #include +#include #include "aether/config.h" @@ -25,10 +26,53 @@ #include "aether/cloud_connections/cloud_request.h" #include "aether/cloud_connections/cloud_subscription.h" +#include "aether/cloud_connections/request_policy.h" +#include "aether/server_connections/client_server_connection.h" #include "aether/client_messages/client_messages_tele.h" namespace ae { +namespace { +P2pSendRouteSnapshot CaptureP2pSendRoute(CloudServerConnections& cloud) { + P2pSendRouteSnapshot snap{}; + cloud.ForServers( + [&](CloudServerConnection* sc) { + if (snap.present || sc == nullptr) { + return; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return; + } + snap.present = true; + snap.server_id = sc->server_id(); + auto info = conn->stream_info(); + snap.linked = info.link_state == LinkState::kLinked; + snap.writable = info.is_writable; + auto channel = conn->server_connection().current_channel(); + if (!channel) { + return; + } + snap.route_generation = + static_cast( + reinterpret_cast(channel.get())); + auto const& stats = + channel->channel_statistics().response_time_statistics(); + snap.ping_sample_count = stats.size(); + if (!stats.empty()) { + snap.min_rtt = stats.min(); + snap.p99_rtt = stats.percentile<99>(); + } + auto const& props = channel->transport_properties(); + snap.protocol = props.connection_type == ConnectionType::kConnectionLess + ? Protocol::kUdp + : Protocol::kTcp; + }, + RequestPolicy::MainServer{}); + return snap; +} +} // namespace + namespace p2p_stream_internal { class MessageSendStream final : public IStream { public: @@ -244,10 +288,24 @@ std::unique_ptr P2pStream::MakeDestinationCloudConn( ae_context_, cloud, std::move(factory), AE_CLOUD_MAX_SERVER_CONNECTIONS); } +P2pSendRouteSnapshot P2pStream::InspectSendRoute() const { + if (!dest_cloud_conn_) { + return {}; + } + return CaptureP2pSendRoute(*dest_cloud_conn_); +} + +P2pSendRouteSnapshot P2pStream::LastSendRoute() const { + return last_send_route_; +} + WriteAction* P2pStream::OnWrite(AeMessage&& message) { if (!message_send_stream_) { return {}; } + if (dest_cloud_conn_) { + last_send_route_ = CaptureP2pSendRoute(*dest_cloud_conn_); + } return &message_send_stream_->Write(std::move(message)); } diff --git a/aether/client_messages/p2p_message_stream.h b/aether/client_messages/p2p_message_stream.h index 64da048b..0a26473d 100644 --- a/aether/client_messages/p2p_message_stream.h +++ b/aether/client_messages/p2p_message_stream.h @@ -28,6 +28,8 @@ #include "aether/client_messages/p2p_port_handle.h" #include "aether/cloud_connections/cloud_server_connections.h" #include "aether/connection_manager/client_cloud_manager.h" +#include "aether/types/address.h" +#include "aether/types/server_id.h" namespace ae { class Client; @@ -36,6 +38,20 @@ namespace p2p_stream_internal { class MessageSendStream; } // namespace p2p_stream_internal +// Diagnostic snapshot of the MainServer destination used by P2pStream::Write. +// Not a public routing API: production Write does not depend on this. +struct P2pSendRouteSnapshot { + bool present{false}; + ServerId server_id{}; + Protocol protocol{Protocol::kTcp}; + std::uint64_t route_generation{0}; + bool linked{false}; + bool writable{false}; + std::size_t ping_sample_count{0}; + Duration min_rtt{}; + Duration p99_rtt{}; +}; + class P2pStream final : public ByteIStream { public: P2pStream(AeContext const& ae_context, Ptr const& client, @@ -53,6 +69,8 @@ class P2pStream final : public ByteIStream { void WriteOut(DataBuffer const& data); Uid const& destination() const; + P2pSendRouteSnapshot InspectSendRoute() const; + P2pSendRouteSnapshot LastSendRoute() const; private: void ConnectReceive(); @@ -79,6 +97,7 @@ class P2pStream final : public ByteIStream { Subscription get_client_cloud_sub_; Subscription out_data_sub_; + P2pSendRouteSnapshot last_send_route_{}; }; } // namespace ae diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index 0a972cc7..bb0518b8 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -71,10 +71,57 @@ void CloudRequest::Failed() { result_event_.Emit(false); } + +void CloudRequest::SucceedAttempt(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return; + } + auto& sr = it->second; + if (sr.succeeded) { + return; + } + sr.state_subs.Reset(); + sr.timeout_sub.Reset(); + sr.succeeded = true; + EnqueueMakeRequest(); +} + +bool CloudRequest::FailAttempt(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return false; + } + auto& sr = it->second; + if (sr.succeeded) { + return false; + } + sr.state_subs.Reset(); + sr.timeout_sub.Reset(); + sr.retry_count++; + if (sr.retry_count >= max_retries_) { + AE_TELED_WARNING("Server {} retry budget exhausted on attempt failure", + sc->server_id()); + sr.exhausted = true; + EmitAttemptExhausted(sc); + } + EnqueueMakeRequest(); + return sr.exhausted; +} + CloudRequest::ResultEvent::Subscriber CloudRequest::result_event() { return EventSubscriber{result_event_}; } +CloudRequest::AttemptExhaustedEvent::Subscriber +CloudRequest::attempt_exhausted_event() { + return EventSubscriber{attempt_exhausted_event_}; +} + +void CloudRequest::EmitAttemptExhausted(CloudServerConnection* sc) { + attempt_exhausted_event_.Emit(sc); +} + void CloudRequest::PrefillServerRequests() { for (auto* sc : cloud_scs_->servers()) { server_requests_.emplace(sc, ServerRequest{}); @@ -92,7 +139,7 @@ void CloudRequest::MakeRequest() { sr = &new_it->second; } else { sr = &it->second; - if (sr->exhausted) { + if (sr->exhausted || sr->succeeded) { return; } } @@ -100,15 +147,17 @@ void CloudRequest::MakeRequest() { }, policy_); - // Check if all server requests are exhausted - bool all_exhausted = !server_requests_.empty(); + bool any_open = false; + bool any_succeeded = false; for (auto const& [sc, sr] : server_requests_) { - if (!sr.exhausted) { - all_exhausted = false; - break; + if (sr.succeeded) { + any_succeeded = true; + } else if (!sr.exhausted) { + any_open = true; } } - if (all_exhausted) { + if (!server_requests_.empty() && + CloudRequestShouldFailAll(any_open, any_succeeded)) { AE_TELED_ERROR("All server requests exhausted, failing"); Failed(); } @@ -181,9 +230,13 @@ void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); EnqueueMakeRequest(); return; } @@ -197,11 +250,15 @@ void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } sr.retry_count++; if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted on write failure", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); } EnqueueMakeRequest(); } @@ -212,10 +269,14 @@ void CloudRequest::OnServerRequestTimeout(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted on timeout", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); EnqueueMakeRequest(); return; } diff --git a/aether/cloud_connections/cloud_request.h b/aether/cloud_connections/cloud_request.h index ceb6d6f5..78a15b52 100644 --- a/aether/cloud_connections/cloud_request.h +++ b/aether/cloud_connections/cloud_request.h @@ -35,12 +35,42 @@ namespace ae { * response. On success, listener must call CloudRequest::Succeeded(). On * failure, listener must call CloudRequest::Failed(). */ +// Testable per-server completion flags used by CloudRequest. +struct CloudRequestAttemptState { + bool exhausted{false}; + bool succeeded{false}; + std::size_t retry_count{0}; + + bool ShouldSkipMake() const noexcept { return exhausted || succeeded; } + + void MarkSucceeded() { succeeded = true; } + + // Returns true when this failure exhausted the retry budget. + bool MarkFailed(std::size_t max_retries) { + if (succeeded) { + return false; + } + ++retry_count; + if (retry_count >= max_retries) { + exhausted = true; + return true; + } + return false; + } +}; + +inline bool CloudRequestShouldFailAll(bool any_open, + bool any_succeeded) noexcept { + return !any_open && !any_succeeded; +} + class CloudRequest final : public Action { struct ServerRequest { MultiSubscription state_subs; TaskSubscription timeout_sub; std::size_t retry_count{0}; bool exhausted{false}; + bool succeeded{false}; }; public: @@ -49,6 +79,7 @@ class CloudRequest final : public Action { std::chrono::milliseconds{AE_CLOUD_REQUEST_TIMEOUT_MS}; using ResultEvent = Event; + using AttemptExhaustedEvent = Event; CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, @@ -66,8 +97,15 @@ class CloudRequest final : public Action { void Succeeded(); void Failed(); + // Per-server success: stop this server's timeout/channel/write + // subscriptions, skip further retries, and do not finish CloudRequest. + void SucceedAttempt(CloudServerConnection* sc); + // Listener-side attempt failure: retry/exhaust this server without + // ending the whole CloudRequest. Returns true when the server is exhausted. + bool FailAttempt(CloudServerConnection* sc); ResultEvent::Subscriber result_event(); + AttemptExhaustedEvent::Subscriber attempt_exhausted_event(); private: void MakeRequest(); @@ -81,6 +119,7 @@ class CloudRequest final : public Action { void RemoveRequest(CloudServerConnection* server_connection); void EnqueueMakeRequest(); + void EmitAttemptExhausted(CloudServerConnection* sc); void Finish(); @@ -95,6 +134,7 @@ class CloudRequest final : public Action { Subscription swa_sub_; Subscription server_changed_sub_; ResultEvent result_event_; + AttemptExhaustedEvent attempt_exhausted_event_; std::map server_requests_; }; diff --git a/aether/cloud_connections/ping_cloud_servers.cpp b/aether/cloud_connections/ping_cloud_servers.cpp index b1128e14..7f5b34d9 100644 --- a/aether/cloud_connections/ping_cloud_servers.cpp +++ b/aether/cloud_connections/ping_cloud_servers.cpp @@ -16,7 +16,11 @@ #include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/cloud_connections/ping_schedule_guard.h" + #include +#include +#include #include #include @@ -24,16 +28,26 @@ # include "aether/channels/channel.h" # include "aether/executors/executors.h" +# if AE_ENABLE_PING_TEST_FAULTS +# include "aether/ae_actions/ping_test_faults.h" +# endif # include "aether/cloud_connections/cloud_connections_tele.h" namespace ae { +namespace { +PingTraceHook g_ping_trace_hook{nullptr}; +} // namespace + +void SetPingTraceHook(PingTraceHook hook) noexcept { g_ping_trace_hook = hook; } PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, + PingCloudServers& owner, ClientConnectivityPolicy& policy, CloudServerConnection& cloud_sc, std::size_t priority) : ae_context_{ae_context}, + owner_{&owner}, policy_{&policy}, cloud_sc_{&cloud_sc}, priority_{priority} { @@ -43,6 +57,23 @@ PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, auto const& timings = policy_->rx_timings()[priority_]; timing_conf_ = timings.conf; + if (timings.next_rx_point != TimePoint{}) { + planned_send_at_ = timings.next_rx_point; + } + + auto& st = Cycle(); + if (st.required_rx_until != TimePoint{}) { + required_rx_until_ = st.required_rx_until; + } + if (st.active && !st.confirmed) { + if (st.first_attempt_at != TimePoint{}) { + planned_send_at_ = st.first_attempt_at; + } + ping_blocker_ = policy_->AcquireSuspendBlock(); + start_sub_ = ae_context_.scheduler().Task([&]() { Start(); }); + return; + } + // if it's to early for next rx wait a bit if ((timings.next_rx_point != TimePoint{}) && (Now() < timings.next_rx_point)) { @@ -71,6 +102,8 @@ void PingCloudServers::ServerPing::Stop() { ping_blocker_.Reset(); rx_window_blocker_.Reset(); restream_blocker_.Reset(); + rx_window_held_ = false; + CloseLocalRx(local_rx_); } template @@ -119,24 +152,184 @@ auto PingCloudServers::ServerPing::MakePing() { return ex::set_error(std::move(ctx.receiver), 2); } - ping_.emplace(ae_context_, *cloud_sc_, timing_conf_.interval, - timing_conf_.rx_window, c->ResponseTimeout()); + auto const& response_stats = + c->channel_statistics().response_time_statistics(); + Duration min_rtt = kPingRttEstimate; + Duration p99_rtt = kPingRttEstimate; + if (!response_stats.empty()) { + min_rtt = response_stats.min(); + p99_rtt = response_stats.template percentile<99>(); + } + auto const guard = ResolvePingSendGuard(min_rtt, p99_rtt, + timing_conf_.interval); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + timing_conf_.interval, + guard, + c->ResponseTimeout(), + p99_rtt, + policy_->ping_retry_count()}); + + if (required_rx_until_.has_value() && + *required_rx_until_ <= Now() && !local_rx_.open) { + required_rx_until_.reset(); + } + + auto& st = Cycle(); +#if AE_ENABLE_PING_TEST_FAULTS + // Test-only: hold a same-cycle retry until a coordinator-chosen + // send time relative to the original nominal Tn. Does not run when + // no hold plan is armed. + if (st.active && !st.confirmed && !announce_unknown_) { + auto const next_attempt = st.attempt_index + 1; + auto const hold_us = PingTestFaults::Instance().RetryHoldOffsetUs( + cloud_sc_->server_id(), st.cycle_id, next_attempt); + if (hold_us.has_value() && st.nominal_ping_at != TimePoint{}) { + TimePoint target = st.nominal_ping_at; + if (*hold_us >= 0) { + target = SaturatingAddTime( + st.nominal_ping_at, + Duration{static_cast(*hold_us)}); + } else { + auto mag = static_cast(-*hold_us); + auto const max_rep = static_cast( + std::numeric_limits::max()); + if (mag > max_rep) { + mag = max_rep; + } + target = SaturatingSubDuration( + st.nominal_ping_at, + Duration{static_cast(mag)}); + } + if (Now() < target) { + start_sub_ = ae_context_.scheduler().DelayedTask( + [this]() { Start(); }, target); + return ex::set_value(std::move(ctx.receiver)); + } + } + } +#endif + auto const actual_send_at = Now(); + if (!announce_unknown_ && owner_->auto_ping_enabled_ && + st.confirmed && actual_send_at < st.next_local_send_at) { + return ex::set_value(std::move(ctx.receiver)); + } + if (required_rx_until_.has_value()) { + st.required_rx_until = *required_rx_until_; + } + LogicalPingAttemptRequest attempt_req{}; + attempt_req.actual_send_at = actual_send_at; + attempt_req.interval = timing_conf_.interval; + attempt_req.guard = guard; + attempt_req.attempt_lead = budget.attempt_lead; + attempt_req.retry_reserve = budget.retry_reserve; + attempt_req.loss_timeout = budget.loss_timeout; + attempt_req.base_rx_window = timing_conf_.rx_window; + attempt_req.predeadline_retry_guaranteed = + budget.predeadline_retry_guaranteed; + attempt_req.announce_unknown = announce_unknown_; + auto const view = ApplyLogicalPingAttempt(st, attempt_req); + if (view.started_new_cycle) { + // Freeze the R99 used for this logical cycle's expected response + // deadline so later RTT samples do not slide it. + st.frozen_p99_rtt = p99_rtt; + st.has_frozen_p99_rtt = true; +#if AE_ENABLE_PING_TEST_FAULTS + PingTestFaults::Instance().BindNextCycle(cloud_sc_->server_id(), + view.cycle_id); +#endif + EmitTrace(PingTraceKind::kCycleStarted); + } + + Duration timeout = budget.loss_timeout; + bool request_sent = true; + bool response_ignored = false; + std::int32_t fault_mode = 0; +#if AE_ENABLE_PING_TEST_FAULTS + auto const fault = PingTestFaults::Instance().Consume(PingFaultContext{ + cloud_sc_->server_id(), view.cycle_id, view.attempt_index, + view.first_attempt_at, actual_send_at}); + if (fault.timeout_override.count() > 0) { + timeout = fault.timeout_override; + } + request_sent = fault.mode != PingFaultMode::kDropRequest; + response_ignored = fault.mode == PingFaultMode::kIgnoreResponse; + fault_mode = static_cast(fault.mode); +#endif + + PingAttempt attempt{}; + attempt.server_id = cloud_sc_->server_id(); + attempt.planned_send_at = planned_send_at_; + attempt.actual_send_at = actual_send_at; + attempt.early_by = view.rx.early_by; + attempt.base_rx_window = timing_conf_.rx_window; + attempt.effective_wire_rx_window = view.rx.effective_wire_rx_window; + attempt.required_rx_until_before = required_rx_until_; + attempt.required_rx_until = view.rx.required_rx_until; + attempt.min_rtt = min_rtt; + attempt.p99_rtt = p99_rtt; + attempt.ping_guard = guard; + attempt.channel_generation = ++send_generation_; + attempt.logical_cycle_id = view.cycle_id; + attempt.physical_attempt_index = view.attempt_index; + attempt.fault_mode = fault_mode; + attempt.request_was_sent = request_sent; + attempt.response_was_ignored = response_ignored; + attempt.cycle_anchor = view.cycle_anchor; + attempt.contract_deadline = view.contract_deadline; + attempt.wire_next_connect_ms = view.wire_next_connect_ms; + attempt.retry_delay = + view.is_retry + ? SaturatingSubTime(actual_send_at, st.first_attempt_at) + : Duration{}; + attempt.attempt_lead = view.attempt_lead; + attempt.retry_reserve = view.retry_reserve; + attempt.loss_timeout = view.loss_timeout; + attempt.predeadline_retry_guaranteed = + view.predeadline_retry_guaranteed; + attempt.next_local_send_at = view.next_local_send; + + next_ping_time_ = view.next_local_send; + attempt.next_planned_send = next_ping_time_; + required_rx_until_ = view.rx.required_rx_until; + in_flight_ = attempt; + EmitTrace(PingTraceKind::kAttemptPrepared); + EmitTrace(PingTraceKind::kPrepared); + + ping_.emplace(ae_context_, *cloud_sc_, view.wire_next_connect, + view.rx.effective_wire_rx_window, timeout); +#if AE_ENABLE_PING_TEST_FAULTS + ping_->ApplyTestFault(static_cast(fault_mode)); +#endif - ping_blocker_ = policy_->AcquireSuspendBlock(); + if (request_sent) { + ping_blocker_ = policy_->AcquireSuspendBlock(); + } ping_->result_event().Subscribe( [this](Ping::PingResult const& res) noexcept { OnPingResult(res); ping_blocker_.Reset(); }); - // run ping request and open rx window - auto const current_time = Now(); - ping_->Start(current_time); - OpenRxWindow(current_time); - next_ping_time_ = current_time + timing_conf_.interval; - policy_->ReportNextServiceTime(priority_, next_ping_time_); - AE_TELED_DEBUG("Next ping time for priority {} at {} after {}", - priority_, next_ping_time_, timing_conf_.interval); + ping_->Start(actual_send_at); + if (request_sent) { + OpenRxWindow(); + EmitTrace(PingTraceKind::kSent); + EmitTrace(PingTraceKind::kRequestSent); + if (response_ignored) { + EmitTrace(PingTraceKind::kResponseIgnored); + } + } else { + EmitTrace(PingTraceKind::kRequestDropped); + } + if (timing_conf_.interval > Duration{}) { + policy_->ReportNextServiceTime(priority_, next_ping_time_); + } + AE_TELED_DEBUG( + "Next ping time for priority {} at {} after {} (guard {}) " + "early_by {} wire_rx {} cycle {} attempt {}", + priority_, next_ping_time_, timing_conf_.interval, guard, + view.rx.early_by, view.rx.effective_wire_rx_window, view.cycle_id, + view.attempt_index); return ex::set_value(std::move(ctx.receiver)); }); @@ -147,6 +340,11 @@ void PingCloudServers::ServerPing::Start() { if (stop_) { return; } + auto const& st = Cycle(); + if (!announce_unknown_ && owner_->auto_ping_enabled_ && st.confirmed && + Now() < st.next_local_send_at) { + return; + } waiter_.emplace( ae_context_, EnsureLinked() | @@ -172,10 +370,7 @@ void PingCloudServers::ServerPing::Start() { }), [this](std::optional&& res) noexcept { if (res && res->IsOk()) { - // repeat start on next_ping_time_ - start_sub_ = ae_context_.scheduler().DelayedTask( - [&]() noexcept { Start(); }, // ~['_']~ - next_ping_time_); + // Next logical cycle is scheduled only after confirmation. } else if (res && res->IsErr()) { AE_TELED_ERROR("Ping start error {}", std::move(res)->error()); } else { @@ -197,28 +392,182 @@ void PingCloudServers::ServerPing::OnPingResult(Ping::PingResult const& res) { return; } + auto& st = Cycle(); + auto const attempt = + in_flight_.has_value() ? in_flight_->physical_attempt_index : 0; + auto const cycle_id = + in_flight_.has_value() ? in_flight_->logical_cycle_id : 0; + auto const wire = in_flight_.has_value() ? in_flight_->effective_wire_rx_window + : timing_conf_.rx_window; + bool const request_sent = + !in_flight_.has_value() || in_flight_->request_was_sent; + std::visit( - [this, c](auto const& value) { + [this, c, wire, attempt, cycle_id, request_sent, &st](auto const& value) { using T = std::decay_t; - if constexpr (std::is_same_v>) { - c->channel_statistics().AddResponseTime(value.value); - } else if constexpr (std::is_same_v) { - AE_TELED_DEBUG("Got late ping duration"); - c->channel_statistics().AddResponseTime(value.duration); + if constexpr (std::is_same_v> || + std::is_same_v) { + if (!ShouldAcceptCycleResult(st.active, st.confirmed, st.cycle_id, + cycle_id, st.attempt_index, attempt, + st.current_attempt_timed_out, true)) { + return; + } + if constexpr (std::is_same_v>) { + c->channel_statistics().AddResponseTime(value.value); + EmitTrace(PingTraceKind::kResult, 0); + } else { + AE_TELED_DEBUG("Got late ping duration"); + c->channel_statistics().AddResponseTime(value.duration); + EmitTrace(PingTraceKind::kResult, 4); + } + if (request_sent) { + ScheduleRxWindowClose(ComputeRxWindowCloseTime(Now(), wire)); + } + ConfirmCycleAndScheduleNext(); + } else if constexpr (std::is_same_v>) { + if (!ShouldAcceptCycleResult(st.active, st.confirmed, st.cycle_id, + cycle_id, st.attempt_index, attempt, + st.current_attempt_timed_out, false)) { + return; + } + AE_TELED_ERROR("Ping error!"); + if (value.error == 2) { + MarkLogicalPingAttemptTimedOut(st); + EmitTrace(PingTraceKind::kAttemptTimeout, 2); + EmitTrace(PingTraceKind::kResult, 2); + // Keep the RX window open: a late pong or same-cycle retry still + // needs it. Closing here drops the next attempt's response. + ScheduleSameCycleRetryWithPreDeadlinePolicy(false); + return; + } + if (value.error == 1) { + MaybeCloseAfterWriteFailure(); + st.required_rx_until = required_rx_until_.value_or(TimePoint{}); + EmitTrace(PingTraceKind::kResult, 1); + ScheduleSameCycleRetryWithPreDeadlinePolicy(true); + return; + } + if (announce_unknown_) { + owner_->OnAnnounceServerDone(false); + return; + } + if (request_sent) { + ScheduleRxWindowClose(ComputeRxWindowCloseTime(Now(), wire)); + } + EmitTrace(PingTraceKind::kResult, value.error); + st.active = false; + st.confirmed = false; + ScheduleRestream(); + if (!owner_->auto_ping_enabled_) { + return; + } + planned_send_at_ = st.next_local_send_at; + next_ping_time_ = st.next_local_send_at; + if (timing_conf_.interval > Duration{}) { + policy_->ReportNextServiceTime(priority_, next_ping_time_); + start_sub_ = ae_context_.scheduler().DelayedTask( + [this]() { Start(); }, next_ping_time_); + } } else { AE_TELED_ERROR("Ping error!"); + if (request_sent) { + ScheduleRxWindowClose(ComputeRxWindowCloseTime(Now(), wire)); + } + EmitTrace(PingTraceKind::kResult, 3); + st.active = false; + st.confirmed = false; + if (announce_unknown_) { + owner_->OnAnnounceServerDone(false); + return; + } ScheduleRestream(); } }, res); } -void PingCloudServers::ServerPing::OpenRxWindow(TimePoint sent_time) { - // keep rx window suspend block for timing_.rx_window time - rx_window_blocker_ = policy_->AcquireSuspendBlock(); +void PingCloudServers::ServerPing::OpenRxWindow() { + if (!rx_window_held_) { + rx_window_blocker_ = policy_->AcquireSuspendBlock(); + rx_window_held_ = true; + } +} + +void PingCloudServers::ServerPing::ScheduleRxWindowClose(TimePoint close_time) { + OpenRxWindow(); + if (!ExtendLocalRxUntil(local_rx_, close_time)) { + return; + } + auto const gen = local_rx_.generation; rx_window_sub_ = ae_context_.scheduler().DelayedTask( - [this]() { rx_window_blocker_.Reset(); }, - sent_time + timing_conf_.rx_window); + [this, gen]() { + if (!ShouldApplyCloseTimer(local_rx_, gen, Now())) { + return; + } + CloseRxWindowNow(); + EmitTrace(PingTraceKind::kRxClosed); + }, + close_time); + EmitTrace(PingTraceKind::kRxCloseScheduled); +} + +void PingCloudServers::ServerPing::CloseRxWindowNow() { + rx_window_sub_.Reset(); + rx_window_blocker_.Reset(); + rx_window_held_ = false; + CloseLocalRx(local_rx_); +} + +void PingCloudServers::ServerPing::MaybeCloseAfterWriteFailure() { + if (in_flight_.has_value()) { + in_flight_->write_failed = true; + required_rx_until_ = in_flight_->required_rx_until_before; + } + auto const now = Now(); + if (ShouldCloseLocalRxAfterWriteFailure( + local_rx_, required_rx_until_.has_value(), + required_rx_until_.value_or(TimePoint{}), now)) { + CloseRxWindowNow(); + } +} + +void PingCloudServers::ServerPing::EmitTrace(PingTraceKind kind, + int result_type) const { + if (g_ping_trace_hook == nullptr || !in_flight_.has_value()) { + return; + } + auto const& a = *in_flight_; + PingTraceEvent event{}; + event.kind = kind; + event.server_id = a.server_id; + event.planned_send_at = a.planned_send_at.value_or(a.actual_send_at); + event.actual_send_at = a.actual_send_at; + event.early_by = a.early_by; + event.base_rx_window = a.base_rx_window; + event.effective_wire_rx_window = a.effective_wire_rx_window; + event.required_rx_until = a.required_rx_until; + event.next_planned_send = a.next_planned_send; + event.min_rtt = a.min_rtt; + event.p99_rtt = a.p99_rtt; + event.ping_guard = a.ping_guard; + event.channel_generation = a.channel_generation; + event.result_type = result_type; + event.event_time = Now(); + event.logical_cycle_id = a.logical_cycle_id; + event.physical_attempt_index = a.physical_attempt_index; + event.fault_mode = a.fault_mode; + event.request_was_sent = a.request_was_sent; + event.response_was_ignored = a.response_was_ignored; + event.cycle_anchor = a.cycle_anchor; + event.contract_deadline = a.contract_deadline; + event.wire_next_connect_ms = a.wire_next_connect_ms; + event.retry_delay = a.retry_delay; + event.next_local_send_at = a.next_local_send_at; + event.attempt_lead = a.attempt_lead; + event.retry_reserve = a.retry_reserve; + event.loss_timeout = a.loss_timeout; + event.predeadline_retry_guaranteed = a.predeadline_retry_guaranteed; + g_ping_trace_hook(event); } void PingCloudServers::ServerPing::ScheduleRestream() { @@ -237,6 +586,114 @@ void PingCloudServers::ServerPing::ScheduleRestream() { }); } +LogicalPingCycleState& PingCloudServers::ServerPing::Cycle() { + return owner_->cycle_states_[cloud_sc_->server_id()]; +} + +void PingCloudServers::ServerPing::ConfirmCycleAndScheduleNext() { + auto& st = Cycle(); + ConfirmLogicalPingCycle(st); + EmitTrace(PingTraceKind::kCycleConfirmed); + if (announce_unknown_) { + announce_unknown_ = false; + owner_->OnAnnounceServerDone(true); + return; + } + if (!owner_->auto_ping_enabled_ || timing_conf_.interval == Duration{}) { + return; + } + planned_send_at_ = st.next_local_send_at; + next_ping_time_ = st.next_local_send_at; + policy_->ReportNextServiceTime(priority_, next_ping_time_); + EmitTrace(PingTraceKind::kNextCycleScheduled); + if (Now() >= next_ping_time_) { + start_sub_ = ae_context_.scheduler().Task([this]() { Start(); }); + } else { + start_sub_ = ae_context_.scheduler().DelayedTask([this]() { Start(); }, + next_ping_time_); + } +} + +bool PingCloudServers::ServerPing::ChannelLinkedAndWritable() const { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { + return false; + } + auto const info = cc->stream_info(); + return info.link_state == LinkState::kLinked && info.is_writable; +} + +void PingCloudServers::ServerPing::ScheduleSameCycleRetryWithPreDeadlinePolicy( + bool restream_first) { + if (stop_) { + return; + } + if (announce_unknown_) { + ScheduleSameCycleRetry(restream_first); + return; + } + auto& st = Cycle(); + auto const now = Now(); + auto const deadline = st.next_nominal_ping_at; + if (CanSchedulePreDeadlineSameCycleRetry(now, deadline, st.attempt_index, + policy_->ping_retry_count())) { + ScheduleSameCycleRetry(restream_first); + return; + } + if (deadline != TimePoint{} && now < deadline) { + EmitTrace(PingTraceKind::kRetryScheduled); + start_sub_ = ae_context_.scheduler().DelayedTask( + [this, restream_first]() { + ScheduleSameCycleRetry(restream_first); + }, + deadline); + return; + } + ScheduleSameCycleRetry(restream_first); +} + +void PingCloudServers::ServerPing::ScheduleSameCycleRetry(bool restream_first) { + if (stop_) { + return; + } + auto& st = Cycle(); + st.awaiting_relink_retry = restream_first || !ChannelLinkedAndWritable(); + EmitTrace(PingTraceKind::kRetryScheduled); + auto kick_start = [this]() { + start_sub_ = ae_context_.scheduler().Task([this]() { Start(); }); + }; + if (restream_first) { + restream_blocker_ = policy_->AcquireSuspendBlock(); + restream_sub_ = ae_context_.scheduler().Task([this, kick_start]() { + auto* cc = cloud_sc_->client_connection(); + if (cc != nullptr) { + cc->Restream(); + } + restream_blocker_.Reset(); + cc = cloud_sc_->client_connection(); + if (cc != nullptr && + cc->stream_info().link_state == LinkState::kLinked) { + kick_start(); + } else if (cc != nullptr) { + WaitForLink(*cc, kick_start); + } else { + kick_start(); + } + }); + return; + } + if (ChannelLinkedAndWritable()) { + kick_start(); + return; + } + auto* cc = cloud_sc_->client_connection(); + if (cc != nullptr) { + WaitForLink(*cc, kick_start); + } else { + kick_start(); + } +} + PingCloudServers::PingCloudServers( AeContext const& ae_context, CloudServerConnections& cloud_server_connections, @@ -297,8 +754,9 @@ void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { it->second.reset(); } server_pings_.insert_or_assign( - server_id, std::make_unique(ae_context_, *policy_, cloud_sc, - priority)); + server_id, + std::make_unique(ae_context_, *this, *policy_, cloud_sc, + priority)); return; } } @@ -323,6 +781,89 @@ void PingCloudServers::ServerQuarantineReleased( server_pings_.erase(it); } } + +void PingCloudServers::ServerPing::AnnounceUnknown() { + announce_unknown_ = true; + start_sub_.Reset(); + ping_blocker_ = policy_->AcquireSuspendBlock(); + start_sub_ = ae_context_.scheduler().Task([this]() { Start(); }); +} + +bool PingCloudServers::ServerPing::quarantined() const noexcept { + return cloud_sc_ != nullptr && cloud_sc_->quarantine(); +} + +void PingCloudServers::StopAutomaticPing() noexcept { + auto_ping_enabled_ = false; +} + +std::optional PingCloudServers::expected_ping_response_time() + const noexcept { + if (!auto_ping_enabled_) { + return std::nullopt; + } + std::optional latest_expected_response; + for (auto const& [server_id, server_ping] : server_pings_) { + if (server_ping == nullptr || server_ping->stopped() || + server_ping->quarantined()) { + continue; + } + auto const cycle_it = cycle_states_.find(server_id); + if (cycle_it == cycle_states_.end()) { + continue; + } + AccumulateLatestExpectedPingResponse( + latest_expected_response, + ExpectedPingResponseTimeForCycle(cycle_it->second)); + } + return latest_expected_response; +} + +PingCloudServers::AnnounceEvent::Subscriber +PingCloudServers::announce_event() { + return EventSubscriber{announce_event_}; +} + +void PingCloudServers::BeginAnnounceUnknown() { + StopAutomaticPing(); + if (announce_in_progress_) { + return; + } + announce_in_progress_ = true; + announce_any_ok_ = false; + announce_pending_ = 0; + for (auto& [id, sp] : server_pings_) { + if (sp == nullptr || sp->stopped() || sp->quarantined()) { + continue; + } + ++announce_pending_; + sp->AnnounceUnknown(); + } + if (announce_pending_ == 0) { + announce_in_progress_ = false; + announce_event_.Emit(Ok{std::monostate{}}); + } +} + +void PingCloudServers::OnAnnounceServerDone(bool ok) { + if (!announce_in_progress_) { + return; + } + if (ok) { + announce_any_ok_ = true; + } + if (announce_pending_ > 0) { + --announce_pending_; + } + if (announce_pending_ == 0) { + announce_in_progress_ = false; + if (announce_any_ok_) { + announce_event_.Emit(Ok{std::monostate{}}); + } else { + announce_event_.Emit(Error{1}); + } + } +} } // namespace ae #endif diff --git a/aether/cloud_connections/ping_cloud_servers.h b/aether/cloud_connections/ping_cloud_servers.h index dd55ef65..0ea07c1c 100644 --- a/aether/cloud_connections/ping_cloud_servers.h +++ b/aether/cloud_connections/ping_cloud_servers.h @@ -20,9 +20,11 @@ #include "aether/config.h" #if AE_ENABLE_PING +# include # include # include # include +# include # include "aether/ae_context.h" # include "aether/events/event_subscription.h" @@ -33,18 +35,77 @@ # include "aether/ae_actions/ping.h" # include "aether/client_connectivity_policy.h" # include "aether/cloud_connections/cloud_server_connections.h" +# include "aether/cloud_connections/ping_schedule_guard.h" + +# include "aether-miscpp/types/result.h" namespace ae { + +enum class PingTraceKind : std::uint8_t { + kPrepared = 0, + kSent = 1, + kResult = 2, + kRxCloseScheduled = 3, + kRxClosed = 4, + kCycleStarted = 5, + kAttemptPrepared = 6, + kRequestDropped = 7, + kRequestSent = 8, + kResponseIgnored = 9, + kAttemptTimeout = 10, + kRetryScheduled = 11, + kCycleConfirmed = 12, + kNextCycleScheduled = 13, +}; + +struct PingTraceEvent { + PingTraceKind kind{PingTraceKind::kPrepared}; + ServerId server_id{}; + TimePoint planned_send_at{}; + TimePoint actual_send_at{}; + Duration early_by{}; + Duration base_rx_window{}; + Duration effective_wire_rx_window{}; + TimePoint required_rx_until{}; + TimePoint next_planned_send{}; + Duration min_rtt{}; + Duration p99_rtt{}; + Duration ping_guard{}; + std::uint64_t channel_generation{0}; + int result_type{-1}; + TimePoint event_time{}; + std::uint64_t logical_cycle_id{0}; + std::uint32_t physical_attempt_index{0}; + std::int32_t fault_mode{0}; + bool request_was_sent{false}; + bool response_was_ignored{false}; + TimePoint cycle_anchor{}; + TimePoint contract_deadline{}; + std::int64_t wire_next_connect_ms{0}; + Duration retry_delay{}; + TimePoint next_local_send_at{}; + Duration attempt_lead{}; + Duration retry_reserve{}; + Duration loss_timeout{}; + bool predeadline_retry_guaranteed{true}; +}; + +using PingTraceHook = void (*)(PingTraceEvent const&); +void SetPingTraceHook(PingTraceHook hook) noexcept; + class PingCloudServers { class ServerPing { public: - ServerPing(AeContext const& ae_context, ClientConnectivityPolicy& policy, + ServerPing(AeContext const& ae_context, PingCloudServers& owner, + ClientConnectivityPolicy& policy, CloudServerConnection& cloud_sc, std::size_t priority); ~ServerPing(); AE_CLASS_NO_COPY_MOVE(ServerPing) void Stop(); + void AnnounceUnknown(); + bool quarantined() const noexcept; TimePoint next_service_time() const noexcept { return next_ping_time_; } std::size_t priority() const noexcept { return priority_; } @@ -61,10 +122,20 @@ class PingCloudServers { auto MakePing(); void OnPingResult(Ping::PingResult const& res); - void OpenRxWindow(TimePoint sent_time); + void OpenRxWindow(); + void ScheduleRxWindowClose(TimePoint close_time); + void CloseRxWindowNow(); + void MaybeCloseAfterWriteFailure(); + void EmitTrace(PingTraceKind kind, int result_type = -1) const; void ScheduleRestream(); + LogicalPingCycleState& Cycle(); + void ConfirmCycleAndScheduleNext(); + void ScheduleSameCycleRetry(bool restream_first); + void ScheduleSameCycleRetryWithPreDeadlinePolicy(bool restream_first); + bool ChannelLinkedAndWritable() const; AeContext ae_context_; + PingCloudServers* owner_{}; ClientConnectivityPolicy* policy_; CloudServerConnection* cloud_sc_; RxTimingConf timing_conf_{}; @@ -81,7 +152,45 @@ class PingCloudServers { ClientConnectivityPolicy::SuspendBlocker ping_blocker_; ClientConnectivityPolicy::SuspendBlocker rx_window_blocker_; ClientConnectivityPolicy::SuspendBlocker restream_blocker_; - TimePoint next_ping_time_; + TimePoint next_ping_time_{}; + std::optional planned_send_at_{}; + std::optional required_rx_until_{}; + LocalRxWindowState local_rx_{}; + bool rx_window_held_{false}; + bool announce_unknown_{false}; + + struct PingAttempt { + ServerId server_id{}; + std::optional planned_send_at{}; + TimePoint actual_send_at{}; + Duration early_by{}; + Duration base_rx_window{}; + Duration effective_wire_rx_window{}; + TimePoint required_rx_until{}; + std::optional required_rx_until_before{}; + TimePoint next_planned_send{}; + Duration min_rtt{}; + Duration p99_rtt{}; + Duration ping_guard{}; + std::uint64_t channel_generation{0}; + bool write_failed{false}; + std::uint64_t logical_cycle_id{0}; + std::uint32_t physical_attempt_index{0}; + std::int32_t fault_mode{0}; + bool request_was_sent{true}; + bool response_was_ignored{false}; + TimePoint cycle_anchor{}; + TimePoint contract_deadline{}; + std::int64_t wire_next_connect_ms{0}; + Duration retry_delay{}; + TimePoint next_local_send_at{}; + Duration attempt_lead{}; + Duration retry_reserve{}; + Duration loss_timeout{}; + bool predeadline_retry_guaranteed{true}; + }; + std::optional in_flight_{}; + std::uint64_t send_generation_{0}; }; public: @@ -90,12 +199,23 @@ class PingCloudServers { ClientConnectivityPolicy& policy); ~PingCloudServers(); + void StopAutomaticPing() noexcept; + void BeginAnnounceUnknown(); + using AnnounceEvent = Event)>; + AnnounceEvent::Subscriber announce_event(); + + // Expected local receive time of the scheduled ping response for the + // current/next logical contract (Tn + frozen p99_RTT/2). nullopt when ping + // scheduling is disabled or no contract deadline exists yet. + std::optional expected_ping_response_time() const noexcept; + private: void ServersUpdate(); void DispatchToServers(); void ReconcileServer(CloudServerConnection& cloud_sc); void ServerQuarantined(CloudServerConnection* cloud_sc); void ServerQuarantineReleased(CloudServerConnection* cloud_sc); + void OnAnnounceServerDone(bool ok); AeContext ae_context_; CloudServerConnections* cloud_server_connections_; @@ -106,7 +226,13 @@ class PingCloudServers { Subscription server_quarantine_released_sub_; TaskSubscription task_sub_; + std::map cycle_states_; std::map> server_pings_; + bool auto_ping_enabled_{true}; + std::size_t announce_pending_{0}; + bool announce_in_progress_{false}; + bool announce_any_ok_{false}; + AnnounceEvent announce_event_; }; } // namespace ae diff --git a/aether/cloud_connections/ping_schedule_guard.h b/aether/cloud_connections/ping_schedule_guard.h new file mode 100644 index 00000000..87c9524f --- /dev/null +++ b/aether/cloud_connections/ping_schedule_guard.h @@ -0,0 +1,859 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_PING_SCHEDULE_GUARD_H_ +#define AETHER_CLOUD_CONNECTIONS_PING_SCHEDULE_GUARD_H_ + +#include +#include +#include +#include + +#include "aether/clock.h" +#include "aether/config.h" +#include "aether/receive_schedule.h" + +namespace ae { + +inline constexpr Duration kPingGuardFloor = + std::chrono::duration_cast(std::chrono::milliseconds{10}); + +// guard = max(0, (p99 - min) / 2) + 10ms +inline Duration ComputePingSendGuard(Duration min_rtt, + Duration p99_rtt) noexcept { + Duration spread{}; + if (p99_rtt > min_rtt) { + spread = (p99_rtt - min_rtt) / 2; + } + return spread + kPingGuardFloor; +} + +// Guard cannot be >= ping_interval; leave at least 1ms before nominal interval. +inline Duration ClampPingSendGuard(Duration guard, + Duration ping_interval) noexcept { + auto const min_remaining = + std::chrono::duration_cast(std::chrono::milliseconds{1}); + if (ping_interval <= min_remaining) { + return Duration{}; + } + auto const max_guard = ping_interval - min_remaining; + return guard > max_guard ? max_guard : guard; +} + +#ifdef AE_PING_GUARD_OVERRIDE_US +inline Duration ResolvePingSendGuard(Duration min_rtt, Duration p99_rtt, + Duration ping_interval) noexcept { + (void)min_rtt; + (void)p99_rtt; + auto const fixed = + Duration{static_cast(AE_PING_GUARD_OVERRIDE_US)}; + return ClampPingSendGuard(fixed, ping_interval); +} +#else +inline Duration ResolvePingSendGuard(Duration min_rtt, Duration p99_rtt, + Duration ping_interval) noexcept { + return ClampPingSendGuard(ComputePingSendGuard(min_rtt, p99_rtt), + ping_interval); +} +#endif + +// Empty stats: min = p99 = 200ms → guard = 10ms. +// One sample: min == p99 → guard = 10ms. +template +inline Duration ComputePingSendGuardFromStats( + Stats const& stats, Duration ping_interval) noexcept { + auto const estimate = + std::chrono::duration_cast(std::chrono::milliseconds{200}); + Duration min_rtt = estimate; + Duration p99_rtt = estimate; + if (!stats.empty()) { + min_rtt = stats.min(); + p99_rtt = stats.template percentile<99>(); + } + return ResolvePingSendGuard(min_rtt, p99_rtt, ping_interval); +} + +inline Duration SaturatingSubDurationValue(Duration a, Duration b) noexcept { + if (b.count() == 0) { + return a; + } + if (a.count() <= b.count()) { + return Duration{}; + } + return Duration{ + static_cast(a.count() - b.count())}; +} + +inline Duration SaturatingAddDuration(Duration a, Duration b) noexcept; + +inline Duration SaturatingMulDuration(Duration d, + std::uint32_t factor) noexcept { + if (factor == 0 || d.count() <= 0) { + return Duration{}; + } + auto const max = Duration::max(); + std::uint64_t acc = static_cast(d.count()); + auto const step = static_cast(d.count()); + auto const max_count = static_cast(max.count()); + for (std::uint32_t i = 1; i < factor; ++i) { + if (acc > max_count - step) { + return max; + } + acc += step; + } + if (acc > max_count) { + return max; + } + return Duration{static_cast(acc)}; +} + +inline Duration DivideDurationFloor(Duration total, + std::uint32_t divisor) noexcept { + if (divisor == 0 || total.count() <= 0) { + return Duration{}; + } + return Duration{static_cast(total.count() / divisor)}; +} + +inline bool CanSchedulePreDeadlineSameCycleRetry( + TimePoint now, TimePoint contract_deadline, std::uint32_t attempt_index, + std::uint8_t pre_deadline_retry_count) noexcept { + if (contract_deadline == TimePoint{} || now >= contract_deadline) { + return true; + } + return attempt_index <= pre_deadline_retry_count; +} + +inline void IncrementLogicalPingAttemptIndex( + std::uint32_t& attempt_index) noexcept { + if (attempt_index < std::numeric_limits::max()) { + ++attempt_index; + } +} + +inline constexpr Duration kPingSchedulerMargin = + std::chrono::duration_cast(std::chrono::milliseconds{10}); +// Fixed allowance for local timeout-to-retry dispatch/scheduling latency. +// Network uncertainty is accounted for separately by RTT p99 and the ping +// guard. Sized from first-request-loss p99 characterization (combined +// required-extra p99 ~40ms, observed max ~48ms); raised to 60ms after a +// residual ~3ms TCP outlier at 50ms. +inline constexpr Duration kPingRetryDispatchMargin = + std::chrono::duration_cast(std::chrono::milliseconds{60}); +inline constexpr Duration kPingMinLossTimeout = + std::chrono::duration_cast(std::chrono::milliseconds{50}); +inline constexpr Duration kPingP99TimeoutMargin = + std::chrono::duration_cast(std::chrono::milliseconds{10}); +inline constexpr Duration kPingRttEstimate = + std::chrono::duration_cast(std::chrono::milliseconds{200}); + +struct PingRetryBudgetInput { + Duration interval{}; + Duration guard{}; + Duration raw_timeout{}; + Duration p99_rtt{}; + std::uint8_t retry_count{kDefaultPingRetryCount}; +}; + +struct PingRetryBudget { + Duration retry_one_way_budget{}; + Duration scheduler_margin{}; + Duration retry_dispatch_margin{}; + Duration loss_timeout{}; + Duration retry_reserve{}; + Duration attempt_lead{}; + Duration max_timeout_for_predeadline_retry{}; + bool predeadline_retry_guaranteed{true}; +}; + +// loss_timeout = max(raw, p99+10ms, 50ms), capped so N retries can still +// finish before Tn when the interval allows it. +// retry_reserve = N*loss_timeout + p99/2 + N*scheduler_margin +// + N*retry_dispatch_margin +// ≈ (N + 0.5)*p99 + N*D + N*scheduler (when uncapped) +// attempt_lead = guard + retry_reserve +inline PingRetryBudget ComputePingRetryBudget( + PingRetryBudgetInput const& in) noexcept { + PingRetryBudget out{}; + auto const retry_count = + in.retry_count > kMaxPingRetryCount ? kMaxPingRetryCount : in.retry_count; + out.scheduler_margin = kPingSchedulerMargin; + out.retry_dispatch_margin = kPingRetryDispatchMargin; + out.retry_one_way_budget = in.p99_rtt / 2; + + Duration loss = in.raw_timeout; + auto const p99_with_margin = + SaturatingAddDuration(in.p99_rtt, kPingP99TimeoutMargin); + if (p99_with_margin > loss) { + loss = p99_with_margin; + } + if (kPingMinLossTimeout > loss) { + loss = kPingMinLossTimeout; + } + + auto const one_ms = + std::chrono::duration_cast(std::chrono::milliseconds{1}); + auto const scheduler_total = + SaturatingMulDuration(out.scheduler_margin, retry_count); + auto const dispatch_total = + SaturatingMulDuration(out.retry_dispatch_margin, retry_count); + auto remaining = in.interval; + auto subtract_ok = [&](Duration d) { + if (d.count() == 0) { + return true; + } + if (remaining.count() <= d.count()) { + remaining = Duration{}; + return false; + } + remaining = SaturatingSubDurationValue(remaining, d); + return remaining.count() > 0; + }; + bool budget_fits = subtract_ok(in.guard) && + subtract_ok(out.retry_one_way_budget); + if (retry_count > 0) { + budget_fits = budget_fits && subtract_ok(scheduler_total) && + subtract_ok(dispatch_total); + } + budget_fits = budget_fits && subtract_ok(one_ms); + if (retry_count > 0) { + out.max_timeout_for_predeadline_retry = + DivideDurationFloor(remaining, retry_count); + out.predeadline_retry_guaranteed = budget_fits && remaining.count() > 0; + if (out.predeadline_retry_guaranteed && + loss > out.max_timeout_for_predeadline_retry) { + loss = out.max_timeout_for_predeadline_retry; + } + } else { + out.max_timeout_for_predeadline_retry = Duration{}; + out.predeadline_retry_guaranteed = budget_fits; + } + out.loss_timeout = loss.count() == 0 ? kPingMinLossTimeout : loss; + auto const loss_total = SaturatingMulDuration(out.loss_timeout, retry_count); + out.retry_reserve = SaturatingAddDuration( + SaturatingAddDuration( + SaturatingAddDuration(loss_total, out.retry_one_way_budget), + scheduler_total), + dispatch_total); + out.attempt_lead = SaturatingAddDuration(in.guard, out.retry_reserve); + if (in.interval > one_ms) { + auto const max_lead = SaturatingSubDurationValue(in.interval, one_ms); + if (out.attempt_lead > max_lead) { + out.attempt_lead = max_lead; + out.predeadline_retry_guaranteed = false; + } + } else { + out.attempt_lead = Duration{}; + out.predeadline_retry_guaranteed = false; + } + if (out.loss_timeout.count() == 0) { + out.loss_timeout = kPingMinLossTimeout; + out.predeadline_retry_guaranteed = false; + } + return out; +} + +template +inline PingRetryBudget ComputePingRetryBudgetFromStats( + Stats const& stats, Duration ping_interval, Duration raw_timeout, + std::uint8_t retry_count = kDefaultPingRetryCount) noexcept { + Duration p99_rtt = kPingRttEstimate; + Duration min_rtt = kPingRttEstimate; + if (!stats.empty()) { + min_rtt = stats.min(); + p99_rtt = stats.template percentile<99>(); + } + auto const guard = ResolvePingSendGuard(min_rtt, p99_rtt, ping_interval); + Duration raw = raw_timeout; + if (raw.count() == 0) { + raw = kPingRttEstimate; + } + return ComputePingRetryBudget(PingRetryBudgetInput{ + ping_interval, guard, raw, p99_rtt, retry_count}); +} + +inline Duration SaturatingAddDuration(Duration a, Duration b) noexcept { + auto const max = Duration::max(); + if (b > max - a) { + return max; + } + return a + b; +} + +inline TimePoint SaturatingAddTime(TimePoint t, Duration d) noexcept { + if (d.count() == 0) { + return t; + } + using Tick = typename TimePoint::duration; + auto const add = std::chrono::duration_cast(d); + auto const max_t = TimePoint::max(); + if (t >= max_t) { + return max_t; + } + if (add > max_t - t) { + return max_t; + } + return t + add; +} + +// One-way return-path estimate for expected ping response receive time: +// R99/2 (same bootstrap RTT as ping scheduling when stats are empty). +inline Duration OneWayReturnEstimateFromP99(Duration p99_rtt) noexcept { + return DivideDurationFloor(p99_rtt, 2); +} + +// Expected local receive time of the scheduled ping response. +// Tn is the request-arrival contract deadline; response return time is +// estimated as Tn + p99_RTT/2. Guard and scheduler/dispatch margins are not +// included. +inline TimePoint ExpectedPingResponseTime(TimePoint contract_deadline_tn, + Duration p99_rtt) noexcept { + return SaturatingAddTime(contract_deadline_tn, + OneWayReturnEstimateFromP99(p99_rtt)); +} + +// Positive later-earlier as Duration; zero if later <= earlier. Saturates. +inline Duration SaturatingSubTime(TimePoint later, TimePoint earlier) noexcept { + if (later <= earlier) { + return Duration{}; + } + auto const delta = later - earlier; + auto const us = + std::chrono::duration_cast(delta); + if (us.count() <= 0) { + return Duration{}; + } + auto const max_us = static_cast( + std::numeric_limits::max()); + if (us.count() >= max_us) { + return Duration::max(); + } + return Duration{static_cast(us.count())}; +} + +inline std::int64_t DurationToSaturatedInt64Ms(Duration d) noexcept { + if (d.count() <= 0) { + return 0; + } + auto const us = + std::chrono::duration_cast(d).count(); + if (us <= 0) { + return std::numeric_limits::max(); + } + constexpr auto max_ms = std::numeric_limits::max(); + if (us > max_ms - 999) { + return max_ms; + } + return static_cast(us / 1000); +} + +struct EarlyRxWindowInput { + bool has_nominal_ping{false}; + TimePoint nominal_ping_at{}; + TimePoint actual_send_at{}; + Duration base_rx_window{}; + bool has_required_rx_until{false}; + TimePoint required_rx_until{}; +}; + +struct EarlyRxWindowOutput { + Duration early_by{}; + Duration effective_wire_rx_window{}; + TimePoint required_rx_until{}; +}; + +// required_end = max(Tn + W, previous required, Ai + W). Negative diffs are 0. +inline EarlyRxWindowOutput ComputeEarlyRxWindow( + EarlyRxWindowInput const& in) noexcept { + EarlyRxWindowOutput out{}; + if (in.has_nominal_ping && in.nominal_ping_at > in.actual_send_at) { + out.early_by = SaturatingSubTime(in.nominal_ping_at, in.actual_send_at); + } + + auto raise = [&](TimePoint candidate) { + if (out.required_rx_until == TimePoint{} || + candidate > out.required_rx_until) { + out.required_rx_until = candidate; + } + }; + + if (in.has_required_rx_until) { + raise(in.required_rx_until); + } + raise(SaturatingAddTime(in.actual_send_at, in.base_rx_window)); + if (in.has_nominal_ping) { + raise(SaturatingAddTime(in.nominal_ping_at, in.base_rx_window)); + } + + out.effective_wire_rx_window = + SaturatingSubTime(out.required_rx_until, in.actual_send_at); + if (in.base_rx_window.count() > 0 && + out.effective_wire_rx_window < in.base_rx_window) { + out.effective_wire_rx_window = in.base_rx_window; + raise(SaturatingAddTime(in.actual_send_at, in.base_rx_window)); + } + return out; +} + +struct LocalRxWindowState { + bool open{false}; + TimePoint close_at{}; + std::uint64_t generation{0}; +}; + +// Returns true if the close timer must be (re)scheduled. Never shortens. +inline bool ExtendLocalRxUntil(LocalRxWindowState& state, + TimePoint close_at) noexcept { + if (state.open && close_at <= state.close_at) { + return false; + } + state.open = true; + state.close_at = close_at; + ++state.generation; + return true; +} + +inline bool ShouldApplyCloseTimer(LocalRxWindowState const& state, + std::uint64_t generation, + TimePoint now) noexcept { + return state.open && state.generation == generation && now >= state.close_at; +} + +inline void CloseLocalRx(LocalRxWindowState& state) noexcept { + state.open = false; +} + +// Write failure before the ping is recorded must not close a still-open window. +inline bool ShouldCloseLocalRxAfterWriteFailure( + LocalRxWindowState const& state, bool has_required_until, + TimePoint required_until, TimePoint now) noexcept { + bool const previous_active = + (has_required_until && required_until > now) || + (state.open && state.close_at > now); + return !previous_active; +} + +// Local RX capability closes at pong/timeout receive + receive_window +// (not at ping send + receive_window). +inline TimePoint ComputeRxWindowCloseTime(TimePoint receive_or_timeout_time, + Duration receive_window) noexcept { + return SaturatingAddTime(receive_or_timeout_time, receive_window); +} + +inline TimePoint SaturatingSubDuration(TimePoint t, Duration d) noexcept { + if (d.count() == 0) { + return t; + } + using Tick = typename TimePoint::duration; + using Rep = typename Tick::rep; + using URep = std::make_unsigned_t; + auto const sub = std::chrono::duration_cast(d); + if (sub.count() <= 0) { + return d.count() > 0 ? TimePoint::min() : t; + } + auto const t_count = t.time_since_epoch().count(); + auto const sub_count = sub.count(); + auto const min_count = TimePoint::min().time_since_epoch().count(); + auto const room = static_cast(t_count) - static_cast(min_count); + if (static_cast(sub_count) > room) { + return TimePoint::min(); + } + return t - sub; +} + +// Floor remaining to whole milliseconds. Known schedules never wire 0. +inline std::int64_t FloorDurationToPositiveInt64Ms(Duration d) noexcept { + if (d.count() <= 0) { + return 1; + } + auto const us = + std::chrono::duration_cast(d).count(); + if (us <= 0) { + return 1; + } + constexpr auto max_ms = std::numeric_limits::max(); + if (us / 1000 >= max_ms) { + return max_ms; + } + auto const ms = static_cast(us / 1000); + return ms < 1 ? 1 : ms; +} + +// Ceil to whole milliseconds so the server RX window is not shorter. +inline std::int64_t CeilDurationToSaturatedInt64Ms(Duration d) noexcept { + if (d.count() <= 0) { + return 0; + } + auto const us = + std::chrono::duration_cast(d).count(); + if (us <= 0) { + return std::numeric_limits::max(); + } + constexpr auto max_ms = std::numeric_limits::max(); + if (us > max_ms - 999) { + return max_ms; + } + return static_cast((us + 999) / 1000); +} + +inline TimePoint SaturatingAddTicks( + TimePoint t, typename TimePoint::duration add) noexcept { + auto const max_t = TimePoint::max(); + if (add.count() <= 0) { + return t; + } + if (t >= max_t) { + return max_t; + } + if (add > max_t - t) { + return max_t; + } + return t + add; +} + +// Advance deadline by whole intervals until it is strictly after retry_actual. +// Uses ceil-style division; never iterates once per interval. +inline TimePoint AdvanceContractDeadlinePast(TimePoint deadline, + TimePoint retry_actual, + Duration interval) noexcept { + if (retry_actual < deadline) { + return deadline; + } + using Tick = typename TimePoint::duration; + auto interval_ticks = std::chrono::duration_cast(interval); + if (interval_ticks.count() <= 0) { + interval_ticks = + std::chrono::duration_cast(std::chrono::milliseconds{1}); + } + if (interval_ticks.count() <= 0) { + return TimePoint::max(); + } + auto const elapsed = + retry_actual >= deadline ? (retry_actual - deadline) : Tick{}; + auto const iv = interval_ticks.count(); + auto const el = elapsed.count(); + auto const max_rep = std::numeric_limits::max(); + typename Tick::rep n = 1; + if (el > 0) { + if (el / iv > max_rep - 1) { + return TimePoint::max(); + } + n = el / iv + 1; + } + if (iv > 0 && n > max_rep / iv) { + return TimePoint::max(); + } + return SaturatingAddTicks(deadline, Tick{iv * n}); +} + +enum class PingErrorRetryAction : std::uint8_t { + kImmediateSameCycle = 0, + kRestreamThenSameCycle = 1, +}; + +inline PingErrorRetryAction PingErrorRetryActionFor(int error_code) noexcept { + if (error_code == 2) { + return PingErrorRetryAction::kImmediateSameCycle; + } + return PingErrorRetryAction::kRestreamThenSameCycle; +} + +inline bool ShouldAcceptCycleResult(bool cycle_active, bool cycle_confirmed, + std::uint64_t current_cycle_id, + std::uint64_t result_cycle_id, + std::uint32_t current_attempt, + std::uint32_t result_attempt, + bool current_attempt_timed_out, + bool result_is_ok_or_late) noexcept { + if (!cycle_active || cycle_confirmed) { + return false; + } + if (result_cycle_id != current_cycle_id || result_attempt == 0) { + return false; + } + (void)current_attempt_timed_out; + if (result_is_ok_or_late) { + return true; + } + return result_attempt == current_attempt; +} + +struct LogicalPingCycleState { + std::uint64_t cycle_id{0}; + bool active{false}; + bool confirmed{false}; + bool has_schedule{false}; + bool bootstrap{false}; + bool current_attempt_timed_out{false}; + bool awaiting_relink_retry{false}; + bool predeadline_retry_guaranteed{true}; + std::uint32_t attempt_index{0}; + TimePoint first_attempt_at{}; + TimePoint actual_attempt_send_at{}; + TimePoint nominal_ping_at{}; + TimePoint next_nominal_ping_at{}; + TimePoint next_local_send_at{}; + TimePoint required_rx_until{}; + Duration configured_interval{}; + Duration base_rx_window{}; + Duration current_guard{}; + Duration current_retry_reserve{}; + Duration current_attempt_lead{}; + Duration current_loss_timeout{}; + // R99 frozen when this logical cycle's schedule budget was established so + // expected_ping_response_time does not slide on later RTT samples. + bool has_frozen_p99_rtt{false}; + Duration frozen_p99_rtt{kPingRttEstimate}; +}; + +// Tn for the current outstanding scheduled ping response expectation. +inline std::optional LogicalPingContractDeadline( + LogicalPingCycleState const& st) noexcept { + if (!st.has_schedule) { + return std::nullopt; + } + if (st.active && !st.confirmed) { + // Request-arrival contract for the in-flight logical cycle. + if (st.nominal_ping_at == TimePoint{}) { + return std::nullopt; + } + return st.nominal_ping_at; + } + // After confirm (or before the next attempt starts): next nominal Tn. + if (st.next_nominal_ping_at == TimePoint{}) { + return std::nullopt; + } + return st.next_nominal_ping_at; +} + +inline std::optional ExpectedPingResponseTimeForCycle( + LogicalPingCycleState const& st) noexcept { + auto const tn = LogicalPingContractDeadline(st); + if (!tn.has_value() || !st.has_frozen_p99_rtt) { + return std::nullopt; + } + return ExpectedPingResponseTime(*tn, st.frozen_p99_rtt); +} + +// Among active-server expected response times, keep the latest deadline. +inline void AccumulateLatestExpectedPingResponse( + std::optional& latest_expected_response, + std::optional candidate) noexcept { + if (!candidate.has_value()) { + return; + } + if (!latest_expected_response.has_value() || + *candidate > *latest_expected_response) { + latest_expected_response = candidate; + } +} + +// Monotonic update for client last-online runtime timestamp. +inline void UpdateMonotonicLastOnlineTime( + std::optional& last_online_time, TimePoint when) noexcept { + if (!last_online_time.has_value() || when > *last_online_time) { + last_online_time = when; + } +} + +struct LogicalPingAttemptRequest { + TimePoint actual_send_at{}; + Duration interval{}; + Duration guard{}; + Duration attempt_lead{}; + Duration retry_reserve{}; + Duration loss_timeout{}; + Duration base_rx_window{}; + bool predeadline_retry_guaranteed{true}; + bool announce_unknown{false}; +}; + +struct LogicalPingAttemptView { + std::uint64_t cycle_id{0}; + std::uint32_t attempt_index{0}; + bool is_retry{false}; + bool started_new_cycle{false}; + bool bootstrap{false}; + TimePoint first_attempt_at{}; + TimePoint actual_attempt_send_at{}; + TimePoint nominal_ping_at{}; + TimePoint next_nominal_ping_at{}; + TimePoint cycle_anchor{}; + TimePoint contract_deadline{}; + TimePoint next_local_send{}; + Duration wire_next_connect{}; + std::int64_t wire_next_connect_ms{0}; + Duration attempt_lead{}; + Duration retry_reserve{}; + Duration loss_timeout{}; + bool predeadline_retry_guaranteed{true}; + EarlyRxWindowOutput rx{}; +}; + +inline Duration ScheduleLeadFor(LogicalPingAttemptRequest const& req, + Duration guard) noexcept { + if (req.attempt_lead.count() > 0) { + return req.attempt_lead; + } + return guard; +} + +inline LogicalPingAttemptView ApplyLogicalPingAttempt( + LogicalPingCycleState& st, LogicalPingAttemptRequest const& req) noexcept { + LogicalPingAttemptView view{}; + bool const new_cycle = !st.active || st.confirmed; + view.started_new_cycle = new_cycle; + view.is_retry = !new_cycle; + + auto const guard = ClampPingSendGuard(req.guard, req.interval); + auto const lead = ScheduleLeadFor(req, guard); + + st.current_guard = guard; + st.current_attempt_lead = lead; + st.current_retry_reserve = req.retry_reserve; + st.current_loss_timeout = req.loss_timeout; + st.predeadline_retry_guaranteed = req.predeadline_retry_guaranteed; + st.configured_interval = req.interval; + st.base_rx_window = req.base_rx_window; + st.actual_attempt_send_at = req.actual_send_at; + + if (req.announce_unknown) { + if (new_cycle) { + st.cycle_id += 1; + if (st.cycle_id == 0) { + st.cycle_id = 1; + } + st.active = true; + st.confirmed = false; + st.current_attempt_timed_out = false; + st.awaiting_relink_retry = false; + st.attempt_index = 1; + st.first_attempt_at = req.actual_send_at; + st.nominal_ping_at = req.actual_send_at; + st.next_nominal_ping_at = req.actual_send_at; + st.bootstrap = false; + } else { + IncrementLogicalPingAttemptIndex(st.attempt_index); + st.current_attempt_timed_out = false; + st.awaiting_relink_retry = false; + } + view.wire_next_connect = Duration{}; + view.wire_next_connect_ms = 0; + st.next_local_send_at = TimePoint::max(); + } else if (new_cycle) { + st.cycle_id += 1; + if (st.cycle_id == 0) { + st.cycle_id = 1; + } + st.active = true; + st.confirmed = false; + st.current_attempt_timed_out = false; + st.awaiting_relink_retry = false; + st.attempt_index = 1; + + if (!st.has_schedule) { + st.bootstrap = true; + st.nominal_ping_at = req.actual_send_at; + st.next_nominal_ping_at = + SaturatingAddTime(st.nominal_ping_at, req.interval); + st.first_attempt_at = req.actual_send_at; + view.wire_next_connect = req.interval; + st.has_schedule = req.interval.count() != 0; + } else { + st.bootstrap = false; + auto tn = st.next_nominal_ping_at; + auto tn1 = SaturatingAddTime(tn, req.interval); + if (req.actual_send_at >= tn1) { + tn1 = AdvanceContractDeadlinePast(tn1, req.actual_send_at, + req.interval); + tn = SaturatingSubDuration(tn1, req.interval); + } + st.nominal_ping_at = tn; + st.next_nominal_ping_at = tn1; + st.first_attempt_at = SaturatingSubDuration(tn, lead); + view.wire_next_connect = + SaturatingSubTime(st.next_nominal_ping_at, req.actual_send_at); + } + st.next_local_send_at = + SaturatingSubDuration(st.next_nominal_ping_at, lead); + } else { + IncrementLogicalPingAttemptIndex(st.attempt_index); + st.current_attempt_timed_out = false; + st.awaiting_relink_retry = false; + if (req.actual_send_at >= st.next_nominal_ping_at) { + auto const old_next = st.next_nominal_ping_at; + st.next_nominal_ping_at = AdvanceContractDeadlinePast( + st.next_nominal_ping_at, req.actual_send_at, req.interval); + auto const shift = + SaturatingSubTime(st.next_nominal_ping_at, old_next); + st.nominal_ping_at = SaturatingAddTime(st.nominal_ping_at, shift); + st.next_local_send_at = + SaturatingSubDuration(st.next_nominal_ping_at, lead); + } + view.wire_next_connect = + SaturatingSubTime(st.next_nominal_ping_at, req.actual_send_at); + } + + EarlyRxWindowInput rx_in{}; + rx_in.has_nominal_ping = st.nominal_ping_at != TimePoint{} || + st.has_schedule || new_cycle; + rx_in.nominal_ping_at = st.nominal_ping_at; + rx_in.actual_send_at = req.actual_send_at; + rx_in.base_rx_window = req.base_rx_window; + rx_in.has_required_rx_until = st.required_rx_until != TimePoint{}; + rx_in.required_rx_until = st.required_rx_until; + view.rx = ComputeEarlyRxWindow(rx_in); + st.required_rx_until = view.rx.required_rx_until; + + if (req.announce_unknown || + (req.interval.count() == 0 && new_cycle)) { + view.wire_next_connect_ms = 0; + } else { + view.wire_next_connect_ms = + FloorDurationToPositiveInt64Ms(view.wire_next_connect); + } + + view.cycle_id = st.cycle_id; + view.attempt_index = st.attempt_index; + view.bootstrap = st.bootstrap; + view.first_attempt_at = st.first_attempt_at; + view.actual_attempt_send_at = st.actual_attempt_send_at; + view.nominal_ping_at = st.nominal_ping_at; + view.next_nominal_ping_at = st.next_nominal_ping_at; + view.cycle_anchor = st.nominal_ping_at; + view.contract_deadline = st.next_nominal_ping_at; + view.next_local_send = st.next_local_send_at; + view.attempt_lead = lead; + view.retry_reserve = req.retry_reserve; + view.loss_timeout = req.loss_timeout; + view.predeadline_retry_guaranteed = st.predeadline_retry_guaranteed; + return view; +} + +inline void ConfirmLogicalPingCycle(LogicalPingCycleState& st) noexcept { + st.confirmed = true; + st.active = false; + st.current_attempt_timed_out = false; + st.awaiting_relink_retry = false; +} + +inline void MarkLogicalPingAttemptTimedOut(LogicalPingCycleState& st) noexcept { + st.current_attempt_timed_out = true; +} + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_PING_SCHEDULE_GUARD_H_ diff --git a/aether/config.h b/aether/config.h index f9ef0d3c..f6532be4 100644 --- a/aether/config.h +++ b/aether/config.h @@ -290,11 +290,22 @@ # define AE_ENABLE_PING 1 #endif +// Test-only ping request/response fault injection. Production default is 0. +#ifndef AE_ENABLE_PING_TEST_FAULTS +# define AE_ENABLE_PING_TEST_FAULTS 0 +#endif + // Send ping interval, ms #ifndef AE_PING_INTERVAL_MS # define AE_PING_INTERVAL_MS AE_DEFAULT_RESPONSE_TIMEOUT_MS + 1000 #endif +// Optional fixed ping send guard in microseconds. When defined, overrides the +// dynamic ComputePingSendGuard() result. When undefined, production uses the +// dynamic guard. Defining this macro to 0 means a fixed zero guard, not +// "override disabled". +// #define AE_PING_GUARD_OVERRIDE_US 10000 + // window size for safe stream response time statistics #ifndef AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE # define AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE 100 diff --git a/aether/connection_manager/client_cloud_manager.cpp b/aether/connection_manager/client_cloud_manager.cpp index 5de4863e..a128ad46 100644 --- a/aether/connection_manager/client_cloud_manager.cpp +++ b/aether/connection_manager/client_cloud_manager.cpp @@ -162,6 +162,10 @@ ClientCloudManager::ClientCloudManager(ObjProp prop, ObjPtr aether, }); }); assert(cache_initialized && "Client did not load"); + + // Action pools only — do not open cloud_connection()/ping here. Callers may + // still need SetReceiveSchedule() before the first cloud_connection(). + Init(); } ClientCloudManager::CloudUpdateEvent::Subscriber @@ -174,9 +178,8 @@ GetCloudAction& ClientCloudManager::GetCloud(Uid client_uid) { auto aether = aether_.Load(); assert(aether && "Aether did not loaded"); - if (!cloud_actions_) { - cloud_actions_.emplace(*aether); - } + + assert(cloud_actions_ && "Cloud actions did not initiated"); auto cached = cloud_cache_.find(client_uid); if ((cached != cloud_cache_.end()) && cached->second.cloud.is_valid()) { @@ -199,11 +202,26 @@ GetCloudAction& ClientCloudManager::GetCloud(Uid client_uid) { return *action; } -void ClientCloudManager::StartListenForCloudUpdate() { +void ClientCloudManager::Init() { + if (cloud_actions_) { + return; + } auto aether = aether_.Load(); assert(aether && "Aether must be loaded"); + cloud_actions_.emplace(*aether); get_servers_pool_.emplace(*aether); +} + +void ClientCloudManager::StartCloudUpdateListener() { + Init(); + if (cloud_update_listening_) { + return; + } + cloud_update_listening_ = true; + ListenForCloudUpdate(); +} +void ClientCloudManager::ListenForCloudUpdate() { auto client = client_.Load(); assert(client && "Client does not loaded"); diff --git a/aether/connection_manager/client_cloud_manager.h b/aether/connection_manager/client_cloud_manager.h index f7b86293..72a002e6 100644 --- a/aether/connection_manager/client_cloud_manager.h +++ b/aether/connection_manager/client_cloud_manager.h @@ -91,11 +91,15 @@ class ClientCloudManager : public Obj { GetCloudAction& GetCloud(Uid client_uid); - AE_OBJECT_REFLECT(AE_MMBRS(aether_, client_, cloud_cache_)) + // Subscribe to server cloud-config pushes. Safe to call after + // Client::cloud_connection() has been created; no-op if already listening. + void StartCloudUpdateListener(); - void StartListenForCloudUpdate(); + AE_OBJECT_REFLECT(AE_MMBRS(aether_, client_, cloud_cache_)) private: + void Init(); + void ListenForCloudUpdate(); void CloudConfigs(std::vector const& configs); void FinalizeCloudConfig(CloudConfig const& conf); auto MakeServersSender(std::vector const& sids); @@ -109,6 +113,7 @@ class ClientCloudManager : public Obj { CloudUpdateEvent cloud_update_event_; CloudEventListener cloud_update_sub_; + bool cloud_update_listening_{false}; std::optional cloud_actions_; std::optional get_servers_pool_; std::vector +#include + +#include "aether/clock.h" + +namespace ae { + +inline constexpr std::uint8_t kDefaultPingRetryCount = 0; +inline constexpr std::uint8_t kMaxPingRetryCount = 8; + +struct ReceiveSchedule { + Duration ping_interval{}; + Duration receive_window{}; + // Additional same-cycle retries reserved in the pre-deadline send budget + // before Tn. Does not cap post-deadline recovery retries after Tn. + std::uint8_t ping_retry_count{kDefaultPingRetryCount}; +}; + +enum class PeerScheduleState { + kExpected, + kMissedDeadline, + kUnknown, +}; + +// Library TimePoint / ae::Now() local timeline only (relative offsets; no +// Unix-epoch wall remapping of server timestamps). +// last_online is converted from lastConnectDeltaMs (online/activity, not +// necessarily a ping). +struct PeerReceiveSchedule { + TimePoint last_online{}; + std::optional next_ping_deadline{}; + PeerScheduleState state{PeerScheduleState::kUnknown}; +}; + +enum class SetReceiveScheduleError : int { + kPingAlreadyStarted = 1, +}; + +enum class ReceiveSendPhase { + kInsideReceiveWindow, + kOutsideReceiveWindow, +}; + +inline ReceiveSendPhase ClassifyReceiveSendOffset( + Duration offset_from_last_ping, Duration receive_window) noexcept { + return offset_from_last_ping < receive_window + ? ReceiveSendPhase::kInsideReceiveWindow + : ReceiveSendPhase::kOutsideReceiveWindow; +} + +} // namespace ae + +#endif // AETHER_RECEIVE_SCHEDULE_H_ diff --git a/aether/server_connections/client_server_connection.cpp b/aether/server_connections/client_server_connection.cpp index d6a96675..7b939842 100644 --- a/aether/server_connections/client_server_connection.cpp +++ b/aether/server_connections/client_server_connection.cpp @@ -142,6 +142,7 @@ ClientServerConnection::ClientServerConnection(AeContext const& ae_context, Ptr const& client, Ptr const& server) : ae_context_{ae_context}, + client_{client}, server_{server}, uid_{client->uid()}, ephemeral_uid_{client->ephemeral_uid()}, @@ -154,6 +155,17 @@ ClientServerConnection::ClientServerConnection(AeContext const& ae_context, AE_TELED_DEBUG("Client server connection from {}:e-{} to {}", uid_, ephemeral_uid_, server->server_id); + protocol_context_.set_inbound_server_response_hook( + [](void* user) noexcept { + auto* self = static_cast(user); + auto client_ptr = self->client_.Lock(); + if (!client_ptr) { + return; + } + client_ptr->MarkServerResponseReceived(Now()); + }, + this); + server_connection_.out_data_event().Subscribe( MethodPtr<&ClientServerConnection::OutData>{this}); } diff --git a/aether/server_connections/client_server_connection.h b/aether/server_connections/client_server_connection.h index 6996fc8a..9d12ac87 100644 --- a/aether/server_connections/client_server_connection.h +++ b/aether/server_connections/client_server_connection.h @@ -78,6 +78,7 @@ class ClientServerConnection { void OutData(DataBuffer const& data); AeContext ae_context_; + PtrView client_; PtrView server_; Uid uid_; Uid ephemeral_uid_; diff --git a/aether/work_cloud_api/client_timing.h b/aether/work_cloud_api/client_timing.h new file mode 100644 index 00000000..a0fd19a8 --- /dev/null +++ b/aether/work_cloud_api/client_timing.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ +#define AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ + +#include + +#include "aether-miscpp/reflect/reflect.h" + +namespace ae { + +// Wire DTO for AuthorizedApi.get_client_timing. Field order matches ADSL: +// nextPingDeltaMs then lastConnectDeltaMs. +struct ClientTiming { + AE_REFLECT_MEMBERS(next_ping_delta_ms, last_connect_delta_ms) + + std::int64_t next_ping_delta_ms{}; + std::int64_t last_connect_delta_ms{}; +}; + +} // namespace ae + +#endif // AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ diff --git a/aether/work_cloud_api/uap.h b/aether/work_cloud_api/uap.h new file mode 100644 index 00000000..02cda24c --- /dev/null +++ b/aether/work_cloud_api/uap.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_WORK_CLOUD_API_UAP_H_ +#define AETHER_WORK_CLOUD_API_UAP_H_ + +#include + +#include "aether-miscpp/reflect/reflect.h" + +namespace ae { + +// Wire DTO for AuthorizedApi.get_uap. Field order matches ADSL: +// deltaMs then lastReadTimestamp. +struct Uap { + AE_REFLECT_MEMBERS(delta_ms, last_read_timestamp_ms) + + std::int64_t delta_ms{}; + std::int64_t last_read_timestamp_ms{}; +}; + +} // namespace ae + +#endif // AETHER_WORK_CLOUD_API_UAP_H_ diff --git a/aether/work_cloud_api/work_server_api/authorized_api.cpp b/aether/work_cloud_api/work_server_api/authorized_api.cpp index 22542c94..2ceb0b38 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.cpp +++ b/aether/work_cloud_api/work_server_api/authorized_api.cpp @@ -26,5 +26,8 @@ AuthorizedApi::AuthorizedApi(ProtocolContext& protocol_context) resolver_servers{protocol_context}, resolver_clouds{protocol_context}, send_telemetry{protocol_context}, + set_next_read_delay{protocol_context}, + get_uap{protocol_context}, + get_client_timing{protocol_context}, report_applied_config{protocol_context} {} } // namespace ae diff --git a/aether/work_cloud_api/work_server_api/authorized_api.h b/aether/work_cloud_api/work_server_api/authorized_api.h index d72c4d66..e0096fe6 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.h +++ b/aether/work_cloud_api/work_server_api/authorized_api.h @@ -26,6 +26,8 @@ #include "aether/work_cloud_api/ae_message.h" #include "aether/work_cloud_api/telemetric.h" #include "aether/work_cloud_api/cloud_configs.h" +#include "aether/work_cloud_api/client_timing.h" +#include "aether/work_cloud_api/uap.h" namespace ae { @@ -33,8 +35,8 @@ class AuthorizedApi : public ApiClass { public: explicit AuthorizedApi(ProtocolContext& protocol_context); - Method<4, ApiPromise(std::uint64_t next_connect_ms_duration, - std::uint64_t rx_window_ms)> + Method<4, ApiPromise(std::int64_t next_connect_ms_duration, + std::int64_t rx_window_ms)> ping; Method<6, void(AeMessage message)> send_message; Method<7, void(std::vector messages)> send_messages; @@ -44,6 +46,12 @@ class AuthorizedApi : public ApiClass { Method<18, void(Telemetric telemetric)> send_telemetry; + // Legacy: mutates primary nextReadDelay; not a temporary RX window. + // Kept for wire compatibility. The schedule path must not call this. + Method<33, void(std::int64_t delay_ms)> set_next_read_delay; + Method<34, ApiPromise(Uid uid)> get_uap; + Method<35, ApiPromise(Uid uid)> get_client_timing; + Method<38, void(std::vector configs)> report_applied_config; }; } // namespace ae diff --git a/aether/work_cloud_api/work_server_api/login_api.cpp b/aether/work_cloud_api/work_server_api/login_api.cpp index 07e45bc1..12e3739f 100644 --- a/aether/work_cloud_api/work_server_api/login_api.cpp +++ b/aether/work_cloud_api/work_server_api/login_api.cpp @@ -22,6 +22,7 @@ namespace ae { LoginApi::LoginApi(ProtocolContext& protocol_context, IEncryptProvider& encrypt_provider) : ApiClass{protocol_context}, + get_time_utc{protocol_context}, login_by_uid{protocol_context, LoginProc{*this}}, login_by_alias{protocol_context, LoginProc{*this}}, get_my_ip{protocol_context}, diff --git a/aether/work_cloud_api/work_server_api/login_api.h b/aether/work_cloud_api/work_server_api/login_api.h index a41bf102..4a7fc3ac 100644 --- a/aether/work_cloud_api/work_server_api/login_api.h +++ b/aether/work_cloud_api/work_server_api/login_api.h @@ -17,6 +17,8 @@ #ifndef AETHER_WORK_CLOUD_API_WORK_SERVER_API_LOGIN_API_H_ #define AETHER_WORK_CLOUD_API_WORK_SERVER_API_LOGIN_API_H_ +#include + #include "aether/types/uid.h" #include "aether/types/data_buffer.h" #include "aether/crypto/icrypto_provider.h" @@ -44,6 +46,9 @@ class LoginApi : public ApiClass { explicit LoginApi(ProtocolContext& protocol_context, IEncryptProvider& encrypt_provider); + // Initialized before login_by_uid (ADSL method order). + Method<3, ApiPromise()> get_time_utc; + Method<4, void(Uid uid, SubApi sub_api), LoginProc> login_by_uid; Method<5, void(Uid alias, SubApi sub_api), LoginProc> diff --git a/config/user_config_uap_delivery_tcp.h b/config/user_config_uap_delivery_tcp.h new file mode 100644 index 00000000..75d9700f --- /dev/null +++ b/config/user_config_uap_delivery_tcp.h @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONFIG_USER_CONFIG_UAP_DELIVERY_TCP_H_ +#define CONFIG_USER_CONFIG_UAP_DELIVERY_TCP_H_ + +#include "aether/config_consts.h" + +#define AE_CRYPTO_ASYNC AE_HYDRO_CRYPTO_PK +#define AE_CRYPTO_SYNC AE_HYDRO_CRYPTO_SK +#define AE_SIGNATURE AE_HYDRO_SIGNATURE +#define AE_KDF AE_HYDRO_KDF + +// TCP-only work path: registration and work servers use TCP. +#define AE_SUPPORT_UDP 0 +#define AE_SUPPORT_TCP 1 +#define AE_UAP_DELIVERY_REQUIRE_UDP 0 + +#if ESP_PLATFORM +# define AE_CLOUD_MAX_SERVER_CONNECTIONS 1 +# define AE_SAFE_STREAM_CAPACITY 2 * 1024 +#endif + +#if !ESP_PLATFORM +# define AE_SUPPORT_WIFIS 0 +#endif + +// Quiet console tele for bench runs. +#define AE_TELE_ENABLED 1 +#define AE_TELE_LOG_CONSOLE 0 +#define AE_TELE_COMPILATION_INFO 0 +#define AE_TELE_LOG_TO_STATISTICS 1 +#define AE_STATISTICS_MAX_SIZE 1024 + +// all except MLog +#define AE_TELE_METRICS_MODULES_EXCLUDE {AE_LOG_MODULE} +#define AE_TELE_METRICS_DURATION_EXCLUDE {AE_LOG_MODULE} + +#define AE_TELE_LOG_MODULES AE_ALL +#define AE_TELE_DEBUG_MODULES AE_ALL +#define AE_TELE_INFO_MODULES AE_ALL +#define AE_TELE_WARN_MODULES AE_ALL +#define AE_TELE_ERROR_MODULES AE_ALL + +#define AE_TELE_LOG_TIME_POINT AE_ALL +// location only for kLog module +#define AE_TELE_LOG_LOCATION {AE_LOG_MODULE} +// tag name for all except kLog +#define AE_TELE_LOG_NAME_EXCLUDE {AE_LOG_MODULE} +#define AE_TELE_LOG_LEVEL_MODULE AE_ALL +#define AE_TELE_LOG_BLOB AE_ALL + +#if AE_DISTILLATION || AE_FILTRATION +# define AE_SUPPORT_REGISTRATION 1 +# define AE_SUPPORT_CLOUD_DNS 1 +#else +# define AE_SUPPORT_REGISTRATION 0 +# define AE_SUPPORT_CLOUD_DNS 0 +#endif + +#endif /* CONFIG_USER_CONFIG_UAP_DELIVERY_TCP_H_ */ diff --git a/config/user_config_uap_delivery_udp.h b/config/user_config_uap_delivery_udp.h new file mode 100644 index 00000000..bb7595d0 --- /dev/null +++ b/config/user_config_uap_delivery_udp.h @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONFIG_USER_CONFIG_UAP_DELIVERY_UDP_H_ +#define CONFIG_USER_CONFIG_UAP_DELIVERY_UDP_H_ + +#include "aether/config_consts.h" + +#define AE_CRYPTO_ASYNC AE_HYDRO_CRYPTO_PK +#define AE_CRYPTO_SYNC AE_HYDRO_CRYPTO_SK +#define AE_SIGNATURE AE_HYDRO_SIGNATURE +#define AE_KDF AE_HYDRO_KDF + +// TCP+UDP: registration cloud is TCP-only; work servers also advertise UDP. +#define AE_SUPPORT_UDP 1 +#define AE_SUPPORT_TCP 1 +// Bench measured work path must be UDP (registration may still be TCP). +#define AE_UAP_DELIVERY_REQUIRE_UDP 1 + +#if ESP_PLATFORM +# define AE_CLOUD_MAX_SERVER_CONNECTIONS 1 +# define AE_SAFE_STREAM_CAPACITY 2 * 1024 +#endif + +#if !ESP_PLATFORM +# define AE_SUPPORT_WIFIS 0 +#endif + +// Quiet console tele for bench runs. +#define AE_TELE_ENABLED 1 +#define AE_TELE_LOG_CONSOLE 0 +#define AE_TELE_COMPILATION_INFO 0 +#define AE_TELE_LOG_TO_STATISTICS 1 +#define AE_STATISTICS_MAX_SIZE 1024 + +// all except MLog +#define AE_TELE_METRICS_MODULES_EXCLUDE {AE_LOG_MODULE} +#define AE_TELE_METRICS_DURATION_EXCLUDE {AE_LOG_MODULE} + +#define AE_TELE_LOG_MODULES AE_ALL +#define AE_TELE_DEBUG_MODULES AE_ALL +#define AE_TELE_INFO_MODULES AE_ALL +#define AE_TELE_WARN_MODULES AE_ALL +#define AE_TELE_ERROR_MODULES AE_ALL + +#define AE_TELE_LOG_TIME_POINT AE_ALL +// location only for kLog module +#define AE_TELE_LOG_LOCATION {AE_LOG_MODULE} +// tag name for all except kLog +#define AE_TELE_LOG_NAME_EXCLUDE {AE_LOG_MODULE} +#define AE_TELE_LOG_LEVEL_MODULE AE_ALL +#define AE_TELE_LOG_BLOB AE_ALL + +#if AE_DISTILLATION || AE_FILTRATION +# define AE_SUPPORT_REGISTRATION 1 +# define AE_SUPPORT_CLOUD_DNS 1 +#else +# define AE_SUPPORT_REGISTRATION 0 +# define AE_SUPPORT_CLOUD_DNS 0 +#endif + +#endif /* CONFIG_USER_CONFIG_UAP_DELIVERY_UDP_H_ */ diff --git a/examples/aether_uap_1s_timing_characterization/CMakeLists.txt b/examples/aether_uap_1s_timing_characterization/CMakeLists.txt new file mode 100644 index 00000000..3f41f362 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM AND WIN32) + project("aether_uap_1s_timing_characterization" VERSION "1.0.0" LANGUAGES C CXX) + + add_executable(aether_uap_1s_timing_characterization + main.cpp + coordinator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../aether_uap_ping_retry_window_test/client_role.cpp + ) + target_link_libraries(aether_uap_1s_timing_characterization PRIVATE + aether + aether_uap_delivery_timing_bench_common + ) + target_include_directories(aether_uap_1s_timing_characterization PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../aether_uap_ping_retry_window_test + ${CMAKE_CURRENT_SOURCE_DIR}/../.. + ${CMAKE_CURRENT_SOURCE_DIR}/../benches/aether_uap_delivery_timing_bench + ) + target_compile_definitions(aether_uap_1s_timing_characterization PRIVATE + _CRT_SECURE_NO_WARNINGS + ) + if(MSVC) + target_compile_options(aether_uap_1s_timing_characterization PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + endif() +else() + message(WARNING "aether_uap_1s_timing_characterization is Windows desktop only; skipped") +endif() diff --git a/examples/aether_uap_1s_timing_characterization/coordinator.cpp b/examples/aether_uap_1s_timing_characterization/coordinator.cpp new file mode 100644 index 00000000..06411721 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/coordinator.cpp @@ -0,0 +1,1869 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "coordinator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#if defined(RegisterClass) +# undef RegisterClass +#endif + +#include "common/bench_ipc.h" +#include "common/bench_types.h" +#include "common/udp_proof_types.h" + +#include "aether/config.h" +#include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/ae_actions/ping_test_faults.h" +#include "aether/receive_schedule.h" + +namespace ae::test_uap_ping_retry_window { + +#if AE_ENABLE_PING_TEST_FAULTS +using ae::PingFaultMode; +#endif +using ae::PingTraceKind; +using ae::bench::uap::BenchProtocol; +using ae::bench::uap::ChannelProof; +using ae::bench::uap::EventKind; +using ae::bench::uap::IpcFrame; +using ae::bench::uap::IpcType; +using ae::bench::uap::NamedPipeServer; +using ae::bench::uap::PipeNameFor; +using ae::bench::uap::UdpProofPath; +using ae::bench::uap::UnpackUdpProofFrame; +using IpcSide = ae::bench::uap::Side; + +namespace { +constexpr std::uint8_t kIpcArmFault = 13; +constexpr std::uint8_t kIpcSendTagged = 14; +constexpr std::uint8_t kIpcQueryNow = 15; +constexpr std::uint8_t kIpcPingTraceEx = 16; +constexpr std::uint8_t kIpcAnnounceUnknown = 17; +constexpr std::uint8_t kIpcScheduleState = 18; +constexpr std::uint8_t kIpcPingBudget = 19; +constexpr std::uint8_t kIpcQueryStats = 20; +constexpr std::uint8_t kIpcFaultTrace = 21; + +struct BobFaultTraceEvent { + std::uint8_t kind{0}; + std::int64_t server_id{0}; + std::int64_t logical_cycle_id{0}; + std::int64_t physical_attempt_index{0}; + std::int64_t mode{0}; + std::int64_t harness_state{0}; + std::int64_t trace_kind{0}; + std::int64_t steady_us{0}; +}; + +struct BobPingEvent { + std::uint8_t kind{0}; + std::int64_t server_id{0}; + std::int64_t planned_us{0}; + std::int64_t actual_us{0}; + std::int64_t early_by_us{0}; + std::int64_t base_window_us{0}; + std::int64_t effective_window_us{0}; + std::int64_t required_until_us{0}; + std::int64_t next_planned_us{0}; + std::int64_t guard_us{0}; + std::int64_t min_rtt_us{0}; + std::int64_t p99_rtt_us{0}; + std::int64_t channel_generation{0}; + std::int64_t result_type{0}; + std::int64_t event_steady_us{0}; + std::int64_t logical_cycle_id{0}; + std::int64_t physical_attempt_index{0}; + std::int64_t fault_mode{0}; + std::int64_t wire_next_connect_ms{0}; + std::int64_t cycle_anchor_us{0}; + std::int64_t contract_deadline_us{0}; + std::int64_t next_local_send_us{0}; + std::int64_t request_was_sent{0}; + std::int64_t response_was_ignored{0}; + std::int64_t event_qpc{0}; + std::int64_t attempt_lead_us{0}; + std::int64_t retry_reserve_us{0}; + std::int64_t loss_timeout_us{0}; + std::int64_t predeadline_retry_guaranteed{1}; +}; + +struct ScheduleSnap { + std::int64_t state{-2}; + std::int64_t next_us{0}; + std::int64_t last_online_us{0}; + std::int64_t selected{0}; + std::int64_t queried{0}; + std::int64_t successful{0}; + std::int64_t failed{0}; + std::int64_t skipped{0}; + std::int64_t qpc{0}; + std::int64_t steady_us{0}; + std::int64_t checkpoint{-1}; + std::int64_t next_ping_delta_ms{std::numeric_limits::min()}; + std::int64_t last_connect_delta_ms{std::numeric_limits::min()}; +}; + +struct QueryStatsSnap { + std::int64_t attempts{0}; + std::int64_t created{0}; + std::int64_t reused{0}; + std::int64_t skipped{0}; + std::int64_t extra{0}; + std::int64_t checkpoint{-1}; + std::int64_t qpc{0}; + std::int64_t steady_us{0}; +}; + +struct SampleRec { + std::string offset_name; + std::int64_t offset_ms{0}; + std::int64_t window_ms{0}; + std::uint32_t sequence{0}; + std::int64_t tn_us{0}; + std::int64_t next_us{0}; + std::int64_t send_qpc{0}; + std::int64_t recv_qpc{0}; + std::int64_t recv_steady_us{0}; + std::int64_t schedule_server{0}; + std::int64_t actual_server{0}; + std::int64_t route_generation{0}; + std::int64_t protocol{0}; + std::int64_t raw_delta_ms{0}; + std::int64_t last_connect_ms{0}; + std::int64_t one_way_us{0}; + int recv_count{0}; + std::string classification{"LOST"}; + double delivery_ms{0}; + double deadline_error_ms{0}; + double window_end_slack_ms{0}; + bool premature{false}; +}; + +struct CycleRec { + int planned_fault{0}; + std::int64_t cycle_id{0}; + std::int64_t first_wire_next{0}; + std::int64_t retry_wire_next{0}; + std::int64_t tn_us{0}; + std::int64_t tn1_us{0}; + std::int64_t first_attempt_us{0}; + std::int64_t retry_attempt_us{0}; + std::int64_t timeout_us{0}; + std::int64_t timeout_qpc{0}; + std::int64_t retry_qpc{0}; + std::int64_t early_by_us{0}; + std::int64_t guard_us{0}; + std::int64_t retry_reserve_us{0}; + std::int64_t loss_timeout_us{0}; + std::int64_t attempt_lead_us{0}; + bool predeadline{true}; + bool retry_before_nominal{false}; + bool confirmed{false}; +}; + +struct OfflineRec { + std::string condition; + double deadline_to_state_ms{0}; + double start_to_state_ms{0}; + int query_count{0}; + std::int64_t final_state{-2}; + bool false_state{false}; +}; + +struct ChildProc { + IpcSide side{}; + NamedPipeServer pipe; + PROCESS_INFORMATION pi{}; + std::uint64_t uid_lo{0}; + std::uint64_t uid_hi{0}; + bool ready{false}; + bool uid_ok{false}; + std::uint32_t seq{0}; + ChannelProof own_proof{}; + ChannelProof dest_proof{}; + bool got_own_proof{false}; + bool got_dest_proof{false}; + std::vector ping_events; + std::vector fault_traces; + std::vector schedules; + std::vector query_stats; + std::map recv_counts; + std::map recv_qpc; + std::map recv_steady; + std::map send_qpc; + std::map sent_samples; + std::int64_t last_ack_a{0}; + std::int64_t last_ack_b{0}; + bool got_ack{false}; + bool warmup_done{false}; + std::int64_t warmup_n{0}; + std::int64_t warmup_min{0}; + std::int64_t warmup_p99{0}; + std::int64_t warmup_d{0}; + std::int64_t warmup_e{0}; + std::uint32_t warmup_guard{0}; +}; + +void ResetChildRuntime(ChildProc& child) { + child.uid_lo = 0; + child.uid_hi = 0; + child.ready = false; + child.uid_ok = false; + child.seq = 0; + child.own_proof = {}; + child.dest_proof = {}; + child.got_own_proof = false; + child.got_dest_proof = false; + child.ping_events.clear(); + child.schedules.clear(); + child.query_stats.clear(); + child.recv_counts.clear(); + child.recv_qpc.clear(); + child.recv_steady.clear(); + child.send_qpc.clear(); + child.sent_samples.clear(); + child.got_ack = false; + child.warmup_done = false; +} + +std::string MakeRunId() { + SYSTEMTIME st{}; + GetSystemTime(&st); + char buf[64]; + std::snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u", st.wYear, + st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond); + return buf; +} + +std::string DefaultExePath() { + char path[MAX_PATH]{}; + GetModuleFileNameA(nullptr, path, MAX_PATH); + return path; +} + +bool SendCmd(ChildProc& child, IpcType type, std::uint32_t sequence = 0, + std::uint32_t offset_ms = 0, std::int64_t a = 0, + std::int64_t b = 0, std::int64_t c = 0) { + IpcFrame f{}; + f.type = static_cast(type); + f.side = static_cast(IpcSide::kCoordinator); + f.seq = ++child.seq; + f.sequence = sequence; + f.offset_ms = offset_ms; + f.a = a; + f.b = b; + f.c = c; + return child.pipe.WriteFrame(f); +} + +bool SendRaw(ChildProc& child, std::uint8_t type, std::uint32_t sequence = 0, + std::int64_t a = 0, std::int64_t b = 0, std::int64_t c = 0, + std::int64_t d = 0, std::int64_t e = 0, std::int64_t f = 0, + std::int64_t g = 0, std::int64_t h = 0) { + IpcFrame frame{}; + frame.type = type; + frame.side = static_cast(IpcSide::kCoordinator); + frame.seq = ++child.seq; + frame.sequence = sequence; + frame.a = a; + frame.b = b; + frame.c = c; + frame.d = d; + frame.e = e; + frame.f = f; + frame.g = g; + frame.h = h; + return child.pipe.WriteFrame(frame); +} + +void HandleChildFrame(ChildProc& child, IpcFrame const& frame) { + auto const type = static_cast(frame.type); + if (type == IpcType::kUidReport) { + std::memcpy(&child.uid_lo, &frame.a, 8); + std::memcpy(&child.uid_hi, &frame.b, 8); + child.uid_ok = true; + } + if (type == IpcType::kChildReady || type == IpcType::kUidReport) { + child.ready = true; + } + if (type == IpcType::kAck) { + child.got_ack = true; + child.last_ack_a = frame.a; + child.last_ack_b = frame.b; + } + if (type == IpcType::kWarmupDone) { + child.warmup_done = true; + child.warmup_n = frame.a; + child.warmup_min = frame.b; + child.warmup_p99 = frame.c; + child.warmup_d = frame.d; + child.warmup_e = frame.e; + child.warmup_guard = frame.offset_ms; + } + if (type == IpcType::kUdpProof) { + auto proof = UnpackUdpProofFrame(frame); + auto const path = static_cast(frame.event_kind); + if (path == UdpProofPath::kOwn) { + child.own_proof = proof; + child.got_own_proof = true; + } else if (path == UdpProofPath::kDestination) { + child.dest_proof = proof; + child.got_dest_proof = true; + } + } + if (type == IpcType::kEvent && + static_cast(frame.event_kind) == EventKind::kSampleReceived) { + auto count = static_cast(frame.c); + if (count <= 0) { + count = 1; + } + child.recv_counts[frame.sequence] = count; + child.recv_qpc[frame.sequence] = frame.b; + child.recv_steady[frame.sequence] = frame.local_steady_us; + } + if (type == IpcType::kSampleResult && + static_cast(frame.event_kind) == EventKind::kSampleSent) { + auto send_qpc = frame.c; + if (send_qpc == 0) { + send_qpc = frame.e; + } + if (send_qpc != 0) { + child.send_qpc[frame.sequence] = send_qpc; + } + SampleRec rec{}; + rec.sequence = frame.sequence; + rec.tn_us = frame.a; + rec.next_us = frame.b; + rec.send_qpc = send_qpc; + rec.schedule_server = frame.d; + rec.actual_server = frame.e; + rec.route_generation = frame.f; + rec.protocol = frame.g; + rec.raw_delta_ms = frame.h; + rec.last_connect_ms = frame.i; + rec.one_way_us = frame.k; + child.sent_samples[frame.sequence] = rec; + } + if (type == IpcType::kPingTrace) { + BobPingEvent e{}; + e.kind = frame.event_kind; + e.server_id = frame.a; + e.planned_us = frame.b; + e.actual_us = frame.c; + e.early_by_us = frame.d; + e.base_window_us = frame.e; + e.effective_window_us = frame.f; + e.required_until_us = frame.g; + e.next_planned_us = frame.h; + e.guard_us = frame.i; + e.channel_generation = frame.j; + e.min_rtt_us = frame.k; + e.p99_rtt_us = frame.l; + e.result_type = static_cast(frame.offset_ms); + e.event_steady_us = frame.local_steady_us; + child.ping_events.push_back(e); + } + if (frame.type == kIpcPingTraceEx && !child.ping_events.empty()) { + auto& e = child.ping_events.back(); + e.logical_cycle_id = frame.a; + e.physical_attempt_index = frame.b; + e.fault_mode = frame.c; + e.wire_next_connect_ms = frame.d; + e.cycle_anchor_us = frame.e; + e.contract_deadline_us = frame.f; + e.next_local_send_us = frame.g; + e.request_was_sent = frame.h; + e.response_was_ignored = frame.i; + e.event_qpc = frame.k; + e.retry_reserve_us = frame.l; + } + if (frame.type == kIpcPingBudget && !child.ping_events.empty()) { + auto& e = child.ping_events.back(); + e.attempt_lead_us = frame.a; + e.retry_reserve_us = frame.b; + e.loss_timeout_us = frame.c; + e.predeadline_retry_guaranteed = frame.d; + if (e.cycle_anchor_us == 0) { + e.cycle_anchor_us = frame.e; + } + if (e.contract_deadline_us == 0) { + e.contract_deadline_us = frame.f; + } + if (e.guard_us == 0) { + e.guard_us = frame.g; + } + } + if (frame.type == kIpcFaultTrace) { + BobFaultTraceEvent t{}; + t.kind = static_cast(frame.f); + t.server_id = frame.a; + t.logical_cycle_id = frame.b; + t.physical_attempt_index = frame.c; + t.mode = frame.d; + t.harness_state = frame.e; + t.trace_kind = frame.f; + t.steady_us = frame.g != 0 ? frame.g : frame.local_steady_us; + child.fault_traces.push_back(t); + } + if (frame.type == kIpcScheduleState) { + ScheduleSnap s{}; + s.state = frame.a; + s.next_us = frame.b; + s.last_online_us = frame.c; + s.selected = frame.d; + s.queried = frame.e; + s.successful = frame.f; + s.failed = frame.g; + s.skipped = frame.h; + s.qpc = frame.i; + s.steady_us = frame.local_steady_us; + s.checkpoint = frame.j; + s.next_ping_delta_ms = frame.k; + s.last_connect_delta_ms = frame.l; + child.schedules.push_back(s); + } + if (frame.type == kIpcQueryStats) { + QueryStatsSnap q{}; + q.attempts = frame.a; + q.created = frame.b; + q.reused = frame.c; + q.skipped = frame.d; + q.extra = frame.e; + q.checkpoint = frame.f; + q.qpc = frame.i; + q.steady_us = frame.local_steady_us; + child.query_stats.push_back(q); + } +} + +bool SpawnChild(ChildProc& child, CharacterizationArgs const& args, + std::string const& state_dir, std::string const& pipe_name, + std::string const& client_name, + std::string const& child_log_path, std::int64_t interval_ms, + std::int64_t window_ms) { + if (!child.pipe.Create(pipe_name)) { + std::cerr << "CreateNamedPipe failed for " << pipe_name << "\n"; + return false; + } + auto cmd = "\"" + args.exe_path + "\" --role client --side " + + std::string(child.side == IpcSide::kA ? "A" : "B") + " --run-id " + + args.run_id + " --state-dir \"" + state_dir + "\" --pipe \"" + + pipe_name + "\" --client-name " + client_name + " --parent-uid " + + args.parent_uid + " --ping-interval-ms " + + std::to_string(interval_ms) + " --receive-window-ms " + + std::to_string(window_ms); + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE log = CreateFileA(child_log_path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, + &sa, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (log == INVALID_HANDLE_VALUE) { + std::cerr << "CreateFile child log failed: " << child_log_path << "\n"; + return false; + } + STARTUPINFOA si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = log; + si.hStdError = log; + std::vector cmdline(cmd.begin(), cmd.end()); + cmdline.push_back('\0'); + if (!CreateProcessA(nullptr, cmdline.data(), nullptr, nullptr, TRUE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &child.pi)) { + CloseHandle(log); + std::cerr << "CreateProcess failed: " << GetLastError() << "\n"; + return false; + } + CloseHandle(log); + if (!child.pipe.WaitForClient(120000)) { + std::cerr << "WaitForClient timeout side=" + << (child.side == IpcSide::kA ? "A" : "B") << "\n"; + return false; + } + return true; +} + +void StopChild(ChildProc& child) { + if (child.pi.hProcess == nullptr) { + child.pipe.Close(); + return; + } + SendCmd(child, IpcType::kShutdown); + if (WaitForSingleObject(child.pi.hProcess, 15000) != WAIT_OBJECT_0) { + TerminateProcess(child.pi.hProcess, 1); + } + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.pi = {}; + child.pipe.Close(); +} + +void KillChild(ChildProc& child) { + if (child.pi.hProcess != nullptr) { + TerminateProcess(child.pi.hProcess, 1); + WaitForSingleObject(child.pi.hProcess, 10000); + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.pi = {}; + } + child.pipe.Close(); + child.ready = false; + child.uid_ok = false; +} + +double QpcToMs(std::uint64_t delta_ticks) { + LARGE_INTEGER freq{}; + QueryPerformanceFrequency(&freq); + return (1000.0 * static_cast(delta_ticks)) / + static_cast(freq.QuadPart); +} + +std::int64_t QpcNow() { + LARGE_INTEGER v{}; + QueryPerformanceCounter(&v); + return v.QuadPart; +} + +double Percentile(std::vector values, double p) { + if (values.empty()) { + return 0; + } + std::sort(values.begin(), values.end()); + auto const idx = static_cast( + std::ceil(p * static_cast(values.size() - 1))); + return values[std::min(idx, values.size() - 1)]; +} + +std::string ClassifySample(SampleRec const& rec, std::int64_t dest, + std::vector const& pings) { + if (rec.recv_count > 1) { + return "DUPLICATE"; + } + if (rec.schedule_server != 0 && rec.actual_server != 0 && + rec.schedule_server != rec.actual_server) { + return "ROUTE_CHANGED"; + } + if (rec.recv_count <= 0) { + return "LOST"; + } + BobPingEvent const* covering = nullptr; + for (auto const& p : pings) { + if (p.kind != static_cast(PingTraceKind::kRequestSent)) { + continue; + } + if (dest != 0 && p.server_id != dest && p.server_id != rec.actual_server) { + continue; + } + if (p.event_steady_us > rec.recv_steady_us) { + continue; + } + auto const end = p.event_steady_us + p.effective_window_us; + if (rec.recv_steady_us <= end) { + covering = &p; + } + } + if (covering == nullptr) { + return "OUTSIDE_WINDOW_UNEXPECTED_DELIVERY"; + } + auto const tn = covering->cycle_anchor_us; + auto const w = covering->base_window_us; + if (covering->physical_attempt_index >= 2) { + return "WAITED_FOR_RETRY_WINDOW"; + } + if (tn > 0 && rec.recv_steady_us < tn && covering->early_by_us > 0 && + covering->effective_window_us > covering->base_window_us) { + return "EARLY_EXTENDED_WINDOW"; + } + if (tn > 0 && w > 0 && rec.recv_steady_us >= tn && + rec.recv_steady_us <= tn + w) { + return "BASE_WINDOW"; + } + if (tn > 0 && rec.recv_steady_us > tn + w) { + return "WAITED_FOR_NEXT_LOGICAL_WINDOW"; + } + return "BASE_WINDOW"; +} + +void WriteMdTable(std::ostream& out, std::vector const& header, + std::vector> const& rows) { + out << "|"; + for (auto const& h : header) { + out << " " << h << " |"; + } + out << "\n|"; + for (std::size_t i = 0; i < header.size(); ++i) { + out << " --- |"; + } + out << "\n"; + for (auto const& row : rows) { + out << "|"; + for (std::size_t i = 0; i < header.size(); ++i) { + out << " " << (i < row.size() ? row[i] : "") << " |"; + } + out << "\n"; + } +} + +std::string F3(double v) { + std::ostringstream os; + os << std::fixed << std::setprecision(3) << v; + return os.str(); +} + +std::string I64(std::int64_t v) { return std::to_string(v); } + +} // namespace + +int RunCharacterization(CharacterizationArgs const& in_args) { + CharacterizationArgs args = in_args; + if (args.run_id.empty()) { + args.run_id = MakeRunId(); + } + if (args.exe_path.empty()) { + args.exe_path = DefaultExePath(); + } + if (args.artifact_dir.empty()) { + args.artifact_dir = "artifacts/uap-1s-characterization/" + args.run_id; + } + if (args.seed == 0) { + args.seed = 1; + } + if (args.quick) { + args.skip_long_characterization = true; + args.window_samples_main = 0; + args.window_samples_extra = 0; + } + + std::filesystem::create_directories(args.artifact_dir); + auto const state_root = + std::filesystem::path{args.artifact_dir} / "persistent-state"; + auto const state_a = (state_root / "state-a").string(); + auto const state_b = (state_root / "state-b").string(); + std::filesystem::create_directories(state_a); + std::filesystem::create_directories(state_b); + + ChildProc alice; + alice.side = IpcSide::kA; + ChildProc bob; + bob.side = IpcSide::kB; + + auto pipe_a = PipeNameFor(args.run_id, IpcSide::kA); + auto pipe_b = PipeNameFor(args.run_id, IpcSide::kB); + + std::cout << "Spawning Alice/Bob run_id=" << args.run_id + << " interval_ms=" << args.ping_interval_ms + << " window_ms=" << args.receive_window_ms + << " seed=" << args.seed + << (args.quick ? " quick=1" : "") << std::endl; + std::cout << std::unitbuf; + auto const log_a = + (std::filesystem::path{args.artifact_dir} / "alice.log").string(); + auto const log_b = + (std::filesystem::path{args.artifact_dir} / "bob.log").string(); + if (!SpawnChild(alice, args, state_a, pipe_a, "uap-1s-alice", log_a, + args.ping_interval_ms, args.receive_window_ms) || + !SpawnChild(bob, args, state_b, pipe_b, "uap-1s-bob", log_b, + args.ping_interval_ms, args.receive_window_ms)) { + return 2; + } + + auto drain = [&](DWORD slice_ms) { + std::optional last; + if (auto f = bob.pipe.TryReadFrame(slice_ms)) { + HandleChildFrame(bob, *f); + last = f; + } + for (;;) { + bool got = false; + if (auto f = bob.pipe.TryReadFrame(0)) { + HandleChildFrame(bob, *f); + last = f; + got = true; + } + if (auto f = alice.pipe.TryReadFrame(0)) { + HandleChildFrame(alice, *f); + last = f; + got = true; + } + if (!got) { + break; + } + } + return last; + }; + + auto wait_ready = [&](ChildProc& c, char const* name) { + auto const deadline = GetTickCount64() + 180000; + while (GetTickCount64() < deadline && !(c.ready && c.uid_ok)) { + drain(200); + } + if (!(c.ready && c.uid_ok)) { + std::cerr << name << " not ready" << std::endl; + return false; + } + return true; + }; + if (!wait_ready(alice, "Alice") || !wait_ready(bob, "Bob")) { + StopChild(alice); + StopChild(bob); + return 3; + } + + auto exchange_uids = [&]() { + std::int64_t a_lo = 0; + std::int64_t a_hi = 0; + std::int64_t b_lo = 0; + std::int64_t b_hi = 0; + std::memcpy(&a_lo, &alice.uid_lo, 8); + std::memcpy(&a_hi, &alice.uid_hi, 8); + std::memcpy(&b_lo, &bob.uid_lo, 8); + std::memcpy(&b_hi, &bob.uid_hi, 8); + SendCmd(alice, IpcType::kSetPeerUid, 0, 0, b_lo, b_hi); + SendCmd(bob, IpcType::kSetPeerUid, 0, 0, a_lo, a_hi); + for (int i = 0; i < 20; ++i) { + drain(100); + } + }; + exchange_uids(); + + auto wait_warmup = [&](ChildProc& c, bool is_alice, std::int64_t* n, + std::int64_t* min_rtt, std::int64_t* p99, + std::uint32_t* guard, std::int64_t* server, + std::int64_t* proto) { + c.warmup_done = false; + SendCmd(c, IpcType::kWaitWarmup); + auto const deadline = GetTickCount64() + 300000; + while (GetTickCount64() < deadline) { + drain(500); + if (c.warmup_done) { + *n = c.warmup_n; + *min_rtt = c.warmup_min; + *p99 = c.warmup_p99; + if (is_alice) { + if (server != nullptr) { + *server = c.warmup_d; + } + if (proto != nullptr) { + *proto = c.warmup_e; + } + } else if (guard != nullptr) { + *guard = c.warmup_guard; + } + return true; + } + } + return false; + }; + + std::int64_t warmup_n = 0; + std::int64_t warmup_min = 0; + std::int64_t warmup_p99 = 0; + std::uint32_t warmup_guard = 0; + std::int64_t alice_n = 0; + std::int64_t alice_min = 0; + std::int64_t alice_p99 = 0; + std::int64_t dest = 0; + std::int64_t dest_proto = 0; + std::cout << "Waiting Bob/Alice warm-up..." << std::endl; + if (!wait_warmup(bob, false, &warmup_n, &warmup_min, &warmup_p99, + &warmup_guard, nullptr, nullptr) || + !wait_warmup(alice, true, &alice_n, &alice_min, &alice_p99, nullptr, + &dest, &dest_proto)) { + std::cerr << "warm-up timed out\n"; + StopChild(alice); + StopChild(bob); + return 4; + } + + std::cout << "## Bob ping statistics\nsamples=" << warmup_n + << " min_rtt_ms=" << warmup_min << " p99_rtt_ms=" << warmup_p99 + << " guard_ms=" << warmup_guard << " dest_server=" << dest + << std::endl; + auto const proto = static_cast(dest_proto); + if (!IsMeasuredProtocolOk(alice.own_proof.protocol) || + !IsMeasuredProtocolOk(bob.own_proof.protocol) || + !IsMeasuredProtocolOk(proto)) { + std::cerr << "FAIL: measured work path protocol mismatch\n"; + StopChild(alice); + StopChild(bob); + return 6; + } + +#if !AE_ENABLE_PING_TEST_FAULTS + std::cerr << "AE_ENABLE_PING_TEST_FAULTS is required\n"; + StopChild(alice); + StopChild(bob); + return 2; +#else + bool ok = true; + std::size_t ping_cursor = 0; + auto wait_ping = [&](std::uint8_t kind, DWORD timeout_ms) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(50); + while (ping_cursor < bob.ping_events.size()) { + auto const& e = bob.ping_events[ping_cursor++]; + if (e.kind == kind && (dest == 0 || e.server_id == dest)) { + return e; + } + } + } + return std::nullopt; + }; + + auto window_open = [&]() { + std::int64_t last_sent = -1; + std::int64_t last_closed = -1; + for (auto const& e : bob.ping_events) { + if (e.server_id != dest) { + continue; + } + if (e.kind == static_cast(PingTraceKind::kRequestSent) || + e.kind == static_cast(PingTraceKind::kRequestDropped)) { + last_sent = e.event_steady_us; + } + if (e.kind == static_cast(PingTraceKind::kRxClosed)) { + last_closed = e.event_steady_us; + } + } + return last_sent > last_closed; + }; + auto wait_window_closed = [&](DWORD timeout_ms) { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(50); + if (!window_open()) { + return true; + } + } + return !window_open(); + }; + + auto arm_next = [&](std::int64_t mode, std::int64_t attempt, std::int64_t e, + std::int64_t timeout_us) { + drain(50); + SendRaw(bob, kIpcArmFault, 0, dest, attempt, mode, timeout_us, e, 0); + drain(150); + }; + + if (args.phase_preservation) { +#include "phase_preservation.inc.cpp" + } + + if (args.first_request_loss_p99) { +#include "first_request_loss_p99.inc.cpp" + } + + if (args.retry_count_zero_runtime) { +#include "retry_count_zero_runtime.inc.cpp" + } + + std::uint32_t rng = args.seed; + auto rnd = [&]() { + rng = rng * 1664525u + 1013904223u; + return rng; + }; + int const n_nominal = args.logical_cycles < 0 ? 0 : args.logical_cycles; + int n_drop = 0; + int n_ignore = 0; + bool const explicit_split = + args.request_loss_cases >= 0 || args.response_loss_cases >= 0; + if (explicit_split) { + n_drop = args.request_loss_cases > 0 ? args.request_loss_cases : 0; + n_ignore = args.response_loss_cases > 0 ? args.response_loss_cases : 0; + } else if (args.loss_cases > 0) { + n_drop = args.loss_cases; + n_ignore = args.loss_cases; + } else if (!args.quick && !args.skip_long_characterization) { + n_drop = std::max(1, n_nominal / 10); + n_ignore = n_drop; + if (n_nominal >= 20) { + n_drop = 10; + n_ignore = 10; + } + } + int const n_cycles = + explicit_split ? (n_nominal + n_drop + n_ignore) : n_nominal; + std::vector fault_plan(static_cast(n_cycles), 0); + if (explicit_split) { + for (int i = 0; i < n_drop && i < n_cycles; ++i) { + fault_plan[static_cast(i)] = 1; + } + for (int i = 0; i < n_ignore && n_drop + i < n_cycles; ++i) { + fault_plan[static_cast(n_drop + i)] = 2; + } + } else { + for (int i = 0; i < n_drop && i < n_cycles; ++i) { + fault_plan[static_cast(i)] = 1; + } + for (int i = 0; i < n_ignore && n_drop + i < n_cycles; ++i) { + fault_plan[static_cast(n_drop + i)] = 2; + } + } + for (int i = n_cycles - 1; i > 0; --i) { + auto j = static_cast(rnd() % static_cast(i + 1)); + std::swap(fault_plan[static_cast(i)], + fault_plan[static_cast(j)]); + } + + std::cout << "COUNTS nominal=" << n_nominal << " request_loss=" << n_drop + << " response_loss=" << n_ignore + << " hard_stop=" << args.hard_stop_runs + << " graceful=" << args.graceful_runs << std::endl; + + std::vector cycles; + cycles.reserve(static_cast(n_cycles)); + std::int64_t first_tn = 0; + std::int64_t first_tn1 = 0; + std::int64_t prev_tn_us = 0; + double phase_drift_max = 0; + std::vector phase_drifts; + int invalid_metric_count = 0; + int retries_before = 0; + int retries_after = 0; + int live_false_missed = 0; + int live_false_unknown = 0; + int query_failures = 0; + int route_changes = 0; + + std::cout << "Running " << n_cycles << " logical cycles..." << std::endl; + if (n_cycles > 0) { + wait_window_closed(8000); + } + + for (int ci = 0; ci < n_cycles; ++ci) { + CycleRec rec{}; + rec.planned_fault = fault_plan[static_cast(ci)]; + wait_window_closed(4000); + if (rec.planned_fault != 0) { + arm_next(rec.planned_fault, 1, 0, 0); + } else { + SendRaw(bob, kIpcArmFault, 0, dest, 1, 0, 0, 0, 0); + drain(50); + } + + auto started = wait_ping( + static_cast(PingTraceKind::kCycleStarted), 8000); + if (!started) { + started = wait_ping( + static_cast(PingTraceKind::kRequestSent), 8000); + } + if (!started) { + started = wait_ping( + static_cast(PingTraceKind::kRequestDropped), 8000); + } + if (!started) { + std::cerr << "FAIL cycle " << ci << ": no start\n"; + ok = false; + break; + } + rec.cycle_id = started->logical_cycle_id; + rec.tn_us = started->cycle_anchor_us; + rec.tn1_us = started->contract_deadline_us; + rec.first_attempt_us = started->actual_us != 0 ? started->actual_us + : started->event_steady_us; + rec.first_wire_next = started->wire_next_connect_ms; + rec.early_by_us = started->early_by_us; + rec.guard_us = started->guard_us; + rec.retry_reserve_us = started->retry_reserve_us; + rec.loss_timeout_us = started->loss_timeout_us; + rec.attempt_lead_us = started->attempt_lead_us; + rec.predeadline = started->predeadline_retry_guaranteed != 0; + + std::optional timeout_ev; + if (rec.planned_fault != 0) { + timeout_ev = wait_ping( + static_cast(PingTraceKind::kAttemptTimeout), 4000); + } + if (timeout_ev) { + rec.timeout_us = timeout_ev->event_steady_us; + rec.timeout_qpc = timeout_ev->event_qpc; + auto retry = wait_ping( + static_cast(PingTraceKind::kRequestSent), 4000); + if (!retry) { + retry = wait_ping( + static_cast(PingTraceKind::kRequestDropped), 1000); + } + if (retry && retry->physical_attempt_index >= 2) { + rec.retry_attempt_us = retry->event_steady_us; + rec.retry_qpc = retry->event_qpc; + rec.retry_wire_next = retry->wire_next_connect_ms; + rec.retry_before_nominal = + retry->wire_next_connect_ms > args.ping_interval_ms; + if (rec.retry_before_nominal) { + ++retries_before; + } else { + ++retries_after; + } + } + } + + auto confirmed = wait_ping( + static_cast(PingTraceKind::kCycleConfirmed), 8000); + rec.confirmed = confirmed.has_value(); + if (!rec.confirmed) { + std::cerr << "WARN cycle " << ci << " not confirmed\n"; + } + if (ci == 0) { + first_tn = rec.tn_us; + first_tn1 = rec.tn1_us; + prev_tn_us = rec.tn_us; + (void)first_tn; + (void)first_tn1; + } else if (rec.tn_us > 0 && prev_tn_us > 0) { + auto const delta_us = rec.tn_us - prev_tn_us; + auto const drift_ms = + static_cast(delta_us - args.ping_interval_ms * 1000) / + 1000.0; + if (std::abs(drift_ms) < 2000.0) { + phase_drifts.push_back(std::abs(drift_ms)); + phase_drift_max = std::max(phase_drift_max, std::abs(drift_ms)); + } else { + ++invalid_metric_count; + } + prev_tn_us = rec.tn_us; + } + cycles.push_back(rec); + + if (ci % 5 == 0) { + std::size_t const before = alice.schedules.size(); + SendRaw(alice, kIpcQueryNow); + auto const q_deadline = GetTickCount64() + 2000; + while (GetTickCount64() < q_deadline && + alice.schedules.size() == before) { + drain(20); + } + if (alice.schedules.size() == before) { + ++query_failures; + } else { + auto const& s = alice.schedules.back(); + if (s.state < 0) { + ++query_failures; + } else if (s.state == 1) { + ++live_false_missed; + } else if (s.state == 2) { + ++live_false_unknown; + } + } + } + } + + auto collect_budget_from_cycles = [&](CycleRec& into) { + for (auto const& c : cycles) { + if (c.attempt_lead_us > 0 && c.early_by_us > 0) { + into = c; + return; + } + } + for (auto const& c : cycles) { + if (c.guard_us > 0 || c.attempt_lead_us > 0) { + into = c; + return; + } + } + }; + CycleRec budget_row{}; + collect_budget_from_cycles(budget_row); + + struct OffsetSpec { + char const* name; + std::int64_t offset_ms; + }; + auto make_offsets = [&](std::int64_t window_ms) { + std::int64_t lead_ms = budget_row.attempt_lead_us / 1000; + if (lead_ms <= 0) { + lead_ms = static_cast(warmup_guard) + 80; + } + return std::vector{ + {"EARLY_EXTENDED", -(lead_ms / 2)}, + {"EARLY_IN_WINDOW", 25}, + {"NEAR_WINDOW_END", window_ms - 20}, + {"JUST_OUTSIDE", window_ms + 20}, + {"BEFORE_NEXT_PING", args.ping_interval_ms - 50}, + }; + }; + + std::vector samples; + std::uint32_t next_seq = 1000; + auto run_window_samples = [&](std::int64_t window_ms, int per_offset) { + auto const offsets = make_offsets(window_ms); + for (auto const& off : offsets) { + int got = 0; + int attempts = 0; + std::cout << "Window " << window_ms << " offset " << off.name + << " ms=" << off.offset_ms << std::endl; + while (got < per_offset && attempts < per_offset * 6) { + ++attempts; + wait_window_closed(3000); + auto const seq = next_seq++; + SendCmd(alice, IpcType::kRunSample, seq, 0, off.offset_ms); + auto const deadline = GetTickCount64() + 20000; + bool sent = false; + SampleRec rec{}; + rec.offset_name = off.name; + rec.offset_ms = off.offset_ms; + rec.window_ms = window_ms; + rec.sequence = seq; + while (GetTickCount64() < deadline && !sent) { + drain(50); + auto it = alice.sent_samples.find(seq); + if (it != alice.sent_samples.end()) { + rec = it->second; + rec.offset_name = off.name; + rec.offset_ms = off.offset_ms; + rec.window_ms = window_ms; + sent = true; + } + } + if (!sent) { + continue; + } + auto const recv_deadline = GetTickCount64() + 8000; + while (GetTickCount64() < recv_deadline) { + drain(50); + auto rit = bob.recv_counts.find(seq); + if (rit != bob.recv_counts.end()) { + rec.recv_count = rit->second; + rec.recv_qpc = bob.recv_qpc[seq]; + rec.recv_steady_us = bob.recv_steady[seq]; + break; + } + } + rec.classification = ClassifySample(rec, dest, bob.ping_events); + if (rec.recv_qpc > 0 && rec.send_qpc > 0 && + rec.recv_qpc >= rec.send_qpc) { + rec.delivery_ms = + QpcToMs(static_cast(rec.recv_qpc - rec.send_qpc)); + } + std::int64_t bob_wire = 0; + std::int64_t bob_rx = 0; + std::int64_t bob_tn1 = 0; + std::int64_t bob_tn = 0; + for (auto const& p : bob.ping_events) { + if (p.kind == static_cast(PingTraceKind::kRequestSent) && + p.server_id == dest && p.cycle_anchor_us > 0) { + bob_wire = p.wire_next_connect_ms; + bob_rx = p.effective_window_us / 1000; + bob_tn1 = p.contract_deadline_us; + bob_tn = p.cycle_anchor_us; + } + } + rec.deadline_error_ms = + static_cast(rec.raw_delta_ms - bob_wire); + auto const est_end_ms = rec.raw_delta_ms + bob_rx; + auto const nominal_end_from_query = + rec.raw_delta_ms - + (bob_tn1 > bob_tn ? (bob_tn1 - bob_tn) / 1000 : args.ping_interval_ms) + + window_ms; + rec.window_end_slack_ms = + static_cast(est_end_ms - (rec.raw_delta_ms - + (args.ping_interval_ms - window_ms))); + (void)nominal_end_from_query; + rec.premature = rec.classification == "OUTSIDE_WINDOW_UNEXPECTED_DELIVERY"; + if (rec.schedule_server != rec.actual_server) { + ++route_changes; + } + if (rec.classification != "LOST") { + ++got; + } + samples.push_back(rec); + std::cout << " seq=" << seq << " class=" << rec.classification + << " recv=" << rec.recv_count << " got=" << got << "/" + << per_offset << std::endl; + } + std::cout << " done " << off.name << " collected=" << got + << " attempts=" << attempts << std::endl; + } + }; + + if (args.window_samples_main > 0) { + std::cout << "Window samples window=" << args.receive_window_ms << std::endl; + run_window_samples(args.receive_window_ms, args.window_samples_main); + } + + auto run_targeted = [&](int mode1, int mode2, char const* name) { + std::cout << "Targeted " << name << std::endl; + wait_window_closed(4000); + arm_next(mode1, 1, 0, 0); + if (mode2 != 0) { + arm_next(mode2, 2, 1, 0); + } + auto started = wait_ping( + static_cast(PingTraceKind::kCycleStarted), 8000); + (void)started; + (void)wait_ping(static_cast(PingTraceKind::kCycleConfirmed), + 12000); + }; + if (!args.quick && !args.skip_long_characterization) { + run_targeted(1, 1, "double_drop"); + run_targeted(2, 2, "double_ignore"); + run_targeted(1, 2, "drop_then_ignore"); + } + + auto restart_pair = [&](std::int64_t window_ms, int suffix) -> bool { + if (alice.pi.hProcess != nullptr) { + StopChild(alice); + } + if (bob.pi.hProcess != nullptr) { + StopChild(bob); + } + Sleep(1000); + ResetChildRuntime(alice); + ResetChildRuntime(bob); + alice.side = IpcSide::kA; + bob.side = IpcSide::kB; + auto const pipe_a2 = + PipeNameFor(args.run_id, IpcSide::kA, "a" + std::to_string(suffix)); + auto const pipe_b2 = + PipeNameFor(args.run_id, IpcSide::kB, "b" + std::to_string(suffix)); + auto const log_a = + (std::filesystem::path{args.artifact_dir} / + ("alice-" + std::to_string(suffix) + ".log")) + .string(); + auto const log_b = + (std::filesystem::path{args.artifact_dir} / + ("bob-" + std::to_string(suffix) + ".log")) + .string(); + auto const state_a2 = + (state_root / ("state-a-" + std::to_string(suffix))).string(); + auto const state_b2 = + (state_root / ("state-b-" + std::to_string(suffix))).string(); + std::filesystem::create_directories(state_a2); + std::filesystem::create_directories(state_b2); + if (!SpawnChild(alice, args, state_a2, pipe_a2, "uap-1s-alice", log_a, + args.ping_interval_ms, window_ms) || + !SpawnChild(bob, args, state_b2, pipe_b2, "uap-1s-bob", log_b, + args.ping_interval_ms, window_ms)) { + return false; + } + ping_cursor = 0; + if (!wait_ready(alice, "Alice-restart") || + !wait_ready(bob, "Bob-restart")) { + return false; + } + exchange_uids(); + if (args.quick) { + auto const confirmed = wait_ping( + static_cast(PingTraceKind::kCycleConfirmed), 20000); + if (!confirmed) { + std::cerr << "FAIL no confirmed ping after pair restart\n"; + return false; + } + std::cout << "Pair restarted (quick) window_ms=" << window_ms + << " dest=" << dest << std::endl; + return true; + } + std::int64_t n = 0; + std::int64_t mn = 0; + std::int64_t p99 = 0; + std::uint32_t g = 0; + if (!wait_warmup(bob, false, &n, &mn, &p99, &g, nullptr, nullptr)) { + return false; + } + std::int64_t alice_n2 = 0; + std::int64_t alice_min2 = 0; + std::int64_t alice_p992 = 0; + std::int64_t alice_srv = 0; + std::int64_t alice_proto = 0; + if (!wait_warmup(alice, true, &alice_n2, &alice_min2, &alice_p992, nullptr, + &alice_srv, &alice_proto)) { + std::cerr << "FAIL Alice warmup after pair restart\n"; + return false; + } + if (alice_srv != 0) { + dest = alice_srv; + } + warmup_guard = g; + std::cout << "Pair restarted window_ms=" << window_ms << " dest=" << dest + << " alice_samples=" << alice_n2 << std::endl; + return true; + }; + + if (args.window_samples_extra > 0) { + for (std::int64_t extra_w : {std::int64_t{100}, std::int64_t{500}}) { + std::cout << "Extra window " << extra_w << " ms" << std::endl; + if (!restart_pair(extra_w, static_cast(extra_w))) { + std::cerr << "FAIL restart pair for window " << extra_w << "\n"; + ok = false; + break; + } + run_window_samples(extra_w, args.window_samples_extra); + } + if (!restart_pair(args.receive_window_ms, 250)) { + std::cerr << "FAIL restore 250ms pair\n"; + ok = false; + } + } + + std::vector offline; + auto poll_until_state = [&](std::int64_t want, DWORD timeout_ms, int* qcount, + std::int64_t* got_state, std::int64_t* got_qpc) { + *qcount = 0; + *got_state = -2; + *got_qpc = 0; + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + std::size_t const before = alice.schedules.size(); + SendRaw(alice, kIpcQueryNow); + ++*qcount; + auto const wait_deadline = GetTickCount64() + 2000; + while (GetTickCount64() < wait_deadline && + alice.schedules.size() == before) { + drain(50); + } + if (alice.schedules.size() > before) { + auto const& s = alice.schedules.back(); + *got_state = s.state; + *got_qpc = s.qpc; + if (s.state == want) { + return true; + } + } + } + return false; + }; + + std::cout << "Hard-stop runs=" << args.hard_stop_runs << std::endl; + for (int r = 0; r < args.hard_stop_runs; ++r) { + int stable = 0; + int const need_stable = args.quick ? 2 : 10; + while (stable < need_stable) { + if (wait_ping(static_cast(PingTraceKind::kCycleConfirmed), + 8000)) { + ++stable; + } else { + break; + } + } + std::size_t const before = alice.schedules.size(); + SendRaw(alice, kIpcQueryNow); + auto const q_deadline = GetTickCount64() + 2000; + while (GetTickCount64() < q_deadline && alice.schedules.size() == before) { + drain(20); + } + ScheduleSnap last{}; + if (!alice.schedules.empty()) { + last = alice.schedules.back(); + if (last.state == 1) { + ++live_false_missed; + } + if (last.state == 2) { + ++live_false_unknown; + } + } + double remaining_ms = 0; + if (last.next_us > last.steady_us) { + remaining_ms = + static_cast(last.next_us - last.steady_us) / 1000.0; + } + DWORD poll_ms = args.quick ? 20000 : 45000; + if (remaining_ms > 0 && remaining_ms < 60000) { + auto const need = + static_cast(remaining_ms + (args.quick ? 8000 : 15000)); + if (need > poll_ms) { + poll_ms = need; + } + } + if (args.quick && poll_ms > 25000) { + poll_ms = 25000; + } + auto const kill_qpc = QpcNow(); + KillChild(bob); + int qcount = 0; + std::int64_t got_state = -2; + std::int64_t got_qpc = 0; + bool const hit = + poll_until_state(1, poll_ms, &qcount, &got_state, &got_qpc); + std::cout << "hard-stop " << r << " pre_state=" << last.state + << " remaining_ms=" << remaining_ms << " poll_ms=" << poll_ms + << " state=" << got_state << " hit=" << hit + << " queries=" << qcount << std::endl; + OfflineRec o{}; + o.condition = "hard_stop"; + o.query_count = qcount; + o.final_state = got_state; + auto const deadline_qpc_est = + last.qpc + static_cast(remaining_ms * + (static_cast(QpcNow() - QpcNow() + 1))); + (void)deadline_qpc_est; + LARGE_INTEGER freq{}; + QueryPerformanceFrequency(&freq); + auto const deadline_qpc = + last.qpc + + static_cast(remaining_ms * freq.QuadPart / 1000.0); + if (hit && got_qpc > deadline_qpc) { + o.deadline_to_state_ms = + QpcToMs(static_cast(got_qpc - deadline_qpc)); + } else if (hit && got_qpc > kill_qpc) { + o.deadline_to_state_ms = + QpcToMs(static_cast(got_qpc - kill_qpc)); + } + if (got_qpc > kill_qpc) { + o.start_to_state_ms = + QpcToMs(static_cast(got_qpc - kill_qpc)); + } + o.false_state = !hit; + offline.push_back(o); + if (!restart_pair(args.receive_window_ms, 1000 + r)) { + std::cerr << "FAIL respawn after hard-stop " << r << "\n"; + ok = false; + break; + } + Sleep(args.quick ? 1500 : 6000); + } + + std::cout << "Graceful-close runs=" << args.graceful_runs << std::endl; + for (int r = 0; r < args.graceful_runs; ++r) { + int variant = 0; + if (!args.quick && !args.skip_long_characterization) { + if (r % 20 < 8) { + variant = 0; + } else if (r % 20 < 14) { + variant = 1; + } else { + variant = 2; + } + } + wait_window_closed(4000); + if (variant == 1) { + arm_next(1, 1, 0, 0); + } else if (variant == 2) { + arm_next(2, 1, 0, 0); + } + auto const start_qpc = QpcNow(); + alice.got_ack = false; + SendRaw(bob, kIpcAnnounceUnknown); + auto const ack_deadline = GetTickCount64() + 10000; + while (GetTickCount64() < ack_deadline && !alice.got_ack) { + drain(50); + if (bob.got_ack) { + break; + } + } + std::int64_t ping0_qpc = 0; + for (auto it = bob.ping_events.rbegin(); it != bob.ping_events.rend(); + ++it) { + if (it->server_id == dest && it->wire_next_connect_ms == 0 && + (it->kind == static_cast(PingTraceKind::kRequestSent) || + it->kind == + static_cast(PingTraceKind::kRequestDropped))) { + ping0_qpc = it->event_qpc; + break; + } + } + int qcount = 0; + std::int64_t got_state = -2; + std::int64_t got_qpc = 0; + bool const hit = poll_until_state(2, args.quick ? 8000 : 20000, &qcount, + &got_state, &got_qpc); + std::cout << "graceful " << r << " var=" << variant + << " state=" << got_state << " hit=" << hit + << " queries=" << qcount << std::endl; + OfflineRec o{}; + if (variant == 0) { + o.condition = "graceful_unknown"; + } else if (variant == 1) { + o.condition = "drop_request"; + } else { + o.condition = "ignore_response"; + } + o.query_count = qcount; + o.final_state = got_state; + if (hit && got_qpc > start_qpc) { + o.start_to_state_ms = + QpcToMs(static_cast(got_qpc - start_qpc)); + } + if (hit && ping0_qpc != 0 && got_qpc > ping0_qpc) { + o.deadline_to_state_ms = + QpcToMs(static_cast(got_qpc - ping0_qpc)); + } + if (got_state == 1) { + o.false_state = true; + } + if (!hit) { + o.false_state = true; + } + offline.push_back(o); + if (r + 1 < args.graceful_runs) { + if (!restart_pair(args.receive_window_ms, 2000 + r)) { + std::cerr << "FAIL respawn after graceful " << r << "\n"; + ok = false; + break; + } + Sleep(args.quick ? 1500 : 6000); + } + } + + auto values_of = [&](std::string const& cond) { + std::vector v; + for (auto const& o : offline) { + if (o.condition == cond && o.deadline_to_state_ms > 0) { + v.push_back(o.deadline_to_state_ms); + } + } + return v; + }; + + int premature = 0; + int duplicates = 0; + int delivered_ok = 0; + int window_n = 0; + for (auto const& s : samples) { + if (s.window_ms != args.receive_window_ms) { + continue; + } + ++window_n; + if (s.premature) { + ++premature; + } + if (s.recv_count > 1) { + ++duplicates; + } + if (s.classification == "EARLY_EXTENDED_WINDOW" || + s.classification == "BASE_WINDOW" || + s.classification == "WAITED_FOR_RETRY_WINDOW" || + s.classification == "WAITED_FOR_NEXT_LOGICAL_WINDOW") { + ++delivered_ok; + } + } + + int drop_ok = 0; + int drop_n = 0; + int ign_ok = 0; + int ign_n = 0; + for (auto const& c : cycles) { + if (c.planned_fault == 1) { + ++drop_n; + if (c.confirmed && c.retry_attempt_us != 0) { + ++drop_ok; + } + } + if (c.planned_fault == 2) { + ++ign_n; + if (c.confirmed) { + ++ign_ok; + } + } + } + + int hard_hit = 0; + int hard_n = 0; + int grace_hit = 0; + int grace_n = 0; + for (auto const& o : offline) { + if (o.condition == "hard_stop") { + ++hard_n; + if (o.final_state == 1) { + ++hard_hit; + } + } + if (o.condition == "graceful_unknown") { + ++grace_n; + if (o.final_state == 2) { + ++grace_hit; + } + } + } + + auto const proto_name = BenchProtocolName(proto); + std::ofstream report(std::filesystem::path{args.artifact_dir} / "report.md"); + report << "# UAP 1s timing characterization\n\n"; + report << "- protocol: " << proto_name << "\n"; + report << "- seed: " << args.seed << "\n"; + report << "- interval_ms: " << args.ping_interval_ms << "\n"; + report << "- base_window_ms: " << args.receive_window_ms << "\n"; + report << "- dest_server: " << dest << "\n"; + report << "- cycles: " << cycles.size() << "\n\n"; + + report << "## Ping timing by transport\n\n"; + auto const guard_ms_report = [&]() -> std::int64_t { + auto const from_budget = budget_row.guard_us / 1000; + if (from_budget <= 10 && warmup_p99 > 50) { + ++invalid_metric_count; + return warmup_guard; + } + if (from_budget > 0) { + return from_budget; + } + return warmup_guard; + }(); + + WriteMdTable(report, + {"protocol", "cycles", "interval", "base_window", "min_rtt", + "p99_rtt", "guard", "loss_timeout", "retry_reserve", + "attempt_lead", "first_attempt_early_by", + "predeadline_retry_guaranteed"}, + {{proto_name, I64(static_cast(cycles.size())), + I64(args.ping_interval_ms), I64(args.receive_window_ms), + I64(warmup_min), I64(warmup_p99), I64(guard_ms_report), + I64(budget_row.loss_timeout_us / 1000), + I64(budget_row.retry_reserve_us / 1000), + I64(budget_row.attempt_lead_us / 1000), + I64(budget_row.early_by_us / 1000), + budget_row.predeadline ? "true" : "false"}}); + + auto deadline_errs = [&](int fault) { + std::vector v; + for (auto const& s : samples) { + if (s.window_ms != args.receive_window_ms) { + continue; + } + (void)fault; + v.push_back(s.deadline_error_ms); + } + return v; + }; + auto de = deadline_errs(0); + report << "\n## Deadline accuracy\n\n"; + WriteMdTable(report, + {"protocol", "fault", "samples", "error_min", "error_p50", + "error_p95", "error_max"}, + {{proto_name, "mixed", I64(static_cast(de.size())), + de.empty() ? "0" : F3(*std::min_element(de.begin(), de.end())), + F3(Percentile(de, 0.50)), F3(Percentile(de, 0.95)), + de.empty() ? "0" + : F3(*std::max_element(de.begin(), de.end()))}}); + + report << "\n## Window accuracy\n\n"; + { + std::vector> rows; + std::map> grouped; + for (auto& s : samples) { + grouped[std::to_string(s.window_ms) + "/" + s.offset_name + "/" + + s.classification] + .push_back(&s); + } + for (auto& [key, vec] : grouped) { + std::vector d; + int prem = 0; + int dup = 0; + for (auto* s : vec) { + if (s->delivery_ms > 0) { + d.push_back(s->delivery_ms); + } + prem += s->premature ? 1 : 0; + dup += s->recv_count > 1 ? 1 : 0; + } + rows.push_back({proto_name, I64(vec.front()->window_ms), + vec.front()->offset_name, + I64(static_cast(vec.size())), + vec.front()->classification, F3(Percentile(d, 0.50)), + F3(Percentile(d, 0.95)), + d.empty() + ? "0" + : F3(*std::max_element(d.begin(), d.end())), + I64(prem), I64(dup)}); + } + WriteMdTable(report, + {"protocol", "window_ms", "offset", "samples", "classification", + "delivery_p50", "delivery_p95", "delivery_max", "premature", + "duplicates"}, + rows); + } + + std::vector retry_delays; + int retry_before_n = 0; + int retry_n = 0; + for (auto const& c : cycles) { + if (c.timeout_us == 0 || c.retry_attempt_us == 0) { + continue; + } + double delay_ms = -1; + if (c.timeout_qpc != 0 && c.retry_qpc != 0 && c.retry_qpc > c.timeout_qpc) { + delay_ms = QpcToMs(static_cast(c.retry_qpc - c.timeout_qpc)); + } else if (c.retry_attempt_us > c.timeout_us) { + delay_ms = static_cast(c.retry_attempt_us - c.timeout_us) / 1000.0; + } + if (delay_ms >= 0 && delay_ms < 5000) { + retry_delays.push_back(delay_ms); + ++retry_n; + if (c.retry_before_nominal) { + ++retry_before_n; + } + } else { + ++invalid_metric_count; + } + } + report << "\n## Retry\n\n"; + WriteMdTable( + report, + {"protocol", "fault", "samples", "timeout_to_retry_p50", + "timeout_to_retry_p95", "retry_before_nominal_percent", "phase_drift_p50", + "phase_drift_max"}, + {{proto_name, "mixed", I64(retry_n), F3(Percentile(retry_delays, 0.50)), + F3(Percentile(retry_delays, 0.95)), + retry_n == 0 + ? "0" + : F3(100.0 * static_cast(retry_before_n) / + static_cast(retry_n)), + F3(Percentile(phase_drifts, 0.50)), F3(phase_drift_max)}}); + + report << "\n## Offline\n\n"; + { + std::vector> rows; + for (auto const* cond : + {"hard_stop", "graceful_unknown", "drop_request", "ignore_response", + "double_drop"}) { + auto v = values_of(cond); + int false_n = 0; + int n = 0; + for (auto const& o : offline) { + if (o.condition == cond) { + ++n; + false_n += o.false_state ? 1 : 0; + } + } + rows.push_back({proto_name, cond, I64(n), F3(Percentile(v, 0.50)), + F3(Percentile(v, 0.95)), + v.empty() ? "0" + : F3(*std::max_element(v.begin(), v.end())), + I64(false_n)}); + } + WriteMdTable(report, + {"protocol", "condition", "samples", "deadline_to_state_p50", + "deadline_to_state_p95", "deadline_to_state_max", + "false_state_count"}, + rows); + } + + report << "\n## Reliability\n\n"; + report << "- live_false_MissedDeadline: " << live_false_missed << "\n"; + report << "- live_false_Unknown: " << live_false_unknown << "\n"; + report << "- missed_deadline_detection_rate: " + << (hard_n == 0 ? 0.0 + : static_cast(hard_hit) / + static_cast(hard_n)) + << " (" << hard_hit << "/" << hard_n << ")\n"; + report << "- graceful_unknown_detection_rate: " + << (grace_n == 0 ? 0.0 + : static_cast(grace_hit) / + static_cast(grace_n)) + << " (" << grace_hit << "/" << grace_n << ")\n"; + report << "- single_request_loss_recovery: " + << (drop_n == 0 ? 0.0 + : static_cast(drop_ok) / + static_cast(drop_n)) + << " (" << drop_ok << "/" << drop_n << ")\n"; + report << "- single_response_loss_recovery: " + << (ign_n == 0 ? 0.0 + : static_cast(ign_ok) / static_cast(ign_n)) + << " (" << ign_ok << "/" << ign_n << ")\n"; + report << "- retries_before_nominal: " << retries_before << "\n"; + report << "- retries_after_nominal: " << retries_after << "\n"; + report << "- correct_window_delivery_rate: " + << (window_n == 0 ? 0.0 + : static_cast(delivered_ok) / + static_cast(window_n)) + << "\n"; + report << "- premature_delivery_count: " << premature << "\n"; + report << "- duplicates: " << duplicates << "\n"; + report << "- p99_rtt_ms: " << warmup_p99 << "\n"; + report << "- guard_ms: " << guard_ms_report << "\n"; + report << "- query_failures: " << query_failures << "\n"; + report << "- route_changes: " << route_changes << "\n"; + if (!alice.schedules.empty()) { + auto const& s = alice.schedules.back(); + report << "- coverage selected/queried/successful/failed/quarantined_skipped: " + << s.selected << "/" << s.queried << "/" << s.successful << "/" + << s.failed << "/" << s.skipped << "\n"; + } + report << "- phase_drift_max_ms: " << F3(phase_drift_max) << "\n"; + report << "- invalid_metric_count: " << invalid_metric_count << "\n"; + report << "- predeadline_retry_guaranteed: " + << (budget_row.predeadline ? "true" : "false") << "\n"; + if (!budget_row.predeadline) { + report << "\n**Blocker:** pre-deadline retry not guaranteed. min_rtt_ms=" + << warmup_min << " p99_rtt_ms=" << warmup_p99 + << " guard_ms=" << guard_ms_report + << " loss_timeout_ms=" << (budget_row.loss_timeout_us / 1000) + << " retry_reserve_ms=" << (budget_row.retry_reserve_us / 1000) + << " attempt_lead_ms=" << (budget_row.attempt_lead_us / 1000) + << " interval_ms=" << args.ping_interval_ms << "\n"; + if (args.transport == "tcp" || args.transport == "TCP") { + report << "KNOWN BLOCKER: TCP pre-deadline retry is limited by " + "RTT/timeout policy; not fixed in this quick loop.\n"; + } + } + + { + std::ofstream csv(std::filesystem::path{args.artifact_dir} / + "bob_ping_trace.csv"); + csv << "kind,server_id,cycle,attempt,wire_next,tn_us,tn1_us,early_by_us," + "guard_us,retry_reserve_us,loss_timeout_us,attempt_lead_us," + "predeadline,event_steady_us,event_qpc\n"; + for (auto const& e : bob.ping_events) { + csv << static_cast(e.kind) << "," << e.server_id << "," + << e.logical_cycle_id << "," << e.physical_attempt_index << "," + << e.wire_next_connect_ms << "," << e.cycle_anchor_us << "," + << e.contract_deadline_us << "," << e.early_by_us << "," << e.guard_us + << "," << e.retry_reserve_us << "," << e.loss_timeout_us << "," + << e.attempt_lead_us << "," << e.predeadline_retry_guaranteed << "," + << e.event_steady_us << "," << e.event_qpc << "\n"; + } + } + { + std::ofstream csv(std::filesystem::path{args.artifact_dir} / "samples.csv"); + csv << "window,offset_name,offset_ms,seq,class,delivery_ms,deadline_error_" + "ms,recv_count,premature\n"; + for (auto const& s : samples) { + csv << s.window_ms << "," << s.offset_name << "," << s.offset_ms << "," + << s.sequence << "," << s.classification << "," << s.delivery_ms << "," + << s.deadline_error_ms << "," << s.recv_count << "," + << (s.premature ? 1 : 0) << "\n"; + } + } + + std::cout << "report=" << (std::filesystem::path{args.artifact_dir} / "report.md") + << std::endl; + std::cout << (ok ? "PASS characterization" : "FAIL characterization") + << std::endl; + + StopChild(alice); + StopChild(bob); + if (duplicates != 0) { + return 7; + } + return ok ? 0 : 7; +#endif +} + +} // namespace ae::test_uap_ping_retry_window diff --git a/examples/aether_uap_1s_timing_characterization/coordinator.h b/examples/aether_uap_1s_timing_characterization/coordinator.h new file mode 100644 index 00000000..8e1e8300 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/coordinator.h @@ -0,0 +1,62 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_1S_TIMING_CHARACTERIZATION_COORDINATOR_H_ +#define AETHER_UAP_1S_TIMING_CHARACTERIZATION_COORDINATOR_H_ + +#include +#include + +namespace ae::test_uap_ping_retry_window { + +struct CharacterizationArgs { + std::string run_id; + std::string artifact_dir; + std::string exe_path; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; + std::string transport{"tcp"}; + std::int64_t ping_interval_ms{1000}; + std::int64_t receive_window_ms{250}; + std::uint32_t seed{1}; + int logical_cycles{100}; + int hard_stop_runs{20}; + int graceful_runs{20}; + int window_samples_main{30}; + int window_samples_extra{10}; + int loss_cases{0}; + int request_loss_cases{-1}; + int response_loss_cases{-1}; + bool quick{false}; + bool skip_long_characterization{false}; + bool phase_preservation{false}; + bool phase_preservation_stress{false}; + // When >0 with --phase-preservation, stop arming new cycles after this + // many wall-clock seconds (shard budget). Fixed fast/stress step lists are + // replaced by a repeating mix sized for long characterization. + int phase_preservation_budget_sec{0}; + // Focused first-request-loss vs p99 timing policy (Alice query only after Tn). + bool first_request_loss_p99{false}; + int first_request_loss_cases{100}; + // ping_retry_count=0 post-deadline recovery acceptance (deterministic drop #1). + bool retry_count_zero_runtime{false}; + int retry_count_zero_cases{10}; +}; + +int RunCharacterization(CharacterizationArgs const& args); + +} // namespace ae::test_uap_ping_retry_window + +#endif // AETHER_UAP_1S_TIMING_CHARACTERIZATION_COORDINATOR_H_ diff --git a/examples/aether_uap_1s_timing_characterization/first_request_loss_p99.inc.cpp b/examples/aether_uap_1s_timing_characterization/first_request_loss_p99.inc.cpp new file mode 100644 index 00000000..5b963b9d --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/first_request_loss_p99.inc.cpp @@ -0,0 +1,1412 @@ + // Focused first-request-loss vs intended 1.5*p99+G policy. + // Alice queries only AFTER original deadline Tn. + // Included inside RunCharacterization after Alice/Bob spawn + warmup. + // + // Production formula (ping_cloud_servers.cpp / ping_schedule_guard.h): + // p99 = response_time_statistics().percentile<99>() + // min = response_time_statistics().min() + // G = Clamp(max(0,(p99-min)/2)+10ms) + // raw = Channel::ResponseTimeout() = same p99 when stats non-empty + // loss_timeout = max(raw, p99+10ms, 50ms) [may cap for interval] + // attempt_lead = G + loss_timeout + p99/2 + scheduler(10) + // + retry_dispatch(60) + // ≈ 1.5*p99 + G + 80ms when uncapped + // Intended invariant under test: + // first_send <= Tn - (1.5*R99 + G) + // retry_send <= Tn - (R99/2 + G) + // retry_send + R99/2 <= Tn - G + + struct FrlCase { + int index{0}; + std::int64_t seed{0}; + int target_cycle_index{0}; + std::int64_t logical_ping_id{0}; + std::int64_t target_logical_ping_id{0}; + std::int64_t phase_anchor_us{0}; + std::int64_t tn_us{0}; + std::int64_t rtt_sample_hint{0}; + double rtt_p99_used_ms{std::numeric_limits::quiet_NaN()}; + double guard_used_ms{std::numeric_limits::quiet_NaN()}; + double loss_timeout_ms{std::numeric_limits::quiet_NaN()}; + double attempt_lead_ms{std::numeric_limits::quiet_NaN()}; + double expected_retry_budget_ms{std::numeric_limits::quiet_NaN()}; + double expected_first_send_us{ + std::numeric_limits::quiet_NaN()}; + std::int64_t actual_first_send_us{ + std::numeric_limits::min()}; + double first_send_margin_ms{std::numeric_limits::quiet_NaN()}; + int first_request_dropped{-1}; + std::int64_t fault_arm_time{0}; + int fault_state_before_send{-1}; + std::int64_t first_request_attempt_number{0}; + std::int64_t first_request_send_time{0}; + std::int64_t fault_match_time{0}; + std::int64_t fault_drop_time{0}; + int fault_consumed{0}; + std::int64_t retry_decision_us{std::numeric_limits::min()}; + std::int64_t actual_retry_send_us{ + std::numeric_limits::min()}; + std::int64_t retry_logical_ping_id{0}; + std::int64_t retry_attempt_number{0}; + std::int64_t retry_send_time{0}; + int fault_state_end{-1}; + double expected_latest_retry_send_us{ + std::numeric_limits::quiet_NaN()}; + double retry_send_margin_ms{std::numeric_limits::quiet_NaN()}; + double estimated_retry_server_arrival_us{ + std::numeric_limits::quiet_NaN()}; + double estimated_retry_server_margin_ms{ + std::numeric_limits::quiet_NaN()}; + std::int64_t next_nominal_us{0}; + double next_nominal_phase_error_ms{ + std::numeric_limits::quiet_NaN()}; + std::int64_t next2_us{0}; + std::int64_t next3_us{0}; + double next2_phase_error_ms{std::numeric_limits::quiet_NaN()}; + double next3_phase_error_ms{std::numeric_limits::quiet_NaN()}; + std::int64_t alice_query_us{std::numeric_limits::min()}; + std::int64_t alice_last_request_us{ + std::numeric_limits::min()}; + std::int64_t alice_deadline_us{std::numeric_limits::min()}; + std::int64_t alice_next_deadline_us{ + std::numeric_limits::min()}; + std::int64_t alice_state{-2}; + std::int64_t alice_next_ping_delta_ms{ + std::numeric_limits::min()}; + std::int64_t alice_last_connect_delta_ms{ + std::numeric_limits::min()}; + bool alice_missed_deadline{false}; + bool harness_sync_ok{false}; + std::vector failures; + }; + + auto const kMissing = std::numeric_limits::min(); + auto abs_d = [](double v) { return v < 0 ? -v : v; }; + auto csv_d = [&](double v) -> std::string { + if (!std::isfinite(v)) { + return {}; + } + return F3(v); + }; + auto csv_i = [&](std::int64_t v) -> std::string { + if (v == kMissing) { + return {}; + } + return std::to_string(v); + }; + + // Production uses response_stats.percentile<99>(). After warmup, p99 must + // not still be the empty-stats estimate alone unless samples exist. + if (warmup_n <= 0) { + std::cerr << "FAIL FIRST_REQUEST_LOSS_P99: production p99 statistic has " + "no warm-up samples (rtt_sample_count=0)\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + + std::int64_t const period_us = args.ping_interval_ms * 1000; + // Existing scheduler tick / characterization resolution (1 ms). + double const tick_ms = 1.0; + int const n_cases = args.first_request_loss_cases > 0 + ? args.first_request_loss_cases + : 100; + + std::filesystem::create_directories(args.artifact_dir); + std::ofstream samples_csv(std::filesystem::path{args.artifact_dir} / + "samples.csv"); + std::ofstream samples_jsonl(std::filesystem::path{args.artifact_dir} / + "samples.jsonl"); + std::ofstream failed_json(std::filesystem::path{args.artifact_dir} / + "failed-cases.json"); + std::ofstream report(std::filesystem::path{args.artifact_dir} / + "report.md"); + std::ofstream summary_json(std::filesystem::path{args.artifact_dir} / + "summary.json"); + std::ofstream fault_trace_csv(std::filesystem::path{args.artifact_dir} / + "fault-trace.csv"); + if (!samples_csv || !samples_jsonl || !failed_json || !report || + !summary_json || !fault_trace_csv) { + std::cerr << "FAIL cannot open first-request-loss-p99 outputs\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + double const dispatch_margin_ms = 60.0; + fault_trace_csv + << "case_index,transport,seed,target_logical_ping_id," + "target_cycle_index,fault_arm_time,fault_state_before_send," + "first_request_logical_ping_id,first_request_attempt_number," + "first_request_send_time,fault_match_time,fault_drop_time," + "fault_consumed,retry_logical_ping_id,retry_attempt_number," + "retry_send_time,fault_state_end\n"; + samples_csv + << "transport,seed,case_index,logical_ping_id,rtt_sample_count," + "rtt_p99_used_ms,guard_used_ms,dispatch_margin_ms,loss_timeout_ms," + "attempt_lead_ms," + "Tn_us,expected_retry_budget_ms,expected_first_send_us," + "actual_first_send_us,first_send_margin_ms,first_request_dropped," + "retry_decision_us,expected_latest_retry_send_us," + "actual_retry_send_us,retry_send_margin_ms," + "estimated_retry_server_arrival_us," + "estimated_retry_server_margin_to_Tn_ms," + "estimated_retry_server_margin_to_Tn_minus_guard_ms," + "next_nominal_us,next_nominal_phase_error_ms,next2_us," + "next2_phase_error_ms,next3_us,next3_phase_error_ms," + "alice_query_us,alice_last_request_us,alice_deadline_us," + "alice_next_deadline_us,alice_state,alice_missed_deadline," + "alice_next_ping_delta_ms,alice_last_connect_delta_ms," + "failure_classes,harness_invalid\n"; + + using EvId = std::tuple; + std::set seen_ev; + auto ev_id = [](BobPingEvent const& e) { + return EvId{e.event_qpc, e.kind, e.logical_cycle_id, + e.physical_attempt_index, e.server_id}; + }; + for (auto const& e : bob.ping_events) { + seen_ev.insert(ev_id(e)); + } + auto take_ev = [&](BobPingEvent const& e) { + seen_ev.insert(ev_id(e)); + return e; + }; + auto dest_ok = [&](BobPingEvent const& e) { + return dest == 0 || e.server_id == dest; + }; + auto find_ev = [&](std::uint8_t kind, DWORD timeout_ms, + std::int64_t cycle_id, std::int64_t min_attempt) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.kind != kind) { + continue; + } + if (cycle_id != 0 && e.logical_cycle_id != 0 && + e.logical_cycle_id != cycle_id) { + continue; + } + if (min_attempt > 0 && e.physical_attempt_index < min_attempt) { + continue; + } + return take_ev(e); + } + } + return std::nullopt; + }; + auto wait_first = [&](DWORD timeout_ms) -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.physical_attempt_index > 1) { + continue; + } + if (e.kind == static_cast(PingTraceKind::kRequestSent) || + e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + return take_ev(e); + } + } + } + return std::nullopt; + }; + auto wait_sched = [&](DWORD timeout_ms) -> std::optional { + auto const before = alice.schedules.size(); + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + if (alice.schedules.size() > before) { + return alice.schedules.back(); + } + } + return std::nullopt; + }; + + constexpr std::uint8_t kFaultTraceArmed = 1; + constexpr std::uint8_t kFaultTraceMatched = 3; + constexpr std::uint8_t kFaultTraceDropped = 4; + + std::int64_t last_settled_cycle_id = 0; + for (auto const& e : bob.ping_events) { + if (e.logical_cycle_id > last_settled_cycle_id) { + last_settled_cycle_id = e.logical_cycle_id; + } + } + + int harness_armed = 0; + int harness_matched = 0; + int harness_dropped = 0; + int harness_wrong_request = 0; + int harness_unconsumed = 0; + int harness_leaked = 0; + int harness_sync_errors = 0; + + auto wait_arm_ack = [&](DWORD timeout_ms) -> bool { + bob.got_ack = false; + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + if (bob.got_ack) { + return true; + } + } + return false; + }; + + auto disarm_fault = [&]() -> bool { + bob.got_ack = false; + SendRaw(bob, kIpcArmFault, 0, dest, 1, 0, 0, 0, 0); + return wait_arm_ack(3000); + }; + + auto arm_drop_next_first = [&]() -> bool { + if (!disarm_fault()) { + return false; + } + bob.got_ack = false; + SendRaw(bob, kIpcArmFault, 0, dest, 1, 1, 0, 0, 0); + return wait_arm_ack(3000); + }; + + struct DropWaitResult { + std::optional drop{}; + bool sent_instead{false}; + bool armed{false}; + bool matched{false}; + bool dropped_trace{false}; + std::int64_t arm_time{0}; + std::int64_t match_time{0}; + std::int64_t drop_time{0}; + std::int64_t cycle_id{0}; + }; + + auto wait_intended_drop = [&](std::int64_t min_cycle_id, + std::size_t trace_after, DWORD timeout_ms) + -> DropWaitResult { + DropWaitResult out{}; + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (std::size_t i = trace_after; i < bob.fault_traces.size(); ++i) { + auto const& t = bob.fault_traces[i]; + if (dest != 0 && t.server_id != dest) { + continue; + } + if (t.kind == kFaultTraceArmed && !out.armed) { + out.armed = true; + out.arm_time = t.steady_us; + } + if (t.kind == kFaultTraceMatched && t.physical_attempt_index == 1) { + if (out.drop.has_value()) { + if (t.logical_cycle_id == out.drop->logical_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + } + } else if (t.logical_cycle_id > min_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + out.cycle_id = t.logical_cycle_id; + } + } + if (t.kind == kFaultTraceDropped && t.physical_attempt_index == 1) { + if (out.drop.has_value()) { + if (t.logical_cycle_id == out.drop->logical_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + } + } else if (t.logical_cycle_id > min_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + out.cycle_id = t.logical_cycle_id; + } + } + } + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.logical_cycle_id == 0) { + continue; + } + if (e.logical_cycle_id <= min_cycle_id) { + continue; + } + if (e.physical_attempt_index != 1) { + continue; + } + if (e.kind == + static_cast(PingTraceKind::kRequestSent)) { + seen_ev.insert(ev_id(e)); + out.sent_instead = true; + out.cycle_id = e.logical_cycle_id; + return out; + } + if (e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + out.drop = take_ev(e); + out.cycle_id = e.logical_cycle_id; + } + } + if (out.drop.has_value() && out.armed && out.matched && + out.dropped_trace) { + return out; + } + if (out.sent_instead) { + return out; + } + } + return out; + }; + + auto wait_next_first_after = [&](std::int64_t after_cycle_id, DWORD timeout_ms) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.logical_cycle_id == 0) { + continue; + } + if (e.logical_cycle_id <= after_cycle_id) { + continue; + } + if (e.physical_attempt_index != 1) { + continue; + } + if (e.kind == + static_cast(PingTraceKind::kRequestSent) || + e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + return take_ev(e); + } + } + } + return std::nullopt; + }; + + auto refresh_fault_flags = [&](DropWaitResult& out, + std::size_t trace_after) { + if (!out.drop.has_value()) { + return; + } + for (std::size_t i = trace_after; i < bob.fault_traces.size(); ++i) { + auto const& t = bob.fault_traces[i]; + if (dest != 0 && t.server_id != dest) { + continue; + } + if (t.kind == kFaultTraceArmed) { + out.armed = true; + out.arm_time = t.steady_us; + } + if (t.kind == kFaultTraceMatched && + t.physical_attempt_index == out.drop->physical_attempt_index && + t.logical_cycle_id == out.drop->logical_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + } + if (t.kind == kFaultTraceDropped && + t.physical_attempt_index == out.drop->physical_attempt_index && + t.logical_cycle_id == out.drop->logical_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + } + } + }; + + auto write_fault_trace_row = [&](FrlCase const& c) { + fault_trace_csv << c.index << "," << args.transport << "," << c.seed << "," + << c.target_logical_ping_id << "," << c.target_cycle_index + << "," << c.fault_arm_time << "," + << c.fault_state_before_send << "," + << c.logical_ping_id << "," << c.first_request_attempt_number + << "," << c.first_request_send_time << "," + << c.fault_match_time << "," << c.fault_drop_time << "," + << c.fault_consumed << "," << c.retry_logical_ping_id + << "," << c.retry_attempt_number << "," + << c.retry_send_time << "," << c.fault_state_end << "\n"; + fault_trace_csv.flush(); + }; + + auto fail = [&](FrlCase& c, char const* cls) { + c.failures.push_back(cls); + std::cerr << "FAIL case " << c.index << " " << cls << std::endl; + }; + + std::vector cases; + std::int64_t phase_anchor_us = 0; + bool ok_all = true; + + std::cout << "FIRST_REQUEST_LOSS_P99 cases=" << n_cases + << " seed=" << args.seed << " transport=" << args.transport + << " warmup_n=" << warmup_n << " warmup_p99_ms=" << warmup_p99 + << " warmup_min_ms=" << warmup_min << std::endl; + + // Document production statistic source once. + std::cout + << "PRODUCTION_P99_SOURCE=channel_statistics().response_time_" + "statistics().percentile<99>() " + "via PingCloudServers::MakePing; " + "guard=Clamp(max(0,(p99-min)/2)+10ms); " + "attempt_lead=G+loss_timeout+p99/2+scheduler+dispatch; " + "loss_timeout=max(ResponseTimeout(=p99),p99+10ms,50ms)\n"; + + wait_window_closed(8000); + + for (int ci = 0; ci < n_cases; ++ci) { + FrlCase rec{}; + rec.index = ci; + rec.seed = args.seed; + rec.rtt_sample_hint = warmup_n; + rec.target_cycle_index = ci; + rec.fault_state_before_send = 0; + + std::int64_t settle_before = last_settled_cycle_id; + std::size_t fault_trace_before = bob.fault_traces.size(); + + wait_window_closed(8000); + + bool armed_ok = false; + DropWaitResult dw{}; + for (int arm_try = 0; arm_try < 4; ++arm_try) { + armed_ok = arm_drop_next_first(); + if (armed_ok && arm_try == 0) { + ++harness_armed; + } + dw = wait_intended_drop(settle_before, fault_trace_before, 8000); + if (dw.drop.has_value()) { + refresh_fault_flags(dw, fault_trace_before); + drain(200); + refresh_fault_flags(dw, fault_trace_before); + } + if (armed_ok && dw.drop.has_value() && dw.armed && dw.matched && + dw.dropped_trace && dw.drop->physical_attempt_index == 1 && + dw.drop->logical_cycle_id > settle_before) { + break; + } + if (dw.sent_instead) { + if (dw.cycle_id > settle_before) { + (void)find_ev( + static_cast(PingTraceKind::kCycleConfirmed), 3000, + dw.cycle_id, 0); + settle_before = dw.cycle_id; + last_settled_cycle_id = dw.cycle_id; + } + disarm_fault(); + fault_trace_before = bob.fault_traces.size(); + dw = DropWaitResult{}; + continue; + } + disarm_fault(); + fault_trace_before = bob.fault_traces.size(); + dw = DropWaitResult{}; + } + + bool sync_ok = armed_ok && dw.drop.has_value() && dw.armed && + dw.matched && dw.dropped_trace && + dw.drop->physical_attempt_index == 1 && + dw.drop->logical_cycle_id > settle_before; + + rec.fault_arm_time = dw.arm_time; + rec.fault_match_time = dw.match_time; + rec.fault_drop_time = dw.drop_time; + rec.fault_state_end = sync_ok ? 4 : -1; + + if (dw.matched) { + ++harness_matched; + } + if (dw.dropped_trace) { + ++harness_dropped; + } + if (dw.sent_instead) { + ++harness_wrong_request; + } + + if (!sync_ok) { + fail(rec, "HARNESS_FAULT_SYNC_ERROR"); + ++harness_sync_errors; + ok_all = false; + if (armed_ok && !dw.dropped_trace && !dw.sent_instead) { + ++harness_unconsumed; + } + if (dw.sent_instead) { + rec.first_request_dropped = 0; + rec.logical_ping_id = dw.cycle_id; + } else { + rec.first_request_dropped = 0; + } + write_fault_trace_row(rec); + cases.push_back(std::move(rec)); + disarm_fault(); + continue; + } + + rec.harness_sync_ok = true; + rec.fault_consumed = 1; + rec.first_request_dropped = 1; + + BobPingEvent const first = *dw.drop; + rec.logical_ping_id = first.logical_cycle_id; + rec.target_logical_ping_id = first.logical_cycle_id; + rec.tn_us = first.cycle_anchor_us; + rec.first_request_attempt_number = first.physical_attempt_index; + rec.first_request_send_time = + first.actual_us != 0 ? first.actual_us : first.event_steady_us; + if (phase_anchor_us == 0 && rec.tn_us != 0) { + phase_anchor_us = rec.tn_us; + } + rec.phase_anchor_us = phase_anchor_us; + rec.actual_first_send_us = rec.first_request_send_time; + + // Production values used for THIS attempt (from Bob ping trace). + if (first.p99_rtt_us <= 0) { + fail(rec, "HARNESS_FAILURE"); + std::cerr << " reason: production p99_rtt not present on attempt " + "trace (expected response_stats.percentile<99>)\n"; + ok_all = false; + write_fault_trace_row(rec); + cases.push_back(std::move(rec)); + disarm_fault(); + continue; + } + rec.rtt_p99_used_ms = first.p99_rtt_us / 1000.0; + rec.guard_used_ms = first.guard_us / 1000.0; + rec.loss_timeout_ms = first.loss_timeout_us / 1000.0; + rec.attempt_lead_ms = first.attempt_lead_us / 1000.0; + + // Intended policy under test (not production's +20ms extras). + rec.expected_retry_budget_ms = + 1.5 * rec.rtt_p99_used_ms + rec.guard_used_ms; + rec.expected_first_send_us = + static_cast(rec.tn_us) - + rec.expected_retry_budget_ms * 1000.0; + rec.first_send_margin_ms = + (rec.expected_first_send_us - + static_cast(rec.actual_first_send_us)) / + 1000.0; + rec.expected_latest_retry_send_us = + static_cast(rec.tn_us) - + (rec.rtt_p99_used_ms / 2.0 + rec.guard_used_ms) * 1000.0; + + if (!(rec.first_send_margin_ms >= -tick_ms)) { + // actual_first_send must be <= expected_first_send (+1ms tick) + fail(rec, "FIRST_SEND_TOO_LATE"); + ok_all = false; + } + + // Wait for attempt timeout then retry send. No Alice QueryNow here. + auto to = find_ev( + static_cast(PingTraceKind::kAttemptTimeout), 2000, + rec.logical_ping_id, 0); + if (to) { + rec.retry_decision_us = to->event_steady_us; + } + auto retry = find_ev( + static_cast(PingTraceKind::kRequestSent), 2000, + rec.logical_ping_id, 2); + if (!retry || retry->request_was_sent == 0) { + fail(rec, "NO_RETRY"); + ok_all = false; + } else { + rec.actual_retry_send_us = + retry->actual_us != 0 ? retry->actual_us : retry->event_steady_us; + rec.retry_logical_ping_id = retry->logical_cycle_id; + rec.retry_attempt_number = retry->physical_attempt_index; + rec.retry_send_time = rec.actual_retry_send_us; + rec.retry_send_margin_ms = + (rec.expected_latest_retry_send_us - + static_cast(rec.actual_retry_send_us)) / + 1000.0; + if (!(rec.retry_send_margin_ms >= -tick_ms)) { + fail(rec, "RETRY_SEND_TOO_LATE"); + ok_all = false; + } + rec.estimated_retry_server_arrival_us = + static_cast(rec.actual_retry_send_us) + + (rec.rtt_p99_used_ms / 2.0) * 1000.0; + rec.estimated_retry_server_margin_ms = + (static_cast(rec.tn_us) - + rec.estimated_retry_server_arrival_us) / + 1000.0; + // Require arrival <= Tn - G => margin >= G + if (!(rec.estimated_retry_server_margin_ms + tick_ms >= + rec.guard_used_ms)) { + fail(rec, "ESTIMATED_RETRY_ARRIVAL_LATE"); + ok_all = false; + } + if (retry->contract_deadline_us != 0) { + rec.next_nominal_us = retry->contract_deadline_us; + } + } + + // Confirm cycle / capture next nominal from confirm if needed. + auto conf = find_ev( + static_cast(PingTraceKind::kCycleConfirmed), 8000, + rec.logical_ping_id, 0); + if (conf) { + if (rec.next_nominal_us == 0) { + rec.next_nominal_us = conf->contract_deadline_us != 0 + ? conf->contract_deadline_us + : (rec.tn_us + period_us); + } + } else if (rec.next_nominal_us == 0 && rec.tn_us != 0) { + rec.next_nominal_us = rec.tn_us + period_us; + } + if (rec.tn_us != 0 && rec.next_nominal_us != 0) { + rec.next_nominal_phase_error_ms = + (static_cast(rec.next_nominal_us) - + static_cast(rec.tn_us + period_us)) / + 1000.0; + if (abs_d(rec.next_nominal_phase_error_ms) > tick_ms) { + fail(rec, "POST_RECOVERY_PHASE_DRIFT"); + ok_all = false; + } + } + + // Wait until after Tn, then Alice queries exactly once. + if (rec.tn_us != 0 && first.event_qpc != 0 && first.actual_us != 0 && + rec.tn_us > first.actual_us) { + LARGE_INTEGER qfreq{}; + QueryPerformanceFrequency(&qfreq); + double const qpc_per_us = + static_cast(qfreq.QuadPart) / 1000000.0; + auto const tn_qpc = + first.event_qpc + + static_cast( + static_cast(rec.tn_us - first.actual_us) * qpc_per_us); + auto now_qpc = QpcNow(); + if (now_qpc < tn_qpc) { + auto wait_ms = + QpcToMs(static_cast(tn_qpc - now_qpc)); + if (wait_ms < 5000) { + Sleep(static_cast(wait_ms + + static_cast(tick_ms) + 2)); + } + } else { + Sleep(static_cast(tick_ms) + 2); + } + } else { + Sleep(static_cast(tick_ms) + 2); + } + + SendRaw(alice, kIpcQueryNow, 0, /*checkpoint*/ 4, 0, 0); + auto aq = wait_sched(3000); + if (!aq) { + fail(rec, "HARNESS_FAILURE"); + ok_all = false; + } else { + rec.alice_query_us = aq->steady_us; + rec.alice_state = aq->state; + rec.alice_deadline_us = aq->next_us; + rec.alice_next_deadline_us = aq->next_us; + rec.alice_last_request_us = aq->last_online_us; + rec.alice_next_ping_delta_ms = aq->next_ping_delta_ms; + rec.alice_last_connect_delta_ms = aq->last_connect_delta_ms; + rec.alice_missed_deadline = (aq->state == 1); + if (rec.alice_missed_deadline) { + fail(rec, "ALICE_FALSE_MISSED_DEADLINE"); + ok_all = false; + } + // last_request vs Tn is statistical only (p99 leaves a tail). + // Do NOT fail the case solely because last_request >= Tn. + if (!(rec.alice_last_request_us != 0 && rec.tn_us != 0) && + !(aq->last_connect_delta_ms != + std::numeric_limits::min() && + aq->last_connect_delta_ms >= 0)) { + fail(rec, "HARNESS_FAILURE"); + ok_all = false; + } + // next deadline on Tn+1000 + if (rec.alice_next_deadline_us != 0 && rec.tn_us != 0) { + double phase_err = + (static_cast(rec.alice_next_deadline_us) - + static_cast(rec.tn_us + period_us)) / + 1000.0; + if (abs_d(phase_err) > tick_ms) { + // Also accept next_ping_delta ≈ remaining to Tn+1000 + bool delta_ok = false; + if (aq->next_ping_delta_ms != + std::numeric_limits::min()) { + // After Tn, next should be ~Tn+1000 => delta roughly + // (Tn+1000 - query_time). Soft check: delta > 0 and state Expected. + delta_ok = aq->next_ping_delta_ms > 0 && aq->state == 0; + } + if (!delta_ok) { + fail(rec, "ALICE_NEXT_DEADLINE_PHASE_SHIFT"); + ok_all = false; + } + } + } else if (aq->next_ping_delta_ms <= 0 || aq->state != 0) { + fail(rec, "ALICE_NEXT_DEADLINE_PHASE_SHIFT"); + ok_all = false; + } + } + + // Observe next three nominal anchors on the same phase. + disarm_fault(); + std::int64_t observe_after = rec.logical_ping_id; + std::vector next_tns; + for (int k = 0; k < 3; ++k) { + auto nx = wait_next_first_after(observe_after, 4000); + if (!nx) { + break; + } + observe_after = nx->logical_cycle_id; + (void)find_ev( + static_cast(PingTraceKind::kCycleConfirmed), 4000, + nx->logical_cycle_id, 0); + if (nx->cycle_anchor_us != 0) { + next_tns.push_back(nx->cycle_anchor_us); + } + } + last_settled_cycle_id = observe_after; + for (std::size_t i = fault_trace_before; i < bob.fault_traces.size(); ++i) { + auto const& t = bob.fault_traces[i]; + if (t.kind == kFaultTraceArmed && + (t.logical_cycle_id == 0 || t.logical_cycle_id > settle_before)) { + ++harness_leaked; + } + } + if (next_tns.size() >= 1) { + rec.next2_us = next_tns[0]; + rec.next2_phase_error_ms = + (static_cast(rec.next2_us) - + static_cast(rec.tn_us + 2 * period_us)) / + 1000.0; + } + if (next_tns.size() >= 2) { + // If we already captured next_nominal from contract, first observed + // after confirm may be Tn+1000 or Tn+2000 depending on timing. + // Align by snapping to nearest grid slot. + auto snap_err = [&](std::int64_t t, int slot) { + return (static_cast(t) - + static_cast(rec.tn_us + slot * period_us)) / + 1000.0; + }; + // Recompute best slot assignment for observed anchors + for (std::size_t i = 0; i < next_tns.size(); ++i) { + double best = 1e300; + int best_slot = static_cast(i + 1); + for (int slot = 1; slot <= 4; ++slot) { + auto e = abs_d(snap_err(next_tns[i], slot)); + if (e < best) { + best = e; + best_slot = slot; + } + } + if (best_slot == 1 && i == 0) { + // already have next_nominal; prefer observed if closer + if (!std::isfinite(rec.next_nominal_phase_error_ms) || + abs_d(snap_err(next_tns[i], 1)) < + abs_d(rec.next_nominal_phase_error_ms)) { + rec.next_nominal_us = next_tns[i]; + rec.next_nominal_phase_error_ms = snap_err(next_tns[i], 1); + } + } else if (best_slot == 2) { + rec.next2_us = next_tns[i]; + rec.next2_phase_error_ms = snap_err(next_tns[i], 2); + } else if (best_slot == 3) { + rec.next3_us = next_tns[i]; + rec.next3_phase_error_ms = snap_err(next_tns[i], 3); + } + } + } + if (next_tns.size() >= 3) { + auto snap_err = [&](std::int64_t t, int slot) { + return (static_cast(t) - + static_cast(rec.tn_us + slot * period_us)) / + 1000.0; + }; + rec.next3_us = next_tns.back(); + rec.next3_phase_error_ms = snap_err(rec.next3_us, 3); + } + for (double e : + {rec.next_nominal_phase_error_ms, rec.next2_phase_error_ms, + rec.next3_phase_error_ms}) { + if (std::isfinite(e) && abs_d(e) > tick_ms) { + fail(rec, "POST_RECOVERY_PHASE_DRIFT"); + ok_all = false; + break; + } + } + + // CSV + JSONL row (flushed each case) + std::string classes; + for (std::size_t fi = 0; fi < rec.failures.size(); ++fi) { + if (fi) { + classes += "|"; + } + classes += rec.failures[fi]; + } + auto is_harness_cls = [](std::string const& s) { + return s.find("HARNESS") != std::string::npos || + s == "FIRST_REQUEST_NOT_ACTUALLY_DROPPED"; + }; + bool harness_invalid = false; + for (auto const& f : rec.failures) { + if (is_harness_cls(f)) { + harness_invalid = true; + break; + } + } + double est_to_tn_g = std::numeric_limits::quiet_NaN(); + if (std::isfinite(rec.estimated_retry_server_margin_ms) && + std::isfinite(rec.guard_used_ms)) { + est_to_tn_g = + rec.estimated_retry_server_margin_ms - rec.guard_used_ms; + } + samples_csv << args.transport << "," << args.seed << "," << ci << "," + << rec.logical_ping_id << "," << rec.rtt_sample_hint << "," + << csv_d(rec.rtt_p99_used_ms) << "," + << csv_d(rec.guard_used_ms) << "," << csv_d(dispatch_margin_ms) + << "," << csv_d(rec.loss_timeout_ms) << "," + << csv_d(rec.attempt_lead_ms) << "," << rec.tn_us << "," + << csv_d(rec.expected_retry_budget_ms) << "," + << csv_d(rec.expected_first_send_us) << "," + << csv_i(rec.actual_first_send_us) << "," + << csv_d(rec.first_send_margin_ms) << "," + << rec.first_request_dropped << "," + << csv_i(rec.retry_decision_us) << "," + << csv_d(rec.expected_latest_retry_send_us) << "," + << csv_i(rec.actual_retry_send_us) << "," + << csv_d(rec.retry_send_margin_ms) << "," + << csv_d(rec.estimated_retry_server_arrival_us) << "," + << csv_d(rec.estimated_retry_server_margin_ms) << "," + << csv_d(est_to_tn_g) << "," << rec.next_nominal_us << "," + << csv_d(rec.next_nominal_phase_error_ms) << "," + << rec.next2_us << "," << csv_d(rec.next2_phase_error_ms) + << "," << rec.next3_us << "," + << csv_d(rec.next3_phase_error_ms) << "," + << csv_i(rec.alice_query_us) << "," + << csv_i(rec.alice_last_request_us) << "," + << csv_i(rec.alice_deadline_us) << "," + << csv_i(rec.alice_next_deadline_us) << "," + << rec.alice_state << "," + << (rec.alice_missed_deadline ? 1 : 0) << "," + << csv_i(rec.alice_next_ping_delta_ms) << "," + << csv_i(rec.alice_last_connect_delta_ms) << "," << classes + << "," << (harness_invalid ? 1 : 0) << "\n"; + samples_csv.flush(); + samples_jsonl + << "{\"case_index\":" << ci << ",\"transport\":\"" << args.transport + << "\",\"seed\":" << args.seed << ",\"Tn\":" << rec.tn_us + << ",\"R99_used_ms\":" << csv_d(rec.rtt_p99_used_ms) + << ",\"guard_ms\":" << csv_d(rec.guard_used_ms) + << ",\"dispatch_margin_ms\":" << csv_d(dispatch_margin_ms) + << ",\"expected_first_send_ms\":" + << csv_d(std::isfinite(rec.expected_first_send_us) + ? rec.expected_first_send_us / 1000.0 + : std::numeric_limits::quiet_NaN()) + << ",\"actual_first_send_ms\":" + << (rec.actual_first_send_us == kMissing + ? std::string{} + : std::to_string(rec.actual_first_send_us / 1000.0)) + << ",\"first_request_dropped\":" << rec.first_request_dropped + << ",\"retry_decision_ms\":" + << (rec.retry_decision_us == kMissing + ? std::string{} + : std::to_string(rec.retry_decision_us / 1000.0)) + << ",\"expected_latest_retry_send_ms\":" + << csv_d(std::isfinite(rec.expected_latest_retry_send_us) + ? rec.expected_latest_retry_send_us / 1000.0 + : std::numeric_limits::quiet_NaN()) + << ",\"actual_retry_send_ms\":" + << (rec.actual_retry_send_us == kMissing + ? std::string{} + : std::to_string(rec.actual_retry_send_us / 1000.0)) + << ",\"retry_send_margin_ms\":" << csv_d(rec.retry_send_margin_ms) + << ",\"estimated_retry_server_arrival_ms\":" + << csv_d(std::isfinite(rec.estimated_retry_server_arrival_us) + ? rec.estimated_retry_server_arrival_us / 1000.0 + : std::numeric_limits::quiet_NaN()) + << ",\"estimated_retry_server_margin_to_Tn_ms\":" + << csv_d(rec.estimated_retry_server_margin_ms) + << ",\"estimated_retry_server_margin_to_Tn_minus_guard_ms\":" + << csv_d(est_to_tn_g) + << ",\"next_nominal_deadline_ms\":" + << (rec.next_nominal_us == 0 + ? std::string{} + : std::to_string(rec.next_nominal_us / 1000.0)) + << ",\"next_nominal_phase_error_ms\":" + << csv_d(rec.next_nominal_phase_error_ms) + << ",\"alice_query_time_ms\":" + << (rec.alice_query_us == kMissing + ? std::string{} + : std::to_string(rec.alice_query_us / 1000.0)) + << ",\"alice_last_request_time_ms\":" + << (rec.alice_last_request_us == kMissing || + rec.alice_last_request_us == 0 + ? std::string{} + : std::to_string(rec.alice_last_request_us / 1000.0)) + << ",\"alice_state\":" << rec.alice_state + << ",\"alice_missed_deadline\":" + << (rec.alice_missed_deadline ? "true" : "false") + << ",\"alice_next_deadline_ms\":" + << (rec.alice_next_deadline_us == kMissing || + rec.alice_next_deadline_us == 0 + ? std::string{} + : std::to_string(rec.alice_next_deadline_us / 1000.0)) + << ",\"harness_invalid\":" << (harness_invalid ? "true" : "false") + << ",\"failure_classes\":\"" << classes << "\"}\n"; + samples_jsonl.flush(); + write_fault_trace_row(rec); + cases.push_back(std::move(rec)); + } + + // Aggregate helpers + auto collect = [&](auto proj) { + std::vector v; + for (auto const& c : cases) { + double x = proj(c); + if (std::isfinite(x)) { + v.push_back(x); + } + } + return v; + }; + auto pct = [](std::vector v, double p) { + if (v.empty()) { + return std::numeric_limits::quiet_NaN(); + } + std::sort(v.begin(), v.end()); + double k = (v.size() - 1) * (p / 100.0); + auto f = static_cast(std::floor(k)); + auto c = static_cast(std::ceil(k)); + if (f == c) { + return v[f]; + } + return v[f] * (c - k) + v[c] * (k - f); + }; + auto dstat = [&](std::vector const& v) { + if (v.empty()) { + return std::string("n=0"); + } + auto s = v; + std::sort(s.begin(), s.end()); + return "n=" + std::to_string(s.size()) + " min=" + F3(s.front()) + + " p1=" + F3(pct(s, 1)) + " p5=" + F3(pct(s, 5)) + + " p50=" + F3(pct(s, 50)) + " p95=" + F3(pct(s, 95)) + + " p99=" + F3(pct(s, 99)) + " max=" + F3(s.back()); + }; + + auto collect_sync = [&](auto proj) { + std::vector v; + for (auto const& c : cases) { + if (!c.harness_sync_ok) { + continue; + } + double x = proj(c); + if (std::isfinite(x)) { + v.push_back(x); + } + } + return v; + }; + + auto p99s = collect_sync([](FrlCase const& c) { return c.rtt_p99_used_ms; }); + auto guards = collect_sync([](FrlCase const& c) { return c.guard_used_ms; }); + auto fs_m = + collect_sync([](FrlCase const& c) { return c.first_send_margin_ms; }); + auto rs_m = + collect_sync([](FrlCase const& c) { return c.retry_send_margin_ms; }); + auto est_m = collect_sync([](FrlCase const& c) { + return c.estimated_retry_server_margin_ms; + }); + auto est_to_tn_g_m = collect_sync([](FrlCase const& c) { + if (!std::isfinite(c.estimated_retry_server_margin_ms) || + !std::isfinite(c.guard_used_ms)) { + return std::numeric_limits::quiet_NaN(); + } + return c.estimated_retry_server_margin_ms - c.guard_used_ms; + }); + auto phase_e = collect_sync([](FrlCase const& c) { + return c.next_nominal_phase_error_ms; + }); + + int drop_ok = 0, first_late = 0, retry_late = 0, est_after_tn = 0, + est_after_tn_g = 0, alice_last_ok = 0, alice_md_false = 0, + alice_phase_ok = 0, alice_n = 0, retry_on_time_dropped = 0, + est_by_tn_g_dropped = 0, drift_1000 = 0; + for (auto const& c : cases) { + if (!c.harness_sync_ok) { + continue; + } + if (c.first_request_dropped == 1) { + ++drop_ok; + } + if (std::isfinite(c.first_send_margin_ms) && + c.first_send_margin_ms < -tick_ms) { + ++first_late; + } + if (std::isfinite(c.retry_send_margin_ms) && + c.retry_send_margin_ms < -tick_ms) { + ++retry_late; + } + if (std::isfinite(c.estimated_retry_server_margin_ms) && + c.estimated_retry_server_margin_ms < 0) { + ++est_after_tn; + } + if (std::isfinite(c.estimated_retry_server_margin_ms) && + std::isfinite(c.guard_used_ms) && + c.estimated_retry_server_margin_ms + tick_ms < c.guard_used_ms) { + ++est_after_tn_g; + } + for (double e : {c.next_nominal_phase_error_ms, c.next2_phase_error_ms, + c.next3_phase_error_ms}) { + if (std::isfinite(e) && abs_d(e) >= 1000.0 - tick_ms) { + ++drift_1000; + break; + } + } + if (c.alice_query_us != kMissing) { + ++alice_n; + if (c.alice_last_request_us != 0 && c.tn_us != 0 && + c.alice_last_request_us < c.tn_us) { + ++alice_last_ok; + } + if (!c.alice_missed_deadline) { + ++alice_md_false; + } + bool phase_ok = false; + if (c.alice_next_deadline_us != 0 && c.tn_us != 0) { + double e = (static_cast(c.alice_next_deadline_us) - + static_cast(c.tn_us + period_us)) / + 1000.0; + phase_ok = abs_d(e) <= tick_ms; + } + if (!phase_ok && c.alice_next_ping_delta_ms > 0 && c.alice_state == 0) { + phase_ok = true; + } + if (phase_ok) { + ++alice_phase_ok; + } + } + } + retry_on_time_dropped = 0; + est_by_tn_g_dropped = 0; + for (auto const& c : cases) { + if (!c.harness_sync_ok) { + continue; + } + if (std::isfinite(c.retry_send_margin_ms) && + c.retry_send_margin_ms >= -tick_ms) { + ++retry_on_time_dropped; + } + if (std::isfinite(c.estimated_retry_server_margin_ms) && + std::isfinite(c.guard_used_ms) && + c.estimated_retry_server_margin_ms + tick_ms >= c.guard_used_ms) { + ++est_by_tn_g_dropped; + } + } + + // Does production satisfy intended 1.5*p99+G? + // Compare production attempt_lead to intended budget. + int lead_ge_budget = 0; + int lead_n = 0; + for (auto const& c : cases) { + if (std::isfinite(c.attempt_lead_ms) && + std::isfinite(c.expected_retry_budget_ms)) { + ++lead_n; + if (c.attempt_lead_ms + tick_ms >= c.expected_retry_budget_ms) { + ++lead_ge_budget; + } + } + } + + int const sync_ok_n = drop_ok; + if (harness_sync_errors > 0) { + ok_all = false; + } + + report << "# First-request-loss p99 timing (deterministic fault)\n\n"; + report << "- transport: " << args.transport << "\n"; + report << "- seed: " << args.seed << "\n"; + report << "- cases: " << n_cases << "\n"; + report << "- warmup_rtt_samples: " << warmup_n << "\n"; + report << "- warmup_min_rtt_ms: " << warmup_min << "\n"; + report << "- warmup_p99_rtt_ms: " << warmup_p99 << "\n"; + report << "\n## Harness\n\n"; + report << "- planned: " << n_cases << "\n"; + report << "- fault armed correctly: " << harness_armed << "\n"; + report << "- matched correct logical ping: " << sync_ok_n << "\n"; + report << "- dropped correct attempt #1: " << sync_ok_n << "\n"; + report << "- wrong-request drops: " << harness_wrong_request << "\n"; + report << "- unconsumed faults: " << harness_unconsumed << "\n"; + report << "- leaked faults: " << harness_leaked << "\n"; + report << "- HARNESS_FAULT_SYNC_ERROR: " << harness_sync_errors << "\n"; + report << "\n## Production formula\n\n"; + report << "Source: `PingCloudServers::MakePing` +\n" + "`ComputePingRetryBudget` / `ComputePingSendGuard`.\n\n"; + report << "- RTT statistic: " + "`channel_statistics().response_time_statistics()." + "percentile<99>()`\n"; + report << "- min RTT: `response_time_statistics().min()`\n"; + report << "- guard G: `Clamp(max(0,(p99-min)/2)+10ms)`\n"; + report << "- `ResponseTimeout()`: same p99 when stats non-empty\n"; + report << "- `loss_timeout = max(raw, p99+10ms, 50ms)` (interval-capped)\n"; + report << "- `attempt_lead = G + loss_timeout + p99/2 + scheduler(10ms) " + "+ retry_dispatch(60ms)` " + "(≈ `1.5*p99 + G + 80ms` when uncapped)\n"; + report << "- Intended invariant under test: " + "`retry_send <= Tn-(R99/2+G)` and " + "`retry_send+R99/2 <= Tn-G`\n"; + report << "- Alice `last_request < Tn` is reported but not a hard gate\n"; + report << "- cases where production attempt_lead >= intended budget: " + << lead_ge_budget << "/" << lead_n << "\n"; + report << "\n## R99 / scheduling\n\n"; + report << "- R99: " << dstat(p99s) << "\n"; + report << "- guard: " << dstat(guards) << "\n"; + report << "\n## First send\n\n"; + report << "- first_send_margin (intended): " << dstat(fs_m) << "\n"; + report << "- cases first send too late: " << first_late << "\n"; + report << "- first request actually dropped (sync-valid): " << drop_ok + << "/" << sync_ok_n << "\n"; + report << "\n## Retry timing (sync-valid drops only)\n\n"; + report << "- retry on time: " << retry_on_time_dropped << "/" << sync_ok_n + << "\n"; + report << "- retry late: " << retry_late << "\n"; + report << "- retry_send_margin: " << dstat(rs_m) << "\n"; + report << "\n## Estimated server arrival (sync-valid drops only)\n\n"; + report << "- arrival by Tn-G: " << est_by_tn_g_dropped << "/" << sync_ok_n + << "\n"; + report << "- arrival after Tn-G: " << est_after_tn_g << "\n"; + report << "- arrival after Tn: " << est_after_tn << "\n"; + report << "- margin_to_Tn_minus_G: " << dstat(est_to_tn_g_m) << "\n"; + report << "- estimated_retry_server_margin: " << dstat(est_m) << "\n"; + report << "\n## Alice (sync-valid drops only)\n\n"; + report << "- Alice queries: " << alice_n << "\n"; + report << "- last_request before Tn: " << alice_last_ok << "/" << alice_n + << "\n"; + report << "- MissedDeadline false: " << alice_md_false << "/" << alice_n + << "\n"; + report << "- next deadline on Tn+1000 (or Expected delta): " << alice_phase_ok + << "/" << alice_n << "\n"; + report << "\n## Phase\n\n"; + report << "- valid queries: " << alice_n << "\n"; + report << "- last_request < Tn: " << alice_last_ok << "/" << alice_n + << "\n"; + report << "- last_request >= Tn: " << (alice_n - alice_last_ok) << "/" + << alice_n << "\n"; + report << "- MissedDeadline true: " << (alice_n - alice_md_false) << "/" + << alice_n << "\n"; + report << "- MissedDeadline false: " << alice_md_false << "/" << alice_n + << "\n"; + report << "- next deadline phase correct: " << alice_phase_ok << "/" + << alice_n << "\n"; + report << "\n## Phase (sync-valid drops only)\n\n"; + report << "- next_nominal_phase_error: " << dstat(phase_e) << "\n"; + { + std::vector abs_drift; + for (auto const& c : cases) { + if (!c.harness_sync_ok) { + continue; + } + for (double e : {c.next_nominal_phase_error_ms, c.next2_phase_error_ms, + c.next3_phase_error_ms}) { + if (std::isfinite(e)) { + abs_drift.push_back(abs_d(e)); + } + } + } + report << "- next-3 max drift: " + << (abs_drift.empty() + ? std::string("n/a") + : F3(*std::max_element(abs_drift.begin(), + abs_drift.end()))) + << "\n"; + } + report << "- 1000-ms drift count: " << drift_1000 << "\n"; + report << "\n## Result\n\n"; + report << (ok_all ? "PASS" : "FAIL") << " first-request-loss-p99\n"; + report.flush(); + + // failed-cases.json + failed_json << "[\n"; + bool first_fail = true; + for (auto const& c : cases) { + if (c.failures.empty()) { + continue; + } + if (!first_fail) { + failed_json << ",\n"; + } + first_fail = false; + failed_json << " {\"transport\":\"" << args.transport + << "\",\"seed\":" << c.seed << ",\"case_index\":" << c.index + << ",\"R99\":" << csv_d(c.rtt_p99_used_ms) + << ",\"guard\":" << csv_d(c.guard_used_ms) + << ",\"Tn\":" << c.tn_us + << ",\"expected_first_send\":" << csv_d(c.expected_first_send_us) + << ",\"actual_first_send\":" << csv_i(c.actual_first_send_us) + << ",\"retry_decision\":" << csv_i(c.retry_decision_us) + << ",\"expected_latest_retry_send\":" + << csv_d(c.expected_latest_retry_send_us) + << ",\"actual_retry_send\":" << csv_i(c.actual_retry_send_us) + << ",\"estimated_retry_server_arrival\":" + << csv_d(c.estimated_retry_server_arrival_us) + << ",\"estimated_retry_server_margin\":" + << csv_d(c.estimated_retry_server_margin_ms) + << ",\"alice_query_time\":" << csv_i(c.alice_query_us) + << ",\"alice_last_request_time\":" + << csv_i(c.alice_last_request_us) + << ",\"alice_state\":" << c.alice_state + << ",\"alice_deadline\":" << csv_i(c.alice_deadline_us) + << ",\"alice_next_deadline\":" + << csv_i(c.alice_next_deadline_us) + << ",\"next_nominal\":" << c.next_nominal_us + << ",\"next2\":" << c.next2_us << ",\"next3\":" << c.next3_us + << ",\"phase_errors\":[" + << csv_d(c.next_nominal_phase_error_ms) << "," + << csv_d(c.next2_phase_error_ms) << "," + << csv_d(c.next3_phase_error_ms) << "],\"classes\":["; + for (std::size_t i = 0; i < c.failures.size(); ++i) { + if (i) { + failed_json << ","; + } + failed_json << "\"" << c.failures[i] << "\""; + } + failed_json << "]}"; + } + failed_json << "\n]\n"; + failed_json.flush(); + + int harness_invalid_n = 0; + int production_fail_n = 0; + int valid_n = 0; + for (auto const& c : cases) { + if (!c.harness_sync_ok) { + continue; + } + ++valid_n; + bool prod = false; + for (auto const& f : c.failures) { + if (f.find("HARNESS") == std::string::npos) { + prod = true; + } + } + if (prod) { + ++production_fail_n; + } + } + harness_invalid_n = harness_sync_errors; + bool const prod_ok = + (production_fail_n == 0 && harness_sync_errors == 0 && sync_ok_n == n_cases); + summary_json << "{\n" + << " \"transport\": \"" << args.transport << "\",\n" + << " \"seed\": " << args.seed << ",\n" + << " \"dispatch_margin_ms\": " << dispatch_margin_ms << ",\n" + << " \"planned_cases\": " << n_cases << ",\n" + << " \"harness_armed\": " << harness_armed << ",\n" + << " \"harness_sync_ok\": " << sync_ok_n << ",\n" + << " \"harness_sync_errors\": " << harness_sync_errors << ",\n" + << " \"harness_wrong_request\": " << harness_wrong_request + << ",\n" + << " \"harness_unconsumed\": " << harness_unconsumed << ",\n" + << " \"harness_leaked\": " << harness_leaked << ",\n" + << " \"drift_1000_ms\": " << drift_1000 << ",\n" + << " \"valid_cases\": " << sync_ok_n << ",\n" + << " \"harness_invalid_cases\": " << harness_sync_errors + << ",\n" + << " \"production_timing_failures\": " << production_fail_n + << ",\n" + << " \"retry_on_time\": \"" << retry_on_time_dropped << "/" + << sync_ok_n << "\",\n" + << " \"retry_late\": " << retry_late << ",\n" + << " \"estimated_arrival_by_Tn_minus_G\": \"" + << est_by_tn_g_dropped << "/" << sync_ok_n << "\",\n" + << " \"first_request_dropped\": \"" << drop_ok << "/" + << sync_ok_n << "\",\n" + << " \"alice_queries\": " << alice_n << ",\n" + << " \"alice_last_request_before_Tn\": \"" << alice_last_ok + << "/" << alice_n << "\",\n" + << " \"alice_missed_deadline_false\": \"" << alice_md_false + << "/" << alice_n << "\",\n" + << " \"pass\": " << (prod_ok ? "true" : "false") << "\n" + << "}\n"; + summary_json.flush(); + + { + auto root = std::filesystem::path{args.artifact_dir}.parent_path(); + std::ofstream cmp(root / "comparison.md"); + if (cmp) { + cmp << "# Comparison: 500-case race vs deterministic 200-case\n\n"; + cmp << "| Metric | TCP old 500 | TCP deterministic 200 | UDP old 500 | " + "UDP deterministic 200 |\n"; + cmp << "| --- | ---: | ---: | ---: | ---: |\n"; + cmp << "| intended first request dropped | 464/500 | "; + if (args.transport == "tcp") { + cmp << sync_ok_n << "/" << n_cases; + } + cmp << " | 390/500 | "; + if (args.transport == "udp") { + cmp << sync_ok_n << "/" << n_cases; + } + cmp << " |\n"; + cmp << "| fault sync failures | 36 | "; + cmp << (args.transport == "tcp" ? std::to_string(harness_sync_errors) + : std::string()); + cmp << " | 110 | "; + cmp << (args.transport == "udp" ? std::to_string(harness_sync_errors) + : std::string()); + cmp << " |\n"; + cmp << "| retry timing pass among real drops | 460/464 | "; + if (args.transport == "tcp") { + cmp << retry_on_time_dropped << "/" << sync_ok_n; + } + cmp << " | 386/390 | "; + if (args.transport == "udp") { + cmp << retry_on_time_dropped << "/" << sync_ok_n; + } + cmp << " |\n"; + cmp << "| retry timing failures | 4 | "; + cmp << (args.transport == "tcp" ? std::to_string(retry_late) + : std::string()); + cmp << " | 4 | "; + cmp << (args.transport == "udp" ? std::to_string(retry_late) + : std::string()); + cmp << " |\n"; + cmp << "| 1000ms drift cases | 6 | "; + cmp << (args.transport == "tcp" ? std::to_string(drift_1000) + : std::string()); + cmp << " | 3 | "; + cmp << (args.transport == "udp" ? std::to_string(drift_1000) + : std::string()); + cmp << " |\n"; + cmp << "| false MissedDeadline | 0 | "; + cmp << (args.transport == "tcp" + ? std::to_string(alice_n - alice_md_false) + : std::string()); + cmp << " | 0 | "; + cmp << (args.transport == "udp" + ? std::to_string(alice_n - alice_md_false) + : std::string()); + cmp << " |\n"; + cmp.flush(); + } + } + + std::cout << (prod_ok ? "PASS" : "FAIL") << " first-request-loss-p99 cases=" + << cases.size() << " sync_ok=" << sync_ok_n + << " harness_sync_errors=" << harness_sync_errors + << " production_fail=" << production_fail_n + << " first_late=" << first_late << " retry_late=" << retry_late + << " est_late=" << est_after_tn_g + << " drift_1000=" << drift_1000 + << " alice_md_ok=" << alice_md_false << "/" << alice_n + << std::endl; + + StopChild(alice); + StopChild(bob); + return prod_ok ? 0 : 7; diff --git a/examples/aether_uap_1s_timing_characterization/main.cpp b/examples/aether_uap_1s_timing_characterization/main.cpp new file mode 100644 index 00000000..50337d52 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/main.cpp @@ -0,0 +1,255 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "client_role.h" +#include "coordinator.h" + +namespace { + +std::string_view ArgValue(int argc, char** argv, std::string_view key) { + for (int i = 1; i < argc; ++i) { + std::string_view a = argv[i]; + if (a == key && i + 1 < argc) { + return argv[i + 1]; + } + if (a.size() > key.size() && a.substr(0, key.size()) == key && + a[key.size()] == '=') { + return a.substr(key.size() + 1); + } + } + return {}; +} + +bool HasFlag(int argc, char** argv, std::string_view key) { + for (int i = 1; i < argc; ++i) { + if (key == argv[i]) { + return true; + } + } + return false; +} + +} // namespace + +int main(int argc, char** argv) { + using namespace ae::test_uap_ping_retry_window; + + if (HasFlag(argc, argv, "--help") || HasFlag(argc, argv, "-h")) { + std::cout + << "aether_uap_1s_timing_characterization\n" + "Count/seed/output:\n" + " --artifact-dir DIR | --output DIR\n" + " --seed N\n" + " --cycles N | --nominal-cycles N\n" + " --request-loss-cases N\n" + " --response-loss-cases N\n" + " --loss-cases N\n" + " --hard-stop-cases N | --hard-stop-runs N\n" + " --graceful-stop-cases N | --graceful-runs N\n" + " --window-samples-main N\n" + " --window-samples-extra N\n" + "Other:\n" + " --quick\n" + " --no-long-characterization\n" + " --phase-preservation\n" + " --phase-preservation-stress\n" + " --phase-preservation-budget-sec N\n" + " --first-request-loss-p99\n" + " --first-request-loss-cases N\n" + " --retry-count-zero-runtime\n" + " --retry-count-zero-cases N\n" + " --transport tcp|udp\n" + " --run-id ID\n" + " --exe PATH\n" + " --parent-uid UID\n" + " --ping-interval-ms N\n" + " --receive-window-ms N\n" + " --role coordinator|client\n"; + return 0; + } + + auto role = ArgValue(argc, argv, "--role"); + if (role.empty() || role == "coordinator") { + CharacterizationArgs args; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + if (args.artifact_dir.empty()) { + args.artifact_dir = std::string{ArgValue(argc, argv, "--output")}; + } + args.exe_path = std::string{ArgValue(argc, argv, "--exe")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + auto transport = ArgValue(argc, argv, "--transport"); + if (!transport.empty()) { + args.transport = std::string{transport}; + } + auto interval = ArgValue(argc, argv, "--ping-interval-ms"); + if (!interval.empty()) { + args.ping_interval_ms = std::strtoll(interval.data(), nullptr, 10); + } + auto window = ArgValue(argc, argv, "--receive-window-ms"); + if (!window.empty()) { + args.receive_window_ms = std::strtoll(window.data(), nullptr, 10); + } + auto seed = ArgValue(argc, argv, "--seed"); + if (!seed.empty()) { + args.seed = static_cast(std::strtoul(seed.data(), nullptr, 10)); + } + auto cycles = ArgValue(argc, argv, "--cycles"); + if (cycles.empty()) { + cycles = ArgValue(argc, argv, "--nominal-cycles"); + } + if (!cycles.empty()) { + args.logical_cycles = std::atoi(cycles.data()); + } + auto hard_stop = ArgValue(argc, argv, "--hard-stop-runs"); + if (hard_stop.empty()) { + hard_stop = ArgValue(argc, argv, "--hard-stop-cases"); + } + if (!hard_stop.empty()) { + args.hard_stop_runs = std::atoi(hard_stop.data()); + } + auto graceful = ArgValue(argc, argv, "--graceful-runs"); + if (graceful.empty()) { + graceful = ArgValue(argc, argv, "--graceful-stop-cases"); + } + if (!graceful.empty()) { + args.graceful_runs = std::atoi(graceful.data()); + } + auto main_n = ArgValue(argc, argv, "--window-samples-main"); + if (!main_n.empty()) { + args.window_samples_main = std::atoi(main_n.data()); + } + auto extra_n = ArgValue(argc, argv, "--window-samples-extra"); + if (!extra_n.empty()) { + args.window_samples_extra = std::atoi(extra_n.data()); + } + auto loss = ArgValue(argc, argv, "--loss-cases"); + if (!loss.empty()) { + args.loss_cases = std::atoi(loss.data()); + } + auto req_loss = ArgValue(argc, argv, "--request-loss-cases"); + if (!req_loss.empty()) { + args.request_loss_cases = std::atoi(req_loss.data()); + } + auto resp_loss = ArgValue(argc, argv, "--response-loss-cases"); + if (!resp_loss.empty()) { + args.response_loss_cases = std::atoi(resp_loss.data()); + } + args.quick = HasFlag(argc, argv, "--quick"); + args.phase_preservation = HasFlag(argc, argv, "--phase-preservation") || + HasFlag(argc, argv, "--phase-preservation-stress"); + args.phase_preservation_stress = + HasFlag(argc, argv, "--phase-preservation-stress"); + auto budget_sec = ArgValue(argc, argv, "--phase-preservation-budget-sec"); + if (!budget_sec.empty()) { + args.phase_preservation_budget_sec = std::atoi(budget_sec.data()); + if (args.phase_preservation_budget_sec > 0) { + args.phase_preservation = true; + } + } + args.first_request_loss_p99 = + HasFlag(argc, argv, "--first-request-loss-p99"); + auto frl_cases = ArgValue(argc, argv, "--first-request-loss-cases"); + if (!frl_cases.empty()) { + args.first_request_loss_cases = std::atoi(frl_cases.data()); + } + if (args.first_request_loss_p99 && args.first_request_loss_cases <= 0) { + args.first_request_loss_cases = 100; + } + args.retry_count_zero_runtime = + HasFlag(argc, argv, "--retry-count-zero-runtime"); + auto rcz_cases = ArgValue(argc, argv, "--retry-count-zero-cases"); + if (!rcz_cases.empty()) { + args.retry_count_zero_cases = std::atoi(rcz_cases.data()); + } + if (args.retry_count_zero_runtime && args.retry_count_zero_cases <= 0) { + args.retry_count_zero_cases = 10; + } + args.skip_long_characterization = + HasFlag(argc, argv, "--no-long-characterization") || args.quick || + args.phase_preservation || args.first_request_loss_p99 || + args.retry_count_zero_runtime; + if (args.quick) { + if (ArgValue(argc, argv, "--cycles").empty() && + ArgValue(argc, argv, "--nominal-cycles").empty()) { + args.logical_cycles = 10; + } + if (ArgValue(argc, argv, "--hard-stop-runs").empty() && + ArgValue(argc, argv, "--hard-stop-cases").empty()) { + args.hard_stop_runs = 3; + } + if (ArgValue(argc, argv, "--graceful-runs").empty() && + ArgValue(argc, argv, "--graceful-stop-cases").empty()) { + args.graceful_runs = 3; + } + if (ArgValue(argc, argv, "--window-samples-main").empty()) { + args.window_samples_main = 0; + } + if (ArgValue(argc, argv, "--window-samples-extra").empty()) { + args.window_samples_extra = 0; + } + if (ArgValue(argc, argv, "--loss-cases").empty() && + ArgValue(argc, argv, "--request-loss-cases").empty() && + ArgValue(argc, argv, "--response-loss-cases").empty()) { + args.loss_cases = 2; + } + } + return RunCharacterization(args); + } + + if (role == "client") { + ClientArgs args; + auto side = ArgValue(argc, argv, "--side"); + args.side = (side == "B" || side == "b") ? Side::kB : Side::kA; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.state_dir = std::string{ArgValue(argc, argv, "--state-dir")}; + args.pipe_name = std::string{ArgValue(argc, argv, "--pipe")}; + args.client_name = std::string{ArgValue(argc, argv, "--client-name")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + if (args.client_name.empty()) { + args.client_name = + args.side == Side::kA ? "uap-1s-alice" : "uap-1s-bob"; + } + auto ping_ms = ArgValue(argc, argv, "--ping-interval-ms"); + if (!ping_ms.empty()) { + args.ping_interval_ms = std::strtoll(ping_ms.data(), nullptr, 10); + } else { + args.ping_interval_ms = 1000; + } + auto rx_ms = ArgValue(argc, argv, "--receive-window-ms"); + if (!rx_ms.empty()) { + args.receive_window_ms = std::strtoll(rx_ms.data(), nullptr, 10); + } else { + args.receive_window_ms = 250; + } + return RunClientRole(args); + } + + std::cerr << "Unknown --role\n"; + return 2; +} diff --git a/examples/aether_uap_1s_timing_characterization/phase_preservation.inc.cpp b/examples/aether_uap_1s_timing_characterization/phase_preservation.inc.cpp new file mode 100644 index 00000000..581fa689 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/phase_preservation.inc.cpp @@ -0,0 +1,1532 @@ + // Phase-preservation mode. Included inside RunCharacterization after + // Alice/Bob spawn, warmup, and wait_ping/arm_next lambdas. + // Terminology mapping: + // Peer/Bob = child B (PingCloudServers / ApplyLogicalPingAttempt) + // Observer/Alice = child A (Client::QueryPeerReceiveSchedule) + // Server = work-cloud AuthorizedApi::ping destination + // Nominal ping = attempt_index 1 of a logical cycle (cycle_anchor = Tn) + // Retry ping = attempt_index >= 2 + // Original deadline Tn = PingTraceEvent.cycle_anchor / nominal_ping_at + // Corrected window = required_rx_until / effective_wire_rx_window after retry + // Nominal phase = T0 + n * 1000ms, never reset from retry completion + struct PhaseStep { + const char* sequence; + const char* fault_type; + int fault_mode; + std::int64_t server_offset_us; + bool use_hold; + bool checkpoints; + }; + struct ObserverHit { + int checkpoint{0}; + std::int64_t query_steady_us{0}; + std::int64_t query_qpc{0}; + std::int64_t rel_deadline_us{0}; + std::int64_t state{-2}; + std::int64_t next_us{0}; + std::int64_t last_online_us{0}; + std::int64_t next_ping_delta_ms{std::numeric_limits::min()}; + std::int64_t last_connect_delta_ms{ + std::numeric_limits::min()}; + std::int64_t expected_state{0}; + bool mismatch{false}; + }; + struct PhaseCycle { + int index{0}; + const char* sequence{"?"}; + const char* fault_type{"none"}; + int fault_mode{0}; + std::int64_t logical_ping_id{0}; + std::int64_t seed{0}; + std::int64_t phase_anchor_us{0}; + std::int64_t expected_nominal_us{0}; + std::int64_t scheduled_nominal_us{0}; + std::int64_t original_deadline_us{0}; + std::int64_t original_window_start_us{0}; + std::int64_t original_window_end_us{0}; + std::int64_t original_window_dur_us{0}; + std::int64_t computed_guard_us{0}; + std::int64_t scheduled_first_us{0}; + std::int64_t actual_first_send_us{0}; + std::int64_t first_attempt_qpc{0}; + std::int64_t first_request_sent{-1}; + std::int64_t first_server_receive_us{ + std::numeric_limits::min()}; + std::int64_t retry_decision_us{std::numeric_limits::min()}; + std::int64_t retry_scheduled_us{std::numeric_limits::min()}; + std::int64_t retry_send_us{std::numeric_limits::min()}; + std::int64_t retry_qpc{0}; + std::int64_t retry_request_sent{-1}; + std::int64_t retry_server_receive_mapped_us{ + std::numeric_limits::min()}; + std::int64_t retry_server_receive_observer_us{ + std::numeric_limits::min()}; + std::int64_t original_deadline_qpc{0}; + std::int64_t retry_mapped_qpc{0}; + double retry_server_margin_ms{ + std::numeric_limits::quiet_NaN()}; + bool retry_before_deadline{false}; + bool retry_after_deadline{false}; + bool no_retry{false}; + std::int64_t corrected_cur_start_us{0}; + std::int64_t corrected_cur_end_us{0}; + std::int64_t corrected_cur_dur_us{0}; + std::int64_t corrected_next_start_us{0}; + std::int64_t next_scheduled_nominal_us{0}; + std::int64_t accepted_attempt{1}; + int confirms{0}; + bool confirmed{false}; + std::vector observers; + std::vector failures; + }; + struct FailedCase { + PhaseCycle c; + std::string invariant; + }; + + auto abs_i64 = [](std::int64_t v) -> std::int64_t { + return v < 0 ? -v : v; + }; + auto const kMissing = std::numeric_limits::min(); + auto csv_i = [&](std::int64_t v) -> std::string { + if (v == kMissing) { + return {}; + } + return std::to_string(v); + }; + auto csv_d = [&](double v) -> std::string { + if (!std::isfinite(v)) { + return {}; + } + return F3(v); + }; + auto json_i = [&](std::int64_t v) -> std::string { + if (v == kMissing) { + return "null"; + } + return std::to_string(v); + }; + auto json_d = [&](double v) -> std::string { + if (!std::isfinite(v)) { + return "null"; + } + return F3(v); + }; + + LARGE_INTEGER qfreq{}; + QueryPerformanceFrequency(&qfreq); + double const qpc_per_us = + static_cast(qfreq.QuadPart) / 1000000.0; + auto map_us_to_qpc = [&](std::int64_t event_qpc, std::int64_t event_us, + std::int64_t target_us) -> std::int64_t { + return event_qpc + static_cast( + static_cast(target_us - event_us) * + qpc_per_us); + }; + std::int64_t const one_way_us = + (warmup_min > 0 ? warmup_min : 100) * 1000 / 2; + std::int64_t const period_us = args.ping_interval_ms * 1000; + auto const tick_us = static_cast(1000); // 1ms scheduler tick + + auto wait_sched = [&](DWORD timeout_ms) -> std::optional { + auto const before = alice.schedules.size(); + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + if (alice.schedules.size() > before) { + return alice.schedules.back(); + } + } + return std::nullopt; + }; + using EvId = std::tuple; + std::set seen_ev; + auto ev_id = [](BobPingEvent const& e) { + return EvId{e.event_qpc, e.kind, e.logical_cycle_id, + e.physical_attempt_index, e.server_id}; + }; + for (auto const& e : bob.ping_events) { + seen_ev.insert(ev_id(e)); + } + auto take_ev = [&](BobPingEvent const& e) { + seen_ev.insert(ev_id(e)); + return e; + }; + auto dest_ok = [&](BobPingEvent const& e) { + return dest == 0 || e.server_id == dest; + }; + auto find_ev = [&](std::uint8_t kind, DWORD timeout_ms, + std::int64_t cycle_id, std::int64_t min_attempt) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.kind != kind) { + continue; + } + if (cycle_id != 0 && e.logical_cycle_id != 0 && + e.logical_cycle_id != cycle_id) { + continue; + } + if (min_attempt > 0 && e.physical_attempt_index < min_attempt) { + continue; + } + return take_ev(e); + } + } + return std::nullopt; + }; + auto wait_ev = find_ev; + auto wait_first_attempt = [&](DWORD timeout_ms) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.physical_attempt_index > 1) { + continue; + } + if (e.kind == static_cast(PingTraceKind::kRequestSent) || + e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + return take_ev(e); + } + } + } + return std::nullopt; + }; + auto query_ckpt = [&](int ckpt, bool force) -> std::optional { + SendRaw(alice, kIpcQueryNow, 0, ckpt, 0, force ? 1 : 0); + return wait_sched(800); + }; + auto rel_deadline_us = [&](ScheduleSnap const& q, + PhaseCycle const& rec) -> std::int64_t { + if (rec.original_deadline_qpc == 0 || q.qpc == 0) { + return 0; + } + return static_cast( + static_cast(q.qpc - rec.original_deadline_qpc) / qpc_per_us); + }; + auto confirm_count = [&](std::int64_t cycle_id) { + int n = 0; + for (auto const& e : bob.ping_events) { + if (!dest_ok(e)) { + continue; + } + if (e.kind == + static_cast(PingTraceKind::kCycleConfirmed) && + e.logical_cycle_id == cycle_id) { + ++n; + } + } + return n; + }; + auto apply_retry = [&](PhaseCycle& rec, BobPingEvent const& retry) { + rec.retry_send_us = + retry.actual_us != 0 ? retry.actual_us : retry.event_steady_us; + rec.retry_qpc = retry.event_qpc; + rec.retry_request_sent = retry.request_was_sent; + rec.accepted_attempt = retry.physical_attempt_index; + rec.corrected_cur_start_us = retry.actual_us; + rec.corrected_cur_end_us = retry.required_until_us; + rec.corrected_cur_dur_us = retry.effective_window_us; + rec.corrected_next_start_us = retry.next_local_send_us; + rec.next_scheduled_nominal_us = retry.contract_deadline_us; + rec.retry_mapped_qpc = + retry.event_qpc + + static_cast(static_cast(one_way_us) * + qpc_per_us); + rec.retry_server_receive_mapped_us = rec.retry_send_us + one_way_us; + if (rec.retry_mapped_qpc < rec.original_deadline_qpc) { + rec.retry_before_deadline = true; + rec.retry_after_deadline = false; + rec.retry_server_margin_ms = QpcToMs(static_cast( + rec.original_deadline_qpc - rec.retry_mapped_qpc)); + } else { + rec.retry_before_deadline = false; + rec.retry_after_deadline = true; + rec.retry_server_margin_ms = -QpcToMs(static_cast( + rec.retry_mapped_qpc - rec.original_deadline_qpc)); + } + }; + + std::vector steps; + auto add_n = [&](char const* seq, char const* ft, int mode, int n, + bool ckpt) { + for (int i = 0; i < n; ++i) { + steps.push_back(PhaseStep{seq, ft, mode, 0, false, ckpt}); + } + }; + bool const stress = args.phase_preservation_stress; + int const budget_sec = args.phase_preservation_budget_sec; + bool const timed_shard = budget_sec > 0; + ULONGLONG const shard_start_tick = GetTickCount64(); + ULONGLONG const shard_budget_ms = + timed_shard ? static_cast(budget_sec) * 1000ULL : 0; + // Reserve ~120s of a timed shard for in-flight/hard-stop/graceful controls. + ULONGLONG const main_budget_ms = + timed_shard && shard_budget_ms > 120000ULL ? shard_budget_ms - 120000ULL + : shard_budget_ms; + auto budget_remaining = [&]() -> bool { + if (!timed_shard) { + return true; + } + return (GetTickCount64() - shard_start_tick) < main_budget_ms; + }; + auto shard_time_remaining = [&]() -> bool { + if (!timed_shard) { + return true; + } + return (GetTickCount64() - shard_start_tick) < shard_budget_ms; + }; + if (timed_shard) { + // Approximate wall-time mix for long characterization shards. + // Sparse checkpoints: QueryNow RTT must not dominate the 1s loop. + auto sparse = [](int i) { return (i % 8) == 0; }; + add_n("baseline", "none", 0, 250, false); + for (int i = 0; i < 250; ++i) { + steps.push_back(PhaseStep{"request-loss", "request-loss", 1, 0, false, + sparse(i)}); + } + for (int i = 0; i < 200; ++i) { + steps.push_back(PhaseStep{"response-loss", "response-loss", 2, 0, false, + sparse(i)}); + } + for (int n : {2, 5, 10}) { + add_n("consecutive-request-loss", "request-loss", 1, n, sparse(0)); + add_n("consecutive-response-loss", "response-loss", 2, n, sparse(0)); + } + for (int n : {10, 20, 50}) { + for (int i = 0; i < n; ++i) { + bool const req = (i % 2) == 0; + steps.push_back(PhaseStep{ + "alternating", req ? "request-loss" : "response-loss", + req ? 1 : 2, 0, false, sparse(i)}); + } + } + for (int every : {2, 3, 5, 10}) { + for (int i = 0; i < every * 20; ++i) { + bool const loss = (i % every) == 0; + char const* ft = loss ? ((every % 2) == 0 ? "request-loss" + : "response-loss") + : "none"; + int mode = loss ? ((every % 2) == 0 ? 1 : 2) : 0; + steps.push_back(PhaseStep{"periodic", ft, mode, 0, false, + loss && sparse(i)}); + } + } + for (int pct : {1, 5, 10, 20}) { + std::uint32_t rr = args.seed ^ static_cast(pct * 0x9e37); + for (int i = 0; i < 200; ++i) { + rr = rr * 1664525u + 1013904223u; + bool const loss = (rr % 100u) < static_cast(pct); + bool const req = (rr & 1u) == 0; + char const* ft = + !loss ? "none" : (req ? "request-loss" : "response-loss"); + int mode = !loss ? 0 : (req ? 1 : 2); + steps.push_back(PhaseStep{"random", ft, mode, 0, false, + loss && sparse(i)}); + } + } + // Repeat the block so wall-clock budget, not step count, ends the shard. + auto const block = steps; + while (steps.size() < 20000) { + steps.insert(steps.end(), block.begin(), block.end()); + } + } else { + int const n_base = stress ? 2000 : 100; + int const n_req = stress ? 500 : 30; + int const n_resp = stress ? 500 : 30; + add_n("baseline", "none", 0, n_base, false); + add_n("request-loss", "request-loss", 1, n_req, true); + add_n("response-loss", "response-loss", 2, n_resp, true); + if (!stress) { + std::int64_t const before[] = {-100000, -50000, -20000, -10000, -5000, + -2000, -1000}; + std::int64_t const after[] = {1000, 2000, 5000, 10000}; + for (auto off : before) { + steps.push_back(PhaseStep{"boundary-before", "request-loss", 1, off, + true, true}); + } + for (auto off : after) { + steps.push_back(PhaseStep{"boundary-after", "request-loss", 1, off, + true, true}); + } + add_n("consecutive-request-loss", "request-loss", 1, 10, true); + add_n("consecutive-response-loss", "response-loss", 2, 10, true); + for (int i = 0; i < 20; ++i) { + steps.push_back( + PhaseStep{"alternating", + (i % 2) == 0 ? "request-loss" : "response-loss", + (i % 2) == 0 ? 1 : 2, 0, false, true}); + } + for (int i = 0; i < 20; ++i) { + bool const loss = (i % 2) == 0; + steps.push_back(PhaseStep{"every-2", loss ? "request-loss" : "none", + loss ? 1 : 0, 0, false, loss}); + } + for (int i = 0; i < 25; ++i) { + bool const loss = (i % 5) == 0; + steps.push_back(PhaseStep{"every-5", loss ? "response-loss" : "none", + loss ? 2 : 0, 0, false, loss}); + } + } else { + for (int s = 0; s < 100; ++s) { + steps.push_back(PhaseStep{"burst-request", "request-loss", 1, 0, + false, s % 10 == 0}); + steps.push_back(PhaseStep{"burst-response", "response-loss", 2, 0, + false, s % 10 == 0}); + } + std::uint32_t rr = args.seed ^ 0x9e3779b9u; + for (int i = 0; i < 1000; ++i) { + rr = rr * 1664525u + 1013904223u; + int mode = 0; + char const* ft = "none"; + if ((rr % 5u) == 0) { + mode = 1; + ft = "request-loss"; + } else if ((rr % 5u) == 1) { + mode = 2; + ft = "response-loss"; + } + steps.push_back(PhaseStep{"random-1000", ft, mode, 0, false, false}); + } + } + } + + std::filesystem::create_directories(args.artifact_dir); + std::ofstream jsonl(std::filesystem::path{args.artifact_dir} / + "samples.jsonl"); + std::ofstream samples_csv(std::filesystem::path{args.artifact_dir} / + "samples.csv"); + std::ofstream phase_csv(std::filesystem::path{args.artifact_dir} / + "phase-error-by-cycle.csv"); + std::ofstream win_csv(std::filesystem::path{args.artifact_dir} / + "window-corrections.csv"); + std::ofstream obs_csv(std::filesystem::path{args.artifact_dir} / + "observer-query-results.csv"); + if (!jsonl || !samples_csv || !phase_csv || !win_csv || !obs_csv) { + std::cerr << "FAIL cannot open phase-preservation csv/jsonl outputs\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + samples_csv + << "run_id,transport,shard,seed,cycle_index,logical_ping_id,sequence," + "fault_type,fault_armed,fault_consumed,attempt_number,Tn,Tn_plus_1," + "scheduled_nominal,scheduled_phase_error_ms,guard_ms," + "attempt_lead_ms,first_attempt_scheduled,first_attempt_actual_send," + "first_attempt_offset_from_Tn_ms,first_request_sent," + "estimated_first_server_receive,attempt_timeout,retry_decision," + "retry_actual_send,timeout_to_retry_decision_ms," + "timeout_to_retry_send_ms,first_attempt_to_retry_ms," + "retry_send_offset_from_Tn_ms,retry_client_margin_to_Tn_ms," + "estimated_server_receive,estimated_server_margin_ms," + "one_way_estimate_us,current_window_before,current_window_after," + "next_scheduled_nominal,next_nominal_phase_error_ms," + "alice_state,alice_next_ping_delta_ms,duplicate_count," + "cycle_confirmed,failure_class\n"; + phase_csv << "cycle_index,sequence,fault_type,expected_nominal_us," + "scheduled_nominal_us,scheduled_phase_error_us,actual_first_" + "send_us,actual_send_phase_error_us,actual_interval_error_us," + "contiguous_cycle,skipped_slots,next_window_phase_error_us\n"; + win_csv << "cycle_index,sequence,fault_type,original_window_start_us," + "original_window_end_us,original_window_duration_us,corrected_" + "current_window_start_us,corrected_current_window_end_us," + "corrected_current_window_duration_us,current_window_start_" + "delta_ms,current_window_end_delta_ms,current_window_duration_" + "delta_ms,next_window_start_delta_ms,next_nominal_phase_delta_" + "ms\n"; + obs_csv << "cycle_index,sequence,fault_type,checkpoint,query_time_us," + "rel_deadline_us,state,expected_state,next_us,last_online_us," + "next_ping_delta_ms,last_connect_delta_ms,mismatch," + "alice_query_rtt_ms\n"; + + std::ofstream meta_json(std::filesystem::path{args.artifact_dir} / + "shard-meta.json"); + if (meta_json) { + meta_json << "{\"run_id\":\"" << args.run_id << "\",\"transport\":\"" + << args.transport << "\",\"seed\":" << args.seed + << ",\"budget_sec\":" << budget_sec + << ",\"one_way_estimate_us\":" << one_way_us + << ",\"warmup_min_rtt_ms\":" << warmup_min + << ",\"warmup_p99_rtt_ms\":" << warmup_p99 << "}\n"; + meta_json.flush(); + } + + std::vector cycles; + std::vector failed; + std::int64_t phase_anchor_us = 0; + std::int64_t prev_first_send_us = 0; + std::int64_t prev_tn1_us = 0; + std::int64_t prev_scheduled_us = 0; + int live_false_md = 0; + int live_false_unknown = 0; + int duplicate_logical = 0; + int query_failures = 0; + bool ok_phase = true; + int graceful_hit = 0; + int hard_hit = 0; + + std::cout << "PHASE_PRESERVATION mode=" + << (timed_shard ? "budget-shard" + : (stress ? "stress" : "fast")) + << " steps=" << steps.size() << " seed=" << args.seed + << " budget_sec=" << budget_sec << std::endl; + wait_window_closed(8000); + + auto fail_inv = [&](PhaseCycle& rec, char const* inv) { + rec.failures.push_back(inv); + FailedCase fc{}; + fc.c = rec; + fc.invariant = inv; + failed.push_back(fc); + ok_phase = false; + std::cerr << "FAIL cycle " << rec.index << " " << rec.sequence << " " + << rec.fault_type << ": " << inv << std::endl; + }; + + int armed_si = -1; + auto arm_step = [&](PhaseStep const& step, int index) { + if (step.use_hold) { + auto const send_hold_us = step.server_offset_us - one_way_us; + SendRaw(bob, kIpcArmFault, 0, dest, 1, step.fault_mode, 20000, 0, 0); + drain(40); + SendRaw(bob, kIpcArmFault, 0, dest, 2, 0, 0, 1, 0, send_hold_us, 1); + drain(40); + } else if (step.fault_mode != 0) { + arm_next(step.fault_mode, 1, 0, 0); + } else { + SendRaw(bob, kIpcArmFault, 0, dest, 1, 0, 0, 0, 0); + drain(20); + } + armed_si = index; + }; + + for (int si = 0; si < static_cast(steps.size()); ++si) { + if (!budget_remaining()) { + std::cout << "PHASE_PRESERVATION budget exhausted after " << si + << " steps elapsed_ms=" + << (GetTickCount64() - shard_start_tick) << std::endl; + break; + } + auto const& st = steps[static_cast(si)]; + PhaseCycle rec{}; + rec.index = si; + rec.sequence = st.sequence; + rec.fault_type = st.fault_type; + rec.fault_mode = st.fault_mode; + rec.seed = args.seed; + std::optional started; + int const arm_tries = st.use_hold ? 1 : 4; + for (int arm_try = 0; arm_try < arm_tries; ++arm_try) { + if (armed_si != si || arm_try > 0) { + arm_step(st, si); + } + started = wait_first_attempt(8000); + if (!started) { + break; + } + if (st.use_hold) { + break; + } + bool const drop_missed = + st.fault_mode == 1 && started->request_was_sent == 1; + bool const ignore_missed = + st.fault_mode == 2 && started->request_was_sent == 0; + bool ignore_not_applied = false; + if (st.fault_mode == 2 && started->request_was_sent == 1) { + auto to = find_ev( + static_cast(PingTraceKind::kAttemptTimeout), 400, + started->logical_cycle_id, 0); + ignore_not_applied = !to; + if (to) { + rec.retry_decision_us = to->event_steady_us; + } + } + if (!drop_missed && !ignore_missed && !ignore_not_applied) { + break; + } + (void)find_ev(static_cast(PingTraceKind::kCycleConfirmed), + 2000, started->logical_cycle_id, 0); + armed_si = -1; + if (arm_try + 1 < arm_tries) { + started.reset(); + } + } + if (!started) { + fail_inv(rec, "reporting/harness failure: no cycle start trace"); + cycles.push_back(std::move(rec)); + continue; + } + rec.logical_ping_id = started->logical_cycle_id; + rec.scheduled_nominal_us = started->cycle_anchor_us; + rec.original_deadline_us = started->cycle_anchor_us; + rec.next_scheduled_nominal_us = started->contract_deadline_us; + rec.actual_first_send_us = started->actual_us != 0 ? started->actual_us + : started->event_steady_us; + rec.first_attempt_qpc = started->event_qpc; + rec.scheduled_first_us = started->planned_us; + rec.computed_guard_us = started->guard_us; + rec.original_window_dur_us = started->base_window_us; + rec.original_window_start_us = started->cycle_anchor_us; + rec.original_window_end_us = + started->cycle_anchor_us + started->base_window_us; + rec.first_request_sent = started->request_was_sent; + rec.corrected_cur_start_us = started->actual_us; + rec.corrected_cur_end_us = started->required_until_us; + rec.corrected_cur_dur_us = started->effective_window_us; + rec.corrected_next_start_us = started->next_local_send_us; + if (phase_anchor_us == 0) { + phase_anchor_us = rec.scheduled_nominal_us; + } + rec.phase_anchor_us = phase_anchor_us; + { + auto const delta = rec.scheduled_nominal_us - phase_anchor_us; + auto n = delta / period_us; + auto rem = delta % period_us; + if (rem < 0) { + rem += period_us; + n -= 1; + } + if (rem > period_us / 2) { + n += 1; + } + rec.expected_nominal_us = phase_anchor_us + n * period_us; + } + rec.original_deadline_qpc = + map_us_to_qpc(started->event_qpc, rec.actual_first_send_us, + rec.original_deadline_us); + + if (st.checkpoints) { + auto q1 = query_ckpt(1, false); + if (!q1) { + ++query_failures; + fail_inv(rec, "observer checkpoint 1: QueryPeerReceiveSchedule did not complete"); + } else { + ObserverHit h{}; + h.checkpoint = 1; + h.query_steady_us = q1->steady_us; + h.query_qpc = q1->qpc; + h.state = q1->state; + h.next_us = q1->next_us; + h.last_online_us = q1->last_online_us; + h.next_ping_delta_ms = q1->next_ping_delta_ms; + h.last_connect_delta_ms = q1->last_connect_delta_ms; + h.rel_deadline_us = rel_deadline_us(*q1, rec); + h.expected_state = 0; + if (q1->state == 1) { + ++live_false_md; + h.mismatch = true; + fail_inv(rec, "observer saw false MissedDeadline before first attempt"); + } else if (q1->state == 2) { + ++live_false_unknown; + h.mismatch = true; + fail_inv(rec, "observer saw false Unknown before first attempt"); + } else if (q1->state != 0) { + h.mismatch = true; + fail_inv(rec, "observer saw the wrong state at checkpoint 1"); + } + rec.observers.push_back(h); + } + } + + if (st.fault_mode == 1 && rec.first_request_sent == 1) { + fail_inv(rec, "request-loss first request was sent to the server"); + } + if (st.fault_mode == 2 && rec.first_request_sent == 0) { + fail_inv(rec, "response-loss first request did not reach send"); + } + if (rec.first_request_sent == 1) { + rec.first_server_receive_us = rec.actual_first_send_us + one_way_us; + } + + std::optional confirmed; + if (st.fault_mode != 0) { + bool const expect_retry = rec.first_request_sent == + (st.fault_mode == 1 ? 0 : 1); + DWORD const retry_wait_ms = + st.use_hold ? 6000 : (expect_retry ? 1500 : 200); + if (rec.retry_decision_us == kMissing) { + auto timeout_ev = find_ev( + static_cast(PingTraceKind::kAttemptTimeout), + retry_wait_ms, rec.logical_ping_id, 0); + if (timeout_ev) { + rec.retry_decision_us = timeout_ev->event_steady_us; + } + } + if (rec.retry_decision_us != kMissing) { + auto retry_sched = find_ev( + static_cast(PingTraceKind::kRetryScheduled), 200, + rec.logical_ping_id, 0); + if (retry_sched) { + rec.retry_scheduled_us = retry_sched->event_steady_us; + } + if (st.checkpoints) { + auto q2 = query_ckpt(2, false); + if (q2) { + ObserverHit h{}; + h.checkpoint = 2; + h.query_steady_us = q2->steady_us; + h.query_qpc = q2->qpc; + h.state = q2->state; + h.next_us = q2->next_us; + h.last_online_us = q2->last_online_us; + h.next_ping_delta_ms = q2->next_ping_delta_ms; + h.last_connect_delta_ms = q2->last_connect_delta_ms; + h.rel_deadline_us = rel_deadline_us(*q2, rec); + h.expected_state = 0; + if (q2->state == 1) { + ++live_false_md; + h.mismatch = true; + fail_inv(rec, + "observer saw false MissedDeadline after loss before retry"); + } + rec.observers.push_back(h); + } else { + ++query_failures; + } + } + } + auto retry = find_ev( + static_cast(PingTraceKind::kRequestSent), + st.use_hold ? 5000 : 800, rec.logical_ping_id, 2); + if (!retry) { + retry = find_ev( + static_cast(PingTraceKind::kRequestDropped), 200, + rec.logical_ping_id, 2); + } + if (!retry) { + rec.no_retry = true; + fail_inv(rec, "retry did not reach server"); + } else { + apply_retry(rec, *retry); + if (retry->request_was_sent == 0) { + rec.no_retry = true; + fail_inv(rec, "retry did not reach server"); + } + } + if (si + 1 < static_cast(steps.size())) { + arm_step(steps[static_cast(si + 1)], si + 1); + } + if (st.checkpoints && !rec.no_retry) { + auto q3 = query_ckpt(3, false); + if (q3) { + ObserverHit h{}; + h.checkpoint = 3; + h.query_steady_us = q3->steady_us; + h.query_qpc = q3->qpc; + h.state = q3->state; + h.next_us = q3->next_us; + h.last_online_us = q3->last_online_us; + h.next_ping_delta_ms = q3->next_ping_delta_ms; + h.last_connect_delta_ms = q3->last_connect_delta_ms; + h.rel_deadline_us = rel_deadline_us(*q3, rec); + h.expected_state = 0; + rec.retry_server_receive_observer_us = q3->last_online_us; + if (q3->state == 1 && rec.retry_before_deadline) { + ++live_false_md; + h.mismatch = true; + fail_inv(rec, "observer saw false MissedDeadline after retry reached server before original deadline"); + } + rec.observers.push_back(h); + } + if (rec.original_deadline_qpc > 0) { + auto const now_qpc = QpcNow(); + if (now_qpc < rec.original_deadline_qpc) { + auto const wait_ms = QpcToMs(static_cast( + rec.original_deadline_qpc - now_qpc)); + if (wait_ms > 0 && wait_ms < 800) { + Sleep(static_cast(wait_ms + 2)); + } + } + } + auto q4 = query_ckpt(4, false); + if (q4) { + ObserverHit h{}; + h.checkpoint = 4; + h.query_steady_us = q4->steady_us; + h.query_qpc = q4->qpc; + h.state = q4->state; + h.next_us = q4->next_us; + h.last_online_us = q4->last_online_us; + h.next_ping_delta_ms = q4->next_ping_delta_ms; + h.last_connect_delta_ms = q4->last_connect_delta_ms; + h.rel_deadline_us = rel_deadline_us(*q4, rec); + h.expected_state = rec.retry_after_deadline ? q4->state : 0; + if (rec.retry_before_deadline && q4->state == 1) { + ++live_false_md; + h.mismatch = true; + fail_inv(rec, "observer saw false MissedDeadline after original deadline following before-deadline retry"); + } + rec.observers.push_back(h); + } + } + } else if (si + 1 < static_cast(steps.size())) { + arm_step(steps[static_cast(si + 1)], si + 1); + } + + if (!confirmed) { + confirmed = wait_ev( + static_cast(PingTraceKind::kCycleConfirmed), 8000, + rec.logical_ping_id, 0); + } + if (confirmed) { + rec.confirmed = true; + rec.confirms = 1; + } else { + fail_inv(rec, "reporting/harness failure: cycle not confirmed"); + } + rec.confirms = confirm_count(rec.logical_ping_id); + if (rec.confirms > 1) { + duplicate_logical += rec.confirms - 1; + fail_inv(rec, "duplicate logical ping / duplicate CycleConfirmed"); + } + + if (st.use_hold && rec.retry_before_deadline == false && + st.server_offset_us < 0 && !rec.no_retry) { + fail_inv(rec, "retry reached server after original deadline"); + } + if (st.use_hold && rec.retry_after_deadline == false && + st.server_offset_us > 0 && !rec.no_retry) { + fail_inv(rec, "retry reached server after original deadline was expected but classified as before-deadline"); + } + if (!st.use_hold && st.fault_mode != 0 && rec.retry_after_deadline) { + fail_inv(rec, "retry reached server after original deadline"); + } + + auto const sched_err = + rec.scheduled_nominal_us - rec.expected_nominal_us; + if (abs_i64(sched_err) > tick_us) { + fail_inv(rec, "next nominal schedule shifted"); + } + if (prev_tn1_us != 0 && rec.scheduled_nominal_us != 0 && + abs_i64(rec.scheduled_nominal_us - prev_scheduled_us - period_us) <= + tick_us && + abs_i64(rec.scheduled_nominal_us - prev_tn1_us) > tick_us) { + fail_inv(rec, "next nominal schedule shifted"); + } + if (st.checkpoints) { + auto q5 = query_ckpt(5, false); + if (q5) { + ObserverHit h{}; + h.checkpoint = 5; + h.query_steady_us = q5->steady_us; + h.query_qpc = q5->qpc; + h.state = q5->state; + h.next_us = q5->next_us; + h.last_online_us = q5->last_online_us; + h.next_ping_delta_ms = q5->next_ping_delta_ms; + h.last_connect_delta_ms = q5->last_connect_delta_ms; + h.rel_deadline_us = rel_deadline_us(*q5, rec); + h.expected_state = 0; + if (q5->state == 1) { + ++live_false_md; + h.mismatch = true; + fail_inv(rec, "observer saw false MissedDeadline after the next nominal ping"); + } else if (q5->state == 2) { + ++live_false_unknown; + h.mismatch = true; + fail_inv(rec, "observer saw false Unknown after the next nominal ping"); + } + rec.observers.push_back(h); + } + } + + std::int64_t interval_err = kMissing; + int contiguous = 0; + std::int64_t skipped_slots = kMissing; + if (prev_first_send_us != 0 && rec.actual_first_send_us != 0) { + auto const raw_interval = + rec.actual_first_send_us - prev_first_send_us; + auto slots = raw_interval / period_us; + auto rem = raw_interval % period_us; + if (rem < 0) { + rem += period_us; + slots -= 1; + } + if (rem > period_us / 2) { + slots += 1; + } + if (slots <= 1) { + contiguous = 1; + interval_err = raw_interval - period_us; + skipped_slots = 0; + } else { + contiguous = 0; + interval_err = raw_interval - period_us; + skipped_slots = slots - 1; + } + } + auto const send_phase_err = + rec.actual_first_send_us - rec.expected_nominal_us; + auto const next_phase_err = + rec.next_scheduled_nominal_us - + (rec.expected_nominal_us + period_us); + auto const sched_err_ms = sched_err / 1000.0; + auto const next_phase_err_ms = next_phase_err / 1000.0; + auto const first_off_ms = + (rec.actual_first_send_us - rec.scheduled_nominal_us) / 1000.0; + auto const attempt_lead_ms = + started->attempt_lead_us > 0 + ? started->attempt_lead_us / 1000.0 + : (rec.scheduled_nominal_us - rec.scheduled_first_us) / 1000.0; + auto const guard_ms = rec.computed_guard_us / 1000.0; + int fault_armed = st.fault_mode != 0 ? 1 : 0; + int fault_consumed = 0; + if (st.fault_mode == 1) { + fault_consumed = rec.first_request_sent == 0 ? 1 : 0; + } else if (st.fault_mode == 2) { + fault_consumed = rec.first_request_sent == 1 ? 1 : 0; + } + std::int64_t timeout_to_decision = kMissing; + std::int64_t timeout_to_retry = kMissing; + std::int64_t first_to_retry = kMissing; + double retry_send_off_ms = std::numeric_limits::quiet_NaN(); + double retry_client_margin_ms = std::numeric_limits::quiet_NaN(); + if (rec.retry_decision_us != kMissing && + rec.retry_send_us != kMissing) { + // retry_decision is attempt timeout time when available + } + if (rec.retry_decision_us != kMissing) { + timeout_to_decision = 0; + } + if (rec.retry_decision_us != kMissing && + rec.retry_send_us != kMissing) { + timeout_to_retry = rec.retry_send_us - rec.retry_decision_us; + } + if (rec.retry_send_us != kMissing && rec.actual_first_send_us != 0) { + first_to_retry = rec.retry_send_us - rec.actual_first_send_us; + retry_send_off_ms = + (rec.retry_send_us - rec.scheduled_nominal_us) / 1000.0; + retry_client_margin_ms = + (rec.scheduled_nominal_us - rec.retry_send_us) / 1000.0; + } + std::string failure_class; + if (!rec.failures.empty()) { + auto const& inv = rec.failures.front(); + if (inv.find("next nominal schedule shifted") != std::string::npos) { + failure_class = "PRODUCTION_PHASE"; + } else if (inv.find("retry reached server after") != std::string::npos) { + failure_class = "PRODUCTION_RETRY_ESTIMATED_LATE_ARRIVAL"; + } else if (inv.find("first request was sent") != std::string::npos || + inv.find("did not reach send") != std::string::npos) { + failure_class = "HARNESS_FAULT_NOT_ARMED"; + } else if (inv.find("cycle not confirmed") != std::string::npos || + inv.find("no cycle start") != std::string::npos) { + failure_class = "HARNESS_CYCLE_CONFIRM"; + } else if (inv.find("QueryPeerReceiveSchedule") != std::string::npos || + inv.find("observer") != std::string::npos) { + failure_class = "HARNESS_QUERY"; + } else if (inv.find("retry did not reach") != std::string::npos) { + failure_class = "HARNESS_FAULT_WRONG_CYCLE"; + } else { + failure_class = "OTHER"; + } + } + std::int64_t alice_state = kMissing; + std::int64_t alice_delta = kMissing; + if (!rec.observers.empty()) { + alice_state = rec.observers.back().state; + alice_delta = rec.observers.back().next_ping_delta_ms; + } + + jsonl << "{\"run_id\":\"" << args.run_id << "\",\"transport\":\"" + << args.transport << "\",\"seed\":" << args.seed + << ",\"cycle_index\":" << si << ",\"logical_ping_id\":" + << rec.logical_ping_id << ",\"attempt_number\":" + << rec.accepted_attempt << ",\"fault_type\":\"" << rec.fault_type + << "\",\"fault_armed\":" << fault_armed + << ",\"fault_consumed\":" << fault_consumed + << ",\"phase_anchor\":" << rec.phase_anchor_us + << ",\"Tn\":" << rec.scheduled_nominal_us + << ",\"Tn_plus_1\":" << (rec.scheduled_nominal_us + period_us) + << ",\"expected_nominal_time\":" << rec.expected_nominal_us + << ",\"original_deadline\":" << rec.original_deadline_us + << ",\"actual_first_attempt_send_time\":" << rec.actual_first_send_us + << ",\"retry_actual_send_time\":" << json_i(rec.retry_send_us) + << ",\"estimated_server_receive\":" + << json_i(rec.retry_server_receive_mapped_us) + << ",\"estimated_server_margin_ms\":" + << json_d(rec.retry_server_margin_ms) + << ",\"one_way_estimate_us\":" << one_way_us + << ",\"next_scheduled_nominal_time\":" + << rec.next_scheduled_nominal_us + << ",\"scheduled_phase_error_ms\":" << json_d(sched_err_ms) + << ",\"next_nominal_phase_error_ms\":" << json_d(next_phase_err_ms) + << ",\"contiguous_cycle\":" << contiguous + << ",\"skipped_slots\":" << json_i(skipped_slots) + << ",\"failure_class\":" + << (failure_class.empty() ? "null" + : ("\"" + failure_class + "\"")) + << ",\"failures\":["; + for (std::size_t fi = 0; fi < rec.failures.size(); ++fi) { + if (fi != 0) { + jsonl << ","; + } + jsonl << "\"" << rec.failures[fi] << "\""; + } + jsonl << "]}\n"; + jsonl.flush(); + + samples_csv << args.run_id << "," << args.transport << "," + << args.run_id << "," << args.seed << "," << si << "," + << rec.logical_ping_id << "," << rec.sequence << "," + << rec.fault_type << "," << fault_armed << "," + << fault_consumed << "," << rec.accepted_attempt << "," + << rec.scheduled_nominal_us << "," + << (rec.scheduled_nominal_us + period_us) << "," + << rec.scheduled_nominal_us << "," << F3(sched_err_ms) << "," + << F3(guard_ms) << "," << F3(attempt_lead_ms) << "," + << rec.scheduled_first_us << "," << rec.actual_first_send_us + << "," << F3(first_off_ms) << "," + << csv_i(rec.first_request_sent) << "," + << csv_i(rec.first_server_receive_us) << "," + << csv_i(rec.retry_decision_us) << "," + << csv_i(rec.retry_scheduled_us) << "," + << csv_i(rec.retry_send_us) << "," + << csv_i(timeout_to_decision) << "," + << csv_i(timeout_to_retry) << "," << csv_i(first_to_retry) + << "," << csv_d(retry_send_off_ms) << "," + << csv_d(retry_client_margin_ms) << "," + << csv_i(rec.retry_server_receive_mapped_us) << "," + << csv_d(rec.retry_server_margin_ms) << "," << one_way_us + << "," << rec.original_window_dur_us << "," + << rec.corrected_cur_dur_us << "," + << rec.next_scheduled_nominal_us << "," + << F3(next_phase_err_ms) << "," << csv_i(alice_state) << "," + << csv_i(alice_delta) << "," + << (rec.confirms > 1 ? rec.confirms - 1 : 0) << "," + << (rec.confirmed ? 1 : 0) << "," << failure_class << "\n"; + samples_csv.flush(); + phase_csv << si << "," << rec.sequence << "," << rec.fault_type << "," + << rec.expected_nominal_us << "," << rec.scheduled_nominal_us + << "," << sched_err << "," << rec.actual_first_send_us << "," + << send_phase_err << "," << csv_i(interval_err) << "," + << contiguous << "," << csv_i(skipped_slots) << "," + << next_phase_err << "\n"; + phase_csv.flush(); + auto const cur_start_d = + (rec.corrected_cur_start_us - rec.original_window_start_us) / 1000.0; + auto const cur_end_d = + (rec.corrected_cur_end_us - rec.original_window_end_us) / 1000.0; + auto const cur_dur_d = + (rec.corrected_cur_dur_us - rec.original_window_dur_us) / 1000.0; + auto const next_start_d = + rec.corrected_next_start_us == 0 + ? 0.0 + : (rec.corrected_next_start_us - + (rec.scheduled_nominal_us + period_us - + started->attempt_lead_us)) / + 1000.0; + win_csv << si << "," << rec.sequence << "," << rec.fault_type << "," + << rec.original_window_start_us << "," + << rec.original_window_end_us << "," << rec.original_window_dur_us + << "," << rec.corrected_cur_start_us << "," + << rec.corrected_cur_end_us << "," << rec.corrected_cur_dur_us + << "," << F3(cur_start_d) << "," << F3(cur_end_d) << "," + << F3(cur_dur_d) << "," << F3(next_start_d) << "," + << F3(next_phase_err / 1000.0) << "\n"; + win_csv.flush(); + for (auto const& h : rec.observers) { + double q_rtt = std::numeric_limits::quiet_NaN(); + obs_csv << si << "," << rec.sequence << "," << rec.fault_type << "," + << h.checkpoint << "," << h.query_steady_us << "," + << h.rel_deadline_us << "," << h.state << "," + << h.expected_state << "," << h.next_us << "," + << h.last_online_us << "," << csv_i(h.next_ping_delta_ms) << "," + << csv_i(h.last_connect_delta_ms) << "," + << (h.mismatch ? 1 : 0) << "," << csv_d(q_rtt) << "\n"; + } + obs_csv.flush(); + + prev_first_send_us = rec.actual_first_send_us; + prev_tn1_us = rec.next_scheduled_nominal_us; + prev_scheduled_us = rec.scheduled_nominal_us; + cycles.push_back(std::move(rec)); + } + + // In-flight QueryNow reuse during one request-loss and one response-loss. + // Timed shards keep controls brief (~5% wall time) and skip if budget gone. + int inflight_reused = 0; + int inflight_skipped = 0; + int inflight_extra = 0; + bool const run_controls = !timed_shard || shard_time_remaining(); + auto reuse_once = [&](int mode) { + if (!run_controls) { + return; + } + wait_window_closed(4000); + arm_next(mode, 1, 0, 0); + auto first = wait_first_attempt(8000); + SendRaw(alice, kIpcQueryNow, 0, 1, 0, 0); + SendRaw(alice, kIpcQueryNow, 0, 1, 0, 1); + drain(200); + if (!alice.query_stats.empty()) { + auto const& q = alice.query_stats.back(); + inflight_reused += static_cast(q.reused); + inflight_skipped += static_cast(q.skipped); + inflight_extra += static_cast(q.extra); + } + auto const cid = first ? first->logical_cycle_id : 0; + (void)wait_ev(static_cast(PingTraceKind::kCycleConfirmed), + 8000, cid, 0); + }; + reuse_once(1); + reuse_once(2); + + // Hard-stop first while Bob is still advertising a live schedule. + // AnnounceUnknown first would leave Alice in Unknown after KillChild. + if (run_controls) { + wait_window_closed(4000); + { + auto live = query_ckpt(0, false); + double remaining_ms = 0; + if (live && live->next_ping_delta_ms > + std::numeric_limits::min() / 2 && + live->next_ping_delta_ms > 0) { + remaining_ms = static_cast(live->next_ping_delta_ms); + } + DWORD poll_ms = 20000; + if (remaining_ms > 0 && remaining_ms < 60000) { + auto const need = static_cast(remaining_ms + 15000); + if (need > poll_ms) { + poll_ms = need; + } + } + if (poll_ms > 45000) { + poll_ms = 45000; + } + std::cout << "hard-stop remaining_ms=" << remaining_ms + << " poll_ms=" << poll_ms << std::endl; + KillChild(bob); + auto const deadline = GetTickCount64() + poll_ms; + while (GetTickCount64() < deadline) { + auto q = query_ckpt(0, false); + if (q && q->state == 1) { + hard_hit = 1; + break; + } + } + } + if (hard_hit != 1) { + ok_phase = false; + std::cerr << "FAIL hard-stop control did not yield MissedDeadline/state 1\n"; + FailedCase fc{}; + fc.invariant = "observer saw the wrong state (hard-stop MissedDeadline expected)"; + failed.push_back(fc); + } + + auto restart_for_graceful = [&]() -> bool { + if (alice.pi.hProcess != nullptr) { + StopChild(alice); + } + if (bob.pi.hProcess != nullptr) { + StopChild(bob); + } + Sleep(1000); + ResetChildRuntime(alice); + ResetChildRuntime(bob); + alice.side = IpcSide::kA; + bob.side = IpcSide::kB; + auto const pipe_a2 = PipeNameFor(args.run_id, IpcSide::kA, "a-phase-g"); + auto const pipe_b2 = PipeNameFor(args.run_id, IpcSide::kB, "b-phase-g"); + auto const log_a = + (std::filesystem::path{args.artifact_dir} / "alice-graceful.log") + .string(); + auto const log_b = + (std::filesystem::path{args.artifact_dir} / "bob-graceful.log") + .string(); + auto const state_a2 = (state_root / "state-a-phase-g").string(); + auto const state_b2 = (state_root / "state-b-phase-g").string(); + std::filesystem::create_directories(state_a2); + std::filesystem::create_directories(state_b2); + if (!SpawnChild(alice, args, state_a2, pipe_a2, "uap-1s-alice", log_a, + args.ping_interval_ms, args.receive_window_ms) || + !SpawnChild(bob, args, state_b2, pipe_b2, "uap-1s-bob", log_b, + args.ping_interval_ms, args.receive_window_ms)) { + return false; + } + ping_cursor = 0; + if (!wait_ready(alice, "Alice-graceful") || + !wait_ready(bob, "Bob-graceful")) { + return false; + } + exchange_uids(); + std::int64_t n = 0; + std::int64_t mn = 0; + std::int64_t p99 = 0; + std::uint32_t g = 0; + if (!wait_warmup(bob, false, &n, &mn, &p99, &g, nullptr, nullptr)) { + return false; + } + std::int64_t alice_n2 = 0; + std::int64_t alice_min2 = 0; + std::int64_t alice_p992 = 0; + std::int64_t alice_srv = 0; + std::int64_t alice_proto = 0; + if (!wait_warmup(alice, true, &alice_n2, &alice_min2, &alice_p992, nullptr, + &alice_srv, &alice_proto)) { + return false; + } + if (alice_srv != 0) { + dest = alice_srv; + } + return true; + }; + if (!restart_for_graceful()) { + ok_phase = false; + std::cerr << "FAIL could not restart pair for graceful Unknown control\n"; + FailedCase fc{}; + fc.invariant = "reporting/harness failure: graceful restart failed"; + failed.push_back(fc); + } else { + wait_window_closed(8000); + SendRaw(bob, kIpcAnnounceUnknown); + auto const deadline = GetTickCount64() + 15000; + while (GetTickCount64() < deadline) { + auto q = query_ckpt(0, false); + if (q && q->state == 2) { + graceful_hit = 1; + break; + } + } + } + if (graceful_hit != 1) { + ok_phase = false; + std::cerr << "FAIL graceful control did not yield Unknown/state 2\n"; + FailedCase fc{}; + fc.invariant = "observer saw the wrong state (graceful Unknown expected)"; + failed.push_back(fc); + } + } // run_controls + + std::vector sched_err_ms; + std::vector send_err_ms; + std::vector interval_err_ms; + std::vector req_margin; + std::vector resp_margin; + int req_n = 0; + int req_before = 0; + int req_after = 0; + int req_none = 0; + int resp_n = 0; + int resp_before = 0; + int resp_after = 0; + int resp_none = 0; + std::vector margin_lt10; + std::vector late_cases; + for (auto const& c : cycles) { + if (c.scheduled_nominal_us == 0 || c.expected_nominal_us == 0) { + continue; + } + sched_err_ms.push_back( + static_cast(c.scheduled_nominal_us - c.expected_nominal_us) / + 1000.0); + if (c.actual_first_send_us != 0) { + send_err_ms.push_back( + static_cast(c.actual_first_send_us - c.expected_nominal_us) / + 1000.0); + } + } + for (std::size_t i = 1; i < cycles.size(); ++i) { + if (cycles[i].actual_first_send_us == 0 || + cycles[i - 1].actual_first_send_us == 0) { + continue; + } + interval_err_ms.push_back( + static_cast(cycles[i].actual_first_send_us - + cycles[i - 1].actual_first_send_us - period_us) / + 1000.0); + } + for (int i = 0; i < static_cast(cycles.size()); ++i) { + auto const& c = cycles[static_cast(i)]; + if (c.fault_mode == 1) { + ++req_n; + if (c.no_retry) { + ++req_none; + } else if (c.retry_before_deadline) { + ++req_before; + if (std::isfinite(c.retry_server_margin_ms)) { + req_margin.push_back(c.retry_server_margin_ms); + if (c.retry_server_margin_ms < 10.0) { + margin_lt10.push_back(i); + } + } + } else if (c.retry_after_deadline) { + ++req_after; + late_cases.push_back(i); + } + } else if (c.fault_mode == 2) { + ++resp_n; + if (c.no_retry) { + ++resp_none; + } else if (c.retry_before_deadline) { + ++resp_before; + if (std::isfinite(c.retry_server_margin_ms)) { + resp_margin.push_back(c.retry_server_margin_ms); + if (c.retry_server_margin_ms < 10.0) { + margin_lt10.push_back(i); + } + } + } else if (c.retry_after_deadline) { + ++resp_after; + late_cases.push_back(i); + } + } + } + auto slope_ms = [&](std::vector const& y) { + auto const n = static_cast(y.size()); + if (n < 2) { + return 0.0; + } + double sum_x = 0; + double sum_y = 0; + double sum_xx = 0; + double sum_xy = 0; + for (std::size_t i = 0; i < y.size(); ++i) { + auto const x = static_cast(i); + sum_x += x; + sum_y += y[i]; + sum_xx += x * x; + sum_xy += x * y[i]; + } + auto const den = n * sum_xx - sum_x * sum_x; + if (den == 0) { + return 0.0; + } + return (n * sum_xy - sum_x * sum_y) / den; + }; + auto mean_of = [&](std::vector const& y) { + if (y.empty()) { + return 0.0; + } + return std::accumulate(y.begin(), y.end(), 0.0) / + static_cast(y.size()); + }; + auto min_of = [&](std::vector const& y) { + if (y.empty()) { + return 0.0; + } + return *std::min_element(y.begin(), y.end()); + }; + auto max_of = [&](std::vector const& y) { + if (y.empty()) { + return 0.0; + } + return *std::max_element(y.begin(), y.end()); + }; + if (std::fabs(slope_ms(sched_err_ms)) > 0.01) { + ok_phase = false; + FailedCase fc{}; + fc.invariant = "next nominal schedule shifted (accumulating phase drift)"; + failed.push_back(fc); + std::cerr << "FAIL accumulating scheduled phase drift slope=" + << F3(slope_ms(sched_err_ms)) << " ms/cycle\n"; + } + + std::ofstream failed_json(std::filesystem::path{args.artifact_dir} / + "failed-cases.json"); + failed_json << "[\n"; + for (std::size_t i = 0; i < failed.size(); ++i) { + auto const& f = failed[i]; + if (i != 0) { + failed_json << ",\n"; + } + failed_json << " {\"cycle\":" << f.c.index << ",\"seed\":" << args.seed + << ",\"transport\":\"" << args.transport + << "\",\"sequence\":\"" << f.c.sequence + << "\",\"fault_type\":\"" << f.c.fault_type + << "\",\"invariant\":\"" << f.invariant + << "\",\"logical_ping_id\":" << f.c.logical_ping_id + << ",\"phase_anchor_us\":" << f.c.phase_anchor_us + << ",\"expected_nominal_us\":" << f.c.expected_nominal_us + << ",\"original_nominal_us\":" << f.c.scheduled_nominal_us + << ",\"original_deadline_us\":" << f.c.original_deadline_us + << ",\"first_attempt_us\":" << f.c.actual_first_send_us + << ",\"first_request_sent\":" << json_i(f.c.first_request_sent) + << ",\"retry_decision_us\":" << json_i(f.c.retry_decision_us) + << ",\"retry_scheduled_us\":" << json_i(f.c.retry_scheduled_us) + << ",\"retry_send_us\":" << json_i(f.c.retry_send_us) + << ",\"retry_server_receive_mapped_us\":" + << json_i(f.c.retry_server_receive_mapped_us) + << ",\"retry_server_margin_ms\":" + << json_d(f.c.retry_server_margin_ms) + << ",\"original_window_start_us\":" + << f.c.original_window_start_us + << ",\"original_window_end_us\":" << f.c.original_window_end_us + << ",\"corrected_current_window_start_us\":" + << f.c.corrected_cur_start_us + << ",\"corrected_current_window_end_us\":" + << f.c.corrected_cur_end_us + << ",\"next_scheduled_nominal_us\":" + << f.c.next_scheduled_nominal_us + << ",\"observer_states\":["; + for (std::size_t oi = 0; oi < f.c.observers.size(); ++oi) { + if (oi != 0) { + failed_json << ","; + } + auto const& h = f.c.observers[oi]; + failed_json << "{\"checkpoint\":" << h.checkpoint + << ",\"rel_deadline_us\":" << h.rel_deadline_us + << ",\"state\":" << h.state + << ",\"expected_state\":" << h.expected_state << "}"; + } + failed_json << "]}"; + } + failed_json << "\n]\n"; + + std::ofstream report(std::filesystem::path{args.artifact_dir} / "report.md"); + if (!report) { + std::cerr << "FAIL cannot write report.md\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + auto pct_block = [&](char const* title, std::vector const& v) { + report << "### " << title << "\n"; + report << "- n: " << v.size() << "\n"; + report << "- min: " << F3(min_of(v)) << "\n"; + report << "- mean: " << F3(mean_of(v)) << "\n"; + report << "- p50: " << F3(Percentile(v, 0.50)) << "\n"; + report << "- p90: " << F3(Percentile(v, 0.90)) << "\n"; + report << "- p95: " << F3(Percentile(v, 0.95)) << "\n"; + report << "- p99: " << F3(Percentile(v, 0.99)) << "\n"; + report << "- max: " << F3(max_of(v)) << "\n\n"; + }; + report << "# UAP 1s nominal phase preservation\n\n"; + report << "- transport: " << args.transport << "\n"; + report << "- seed: " << args.seed << "\n"; + report << "- mode: " << (stress ? "stress" : "fast") << "\n"; + report << "- steps: " << steps.size() << "\n"; + report << "- phase_anchor_us: " << phase_anchor_us << "\n"; + report << "- period_us: " << period_us << "\n"; + report << "- one_way_us (coordinator mapping, min_rtt/2): " << one_way_us + << "\n"; + report << "- first ping is sent at Tn - attempt_lead (PingCloudServers::" + "MakePing / ApplyLogicalPingAttempt first_attempt_at).\n"; + report << "- retry is scheduled immediately on ping error 2 " + "(ScheduleSameCycleRetry); boundary cases hold send until " + "Tn+offset-one_way via PingTestFaults hold.\n"; + report << "- Alice queries via Client::QueryPeerReceiveSchedule at " + "checkpoints 1-5 (IPC QueryNow).\n"; + report << "- retry vs original deadline uses coordinator QPC mapping: " + "retry_send_qpc + one_way vs cycle_anchor mapped from the same " + "Bob trace event. Alice last_online is recorded separately and " + "is not replaced with a raw client send timestamp.\n\n"; + report << "## Schedule phase\n\n"; + pct_block("scheduled phase error (ms)", sched_err_ms); + pct_block("actual first-attempt send phase error (ms)", send_err_ms); + pct_block("actual interval error from 1000 ms (ms)", interval_err_ms); + report << "- final scheduled phase error ms: " + << (sched_err_ms.empty() ? "0" : F3(sched_err_ms.back())) << "\n"; + double max_abs_sched = 0; + for (auto x : sched_err_ms) { + max_abs_sched = std::max(max_abs_sched, std::fabs(x)); + } + report << "- maximum cumulative |scheduled phase error| ms: " + << F3(max_abs_sched) << "\n"; + report << "- linear drift slope ms/cycle: " << F3(slope_ms(sched_err_ms)) + << "\n\n"; + auto retry_block = [&](char const* title, int n, int before, int after, + int none, std::vector const& m) { + report << "### " << title << "\n"; + report << "- attempted cases: " << n << "\n"; + report << "- retry received before original deadline: " << before << "\n"; + report << "- retry received after original deadline: " << after << "\n"; + report << "- no retry received: " << none << "\n"; + report << "- success rate: " + << (n == 0 ? 0.0 : static_cast(before) / n) << "\n"; + pct_block("retry_server_margin ms", m); + }; + report << "## Retry arrival before original deadline\n\n"; + retry_block("request loss", req_n, req_before, req_after, req_none, + req_margin); + retry_block("response loss", resp_n, resp_before, resp_after, resp_none, + resp_margin); + report << "- cases with margin < 10 ms:"; + if (margin_lt10.empty()) { + report << " none\n"; + } else { + report << "\n"; + for (auto i : margin_lt10) { + auto const& c = cycles[static_cast(i)]; + report << " - cycle " << i << " " << c.sequence << " " << c.fault_type + << " margin_ms=" << csv_d(c.retry_server_margin_ms) << "\n"; + } + } + report << "- late cases:"; + if (late_cases.empty()) { + report << " none\n\n"; + } else { + report << "\n"; + for (auto i : late_cases) { + auto const& c = cycles[static_cast(i)]; + report << " - cycle " << i << " " << c.sequence << " " << c.fault_type + << " margin_ms=" << csv_d(c.retry_server_margin_ms) + << " production_observer_state_after_Tn: QueryPeerReceiveSchedule " + "remains kExpected while nextPingDelta still points at Tn+1 " + "unless every expected server returns a negative delta.\n"; + } + report << "\n"; + } + report << "## Window correction\n\n"; + report << "See window-corrections.csv. Next nominal phase delta is " + "next_scheduled_nominal - (expected + 1000ms).\n\n"; + report << "## Observer results\n\n"; + report << "Alice calls Client::QueryPeerReceiveSchedule(peer_uid) through " + "RoleState::QueryNow at:\n"; + report << "1. immediately after the first-attempt trace is captured for the " + "armed ping (Bob has already advertised Tn; QueryNow is not " + "inserted before Consume)\n"; + report << "2. after AttemptTimeout, before retry send (hold provides the " + "race window on boundary cases)\n"; + report << "3. after retry RequestSent, before original deadline\n"; + report << "4. immediately after original deadline\n"; + report << "5. after CycleConfirmed / next nominal\n\n"; + report << "- query_failures: " << query_failures << "\n"; + report << "- false live MissedDeadline: " << live_false_md << "\n"; + report << "- false live Unknown: " << live_false_unknown << "\n\n"; + report << "## Reliability\n\n"; + report << "- request-loss recovery: " << req_before << "/" << req_n << "\n"; + report << "- response-loss recovery: " << resp_before << "/" << resp_n + << "\n"; + report << "- duplicate logical ping count: " << duplicate_logical << "\n"; + report << "- in-flight query reused: " << inflight_reused << "\n"; + report << "- in-flight query skipped: " << inflight_skipped << "\n"; + report << "- extra in-flight subscribers: " << inflight_extra << "\n"; + report << "- graceful Unknown: " << graceful_hit << "/1\n"; + report << "- hard-stop MissedDeadline: " << hard_hit << "/1\n"; + report << "- failed invariants: " << failed.size() << "\n"; + report << "\nHard-stop timing is Test-harness MissedDeadline detection " + "latency, not a production SLA.\n"; + report << "Semantic results: hard-stop MissedDeadline/state 1; " + "graceful-stop Unknown/state 2.\n"; + report.flush(); + if (!report) { + std::cerr << "FAIL report.md write failed\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + jsonl.flush(); + samples_csv.flush(); + phase_csv.flush(); + win_csv.flush(); + obs_csv.flush(); + std::cout << "report=" + << (std::filesystem::path{args.artifact_dir} / "report.md") + << std::endl; + std::cout << (ok_phase ? "PASS phase-preservation" : "FAIL phase-preservation") + << std::endl; + StopChild(alice); + StopChild(bob); + return ok_phase ? 0 : 7; diff --git a/examples/aether_uap_1s_timing_characterization/retry_count_zero_runtime.inc.cpp b/examples/aether_uap_1s_timing_characterization/retry_count_zero_runtime.inc.cpp new file mode 100644 index 00000000..64603861 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/retry_count_zero_runtime.inc.cpp @@ -0,0 +1,1103 @@ + // Runtime acceptance: ping_retry_count=0 post-deadline same-cycle recovery. + // Deterministic drop of Bob attempt #1; no pre-deadline retry before Tn. + + struct RczRetryRec { + std::int64_t attempt_index{0}; + std::int64_t actual_send_time{0}; + double offset_from_Tn_ms{std::numeric_limits::quiet_NaN()}; + std::int64_t logical_cycle_id{0}; + }; + + struct RczAliceQuery { + std::int64_t query_time{std::numeric_limits::min()}; + std::int64_t state{-2}; + std::int64_t last_online{std::numeric_limits::min()}; + std::int64_t next_deadline{0}; + bool valid{false}; + }; + + struct RczCase { + int case_index{0}; + std::string transport; + std::int64_t seed{0}; + std::int64_t logical_cycle_id{0}; + std::int64_t tn_us{0}; + std::int64_t interval_ms{0}; + int ping_retry_count{0}; + double r99_ms{std::numeric_limits::quiet_NaN()}; + double guard_ms{std::numeric_limits::quiet_NaN()}; + std::int64_t first_attempt_send_time{0}; + std::int64_t first_attempt_index{0}; + int first_attempt_dropped{-1}; + int retry_attempt_count{0}; + std::vector retries; + std::int64_t first_post_deadline_retry_time{0}; + double first_post_deadline_retry_offset_ms{ + std::numeric_limits::quiet_NaN()}; + std::int64_t confirmation_time{0}; + double confirmation_offset_from_Tn_ms{ + std::numeric_limits::quiet_NaN()}; + std::int64_t confirming_attempt_index{0}; + std::int64_t next_nominal_deadline{0}; + std::int64_t expected_next_grid_deadline{0}; + double phase_error_ms{std::numeric_limits::quiet_NaN()}; + int duplicate_count{0}; + RczAliceQuery alice_q1; + RczAliceQuery alice_q2; + bool fault_armed{false}; + bool fault_matched{false}; + bool fault_dropped{false}; + std::string harness_error; + bool harness_valid{false}; + std::vector production_failures; + bool failed_pre_deadline_retry{false}; + bool failed_new_cycle_before_confirm{false}; + bool failed_duplicate{false}; + }; + + + auto abs_d = [](double v) { return v < 0 ? -v : v; }; + auto fmt3 = [](double v) -> std::string { + if (!std::isfinite(v)) { + return {}; + } + std::ostringstream os; + os << std::fixed << std::setprecision(3) << v; + return os.str(); + }; + + if (warmup_n <= 0) { + std::cerr << "FAIL RETRY_COUNT_ZERO: no warm-up RTT samples\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + + std::int64_t const period_us = args.ping_interval_ms * 1000; + double const tick_ms = 1.0; + int const n_cases = args.retry_count_zero_cases > 0 + ? args.retry_count_zero_cases + : 10; + int const ping_retry_count_cfg = static_cast(kDefaultPingRetryCount); + + std::filesystem::create_directories(args.artifact_dir); + std::ofstream samples_csv(std::filesystem::path{args.artifact_dir} / + "samples.csv"); + std::ofstream samples_jsonl(std::filesystem::path{args.artifact_dir} / + "samples.jsonl"); + std::ofstream failed_json(std::filesystem::path{args.artifact_dir} / + "failed-cases.json"); + std::ofstream report(std::filesystem::path{args.artifact_dir} / + "report.md"); + std::ofstream summary_json(std::filesystem::path{args.artifact_dir} / + "summary.json"); + if (!samples_csv || !samples_jsonl || !failed_json || !report || + !summary_json) { + std::cerr << "FAIL cannot open retry-count-zero outputs\n"; + StopChild(alice); + StopChild(bob); + return 8; + } + + samples_csv + << "case_index,transport,seed,logical_cycle_id,Tn_us,interval_ms," + "ping_retry_count,R99_ms,guard_ms,first_attempt_send_time," + "first_attempt_index,first_attempt_dropped,retry_attempt_count," + "first_post_deadline_retry_time,first_post_deadline_retry_offset_ms," + "confirmation_time,confirmation_offset_from_Tn_ms," + "confirming_attempt_index,next_nominal_deadline," + "expected_next_grid_deadline,phase_error_ms,duplicate_count," + "alice_q1_query_time,alice_q1_state,alice_q1_last_online," + "alice_q1_next_deadline,alice_q2_query_time,alice_q2_state," + "alice_q2_last_online,alice_q2_next_deadline,fault_armed," + "fault_matched,fault_dropped,harness_error,production_failures\n"; + + using EvId = std::tuple; + std::set seen_ev; + auto ev_id = [](BobPingEvent const& e) { + return EvId{e.event_qpc, e.kind, e.logical_cycle_id, + e.physical_attempt_index, e.server_id}; + }; + for (auto const& e : bob.ping_events) { + seen_ev.insert(ev_id(e)); + } + auto take_ev = [&](BobPingEvent const& e) { + seen_ev.insert(ev_id(e)); + return e; + }; + auto dest_ok = [&](BobPingEvent const& e) { + return dest == 0 || e.server_id == dest; + }; + auto send_us = [](BobPingEvent const& e) -> std::int64_t { + return e.actual_us != 0 ? e.actual_us : e.event_steady_us; + }; + auto find_ev = [&](std::uint8_t kind, DWORD timeout_ms, + std::int64_t cycle_id, std::int64_t min_attempt) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.kind != kind) { + continue; + } + if (cycle_id != 0 && e.logical_cycle_id != 0 && + e.logical_cycle_id != cycle_id) { + continue; + } + if (min_attempt > 0 && e.physical_attempt_index < min_attempt) { + continue; + } + return take_ev(e); + } + } + return std::nullopt; + }; + auto wait_sched = [&](DWORD timeout_ms) -> std::optional { + auto const before = alice.schedules.size(); + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + if (alice.schedules.size() > before) { + return alice.schedules.back(); + } + } + return std::nullopt; + }; + + constexpr std::uint8_t kFaultTraceArmed = 1; + constexpr std::uint8_t kFaultTraceMatched = 3; + constexpr std::uint8_t kFaultTraceDropped = 4; + + std::int64_t last_settled_cycle_id = 0; + for (auto const& e : bob.ping_events) { + if (e.logical_cycle_id > last_settled_cycle_id) { + last_settled_cycle_id = e.logical_cycle_id; + } + } + + auto wait_arm_ack = [&](DWORD timeout_ms) -> bool { + bob.got_ack = false; + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + if (bob.got_ack) { + return true; + } + } + return false; + }; + auto disarm_fault = [&]() -> bool { + bob.got_ack = false; + SendRaw(bob, kIpcArmFault, 0, dest, 1, 0, 0, 0, 0); + return wait_arm_ack(3000); + }; + auto arm_drop_next_first = [&]() -> bool { + if (!disarm_fault()) { + return false; + } + bob.got_ack = false; + SendRaw(bob, kIpcArmFault, 0, dest, 1, 1, 0, 0, 0); + return wait_arm_ack(3000); + }; + + struct DropWaitResult { + std::optional drop{}; + bool sent_instead{false}; + bool armed{false}; + bool matched{false}; + bool dropped_trace{false}; + std::int64_t arm_time{0}; + std::int64_t match_time{0}; + std::int64_t drop_time{0}; + std::int64_t cycle_id{0}; + }; + + auto wait_intended_drop = [&](std::int64_t min_cycle_id, + std::size_t trace_after, DWORD timeout_ms) + -> DropWaitResult { + DropWaitResult out{}; + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (std::size_t i = trace_after; i < bob.fault_traces.size(); ++i) { + auto const& t = bob.fault_traces[i]; + if (dest != 0 && t.server_id != dest) { + continue; + } + if (t.kind == kFaultTraceArmed && !out.armed) { + out.armed = true; + out.arm_time = t.steady_us; + } + if (t.kind == kFaultTraceMatched && t.physical_attempt_index == 1) { + if (out.drop.has_value()) { + if (t.logical_cycle_id == out.drop->logical_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + } + } else if (t.logical_cycle_id > min_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + out.cycle_id = t.logical_cycle_id; + } + } + if (t.kind == kFaultTraceDropped && t.physical_attempt_index == 1) { + if (out.drop.has_value()) { + if (t.logical_cycle_id == out.drop->logical_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + } + } else if (t.logical_cycle_id > min_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + out.cycle_id = t.logical_cycle_id; + } + } + } + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.logical_cycle_id == 0 || e.logical_cycle_id <= min_cycle_id) { + continue; + } + if (e.physical_attempt_index != 1) { + continue; + } + if (e.kind == + static_cast(PingTraceKind::kRequestSent)) { + seen_ev.insert(ev_id(e)); + out.sent_instead = true; + out.cycle_id = e.logical_cycle_id; + return out; + } + if (e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + out.drop = take_ev(e); + out.cycle_id = e.logical_cycle_id; + } + } + if (out.drop.has_value() && out.armed && out.matched && + out.dropped_trace) { + return out; + } + if (out.sent_instead) { + return out; + } + } + return out; + }; + + auto wait_next_first_after = [&](std::int64_t after_cycle_id, + DWORD timeout_ms) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(20); + for (auto const& e : bob.ping_events) { + if (seen_ev.count(ev_id(e)) != 0 || !dest_ok(e)) { + continue; + } + if (e.logical_cycle_id == 0 || + e.logical_cycle_id <= after_cycle_id) { + continue; + } + if (e.physical_attempt_index != 1) { + continue; + } + if (e.kind == + static_cast(PingTraceKind::kRequestSent) || + e.kind == + static_cast(PingTraceKind::kRequestDropped)) { + return take_ev(e); + } + } + } + return std::nullopt; + }; + + auto refresh_fault_flags = [&](DropWaitResult& out, + std::size_t trace_after) { + if (!out.drop.has_value()) { + return; + } + for (std::size_t i = trace_after; i < bob.fault_traces.size(); ++i) { + auto const& t = bob.fault_traces[i]; + if (dest != 0 && t.server_id != dest) { + continue; + } + if (t.kind == kFaultTraceArmed) { + out.armed = true; + out.arm_time = t.steady_us; + } + if (t.kind == kFaultTraceMatched && + t.physical_attempt_index == out.drop->physical_attempt_index && + t.logical_cycle_id == out.drop->logical_cycle_id) { + out.matched = true; + out.match_time = t.steady_us; + } + if (t.kind == kFaultTraceDropped && + t.physical_attempt_index == out.drop->physical_attempt_index && + t.logical_cycle_id == out.drop->logical_cycle_id) { + out.dropped_trace = true; + out.drop_time = t.steady_us; + } + } + }; + + auto expected_grid_after = [&](std::int64_t tn, std::int64_t ref_us) { + for (int k = 1; k < 64; ++k) { + auto const g = tn + static_cast(k) * period_us; + if (g > ref_us) { + return g; + } + } + return tn + period_us; + }; + + auto prod_fail = [&](RczCase& c, char const* cls) { + c.production_failures.push_back(cls); + std::cerr << "FAIL case " << c.case_index << " " << cls << std::endl; + }; + + auto alice_query = [&](int checkpoint) -> std::optional { + for (int try_i = 0; try_i < 6; ++try_i) { + // force=1 so a leftover in-flight query still emits schedule state. + SendRaw(alice, kIpcQueryNow, 0, checkpoint, 0, 1); + if (auto aq = wait_sched(1500)) { + return aq; + } + } + return std::nullopt; + }; + + auto scan_cycle_events = [&](RczCase& c, bool confirmed) { + int confirms = 0; + bool saw_our_confirm = false; + for (auto const& e : bob.ping_events) { + if (!dest_ok(e)) { + continue; + } + if (e.logical_cycle_id == c.logical_cycle_id && + e.kind == + static_cast(PingTraceKind::kCycleConfirmed)) { + saw_our_confirm = true; + ++confirms; + } + } + for (auto const& e : bob.ping_events) { + if (!dest_ok(e)) { + continue; + } + // New logical cycle before OUR confirmation is a production failure. + // Ignore CycleStarted events that appear after we already confirmed. + if (!confirmed && !saw_our_confirm && + e.logical_cycle_id > c.logical_cycle_id && + e.kind == static_cast(PingTraceKind::kCycleStarted) && + !c.failed_new_cycle_before_confirm) { + c.failed_new_cycle_before_confirm = true; + prod_fail(c, "NEW_CYCLE_BEFORE_CONFIRM"); + } + if (e.logical_cycle_id != c.logical_cycle_id) { + continue; + } + if (e.physical_attempt_index < 2) { + continue; + } + if (e.kind != static_cast(PingTraceKind::kRequestSent) && + e.kind != + static_cast(PingTraceKind::kRequestDropped)) { + continue; + } + auto const t = send_us(e); + if (c.tn_us != 0 && t < c.tn_us && !c.failed_pre_deadline_retry) { + c.failed_pre_deadline_retry = true; + prod_fail(c, "PRE_DEADLINE_RETRY"); + } + bool already = false; + for (auto const& r : c.retries) { + if (r.attempt_index == e.physical_attempt_index && + r.actual_send_time == t) { + already = true; + break; + } + } + if (already) { + continue; + } + RczRetryRec rr{}; + rr.attempt_index = e.physical_attempt_index; + rr.actual_send_time = t; + rr.logical_cycle_id = e.logical_cycle_id; + if (c.tn_us != 0) { + rr.offset_from_Tn_ms = + static_cast(t - c.tn_us) / 1000.0; + } + c.retries.push_back(rr); + if (c.tn_us != 0 && t >= c.tn_us && + c.first_post_deadline_retry_time == 0) { + c.first_post_deadline_retry_time = t; + c.first_post_deadline_retry_offset_ms = rr.offset_from_Tn_ms; + } + } + if (confirms > 1 && !c.failed_duplicate) { + c.failed_duplicate = true; + c.duplicate_count += confirms - 1; + prod_fail(c, "DUPLICATE_LOGICAL_PING"); + } else if (confirms > 1) { + c.duplicate_count = confirms - 1; + } + c.retry_attempt_count = static_cast(c.retries.size()); + }; + + std::vector cases; + int harness_armed_n = 0; + int harness_matched_n = 0; + int harness_dropped_n = 0; + int harness_invalid_n = 0; + int harness_wrong_request = 0; + bool prod_ok = true; + + std::cout << "RETRY_COUNT_ZERO_RUNTIME cases=" << n_cases + << " seed=" << args.seed << " transport=" << args.transport + << " ping_retry_count=" << ping_retry_count_cfg << std::endl; + + wait_window_closed(8000); + + for (int ci = 0; ci < n_cases; ++ci) { + RczCase rec{}; + rec.case_index = ci; + rec.transport = args.transport; + rec.seed = args.seed; + rec.interval_ms = args.ping_interval_ms; + rec.ping_retry_count = ping_retry_count_cfg; + + std::int64_t settle_before = last_settled_cycle_id; + std::size_t fault_trace_before = bob.fault_traces.size(); + + wait_window_closed(8000); + + bool armed_ok = false; + DropWaitResult dw{}; + for (int arm_try = 0; arm_try < 4; ++arm_try) { + armed_ok = arm_drop_next_first(); + if (armed_ok && arm_try == 0) { + ++harness_armed_n; + } + dw = wait_intended_drop(settle_before, fault_trace_before, 8000); + if (dw.drop.has_value()) { + refresh_fault_flags(dw, fault_trace_before); + drain(200); + refresh_fault_flags(dw, fault_trace_before); + } + if (armed_ok && dw.drop.has_value() && dw.armed && dw.matched && + dw.dropped_trace && dw.drop->physical_attempt_index == 1 && + dw.drop->logical_cycle_id > settle_before) { + break; + } + if (dw.sent_instead) { + ++harness_wrong_request; + if (dw.cycle_id > settle_before) { + (void)find_ev( + static_cast(PingTraceKind::kCycleConfirmed), 3000, + dw.cycle_id, 0); + settle_before = dw.cycle_id; + last_settled_cycle_id = dw.cycle_id; + } + disarm_fault(); + fault_trace_before = bob.fault_traces.size(); + dw = DropWaitResult{}; + continue; + } + disarm_fault(); + fault_trace_before = bob.fault_traces.size(); + dw = DropWaitResult{}; + } + + rec.fault_armed = armed_ok && dw.armed; + rec.fault_matched = dw.matched; + rec.fault_dropped = dw.dropped_trace; + + bool sync_ok = armed_ok && dw.drop.has_value() && dw.armed && + dw.matched && dw.dropped_trace && + dw.drop->physical_attempt_index == 1 && + dw.drop->logical_cycle_id > settle_before; + + if (!sync_ok) { + rec.harness_error = "HARNESS_FAULT_SYNC_ERROR"; + ++harness_invalid_n; + prod_ok = false; + if (dw.sent_instead) { + rec.first_attempt_dropped = 0; + } + failed_json << "{\"case_index\":" << ci + << ",\"harness_error\":\"HARNESS_FAULT_SYNC_ERROR\"}\n"; + failed_json.flush(); + cases.push_back(std::move(rec)); + disarm_fault(); + continue; + } + + rec.harness_valid = true; + ++harness_matched_n; + ++harness_dropped_n; + + BobPingEvent const first = *dw.drop; + rec.logical_cycle_id = first.logical_cycle_id; + rec.tn_us = first.cycle_anchor_us != 0 ? first.cycle_anchor_us + : first.contract_deadline_us; + rec.first_attempt_index = first.physical_attempt_index; + rec.first_attempt_send_time = send_us(first); + rec.first_attempt_dropped = 1; + if (first.p99_rtt_us > 0) { + rec.r99_ms = first.p99_rtt_us / 1000.0; + } + if (first.guard_us > 0) { + rec.guard_ms = first.guard_us / 1000.0; + } + + bool confirmed = false; + bool q1_done = false; + bool tn_passed = false; + auto const case_deadline = GetTickCount64() + 20000; + + while (GetTickCount64() < case_deadline) { + drain(20); + scan_cycle_events(rec, confirmed); + + if (!confirmed) { + for (auto const& e : bob.ping_events) { + if (!dest_ok(e)) { + continue; + } + if (e.logical_cycle_id != rec.logical_cycle_id) { + continue; + } + if (e.kind != + static_cast(PingTraceKind::kCycleConfirmed)) { + continue; + } + confirmed = true; + // Prefer ae-clock send time; CycleConfirmed may only have steady. + rec.confirmation_time = send_us(e); + if (rec.confirmation_time == 0 && + rec.first_post_deadline_retry_time != 0) { + rec.confirmation_time = rec.first_post_deadline_retry_time; + } + rec.confirming_attempt_index = e.physical_attempt_index; + auto const phase_ref = + rec.confirmation_time != 0 + ? rec.confirmation_time + : (rec.first_post_deadline_retry_time != 0 + ? rec.first_post_deadline_retry_time + : rec.tn_us); + if (rec.tn_us != 0 && phase_ref != 0) { + rec.confirmation_offset_from_Tn_ms = + static_cast(phase_ref - rec.tn_us) / 1000.0; + } + rec.next_nominal_deadline = + e.contract_deadline_us != 0 ? e.contract_deadline_us + : expected_grid_after(rec.tn_us, + phase_ref); + rec.expected_next_grid_deadline = + expected_grid_after(rec.tn_us, phase_ref); + rec.phase_error_ms = + static_cast(rec.next_nominal_deadline - + rec.expected_next_grid_deadline) / + 1000.0; + if (abs_d(rec.phase_error_ms) > tick_ms) { + prod_fail(rec, "POST_RECOVERY_PHASE_DRIFT"); + } + break; + } + if (confirmed) { + break; + } + } + + if (rec.tn_us != 0 && first.event_qpc != 0 && first.actual_us != 0) { + LARGE_INTEGER qfreq{}; + QueryPerformanceFrequency(&qfreq); + double const qpc_per_us = + static_cast(qfreq.QuadPart) / 1000000.0; + auto const tn_qpc = + first.event_qpc + + static_cast( + static_cast(rec.tn_us - first.actual_us) * + qpc_per_us); + auto now_qpc = QpcNow(); + if (now_qpc >= tn_qpc) { + tn_passed = true; + } + } else if (rec.tn_us != 0) { + for (auto const& e : bob.ping_events) { + if (send_us(e) >= rec.tn_us) { + tn_passed = true; + break; + } + } + } + + if (tn_passed && !q1_done && !confirmed) { + q1_done = true; + // Q1 is diagnostic only. Skip QueryPeer here so Q2 (required) is not + // blocked by a stuck in-flight Alice query from the race window. + } + } + + scan_cycle_events(rec, confirmed); + + if (!confirmed) { + prod_fail(rec, "NO_CONFIRMATION"); + } + if (rec.first_post_deadline_retry_time == 0 && confirmed == false) { + // still check post-deadline even if no confirm yet + } + bool has_post_deadline = false; + for (auto const& r : rec.retries) { + if (r.actual_send_time >= rec.tn_us) { + has_post_deadline = true; + break; + } + } + if (!has_post_deadline) { + prod_fail(rec, "NO_POST_DEADLINE_RETRY"); + } + + if (confirmed) { + if (auto aq2 = alice_query(/*checkpoint*/ 42)) { + rec.alice_q2.valid = true; + rec.alice_q2.query_time = aq2->steady_us; + rec.alice_q2.state = aq2->state; + rec.alice_q2.last_online = aq2->last_online_us; + rec.alice_q2.next_deadline = aq2->next_us; + if (rec.alice_q2.state == 1) { + prod_fail(rec, "ALICE_Q2_MISSED_DEADLINE"); + } + if (rec.alice_q2.state == 0 && rec.alice_q2.next_deadline != 0 && + rec.tn_us != 0) { + // next_deadline is ae-clock; snap to original grid (not query_time). + auto const expected = expected_grid_after( + rec.tn_us, rec.alice_q2.next_deadline - 1); + double const alice_phase = + static_cast(rec.alice_q2.next_deadline - expected) / + 1000.0; + if (abs_d(alice_phase) > tick_ms) { + prod_fail(rec, "ALICE_Q2_PHASE_SHIFT"); + } + } + } else { + rec.harness_error = "ALICE_Q2_QUERY_FAILED"; + std::cerr << "HARNESS case " << rec.case_index + << " ALICE_Q2_QUERY_FAILED (Bob recovery still valid)\n"; + } + last_settled_cycle_id = rec.logical_cycle_id; + // Let the next nominal cycle settle so the following arm is clean. + disarm_fault(); + wait_window_closed(8000); + auto next_first = wait_next_first_after(last_settled_cycle_id, 5000); + if (next_first) { + (void)find_ev( + static_cast(PingTraceKind::kCycleConfirmed), 5000, + next_first->logical_cycle_id, 0); + last_settled_cycle_id = next_first->logical_cycle_id; + } + wait_window_closed(8000); + } else { + disarm_fault(); + wait_window_closed(8000); + } + + disarm_fault(); + + if (!rec.production_failures.empty()) { + prod_ok = false; + failed_json << "{\"case_index\":" << ci + << ",\"logical_cycle_id\":" << rec.logical_cycle_id + << ",\"failures\":["; + for (std::size_t fi = 0; fi < rec.production_failures.size(); ++fi) { + if (fi != 0) { + failed_json << ","; + } + failed_json << "\"" << rec.production_failures[fi] << "\""; + } + failed_json << "]}\n"; + failed_json.flush(); + } + + std::string prod_cls; + for (std::size_t fi = 0; fi < rec.production_failures.size(); ++fi) { + if (fi != 0) { + prod_cls += ";"; + } + prod_cls += rec.production_failures[fi]; + } + + samples_csv << rec.case_index << "," << rec.transport << "," << rec.seed + << "," << rec.logical_cycle_id << "," << rec.tn_us << "," + << rec.interval_ms << "," << rec.ping_retry_count << "," + << fmt3(rec.r99_ms) << "," << fmt3(rec.guard_ms) << "," + << rec.first_attempt_send_time << "," + << rec.first_attempt_index << "," + << rec.first_attempt_dropped << "," + << rec.retry_attempt_count << "," + << rec.first_post_deadline_retry_time << "," + << fmt3(rec.first_post_deadline_retry_offset_ms) << "," + << rec.confirmation_time << "," + << fmt3(rec.confirmation_offset_from_Tn_ms) << "," + << rec.confirming_attempt_index << "," + << rec.next_nominal_deadline << "," + << rec.expected_next_grid_deadline << "," + << fmt3(rec.phase_error_ms) << "," << rec.duplicate_count + << ","; + if (rec.alice_q1.valid) { + samples_csv << rec.alice_q1.query_time << "," << rec.alice_q1.state + << "," << rec.alice_q1.last_online << "," + << rec.alice_q1.next_deadline << ","; + } else { + samples_csv << ",,,,"; + } + if (rec.alice_q2.valid) { + samples_csv << rec.alice_q2.query_time << "," << rec.alice_q2.state + << "," << rec.alice_q2.last_online << "," + << rec.alice_q2.next_deadline << ","; + } else { + samples_csv << ",,,,"; + } + samples_csv << (rec.fault_armed ? 1 : 0) << "," + << (rec.fault_matched ? 1 : 0) << "," + << (rec.fault_dropped ? 1 : 0) << ",\"" << rec.harness_error + << "\",\"" << prod_cls << "\"\n"; + samples_csv.flush(); + + samples_jsonl << "{\"case_index\":" << rec.case_index + << ",\"transport\":\"" << rec.transport << "\"" + << ",\"seed\":" << rec.seed + << ",\"logical_cycle_id\":" << rec.logical_cycle_id + << ",\"Tn_us\":" << rec.tn_us + << ",\"interval_ms\":" << rec.interval_ms + << ",\"ping_retry_count\":" << rec.ping_retry_count + << ",\"R99_ms\":" << fmt3(rec.r99_ms) + << ",\"guard_ms\":" << fmt3(rec.guard_ms) + << ",\"first_attempt_send_time\":" + << rec.first_attempt_send_time + << ",\"first_attempt_index\":" << rec.first_attempt_index + << ",\"first_attempt_dropped\":" << rec.first_attempt_dropped + << ",\"retry_attempt_count\":" << rec.retry_attempt_count + << ",\"retries\":["; + for (std::size_t ri = 0; ri < rec.retries.size(); ++ri) { + if (ri != 0) { + samples_jsonl << ","; + } + auto const& r = rec.retries[ri]; + samples_jsonl << "{\"attempt_index\":" << r.attempt_index + << ",\"actual_send_time\":" << r.actual_send_time + << ",\"offset_from_Tn_ms\":" << fmt3(r.offset_from_Tn_ms) + << ",\"logical_cycle_id\":" << r.logical_cycle_id << "}"; + } + samples_jsonl << "],\"first_post_deadline_retry_time\":" + << rec.first_post_deadline_retry_time + << ",\"first_post_deadline_retry_offset_ms\":" + << fmt3(rec.first_post_deadline_retry_offset_ms) + << ",\"confirmation_time\":" << rec.confirmation_time + << ",\"confirmation_offset_from_Tn_ms\":" + << fmt3(rec.confirmation_offset_from_Tn_ms) + << ",\"confirming_attempt_index\":" + << rec.confirming_attempt_index + << ",\"next_nominal_deadline\":" + << rec.next_nominal_deadline + << ",\"expected_next_grid_deadline\":" + << rec.expected_next_grid_deadline + << ",\"phase_error_ms\":" << fmt3(rec.phase_error_ms) + << ",\"duplicate_count\":" << rec.duplicate_count + << ",\"alice_q1\":{"; + if (rec.alice_q1.valid) { + samples_jsonl << "\"query_time\":" << rec.alice_q1.query_time + << ",\"state\":" << rec.alice_q1.state + << ",\"last_online\":" << rec.alice_q1.last_online + << ",\"next_deadline\":" << rec.alice_q1.next_deadline; + } + samples_jsonl << "},\"alice_q2\":{"; + if (rec.alice_q2.valid) { + samples_jsonl << "\"query_time\":" << rec.alice_q2.query_time + << ",\"state\":" << rec.alice_q2.state + << ",\"last_online\":" << rec.alice_q2.last_online + << ",\"next_deadline\":" << rec.alice_q2.next_deadline; + } + samples_jsonl << "},\"fault_armed\":" << (rec.fault_armed ? "true" : "false") + << ",\"fault_matched\":" << (rec.fault_matched ? "true" : "false") + << ",\"fault_dropped\":" << (rec.fault_dropped ? "true" : "false") + << ",\"harness_error\":\"" << rec.harness_error << "\"" + << ",\"production_failures\":\"" << prod_cls << "\"}\n"; + samples_jsonl.flush(); + + cases.push_back(std::move(rec)); + } + + auto collect_valid = [&](auto proj) { + std::vector v; + for (auto const& c : cases) { + if (!c.harness_valid) { + continue; + } + double x = proj(c); + if (std::isfinite(x)) { + v.push_back(x); + } + } + return v; + }; + auto pct = [](std::vector v, double p) { + if (v.empty()) { + return std::numeric_limits::quiet_NaN(); + } + std::sort(v.begin(), v.end()); + double k = (v.size() - 1) * (p / 100.0); + auto f = static_cast(std::floor(k)); + auto c = static_cast(std::ceil(k)); + if (f == c) { + return v[f]; + } + return v[f] * (c - k) + v[c] * (k - f); + }; + auto dstat = [&](std::vector const& v) { + if (v.empty()) { + return std::string("n=0"); + } + auto s = v; + std::sort(s.begin(), s.end()); + return "n=" + std::to_string(s.size()) + " min=" + fmt3(s.front()) + + " p50=" + fmt3(pct(s, 50)) + " p95=" + fmt3(pct(s, 95)) + + " max=" + fmt3(s.back()); + }; + + int valid_n = 0; + int fault_drop_ok = 0; + int pre_deadline_retry_cases = 0; + int post_deadline_recovery_cases = 0; + int same_cycle_cases = 0; + int new_cycle_cases = 0; + int phase_error_nonzero = 0; + int duplicate_total = 0; + int duplicate_cases = 0; + int production_fail_n = 0; + int q1_valid = 0, q1_expected = 0, q1_missed = 0, q1_unknown = 0; + int q2_valid = 0, q2_expected = 0, q2_missed = 0, q2_unknown = 0; + int q2_phase_ok = 0; + int q2_query_failed = 0; + + for (auto const& c : cases) { + if (c.harness_valid) { + ++valid_n; + } + if (c.harness_valid && c.first_attempt_dropped == 1) { + ++fault_drop_ok; + } + bool pre = false; + bool post = false; + for (auto const& r : c.retries) { + if (r.actual_send_time < c.tn_us) { + pre = true; + } + if (r.actual_send_time >= c.tn_us) { + post = true; + } + } + if (pre) { + ++pre_deadline_retry_cases; + } + if (post) { + ++post_deadline_recovery_cases; + } + bool new_cycle = false; + for (auto const& f : c.production_failures) { + if (f == "NEW_CYCLE_BEFORE_CONFIRM") { + new_cycle = true; + } + } + if (c.harness_valid && !new_cycle && post) { + ++same_cycle_cases; + } + if (new_cycle) { + ++new_cycle_cases; + } + if (std::isfinite(c.phase_error_ms) && abs_d(c.phase_error_ms) > tick_ms) { + ++phase_error_nonzero; + } + duplicate_total += c.duplicate_count; + if (c.duplicate_count > 0) { + ++duplicate_cases; + } + if (!c.production_failures.empty()) { + ++production_fail_n; + } + if (c.alice_q1.valid) { + ++q1_valid; + if (c.alice_q1.state == 0) { + ++q1_expected; + } else if (c.alice_q1.state == 1) { + ++q1_missed; + } else { + ++q1_unknown; + } + } + if (c.harness_valid && c.harness_error == "ALICE_Q2_QUERY_FAILED") { + ++q2_query_failed; + } + if (c.alice_q2.valid) { + ++q2_valid; + if (c.alice_q2.state == 0) { + ++q2_expected; + if (c.tn_us != 0 && c.alice_q2.next_deadline != 0) { + auto const expected = expected_grid_after( + c.tn_us, c.alice_q2.next_deadline - 1); + double const e = + static_cast(c.alice_q2.next_deadline - expected) / + 1000.0; + if (abs_d(e) <= tick_ms) { + ++q2_phase_ok; + } + } + } else if (c.alice_q2.state == 1) { + ++q2_missed; + } else { + ++q2_unknown; + } + } + } + + auto post_retry_offsets = collect_valid( + [](RczCase const& c) { return c.first_post_deadline_retry_offset_ms; }); + auto attempts_until = collect_valid([&](RczCase const& c) { + if (c.confirming_attempt_index <= 0) { + return std::numeric_limits::quiet_NaN(); + } + return static_cast(c.confirming_attempt_index); + }); + auto confirm_offsets = + collect_valid([](RczCase const& c) { + return c.confirmation_offset_from_Tn_ms; + }); + auto phase_errors = collect_valid([](RczCase const& c) { + return c.phase_error_ms; + }); + + report << "# ping_retry_count=0 runtime acceptance\n\n"; + report << "- transport: " << args.transport << "\n"; + report << "- seed: " << args.seed << "\n"; + report << "- planned: " << n_cases << "\n"; + report << "- valid: " << valid_n << "\n\n"; + report << "## Harness\n\n"; + report << "- fault correctly dropped: " << fault_drop_ok << "/" << valid_n + << "\n"; + report << "- harness-invalid: " << harness_invalid_n << "\n"; + report << "- wrong-request drops: " << harness_wrong_request << "\n\n"; + report << "## Pre-deadline retries (target 0 cases)\n\n"; + report << "- cases with retry before Tn: " << pre_deadline_retry_cases + << "\n\n"; + report << "## Post-deadline recovery\n\n"; + report << "- cases with retry after Tn: " << post_deadline_recovery_cases + << "/" << valid_n << "\n"; + report << "- first post-deadline retry offset: " + << dstat(post_retry_offsets) << "\n\n"; + report << "## Attempts until confirmation\n\n"; + report << "- " << dstat(attempts_until) << "\n\n"; + report << "## Confirmation offset from Tn\n\n"; + report << "- " << dstat(confirm_offsets) << "\n\n"; + report << "## Logical-cycle integrity\n\n"; + report << "- same-cycle recovery: " << same_cycle_cases << "/" << valid_n + << "\n"; + report << "- unexpected new-cycle: " << new_cycle_cases << "\n\n"; + report << "## Phase\n\n"; + report << "- phase_error: " << dstat(phase_errors) << "\n"; + report << "- phase_error != 0: " << phase_error_nonzero << "\n\n"; + report << "## Duplicates\n\n"; + report << "- total: " << duplicate_total << "\n"; + report << "- cases with duplicates: " << duplicate_cases << "\n\n"; + report << "## Alice Q1 (diagnostic)\n\n"; + report << "- valid: " << q1_valid << " Expected: " << q1_expected + << " MissedDeadline: " << q1_missed << " Unknown: " << q1_unknown + << "\n\n"; + report << "## Alice Q2 (recovery check)\n\n"; + report << "- valid: " << q2_valid << " Expected: " << q2_expected + << " MissedDeadline: " << q2_missed << " Unknown: " << q2_unknown + << "\n"; + report << "- Q2 future deadline phase-correct: " << q2_phase_ok << "/" + << q2_expected << "\n"; + report << "- Q2 query failed (harness): " << q2_query_failed << "\n\n"; + report << "## Failures\n\n"; + report << "- production failures: " << production_fail_n << "\n"; + report << "- harness failures: " << harness_invalid_n << "\n"; + report << "- alice Q2 query failures: " << q2_query_failed << "\n"; + + summary_json << "{\n" + << " \"transport\": \"" << args.transport << "\",\n" + << " \"seed\": " << args.seed << ",\n" + << " \"planned\": " << n_cases << ",\n" + << " \"valid\": " << valid_n << ",\n" + << " \"harness\": {\n" + << " \"fault_correctly_dropped\": " << fault_drop_ok + << ",\n" + << " \"harness_invalid\": " << harness_invalid_n << ",\n" + << " \"alice_q2_query_failed\": " << q2_query_failed << "\n" + << " },\n" + << " \"pre_deadline_retry_cases\": " + << pre_deadline_retry_cases << ",\n" + << " \"post_deadline_recovery_cases\": " + << post_deadline_recovery_cases << ",\n" + << " \"same_cycle_recovery_cases\": " << same_cycle_cases + << ",\n" + << " \"unexpected_new_cycle_cases\": " << new_cycle_cases + << ",\n" + << " \"phase_error_nonzero_count\": " << phase_error_nonzero + << ",\n" + << " \"duplicate_total\": " << duplicate_total << ",\n" + << " \"production_failures\": " << production_fail_n << ",\n" + << " \"harness_failures\": " << harness_invalid_n << ",\n" + << " \"alice_q1\": {\"valid\": " << q1_valid + << ", \"expected\": " << q1_expected + << ", \"missed_deadline\": " << q1_missed + << ", \"unknown\": " << q1_unknown << "},\n" + << " \"alice_q2\": {\"valid\": " << q2_valid + << ", \"expected\": " << q2_expected + << ", \"missed_deadline\": " << q2_missed + << ", \"unknown\": " << q2_unknown + << ", \"phase_correct\": " << q2_phase_ok << "}\n" + << "}\n"; + + { + auto const parent = + std::filesystem::path{args.artifact_dir}.parent_path(); + auto const sibling_name = + args.transport == "tcp" ? "udp/summary.json" : "tcp/summary.json"; + auto const sibling = parent / sibling_name; + if (std::filesystem::exists(sibling)) { + std::ofstream cmp(parent / "comparison.md"); + cmp << "# TCP vs UDP — ping_retry_count=0 runtime acceptance\n\n"; + cmp << "| metric | tcp | udp |\n"; + cmp << "| --- | --- | --- |\n"; + cmp << "| artifact | tcp/ | udp/ |\n"; + cmp << "| run order | first | second |\n"; + cmp.flush(); + } + } + + std::cout << (prod_ok && harness_invalid_n == 0 && q2_query_failed == 0 + ? "PASS" + : "FAIL") + << " retry-count-zero-runtime cases=" << cases.size() + << " valid=" << valid_n + << " pre_deadline_retry_cases=" << pre_deadline_retry_cases + << " post_deadline_recovery=" << post_deadline_recovery_cases + << " production_fail=" << production_fail_n + << " harness_invalid=" << harness_invalid_n + << " alice_q2_query_failed=" << q2_query_failed << std::endl; + + StopChild(alice); + StopChild(bob); + return (prod_ok && harness_invalid_n == 0 && q2_query_failed == 0) ? 0 + : 7; diff --git a/examples/aether_uap_1s_timing_characterization/tele_off.h b/examples/aether_uap_1s_timing_characterization/tele_off.h new file mode 100644 index 00000000..89da4ca9 --- /dev/null +++ b/examples/aether_uap_1s_timing_characterization/tele_off.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Load USER_CONFIG (and the rest of aether/config.h) first, then override +// console telemetry for this benchmark/example target. Force-include this +// header so the override wins without a conflicting /D AE_TELE_LOG_CONSOLE. +#include "aether/config.h" + +#undef AE_TELE_LOG_CONSOLE +#define AE_TELE_LOG_CONSOLE 0 diff --git a/examples/aether_uap_peer_deadline_test/CMakeLists.txt b/examples/aether_uap_peer_deadline_test/CMakeLists.txt new file mode 100644 index 00000000..f1fcd14e --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/CMakeLists.txt @@ -0,0 +1,59 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM AND WIN32) + project("aether_uap_peer_deadline_test" VERSION "1.0.0" LANGUAGES C CXX) + + add_library(aether_uap_peer_deadline_test_common STATIC + common/deadline_ipc.cpp + ) + target_include_directories(aether_uap_peer_deadline_test_common PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/common + ) + target_link_libraries(aether_uap_peer_deadline_test_common PUBLIC aether) + + add_executable(aether_uap_peer_deadline_test + main.cpp + client_role.cpp + coordinator.cpp + ) + target_link_libraries(aether_uap_peer_deadline_test PRIVATE + aether_uap_peer_deadline_test_common + ) + target_include_directories(aether_uap_peer_deadline_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../.. + ) + target_compile_definitions(aether_uap_peer_deadline_test PRIVATE + _CRT_SECURE_NO_WARNINGS + ) + if(MSVC) + target_compile_options(aether_uap_peer_deadline_test PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + target_compile_options(aether_uap_peer_deadline_test_common PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + endif() +else() + message(WARNING "aether_uap_peer_deadline_test is Windows desktop only; skipped") +endif() diff --git a/examples/aether_uap_peer_deadline_test/client_role.cpp b/examples/aether_uap_peer_deadline_test/client_role.cpp new file mode 100644 index 00000000..0df72421 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/client_role.cpp @@ -0,0 +1,844 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client_role.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#if defined(RegisterClass) +# undef RegisterClass +#endif + +#define AE_EXAMPLE_ETHERNET 1 +#include "aether/all.h" +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/receive_schedule.h" + +#include "common/deadline_ipc.h" +#include "common/directory_domain_storage.h" +#include "missed_deadline.h" + +namespace ae::test_uap_peer_deadline { +namespace { + +constexpr auto kBobPingInterval = std::chrono::milliseconds{3000}; +constexpr auto kBobReceiveWindow = std::chrono::milliseconds{1000}; +constexpr auto kQueryRetry = std::chrono::milliseconds{250}; +constexpr auto kPastDeadlineMargin = std::chrono::milliseconds{500}; +constexpr auto kAfterMissMargin = std::chrono::milliseconds{250}; +constexpr auto kConfirmGap = std::chrono::milliseconds{1000}; +constexpr auto kBeforeDeadlineLead = std::chrono::milliseconds{500}; +constexpr auto kRecoveryTimeout = std::chrono::seconds{5}; +constexpr auto kStabilizeTimeout = std::chrono::seconds{45}; +constexpr auto kLiveCycleTimeout = std::chrono::seconds{20}; + +inline std::int64_t TimePointUs(TimePoint tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +inline std::int64_t MsBetween(TimePoint later, TimePoint earlier) { + return std::chrono::duration_cast(later - earlier) + .count(); +} + +inline void UidToHalves(Uid const& uid, std::int64_t& lo, std::int64_t& hi) { + std::memcpy(&lo, uid.value.data(), 8); + std::memcpy(&hi, uid.value.data() + 8, 8); +} + +inline Uid UidFromHalves(std::int64_t lo, std::int64_t hi) { + Uid uid{}; + std::memcpy(uid.value.data(), &lo, 8); + std::memcpy(uid.value.data() + 8, &hi, 8); + return uid; +} + +struct QueryRow { + std::string phase; + std::uint32_t query_index{0}; + std::int64_t query_begin{0}; + std::int64_t query_end{0}; + std::int64_t last_online{0}; + std::int64_t next_ping_deadline{-1}; + std::int64_t now{0}; + int last_online_advanced{0}; + int deadline_passed{0}; + int query_success{0}; +}; + +struct QueryOutcome { + bool success{false}; + int error{0}; + PeerReceiveSchedule schedule{}; + TimePoint begin{}; + TimePoint end{}; + TimePoint now{}; +}; + +enum class TestPhase { + kIdle, + kStabilize, + kLive, + kBeforeKill, + kWaitKillAck, + kBeforeDeadline, + kAfterMiss, + kAfterMissConfirm, + kWaitRestartAck, + kRecovery, + kUnknown, + kDone, +}; + +struct RoleState { + Side side{}; + std::uint32_t run_id_hash{0}; + NamedPipeClient pipe; + std::unique_ptr app; + Client::ptr client; + Uid peer_uid{}; + bool peer_set{false}; + Subscription select_sub; + Subscription query_sub; + std::uint32_t ipc_seq{0}; + bool exit_requested{false}; + bool client_ready{false}; + bool cloud_started{false}; + bool test_running{false}; + TestPhase phase{TestPhase::kIdle}; + TimePoint test_start_{}; + TimePoint phase_deadline_{}; + TimePoint wait_until_{}; + TimePoint missed_deadline_{}; + TimePoint missed_anchor_last_online_{}; + TimePoint recovery_start_{}; + PeerReceiveSchedule previous_{}; + PeerReceiveSchedule p0_{}; + bool have_previous_{false}; + bool have_p0_{false}; + int live_cycle_{0}; + int advances_seen_{0}; + int false_missed_{0}; + std::uint32_t query_index_{0}; + bool query_inflight_{false}; + std::optional last_query_{}; + std::vector csv_rows_; + std::string artifact_dir_; + std::string fail_reason_; + bool passed_{false}; + std::int64_t recovery_ms_{-1}; + std::int64_t ping_interval_ms{3000}; + std::int64_t deadline_late_by_ms_{-1}; + bool skip_before_deadline_{false}; + + bool Emit(IpcType type, std::uint32_t code = 0, std::int64_t a = 0, + std::int64_t b = 0, std::int64_t c = 0) { + IpcFrame f{}; + f.type = static_cast(type); + f.side = static_cast(side); + f.run_id_hash = run_id_hash; + f.seq = ++ipc_seq; + f.code = code; + f.local_us = TimePointUs(Now()); + f.a = a; + f.b = b; + f.c = c; + return pipe.WriteFrame(f); + } + + void Fail(std::string reason) { + fail_reason_ = std::move(reason); + passed_ = false; + phase = TestPhase::kDone; + Emit(IpcType::kTestDone, 1); + exit_requested = true; + } + + void Pass() { + passed_ = true; + phase = TestPhase::kDone; + Emit(IpcType::kTestDone, 0, false_missed_, recovery_ms_, + deadline_late_by_ms_); + exit_requested = true; + } + + void WriteCsv() { + if (artifact_dir_.empty()) { + return; + } + auto path = artifact_dir_ + "/queries.csv"; + std::ofstream out(path, std::ios::out | std::ios::trunc); + out << "phase,query_index,query_begin,query_end,last_online," + "next_ping_deadline,now,last_online_advanced,deadline_passed," + "query_success\n"; + auto const origin = TimePointUs(test_start_); + for (auto const& r : csv_rows_) { + out << r.phase << ',' << r.query_index << ',' << (r.query_begin - origin) + << ',' << (r.query_end - origin) << ',' << (r.last_online - origin) + << ',' + << (r.next_ping_deadline < 0 ? -1 : (r.next_ping_deadline - origin)) + << ',' << (r.now - origin) << ',' << r.last_online_advanced << ',' + << r.deadline_passed << ',' << r.query_success << '\n'; + } + } + + void RecordCsv(std::string const& phase_name, QueryOutcome const& q, + PeerReceiveSchedule const* prev) { + QueryRow row; + row.phase = phase_name; + row.query_index = query_index_; + row.query_begin = TimePointUs(q.begin); + row.query_end = TimePointUs(q.end); + row.now = TimePointUs(q.now); + row.query_success = q.success ? 1 : 0; + if (q.success) { + row.last_online = TimePointUs(q.schedule.last_online); + row.next_ping_deadline = + q.schedule.next_ping_deadline.has_value() + ? TimePointUs(*q.schedule.next_ping_deadline) + : -1; + if (prev != nullptr) { + row.last_online_advanced = + IsLastOnlineAdvanced(prev->last_online, q.schedule.last_online) ? 1 : 0; + if (prev->next_ping_deadline.has_value()) { + row.deadline_passed = + q.now > *prev->next_ping_deadline ? 1 : 0; + } + } + } + csv_rows_.push_back(row); + } + + void BeginQuery() { + if (!client || !peer_set || query_inflight_) { + return; + } + query_inflight_ = true; + last_query_.reset(); + auto const begin = Now(); + ++query_index_; + query_sub.Reset(); + auto& action = client->QueryPeerReceiveSchedule(peer_uid); + query_sub = action.result_event().Subscribe( + [this, begin, &action](Result const& res) { + QueryOutcome out; + out.begin = begin; + out.end = Now(); + out.now = out.end; + if (!res) { + out.success = false; + out.error = res.error(); + } else { + out.success = true; + out.schedule = res.value(); + } + std::cerr << "timing_diag query=" << query_index_; + for (auto const& d : action.server_diagnostics()) { + std::cerr << " srv=" << d.server_id + << " status=" << static_cast(d.status); + if (d.has_raw) { + std::cerr << " next_delta_ms=" << d.raw.next_ping_delta_ms + << " last_connect_ms=" << d.raw.last_connect_delta_ms; + } + } + std::cerr << std::endl; + last_query_ = out; + query_inflight_ = false; + }); + } + + bool WaitUntilAe(TimePoint until) { + return Now() >= until; + } + + void StartStabilize() { + phase = TestPhase::kStabilize; + phase_deadline_ = Now() + kStabilizeTimeout; + have_previous_ = false; + advances_seen_ = 0; + wait_until_ = Now(); + BeginQuery(); + } + + void StartUnknown() { + phase = TestPhase::kUnknown; + phase_deadline_ = Now() + std::chrono::seconds{8}; + wait_until_ = Now(); + BeginQuery(); + } + + void OnStabilizeTick() { + if (query_inflight_) { + return; + } + if (last_query_.has_value()) { + auto const& q = *last_query_; + RecordCsv("stabilize", q, have_previous_ ? &previous_ : nullptr); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + // Do not apply MISSED_DEADLINE during stabilize: a slow query can finish + // after the previous deadline without implying Bob is offline. + if (!q.schedule.next_ping_deadline.has_value() || + !(*q.schedule.next_ping_deadline > q.now)) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (have_previous_ && + IsLastOnlineAdvanced(previous_.last_online, q.schedule.last_online)) { + ++advances_seen_; + } + previous_ = q.schedule; + have_previous_ = true; + // Need initial valid schedule + at least one advance (= 2 ping cycles). + if (advances_seen_ >= 1) { + p0_ = q.schedule; + have_p0_ = true; + auto const now = q.now; + auto const age_ms = MsBetween(now, p0_.last_online); + auto const until_ms = MsBetween(*p0_.next_ping_deadline, now); + std::cout << "INITIAL last_online_age_ms=" << age_ms + << " until_next_ping_ms=" << until_ms << std::endl; + std::cout << "## Live Bob\n" + << "cycle last_online_advanced until_next_ping_ms\n"; + std::cout << "0 yes " << until_ms << std::endl; + live_cycle_ = 0; + phase = TestPhase::kLive; + wait_until_ = *p0_.next_ping_deadline + kPastDeadlineMargin; + phase_deadline_ = wait_until_ + kLiveCycleTimeout; + last_query_.reset(); + return; + } + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (WaitUntilAe(wait_until_)) { + if (Now() > phase_deadline_) { + Fail("stabilize timeout"); + return; + } + BeginQuery(); + } + } + + void OnLiveTick() { + if (!WaitUntilAe(wait_until_)) { + return; + } + if (query_inflight_) { + return; + } + if (!last_query_.has_value()) { + BeginQuery(); + return; + } + auto const& q = *last_query_; + RecordCsv("live", q, &previous_); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (IsMissedDeadline(previous_, q.schedule, q.now)) { + // Only after we intentionally waited past the promised deadline. + ++false_missed_; + auto const prev_until = + previous_.next_ping_deadline.has_value() + ? MsBetween(*previous_.next_ping_deadline, q.now) + : -1; + std::cerr << "FALSE_MISSED_DEADLINE previous_last_online_rel_us=" + << (TimePointUs(previous_.last_online) - TimePointUs(test_start_)) + << " previous_deadline_rel_ms=" << prev_until + << " query_last_online_rel_us=" + << (TimePointUs(q.schedule.last_online) - TimePointUs(test_start_)) + << " query_latency_ms=" << MsBetween(q.end, q.begin) + << std::endl; + Fail("FALSE_MISSED_DEADLINE"); + return; + } + if (!IsLastOnlineAdvanced(previous_.last_online, q.schedule.last_online) || + !q.schedule.next_ping_deadline.has_value() || + !(*q.schedule.next_ping_deadline > q.now)) { + // Harness race: deadline+margin may still land before the server + // publishes the new last_online. Retry until advance or false miss. + if (Now() > phase_deadline_) { + Fail("live last_online did not advance"); + return; + } + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + ++live_cycle_; + auto const until_ms = MsBetween(*q.schedule.next_ping_deadline, q.now); + std::cout << live_cycle_ << " yes " << until_ms + << std::endl; + previous_ = q.schedule; + last_query_.reset(); + if (live_cycle_ >= 2) { + // P0, P1, P2 => live_cycle_ 0 printed at stabilize, then 1 and 2 here. + phase = TestPhase::kBeforeKill; + wait_until_ = Now(); + phase_deadline_ = Now() + kLiveCycleTimeout; + return; + } + wait_until_ = *q.schedule.next_ping_deadline + kPastDeadlineMargin; + phase_deadline_ = wait_until_ + kLiveCycleTimeout; + } + + void OnBeforeKillTick() { + if (query_inflight_) { + return; + } + if (!last_query_.has_value()) { + BeginQuery(); + return; + } + auto const& q = *last_query_; + RecordCsv("before_kill", q, &previous_); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (!q.schedule.next_ping_deadline.has_value() || + !(*q.schedule.next_ping_deadline > q.now)) { + Fail("BeforeKill missing future deadline"); + return; + } + // Need headroom so an in-flight Bob ping is unlikely to land after kill + // and look like a post-deadline advance. + auto const until_ms = + MsBetween(*q.schedule.next_ping_deadline, q.now); + if (until_ms < 1500) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + if (Now() > phase_deadline_) { + Fail("BeforeKill could not obtain schedule with headroom"); + } + return; + } + missed_anchor_last_online_ = q.schedule.last_online; + missed_deadline_ = *q.schedule.next_ping_deadline; + previous_ = q.schedule; + last_query_.reset(); + phase = TestPhase::kWaitKillAck; + Emit(IpcType::kRequestBobKill); + } + + void OnBeforeDeadlineTick() { + if (skip_before_deadline_) { + phase = TestPhase::kAfterMiss; + wait_until_ = missed_deadline_ + kAfterMissMargin; + return; + } + auto const target = missed_deadline_ - kBeforeDeadlineLead; + if (!WaitUntilAe(target)) { + return; + } + if (Now() >= missed_deadline_) { + skip_before_deadline_ = true; + phase = TestPhase::kAfterMiss; + wait_until_ = missed_deadline_ + kAfterMissMargin; + return; + } + if (query_inflight_) { + return; + } + if (!last_query_.has_value()) { + BeginQuery(); + return; + } + auto const& q = *last_query_; + RecordCsv("before_deadline", q, &previous_); + if (!q.success) { + // Skip phase rather than stall near deadline. + phase = TestPhase::kAfterMiss; + wait_until_ = missed_deadline_ + kAfterMissMargin; + last_query_.reset(); + return; + } + // Still not offline; last_online may equal anchor. + previous_ = q.schedule; + last_query_.reset(); + phase = TestPhase::kAfterMiss; + wait_until_ = missed_deadline_ + kAfterMissMargin; + } + + void OnAfterMissTick() { + if (!WaitUntilAe(wait_until_)) { + return; + } + if (query_inflight_) { + return; + } + if (!last_query_.has_value()) { + BeginQuery(); + return; + } + auto const& q = *last_query_; + RecordCsv("after_miss", q, &previous_); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + if (Now() > missed_deadline_ + std::chrono::seconds{10}) { + Fail("after_miss query timeout"); + } + return; + } + if (!(q.now > missed_deadline_)) { + wait_until_ = missed_deadline_ + kAfterMissMargin; + last_query_.reset(); + return; + } + if (IsLastOnlineAdvanced(missed_anchor_last_online_, q.schedule.last_online)) { + Fail("AfterMiss last_online advanced unexpectedly"); + return; + } + if (q.schedule.state == PeerScheduleState::kUnknown) { + std::cerr << "AfterMiss aggregate Unknown (non-deterministic servers); " + "last_online frozen, continuing\n"; + } else if (!IsMissedDeadline(previous_, q.schedule, q.now)) { + Fail("AfterMiss classification failed"); + return; + } + deadline_late_by_ms_ = MsBetween(q.now, missed_deadline_); + previous_ = q.schedule; + last_query_.reset(); + phase = TestPhase::kAfterMissConfirm; + wait_until_ = Now() + kConfirmGap; + } + + void OnAfterMissConfirmTick() { + if (!WaitUntilAe(wait_until_)) { + return; + } + if (query_inflight_) { + return; + } + if (!last_query_.has_value()) { + BeginQuery(); + return; + } + auto const& q = *last_query_; + RecordCsv("after_miss_confirm", q, &previous_); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (IsLastOnlineAdvanced(missed_anchor_last_online_, q.schedule.last_online)) { + Fail("AfterMissConfirm last_online not frozen"); + return; + } + std::cout << "## Missed deadline\n" + << "last_online_frozen: yes\n" + << "deadline_passed_ms: " << deadline_late_by_ms_ << "\n" + << "confirmation_after_ms: 1000\n"; + std::cout << "MISSED_DEADLINE deadline_late_by_ms=" << deadline_late_by_ms_ + << " last_online_unchanged=1" << std::endl; + previous_ = q.schedule; + last_query_.reset(); + phase = TestPhase::kWaitRestartAck; + Emit(IpcType::kRequestBobRestart); + } + + void OnRecoveryTick() { + if (query_inflight_) { + return; + } + if (last_query_.has_value()) { + auto const& q = *last_query_; + RecordCsv("recovery", q, &previous_); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (IsLastOnlineAdvanced(missed_anchor_last_online_, q.schedule.last_online) && + q.schedule.next_ping_deadline.has_value() && + *q.schedule.next_ping_deadline > q.now) { + recovery_ms_ = MsBetween(q.now, recovery_start_); + auto const until_ms = + MsBetween(*q.schedule.next_ping_deadline, q.now); + std::cout << "## Bob return\n" + << "last_online_advanced: yes\n" + << "recovery_ms: " << recovery_ms_ << "\n"; + std::cout << "RETURNED new_last_online=1 until_next_ping_ms=" << until_ms + << std::endl; + WriteCsv(); + if (false_missed_ != 0) { + Fail("false missed deadline count != 0"); + return; + } + std::cout << "UAP_PEER_DEADLINE_TEST PASS" << std::endl; + Pass(); + return; + } + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (Now() > recovery_start_ + kRecoveryTimeout) { + Fail("recovery timeout"); + return; + } + if (WaitUntilAe(wait_until_)) { + BeginQuery(); + } + } + + void OnUnknownTick() { + if (query_inflight_) { + return; + } + if (last_query_.has_value()) { + auto const& q = *last_query_; + RecordCsv("unknown", q, have_previous_ ? &previous_ : nullptr); + if (!q.success) { + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (q.schedule.state == PeerScheduleState::kUnknown && + !q.schedule.next_ping_deadline.has_value()) { + std::cout << "UNKNOWN_SCHEDULE state=Unknown next_ping_deadline=nullopt" + << std::endl; + WriteCsv(); + std::cout << "UAP_PEER_DEADLINE_UNKNOWN PASS" << std::endl; + Pass(); + return; + } + wait_until_ = Now() + kQueryRetry; + last_query_.reset(); + return; + } + if (Now() > phase_deadline_) { + Fail("unknown schedule timeout"); + return; + } + if (WaitUntilAe(wait_until_)) { + BeginQuery(); + } + } + + void TickTest() { + if (!test_running || side != Side::kA) { + return; + } + switch (phase) { + case TestPhase::kStabilize: + OnStabilizeTick(); + break; + case TestPhase::kLive: + OnLiveTick(); + break; + case TestPhase::kBeforeKill: + OnBeforeKillTick(); + break; + case TestPhase::kWaitKillAck: + case TestPhase::kWaitRestartAck: + break; + case TestPhase::kBeforeDeadline: + OnBeforeDeadlineTick(); + break; + case TestPhase::kAfterMiss: + OnAfterMissTick(); + break; + case TestPhase::kAfterMissConfirm: + OnAfterMissConfirmTick(); + break; + case TestPhase::kRecovery: + OnRecoveryTick(); + break; + case TestPhase::kUnknown: + OnUnknownTick(); + break; + default: + break; + } + } + + void HandleIpc(IpcFrame const& f) { + auto const type = static_cast(f.type); + switch (type) { + case IpcType::kSetPeerUid: + peer_uid = UidFromHalves(f.a, f.b); + peer_set = true; + Emit(IpcType::kAck); + break; + case IpcType::kStartCloud: + if (client) { + (void)client->cloud_connection(); + cloud_started = true; + Emit(IpcType::kCloudStarted); + } + break; + case IpcType::kRunTest: + if (side != Side::kA || !client || !peer_set) { + Fail("Alice not ready for RunTest"); + return; + } + (void)client->cloud_connection(); + test_start_ = Now(); + test_running = true; + if (f.code == 1) { + StartUnknown(); + } else { + StartStabilize(); + } + break; + case IpcType::kBobKilled: + if (phase == TestPhase::kWaitKillAck) { + auto const remaining = missed_deadline_ - Now(); + if (remaining > kBeforeDeadlineLead + std::chrono::milliseconds{50}) { + phase = TestPhase::kBeforeDeadline; + skip_before_deadline_ = false; + } else { + phase = TestPhase::kAfterMiss; + wait_until_ = missed_deadline_ + kAfterMissMargin; + skip_before_deadline_ = true; + } + } + break; + case IpcType::kBobRestarted: + if (phase == TestPhase::kWaitRestartAck) { + phase = TestPhase::kRecovery; + recovery_start_ = Now(); + wait_until_ = Now(); + } + break; + case IpcType::kFlushState: + // Persist domain only — no UAP graceful announce / setNextReadDelay(0). + if (app && app->aether()) { + app->aether().Save(); + } + Emit(IpcType::kAck); + break; + case IpcType::kShutdown: + exit_requested = true; + Emit(IpcType::kAck); + break; + default: + break; + } + } +}; + +std::unique_ptr MakeApp(std::string const& state_dir) { + return AetherApp::Construct( + AetherAppContext{[state_dir]() { + return std::unique_ptr{ + std::make_unique(state_dir)}; + }} +#if AE_DISTILLATION + .AddAdapterFactory([](AetherAppContext const& context) { + return EthernetAdapter::ptr::Create( + CreateWith{context.domain()}.with_id( + GlobalId::kEthernetAdapter), + context.aether(), context.poller(), context.dns_resolver()); + }) +#endif + ); +} + +} // namespace + +int RunClientRole(ClientArgs const& args) { + RoleState state; + state.side = args.side; + state.run_id_hash = HashRunId(args.run_id); + state.artifact_dir_ = args.artifact_dir; + + if (!state.pipe.Connect(args.pipe_name, 60000)) { + std::cerr << "pipe connect failed: " << args.pipe_name << "\n"; + return 2; + } + + state.ping_interval_ms = args.ping_interval_ms; + state.app = MakeApp(args.state_dir); + auto parent = Uid::FromString(args.parent_uid); + auto& select = + state.app->aether()->SelectClient(parent, args.client_name); + state.select_sub = select.result_event().Subscribe( + [&](Result const& res) { + if (!res) { + state.Emit(IpcType::kTestDone, 2); + state.exit_requested = true; + return; + } + state.client = res.value(); + if (state.side == Side::kB) { + auto ok = state.client->SetReceiveSchedule(ReceiveSchedule{ + .ping_interval = std::chrono::duration_cast( + std::chrono::milliseconds{state.ping_interval_ms}), + .receive_window = + std::chrono::duration_cast(kBobReceiveWindow), + }); + if (!ok) { + state.Emit(IpcType::kTestDone, 3); + state.exit_requested = true; + return; + } + } + state.client_ready = true; + // Ensure clients_ map is on disk before a later hard-kill. + state.app->aether().Save(); + std::int64_t lo = 0; + std::int64_t hi = 0; + UidToHalves(state.client->uid(), lo, hi); + state.Emit(IpcType::kUidReport, 0, lo, hi); + state.Emit(IpcType::kChildReady); + }); + + while (!state.exit_requested && !state.app->IsExited()) { + auto const now = Now(); + auto next = state.app->Update(now); + if (auto frame = state.pipe.TryReadFrame(0)) { + state.HandleIpc(*frame); + } + state.TickTest(); + state.app->WaitUntil( + std::min(next, now + std::chrono::milliseconds{5})); + } + + if (state.side == Side::kA && !state.fail_reason_.empty()) { + std::cerr << "UAP_PEER_DEADLINE_TEST FAIL " << state.fail_reason_ + << std::endl; + state.WriteCsv(); + } + return state.passed_ ? 0 : (state.side == Side::kA ? 1 : 0); +} + +} // namespace ae::test_uap_peer_deadline diff --git a/examples/aether_uap_peer_deadline_test/client_role.h b/examples/aether_uap_peer_deadline_test/client_role.h new file mode 100644 index 00000000..2fbae6ad --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/client_role.h @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_CLIENT_ROLE_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_CLIENT_ROLE_H_ + +#include +#include + +#include "common/deadline_types.h" + +namespace ae::test_uap_peer_deadline { + +struct ClientArgs { + Side side{Side::kA}; + std::string run_id; + std::string state_dir; + std::string pipe_name; + std::string client_name; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; + std::string artifact_dir; + std::int64_t ping_interval_ms{3000}; +}; + +int RunClientRole(ClientArgs const& args); + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_CLIENT_ROLE_H_ diff --git a/examples/aether_uap_peer_deadline_test/common/crc32.h b/examples/aether_uap_peer_deadline_test/common/crc32.h new file mode 100644 index 00000000..9ec220fa --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/common/crc32.h @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_CRC32_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_CRC32_H_ + +#include +#include + +namespace ae::test_uap_peer_deadline { + +inline std::uint32_t Crc32(void const* data, std::size_t size) noexcept { + auto const* bytes = static_cast(data); + std::uint32_t crc = 0xFFFFFFFFu; + for (std::size_t i = 0; i < size; ++i) { + crc ^= bytes[i]; + for (int b = 0; b < 8; ++b) { + auto const mask = + static_cast(-(static_cast(crc & 1u))); + crc = (crc >> 1) ^ (0xEDB88320u & mask); + } + } + return ~crc; +} + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_CRC32_H_ diff --git a/examples/aether_uap_peer_deadline_test/common/deadline_ipc.cpp b/examples/aether_uap_peer_deadline_test/common/deadline_ipc.cpp new file mode 100644 index 00000000..3eafbb97 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/common/deadline_ipc.cpp @@ -0,0 +1,268 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "deadline_ipc.h" + +#include + +#if defined(_WIN32) +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#endif + +namespace ae::test_uap_peer_deadline { + +std::uint32_t IpcFrameCrc(IpcFrame const& frame) noexcept { + IpcFrame tmp = frame; + tmp.crc = 0; + return Crc32(&tmp, sizeof(tmp)); +} + +void EncodeIpcFrame(IpcFrame& frame) noexcept { + frame.magic = kIpcMagic; + frame.version = kIpcVersion; + frame.crc = IpcFrameCrc(frame); +} + +bool DecodeIpcFrame(void const* data, std::size_t size, + IpcFrame& out) noexcept { + if (data == nullptr || size < sizeof(IpcFrame)) { + return false; + } + std::memcpy(&out, data, sizeof(IpcFrame)); + if (out.magic != kIpcMagic || out.version != kIpcVersion) { + return false; + } + return IpcFrameCrc(out) == out.crc; +} + +std::string PipeNameFor(std::string const& run_id, Side side) { + auto const s = side == Side::kA ? "a" : (side == Side::kB ? "b" : "c"); + return "\\\\.\\pipe\\aether-uap-deadline-" + run_id + "-" + s; +} + +std::uint32_t HashRunId(std::string const& run_id) { + std::uint32_t h = 2166136261u; + for (char c : run_id) { + h ^= static_cast(c); + h *= 16777619u; + } + return h; +} + +#if defined(_WIN32) + +namespace { + +bool OverlappedWait(HANDLE handle, OVERLAPPED& ov, DWORD timeout_ms, + DWORD* transferred) { + DWORD bytes = 0; + if (GetOverlappedResult(handle, &ov, &bytes, FALSE)) { + if (transferred != nullptr) { + *transferred = bytes; + } + return true; + } + if (GetLastError() != ERROR_IO_INCOMPLETE) { + return false; + } + auto const wait = WaitForSingleObject(ov.hEvent, timeout_ms); + if (wait != WAIT_OBJECT_0) { + CancelIoEx(handle, &ov); + return false; + } + if (!GetOverlappedResult(handle, &ov, &bytes, FALSE)) { + return false; + } + if (transferred != nullptr) { + *transferred = bytes; + } + return true; +} + +} // namespace + +NamedPipeServer::~NamedPipeServer() { Close(); } + +bool NamedPipeServer::Create(std::string const& pipe_name) { + Close(); + handle_ = CreateNamedPipeA( + pipe_name.c_str(), PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, 1, 4096, 4096, 0, + nullptr); + return handle_ != INVALID_HANDLE_VALUE && handle_ != nullptr; +} + +bool NamedPipeServer::WaitForClient(std::uint32_t timeout_ms) { + if (handle_ == nullptr || handle_ == INVALID_HANDLE_VALUE) { + return false; + } + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return false; + } + auto connected = ConnectNamedPipe(static_cast(handle_), &ov); + if (connected) { + CloseHandle(ov.hEvent); + return true; + } + auto const err = GetLastError(); + if (err == ERROR_PIPE_CONNECTED) { + CloseHandle(ov.hEvent); + return true; + } + if (err != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return false; + } + auto const ok = OverlappedWait(static_cast(handle_), ov, timeout_ms, + nullptr); + CloseHandle(ov.hEvent); + return ok; +} + +bool NamedPipeServer::WriteFrame(IpcFrame frame) { + EncodeIpcFrame(frame); + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return false; + } + DWORD written = 0; + auto ok = WriteFile(static_cast(handle_), &frame, sizeof(frame), + &written, &ov); + if (!ok) { + if (GetLastError() != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return false; + } + ok = OverlappedWait(static_cast(handle_), ov, 5000, &written); + } + CloseHandle(ov.hEvent); + return ok && written == sizeof(frame); +} + +std::optional NamedPipeServer::TryReadFrame( + std::uint32_t timeout_ms) { + IpcFrame frame{}; + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return std::nullopt; + } + DWORD read = 0; + auto ok = ReadFile(static_cast(handle_), &frame, sizeof(frame), &read, + &ov); + if (!ok) { + if (GetLastError() != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return std::nullopt; + } + ok = OverlappedWait(static_cast(handle_), ov, timeout_ms, &read); + } + CloseHandle(ov.hEvent); + if (!ok || read < sizeof(IpcFrame)) { + return std::nullopt; + } + IpcFrame out{}; + if (!DecodeIpcFrame(&frame, sizeof(frame), out)) { + return std::nullopt; + } + return out; +} + +void NamedPipeServer::Close() { + if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(static_cast(handle_)); + } + handle_ = nullptr; +} + +NamedPipeClient::~NamedPipeClient() { Close(); } + +bool NamedPipeClient::Connect(std::string const& pipe_name, + std::uint32_t timeout_ms) { + Close(); + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + handle_ = CreateFileA(pipe_name.c_str(), GENERIC_READ | GENERIC_WRITE, 0, + nullptr, OPEN_EXISTING, 0, nullptr); + if (handle_ != INVALID_HANDLE_VALUE) { + DWORD mode = PIPE_READMODE_MESSAGE; + SetNamedPipeHandleState(static_cast(handle_), &mode, nullptr, + nullptr); + return true; + } + if (GetLastError() != ERROR_PIPE_BUSY) { + Sleep(50); + continue; + } + WaitNamedPipeA(pipe_name.c_str(), 200); + } + handle_ = nullptr; + return false; +} + +bool NamedPipeClient::WriteFrame(IpcFrame frame) { + EncodeIpcFrame(frame); + DWORD written = 0; + return WriteFile(static_cast(handle_), &frame, sizeof(frame), + &written, nullptr) && + written == sizeof(frame); +} + +std::optional NamedPipeClient::TryReadFrame( + std::uint32_t timeout_ms) { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() <= deadline) { + DWORD avail = 0; + if (!PeekNamedPipe(static_cast(handle_), nullptr, 0, nullptr, + &avail, nullptr)) { + return std::nullopt; + } + if (avail < sizeof(IpcFrame)) { + Sleep(5); + continue; + } + IpcFrame frame{}; + DWORD read = 0; + if (!ReadFile(static_cast(handle_), &frame, sizeof(frame), &read, + nullptr) || + read < sizeof(IpcFrame)) { + return std::nullopt; + } + IpcFrame out{}; + if (!DecodeIpcFrame(&frame, sizeof(frame), out)) { + return std::nullopt; + } + return out; + } + return std::nullopt; +} + +void NamedPipeClient::Close() { + if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(static_cast(handle_)); + } + handle_ = nullptr; +} + +#endif + +} // namespace ae::test_uap_peer_deadline diff --git a/examples/aether_uap_peer_deadline_test/common/deadline_ipc.h b/examples/aether_uap_peer_deadline_test/common/deadline_ipc.h new file mode 100644 index 00000000..97e7067c --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/common/deadline_ipc.h @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_IPC_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_IPC_H_ + +#include +#include +#include + +#include "crc32.h" +#include "deadline_types.h" + +namespace ae::test_uap_peer_deadline { + +#pragma pack(push, 1) +struct IpcFrame { + std::uint32_t magic{kIpcMagic}; + std::uint8_t version{kIpcVersion}; + std::uint8_t type{0}; + std::uint8_t side{0}; + std::uint8_t flags{0}; + std::uint32_t run_id_hash{0}; + std::uint32_t seq{0}; + std::uint32_t sequence{0}; + std::uint32_t code{0}; + std::int64_t local_us{0}; + std::int64_t a{0}; + std::int64_t b{0}; + std::int64_t c{0}; + std::uint32_t crc{0}; +}; +#pragma pack(pop) + +static_assert(sizeof(IpcFrame) < 128); + +std::uint32_t IpcFrameCrc(IpcFrame const& frame) noexcept; +void EncodeIpcFrame(IpcFrame& frame) noexcept; +bool DecodeIpcFrame(void const* data, std::size_t size, IpcFrame& out) noexcept; + +std::string PipeNameFor(std::string const& run_id, Side side); +std::uint32_t HashRunId(std::string const& run_id); + +#if defined(_WIN32) +class NamedPipeServer { + public: + NamedPipeServer() = default; + ~NamedPipeServer(); + NamedPipeServer(NamedPipeServer const&) = delete; + NamedPipeServer& operator=(NamedPipeServer const&) = delete; + + bool Create(std::string const& pipe_name); + bool WaitForClient(std::uint32_t timeout_ms); + bool WriteFrame(IpcFrame frame); + std::optional TryReadFrame(std::uint32_t timeout_ms); + void Close(); + + private: + void* handle_{nullptr}; +}; + +class NamedPipeClient { + public: + NamedPipeClient() = default; + ~NamedPipeClient(); + NamedPipeClient(NamedPipeClient const&) = delete; + NamedPipeClient& operator=(NamedPipeClient const&) = delete; + + bool Connect(std::string const& pipe_name, std::uint32_t timeout_ms); + bool WriteFrame(IpcFrame frame); + std::optional TryReadFrame(std::uint32_t timeout_ms); + void Close(); + + private: + void* handle_{nullptr}; +}; +#endif + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_IPC_H_ diff --git a/examples/aether_uap_peer_deadline_test/common/deadline_types.h b/examples/aether_uap_peer_deadline_test/common/deadline_types.h new file mode 100644 index 00000000..474cdbca --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/common/deadline_types.h @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_TYPES_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_TYPES_H_ + +#include +#include + +namespace ae::test_uap_peer_deadline { + +inline constexpr std::uint32_t kIpcMagic = 0x41555044u; // 'AUPD' +inline constexpr std::uint8_t kIpcVersion = 1; + +enum class Side : std::uint8_t { kCoordinator = 0, kA = 1, kB = 2 }; + +enum class IpcType : std::uint8_t { + kChildReady = 1, + kUidReport = 2, + kSetPeerUid = 3, + kStartCloud = 4, + kCloudStarted = 5, + kRunTest = 6, + kRequestBobKill = 7, + kBobKilled = 8, + kRequestBobRestart = 9, + kBobRestarted = 10, + kTestDone = 11, + kShutdown = 12, + kAck = 13, + kLogLine = 14, + kFlushState = 15, +}; + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DEADLINE_TYPES_H_ diff --git a/examples/aether_uap_peer_deadline_test/common/directory_domain_storage.h b/examples/aether_uap_peer_deadline_test/common/directory_domain_storage.h new file mode 100644 index 00000000..67230d50 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/common/directory_domain_storage.h @@ -0,0 +1,150 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ + +#include +#include +#include +#include +#include +#include + +#include "aether-miscpp/types/result.h" +#include "aether/obj/idomain_storage.h" + +namespace ae::test_uap_peer_deadline { + +// File-backed storage rooted at an explicit directory (not CWD). +class DirectoryDomainStorage final : public IDomainStorage { + public: + explicit DirectoryDomainStorage(std::filesystem::path root) + : root_{std::move(root)} { + std::error_code ec; + std::filesystem::create_directories(root_, ec); + } + + std::unique_ptr Store( + DomainQuery const& query) override { + auto class_dir = + root_ / std::to_string(query.id.id()) / std::to_string(query.class_id); + std::filesystem::create_directories(class_dir); + auto path = class_dir / std::to_string(query.version); + std::ofstream f(path, std::ios::out | std::ios::binary | std::ios::trunc); + class Writer final : public IDomainStorageWriter { + public: + explicit Writer(std::ofstream&& file) : file_{std::move(file)} {} + ~Writer() override { file_.close(); } + seri::SeriResult Write(seri::SizeWriteTag data) override { + auto const u_size = static_cast(data.size); + return Write(seri::DataTag{u_size}); + } + seri::SeriResult Write(seri::DataWriteTag data) override { + file_.write(reinterpret_cast(data.data), + static_cast(data.size)); + if (file_.fail()) { + return Error{seri::write_error}; + } + return Ok{seri::good}; + } + + private: + std::ofstream file_; + }; + return std::make_unique(std::move(f)); + } + + ClassList Enumerate(ObjId const& obj_id) override { + std::set classes; + std::error_code ec; + auto obj_dir = root_ / std::to_string(obj_id.id()); + for (auto const& class_dir : + std::filesystem::directory_iterator(obj_dir, ec)) { + classes.insert(static_cast( + std::stoul(class_dir.path().filename().string()))); + } + return ClassList{classes.begin(), classes.end()}; + } + + DomainLoad Load(DomainQuery const& query) override { + auto object_dir = root_ / std::to_string(query.id.id()); + std::error_code ec; + if (!std::filesystem::exists(object_dir, ec)) { + return {DomainLoadResult::kEmpty, {}}; + } + auto path = object_dir / std::to_string(query.class_id) / + std::to_string(query.version); + std::ifstream f(path, std::ios::in | std::ios::binary); + if (!f.good()) { + return {DomainLoadResult::kEmpty, {}}; + } + class Reader final : public IDomainStorageReader { + public: + explicit Reader(std::ifstream&& file) : file_{std::move(file)} {} + ~Reader() override { file_.close(); } + seri::SeriResult Read(seri::SizeReadTag data) override { + std::uint32_t u_size{}; + TRY_RESULT(Read(seri::DataTag{u_size})); + data.size = static_cast(u_size); + return Ok{seri::good}; + } + seri::SeriResult Read(seri::DataReadTag data) override { + if (file_.eof()) { + return Error{seri::read_eof}; + } + file_.read(reinterpret_cast(data.data), + static_cast(data.size)); + if (file_.bad()) { + return Error{seri::read_error}; + } + if (file_.gcount() != static_cast(data.size)) { + return Error{file_.eof() ? seri::read_eof : seri::read_error}; + } + return Ok{seri::good}; + } + + private: + std::ifstream file_; + }; + return {DomainLoadResult::kLoaded, std::make_unique(std::move(f))}; + } + + void Remove(ObjId const& obj_id) override { + auto object_dir = root_ / std::to_string(obj_id.id()); + std::error_code ec; + if (!std::filesystem::exists(object_dir, ec)) { + std::filesystem::create_directory(object_dir, ec); + return; + } + for (auto const& class_dir : + std::filesystem::directory_iterator(object_dir, ec)) { + std::error_code ec2; + std::filesystem::remove_all(class_dir.path(), ec2); + } + } + + void CleanUp() override { + // Preserve state across Bob hard-kill / restart so the same UID remains. + } + + private: + std::filesystem::path root_; +}; + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ diff --git a/examples/aether_uap_peer_deadline_test/coordinator.cpp b/examples/aether_uap_peer_deadline_test/coordinator.cpp new file mode 100644 index 00000000..eb455736 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/coordinator.cpp @@ -0,0 +1,363 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "coordinator.h" + +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include + +#include "common/deadline_ipc.h" +#include "common/deadline_types.h" + +namespace ae::test_uap_peer_deadline { +namespace { + +struct ChildProc { + Side side{}; + NamedPipeServer pipe; + PROCESS_INFORMATION pi{}; + std::uint64_t uid_lo{0}; + std::uint64_t uid_hi{0}; + bool ready{false}; + bool uid_ok{false}; + std::uint32_t seq{0}; + bool process_open{false}; +}; + +std::string MakeRunId() { + SYSTEMTIME st{}; + GetSystemTime(&st); + char buf[64]; + std::snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u", st.wYear, + st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond); + return buf; +} + +std::string DefaultExePath() { + char path[MAX_PATH]{}; + GetModuleFileNameA(nullptr, path, MAX_PATH); + return path; +} + +bool SendCmd(ChildProc& child, IpcType type, std::int64_t a = 0, + std::int64_t b = 0, std::int64_t c = 0, std::uint32_t code = 0) { + IpcFrame f{}; + f.type = static_cast(type); + f.side = static_cast(Side::kCoordinator); + f.seq = ++child.seq; + f.code = code; + f.a = a; + f.b = b; + f.c = c; + return child.pipe.WriteFrame(f); +} + +void HandleChildFrame(ChildProc& child, IpcFrame const& frame) { + auto const type = static_cast(frame.type); + if (type == IpcType::kUidReport) { + std::memcpy(&child.uid_lo, &frame.a, 8); + std::memcpy(&child.uid_hi, &frame.b, 8); + child.uid_ok = true; + child.ready = true; + } + if (type == IpcType::kChildReady) { + child.ready = true; + } +} + +bool SpawnChild(ChildProc& child, CoordinatorArgs const& args, + std::string const& state_dir, std::string const& pipe_name, + std::string const& client_name, + std::string const& child_log_path, + std::string const& artifact_dir, bool inherit_console) { + if (!child.pipe.Create(pipe_name)) { + std::cerr << "CreateNamedPipe failed for " << pipe_name << "\n"; + return false; + } + auto cmd = "\"" + args.exe_path + "\" --role client --side " + + std::string(child.side == Side::kA ? "A" : "B") + " --run-id " + + args.run_id + " --state-dir \"" + state_dir + "\" --pipe \"" + + pipe_name + "\" --client-name " + client_name + " --parent-uid " + + args.parent_uid + " --artifact-dir \"" + artifact_dir + "\""; + if (child.side == Side::kB) { + cmd += " --ping-interval-ms " + + std::to_string(args.unknown_only ? 0 : 3000); + } + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE log = INVALID_HANDLE_VALUE; + STARTUPINFOA si{}; + si.cb = sizeof(si); + if (inherit_console) { + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + si.hStdError = GetStdHandle(STD_ERROR_HANDLE); + } else { + log = CreateFileA(child_log_path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, + &sa, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (log == INVALID_HANDLE_VALUE) { + std::cerr << "CreateFile child log failed: " << child_log_path << "\n"; + return false; + } + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = log; + si.hStdError = log; + } + std::vector cmdline(cmd.begin(), cmd.end()); + cmdline.push_back('\0'); + ZeroMemory(&child.pi, sizeof(child.pi)); + if (!CreateProcessA(nullptr, cmdline.data(), nullptr, nullptr, TRUE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &child.pi)) { + if (log != INVALID_HANDLE_VALUE) { + CloseHandle(log); + } + std::cerr << "CreateProcess failed: " << GetLastError() << "\n"; + return false; + } + if (log != INVALID_HANDLE_VALUE) { + CloseHandle(log); + } + child.process_open = true; + child.ready = false; + child.uid_ok = false; + if (!child.pipe.WaitForClient(120000)) { + std::cerr << "WaitForClient timeout side=" + << (child.side == Side::kA ? "A" : "B") << "\n"; + return false; + } + return true; +} + +bool WaitReady(ChildProc& child, DWORD timeout_ms) { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + if (auto f = child.pipe.TryReadFrame(200)) { + HandleChildFrame(child, *f); + } + if (child.ready && child.uid_ok) { + return true; + } + } + return false; +} + +void HardKill(ChildProc& child) { + if (!child.process_open) { + return; + } + TerminateProcess(child.pi.hProcess, 1); + WaitForSingleObject(child.pi.hProcess, 15000); + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.process_open = false; + child.pipe.Close(); + ZeroMemory(&child.pi, sizeof(child.pi)); +} + +void SoftStop(ChildProc& child) { + if (!child.process_open) { + return; + } + SendCmd(child, IpcType::kShutdown); + if (WaitForSingleObject(child.pi.hProcess, 15000) != WAIT_OBJECT_0) { + TerminateProcess(child.pi.hProcess, 1); + } + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.process_open = false; + child.pipe.Close(); +} + +bool ProcessAlive(ChildProc const& child) { + if (!child.process_open) { + return false; + } + return WaitForSingleObject(child.pi.hProcess, 0) == WAIT_TIMEOUT; +} + +} // namespace + +int RunCoordinator(CoordinatorArgs args) { + if (args.run_id.empty()) { + args.run_id = MakeRunId(); + } + if (args.exe_path.empty()) { + args.exe_path = DefaultExePath(); + } + if (args.artifact_dir.empty()) { + args.artifact_dir = ".artifacts/uap-peer-deadline/" + args.run_id; + } + + std::filesystem::create_directories(args.artifact_dir); + auto const state_root = + std::filesystem::path{args.artifact_dir} / "persistent-state"; + auto const state_a = (state_root / "state-a").string(); + auto const state_b = (state_root / "state-b").string(); + std::filesystem::create_directories(state_a); + std::filesystem::create_directories(state_b); + + ChildProc alice; + alice.side = Side::kA; + ChildProc bob; + bob.side = Side::kB; + + auto const pipe_a = PipeNameFor(args.run_id, Side::kA); + auto const pipe_b = PipeNameFor(args.run_id, Side::kB); + auto const log_a = args.artifact_dir + "/alice.log"; + auto const log_b = args.artifact_dir + "/bob.log"; + + std::cout << "Spawning Alice/Bob run_id=" << args.run_id << std::endl; + if (!SpawnChild(alice, args, state_a, pipe_a, "uap-deadline-alice", log_a, + args.artifact_dir, true) || + !SpawnChild(bob, args, state_b, pipe_b, "uap-deadline-bob", log_b, + args.artifact_dir, false)) { + return 2; + } + if (!WaitReady(alice, 180000) || !WaitReady(bob, 180000)) { + std::cerr << "children not ready\n"; + SoftStop(alice); + SoftStop(bob); + return 3; + } + + std::int64_t a_lo = 0; + std::int64_t a_hi = 0; + std::int64_t b_lo = 0; + std::int64_t b_hi = 0; + std::memcpy(&a_lo, &alice.uid_lo, 8); + std::memcpy(&a_hi, &alice.uid_hi, 8); + std::memcpy(&b_lo, &bob.uid_lo, 8); + std::memcpy(&b_hi, &bob.uid_hi, 8); + + if (!SendCmd(alice, IpcType::kSetPeerUid, b_lo, b_hi) || + !SendCmd(bob, IpcType::kSetPeerUid, a_lo, a_hi)) { + std::cerr << "SetPeerUid failed\n"; + return 4; + } + (void)alice.pipe.TryReadFrame(5000); + (void)bob.pipe.TryReadFrame(5000); + + if (!SendCmd(bob, IpcType::kStartCloud) || + !SendCmd(alice, IpcType::kStartCloud)) { + std::cerr << "StartCloud failed\n"; + return 5; + } + (void)bob.pipe.TryReadFrame(5000); + (void)alice.pipe.TryReadFrame(5000); + + if (!SendCmd(alice, IpcType::kRunTest, 0, 0, 0, + args.unknown_only ? 1 : 0)) { + std::cerr << "RunTest failed\n"; + return 6; + } + + int result = 1; + auto const deadline = GetTickCount64() + 300000; + while (GetTickCount64() < deadline) { + if (auto f = alice.pipe.TryReadFrame(200)) { + auto const type = static_cast(f->type); + if (type == IpcType::kRequestBobKill) { + std::cout << "Flushing Bob state then hard-kill pid=" + << bob.pi.dwProcessId << std::endl; + if (!SendCmd(bob, IpcType::kFlushState)) { + std::cerr << "FlushState send failed\n"; + SoftStop(alice); + return 15; + } + (void)bob.pipe.TryReadFrame(10000); + HardKill(bob); + if (ProcessAlive(bob)) { + std::cerr << "Bob still alive after TerminateProcess\n"; + SoftStop(alice); + return 7; + } + if (!SendCmd(alice, IpcType::kBobKilled)) { + std::cerr << "BobKilled notify failed\n"; + SoftStop(alice); + return 8; + } + } else if (type == IpcType::kRequestBobRestart) { + std::cout << "Restarting Bob same state/UID" << std::endl; + auto const pipe_b2 = PipeNameFor(args.run_id + "-r", Side::kB); + if (!SpawnChild(bob, args, state_b, pipe_b2, "uap-deadline-bob", + args.artifact_dir + "/bob-restart.log", + args.artifact_dir, false)) { + SoftStop(alice); + return 9; + } + if (!WaitReady(bob, 180000)) { + std::cerr << "Bob restart not ready\n"; + SoftStop(alice); + HardKill(bob); + return 10; + } + if (bob.uid_lo != static_cast(b_lo) || + bob.uid_hi != static_cast(b_hi)) { + std::cerr << "Bob UID changed after restart\n"; + SoftStop(alice); + SoftStop(bob); + return 11; + } + if (!SendCmd(bob, IpcType::kSetPeerUid, a_lo, a_hi) || + !SendCmd(bob, IpcType::kStartCloud)) { + SoftStop(alice); + SoftStop(bob); + return 12; + } + (void)bob.pipe.TryReadFrame(5000); + (void)bob.pipe.TryReadFrame(5000); + if (!SendCmd(alice, IpcType::kBobRestarted)) { + SoftStop(alice); + SoftStop(bob); + return 13; + } + } else if (type == IpcType::kTestDone) { + result = (f->code == 0) ? 0 : 1; + if (f->code == 0) { + std::cout << "false_missed_deadline=" << f->a + << " recovery_ms=" << f->b + << " deadline_late_by_ms=" << f->c << std::endl; + } + break; + } + } + if (!ProcessAlive(alice)) { + std::cerr << "Alice exited early\n"; + result = 14; + break; + } + } + + SoftStop(alice); + if (ProcessAlive(bob)) { + SoftStop(bob); + } + return result; +} + +} // namespace ae::test_uap_peer_deadline diff --git a/examples/aether_uap_peer_deadline_test/coordinator.h b/examples/aether_uap_peer_deadline_test/coordinator.h new file mode 100644 index 00000000..6af53120 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/coordinator.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COORDINATOR_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COORDINATOR_H_ + +#include + +namespace ae::test_uap_peer_deadline { + +struct CoordinatorArgs { + std::string run_id; + std::string artifact_dir; + std::string exe_path; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; + bool unknown_only{false}; +}; + +int RunCoordinator(CoordinatorArgs args); + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_COORDINATOR_H_ diff --git a/examples/aether_uap_peer_deadline_test/main.cpp b/examples/aether_uap_peer_deadline_test/main.cpp new file mode 100644 index 00000000..2a4f1e54 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/main.cpp @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "client_role.h" +#include "coordinator.h" + +namespace { + +std::string_view ArgValue(int argc, char** argv, std::string_view key) { + for (int i = 1; i + 1 < argc; ++i) { + if (key == argv[i]) { + return argv[i + 1]; + } + } + return {}; +} + +} // namespace + +int main(int argc, char** argv) { + using namespace ae::test_uap_peer_deadline; + + auto role = ArgValue(argc, argv, "--role"); + if (role.empty() || role == "coordinator") { + CoordinatorArgs args; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + args.exe_path = std::string{ArgValue(argc, argv, "--exe")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + auto mode = ArgValue(argc, argv, "--mode"); + args.unknown_only = (mode == "unknown"); + return RunCoordinator(args); + } + + if (role == "client") { + ClientArgs args; + auto side = ArgValue(argc, argv, "--side"); + args.side = (side == "B" || side == "b") ? Side::kB : Side::kA; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.state_dir = std::string{ArgValue(argc, argv, "--state-dir")}; + args.pipe_name = std::string{ArgValue(argc, argv, "--pipe")}; + args.client_name = std::string{ArgValue(argc, argv, "--client-name")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + if (args.client_name.empty()) { + args.client_name = + args.side == Side::kA ? "uap-deadline-alice" : "uap-deadline-bob"; + } + auto ping_ms = ArgValue(argc, argv, "--ping-interval-ms"); + if (!ping_ms.empty()) { + args.ping_interval_ms = std::strtoll(ping_ms.data(), nullptr, 10); + } + return RunClientRole(args); + } + + std::cerr << "Unknown --role\n"; + return 1; +} diff --git a/examples/aether_uap_peer_deadline_test/missed_deadline.h b/examples/aether_uap_peer_deadline_test/missed_deadline.h new file mode 100644 index 00000000..f365cd8b --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/missed_deadline.h @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_MISSED_DEADLINE_H_ +#define EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_MISSED_DEADLINE_H_ + +#include + +#include "aether/receive_schedule.h" + +namespace ae::test_uap_peer_deadline { + +// PeerReceiveSchedule TimePoints are local-anchor converted per query, so tiny +// cross-query drift is expected. Treat only large moves as a new ping. +inline constexpr auto kLastPingAdvanceEpsilon = std::chrono::milliseconds{500}; + +inline bool IsLastOnlineAdvanced(TimePoint previous_last_online, + TimePoint current_last_online) noexcept { + return current_last_online > previous_last_online + kLastPingAdvanceEpsilon; +} + +// Test-local helper. Production classification is PeerScheduleState. +inline bool IsMissedDeadline(PeerReceiveSchedule const& previous, + PeerReceiveSchedule const& current, + TimePoint now) noexcept { + if (current.state == PeerScheduleState::kMissedDeadline) { + return true; + } + if (!previous.next_ping_deadline.has_value()) { + return false; + } + if (!(now > *previous.next_ping_deadline)) { + return false; + } + return !IsLastOnlineAdvanced(previous.last_online, current.last_online); +} + +} // namespace ae::test_uap_peer_deadline + +#endif // EXAMPLES_AETHER_UAP_PEER_DEADLINE_TEST_MISSED_DEADLINE_H_ diff --git a/examples/aether_uap_peer_deadline_test/tele_off.h b/examples/aether_uap_peer_deadline_test/tele_off.h new file mode 100644 index 00000000..89da4ca9 --- /dev/null +++ b/examples/aether_uap_peer_deadline_test/tele_off.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Load USER_CONFIG (and the rest of aether/config.h) first, then override +// console telemetry for this benchmark/example target. Force-include this +// header so the override wins without a conflicting /D AE_TELE_LOG_CONSOLE. +#include "aether/config.h" + +#undef AE_TELE_LOG_CONSOLE +#define AE_TELE_LOG_CONSOLE 0 diff --git a/examples/aether_uap_ping_retry_window_test/CMakeLists.txt b/examples/aether_uap_ping_retry_window_test/CMakeLists.txt new file mode 100644 index 00000000..5f38460d --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM AND WIN32) + project("aether_uap_ping_retry_window_test" VERSION "1.0.0" LANGUAGES C CXX) + + add_executable(aether_uap_ping_retry_window_test + main.cpp + client_role.cpp + coordinator.cpp + ) + target_link_libraries(aether_uap_ping_retry_window_test PRIVATE + aether + aether_uap_delivery_timing_bench_common + ) + target_include_directories(aether_uap_ping_retry_window_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../.. + ${CMAKE_CURRENT_SOURCE_DIR}/../benches/aether_uap_delivery_timing_bench + ) + target_compile_definitions(aether_uap_ping_retry_window_test PRIVATE + _CRT_SECURE_NO_WARNINGS + ) + if(MSVC) + target_compile_options(aether_uap_ping_retry_window_test PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + endif() +else() + message(WARNING "aether_uap_ping_retry_window_test is Windows desktop only; skipped") +endif() diff --git a/examples/aether_uap_ping_retry_window_test/client_role.cpp b/examples/aether_uap_ping_retry_window_test/client_role.cpp new file mode 100644 index 00000000..3b17b79e --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/client_role.cpp @@ -0,0 +1,1320 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client_role.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +// Windows.h maps RegisterClass -> RegisterClassA/W; aether's Registry uses +// RegisterClass by name. +#if defined(RegisterClass) +# undef RegisterClass +#endif + +#define AE_EXAMPLE_ETHERNET 1 +#include "aether/all.h" +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/ae_actions/announce_next_ping_unknown.h" +#include "aether/channels/channel.h" +#include "aether/client_messages/p2p_message_stream.h" +#include "aether/cloud_connections/ping_schedule_guard.h" +#include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/ae_actions/ping_test_faults.h" +#include "aether/receive_schedule.h" +#include "aether/server_connections/server_connection.h" + +#include "common/bench_ipc.h" +#include "common/bench_message.h" +#include "common/directory_domain_storage.h" +#include "common/udp_proof.h" + +namespace ae::test_uap_ping_retry_window { + +#if AE_ENABLE_PING_TEST_FAULTS +using ae::PingFaultMode; +using ae::PingFaultPlan; +using ae::PingTestFaults; +#endif +using ae::ServerId; +using ae::bench::uap::BenchProtocol; +using ae::bench::uap::ChannelProof; +using ae::bench::uap::CollectDestinationProofFromCloud; +using ae::bench::uap::CollectOwnCloudProof; +using ae::bench::uap::DeliveryBenchMessage; +using ae::bench::uap::DeserializeDeliveryBenchMessage; +using ae::bench::uap::DirectoryDomainStorage; +using ae::bench::uap::EventKind; +using ae::bench::uap::HashRunId; +using ae::bench::uap::IpcFrame; +using ae::bench::uap::IpcType; +using ae::bench::uap::NamedPipeClient; +using ae::bench::uap::PackUdpProofFrame; +using ae::bench::uap::SerializeDeliveryBenchMessage; +using ae::bench::uap::UdpProofPath; +using IpcSide = ae::bench::uap::Side; +namespace { +constexpr std::uint8_t kIpcArmFault = 13; +constexpr std::uint8_t kIpcSendTagged = 14; +constexpr std::uint8_t kIpcQueryNow = 15; +constexpr std::uint8_t kIpcPingTraceEx = 16; +constexpr std::uint8_t kIpcAnnounceUnknown = 17; +constexpr std::uint8_t kIpcScheduleState = 18; +constexpr std::uint8_t kIpcPingBudget = 19; +constexpr std::uint8_t kIpcQueryStats = 20; +constexpr std::uint8_t kIpcFaultTrace = 21; +constexpr std::uint32_t kTagRequestLossQueued = 1; +constexpr std::uint32_t kTagResponseLossFirstWindow = 2; +constexpr std::uint32_t kTagAfterRetryWindow = 3; + + +constexpr auto kBobPingInterval = std::chrono::milliseconds{3000}; +constexpr auto kBobReceiveWindow = std::chrono::milliseconds{1000}; +constexpr auto kSkipIfCloserThan = std::chrono::milliseconds{50}; +constexpr std::size_t kWarmupSamples = 10; + +inline std::int64_t TimePointUs(TimePoint tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +// Duration is unsigned; never add a negative chrono duration to TimePoint. +inline TimePoint AddOffsetMs(TimePoint base, std::int64_t offset_ms) { + auto const mag_ms = offset_ms < 0 ? -offset_ms : offset_ms; + auto const mag = std::chrono::duration_cast( + std::chrono::milliseconds{mag_ms}); + if (offset_ms >= 0) { + return base + mag; + } + return base - mag; +} + +inline std::int64_t DurationUs(Duration d) { + return static_cast(d.count()); +} + +inline std::int64_t BenchProtocolFromAe(Protocol protocol) { + if (protocol == Protocol::kUdp) { + return static_cast(BenchProtocol::kUdp); + } + if (protocol == Protocol::kTcp) { + return static_cast(BenchProtocol::kTcp); + } + return static_cast(BenchProtocol::kUnknown); +} + +inline std::int64_t SteadyUsNow() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +inline std::uint64_t QpcNow() { + LARGE_INTEGER v{}; + QueryPerformanceCounter(&v); + return static_cast(v.QuadPart); +} + +#if AE_ENABLE_PING +struct PendingPingTrace { + PingTraceEvent event; + std::int64_t steady_us{0}; + std::int64_t qpc{0}; +}; +std::vector g_pending_ping_traces; +std::vector g_all_ping_traces; + +void OnPingTrace(PingTraceEvent const& event) { + PendingPingTrace rec{event, SteadyUsNow(), static_cast(QpcNow())}; + g_pending_ping_traces.push_back(rec); + if (g_all_ping_traces.size() < 4096) { + g_all_ping_traces.push_back(rec); + } +} + +#if AE_ENABLE_PING_TEST_FAULTS +struct PendingFaultTrace { + PingFaultTraceEvent event; + std::int64_t steady_us{0}; +}; +std::vector g_pending_fault_traces; + +void OnPingFaultTrace(PingFaultTraceEvent const& event) { + g_pending_fault_traces.push_back( + PendingFaultTrace{event, SteadyUsNow()}); +} +#endif +#endif + +inline void UidToHalves(Uid const& uid, std::int64_t& lo, std::int64_t& hi) { + std::memcpy(&lo, uid.value.data(), 8); + std::memcpy(&hi, uid.value.data() + 8, 8); +} + +inline Uid UidFromHalves(std::int64_t lo, std::int64_t hi) { + Uid uid{}; + std::memcpy(uid.value.data(), &lo, 8); + std::memcpy(uid.value.data() + 8, &hi, 8); + return uid; +} + +struct RoleState { + IpcSide side{}; + std::int64_t ping_interval_ms{3000}; + std::int64_t receive_window_ms{1000}; + std::uint32_t run_id_hash{0}; + NamedPipeClient pipe; + std::unique_ptr app; + Client::ptr client; + Uid peer_uid{}; + bool peer_set{false}; + std::shared_ptr stream; + std::vector> retired_streams_; + Subscription stream_sub; + Subscription new_port_sub; + Subscription select_sub; + Subscription announce_sub; + Subscription query_sub; + bool query_state_only_{false}; + bool query_in_flight_{false}; + std::int64_t pending_query_checkpoint_{-1}; + std::int64_t query_attempts_{0}; + std::int64_t query_created_{0}; + std::int64_t query_reused_{0}; + std::int64_t query_skipped_inflight_{0}; + std::int64_t query_extra_subscribers_{0}; + Subscription extra_query_sub_; + PeerTimingQueryCoverage last_coverage_{}; + Subscription dest_cloud_sub; + bool dest_proof_sent{false}; + bool dest_cloud_failed_{false}; + bool own_proof_sent{false}; + std::optional dest_retry_at_{}; + ChannelProof own_proof{}; + ChannelProof dest_proof{}; + std::unordered_map seen; + std::uint32_t ipc_seq{0}; + bool exit_requested{false}; + bool client_ready{false}; + bool warmup_active{false}; + std::optional warmup_requery_at_{}; + bool sample_in_flight{false}; + std::uint32_t pending_sequence{0}; + std::uint32_t pending_offset_ms{0}; + std::int64_t pending_offset_signed_{0}; + std::optional send_at_{}; + std::optional requery_at_{}; + std::int64_t pending_last_us_{0}; + std::int64_t pending_next_us_{-1}; + std::vector last_diagnostics_{}; + std::int64_t pending_schedule_server_id_{0}; + std::int64_t pending_route_generation_{0}; + std::int64_t pending_protocol_{0}; + std::int64_t pending_raw_delta_ms_{0}; + std::int64_t pending_last_connect_ms_{0}; + std::int64_t pending_qsend_us_{0}; + std::int64_t pending_one_way_us_{0}; + std::int64_t pending_target_us_{0}; + + bool Emit(IpcType type, EventKind kind = EventKind::kAck, + std::uint32_t sequence = 0, std::uint32_t offset_ms = 0, + std::int64_t a = 0, std::int64_t b = 0, std::int64_t c = 0, + std::int64_t d = 0, std::int64_t e = 0, std::int64_t f = 0, + std::int64_t g = 0, std::int64_t h = 0, std::int64_t i = 0, + std::int64_t j = 0, std::int64_t k = 0, std::int64_t l = 0) { + IpcFrame frame{}; + frame.type = static_cast(type); + frame.side = static_cast(side); + frame.event_kind = static_cast(kind); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.sequence = sequence; + frame.offset_ms = offset_ms; + frame.local_steady_us = SteadyUsNow(); + frame.a = a; + frame.b = b; + frame.c = c; + frame.d = d; + frame.e = e; + frame.f = f; + frame.g = g; + frame.h = h; + frame.i = i; + frame.j = j; + frame.k = k; + frame.l = l; + return pipe.WriteFrame(frame); + } + + void DrainPingTraces() { +#if AE_ENABLE_PING + for (auto const& rec : g_pending_ping_traces) { + auto const& e = rec.event; + IpcFrame frame{}; + frame.type = static_cast(IpcType::kPingTrace); + frame.side = static_cast(side); + frame.event_kind = static_cast(e.kind); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.offset_ms = + e.result_type < 0 ? 0 : static_cast(e.result_type); + frame.local_steady_us = rec.steady_us; + frame.a = static_cast(e.server_id); + frame.b = TimePointUs(e.planned_send_at); + frame.c = TimePointUs(e.actual_send_at); + frame.d = DurationUs(e.early_by); + frame.e = DurationUs(e.base_rx_window); + frame.f = DurationUs(e.effective_wire_rx_window); + frame.g = TimePointUs(e.required_rx_until); + frame.h = TimePointUs(e.next_planned_send); + frame.i = DurationUs(e.ping_guard); + frame.j = static_cast(e.channel_generation); + frame.k = DurationUs(e.min_rtt); + frame.l = DurationUs(e.p99_rtt); + pipe.WriteFrame(frame); + IpcFrame extra{}; + extra.type = kIpcPingTraceEx; + extra.side = static_cast(side); + extra.event_kind = static_cast(e.kind); + extra.run_id_hash = run_id_hash; + extra.seq = ++ipc_seq; + extra.local_steady_us = rec.steady_us; + extra.a = static_cast(e.logical_cycle_id); + extra.b = static_cast(e.physical_attempt_index); + extra.c = static_cast(e.fault_mode); + extra.d = e.wire_next_connect_ms; + extra.e = TimePointUs(e.cycle_anchor); + extra.f = TimePointUs(e.contract_deadline); + extra.g = TimePointUs(e.next_local_send_at); + extra.h = e.request_was_sent ? 1 : 0; + extra.i = e.response_was_ignored ? 1 : 0; + extra.j = static_cast(e.server_id); + extra.k = rec.qpc; + extra.l = DurationUs(e.retry_reserve); + pipe.WriteFrame(extra); + IpcFrame budget{}; + budget.type = kIpcPingBudget; + budget.side = static_cast(side); + budget.event_kind = static_cast(e.kind); + budget.run_id_hash = run_id_hash; + budget.seq = ++ipc_seq; + budget.local_steady_us = rec.steady_us; + budget.a = DurationUs(e.attempt_lead); + budget.b = DurationUs(e.retry_reserve); + budget.c = DurationUs(e.loss_timeout); + budget.d = e.predeadline_retry_guaranteed ? 1 : 0; + budget.e = TimePointUs(e.cycle_anchor); + budget.f = TimePointUs(e.contract_deadline); + budget.g = DurationUs(e.ping_guard); + budget.h = static_cast(e.logical_cycle_id); + budget.i = static_cast(e.physical_attempt_index); + budget.j = static_cast(e.server_id); + budget.k = rec.qpc; + pipe.WriteFrame(budget); + } + g_pending_ping_traces.clear(); +#endif + } + + void DrainFaultTraces() { +#if AE_ENABLE_PING_TEST_FAULTS + for (auto const& rec : g_pending_fault_traces) { + auto const& e = rec.event; + IpcFrame frame{}; + frame.type = kIpcFaultTrace; + frame.side = static_cast(side); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.local_steady_us = rec.steady_us; + frame.a = static_cast(e.server_id); + frame.b = static_cast(e.logical_cycle_id); + frame.c = static_cast(e.physical_attempt_index); + frame.d = static_cast(e.mode); + frame.e = static_cast(e.harness_state); + frame.f = static_cast(e.kind); + frame.g = e.steady_us; + pipe.WriteFrame(frame); + } + g_pending_fault_traces.clear(); +#endif + } + + void EmitUdpProof(UdpProofPath path, ChannelProof const& proof) { + if (path == UdpProofPath::kOwn) { + own_proof = proof; + } else if (path == UdpProofPath::kDestination) { + dest_proof = proof; + } + IpcFrame f{}; + PackUdpProofFrame(f, path, proof); + f.side = static_cast(side); + f.run_id_hash = run_id_hash; + f.seq = ++ipc_seq; + f.local_steady_us = SteadyUsNow(); + pipe.WriteFrame(f); + } + + static bool IsClassifiedWorkProtocol(BenchProtocol protocol) noexcept { + return protocol == BenchProtocol::kTcp || protocol == BenchProtocol::kUdp; + } + + void TryEmitOwnProof() { + if (!client) { + return; + } + (void)client->cloud_connection(); + auto proof = CollectOwnCloudProof(*client); + if (!proof.present || !IsClassifiedWorkProtocol(proof.protocol)) { + return; + } + if (own_proof_sent && own_proof.protocol == proof.protocol) { + return; + } + own_proof_sent = true; + EmitUdpProof(UdpProofPath::kOwn, proof); + } + + void TryEmitDestProof() { + if (dest_proof_sent || !client || !peer_set) { + return; + } + if (dest_cloud_failed_) { + dest_cloud_sub.Reset(); + dest_cloud_failed_ = false; + } + if (dest_cloud_sub) { + return; + } + auto const now = Now(); + if (dest_retry_at_ && now < *dest_retry_at_) { + return; + } + dest_retry_at_ = now + std::chrono::milliseconds{250}; + auto& get_cloud = client->cloud_manager()->GetCloud(peer_uid); + dest_cloud_sub = get_cloud.result_event().Subscribe( + [this](Result const& res) { + if (!res) { + dest_cloud_failed_ = true; + dest_retry_at_ = Now() + std::chrono::milliseconds{250}; + return; + } + auto proof = CollectDestinationProofFromCloud(*client, res.value()); + if (!proof.present || !IsClassifiedWorkProtocol(proof.protocol)) { + dest_cloud_failed_ = true; + dest_retry_at_ = Now() + std::chrono::milliseconds{250}; + return; + } + dest_proof_sent = true; + EmitUdpProof(UdpProofPath::kDestination, proof); + }); + } + + void ResetPeerBinding() { + sample_in_flight = false; + send_at_.reset(); + requery_at_.reset(); + stream_sub.Reset(); + if (stream) { + retired_streams_.push_back(std::move(stream)); + } + new_port_sub.Reset(); + dest_cloud_sub.Reset(); + dest_proof_sent = false; + dest_cloud_failed_ = false; + dest_proof = {}; + dest_retry_at_.reset(); + } + + void EnsureStreams() { + if (!client || !peer_set) { + return; + } + if (side == IpcSide::kA && !stream) { + stream = std::make_shared( + AeContext{*app}, client.Load(), peer_uid, + client->message_stream_manager().CreatePort(peer_uid)); + stream_sub = stream->out_data_event().Subscribe( + [this](DataBuffer const& data) { OnReceive(data); }); + } + if (!new_port_sub) { + new_port_sub = client->message_stream_manager().new_port_event().Subscribe( + [this](P2pPortHandle handle) { + if (handle.destination() != peer_uid && side == IpcSide::kB) { + // Bob accepts any inbound port from Alice after peer is set. + } + stream = std::make_shared( + AeContext{*app}, client.Load(), handle.destination(), + std::move(handle)); + stream_sub = stream->out_data_event().Subscribe( + [this](DataBuffer const& data) { OnReceive(data); }); + }); + } + } + + void OnReceive(DataBuffer const& data) { + auto msg = DeserializeDeliveryBenchMessage(data.data(), data.size()); + if (!msg) { + Emit(IpcType::kEvent, EventKind::kError, 0, 0, 1); + return; + } + auto& count = seen[msg->sequence]; + ++count; + auto const recv_qpc = static_cast(QpcNow()); + Emit(IpcType::kEvent, EventKind::kSampleReceived, msg->sequence, + msg->offset_ms, static_cast(msg->send_qpc), recv_qpc, + count); + } + + // Returns max response sample count across active channels, and fills + // min/p99 from the channel with the most samples. + std::size_t CollectResponseStats(Duration* min_out, Duration* p99_out) { + std::size_t best = 0; + Duration best_min{}; + Duration best_p99{}; + auto& csc = client->cloud_connection(); + for (auto* sc : csc.servers()) { + if (sc == nullptr) { + continue; + } + auto* cc = sc->client_connection(); + if (cc == nullptr) { + continue; + } + auto ch = cc->server_connection().current_channel(); + if (!ch) { + continue; + } + auto const& stats = ch->channel_statistics().response_time_statistics(); + if (stats.size() > best) { + best = stats.size(); + if (!stats.empty()) { + best_min = stats.min(); + best_p99 = stats.percentile<99>(); + } + } + } + if (min_out != nullptr) { + *min_out = best_min; + } + if (p99_out != nullptr) { + *p99_out = best_p99; + } + return best; + } + + void PollWarmup() { + if (!warmup_active || !client) { + return; + } + if (side == IpcSide::kA) { + EnsureStreams(); + TryEmitOwnProof(); + TryEmitDestProof(); + auto const now = Now(); + if (!warmup_requery_at_ || now >= *warmup_requery_at_) { + warmup_requery_at_ = now + std::chrono::milliseconds{250}; + if (!query_in_flight_ && peer_set) { + query_state_only_ = true; + BeginQuery(); + } + } + + ServerId server_id{}; + std::size_t sample_count = 0; + Duration min_rtt{}; + Duration p99_rtt{}; + Protocol protocol = Protocol::kTcp; + bool present = false; + + if (stream) { + auto const route = stream->InspectSendRoute(); + if (route.present) { + present = true; + server_id = route.server_id; + sample_count = route.ping_sample_count; + min_rtt = route.min_rtt; + p99_rtt = route.p99_rtt; + protocol = route.protocol; + } + } + // Prefer dest-route stats. Fall back to Alice's own cloud channel so + // warm-up can complete when dest GetCloud is slow; Q2 may still fail + // separately and is classified as harness if QueryPeer is unavailable. + if (sample_count < kWarmupSamples) { + Duration own_min{}; + Duration own_p99{}; + ServerId own_sid{}; + Protocol own_proto = Protocol::kTcp; + std::size_t own_n = 0; + auto& csc = client->cloud_connection(); + for (auto* sc : csc.servers()) { + if (sc == nullptr) { + continue; + } + auto* cc = sc->client_connection(); + if (cc == nullptr) { + continue; + } + auto ch = cc->server_connection().current_channel(); + if (!ch) { + continue; + } + auto const& stats = + ch->channel_statistics().response_time_statistics(); + if (stats.size() > own_n) { + own_n = stats.size(); + own_sid = sc->server_id(); + if (!stats.empty()) { + own_min = stats.min(); + own_p99 = stats.percentile<99>(); + } + auto const& props = ch->transport_properties(); + own_proto = + props.connection_type == ConnectionType::kConnectionLess + ? Protocol::kUdp + : Protocol::kTcp; + } + } + if (own_n > sample_count) { + sample_count = own_n; + min_rtt = own_min; + p99_rtt = own_p99; + if (!present) { + present = true; + server_id = own_sid; + protocol = own_proto; + } + if (own_proof.present) { + server_id = own_proof.server_id; + protocol = own_proof.protocol == BenchProtocol::kUdp + ? Protocol::kUdp + : Protocol::kTcp; + } + } + } + + static std::size_t last_logged = 0; + if (sample_count != last_logged && + (sample_count % 2 == 0 || sample_count >= kWarmupSamples)) { + last_logged = sample_count; + std::cerr << "Alice dest warmup server=" << server_id + << " samples=" << sample_count + << " present=" << present << std::endl; + } + if (!present || sample_count < kWarmupSamples) { + return; + } + auto const min_ms = + std::chrono::duration_cast(min_rtt) + .count(); + auto const p99_ms = + std::chrono::duration_cast(p99_rtt) + .count(); + if ((min_ms == 200 && p99_ms == 200) || min_ms == 5000 || p99_ms == 5000) { + std::cerr << "Alice dest warmup stats look synthetic: min=" << min_ms + << " p99=" << p99_ms << "\n"; + return; + } + warmup_active = false; + warmup_requery_at_.reset(); + std::cout << "## Alice dest-server ping statistics (child)\n" + << "server_id=" << server_id + << " samples=" << sample_count + << " min_rtt_ms=" << min_ms << " p99_rtt_ms=" << p99_ms + << " protocol=" + << (protocol == Protocol::kUdp ? "udp" : "tcp") << "\n"; + Emit(IpcType::kWarmupDone, EventKind::kWarmupDone, 0, 0, + static_cast(sample_count), min_ms, p99_ms, + static_cast(server_id), + BenchProtocolFromAe(protocol)); + return; + } + if (side != IpcSide::kB) { + return; + } + Duration min_rtt{}; + Duration p99_rtt{}; + auto const n = CollectResponseStats(&min_rtt, &p99_rtt); + static std::size_t last_logged = 0; + if (n != last_logged && (n % 2 == 0 || n >= kWarmupSamples)) { + last_logged = n; + std::cerr << "Bob warmup samples=" << n << std::endl; + } + if (n < kWarmupSamples) { + return; + } + auto const interval = std::chrono::duration_cast( + std::chrono::milliseconds{ping_interval_ms}); + auto const guard = ClampPingSendGuard( + ComputePingSendGuard(min_rtt, p99_rtt), interval); + auto const min_ms = + std::chrono::duration_cast(min_rtt).count(); + auto const p99_ms = + std::chrono::duration_cast(p99_rtt).count(); + auto const guard_ms = + std::chrono::duration_cast(guard).count(); + // Reject obvious synthetic seed values (200ms estimate / 5000ms). + if ((min_ms == 200 && p99_ms == 200) || min_ms == 5000 || p99_ms == 5000) { + std::cerr << "warmup stats look synthetic: min=" << min_ms + << " p99=" << p99_ms << "\n"; + return; + } + warmup_active = false; + std::cout << "## Bob ping statistics (child)\n" + << "samples=" << n << " min_rtt_ms=" << min_ms + << " p99_rtt_ms=" << p99_ms << " guard_ms=" << guard_ms << "\n"; + // sequence unused; offset_ms carries guard_ms; a=n b=min c=p99 + Emit(IpcType::kWarmupDone, EventKind::kWarmupDone, 0, + static_cast(guard_ms), static_cast(n), + min_ms, p99_ms); + } + + void StartSample(std::uint32_t sequence, std::int64_t offset_ms) { + if (side != IpcSide::kA || !client || !peer_set) { + return; + } + sample_in_flight = true; + query_state_only_ = false; + pending_sequence = sequence; + pending_offset_signed_ = offset_ms; + pending_offset_ms = offset_ms < 0 ? 0u : static_cast(offset_ms); + send_at_.reset(); + requery_at_.reset(); + EnsureStreams(); + BeginQuery(); + } + + void BeginQuery() { + // Reset subscription before replacing Client-owned action. + query_sub.Reset(); + query_in_flight_ = true; + ++query_created_; + auto& action = client->QueryPeerReceiveSchedule(peer_uid); + query_sub = action.result_event().Subscribe( + [this, &action](Result const& res) { + last_diagnostics_ = action.server_diagnostics(); + last_coverage_ = action.coverage(); + query_in_flight_ = false; + if (query_state_only_) { + EmitScheduleState(res); + query_state_only_ = false; + return; + } + OnSchedule(res); + }); + if (action.is_finished()) { + // Synchronous completion can race Subscribe; clear the stuck flag. + query_in_flight_ = false; + } + } + + ServerTimingDiagnostic const* FindDestDiagnostic( + ServerId server_id) const { + for (auto const& d : last_diagnostics_) { + if (d.server_id == server_id && d.has_raw && + d.status == ServerTimingAttemptStatus::kSuccess) { + return &d; + } + } + return nullptr; + } + + void OnSchedule(Result const& res) { + if (!res) { + sample_in_flight = false; + send_at_.reset(); + requery_at_.reset(); + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, res.error()); + return; + } + TryEmitDestProof(); + EnsureStreams(); + if (!stream) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, 3); + return; + } + auto const route = stream->InspectSendRoute(); + if (!route.present) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, 0, -1, 7); + return; + } + if (route.ping_sample_count < kWarmupSamples) { + requery_at_ = Now() + std::chrono::milliseconds{250}; + return; + } + auto const* diag = FindDestDiagnostic(route.server_id); + if (diag == nullptr || + diag->converted.state != PeerScheduleState::kExpected || + !diag->converted.next_ping_deadline.has_value()) { + sample_in_flight = false; + send_at_.reset(); + requery_at_.reset(); + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, 0, -1, 8); + return; + } + + auto const cycle_start = + *diag->converted.next_ping_deadline - + std::chrono::duration_cast( + std::chrono::milliseconds{ping_interval_ms}); + auto const target = AddOffsetMs(cycle_start, pending_offset_signed_); + auto const now = Now(); + pending_last_us_ = TimePointUs(cycle_start); + pending_next_us_ = TimePointUs(*diag->converted.next_ping_deadline); + pending_schedule_server_id_ = static_cast(route.server_id); + pending_route_generation_ = + static_cast(route.route_generation); + pending_protocol_ = BenchProtocolFromAe(route.protocol); + pending_raw_delta_ms_ = diag->raw.next_ping_delta_ms; + pending_last_connect_ms_ = diag->raw.last_connect_delta_ms; + pending_qsend_us_ = TimePointUs(diag->qsend); + pending_one_way_us_ = DurationUs(diag->one_way); + pending_target_us_ = TimePointUs(target); + + auto const to_next_ms = + std::chrono::duration_cast( + *diag->converted.next_ping_deadline - now) + .count(); + std::cerr << "Alice dest-server schedule server=" << route.server_id + << " gen=" << route.route_generation + << " age_to_cycle_start_ms=" + << std::chrono::duration_cast( + now - cycle_start) + .count() + << " to_target_ms=" + << std::chrono::duration_cast(target - + now) + .count() + << " to_next_ms=" << to_next_ms + << " offset_ms=" << pending_offset_signed_ + << " raw_delta_ms=" << pending_raw_delta_ms_ << std::endl; + + if (now + kSkipIfCloserThan > target && now < target) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 1, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_, pending_protocol_, pending_raw_delta_ms_, + pending_last_connect_ms_); + return; + } + if (now >= target) { + auto const next_target = AddOffsetMs(*diag->converted.next_ping_deadline, + pending_offset_signed_); + if (next_target > now + kSkipIfCloserThan) { + auto const delay_ms = + std::chrono::duration_cast(next_target - + now) + .count(); + if (delay_ms > 15000) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, + pending_sequence, pending_offset_ms, pending_last_us_, + pending_next_us_, 3, pending_schedule_server_id_, + pending_schedule_server_id_, pending_route_generation_); + return; + } + pending_last_us_ = TimePointUs(*diag->converted.next_ping_deadline); + pending_next_us_ = TimePointUs( + *diag->converted.next_ping_deadline + + std::chrono::duration_cast( + std::chrono::milliseconds{ping_interval_ms})); + pending_target_us_ = TimePointUs(next_target); + send_at_ = next_target; + return; + } + auto wait_until = + *diag->converted.next_ping_deadline + std::chrono::milliseconds{150}; + if (wait_until <= now) { + wait_until = now + std::chrono::milliseconds{250}; + } + if (wait_until > now + std::chrono::seconds{15}) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, + pending_sequence, pending_offset_ms, pending_last_us_, + pending_next_us_, 2, pending_schedule_server_id_, + pending_schedule_server_id_, pending_route_generation_); + return; + } + requery_at_ = wait_until; + return; + } + + auto const delay_ms = + std::chrono::duration_cast(target - now) + .count(); + if (delay_ms > 15000) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 3, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_); + return; + } + send_at_ = target; + } + + void PollSampleTiming() { + if (side != IpcSide::kA || !sample_in_flight || !client) { + return; + } + auto const now = Now(); + if (requery_at_ && now >= *requery_at_) { + requery_at_.reset(); + BeginQuery(); + return; + } + if (send_at_ && now >= *send_at_) { + send_at_.reset(); + SendPendingMessage(); + } + } + + void SendPendingMessage() { + if (!stream) { + EnsureStreams(); + } + if (!stream) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, 3); + return; + } + auto const before = stream->InspectSendRoute(); + if (!before.present || + static_cast(before.server_id) != + pending_schedule_server_id_ || + static_cast(before.route_generation) != + pending_route_generation_) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 6, + pending_schedule_server_id_, + before.present ? static_cast(before.server_id) : 0, + before.present ? static_cast(before.route_generation) + : 0, + before.present ? BenchProtocolFromAe(before.protocol) : 0); + return; + } + auto const dest_proto = static_cast(pending_protocol_); + if (RefuseTcpSample(own_proof.protocol, dest_proto)) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 5, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_, pending_protocol_); + return; + } + + DeliveryBenchMessage msg{}; + msg.offset_ms = static_cast(pending_offset_ms); + msg.sequence = pending_sequence; + msg.send_qpc = QpcNow(); + auto bytes = SerializeDeliveryBenchMessage(msg); + DataBuffer payload{bytes.begin(), bytes.end()}; + stream->Write(std::move(payload)); + auto const after = stream->LastSendRoute(); + auto const actual_id = after.present + ? static_cast(after.server_id) + : pending_schedule_server_id_; + auto const actual_gen = + after.present ? static_cast(after.route_generation) + : pending_route_generation_; + auto const actual_proto = + after.present ? BenchProtocolFromAe(after.protocol) : pending_protocol_; + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSent, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, + static_cast(msg.send_qpc), pending_schedule_server_id_, + actual_id, actual_gen, actual_proto, pending_raw_delta_ms_, + pending_last_connect_ms_, pending_qsend_us_, pending_one_way_us_, + pending_target_us_); + std::cerr << "Alice sent seq=" << pending_sequence + << " offset_ms=" << pending_offset_ms + << " schedule_server=" << pending_schedule_server_id_ + << " actual_server=" << actual_id << std::endl; + } + + + void SendTaggedNow(std::uint32_t tag) { + EnsureStreams(); + if (!stream) { + Emit(IpcType::kSampleResult, EventKind::kError, tag, 0, 3); + return; + } + auto const before = stream->InspectSendRoute(); + if (!before.present) { + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, tag, 0, 0, 0, 7); + return; + } + DeliveryBenchMessage msg{}; + msg.offset_ms = 0; + msg.sequence = tag; + msg.send_qpc = QpcNow(); + auto bytes = SerializeDeliveryBenchMessage(msg); + DataBuffer payload{bytes.begin(), bytes.end()}; + auto const dest = static_cast(before.server_id); + auto const gen = static_cast(before.route_generation); + stream->Write(std::move(payload)); + auto const after = stream->LastSendRoute(); + if (after.present && (static_cast(after.server_id) != dest || + static_cast(after.route_generation) != gen)) { + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, tag, 0, 0, 0, 6, + dest, static_cast(after.server_id), gen); + return; + } + Emit(IpcType::kSampleResult, EventKind::kSampleSent, tag, 0, 0, 0, + static_cast(msg.send_qpc), dest, dest, gen, + BenchProtocolFromAe(before.protocol)); + } + + void ArmFault(IpcFrame const& f) { +#if AE_ENABLE_PING_TEST_FAULTS + PingFaultPlan plan{}; + plan.server_id = static_cast(f.a); + plan.logical_cycle_id = 0; + plan.physical_attempt_index = + f.b <= 0 ? 1u : static_cast(f.b); + plan.mode = static_cast(f.c); + if (f.d > 0) { + plan.timeout_override = Duration{static_cast(f.d)}; + } + if (f.f > 0) { + plan.logical_cycle_id = static_cast(f.f); + } + if (f.e == 0) { + PingTestFaults::Instance().Clear(); + PingFaultTraceEvent cleared{}; + cleared.kind = PingFaultTraceKind::kCleared; + cleared.harness_state = PingFaultHarnessState::kIdle; + cleared.steady_us = SteadyUsNow(); + g_pending_fault_traces.push_back({cleared, SteadyUsNow()}); + } + if (f.h != 0) { + plan.hold_enabled = true; + plan.retry_hold_offset_us = f.g; + } + if (plan.mode != PingFaultMode::kNone) { + PingTestFaults::Instance().Arm(plan); + PingFaultTraceEvent armed{}; + armed.kind = PingFaultTraceKind::kArmed; + armed.server_id = plan.server_id; + armed.logical_cycle_id = plan.logical_cycle_id; + armed.physical_attempt_index = plan.physical_attempt_index; + armed.mode = plan.mode; + armed.harness_state = PingFaultHarnessState::kArmed; + armed.steady_us = SteadyUsNow(); + g_pending_fault_traces.push_back({armed, SteadyUsNow()}); + } + DrainFaultTraces(); + Emit(IpcType::kAck, EventKind::kAck, 0, 0, f.a, f.b, f.c); +#else + (void)f; + Emit(IpcType::kEvent, EventKind::kError, 0, 0, 12); +#endif + } + + void EmitScheduleState(Result const& res) { + IpcFrame frame{}; + frame.type = kIpcScheduleState; + frame.side = static_cast(side); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.local_steady_us = SteadyUsNow(); + if (!res) { + frame.a = -1; + frame.b = res.error(); + } else { + frame.a = static_cast(res.value().state); + frame.b = res.value().next_ping_deadline.has_value() + ? TimePointUs(*res.value().next_ping_deadline) + : 0; + frame.c = TimePointUs(res.value().last_online); + } + frame.d = static_cast(last_coverage_.selected_server_count); + frame.e = static_cast(last_coverage_.queried_server_count); + frame.f = static_cast(last_coverage_.successful_server_count); + frame.g = static_cast(last_coverage_.failed_server_count); + frame.h = + static_cast(last_coverage_.quarantined_skipped_count); + frame.i = static_cast(QpcNow()); + frame.j = pending_query_checkpoint_; + frame.k = std::numeric_limits::min(); + frame.l = std::numeric_limits::min(); + for (auto const& d : last_diagnostics_) { + if (d.has_raw && + d.status == ServerTimingAttemptStatus::kSuccess) { + frame.k = d.raw.next_ping_delta_ms; + frame.l = d.raw.last_connect_delta_ms; + break; + } + } + pipe.WriteFrame(frame); + EmitQueryStats(); + } + + void EmitQueryStats() { + IpcFrame stats{}; + stats.type = kIpcQueryStats; + stats.side = static_cast(side); + stats.run_id_hash = run_id_hash; + stats.seq = ++ipc_seq; + stats.local_steady_us = SteadyUsNow(); + stats.a = query_attempts_; + stats.b = query_created_; + stats.c = query_reused_; + stats.d = query_skipped_inflight_; + stats.e = query_extra_subscribers_; + stats.f = pending_query_checkpoint_; + stats.i = static_cast(QpcNow()); + pipe.WriteFrame(stats); + } + + void QueryNow(std::int64_t checkpoint, bool force) { + if (!client || !peer_set) { + return; + } + ++query_attempts_; + pending_query_checkpoint_ = checkpoint; + if (query_in_flight_) { + ++query_skipped_inflight_; + auto& existing = client->QueryPeerReceiveSchedule(peer_uid); + if (existing.is_finished()) { + query_in_flight_ = false; + } else { + ++query_reused_; + if (force) { + ++query_extra_subscribers_; + extra_query_sub_.Reset(); + extra_query_sub_ = existing.result_event().Subscribe( + [this](Result const& res) { + EmitScheduleState(res); + }); + } + EmitQueryStats(); + return; + } + } + query_state_only_ = true; + BeginQuery(); + } + + void StartAnnounceUnknown() { + if (!client) { + Emit(IpcType::kEvent, EventKind::kError, 0, 0, 13); + return; + } + announce_sub.Reset(); + auto& action = client->AnnounceNextPingUnknown(); + announce_sub = action.result_event().Subscribe( + [this](Result const& res) { + Emit(IpcType::kAck, EventKind::kAck, 0, 0, res ? 0 : res.error(), + static_cast(QpcNow())); + }); + } + void HandleIpc(IpcFrame const& f) { + auto const type = static_cast(f.type); + switch (type) { + case IpcType::kSetPeerUid: { + auto const next = UidFromHalves(f.a, f.b); + if (!peer_set || next != peer_uid) { + ResetPeerBinding(); + std::cerr << (side == IpcSide::kA ? "Alice" : "Bob") + << " peer uid changed; rebuilt P2P binding" << std::endl; + } + peer_uid = next; + peer_set = true; + EnsureStreams(); + Emit(IpcType::kAck, EventKind::kAck); + break; + } + case IpcType::kWaitWarmup: + warmup_active = true; + warmup_requery_at_.reset(); + std::cerr << (side == IpcSide::kA ? "Alice" : "Bob") + << " WaitWarmup received" << std::endl; + TryEmitOwnProof(); + TryEmitDestProof(); + break; + case IpcType::kRunSample: { + auto offset = static_cast(f.offset_ms); + if (f.a != 0) { + offset = f.a; + } + StartSample(f.sequence, offset); + break; + } + case IpcType::kShutdown: + exit_requested = true; + Emit(IpcType::kAck, EventKind::kAck); + break; + default: + if (f.type == kIpcArmFault) { + ArmFault(f); + } else if (f.type == kIpcSendTagged) { + SendTaggedNow(f.sequence); + } else if (f.type == kIpcQueryNow) { + QueryNow(f.a, f.c != 0); + } else if (f.type == kIpcAnnounceUnknown) { + StartAnnounceUnknown(); + } + break; + } + } +}; + +std::unique_ptr MakeApp(std::string const& state_dir) { + auto dir = std::make_shared(state_dir); + return AetherApp::Construct( + AetherAppContext{[dir]() { + return std::unique_ptr{ + std::make_unique(*dir)}; + }} +#if AE_DISTILLATION + .AddAdapterFactory([](AetherAppContext const& context) { + return EthernetAdapter::ptr::Create( + CreateWith{context.domain()}.with_id( + GlobalId::kEthernetAdapter), + context.aether(), context.poller(), context.dns_resolver()); + }) +#endif + ); +} + +} // namespace + +int RunClientRole(ClientArgs const& args) { + RoleState state; + state.side = static_cast(args.side) == 1 ? IpcSide::kB : IpcSide::kA; + state.ping_interval_ms = args.ping_interval_ms; + state.receive_window_ms = args.receive_window_ms; + state.run_id_hash = HashRunId(args.run_id); + + if (!state.pipe.Connect(args.pipe_name, 60000)) { + std::cerr << "pipe connect failed: " << args.pipe_name << "\n"; + return 2; + } + + state.app = MakeApp(args.state_dir); + auto parent = Uid::FromString(args.parent_uid); + auto& select = + state.app->aether()->SelectClient(parent, args.client_name); + state.select_sub = select.result_event().Subscribe( + [&](Result const& res) { + if (!res) { + state.Emit(IpcType::kEvent, EventKind::kError, 0, 0, 10); + state.exit_requested = true; + return; + } + state.client = res.value(); + if (state.side == IpcSide::kB) { + auto const ping_ms = state.ping_interval_ms; + auto const rx_ms = state.receive_window_ms; + std::cerr << "Bob SetReceiveSchedule applying ping_interval_ms=" + << ping_ms << " receive_window_ms=" << rx_ms << std::endl; + auto ok = state.client->SetReceiveSchedule(ReceiveSchedule{ + .ping_interval = std::chrono::duration_cast( + std::chrono::milliseconds{state.ping_interval_ms}), + .receive_window = std::chrono::duration_cast( + std::chrono::milliseconds{state.receive_window_ms}), + }); + std::cerr << "Bob SetReceiveSchedule done ok=" << static_cast(ok) + << " (expect ping_interval_ms=" << ping_ms + << " receive_window_ms=" << rx_ms << ")" << std::endl; + if (!ok) { + state.Emit(IpcType::kEvent, EventKind::kError, 0, 0, 11); + state.exit_requested = true; + return; + } +#if AE_ENABLE_PING + SetPingTraceHook(&OnPingTrace); +#endif +#if AE_ENABLE_PING_TEST_FAULTS + SetPingFaultTraceHook(&OnPingFaultTrace); +#endif + } + state.client_ready = true; + std::int64_t lo = 0; + std::int64_t hi = 0; + UidToHalves(state.client->uid(), lo, hi); + std::cerr << "Client ready side=" + << (state.side == IpcSide::kA ? "A" : "B") << std::endl; + state.Emit(IpcType::kUidReport, EventKind::kChildReady, 0, 0, lo, hi); + state.Emit(IpcType::kChildReady, EventKind::kChildReady); + }); + + while (!state.exit_requested && !state.app->IsExited()) { + auto const now = Now(); + auto next = state.app->Update(now); + state.DrainPingTraces(); + state.DrainFaultTraces(); + if (auto frame = state.pipe.TryReadFrame(0)) { + state.HandleIpc(*frame); + } + state.TryEmitOwnProof(); + state.TryEmitDestProof(); + state.PollWarmup(); + state.PollSampleTiming(); + state.app->WaitUntil( + std::min(next, now + std::chrono::milliseconds{5})); + } +#if AE_ENABLE_PING + SetPingTraceHook(nullptr); + if (static_cast(args.side) == 1 && !g_all_ping_traces.empty()) { + std::ofstream csv(args.state_dir + "/bob_ping_trace.csv"); + csv << "kind,server_id,planned_send_us,actual_send_us,early_by_us," + "base_rx_window_us,effective_wire_rx_window_us,required_rx_until_us," + "next_planned_send_us,ping_guard_us,min_rtt_us,p99_rtt_us," + "channel_generation,result_type,steady_us\n"; + for (auto const& rec : g_all_ping_traces) { + auto const& e = rec.event; + csv << static_cast(e.kind) << "," + << static_cast(e.server_id) << "," + << TimePointUs(e.planned_send_at) << "," + << TimePointUs(e.actual_send_at) << "," << DurationUs(e.early_by) + << "," << DurationUs(e.base_rx_window) << "," + << DurationUs(e.effective_wire_rx_window) << "," + << TimePointUs(e.required_rx_until) << "," + << TimePointUs(e.next_planned_send) << "," + << DurationUs(e.ping_guard) << "," << DurationUs(e.min_rtt) << "," + << DurationUs(e.p99_rtt) << "," << e.channel_generation << "," + << e.result_type << "," << rec.steady_us << "\n"; + } + } +#endif + return 0; +} + +} // namespace ae::test_uap_ping_retry_window diff --git a/examples/aether_uap_ping_retry_window_test/client_role.h b/examples/aether_uap_ping_retry_window_test/client_role.h new file mode 100644 index 00000000..8c5deb54 --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/client_role.h @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_PING_RETRY_WINDOW_TEST_CLIENT_ROLE_H_ +#define AETHER_UAP_PING_RETRY_WINDOW_TEST_CLIENT_ROLE_H_ + +#include +#include + +namespace ae::test_uap_ping_retry_window { + +enum class Side : std::uint8_t { kA = 0, kB = 1 }; + +struct ClientArgs { + Side side{Side::kA}; + std::string run_id; + std::string state_dir; + std::string pipe_name; + std::string client_name; + std::string artifact_dir; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; + std::int64_t ping_interval_ms{3000}; + std::int64_t receive_window_ms{1000}; +}; + +int RunClientRole(ClientArgs const& args); + +} // namespace ae::test_uap_ping_retry_window + +#endif // AETHER_UAP_PING_RETRY_WINDOW_TEST_CLIENT_ROLE_H_ diff --git a/examples/aether_uap_ping_retry_window_test/coordinator.cpp b/examples/aether_uap_ping_retry_window_test/coordinator.cpp new file mode 100644 index 00000000..c270a0e2 --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/coordinator.cpp @@ -0,0 +1,962 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "coordinator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#if defined(RegisterClass) +# undef RegisterClass +#endif + +#include "common/bench_ipc.h" +#include "common/bench_types.h" +#include "common/udp_proof_types.h" + +#include "aether/config.h" +#include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/ae_actions/ping_test_faults.h" + +namespace ae::test_uap_ping_retry_window { + +#if AE_ENABLE_PING_TEST_FAULTS +using ae::PingFaultMode; +#endif +using ae::PingTraceKind; +using ae::bench::uap::BenchProtocol; +using ae::bench::uap::ChannelProof; +using ae::bench::uap::EventKind; +using ae::bench::uap::IpcFrame; +using ae::bench::uap::IpcType; +using ae::bench::uap::NamedPipeServer; +using ae::bench::uap::PipeNameFor; +using ae::bench::uap::UdpProofPath; +using ae::bench::uap::UnpackUdpProofFrame; +using IpcSide = ae::bench::uap::Side; +namespace { +constexpr std::uint8_t kIpcArmFault = 13; +constexpr std::uint8_t kIpcSendTagged = 14; +constexpr std::uint8_t kIpcPingTraceEx = 16; +constexpr std::uint32_t kTagRequestLossQueued = 1; +constexpr std::uint32_t kTagResponseLossFirstWindow = 2; +constexpr std::uint32_t kTagAfterRetryWindow = 3; + + +constexpr int kOffsetsMs[] = {500, 800, 1500, 2500}; +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP +constexpr int kSamplesPerOffset = 30; +constexpr int kMinValidPerOffset = 30; +#else +constexpr int kSamplesPerOffset = 20; +constexpr int kMinValidPerOffset = 20; +#endif + +char const* SkipReasonString(std::int64_t code) { + switch (code) { + case 1: + return "skipped_too_close"; + case 2: + return "skipped_stale"; + case 3: + return "skipped_delay_too_long"; + case 5: + return "skipped_tcp_refuse"; + case 6: + return "INVALID_ROUTE_CHANGED"; + case 7: + return "no_dest_route"; + case 8: + return "no_dest_server_timing"; + default: + return "skipped_cycle"; + } +} + +struct BobPingEvent { + std::uint8_t kind{0}; + std::int64_t server_id{0}; + std::int64_t planned_us{0}; + std::int64_t actual_us{0}; + std::int64_t early_by_us{0}; + std::int64_t base_window_us{0}; + std::int64_t effective_window_us{0}; + std::int64_t required_until_us{0}; + std::int64_t next_planned_us{0}; + std::int64_t guard_us{0}; + std::int64_t min_rtt_us{0}; + std::int64_t p99_rtt_us{0}; + std::int64_t channel_generation{0}; + std::int64_t result_type{0}; + std::int64_t event_steady_us{0}; + std::int64_t logical_cycle_id{0}; + std::int64_t physical_attempt_index{0}; + std::int64_t fault_mode{0}; + std::int64_t wire_next_connect_ms{0}; + std::int64_t cycle_anchor_us{0}; + std::int64_t contract_deadline_us{0}; + std::int64_t next_local_send_us{0}; + std::int64_t request_was_sent{0}; + std::int64_t response_was_ignored{0}; + std::int64_t event_qpc{0}; +}; + +#if 0 +void AttachAndClassify(SampleRecord& rec, + std::vector const& pings) { + if (rec.duplicate_count > 1) { + rec.classification = "DUPLICATE"; + return; + } + if (rec.invalid_reason == "INVALID_ROUTE_CHANGED" || + rec.invalid_reason == "ROUTE_MISMATCH") { + rec.classification = "ROUTE_CHANGED"; + return; + } + + BobPingEvent const* covering = nullptr; + if (rec.receive_us > 0) { + for (auto const& p : pings) { + if (p.kind != 1) { + continue; + } + if (p.server_id != rec.actual_send_server_id && + p.server_id != rec.schedule_server_id) { + continue; + } + if (p.event_steady_us > rec.receive_us) { + continue; + } + auto const end = p.event_steady_us + p.effective_window_us; + if (rec.receive_us <= end) { + covering = &p; + } + } + } + + if (covering != nullptr) { + rec.bob_ping_server_id = covering->server_id; + rec.bob_ping_planned_send_us = covering->planned_us; + rec.bob_ping_actual_send_us = covering->actual_us; + rec.bob_ping_early_by_us = covering->early_by_us; + rec.bob_base_rx_window_us = covering->base_window_us; + rec.bob_effective_wire_rx_window_us = covering->effective_window_us; + rec.bob_required_rx_until_us = covering->required_until_us; + rec.bob_ping_guard_us = covering->guard_us; + for (auto const& p : pings) { + if (p.kind == 2 && p.server_id == covering->server_id && + p.actual_us == covering->actual_us) { + rec.bob_ping_result_us = p.event_steady_us; + } + } + bool const early = covering->early_by_us > 0 && + covering->effective_window_us > covering->base_window_us; + if (early) { + rec.classification = "EARLY_PING_EXTENDED_WINDOW"; + } else if (rec.offset_ms >= 1000) { + rec.classification = "WAITED_FOR_NEXT_WINDOW"; + } else { + rec.classification = "CURRENT_NORMAL_WINDOW"; + } + return; + } + + if (rec.receive_us > 0) { + rec.classification = "LATE_UNEXPLAINED"; + return; + } + rec.classification = "LOST"; +} +#endif + +struct ChildProc { + IpcSide side{}; + NamedPipeServer pipe; + PROCESS_INFORMATION pi{}; + std::uint64_t uid_lo{0}; + std::uint64_t uid_hi{0}; + bool ready{false}; + bool uid_ok{false}; + std::uint32_t seq{0}; + ChannelProof own_proof{}; + ChannelProof dest_proof{}; + bool got_own_proof{false}; + bool got_dest_proof{false}; + std::vector ping_events; + std::map recv_counts; + std::map recv_qpc; + std::map send_qpc; +}; + +std::string MakeRunId() { + SYSTEMTIME st{}; + GetSystemTime(&st); + char buf[64]; + std::snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u", st.wYear, + st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond); + return buf; +} + +std::string DefaultExePath() { + char path[MAX_PATH]{}; + GetModuleFileNameA(nullptr, path, MAX_PATH); + return path; +} + +bool SendCmd(ChildProc& child, IpcType type, std::uint32_t sequence = 0, + std::uint32_t offset_ms = 0, std::int64_t a = 0, + std::int64_t b = 0, std::int64_t c = 0) { + IpcFrame f{}; + f.type = static_cast(type); + f.side = static_cast(IpcSide::kCoordinator); + f.seq = ++child.seq; + f.sequence = sequence; + f.offset_ms = offset_ms; + f.a = a; + f.b = b; + f.c = c; + return child.pipe.WriteFrame(f); +} + +void HandleChildFrame(ChildProc& child, IpcFrame const& frame) { + auto const type = static_cast(frame.type); + if (type == IpcType::kUidReport) { + std::memcpy(&child.uid_lo, &frame.a, 8); + std::memcpy(&child.uid_hi, &frame.b, 8); + child.uid_ok = true; + } + if (type == IpcType::kChildReady || type == IpcType::kUidReport) { + child.ready = true; + } + if (type == IpcType::kUdpProof) { + auto proof = UnpackUdpProofFrame(frame); + auto const path = static_cast(frame.event_kind); + if (path == UdpProofPath::kOwn) { + child.own_proof = proof; + child.got_own_proof = true; + } else if (path == UdpProofPath::kDestination) { + child.dest_proof = proof; + child.got_dest_proof = true; + } + } + if (type == IpcType::kEvent && + static_cast(frame.event_kind) == EventKind::kSampleReceived) { + auto count = static_cast(frame.c); + if (count <= 0) { + count = 1; + } + child.recv_counts[frame.sequence] = count; + child.recv_qpc[frame.sequence] = frame.b; + } + if (type == IpcType::kSampleResult && + static_cast(frame.event_kind) == EventKind::kSampleSent) { + auto send_qpc = frame.c; + if (send_qpc == 0) { + send_qpc = frame.e; + } + if (send_qpc != 0) { + child.send_qpc[frame.sequence] = send_qpc; + } + } + if (type == IpcType::kPingTrace) { + BobPingEvent e{}; + e.kind = frame.event_kind; + e.server_id = frame.a; + e.planned_us = frame.b; + e.actual_us = frame.c; + e.early_by_us = frame.d; + e.base_window_us = frame.e; + e.effective_window_us = frame.f; + e.required_until_us = frame.g; + e.next_planned_us = frame.h; + e.guard_us = frame.i; + e.channel_generation = frame.j; + e.min_rtt_us = frame.k; + e.p99_rtt_us = frame.l; + e.result_type = static_cast(frame.offset_ms); + e.event_steady_us = frame.local_steady_us; + child.ping_events.push_back(e); + } + + if (frame.type == kIpcPingTraceEx) { + if (!child.ping_events.empty()) { + auto& e = child.ping_events.back(); + e.logical_cycle_id = frame.a; + e.physical_attempt_index = frame.b; + e.fault_mode = frame.c; + e.wire_next_connect_ms = frame.d; + e.cycle_anchor_us = frame.e; + e.contract_deadline_us = frame.f; + e.next_local_send_us = frame.g; + e.request_was_sent = frame.h; + e.response_was_ignored = frame.i; + e.event_qpc = frame.k; + } + } +} + +bool SpawnChild(ChildProc& child, CoordinatorArgs const& args, + std::string const& state_dir, std::string const& pipe_name, + std::string const& client_name, + std::string const& child_log_path) { + if (!child.pipe.Create(pipe_name)) { + std::cerr << "CreateNamedPipe failed for " << pipe_name << "\n"; + return false; + } + auto cmd = "\"" + args.exe_path + "\" --role client --side " + + std::string(child.side == IpcSide::kA ? "A" : "B") + " --run-id " + + args.run_id + " --state-dir \"" + state_dir + "\" --pipe \"" + + pipe_name + "\" --client-name " + client_name + " --parent-uid " + + args.parent_uid; + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE log = CreateFileA(child_log_path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, + &sa, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (log == INVALID_HANDLE_VALUE) { + std::cerr << "CreateFile child log failed: " << child_log_path << "\n"; + return false; + } + STARTUPINFOA si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = log; + si.hStdError = log; + std::vector cmdline(cmd.begin(), cmd.end()); + cmdline.push_back('\0'); + if (!CreateProcessA(nullptr, cmdline.data(), nullptr, nullptr, TRUE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &child.pi)) { + CloseHandle(log); + std::cerr << "CreateProcess failed: " << GetLastError() << "\n"; + return false; + } + CloseHandle(log); + if (!child.pipe.WaitForClient(120000)) { + std::cerr << "WaitForClient timeout side=" + << (child.side == IpcSide::kA ? "A" : "B") << "\n"; + return false; + } + return true; +} + +void StopChild(ChildProc& child) { + SendCmd(child, IpcType::kShutdown); + if (WaitForSingleObject(child.pi.hProcess, 15000) != WAIT_OBJECT_0) { + TerminateProcess(child.pi.hProcess, 1); + } + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.pipe.Close(); +} + +double QpcToMs(std::uint64_t delta_ticks) { + LARGE_INTEGER freq{}; + QueryPerformanceFrequency(&freq); + return (1000.0 * static_cast(delta_ticks)) / + static_cast(freq.QuadPart); +} + +double Percentile(std::vector values, double p) { + if (values.empty()) { + return 0; + } + std::sort(values.begin(), values.end()); + auto const idx = static_cast( + std::ceil(p * static_cast(values.size() - 1))); + return values[std::min(idx, values.size() - 1)]; +} + +} // namespace + +int RunCoordinator(CoordinatorArgs const& in_args) { + CoordinatorArgs args = in_args; + if (args.run_id.empty()) { + args.run_id = MakeRunId(); + } + if (args.exe_path.empty()) { + args.exe_path = DefaultExePath(); + } + if (args.artifact_dir.empty()) { + args.artifact_dir = ".artifacts/uap-delivery-timing/" + args.run_id; + } + + std::filesystem::create_directories(args.artifact_dir); + auto const state_root = + std::filesystem::path{args.artifact_dir} / "persistent-state"; + auto const state_a = (state_root / "state-a").string(); + auto const state_b = (state_root / "state-b").string(); + std::filesystem::create_directories(state_a); + std::filesystem::create_directories(state_b); + + ChildProc alice; + alice.side = IpcSide::kA; + ChildProc bob; + bob.side = IpcSide::kB; + + auto const pipe_a = PipeNameFor(args.run_id, IpcSide::kA); + auto const pipe_b = PipeNameFor(args.run_id, IpcSide::kB); + + std::cout << "Spawning Alice/Bob run_id=" << args.run_id + << (args.quick ? " quick=1" : "") << std::endl; + auto const log_a = + (std::filesystem::path{args.artifact_dir} / "alice.log").string(); + auto const log_b = + (std::filesystem::path{args.artifact_dir} / "bob.log").string(); + if (!SpawnChild(alice, args, state_a, pipe_a, "uap-bench-alice", log_a) || + !SpawnChild(bob, args, state_b, pipe_b, "uap-bench-bob", log_b)) { + return 2; + } + + auto wait_ready = [&](ChildProc& c, char const* name) { + auto const deadline = GetTickCount64() + 180000; + while (GetTickCount64() < deadline && !(c.ready && c.uid_ok)) { + if (auto f = c.pipe.TryReadFrame(200)) { + HandleChildFrame(c, *f); + } + } + if (!(c.ready && c.uid_ok)) { + std::cerr << name << " not ready" << std::endl; + return false; + } + return true; + }; + if (!wait_ready(alice, "Alice") || !wait_ready(bob, "Bob")) { + StopChild(alice); + StopChild(bob); + return 3; + } + + std::int64_t a_lo = 0; + std::int64_t a_hi = 0; + std::int64_t b_lo = 0; + std::int64_t b_hi = 0; + std::memcpy(&a_lo, &alice.uid_lo, 8); + std::memcpy(&a_hi, &alice.uid_hi, 8); + std::memcpy(&b_lo, &bob.uid_lo, 8); + std::memcpy(&b_hi, &bob.uid_hi, 8); + SendCmd(alice, IpcType::kSetPeerUid, 0, 0, b_lo, b_hi); + SendCmd(bob, IpcType::kSetPeerUid, 0, 0, a_lo, a_hi); + // Drain acks + for (int i = 0; i < 20; ++i) { + if (auto f = alice.pipe.TryReadFrame(100)) { + HandleChildFrame(alice, *f); + } + if (auto f = bob.pipe.TryReadFrame(100)) { + HandleChildFrame(bob, *f); + } + } + + std::cout << "Waiting Bob warm-up (>=10 ping RTT samples)..." << std::endl; + SendCmd(bob, IpcType::kWaitWarmup); + std::int64_t warmup_n = 0; + std::int64_t warmup_min = 0; + std::int64_t warmup_p99 = 0; + std::uint32_t warmup_guard = 0; + { + auto const deadline = GetTickCount64() + 300000; + bool done = false; + while (GetTickCount64() < deadline && !done) { + if (auto f = bob.pipe.TryReadFrame(500)) { + HandleChildFrame(bob, *f); + if (static_cast(f->type) == IpcType::kWarmupDone) { + warmup_n = f->a; + warmup_min = f->b; + warmup_p99 = f->c; + warmup_guard = f->offset_ms; + done = true; + } + } + if (auto f = alice.pipe.TryReadFrame(0)) { + HandleChildFrame(alice, *f); + } + } + if (!done) { + std::cerr << "Bob warm-up timed out\n"; + StopChild(alice); + StopChild(bob); + return 4; + } + } + + SendCmd(alice, IpcType::kWaitWarmup); + std::int64_t alice_warmup_n = 0; + std::int64_t alice_warmup_min = 0; + std::int64_t alice_warmup_p99 = 0; + std::int64_t alice_dest_server = 0; + std::int64_t alice_dest_protocol = 0; + { + auto const deadline = GetTickCount64() + 300000; + bool done = false; + while (GetTickCount64() < deadline && !done) { + if (auto f = alice.pipe.TryReadFrame(500)) { + HandleChildFrame(alice, *f); + if (static_cast(f->type) == IpcType::kWarmupDone) { + alice_warmup_n = f->a; + alice_warmup_min = f->b; + alice_warmup_p99 = f->c; + alice_dest_server = f->d; + alice_dest_protocol = f->e; + done = true; + } + } + if (auto f = bob.pipe.TryReadFrame(0)) { + HandleChildFrame(bob, *f); + } + } + if (!done) { + std::cerr << "Alice dest-server warm-up timed out\n"; + StopChild(alice); + StopChild(bob); + return 4; + } + } + + std::cout << "## Bob ping statistics\n" + << "samples=" << warmup_n << " min_rtt_ms=" << warmup_min + << " p99_rtt_ms=" << warmup_p99 << " guard_ms=" << warmup_guard + << std::endl + << std::endl; + std::cout << "## Alice dest-server ping statistics\n" + << "server_id=" << alice_dest_server << " samples=" << alice_warmup_n + << " min_rtt_ms=" << alice_warmup_min + << " p99_rtt_ms=" << alice_warmup_p99 << std::endl + << std::endl; + + auto const dest_proto = static_cast(alice_dest_protocol); + std::cout << "AE_SUPPORT_TCP=" << AE_SUPPORT_TCP + << " AE_SUPPORT_UDP=" << AE_SUPPORT_UDP + << " selected alice=" << BenchProtocolName(alice.own_proof.protocol) + << " bob=" << BenchProtocolName(bob.own_proof.protocol) + << " dest=" << BenchProtocolName(dest_proto) << "\n\n"; + if (!IsMeasuredProtocolOk(alice.own_proof.protocol) || + !IsMeasuredProtocolOk(bob.own_proof.protocol) || + !IsMeasuredProtocolOk(dest_proto)) { +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP + std::cerr << "FAIL: measured work path is not UDP " + "(registration may be TCP)\n"; +#else + std::cerr << "FAIL: measured work path is not TCP\n"; +#endif + StopChild(alice); + StopChild(bob); + return 6; + } + + + [[maybe_unused]] bool scenarios_ok = false; +#if !AE_ENABLE_PING_TEST_FAULTS + std::cerr << "AE_ENABLE_PING_TEST_FAULTS is required for this test\n"; + StopChild(alice); + StopChild(bob); + return 2; +#else + bool ok = true; + auto send_raw = [&](ChildProc& child, std::uint8_t type, std::uint32_t sequence = 0, + std::int64_t a = 0, std::int64_t b = 0, std::int64_t c = 0, + std::int64_t d = 0) { + IpcFrame f{}; + f.type = type; + f.side = static_cast(IpcSide::kCoordinator); + f.seq = ++child.seq; + f.sequence = sequence; + f.a = a; + f.b = b; + f.c = c; + f.d = d; + return child.pipe.WriteFrame(f); + }; + + auto drain = [&](DWORD slice_ms) { + // Prefer Bob: Alice ping traces otherwise fill the 4KiB pipe and Bob + // blocks in WriteFrame, so he never reads ArmFault. + std::optional last; + if (auto f = bob.pipe.TryReadFrame(slice_ms)) { + HandleChildFrame(bob, *f); + last = f; + } + for (;;) { + bool got = false; + if (auto f = bob.pipe.TryReadFrame(0)) { + HandleChildFrame(bob, *f); + last = f; + got = true; + } + if (auto f = alice.pipe.TryReadFrame(0)) { + HandleChildFrame(alice, *f); + last = f; + got = true; + } + if (!got) { + break; + } + } + return last; + }; + + std::size_t ping_cursor = 0; + auto wait_ping = [&](std::uint8_t kind, std::int64_t server_id, DWORD timeout_ms) + -> std::optional { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(50); + while (ping_cursor < bob.ping_events.size()) { + auto const& e = bob.ping_events[ping_cursor++]; + if (e.kind == kind && (server_id == 0 || e.server_id == server_id)) { + return e; + } + } + } + return std::nullopt; + }; + auto find_ping = [&](std::uint8_t kind, std::int64_t server_id, + std::int64_t cycle_id, std::int64_t min_attempt, + std::size_t start) -> std::optional { + for (std::size_t i = start; i < bob.ping_events.size(); ++i) { + auto const& e = bob.ping_events[i]; + if (e.kind != kind) { + continue; + } + if (server_id != 0 && e.server_id != server_id) { + continue; + } + if (cycle_id != 0 && e.logical_cycle_id != 0 && + e.logical_cycle_id != cycle_id) { + continue; + } + if (e.physical_attempt_index < min_attempt) { + continue; + } + return e; + } + return std::nullopt; + }; + + auto recv_total = [&](std::uint32_t tag) -> int { + auto const it = bob.recv_counts.find(tag); + return it == bob.recv_counts.end() ? 0 : it->second; + }; + auto wait_until_recv = [&](std::uint32_t tag, int min_count, + DWORD timeout_ms) -> int { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(50); + int const n = recv_total(tag); + if (n >= min_count) { + return n; + } + } + return recv_total(tag); + }; + + auto wait_sent = [&](std::uint32_t tag, DWORD timeout_ms) -> bool { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + if (auto f = alice.pipe.TryReadFrame(50)) { + HandleChildFrame(alice, *f); + auto const type = static_cast(f->type); + auto const kind = static_cast(f->event_kind); + if (type == IpcType::kSampleResult && f->sequence == tag) { + if (kind == EventKind::kSampleSent) { + return true; + } + if (kind == EventKind::kSampleSkipped) { + return false; + } + } + } + drain(0); + } + return false; + }; + + auto const dest = alice_dest_server; + std::cout << "dest_server_id=" << dest << " transport=" << args.transport + << std::endl; + + auto window_open = [&]() -> bool { + std::int64_t last_sent = -1; + std::int64_t last_closed = -1; + for (auto const& e : bob.ping_events) { + if (e.server_id != dest) { + continue; + } + if (e.kind == static_cast(PingTraceKind::kRequestSent)) { + last_sent = e.event_steady_us; + } + if (e.kind == static_cast(PingTraceKind::kRxClosed)) { + last_closed = e.event_steady_us; + } + } + return last_sent > last_closed; + }; + auto wait_window_closed = [&](DWORD timeout_ms) -> bool { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + drain(50); + if (!window_open()) { + return true; + } + } + return !window_open(); + }; + + if (!wait_window_closed(8000)) { + std::cerr << "WARN: dest RX window still open before drop-request" << std::endl; + } + Sleep(250); + + auto arm = [&](std::int64_t mode, std::int64_t timeout_us) { + drain(50); + send_raw(bob, kIpcArmFault, 0, dest, 1, mode, timeout_us); + drain(200); + }; + + // Scenario 1: drop request + arm(static_cast(PingFaultMode::kDropRequest), 400000); + auto dropped = wait_ping(static_cast(PingTraceKind::kRequestDropped), + dest, 20000); + if (!dropped) { + std::cerr << "FAIL drop-request: no REQUEST_DROPPED\n"; + ok = false; + } else { + std::size_t const after_drop = ping_cursor; + Sleep(100); + send_raw(alice, kIpcSendTagged, kTagRequestLossQueued); + if (!wait_sent(kTagRequestLossQueued, 5000)) { + std::cerr << "FAIL drop-request: Alice send skipped/timeout\n"; + ok = false; + } + auto timeout_ev = + wait_ping(static_cast(PingTraceKind::kAttemptTimeout), dest, + 10000); + std::optional retry; + auto const retry_deadline = GetTickCount64() + 10000; + while (GetTickCount64() < retry_deadline && !retry) { + drain(50); + retry = find_ping(static_cast(PingTraceKind::kRequestSent), + dest, dropped->logical_cycle_id, 2, after_drop); + if (!retry) { + retry = find_ping( + static_cast(PingTraceKind::kAttemptPrepared), dest, + dropped->logical_cycle_id, 2, after_drop); + } + } + if (!timeout_ev || !retry) { + std::cerr << "FAIL drop-request: missing timeout/retry\n"; + ok = false; + } else { + auto const retry_delay_ms = + (retry->event_steady_us - timeout_ev->event_steady_us) / 1000; + if (retry_delay_ms > 100) { + std::cerr << "FAIL drop-request: retry delay " << retry_delay_ms + << "ms > 100ms\n"; + ok = false; + } + if (retry->wire_next_connect_ms <= 0) { + std::cerr << "FAIL drop-request: wire_next_connect_ms=" + << retry->wire_next_connect_ms << " (must not be 0)\n"; + ok = false; + } + if (dropped->wire_next_connect_ms > 0 && + retry->wire_next_connect_ms >= dropped->wire_next_connect_ms) { + std::cerr << "FAIL drop-request: retry wire_next_connect_ms=" + << retry->wire_next_connect_ms + << " did not shrink vs first=" << dropped->wire_next_connect_ms + << "\n"; + ok = false; + } + } + int const got = wait_until_recv(kTagRequestLossQueued, 1, 4000); + if (got != 1) { + std::cerr << "FAIL drop-request: expected 1 receive, got " << got << "\n"; + ok = false; + } else if (retry && retry->event_qpc != 0 && + bob.recv_qpc[kTagRequestLossQueued] <= retry->event_qpc) { + std::cerr << "FAIL drop-request: message_receive_qpc=" + << bob.recv_qpc[kTagRequestLossQueued] + << " is not after retry_request_sent_qpc=" << retry->event_qpc + << " cycle=" << retry->logical_cycle_id + << " attempt=" << retry->physical_attempt_index << "\n"; + ok = false; + } + auto confirmed = + wait_ping(static_cast(PingTraceKind::kCycleConfirmed), dest, + 10000); + auto next_sched = wait_ping( + static_cast(PingTraceKind::kNextCycleScheduled), dest, 5000); + if (dropped && next_sched && dropped->next_local_send_us > 0 && + next_sched->next_local_send_us > 0) { + auto const delta = + next_sched->next_local_send_us - dropped->next_local_send_us; + if (delta < -20000 || delta > 20000) { + std::cerr << "FAIL drop-request: next local send phase shift delta_us=" + << delta << "\n"; + ok = false; + } + } + (void)confirmed; + std::cout << "scenario drop-request " << (ok ? "progressing" : "failed") + << std::endl; + } + + // Scenario 2: ignore response (400ms timeout override) + arm(static_cast(PingFaultMode::kIgnoreResponse), 400000); + auto sent1 = wait_ping(static_cast(PingTraceKind::kRequestSent), + dest, 20000); + auto ignored = + wait_ping(static_cast(PingTraceKind::kResponseIgnored), dest, + 5000); + if (!sent1 || !ignored || sent1->request_was_sent == 0) { + std::cerr << "FAIL ignore-response: missing SENT/IGNORED\n"; + ok = false; + } else { + Sleep(100); + send_raw(alice, kIpcSendTagged, kTagResponseLossFirstWindow); + if (!wait_sent(kTagResponseLossFirstWindow, 5000)) { + std::cerr << "FAIL ignore-response: Alice send failed\n"; + ok = false; + } + int const got = wait_until_recv(kTagResponseLossFirstWindow, 1, 3000); + if (got != 1) { + std::cerr << "FAIL ignore-response: expected receive in first window, got " + << got << "\n"; + ok = false; + } + auto retry = wait_ping(static_cast(PingTraceKind::kRequestSent), + dest, 10000); + if (!retry || retry->wire_next_connect_ms <= 0) { + std::cerr << "FAIL ignore-response: retry nextConnect missing/zero\n"; + ok = false; + } else if (sent1->wire_next_connect_ms > 0 && + retry->wire_next_connect_ms >= sent1->wire_next_connect_ms) { + std::cerr << "FAIL ignore-response: retry nextConnect=" + << retry->wire_next_connect_ms + << " did not shrink vs first=" << sent1->wire_next_connect_ms + << "\n"; + ok = false; + } + if (got == 1 && sent1 && retry && sent1->event_qpc != 0 && + retry->event_qpc != 0) { + auto const recv_qpc = bob.recv_qpc[kTagResponseLossFirstWindow]; + if (!(sent1->event_qpc < recv_qpc && recv_qpc < retry->event_qpc)) { + std::cerr << "FAIL ignore-response: recv_qpc=" << recv_qpc + << " not between first_sent=" << sent1->event_qpc + << " and retry_sent=" << retry->event_qpc << "\n"; + ok = false; + } + } + int const dup = wait_until_recv(kTagResponseLossFirstWindow, 2, 500); + if (dup >= 2) { + std::cerr << "FAIL ignore-response: duplicate receive\n"; + ok = false; + } + } + + // Scenario 3: after retry window, before next logical ping + if (!wait_window_closed(8000)) { + std::cerr << "FAIL after-window: retry RX window did not close" << std::endl; + ok = false; + } + Sleep(300); + std::int64_t close_qpc = 0; + for (auto it = bob.ping_events.rbegin(); it != bob.ping_events.rend(); ++it) { + if (it->server_id == dest && + it->kind == static_cast(PingTraceKind::kRxClosed)) { + close_qpc = it->event_qpc; + break; + } + } + send_raw(alice, kIpcSendTagged, kTagAfterRetryWindow); + if (!wait_sent(kTagAfterRetryWindow, 5000)) { + std::cerr << "FAIL after-window: send failed\n"; + ok = false; + } + auto next_sent = + wait_ping(static_cast(PingTraceKind::kRequestSent), dest, + 20000); + int const late = wait_until_recv(kTagAfterRetryWindow, 1, 8000); + if (late != 1) { + std::cerr << "FAIL after-window: expected 1 receive after next ping, got " + << late << "\n"; + ok = false; + } + auto const send_qpc = alice.send_qpc[kTagAfterRetryWindow]; + auto const recv_qpc = bob.recv_qpc[kTagAfterRetryWindow]; + if (late == 1 && close_qpc != 0 && send_qpc != 0 && next_sent && + next_sent->event_qpc != 0) { + if (!(send_qpc > close_qpc)) { + std::cerr << "FAIL after-window: message_send_qpc=" << send_qpc + << " is not after retry_window_close_qpc=" << close_qpc << "\n"; + ok = false; + } + if (!(recv_qpc >= next_sent->event_qpc)) { + std::cerr << "FAIL after-window: message_receive_qpc=" << recv_qpc + << " is before next_logical_ping_request_sent_qpc=" + << next_sent->event_qpc << "\n"; + ok = false; + } + } + (void)next_sent; + + std::cout << (ok ? "PASS ping retry window scenarios" + : "FAIL ping retry window scenarios") + << std::endl; + scenarios_ok = ok; +#endif + { + auto const ping_csv = + (std::filesystem::path{args.artifact_dir} / "bob_ping_trace.csv") + .string(); + std::ofstream out(ping_csv); + out << "kind,server_id,planned_us,actual_us,early_by_us,base_window_us," + "effective_window_us,required_until_us,next_planned_us,guard_us," + "min_rtt_us,p99_rtt_us,channel_generation,result_type," + "event_steady_us\n"; + for (auto const& e : bob.ping_events) { + out << static_cast(e.kind) << "," << e.server_id << "," + << e.planned_us << "," << e.actual_us << "," << e.early_by_us << "," + << e.base_window_us << "," << e.effective_window_us << "," + << e.required_until_us << "," << e.next_planned_us << "," + << e.guard_us << "," << e.min_rtt_us << "," << e.p99_rtt_us << "," + << e.channel_generation << "," << e.result_type << "," + << e.event_steady_us << "\n"; + } + } + + StopChild(alice); + StopChild(bob); +#if AE_ENABLE_PING_TEST_FAULTS + return scenarios_ok ? 0 : 7; +#else + return 2; +#endif +} + +} // namespace ae::test_uap_ping_retry_window diff --git a/examples/aether_uap_ping_retry_window_test/coordinator.h b/examples/aether_uap_ping_retry_window_test/coordinator.h new file mode 100644 index 00000000..38864932 --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/coordinator.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_PING_RETRY_WINDOW_TEST_COORDINATOR_H_ +#define AETHER_UAP_PING_RETRY_WINDOW_TEST_COORDINATOR_H_ + +#include + +namespace ae::test_uap_ping_retry_window { + +struct CoordinatorArgs { + std::string run_id; + std::string artifact_dir; + std::string exe_path; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; + std::string transport{"tcp"}; + bool quick{false}; +}; + +int RunCoordinator(CoordinatorArgs const& args); + +} // namespace ae::test_uap_ping_retry_window + +#endif // AETHER_UAP_PING_RETRY_WINDOW_TEST_COORDINATOR_H_ diff --git a/examples/aether_uap_ping_retry_window_test/main.cpp b/examples/aether_uap_ping_retry_window_test/main.cpp new file mode 100644 index 00000000..31787798 --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/main.cpp @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "client_role.h" +#include "coordinator.h" + +namespace { + +std::string_view ArgValue(int argc, char** argv, std::string_view key) { + for (int i = 1; i < argc; ++i) { + std::string_view a = argv[i]; + if (a == key && i + 1 < argc) { + return argv[i + 1]; + } + if (a.size() > key.size() && a.substr(0, key.size()) == key && + a[key.size()] == '=') { + return a.substr(key.size() + 1); + } + } + return {}; +} + +bool HasFlag(int argc, char** argv, std::string_view key) { + for (int i = 1; i < argc; ++i) { + if (key == argv[i]) { + return true; + } + } + return false; +} + +} // namespace + +int main(int argc, char** argv) { + using namespace ae::test_uap_ping_retry_window; + + auto role = ArgValue(argc, argv, "--role"); + if (role.empty() || role == "coordinator") { + CoordinatorArgs args; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + args.exe_path = std::string{ArgValue(argc, argv, "--exe")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + auto transport = ArgValue(argc, argv, "--transport"); + if (!transport.empty()) { + args.transport = std::string{transport}; + } + args.quick = HasFlag(argc, argv, "--quick"); + return RunCoordinator(args); + } + + if (role == "client") { + ClientArgs args; + auto side = ArgValue(argc, argv, "--side"); + args.side = (side == "B" || side == "b") ? Side::kB : Side::kA; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.state_dir = std::string{ArgValue(argc, argv, "--state-dir")}; + args.pipe_name = std::string{ArgValue(argc, argv, "--pipe")}; + args.client_name = std::string{ArgValue(argc, argv, "--client-name")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + if (args.client_name.empty()) { + args.client_name = + args.side == Side::kA ? "uap-retry-alice" : "uap-retry-bob"; + } + auto ping_ms = ArgValue(argc, argv, "--ping-interval-ms"); + if (!ping_ms.empty()) { + args.ping_interval_ms = std::strtoll(ping_ms.data(), nullptr, 10); + } + auto rx_ms = ArgValue(argc, argv, "--receive-window-ms"); + if (!rx_ms.empty()) { + args.receive_window_ms = std::strtoll(rx_ms.data(), nullptr, 10); + } + return RunClientRole(args); + } + + std::cerr << "Unknown --role\n"; + return 2; +} diff --git a/examples/aether_uap_ping_retry_window_test/tele_off.h b/examples/aether_uap_ping_retry_window_test/tele_off.h new file mode 100644 index 00000000..89da4ca9 --- /dev/null +++ b/examples/aether_uap_ping_retry_window_test/tele_off.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Load USER_CONFIG (and the rest of aether/config.h) first, then override +// console telemetry for this benchmark/example target. Force-include this +// header so the override wins without a conflicting /D AE_TELE_LOG_CONSOLE. +#include "aether/config.h" + +#undef AE_TELE_LOG_CONSOLE +#define AE_TELE_LOG_CONSOLE 0 diff --git a/examples/benches/aether_uap_delivery_timing_bench/CMakeLists.txt b/examples/benches/aether_uap_delivery_timing_bench/CMakeLists.txt new file mode 100644 index 00000000..05100fbd --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/CMakeLists.txt @@ -0,0 +1,59 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM AND WIN32) + project("aether_uap_delivery_timing_bench" VERSION "1.0.0" LANGUAGES C CXX) + + add_library(aether_uap_delivery_timing_bench_common STATIC + common/bench_ipc.cpp + ) + target_include_directories(aether_uap_delivery_timing_bench_common PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/common + ) + target_link_libraries(aether_uap_delivery_timing_bench_common PUBLIC aether) + + add_executable(aether_uap_delivery_timing_bench + main.cpp + client_role.cpp + coordinator.cpp + ) + target_link_libraries(aether_uap_delivery_timing_bench PRIVATE + aether_uap_delivery_timing_bench_common + ) + target_include_directories(aether_uap_delivery_timing_bench PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../.. + ) + target_compile_definitions(aether_uap_delivery_timing_bench PRIVATE + _CRT_SECURE_NO_WARNINGS + ) + if(MSVC) + target_compile_options(aether_uap_delivery_timing_bench PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + target_compile_options(aether_uap_delivery_timing_bench_common PRIVATE + /W4 /WX + "/FI${CMAKE_CURRENT_SOURCE_DIR}/tele_off.h" + ) + endif() +else() + message(WARNING "aether_uap_delivery_timing_bench is Windows desktop only; skipped") +endif() diff --git a/examples/benches/aether_uap_delivery_timing_bench/client_role.cpp b/examples/benches/aether_uap_delivery_timing_bench/client_role.cpp new file mode 100644 index 00000000..e1d9a34e --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/client_role.cpp @@ -0,0 +1,832 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client_role.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +// Windows.h maps RegisterClass -> RegisterClassA/W; aether's Registry uses +// RegisterClass by name. +#if defined(RegisterClass) +# undef RegisterClass +#endif + +#define AE_EXAMPLE_ETHERNET 1 +#include "aether/all.h" +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/channels/channel.h" +#include "aether/client_messages/p2p_message_stream.h" +#include "aether/cloud_connections/ping_schedule_guard.h" +#include "aether/cloud_connections/ping_cloud_servers.h" +#include "aether/receive_schedule.h" +#include "aether/server_connections/server_connection.h" + +#include "common/bench_ipc.h" +#include "common/bench_message.h" +#include "common/directory_domain_storage.h" +#include "common/udp_proof.h" + +namespace ae::bench::uap { +namespace { + +constexpr auto kBobPingInterval = std::chrono::milliseconds{3000}; +constexpr auto kBobReceiveWindow = std::chrono::milliseconds{1000}; +constexpr auto kSkipIfCloserThan = std::chrono::milliseconds{50}; +constexpr std::size_t kWarmupSamples = 10; + +inline std::int64_t TimePointUs(TimePoint tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +inline std::int64_t DurationUs(Duration d) { + return static_cast(d.count()); +} + +inline std::int64_t BenchProtocolFromAe(Protocol protocol) { + if (protocol == Protocol::kUdp) { + return static_cast(BenchProtocol::kUdp); + } + if (protocol == Protocol::kTcp) { + return static_cast(BenchProtocol::kTcp); + } + return static_cast(BenchProtocol::kUnknown); +} + +inline std::int64_t SteadyUsNow() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +#if AE_ENABLE_PING +struct PendingPingTrace { + PingTraceEvent event; + std::int64_t steady_us{0}; +}; +std::vector g_pending_ping_traces; +std::vector g_all_ping_traces; + +void OnPingTrace(PingTraceEvent const& event) { + PendingPingTrace rec{event, SteadyUsNow()}; + g_pending_ping_traces.push_back(rec); + if (g_all_ping_traces.size() < 4096) { + g_all_ping_traces.push_back(rec); + } +} +#endif + +inline void UidToHalves(Uid const& uid, std::int64_t& lo, std::int64_t& hi) { + std::memcpy(&lo, uid.value.data(), 8); + std::memcpy(&hi, uid.value.data() + 8, 8); +} + +inline Uid UidFromHalves(std::int64_t lo, std::int64_t hi) { + Uid uid{}; + std::memcpy(uid.value.data(), &lo, 8); + std::memcpy(uid.value.data() + 8, &hi, 8); + return uid; +} + +inline std::uint64_t QpcNow() { + LARGE_INTEGER v{}; + QueryPerformanceCounter(&v); + return static_cast(v.QuadPart); +} + +struct RoleState { + Side side{}; + std::uint32_t run_id_hash{0}; + NamedPipeClient pipe; + std::unique_ptr app; + Client::ptr client; + Uid peer_uid{}; + bool peer_set{false}; + std::shared_ptr stream; + Subscription stream_sub; + Subscription new_port_sub; + Subscription select_sub; + Subscription query_sub; + Subscription dest_cloud_sub; + bool dest_proof_sent{false}; + bool own_proof_sent{false}; + std::optional dest_retry_at_{}; + ChannelProof own_proof{}; + ChannelProof dest_proof{}; + std::unordered_map seen; + std::uint32_t ipc_seq{0}; + bool exit_requested{false}; + bool client_ready{false}; + bool warmup_active{false}; + bool sample_in_flight{false}; + std::uint32_t pending_sequence{0}; + std::uint32_t pending_offset_ms{0}; + std::optional send_at_{}; + std::optional requery_at_{}; + std::int64_t pending_last_us_{0}; + std::int64_t pending_next_us_{-1}; + std::vector last_diagnostics_{}; + std::int64_t pending_schedule_server_id_{0}; + std::int64_t pending_route_generation_{0}; + std::int64_t pending_protocol_{0}; + std::int64_t pending_raw_delta_ms_{0}; + std::int64_t pending_last_connect_ms_{0}; + std::int64_t pending_qsend_us_{0}; + std::int64_t pending_one_way_us_{0}; + std::int64_t pending_target_us_{0}; + + bool Emit(IpcType type, EventKind kind = EventKind::kAck, + std::uint32_t sequence = 0, std::uint32_t offset_ms = 0, + std::int64_t a = 0, std::int64_t b = 0, std::int64_t c = 0, + std::int64_t d = 0, std::int64_t e = 0, std::int64_t f = 0, + std::int64_t g = 0, std::int64_t h = 0, std::int64_t i = 0, + std::int64_t j = 0, std::int64_t k = 0, std::int64_t l = 0) { + IpcFrame frame{}; + frame.type = static_cast(type); + frame.side = static_cast(side); + frame.event_kind = static_cast(kind); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.sequence = sequence; + frame.offset_ms = offset_ms; + frame.local_steady_us = SteadyUsNow(); + frame.a = a; + frame.b = b; + frame.c = c; + frame.d = d; + frame.e = e; + frame.f = f; + frame.g = g; + frame.h = h; + frame.i = i; + frame.j = j; + frame.k = k; + frame.l = l; + return pipe.WriteFrame(frame); + } + + void DrainPingTraces() { +#if AE_ENABLE_PING + for (auto const& rec : g_pending_ping_traces) { + auto const& e = rec.event; + IpcFrame frame{}; + frame.type = static_cast(IpcType::kPingTrace); + frame.side = static_cast(side); + frame.event_kind = static_cast(e.kind); + frame.run_id_hash = run_id_hash; + frame.seq = ++ipc_seq; + frame.offset_ms = + e.result_type < 0 ? 0 : static_cast(e.result_type); + frame.local_steady_us = rec.steady_us; + frame.a = static_cast(e.server_id); + frame.b = TimePointUs(e.planned_send_at); + frame.c = TimePointUs(e.actual_send_at); + frame.d = DurationUs(e.early_by); + frame.e = DurationUs(e.base_rx_window); + frame.f = DurationUs(e.effective_wire_rx_window); + frame.g = TimePointUs(e.required_rx_until); + frame.h = TimePointUs(e.next_planned_send); + frame.i = DurationUs(e.ping_guard); + frame.j = static_cast(e.channel_generation); + frame.k = DurationUs(e.min_rtt); + frame.l = DurationUs(e.p99_rtt); + pipe.WriteFrame(frame); + } + g_pending_ping_traces.clear(); +#endif + } + + void EmitUdpProof(UdpProofPath path, ChannelProof const& proof) { + if (path == UdpProofPath::kOwn) { + own_proof = proof; + } else if (path == UdpProofPath::kDestination) { + dest_proof = proof; + } + IpcFrame f{}; + PackUdpProofFrame(f, path, proof); + f.side = static_cast(side); + f.run_id_hash = run_id_hash; + f.seq = ++ipc_seq; + f.local_steady_us = SteadyUsNow(); + pipe.WriteFrame(f); + } + + static bool IsClassifiedWorkProtocol(BenchProtocol protocol) noexcept { + return protocol == BenchProtocol::kTcp || protocol == BenchProtocol::kUdp; + } + + void TryEmitOwnProof() { + if (!client) { + return; + } + (void)client->cloud_connection(); + auto proof = CollectOwnCloudProof(*client); + if (!proof.present || !IsClassifiedWorkProtocol(proof.protocol)) { + return; + } + if (own_proof_sent && own_proof.protocol == proof.protocol) { + return; + } + own_proof_sent = true; + EmitUdpProof(UdpProofPath::kOwn, proof); + } + + void TryEmitDestProof() { + if (dest_proof_sent || !client || !peer_set) { + return; + } + auto const now = Now(); + if (dest_retry_at_ && now < *dest_retry_at_) { + return; + } + dest_retry_at_ = now + std::chrono::milliseconds{250}; + dest_cloud_sub.Reset(); + auto& get_cloud = client->cloud_manager()->GetCloud(peer_uid); + dest_cloud_sub = get_cloud.result_event().Subscribe( + [this](Result const& res) { + if (!res) { + return; + } + auto proof = CollectDestinationProofFromCloud(*client, res.value()); + if (!proof.present || !IsClassifiedWorkProtocol(proof.protocol)) { + return; + } + dest_proof_sent = true; + EmitUdpProof(UdpProofPath::kDestination, proof); + }); + } + + void EnsureStreams() { + if (!client || !peer_set) { + return; + } + if (side == Side::kA && !stream) { + stream = std::make_shared( + AeContext{*app}, client.Load(), peer_uid, + client->message_stream_manager().CreatePort(peer_uid)); + stream_sub = stream->out_data_event().Subscribe( + [this](DataBuffer const& data) { OnReceive(data); }); + } + if (!new_port_sub) { + new_port_sub = client->message_stream_manager().new_port_event().Subscribe( + [this](P2pPortHandle handle) { + if (handle.destination() != peer_uid && side == Side::kB) { + // Bob accepts any inbound port from Alice after peer is set. + } + stream = std::make_shared( + AeContext{*app}, client.Load(), handle.destination(), + std::move(handle)); + stream_sub = stream->out_data_event().Subscribe( + [this](DataBuffer const& data) { OnReceive(data); }); + }); + } + } + + void OnReceive(DataBuffer const& data) { + auto msg = DeserializeDeliveryBenchMessage(data.data(), data.size()); + if (!msg) { + Emit(IpcType::kEvent, EventKind::kError, 0, 0, 1); + return; + } + auto& count = seen[msg->sequence]; + ++count; + auto const recv_qpc = static_cast(QpcNow()); + Emit(IpcType::kEvent, EventKind::kSampleReceived, msg->sequence, + msg->offset_ms, static_cast(msg->send_qpc), recv_qpc, + count); + } + + // Returns max response sample count across active channels, and fills + // min/p99 from the channel with the most samples. + std::size_t CollectResponseStats(Duration* min_out, Duration* p99_out) { + std::size_t best = 0; + Duration best_min{}; + Duration best_p99{}; + auto& csc = client->cloud_connection(); + for (auto* sc : csc.servers()) { + if (sc == nullptr) { + continue; + } + auto* cc = sc->client_connection(); + if (cc == nullptr) { + continue; + } + auto ch = cc->server_connection().current_channel(); + if (!ch) { + continue; + } + auto const& stats = ch->channel_statistics().response_time_statistics(); + if (stats.size() > best) { + best = stats.size(); + if (!stats.empty()) { + best_min = stats.min(); + best_p99 = stats.percentile<99>(); + } + } + } + if (min_out != nullptr) { + *min_out = best_min; + } + if (p99_out != nullptr) { + *p99_out = best_p99; + } + return best; + } + + void PollWarmup() { + if (!warmup_active || !client) { + return; + } + if (side == Side::kA) { + EnsureStreams(); + if (!stream) { + return; + } + auto const route = stream->InspectSendRoute(); + static std::size_t last_logged = 0; + if (route.ping_sample_count != last_logged && + (route.ping_sample_count % 2 == 0 || + route.ping_sample_count >= kWarmupSamples)) { + last_logged = route.ping_sample_count; + std::cerr << "Alice dest warmup server=" << route.server_id + << " samples=" << route.ping_sample_count + << " present=" << route.present << std::endl; + } + if (!route.present || route.ping_sample_count < kWarmupSamples) { + return; + } + auto const min_ms = + std::chrono::duration_cast(route.min_rtt) + .count(); + auto const p99_ms = + std::chrono::duration_cast(route.p99_rtt) + .count(); + if ((min_ms == 200 && p99_ms == 200) || min_ms == 5000 || p99_ms == 5000) { + std::cerr << "Alice dest warmup stats look synthetic: min=" << min_ms + << " p99=" << p99_ms << "\n"; + return; + } + warmup_active = false; + std::cout << "## Alice dest-server ping statistics (child)\n" + << "server_id=" << route.server_id + << " samples=" << route.ping_sample_count + << " min_rtt_ms=" << min_ms << " p99_rtt_ms=" << p99_ms + << " protocol=" + << (route.protocol == Protocol::kUdp ? "udp" : "tcp") << "\n"; + Emit(IpcType::kWarmupDone, EventKind::kWarmupDone, 0, 0, + static_cast(route.ping_sample_count), min_ms, p99_ms, + static_cast(route.server_id), + BenchProtocolFromAe(route.protocol)); + return; + } + if (side != Side::kB) { + return; + } + Duration min_rtt{}; + Duration p99_rtt{}; + auto const n = CollectResponseStats(&min_rtt, &p99_rtt); + static std::size_t last_logged = 0; + if (n != last_logged && (n % 2 == 0 || n >= kWarmupSamples)) { + last_logged = n; + std::cerr << "Bob warmup samples=" << n << std::endl; + } + if (n < kWarmupSamples) { + return; + } + auto const interval = + std::chrono::duration_cast(kBobPingInterval); + auto const guard = ClampPingSendGuard( + ComputePingSendGuard(min_rtt, p99_rtt), interval); + auto const min_ms = + std::chrono::duration_cast(min_rtt).count(); + auto const p99_ms = + std::chrono::duration_cast(p99_rtt).count(); + auto const guard_ms = + std::chrono::duration_cast(guard).count(); + // Reject obvious synthetic seed values (200ms estimate / 5000ms). + if ((min_ms == 200 && p99_ms == 200) || min_ms == 5000 || p99_ms == 5000) { + std::cerr << "warmup stats look synthetic: min=" << min_ms + << " p99=" << p99_ms << "\n"; + return; + } + warmup_active = false; + std::cout << "## Bob ping statistics (child)\n" + << "samples=" << n << " min_rtt_ms=" << min_ms + << " p99_rtt_ms=" << p99_ms << " guard_ms=" << guard_ms << "\n"; + // sequence unused; offset_ms carries guard_ms; a=n b=min c=p99 + Emit(IpcType::kWarmupDone, EventKind::kWarmupDone, 0, + static_cast(guard_ms), static_cast(n), + min_ms, p99_ms); + } + + void StartSample(std::uint32_t sequence, std::uint32_t offset_ms) { + if (side != Side::kA || !client || !peer_set) { + return; + } + // Coordinator may time out while we are waiting to requery; always take + // the latest sample request. + sample_in_flight = true; + pending_sequence = sequence; + pending_offset_ms = offset_ms; + send_at_.reset(); + requery_at_.reset(); + EnsureStreams(); + BeginQuery(); + } + + void BeginQuery() { + // Reset subscription before replacing Client-owned action. + query_sub.Reset(); + auto& action = client->QueryPeerReceiveSchedule(peer_uid); + query_sub = action.result_event().Subscribe( + [this, &action](Result const& res) { + last_diagnostics_ = action.server_diagnostics(); + OnSchedule(res); + }); + } + + ServerTimingDiagnostic const* FindDestDiagnostic( + ServerId server_id) const { + for (auto const& d : last_diagnostics_) { + if (d.server_id == server_id && d.has_raw && + d.status == ServerTimingAttemptStatus::kSuccess) { + return &d; + } + } + return nullptr; + } + + void OnSchedule(Result const& res) { + if (!res) { + sample_in_flight = false; + send_at_.reset(); + requery_at_.reset(); + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, res.error()); + return; + } + TryEmitDestProof(); + EnsureStreams(); + if (!stream) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, 3); + return; + } + auto const route = stream->InspectSendRoute(); + if (!route.present) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, 0, -1, 7); + return; + } + if (route.ping_sample_count < kWarmupSamples) { + requery_at_ = Now() + std::chrono::milliseconds{250}; + return; + } + auto const* diag = FindDestDiagnostic(route.server_id); + if (diag == nullptr || + diag->converted.state != PeerScheduleState::kExpected || + !diag->converted.next_ping_deadline.has_value()) { + sample_in_flight = false; + send_at_.reset(); + requery_at_.reset(); + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, 0, -1, 8); + return; + } + + auto const offset = std::chrono::milliseconds{pending_offset_ms}; + auto const cycle_start = + *diag->converted.next_ping_deadline - + std::chrono::duration_cast(kBobPingInterval); + auto const target = cycle_start + offset; + auto const now = Now(); + pending_last_us_ = TimePointUs(cycle_start); + pending_next_us_ = TimePointUs(*diag->converted.next_ping_deadline); + pending_schedule_server_id_ = static_cast(route.server_id); + pending_route_generation_ = + static_cast(route.route_generation); + pending_protocol_ = BenchProtocolFromAe(route.protocol); + pending_raw_delta_ms_ = diag->raw.next_ping_delta_ms; + pending_last_connect_ms_ = diag->raw.last_connect_delta_ms; + pending_qsend_us_ = TimePointUs(diag->qsend); + pending_one_way_us_ = DurationUs(diag->one_way); + pending_target_us_ = TimePointUs(target); + + auto const to_next_ms = + std::chrono::duration_cast( + *diag->converted.next_ping_deadline - now) + .count(); + std::cerr << "Alice dest-server schedule server=" << route.server_id + << " gen=" << route.route_generation + << " age_to_cycle_start_ms=" + << std::chrono::duration_cast( + now - cycle_start) + .count() + << " to_target_ms=" + << std::chrono::duration_cast(target - + now) + .count() + << " to_next_ms=" << to_next_ms + << " offset_ms=" << pending_offset_ms + << " raw_delta_ms=" << pending_raw_delta_ms_ << std::endl; + + if (now + kSkipIfCloserThan > target && now < target) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 1, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_, pending_protocol_, pending_raw_delta_ms_, + pending_last_connect_ms_); + return; + } + if (now >= target) { + auto wait_until = + *diag->converted.next_ping_deadline + std::chrono::milliseconds{150}; + if (wait_until <= now) { + wait_until = now + std::chrono::milliseconds{250}; + } + if (wait_until > now + std::chrono::seconds{15}) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, + pending_sequence, pending_offset_ms, pending_last_us_, + pending_next_us_, 2, pending_schedule_server_id_, + pending_schedule_server_id_, pending_route_generation_); + return; + } + requery_at_ = wait_until; + return; + } + + auto const delay_ms = + std::chrono::duration_cast(target - now) + .count(); + if (delay_ms > 15000) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 3, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_); + return; + } + send_at_ = target; + } + + void PollSampleTiming() { + if (side != Side::kA || !sample_in_flight || !client) { + return; + } + auto const now = Now(); + if (requery_at_ && now >= *requery_at_) { + requery_at_.reset(); + BeginQuery(); + return; + } + if (send_at_ && now >= *send_at_) { + send_at_.reset(); + SendPendingMessage(); + } + } + + void SendPendingMessage() { + if (!stream) { + EnsureStreams(); + } + if (!stream) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kError, pending_sequence, + pending_offset_ms, 3); + return; + } + auto const before = stream->InspectSendRoute(); + if (!before.present || + static_cast(before.server_id) != + pending_schedule_server_id_ || + static_cast(before.route_generation) != + pending_route_generation_) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 6, + pending_schedule_server_id_, + before.present ? static_cast(before.server_id) : 0, + before.present ? static_cast(before.route_generation) + : 0, + before.present ? BenchProtocolFromAe(before.protocol) : 0); + return; + } + auto const dest_proto = static_cast(pending_protocol_); + if (RefuseTcpSample(own_proof.protocol, dest_proto)) { + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSkipped, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, 5, + pending_schedule_server_id_, pending_schedule_server_id_, + pending_route_generation_, pending_protocol_); + return; + } + + DeliveryBenchMessage msg{}; + msg.offset_ms = static_cast(pending_offset_ms); + msg.sequence = pending_sequence; + msg.send_qpc = QpcNow(); + auto bytes = SerializeDeliveryBenchMessage(msg); + DataBuffer payload{bytes.begin(), bytes.end()}; + stream->Write(std::move(payload)); + auto const after = stream->LastSendRoute(); + auto const actual_id = after.present + ? static_cast(after.server_id) + : pending_schedule_server_id_; + auto const actual_gen = + after.present ? static_cast(after.route_generation) + : pending_route_generation_; + auto const actual_proto = + after.present ? BenchProtocolFromAe(after.protocol) : pending_protocol_; + sample_in_flight = false; + Emit(IpcType::kSampleResult, EventKind::kSampleSent, pending_sequence, + pending_offset_ms, pending_last_us_, pending_next_us_, + static_cast(msg.send_qpc), pending_schedule_server_id_, + actual_id, actual_gen, actual_proto, pending_raw_delta_ms_, + pending_last_connect_ms_, pending_qsend_us_, pending_one_way_us_, + pending_target_us_); + std::cerr << "Alice sent seq=" << pending_sequence + << " offset_ms=" << pending_offset_ms + << " schedule_server=" << pending_schedule_server_id_ + << " actual_server=" << actual_id << std::endl; + } + + void HandleIpc(IpcFrame const& f) { + auto const type = static_cast(f.type); + switch (type) { + case IpcType::kSetPeerUid: + peer_uid = UidFromHalves(f.a, f.b); + peer_set = true; + if (client) { + // Bob: schedule already applied; Alice: default schedule. + TryEmitOwnProof(); + TryEmitDestProof(); + } + EnsureStreams(); + Emit(IpcType::kAck, EventKind::kAck); + break; + case IpcType::kWaitWarmup: + warmup_active = true; + std::cerr << (side == Side::kA ? "Alice" : "Bob") + << " WaitWarmup received" << std::endl; + TryEmitOwnProof(); + TryEmitDestProof(); + break; + case IpcType::kRunSample: + StartSample(f.sequence, f.offset_ms); + break; + case IpcType::kShutdown: + exit_requested = true; + Emit(IpcType::kAck, EventKind::kAck); + break; + default: + break; + } + } +}; + +std::unique_ptr MakeApp(std::string const& state_dir) { + return AetherApp::Construct( + AetherAppContext{[state_dir]() { + return std::unique_ptr{ + std::make_unique(state_dir)}; + }} +#if AE_DISTILLATION + .AddAdapterFactory([](AetherAppContext const& context) { + return EthernetAdapter::ptr::Create( + CreateWith{context.domain()}.with_id( + GlobalId::kEthernetAdapter), + context.aether(), context.poller(), context.dns_resolver()); + }) +#endif + ); +} + +} // namespace + +int RunClientRole(ClientArgs const& args) { + RoleState state; + state.side = args.side; + state.run_id_hash = HashRunId(args.run_id); + + if (!state.pipe.Connect(args.pipe_name, 60000)) { + std::cerr << "pipe connect failed: " << args.pipe_name << "\n"; + return 2; + } + + state.app = MakeApp(args.state_dir); + auto parent = Uid::FromString(args.parent_uid); + auto& select = + state.app->aether()->SelectClient(parent, args.client_name); + state.select_sub = select.result_event().Subscribe( + [&](Result const& res) { + if (!res) { + state.Emit(IpcType::kEvent, EventKind::kError, 0, 0, 10); + state.exit_requested = true; + return; + } + state.client = res.value(); + if (state.side == Side::kB) { + auto const ping_ms = kBobPingInterval.count(); + auto const rx_ms = kBobReceiveWindow.count(); + std::cerr << "Bob SetReceiveSchedule applying ping_interval_ms=" + << ping_ms << " receive_window_ms=" << rx_ms << std::endl; + auto ok = state.client->SetReceiveSchedule(ReceiveSchedule{ + .ping_interval = + std::chrono::duration_cast(kBobPingInterval), + .receive_window = + std::chrono::duration_cast(kBobReceiveWindow), + }); + std::cerr << "Bob SetReceiveSchedule done ok=" << static_cast(ok) + << " (expect ping_interval_ms=" << ping_ms + << " receive_window_ms=" << rx_ms << ")" << std::endl; + if (!ok) { + state.Emit(IpcType::kEvent, EventKind::kError, 0, 0, 11); + state.exit_requested = true; + return; + } +#if AE_ENABLE_PING + SetPingTraceHook(&OnPingTrace); +#endif + } + state.client_ready = true; + std::int64_t lo = 0; + std::int64_t hi = 0; + UidToHalves(state.client->uid(), lo, hi); + std::cerr << "Client ready side=" + << (state.side == Side::kA ? "A" : "B") << std::endl; + state.Emit(IpcType::kUidReport, EventKind::kChildReady, 0, 0, lo, hi); + state.Emit(IpcType::kChildReady, EventKind::kChildReady); + }); + + while (!state.exit_requested && !state.app->IsExited()) { + auto const now = Now(); + auto next = state.app->Update(now); + state.DrainPingTraces(); + if (auto frame = state.pipe.TryReadFrame(0)) { + state.HandleIpc(*frame); + } + state.TryEmitOwnProof(); + state.TryEmitDestProof(); + state.PollWarmup(); + state.PollSampleTiming(); + state.app->WaitUntil( + std::min(next, now + std::chrono::milliseconds{5})); + } +#if AE_ENABLE_PING + SetPingTraceHook(nullptr); + if (args.side == Side::kB && !g_all_ping_traces.empty()) { + std::ofstream csv(args.state_dir + "/bob_ping_trace.csv"); + csv << "kind,server_id,planned_send_us,actual_send_us,early_by_us," + "base_rx_window_us,effective_wire_rx_window_us,required_rx_until_us," + "next_planned_send_us,ping_guard_us,min_rtt_us,p99_rtt_us," + "channel_generation,result_type,steady_us\n"; + for (auto const& rec : g_all_ping_traces) { + auto const& e = rec.event; + csv << static_cast(e.kind) << "," + << static_cast(e.server_id) << "," + << TimePointUs(e.planned_send_at) << "," + << TimePointUs(e.actual_send_at) << "," << DurationUs(e.early_by) + << "," << DurationUs(e.base_rx_window) << "," + << DurationUs(e.effective_wire_rx_window) << "," + << TimePointUs(e.required_rx_until) << "," + << TimePointUs(e.next_planned_send) << "," + << DurationUs(e.ping_guard) << "," << DurationUs(e.min_rtt) << "," + << DurationUs(e.p99_rtt) << "," << e.channel_generation << "," + << e.result_type << "," << rec.steady_us << "\n"; + } + } +#endif + return 0; +} + +} // namespace ae::bench::uap diff --git a/examples/benches/aether_uap_delivery_timing_bench/client_role.h b/examples/benches/aether_uap_delivery_timing_bench/client_role.h new file mode 100644 index 00000000..fdc98dfc --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/client_role.h @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_CLIENT_ROLE_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_CLIENT_ROLE_H_ + +#include + +#include "common/bench_types.h" + +namespace ae::bench::uap { + +struct ClientArgs { + Side side{Side::kA}; + std::string run_id; + std::string state_dir; + std::string pipe_name; + std::string client_name; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; +}; + +int RunClientRole(ClientArgs const& args); + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_CLIENT_ROLE_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.cpp b/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.cpp new file mode 100644 index 00000000..304ce785 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.cpp @@ -0,0 +1,301 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bench_ipc.h" + +#include + +#if defined(_WIN32) +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#endif + +namespace ae::bench::uap { + +std::uint32_t IpcFrameCrc(IpcFrame const& frame) noexcept { + IpcFrame tmp = frame; + tmp.crc = 0; + return Crc32(&tmp, sizeof(tmp)); +} + +void EncodeIpcFrame(IpcFrame& frame) noexcept { + frame.magic = kIpcMagic; + frame.version = kIpcVersion; + frame.crc = IpcFrameCrc(frame); +} + +bool DecodeIpcFrame(void const* data, std::size_t size, + IpcFrame& out) noexcept { + if (data == nullptr || size < sizeof(IpcFrame)) { + return false; + } + std::memcpy(&out, data, sizeof(IpcFrame)); + if (out.magic != kIpcMagic || out.version != kIpcVersion) { + return false; + } + return IpcFrameCrc(out) == out.crc; +} + +std::string PipeNameFor(std::string const& run_id, Side side) { + return PipeNameFor(run_id, side, ""); +} + +std::string PipeNameFor(std::string const& run_id, Side side, + std::string const& suffix) { + auto const s = side == Side::kA ? "a" : (side == Side::kB ? "b" : "c"); + auto name = "\\\\.\\pipe\\aether-uap-bench-" + run_id + "-" + s; + if (!suffix.empty()) { + name += "-"; + name += suffix; + } + return name; +} + +std::uint32_t HashRunId(std::string const& run_id) { + std::uint32_t h = 2166136261u; + for (char c : run_id) { + h ^= static_cast(c); + h *= 16777619u; + } + return h; +} + +#if defined(_WIN32) + +namespace { + +bool OverlappedWait(HANDLE handle, OVERLAPPED& ov, DWORD timeout_ms, + DWORD* transferred) { + DWORD bytes = 0; + if (GetOverlappedResult(handle, &ov, &bytes, FALSE)) { + if (transferred != nullptr) { + *transferred = bytes; + } + return true; + } + if (GetLastError() != ERROR_IO_INCOMPLETE) { + return false; + } + auto const wait = WaitForSingleObject(ov.hEvent, timeout_ms); + if (wait != WAIT_OBJECT_0) { + CancelIoEx(handle, &ov); + return false; + } + if (!GetOverlappedResult(handle, &ov, &bytes, FALSE)) { + return false; + } + if (transferred != nullptr) { + *transferred = bytes; + } + return true; +} + +} // namespace + +NamedPipeServer::~NamedPipeServer() { Close(); } + +bool NamedPipeServer::Create(std::string const& pipe_name) { + Close(); + handle_ = CreateNamedPipeA( + pipe_name.c_str(), PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, + 1, 4096, 4096, 0, nullptr); + return handle_ != INVALID_HANDLE_VALUE && handle_ != nullptr; +} + +bool NamedPipeServer::WaitForClient(std::uint32_t timeout_ms) { + if (handle_ == nullptr || handle_ == INVALID_HANDLE_VALUE) { + return false; + } + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return false; + } + auto connected = ConnectNamedPipe(static_cast(handle_), &ov); + if (connected) { + CloseHandle(ov.hEvent); + return true; + } + auto const err = GetLastError(); + if (err == ERROR_PIPE_CONNECTED) { + CloseHandle(ov.hEvent); + return true; + } + if (err != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return false; + } + auto const ok = OverlappedWait(static_cast(handle_), ov, timeout_ms, + nullptr); + CloseHandle(ov.hEvent); + return ok; +} + +bool NamedPipeServer::WriteFrame(IpcFrame frame) { + EncodeIpcFrame(frame); + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return false; + } + DWORD written = 0; + auto ok = WriteFile(static_cast(handle_), &frame, sizeof(frame), + &written, &ov); + if (!ok) { + if (GetLastError() != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return false; + } + ok = OverlappedWait(static_cast(handle_), ov, 5000, &written); + } + CloseHandle(ov.hEvent); + return ok && written == sizeof(frame); +} + +std::optional NamedPipeServer::TryReadFrame( + std::uint32_t timeout_ms) { + IpcFrame frame{}; + OVERLAPPED ov{}; + ov.hEvent = CreateEventA(nullptr, TRUE, FALSE, nullptr); + if (ov.hEvent == nullptr) { + return std::nullopt; + } + DWORD read = 0; + auto ok = ReadFile(static_cast(handle_), &frame, sizeof(frame), &read, + &ov); + if (!ok) { + if (GetLastError() != ERROR_IO_PENDING) { + CloseHandle(ov.hEvent); + return std::nullopt; + } + ok = OverlappedWait(static_cast(handle_), ov, timeout_ms, &read); + } + CloseHandle(ov.hEvent); + if (!ok || read < sizeof(IpcFrame)) { + return std::nullopt; + } + IpcFrame out{}; + if (!DecodeIpcFrame(&frame, sizeof(frame), out)) { + return std::nullopt; + } + return out; +} + +void NamedPipeServer::Close() { + if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(static_cast(handle_)); + } + handle_ = nullptr; +} + +NamedPipeClient::~NamedPipeClient() { Close(); } + +bool NamedPipeClient::Connect(std::string const& pipe_name, + std::uint32_t timeout_ms) { + Close(); + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() < deadline) { + handle_ = CreateFileA(pipe_name.c_str(), GENERIC_READ | GENERIC_WRITE, 0, + nullptr, OPEN_EXISTING, 0, nullptr); + if (handle_ != INVALID_HANDLE_VALUE) { + DWORD mode = PIPE_READMODE_MESSAGE; + SetNamedPipeHandleState(static_cast(handle_), &mode, nullptr, + nullptr); + return true; + } + if (GetLastError() != ERROR_PIPE_BUSY) { + Sleep(50); + continue; + } + WaitNamedPipeA(pipe_name.c_str(), 200); + } + handle_ = nullptr; + return false; +} + +bool NamedPipeClient::WriteFrame(IpcFrame frame) { + EncodeIpcFrame(frame); + DWORD written = 0; + return WriteFile(static_cast(handle_), &frame, sizeof(frame), + &written, nullptr) && + written == sizeof(frame); +} + +std::optional NamedPipeClient::TryReadFrame( + std::uint32_t timeout_ms) { + auto const deadline = GetTickCount64() + timeout_ms; + while (GetTickCount64() <= deadline) { + DWORD avail = 0; + if (!PeekNamedPipe(static_cast(handle_), nullptr, 0, nullptr, + &avail, nullptr)) { + return std::nullopt; + } + if (avail < sizeof(IpcFrame)) { + Sleep(5); + continue; + } + IpcFrame frame{}; + DWORD read = 0; + if (!ReadFile(static_cast(handle_), &frame, sizeof(frame), &read, + nullptr) || + read < sizeof(IpcFrame)) { + return std::nullopt; + } + IpcFrame out{}; + if (!DecodeIpcFrame(&frame, sizeof(frame), out)) { + return std::nullopt; + } + return out; + } + return std::nullopt; +} + +void NamedPipeClient::Close() { + if (handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(static_cast(handle_)); + } + handle_ = nullptr; +} + +bool WaitMsPrecise(std::int64_t delay_ms) { + if (delay_ms <= 0) { + return true; + } + HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, + CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, + TIMER_ALL_ACCESS); + if (timer == nullptr) { + Sleep(static_cast(delay_ms)); + return true; + } + LARGE_INTEGER due; + due.QuadPart = -delay_ms * 10000LL; + if (!SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) { + CloseHandle(timer); + Sleep(static_cast(delay_ms)); + return true; + } + WaitForSingleObject(timer, INFINITE); + CloseHandle(timer); + return true; +} + +#endif + +} // namespace ae::bench::uap diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.h b/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.h new file mode 100644 index 00000000..0bb866e0 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/bench_ipc.h @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_IPC_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_IPC_H_ + +#include +#include +#include + +#include "bench_message.h" +#include "bench_types.h" + +namespace ae::bench::uap { + +#pragma pack(push, 1) +struct IpcFrame { + std::uint32_t magic{kIpcMagic}; + std::uint8_t version{kIpcVersion}; + std::uint8_t type{0}; + std::uint8_t side{0}; + std::uint8_t event_kind{0}; + std::uint32_t run_id_hash{0}; + std::uint32_t seq{0}; + std::uint32_t sequence{0}; + std::uint32_t offset_ms{0}; + std::int64_t local_steady_us{0}; + std::int64_t a{0}; + std::int64_t b{0}; + std::int64_t c{0}; + std::int64_t d{0}; + std::int64_t e{0}; + std::int64_t f{0}; + std::int64_t g{0}; + std::int64_t h{0}; + std::int64_t i{0}; + std::int64_t j{0}; + std::int64_t k{0}; + std::int64_t l{0}; + std::uint32_t crc{0}; +}; +#pragma pack(pop) + +static_assert(sizeof(IpcFrame) < 160); + +std::uint32_t IpcFrameCrc(IpcFrame const& frame) noexcept; +void EncodeIpcFrame(IpcFrame& frame) noexcept; +bool DecodeIpcFrame(void const* data, std::size_t size, IpcFrame& out) noexcept; + +std::string PipeNameFor(std::string const& run_id, Side side); +std::string PipeNameFor(std::string const& run_id, Side side, + std::string const& suffix); +std::uint32_t HashRunId(std::string const& run_id); + +#if defined(_WIN32) +class NamedPipeServer { + public: + NamedPipeServer() = default; + ~NamedPipeServer(); + NamedPipeServer(NamedPipeServer const&) = delete; + NamedPipeServer& operator=(NamedPipeServer const&) = delete; + + bool Create(std::string const& pipe_name); + bool WaitForClient(std::uint32_t timeout_ms); + bool WriteFrame(IpcFrame frame); + std::optional TryReadFrame(std::uint32_t timeout_ms); + void Close(); + + private: + void* handle_{nullptr}; +}; + +class NamedPipeClient { + public: + NamedPipeClient() = default; + ~NamedPipeClient(); + NamedPipeClient(NamedPipeClient const&) = delete; + NamedPipeClient& operator=(NamedPipeClient const&) = delete; + + bool Connect(std::string const& pipe_name, std::uint32_t timeout_ms); + bool WriteFrame(IpcFrame frame); + std::optional TryReadFrame(std::uint32_t timeout_ms); + void Close(); + + private: + void* handle_{nullptr}; +}; + +bool WaitMsPrecise(std::int64_t delay_ms); +#endif + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_IPC_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/bench_message.h b/examples/benches/aether_uap_delivery_timing_bench/common/bench_message.h new file mode 100644 index 00000000..b771a295 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/bench_message.h @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXAMPLES_BENCHES_AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_MESSAGE_H_ +#define EXAMPLES_BENCHES_AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_MESSAGE_H_ + +#include +#include +#include +#include +#include + +namespace ae::bench::uap { + +inline constexpr std::uint32_t kBenchMagic = 0x41555031u; // 'AUP1' +inline constexpr std::uint8_t kBenchVersion = 1; + +#pragma pack(push, 1) +struct DeliveryBenchMessage { + std::uint32_t magic{kBenchMagic}; + std::uint8_t version{kBenchVersion}; + std::uint8_t reserved{0}; + std::uint16_t offset_ms{0}; + std::uint32_t sequence{0}; + std::uint64_t send_qpc{0}; + std::uint32_t crc{0}; +}; +#pragma pack(pop) + +static_assert(sizeof(DeliveryBenchMessage) < 128); + +inline std::uint32_t Crc32(void const* data, std::size_t size) noexcept { + auto const* bytes = static_cast(data); + std::uint32_t crc = 0xFFFFFFFFu; + for (std::size_t i = 0; i < size; ++i) { + crc ^= bytes[i]; + for (int b = 0; b < 8; ++b) { + auto const mask = + static_cast(-(static_cast(crc & 1u))); + crc = (crc >> 1) ^ (0xEDB88320u & mask); + } + } + return ~crc; +} + +inline std::uint32_t DeliveryBenchMessageCrc( + DeliveryBenchMessage const& msg) noexcept { + auto copy = msg; + copy.crc = 0; + return Crc32(©, sizeof(copy)); +} + +inline void EncodeDeliveryBenchMessage(DeliveryBenchMessage& msg) noexcept { + msg.magic = kBenchMagic; + msg.version = kBenchVersion; + msg.crc = DeliveryBenchMessageCrc(msg); +} + +inline bool DecodeDeliveryBenchMessage(std::uint8_t const* data, + std::size_t size, + DeliveryBenchMessage& out) noexcept { + if (data == nullptr || size < sizeof(DeliveryBenchMessage)) { + return false; + } + std::memcpy(&out, data, sizeof(DeliveryBenchMessage)); + if (out.magic != kBenchMagic || out.version != kBenchVersion) { + return false; + } + return out.crc == DeliveryBenchMessageCrc(out); +} + +inline std::vector SerializeDeliveryBenchMessage( + DeliveryBenchMessage msg) { + EncodeDeliveryBenchMessage(msg); + auto const* bytes = reinterpret_cast(&msg); + return {bytes, bytes + sizeof(msg)}; +} + +inline std::optional DeserializeDeliveryBenchMessage( + std::uint8_t const* data, std::size_t size) { + DeliveryBenchMessage out{}; + if (!DecodeDeliveryBenchMessage(data, size, out)) { + return std::nullopt; + } + return out; +} + +} // namespace ae::bench::uap + +#endif // EXAMPLES_BENCHES_AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_MESSAGE_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/bench_types.h b/examples/benches/aether_uap_delivery_timing_bench/common/bench_types.h new file mode 100644 index 00000000..313bceff --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/bench_types.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_TYPES_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_TYPES_H_ + +#include +#include + +namespace ae::bench::uap { + +inline constexpr std::uint32_t kIpcMagic = 0x41555049u; // 'AUPI' +inline constexpr std::uint8_t kIpcVersion = 2; + +enum class Side : std::uint8_t { kCoordinator = 0, kA = 1, kB = 2 }; + +enum class EventKind : std::uint8_t { + kAck = 0, + kError = 1, + kChildReady = 2, + kWarmupDone = 3, + kSampleSent = 4, + kSampleReceived = 5, + kSampleSkipped = 6, +}; + +enum class BenchProtocol : std::uint8_t { + kUnknown = 0, + kUdp = 1, + kTcp = 2, +}; + +enum class UdpProofPath : std::uint8_t { + kOwn = 1, + kDestination = 2, +}; + +enum class IpcType : std::uint8_t { + kChildReady = 1, + kUidReport = 2, + kSetPeerUid = 3, + kWaitWarmup = 4, + kWarmupDone = 5, + kRunSample = 6, + kSampleResult = 7, + kEvent = 8, + kShutdown = 9, + kAck = 10, + kUdpProof = 11, + kPingTrace = 12, +}; + +struct SampleRecord { + std::uint32_t sequence{0}; + std::int64_t offset_ms{0}; + std::int64_t window_start_us{0}; + std::int64_t converted_deadline_us{-1}; + std::uint64_t send_qpc{0}; + std::uint64_t receive_qpc{0}; + double delivery_ms{0}; + int duplicate_count{0}; + std::int64_t schedule_server_id{0}; + std::int64_t actual_send_server_id{0}; + std::int64_t route_generation{0}; + std::int64_t protocol{0}; + std::int64_t raw_next_ping_delta_ms{0}; + std::int64_t last_connect_delta_ms{0}; + std::int64_t query_send_us{0}; + std::int64_t one_way_estimate_us{0}; + std::int64_t target_send_us{0}; + std::int64_t actual_send_us{0}; + std::int64_t receive_us{0}; + bool valid{false}; + std::string invalid_reason; + std::string classification; + std::int64_t bob_ping_server_id{0}; + std::int64_t bob_ping_planned_send_us{0}; + std::int64_t bob_ping_actual_send_us{0}; + std::int64_t bob_ping_early_by_us{0}; + std::int64_t bob_base_rx_window_us{0}; + std::int64_t bob_effective_wire_rx_window_us{0}; + std::int64_t bob_required_rx_until_us{0}; + std::int64_t bob_ping_result_us{0}; + std::int64_t bob_ping_guard_us{0}; +}; + +inline char const* ClassificationForOffset(std::int64_t offset_ms) noexcept { + return offset_ms < 1000 ? "CURRENT_WINDOW" : "NEXT_WINDOW"; +} + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_BENCH_TYPES_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/directory_domain_storage.h b/examples/benches/aether_uap_delivery_timing_bench/common/directory_domain_storage.h new file mode 100644 index 00000000..e29af177 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/directory_domain_storage.h @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ + +#include +#include +#include +#include +#include +#include + +#include "aether-miscpp/types/result.h" +#include "aether/obj/idomain_storage.h" + +namespace ae::bench::uap { + +// File-backed storage rooted at an explicit directory (not CWD). +// Layout matches FileSystemStdStorage: /// +class DirectoryDomainStorage final : public IDomainStorage { + public: + explicit DirectoryDomainStorage(std::filesystem::path root) + : root_{std::move(root)} { + std::error_code ec; + std::filesystem::create_directories(root_, ec); + } + + std::unique_ptr Store( + DomainQuery const& query) override { + auto class_dir = + root_ / std::to_string(query.id.id()) / std::to_string(query.class_id); + std::filesystem::create_directories(class_dir); + auto path = class_dir / std::to_string(query.version); + std::ofstream f(path, std::ios::out | std::ios::binary | std::ios::trunc); + class Writer final : public IDomainStorageWriter { + public: + explicit Writer(std::ofstream&& file) : file_{std::move(file)} {} + ~Writer() override { file_.close(); } + seri::SeriResult Write(seri::SizeWriteTag data) override { + auto const u_size = static_cast(data.size); + return Write(seri::DataTag{u_size}); + } + seri::SeriResult Write(seri::DataWriteTag data) override { + file_.write(reinterpret_cast(data.data), + static_cast(data.size)); + if (file_.fail()) { + return Error{seri::write_error}; + } + return Ok{seri::good}; + } + + private: + std::ofstream file_; + }; + return std::make_unique(std::move(f)); + } + + ClassList Enumerate(ObjId const& obj_id) override { + std::set classes; + std::error_code ec; + auto obj_dir = root_ / std::to_string(obj_id.id()); + for (auto const& class_dir : + std::filesystem::directory_iterator(obj_dir, ec)) { + classes.insert(static_cast( + std::stoul(class_dir.path().filename().string()))); + } + return ClassList{classes.begin(), classes.end()}; + } + + DomainLoad Load(DomainQuery const& query) override { + auto object_dir = root_ / std::to_string(query.id.id()); + std::error_code ec; + if (!std::filesystem::exists(object_dir, ec)) { + return {DomainLoadResult::kEmpty, {}}; + } + auto path = object_dir / std::to_string(query.class_id) / + std::to_string(query.version); + std::ifstream f(path, std::ios::in | std::ios::binary); + if (!f.good()) { + return {DomainLoadResult::kEmpty, {}}; + } + class Reader final : public IDomainStorageReader { + public: + explicit Reader(std::ifstream&& file) : file_{std::move(file)} {} + ~Reader() override { file_.close(); } + seri::SeriResult Read(seri::SizeReadTag data) override { + std::uint32_t u_size{}; + TRY_RESULT(Read(seri::DataTag{u_size})); + data.size = static_cast(u_size); + return Ok{seri::good}; + } + seri::SeriResult Read(seri::DataReadTag data) override { + if (file_.eof()) { + return Error{seri::read_eof}; + } + file_.read(reinterpret_cast(data.data), + static_cast(data.size)); + if (file_.bad()) { + return Error{seri::read_error}; + } + if (file_.gcount() != static_cast(data.size)) { + return Error{file_.eof() ? seri::read_eof : seri::read_error}; + } + return Ok{seri::good}; + } + + private: + std::ifstream file_; + }; + return {DomainLoadResult::kLoaded, + std::make_unique(std::move(f))}; + } + + void Remove(ObjId const& obj_id) override { + auto object_dir = root_ / std::to_string(obj_id.id()); + std::error_code ec; + if (!std::filesystem::exists(object_dir, ec)) { + std::filesystem::create_directory(object_dir, ec); + return; + } + for (auto const& class_dir : + std::filesystem::directory_iterator(object_dir, ec)) { + std::error_code ec2; + std::filesystem::remove_all(class_dir.path(), ec2); + } + } + + void CleanUp() override { + // Bench intentionally preserves state across child restarts so the same + // UIDs remain in state-a / state-b. Distillation still calls CleanUp on + // Construct; do not wipe the root here. + } + + private: + std::filesystem::path root_; +}; + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_DIRECTORY_DOMAIN_STORAGE_H_ + diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof.h b/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof.h new file mode 100644 index 00000000..287395d3 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof.h @@ -0,0 +1,186 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_H_ + +#include +#include +#include +#include + +#include "aether/channels/ethernet_channel.h" +#include "aether/client.h" +#include "aether/cloud.h" +#include "aether/cloud_connections/cloud_server_connections.h" +#include "aether/config.h" +#include "aether/server_connections/client_server_connection.h" +#include "aether/types/address.h" +#include "aether/types/server_id.h" +#include "aether/types/uid.h" + +#if AE_SUPPORT_UDP && defined(WIN_SOCKET_ENABLED) +# include "aether/transport/system_sockets/sockets/win_udp_socket.h" +#endif + +#include "udp_proof_types.h" + +namespace ae::bench::uap { + +inline std::uint64_t CurrentUdpSocketGeneration() noexcept { + return 0; +} + +inline std::string FormatEndpointAddress(Address const& address) { + std::string out; + std::visit( + [&](auto const& value) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + out = std::to_string(value.ipv4_value[0]) + "." + + std::to_string(value.ipv4_value[1]) + "." + + std::to_string(value.ipv4_value[2]) + "." + + std::to_string(value.ipv4_value[3]); + } else if constexpr (std::is_same_v) { + out = "ipv6"; + } else if constexpr (std::is_same_v) { +#if AE_SUPPORT_CLOUD_DNS + out = value.name; +#else + out = "named"; +#endif + } else { + out = "null"; + } + }, + address); + return out; +} + +inline std::uint32_t PackIpv4(Address const& address) noexcept { + std::uint32_t packed = 0; + std::visit( + [&](auto const& value) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + packed = (static_cast(value.ipv4_value[0]) << 24) | + (static_cast(value.ipv4_value[1]) << 16) | + (static_cast(value.ipv4_value[2]) << 8) | + static_cast(value.ipv4_value[3]); + } + }, + address); + return packed; +} + +inline ChannelProof MakeChannelProof(ServerId server_id, + ClientServerConnection& connection) { + ChannelProof proof{}; + proof.present = true; + proof.server_id = server_id; + proof.udp_socket_generation = CurrentUdpSocketGeneration(); + + auto info = connection.stream_info(); + proof.link_state = info.link_state; + proof.is_writable = info.is_writable; + + auto channel = connection.server_connection().current_channel(); + if (!channel) { + proof.protocol = BenchProtocol::kUnknown; + return proof; + } + + auto const& props = channel->transport_properties(); + proof.connection_type = props.connection_type; + proof.reliability = props.reliability; + + auto* ethernet = channel.as(); + if (ethernet == nullptr) { + proof.protocol = BenchProtocol::kUnknown; + return proof; + } + + proof.protocol = ClassifyWireProtocol(ethernet->address.protocol); + proof.port = ethernet->address.port; + proof.endpoint = FormatEndpointAddress(ethernet->address.address) + ":" + + std::to_string(ethernet->address.port); + proof.ipv4_packed = PackIpv4(ethernet->address.address); + return proof; +} + +inline std::vector CollectCloudConnectionProofs( + CloudServerConnections& cloud_connection) { + std::vector out; + for (auto* sc : cloud_connection.servers()) { + if (sc == nullptr) { + continue; + } + auto* cc = sc->client_connection(); + if (cc == nullptr) { + continue; + } + out.push_back(MakeChannelProof(sc->server_id(), *cc)); + } + return out; +} + +inline ChannelProof FirstPresentProof( + std::vector const& proofs) { + for (auto const& p : proofs) { + if (p.present) { + return p; + } + } + return {}; +} + +inline ChannelProof CollectOwnCloudProof(Client& client) { + return FirstPresentProof( + CollectCloudConnectionProofs(client.cloud_connection())); +} + +inline ChannelProof CollectDestinationProofFromCloud(Client& client, + Cloud::ptr const& cloud) { + ChannelProof best{}; + if (!cloud) { + return best; + } + auto const& loaded = cloud.Load(); + if (!loaded) { + return best; + } + auto& scm = client.server_connection_manager(); + for (auto const& [sid, entry] : loaded->servers()) { + (void)entry; + auto conn = scm.FindInCache(sid); + if (!conn) { + continue; + } + auto proof = MakeChannelProof(sid, *conn); + if (!proof.present) { + continue; + } + if (!best.present || (proof.link_state == LinkState::kLinked && + best.link_state != LinkState::kLinked)) { + best = std::move(proof); + } + } + return best; +} + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof_types.h b/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof_types.h new file mode 100644 index 00000000..d413b795 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/common/udp_proof_types.h @@ -0,0 +1,141 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_TYPES_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_TYPES_H_ + +#include +#include + +#include "aether/channels/channels_types.h" +#include "aether/config.h" +#include "aether/stream_api/istream.h" +#include "aether/types/address.h" +#include "aether/types/server_id.h" + +#include "bench_ipc.h" +#include "bench_types.h" + +namespace ae::bench::uap { + +struct ChannelProof { + bool present{false}; + ServerId server_id{0}; + BenchProtocol protocol{BenchProtocol::kUnknown}; + std::uint16_t port{0}; + std::uint32_t ipv4_packed{0}; + std::string endpoint; + ConnectionType connection_type{}; + Reliability reliability{}; + LinkState link_state{}; + bool is_writable{false}; + std::uint64_t udp_socket_generation{0}; +}; + +inline char const* BenchProtocolName(BenchProtocol p) noexcept { + switch (p) { + case BenchProtocol::kUdp: + return "udp"; + case BenchProtocol::kTcp: + return "tcp"; + case BenchProtocol::kUnknown: + default: + return "unknown"; + } +} + +inline BenchProtocol ClassifyWireProtocol(Protocol protocol) noexcept { + if (protocol == Protocol::kUdp) { + return BenchProtocol::kUdp; + } + if (protocol == Protocol::kTcp) { + return BenchProtocol::kTcp; + } + return BenchProtocol::kUnknown; +} + +inline bool RefuseTcpSample(BenchProtocol own, + BenchProtocol destination) noexcept { +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP + return own == BenchProtocol::kTcp || destination == BenchProtocol::kTcp; +#else + static_cast(own); + static_cast(destination); + return false; +#endif +} + +inline bool IsMeasuredProtocolOk(BenchProtocol protocol) noexcept { +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP + return protocol == BenchProtocol::kUdp; +#else + return protocol == BenchProtocol::kTcp; +#endif +} + +inline std::string UnpackIpv4Endpoint(std::uint32_t ipv4, + std::uint16_t port) { + if (ipv4 == 0) { + return std::string("?:") + std::to_string(port); + } + return std::to_string((ipv4 >> 24) & 0xff) + "." + + std::to_string((ipv4 >> 16) & 0xff) + "." + + std::to_string((ipv4 >> 8) & 0xff) + "." + + std::to_string(ipv4 & 0xff) + ":" + std::to_string(port); +} + +// IpcFrame packing for IpcType::kUdpProof (frame stays < 128 bytes): +// event_kind = UdpProofPath +// sequence = server_id +// offset_ms = port +// a = protocol | type<<8 | reliability<<16 | link<<24 | writable<<32 +// b = ipv4 packed +// c = udp_socket_generation +inline void PackUdpProofFrame(IpcFrame& frame, UdpProofPath path, + ChannelProof const& proof) noexcept { + frame.type = static_cast(IpcType::kUdpProof); + frame.event_kind = static_cast(path); + frame.sequence = proof.server_id; + frame.offset_ms = proof.port; + frame.a = static_cast(proof.protocol) | + (static_cast(proof.connection_type) << 8) | + (static_cast(proof.reliability) << 16) | + (static_cast(proof.link_state) << 24) | + (static_cast(proof.is_writable ? 1 : 0) << 32); + frame.b = static_cast(proof.ipv4_packed); + frame.c = static_cast(proof.udp_socket_generation); +} + +inline ChannelProof UnpackUdpProofFrame(IpcFrame const& frame) noexcept { + ChannelProof proof{}; + proof.present = true; + proof.server_id = static_cast(frame.sequence); + proof.port = static_cast(frame.offset_ms); + proof.protocol = static_cast(frame.a & 0xff); + proof.connection_type = + static_cast((frame.a >> 8) & 0xff); + proof.reliability = static_cast((frame.a >> 16) & 0xff); + proof.link_state = static_cast((frame.a >> 24) & 0xff); + proof.is_writable = ((frame.a >> 32) & 0x1) != 0; + proof.ipv4_packed = static_cast(frame.b); + proof.udp_socket_generation = static_cast(frame.c); + proof.endpoint = UnpackIpv4Endpoint(proof.ipv4_packed, proof.port); + return proof; +} + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COMMON_UDP_PROOF_TYPES_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/coordinator.cpp b/examples/benches/aether_uap_delivery_timing_bench/coordinator.cpp new file mode 100644 index 00000000..613e7575 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/coordinator.cpp @@ -0,0 +1,773 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "coordinator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include + +#include "common/bench_ipc.h" +#include "common/bench_types.h" +#include "common/udp_proof_types.h" + +#include "aether/config.h" + +namespace ae::bench::uap { +namespace { + +constexpr int kOffsetsMs[] = {500, 800, 1500, 2500}; +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP +constexpr int kSamplesPerOffset = 30; +constexpr int kMinValidPerOffset = 30; +#else +constexpr int kSamplesPerOffset = 20; +constexpr int kMinValidPerOffset = 20; +#endif + +char const* SkipReasonString(std::int64_t code) { + switch (code) { + case 1: + return "skipped_too_close"; + case 2: + return "skipped_stale"; + case 3: + return "skipped_delay_too_long"; + case 5: + return "skipped_tcp_refuse"; + case 6: + return "INVALID_ROUTE_CHANGED"; + case 7: + return "no_dest_route"; + case 8: + return "no_dest_server_timing"; + default: + return "skipped_cycle"; + } +} + +struct BobPingEvent { + std::uint8_t kind{0}; + std::int64_t server_id{0}; + std::int64_t planned_us{0}; + std::int64_t actual_us{0}; + std::int64_t early_by_us{0}; + std::int64_t base_window_us{0}; + std::int64_t effective_window_us{0}; + std::int64_t required_until_us{0}; + std::int64_t next_planned_us{0}; + std::int64_t guard_us{0}; + std::int64_t min_rtt_us{0}; + std::int64_t p99_rtt_us{0}; + std::int64_t channel_generation{0}; + std::int64_t result_type{0}; + std::int64_t event_steady_us{0}; +}; + +void AttachAndClassify(SampleRecord& rec, + std::vector const& pings) { + if (rec.duplicate_count > 1) { + rec.classification = "DUPLICATE"; + return; + } + if (rec.invalid_reason == "INVALID_ROUTE_CHANGED" || + rec.invalid_reason == "ROUTE_MISMATCH") { + rec.classification = "ROUTE_CHANGED"; + return; + } + + BobPingEvent const* covering = nullptr; + if (rec.receive_us > 0) { + for (auto const& p : pings) { + if (p.kind != 1) { + continue; + } + if (p.server_id != rec.actual_send_server_id && + p.server_id != rec.schedule_server_id) { + continue; + } + if (p.event_steady_us > rec.receive_us) { + continue; + } + auto const end = p.event_steady_us + p.effective_window_us; + if (rec.receive_us <= end) { + covering = &p; + } + } + } + + if (covering != nullptr) { + rec.bob_ping_server_id = covering->server_id; + rec.bob_ping_planned_send_us = covering->planned_us; + rec.bob_ping_actual_send_us = covering->actual_us; + rec.bob_ping_early_by_us = covering->early_by_us; + rec.bob_base_rx_window_us = covering->base_window_us; + rec.bob_effective_wire_rx_window_us = covering->effective_window_us; + rec.bob_required_rx_until_us = covering->required_until_us; + rec.bob_ping_guard_us = covering->guard_us; + for (auto const& p : pings) { + if (p.kind == 2 && p.server_id == covering->server_id && + p.actual_us == covering->actual_us) { + rec.bob_ping_result_us = p.event_steady_us; + } + } + bool const early = covering->early_by_us > 0 && + covering->effective_window_us > covering->base_window_us; + if (early) { + rec.classification = "EARLY_PING_EXTENDED_WINDOW"; + } else if (rec.offset_ms >= 1000) { + rec.classification = "WAITED_FOR_NEXT_WINDOW"; + } else { + rec.classification = "CURRENT_NORMAL_WINDOW"; + } + return; + } + + if (rec.receive_us > 0) { + rec.classification = "LATE_UNEXPLAINED"; + return; + } + rec.classification = "LOST"; +} + +struct ChildProc { + Side side{}; + NamedPipeServer pipe; + PROCESS_INFORMATION pi{}; + std::uint64_t uid_lo{0}; + std::uint64_t uid_hi{0}; + bool ready{false}; + bool uid_ok{false}; + std::uint32_t seq{0}; + ChannelProof own_proof{}; + ChannelProof dest_proof{}; + bool got_own_proof{false}; + bool got_dest_proof{false}; + std::vector ping_events; +}; + +std::string MakeRunId() { + SYSTEMTIME st{}; + GetSystemTime(&st); + char buf[64]; + std::snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u", st.wYear, + st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond); + return buf; +} + +std::string DefaultExePath() { + char path[MAX_PATH]{}; + GetModuleFileNameA(nullptr, path, MAX_PATH); + return path; +} + +bool SendCmd(ChildProc& child, IpcType type, std::uint32_t sequence = 0, + std::uint32_t offset_ms = 0, std::int64_t a = 0, + std::int64_t b = 0, std::int64_t c = 0) { + IpcFrame f{}; + f.type = static_cast(type); + f.side = static_cast(Side::kCoordinator); + f.seq = ++child.seq; + f.sequence = sequence; + f.offset_ms = offset_ms; + f.a = a; + f.b = b; + f.c = c; + return child.pipe.WriteFrame(f); +} + +void HandleChildFrame(ChildProc& child, IpcFrame const& frame) { + auto const type = static_cast(frame.type); + if (type == IpcType::kUidReport) { + std::memcpy(&child.uid_lo, &frame.a, 8); + std::memcpy(&child.uid_hi, &frame.b, 8); + child.uid_ok = true; + } + if (type == IpcType::kChildReady || type == IpcType::kUidReport) { + child.ready = true; + } + if (type == IpcType::kUdpProof) { + auto proof = UnpackUdpProofFrame(frame); + auto const path = static_cast(frame.event_kind); + if (path == UdpProofPath::kOwn) { + child.own_proof = proof; + child.got_own_proof = true; + } else if (path == UdpProofPath::kDestination) { + child.dest_proof = proof; + child.got_dest_proof = true; + } + } + if (type == IpcType::kPingTrace) { + BobPingEvent e{}; + e.kind = frame.event_kind; + e.server_id = frame.a; + e.planned_us = frame.b; + e.actual_us = frame.c; + e.early_by_us = frame.d; + e.base_window_us = frame.e; + e.effective_window_us = frame.f; + e.required_until_us = frame.g; + e.next_planned_us = frame.h; + e.guard_us = frame.i; + e.channel_generation = frame.j; + e.min_rtt_us = frame.k; + e.p99_rtt_us = frame.l; + e.result_type = static_cast(frame.offset_ms); + e.event_steady_us = frame.local_steady_us; + child.ping_events.push_back(e); + } +} + +bool SpawnChild(ChildProc& child, CoordinatorArgs const& args, + std::string const& state_dir, std::string const& pipe_name, + std::string const& client_name, + std::string const& child_log_path) { + if (!child.pipe.Create(pipe_name)) { + std::cerr << "CreateNamedPipe failed for " << pipe_name << "\n"; + return false; + } + auto cmd = "\"" + args.exe_path + "\" --role client --side " + + std::string(child.side == Side::kA ? "A" : "B") + " --run-id " + + args.run_id + " --state-dir \"" + state_dir + "\" --pipe \"" + + pipe_name + "\" --client-name " + client_name + " --parent-uid " + + args.parent_uid; + SECURITY_ATTRIBUTES sa{}; + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE log = CreateFileA(child_log_path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, + &sa, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (log == INVALID_HANDLE_VALUE) { + std::cerr << "CreateFile child log failed: " << child_log_path << "\n"; + return false; + } + STARTUPINFOA si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = log; + si.hStdError = log; + std::vector cmdline(cmd.begin(), cmd.end()); + cmdline.push_back('\0'); + if (!CreateProcessA(nullptr, cmdline.data(), nullptr, nullptr, TRUE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &child.pi)) { + CloseHandle(log); + std::cerr << "CreateProcess failed: " << GetLastError() << "\n"; + return false; + } + CloseHandle(log); + if (!child.pipe.WaitForClient(120000)) { + std::cerr << "WaitForClient timeout side=" + << (child.side == Side::kA ? "A" : "B") << "\n"; + return false; + } + return true; +} + +void StopChild(ChildProc& child) { + SendCmd(child, IpcType::kShutdown); + if (WaitForSingleObject(child.pi.hProcess, 15000) != WAIT_OBJECT_0) { + TerminateProcess(child.pi.hProcess, 1); + } + CloseHandle(child.pi.hThread); + CloseHandle(child.pi.hProcess); + child.pipe.Close(); +} + +double QpcToMs(std::uint64_t delta_ticks) { + LARGE_INTEGER freq{}; + QueryPerformanceFrequency(&freq); + return (1000.0 * static_cast(delta_ticks)) / + static_cast(freq.QuadPart); +} + +double Percentile(std::vector values, double p) { + if (values.empty()) { + return 0; + } + std::sort(values.begin(), values.end()); + auto const idx = static_cast( + std::ceil(p * static_cast(values.size() - 1))); + return values[std::min(idx, values.size() - 1)]; +} + +} // namespace + +int RunCoordinator(CoordinatorArgs args) { + if (args.run_id.empty()) { + args.run_id = MakeRunId(); + } + if (args.exe_path.empty()) { + args.exe_path = DefaultExePath(); + } + if (args.artifact_dir.empty()) { + args.artifact_dir = ".artifacts/uap-delivery-timing/" + args.run_id; + } + + std::filesystem::create_directories(args.artifact_dir); + auto const state_root = + std::filesystem::path{args.artifact_dir} / "persistent-state"; + auto const state_a = (state_root / "state-a").string(); + auto const state_b = (state_root / "state-b").string(); + std::filesystem::create_directories(state_a); + std::filesystem::create_directories(state_b); + + ChildProc alice; + alice.side = Side::kA; + ChildProc bob; + bob.side = Side::kB; + + auto const pipe_a = PipeNameFor(args.run_id, Side::kA); + auto const pipe_b = PipeNameFor(args.run_id, Side::kB); + + std::cout << "Spawning Alice/Bob run_id=" << args.run_id << std::endl; + auto const log_a = + (std::filesystem::path{args.artifact_dir} / "alice.log").string(); + auto const log_b = + (std::filesystem::path{args.artifact_dir} / "bob.log").string(); + if (!SpawnChild(alice, args, state_a, pipe_a, "uap-bench-alice", log_a) || + !SpawnChild(bob, args, state_b, pipe_b, "uap-bench-bob", log_b)) { + return 2; + } + + auto wait_ready = [&](ChildProc& c, char const* name) { + auto const deadline = GetTickCount64() + 180000; + while (GetTickCount64() < deadline && !(c.ready && c.uid_ok)) { + if (auto f = c.pipe.TryReadFrame(200)) { + HandleChildFrame(c, *f); + } + } + if (!(c.ready && c.uid_ok)) { + std::cerr << name << " not ready" << std::endl; + return false; + } + return true; + }; + if (!wait_ready(alice, "Alice") || !wait_ready(bob, "Bob")) { + StopChild(alice); + StopChild(bob); + return 3; + } + + std::int64_t a_lo = 0; + std::int64_t a_hi = 0; + std::int64_t b_lo = 0; + std::int64_t b_hi = 0; + std::memcpy(&a_lo, &alice.uid_lo, 8); + std::memcpy(&a_hi, &alice.uid_hi, 8); + std::memcpy(&b_lo, &bob.uid_lo, 8); + std::memcpy(&b_hi, &bob.uid_hi, 8); + SendCmd(alice, IpcType::kSetPeerUid, 0, 0, b_lo, b_hi); + SendCmd(bob, IpcType::kSetPeerUid, 0, 0, a_lo, a_hi); + // Drain acks + for (int i = 0; i < 20; ++i) { + if (auto f = alice.pipe.TryReadFrame(100)) { + HandleChildFrame(alice, *f); + } + if (auto f = bob.pipe.TryReadFrame(100)) { + HandleChildFrame(bob, *f); + } + } + + std::cout << "Waiting Bob warm-up (>=10 ping RTT samples)..." << std::endl; + SendCmd(bob, IpcType::kWaitWarmup); + std::int64_t warmup_n = 0; + std::int64_t warmup_min = 0; + std::int64_t warmup_p99 = 0; + std::uint32_t warmup_guard = 0; + { + auto const deadline = GetTickCount64() + 300000; + bool done = false; + while (GetTickCount64() < deadline && !done) { + if (auto f = bob.pipe.TryReadFrame(500)) { + HandleChildFrame(bob, *f); + if (static_cast(f->type) == IpcType::kWarmupDone) { + warmup_n = f->a; + warmup_min = f->b; + warmup_p99 = f->c; + warmup_guard = f->offset_ms; + done = true; + } + } + if (auto f = alice.pipe.TryReadFrame(0)) { + HandleChildFrame(alice, *f); + } + } + if (!done) { + std::cerr << "Bob warm-up timed out\n"; + StopChild(alice); + StopChild(bob); + return 4; + } + } + + SendCmd(alice, IpcType::kWaitWarmup); + std::int64_t alice_warmup_n = 0; + std::int64_t alice_warmup_min = 0; + std::int64_t alice_warmup_p99 = 0; + std::int64_t alice_dest_server = 0; + std::int64_t alice_dest_protocol = 0; + { + auto const deadline = GetTickCount64() + 300000; + bool done = false; + while (GetTickCount64() < deadline && !done) { + if (auto f = alice.pipe.TryReadFrame(500)) { + HandleChildFrame(alice, *f); + if (static_cast(f->type) == IpcType::kWarmupDone) { + alice_warmup_n = f->a; + alice_warmup_min = f->b; + alice_warmup_p99 = f->c; + alice_dest_server = f->d; + alice_dest_protocol = f->e; + done = true; + } + } + if (auto f = bob.pipe.TryReadFrame(0)) { + HandleChildFrame(bob, *f); + } + } + if (!done) { + std::cerr << "Alice dest-server warm-up timed out\n"; + StopChild(alice); + StopChild(bob); + return 4; + } + } + + std::cout << "## Bob ping statistics\n" + << "samples=" << warmup_n << " min_rtt_ms=" << warmup_min + << " p99_rtt_ms=" << warmup_p99 << " guard_ms=" << warmup_guard + << std::endl + << std::endl; + std::cout << "## Alice dest-server ping statistics\n" + << "server_id=" << alice_dest_server << " samples=" << alice_warmup_n + << " min_rtt_ms=" << alice_warmup_min + << " p99_rtt_ms=" << alice_warmup_p99 << std::endl + << std::endl; + + auto const dest_proto = static_cast(alice_dest_protocol); + std::cout << "AE_SUPPORT_TCP=" << AE_SUPPORT_TCP + << " AE_SUPPORT_UDP=" << AE_SUPPORT_UDP + << " selected alice=" << BenchProtocolName(alice.own_proof.protocol) + << " bob=" << BenchProtocolName(bob.own_proof.protocol) + << " dest=" << BenchProtocolName(dest_proto) << "\n\n"; + if (!IsMeasuredProtocolOk(alice.own_proof.protocol) || + !IsMeasuredProtocolOk(bob.own_proof.protocol) || + !IsMeasuredProtocolOk(dest_proto)) { +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP + std::cerr << "FAIL: measured work path is not UDP " + "(registration may be TCP)\n"; +#else + std::cerr << "FAIL: measured work path is not TCP\n"; +#endif + StopChild(alice); + StopChild(bob); + return 6; + } + + std::map samples; + std::uint32_t sequence = 1; + auto const csv_path = + (std::filesystem::path{args.artifact_dir} / "samples.csv").string(); + std::ofstream csv(csv_path); + csv << "sequence,offset_ms,schedule_server_id,actual_send_server_id," + "route_generation,protocol,raw_next_ping_delta_ms,last_connect_delta_" + "ms,query_send_us,one_way_estimate_us,converted_deadline_us,window_" + "start_us,target_send_us,actual_send_us,receive_us,delivery_ms," + "duplicate_count,classification,valid,invalid_reason," + "bob_ping_server_id,bob_ping_planned_send_us,bob_ping_actual_send_us," + "bob_ping_early_by_us,bob_base_rx_window_us," + "bob_effective_wire_rx_window_us,bob_required_rx_until_us," + "bob_ping_result_us,bob_ping_guard_us\n"; + + int route_invalid_total = 0; + int duplicate_total = 0; + bool acceptance_fail = false; + for (int offset : kOffsetsMs) { + int valid = 0; + int attempts = 0; + while (valid < kSamplesPerOffset && attempts < kSamplesPerOffset * 5) { + ++attempts; + auto const seq = sequence++; + SendCmd(alice, IpcType::kRunSample, seq, + static_cast(offset)); + + SampleRecord rec{}; + rec.sequence = seq; + rec.offset_ms = offset; + rec.classification = ClassificationForOffset(offset); + bool got_send = false; + bool got_recv = false; + auto const deadline = GetTickCount64() + 45000; + while (GetTickCount64() < deadline && !(got_send && got_recv)) { + if (auto f = alice.pipe.TryReadFrame(100)) { + auto const type = static_cast(f->type); + auto const kind = static_cast(f->event_kind); + if (type == IpcType::kSampleResult && f->sequence == seq) { + if (kind == EventKind::kSampleSent) { + rec.window_start_us = f->a; + rec.converted_deadline_us = f->b; + rec.send_qpc = static_cast(f->c); + rec.schedule_server_id = f->d; + rec.actual_send_server_id = f->e; + rec.route_generation = f->f; + rec.protocol = f->g; + rec.raw_next_ping_delta_ms = f->h; + rec.last_connect_delta_ms = f->i; + rec.query_send_us = f->j; + rec.one_way_estimate_us = f->k; + rec.target_send_us = f->l; + rec.actual_send_us = f->local_steady_us; + got_send = true; + } else if (kind == EventKind::kSampleSkipped) { + rec.window_start_us = f->a; + rec.converted_deadline_us = f->b; + rec.schedule_server_id = f->d; + rec.actual_send_server_id = f->e; + rec.route_generation = f->f; + rec.protocol = f->g; + rec.raw_next_ping_delta_ms = f->h; + rec.last_connect_delta_ms = f->i; + rec.invalid_reason = SkipReasonString(f->c); + got_send = true; + got_recv = true; + } else if (kind == EventKind::kError) { + rec.invalid_reason = "alice_error"; + got_send = true; + got_recv = true; + } + } + } + if (auto f = bob.pipe.TryReadFrame(100)) { + HandleChildFrame(bob, *f); + auto const type = static_cast(f->type); + auto const kind = static_cast(f->event_kind); + if (type == IpcType::kEvent && kind == EventKind::kSampleReceived && + f->sequence == seq) { + rec.send_qpc = static_cast(f->a); + rec.receive_qpc = static_cast(f->b); + rec.duplicate_count = static_cast(f->c); + rec.receive_us = f->local_steady_us; + got_recv = true; + } + } + } + + if (rec.invalid_reason.empty() && got_send && got_recv && + rec.receive_qpc >= rec.send_qpc) { + rec.delivery_ms = QpcToMs(rec.receive_qpc - rec.send_qpc); + if (rec.duplicate_count != 1) { + rec.invalid_reason = rec.duplicate_count == 0 ? "no_receive_count" + : "duplicate"; + if (rec.duplicate_count > 1) { + ++duplicate_total; + } + } else if (rec.schedule_server_id != rec.actual_send_server_id || + rec.schedule_server_id == 0) { + rec.invalid_reason = "ROUTE_MISMATCH"; + ++route_invalid_total; + } else if (!IsMeasuredProtocolOk( + static_cast(rec.protocol))) { + rec.invalid_reason = "protocol_mismatch"; + } else { + rec.valid = true; + ++valid; + } + } else if (rec.invalid_reason.empty()) { + rec.invalid_reason = "timeout_or_incomplete"; + } + if (rec.invalid_reason == "INVALID_ROUTE_CHANGED" || + rec.invalid_reason == "ROUTE_MISMATCH") { + ++route_invalid_total; + } + + AttachAndClassify(rec, bob.ping_events); + samples[seq] = rec; + csv << rec.sequence << "," << rec.offset_ms << "," + << rec.schedule_server_id << "," << rec.actual_send_server_id << "," + << rec.route_generation << "," << rec.protocol << "," + << rec.raw_next_ping_delta_ms << "," << rec.last_connect_delta_ms + << "," << rec.query_send_us << "," << rec.one_way_estimate_us << "," + << rec.converted_deadline_us << "," << rec.window_start_us << "," + << rec.target_send_us << "," << rec.actual_send_us << "," + << rec.receive_us << "," << rec.delivery_ms << "," + << rec.duplicate_count << "," << rec.classification << "," + << (rec.valid ? 1 : 0) << "," << rec.invalid_reason << "," + << rec.bob_ping_server_id << "," << rec.bob_ping_planned_send_us + << "," << rec.bob_ping_actual_send_us << "," + << rec.bob_ping_early_by_us << "," << rec.bob_base_rx_window_us << "," + << rec.bob_effective_wire_rx_window_us << "," + << rec.bob_required_rx_until_us << "," << rec.bob_ping_result_us + << "," << rec.bob_ping_guard_us << "\n"; + csv.flush(); + std::cout << "offset=" << offset << " seq=" << seq + << " valid=" << rec.valid << " delivery_ms=" << rec.delivery_ms + << " sched=" << rec.schedule_server_id + << " actual=" << rec.actual_send_server_id + << " reason=" << rec.invalid_reason << std::endl; + } + if (valid < kMinValidPerOffset) { + std::cerr << "Only " << valid << " valid samples for offset " << offset + << " (need " << kMinValidPerOffset << ")\n"; + acceptance_fail = true; + } + } + + std::cout << "\n## Delivery results\n"; + std::cout << "| offset_ms | valid | route_invalid | min_ms | p50_ms | p90_ms " + "| max_ms | duplicates | classification |\n"; + std::cout << "|-----------|-------|---------------|--------|--------|--------|" + "--------|------------|----------------|\n"; + for (int offset : kOffsetsMs) { + std::vector vals; + int route_invalid = 0; + int extras = 0; + for (auto const& [_, s] : samples) { + if (s.offset_ms != offset) { + continue; + } + if (s.valid) { + vals.push_back(s.delivery_ms); + } + if (s.invalid_reason == "INVALID_ROUTE_CHANGED" || + s.invalid_reason == "ROUTE_MISMATCH") { + ++route_invalid; + } + if (s.duplicate_count > 1) { + extras += s.duplicate_count - 1; + } + } + double min_v = 0; + double max_v = 0; + if (!vals.empty()) { + min_v = *std::min_element(vals.begin(), vals.end()); + max_v = *std::max_element(vals.begin(), vals.end()); + } + auto p50 = Percentile(vals, 0.50); + auto p90 = Percentile(vals, 0.90); + std::cout << "| " << offset << " | " << vals.size() << " | " << route_invalid + << " | " << min_v << " | " << p50 << " | " << p90 << " | " + << max_v << " | " << extras << " | " + << ClassificationForOffset(offset) << " |\n"; + if (static_cast(vals.size()) < kMinValidPerOffset || extras != 0) { + acceptance_fail = true; + } + } + std::cout << "\nCSV: " << csv_path << std::endl; + std::cout << "route_invalid_total=" << route_invalid_total + << " duplicate_total=" << duplicate_total << std::endl; + +#if defined(AE_UAP_DELIVERY_REQUIRE_UDP) && AE_UAP_DELIVERY_REQUIRE_UDP + std::cout << "\n## UDP anomaly traces (if any)\n"; + bool saw_500_3641 = false; + bool saw_1500_286 = false; + for (auto const& [_, s] : samples) { + if (!s.valid) { + continue; + } + bool interesting = false; + if (s.offset_ms == 500 && s.delivery_ms > 3000) { + interesting = true; + saw_500_3641 = true; + } + if (s.offset_ms == 1500 && s.delivery_ms < 500) { + interesting = true; + saw_1500_286 = true; + } + if (!interesting) { + continue; + } + std::cout << "TRACE seq=" << s.sequence << " offset=" << s.offset_ms + << " sched=" << s.schedule_server_id + << " actual=" << s.actual_send_server_id + << " gen=" << s.route_generation << " proto=" << s.protocol + << " raw_delta_ms=" << s.raw_next_ping_delta_ms + << " window_start_us=" << s.window_start_us + << " target_us=" << s.target_send_us + << " send_us=" << s.actual_send_us + << " recv_us=" << s.receive_us + << " delivery_ms=" << s.delivery_ms + << " class=" << s.classification + << " early_by_us=" << s.bob_ping_early_by_us + << " base_win_us=" << s.bob_base_rx_window_us + << " eff_win_us=" << s.bob_effective_wire_rx_window_us + << " planned_us=" << s.bob_ping_planned_send_us + << " ping_actual_us=" << s.bob_ping_actual_send_us + << " required_until_us=" << s.bob_required_rx_until_us << "\n"; + } + int explained_early = 0; + int unexplained_1500 = 0; + for (auto const& [_, s] : samples) { + if (!s.valid || s.offset_ms != 1500 || s.delivery_ms >= 500) { + continue; + } + if (s.classification == "EARLY_PING_EXTENDED_WINDOW") { + ++explained_early; + } else { + ++unexplained_1500; + } + } + std::cout << "reproduced_+500_~3641ms=" << (saw_500_3641 ? "yes" : "no") + << "\nreproduced_+1500_~286ms=" << (saw_1500_286 ? "yes" : "no") + << "\nexplained_by_early_ping_window=" << explained_early + << "\nunexplained_+1500_short=" << unexplained_1500 << "\n"; + if (saw_500_3641) { + acceptance_fail = true; + } +#endif + + { + auto const ping_csv = + (std::filesystem::path{args.artifact_dir} / "bob_ping_trace.csv") + .string(); + std::ofstream out(ping_csv); + out << "kind,server_id,planned_us,actual_us,early_by_us,base_window_us," + "effective_window_us,required_until_us,next_planned_us,guard_us," + "min_rtt_us,p99_rtt_us,channel_generation,result_type," + "event_steady_us\n"; + for (auto const& e : bob.ping_events) { + out << static_cast(e.kind) << "," << e.server_id << "," + << e.planned_us << "," << e.actual_us << "," << e.early_by_us << "," + << e.base_window_us << "," << e.effective_window_us << "," + << e.required_until_us << "," << e.next_planned_us << "," + << e.guard_us << "," << e.min_rtt_us << "," << e.p99_rtt_us << "," + << e.channel_generation << "," << e.result_type << "," + << e.event_steady_us << "\n"; + } + } + + StopChild(alice); + StopChild(bob); + if (acceptance_fail || route_invalid_total != 0 || duplicate_total != 0) { + std::cerr << "FAIL: delivery acceptance checks\n"; + return 7; + } + return 0; +} + +} // namespace ae::bench::uap diff --git a/examples/benches/aether_uap_delivery_timing_bench/coordinator.h b/examples/benches/aether_uap_delivery_timing_bench/coordinator.h new file mode 100644 index 00000000..3b9da6fd --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/coordinator.h @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_UAP_DELIVERY_TIMING_BENCH_COORDINATOR_H_ +#define AETHER_UAP_DELIVERY_TIMING_BENCH_COORDINATOR_H_ + +#include + +namespace ae::bench::uap { + +struct CoordinatorArgs { + std::string run_id; + std::string artifact_dir; + std::string exe_path; + std::string parent_uid{"3ac93165-3d37-4970-87a6-fa4ee27744e4"}; +}; + +int RunCoordinator(CoordinatorArgs args); + +} // namespace ae::bench::uap + +#endif // AETHER_UAP_DELIVERY_TIMING_BENCH_COORDINATOR_H_ diff --git a/examples/benches/aether_uap_delivery_timing_bench/main.cpp b/examples/benches/aether_uap_delivery_timing_bench/main.cpp new file mode 100644 index 00000000..604089aa --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/main.cpp @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "client_role.h" +#include "coordinator.h" + +namespace { + +std::string_view ArgValue(int argc, char** argv, std::string_view key) { + for (int i = 1; i + 1 < argc; ++i) { + if (key == argv[i]) { + return argv[i + 1]; + } + } + return {}; +} + +} // namespace + +int main(int argc, char** argv) { + using namespace ae::bench::uap; + + auto role = ArgValue(argc, argv, "--role"); + if (role.empty() || role == "coordinator") { + CoordinatorArgs args; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.artifact_dir = std::string{ArgValue(argc, argv, "--artifact-dir")}; + args.exe_path = std::string{ArgValue(argc, argv, "--exe")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + return RunCoordinator(args); + } + + if (role == "client") { + ClientArgs args; + auto side = ArgValue(argc, argv, "--side"); + args.side = (side == "B" || side == "b") ? Side::kB : Side::kA; + args.run_id = std::string{ArgValue(argc, argv, "--run-id")}; + args.state_dir = std::string{ArgValue(argc, argv, "--state-dir")}; + args.pipe_name = std::string{ArgValue(argc, argv, "--pipe")}; + args.client_name = std::string{ArgValue(argc, argv, "--client-name")}; + auto parent = ArgValue(argc, argv, "--parent-uid"); + if (!parent.empty()) { + args.parent_uid = std::string{parent}; + } + if (args.client_name.empty()) { + args.client_name = + args.side == Side::kA ? "uap-bench-alice" : "uap-bench-bob"; + } + return RunClientRole(args); + } + + std::cerr << "Unknown --role\n"; + return 1; +} diff --git a/examples/benches/aether_uap_delivery_timing_bench/tele_off.h b/examples/benches/aether_uap_delivery_timing_bench/tele_off.h new file mode 100644 index 00000000..89da4ca9 --- /dev/null +++ b/examples/benches/aether_uap_delivery_timing_bench/tele_off.h @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Load USER_CONFIG (and the rest of aether/config.h) first, then override +// console telemetry for this benchmark/example target. Force-include this +// header so the override wins without a conflicting /D AE_TELE_LOG_CONSOLE. +#include "aether/config.h" + +#undef AE_TELE_LOG_CONSOLE +#define AE_TELE_LOG_CONSOLE 0 diff --git a/scripts/aggregate_uap_phase_preservation_8h.py b/scripts/aggregate_uap_phase_preservation_8h.py new file mode 100644 index 00000000..50c26f01 --- /dev/null +++ b/scripts/aggregate_uap_phase_preservation_8h.py @@ -0,0 +1,810 @@ +#!/usr/bin/env python3 +"""Aggregate 8h UAP phase-preservation shard outputs into final reports.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +def wilson(success: int, n: int, z: float = 1.96) -> tuple[float, float]: + if n <= 0: + return (float("nan"), float("nan")) + p = success / n + den = 1 + z * z / n + centre = p + z * z / (2 * n) + margin = z * math.sqrt((p * (1 - p) + z * z / (4 * n)) / n) + return ((centre - margin) / den, (centre + margin) / den) + + +def percentile(sorted_vals: list[float], p: float) -> float: + if not sorted_vals: + return float("nan") + if len(sorted_vals) == 1: + return sorted_vals[0] + k = (len(sorted_vals) - 1) * (p / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_vals[int(k)] + return sorted_vals[f] * (c - k) + sorted_vals[c] * (k - f) + + +def dist(vals: list[float]) -> dict[str, Any]: + clean = [float(v) for v in vals if v is not None and not (isinstance(v, float) and math.isnan(v))] + clean = [v for v in clean if math.isfinite(v)] + if not clean: + return {"n": 0} + s = sorted(clean) + mean = statistics.fmean(s) + stdev = statistics.pstdev(s) if len(s) > 1 else 0.0 + out = { + "n": len(s), + "min": s[0], + "mean": mean, + "stddev": stdev, + "p1": percentile(s, 1), + "p5": percentile(s, 5), + "p10": percentile(s, 10), + "p25": percentile(s, 25), + "p50": percentile(s, 50), + "p75": percentile(s, 75), + "p90": percentile(s, 90), + "p95": percentile(s, 95), + "p99": percentile(s, 99), + "p99_5": percentile(s, 99.5), + "p99_9": percentile(s, 99.9), + "max": s[-1], + "max_abs": max(abs(x) for x in s), + "final": s[-1], + } + return out + + +def fmt_dist(d: dict[str, Any], indent: str = "") -> str: + if d.get("n", 0) == 0: + return f"{indent}- n: 0\n" + lines = [f"{indent}- n: {d['n']}"] + for k in ( + "min", + "mean", + "stddev", + "p1", + "p5", + "p10", + "p25", + "p50", + "p75", + "p90", + "p95", + "p99", + "p99_5", + "p99_9", + "max", + ): + if k in d: + lines.append(f"{indent}- {k}: {d[k]:.6f}") + if "max_abs" in d: + lines.append(f"{indent}- max_abs: {d['max_abs']:.6f}") + return "\n".join(lines) + "\n" + + +def parse_float(x: str) -> float | None: + if x is None: + return None + s = str(x).strip() + if s == "" or s.lower() in {"null", "nan", "none"}: + return None + try: + v = float(s) + except ValueError: + return None + if not math.isfinite(v): + return None + return v + + +def parse_int(x: str) -> int | None: + v = parse_float(x) + return None if v is None else int(v) + + +def load_jsonl(path: Path) -> list[dict]: + rows = [] + if not path.exists(): + return rows + with path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def linear_drift(xs: list[float], ys: list[float]) -> tuple[float, float]: + n = min(len(xs), len(ys)) + if n < 2: + return (0.0, 0.0) + x = xs[:n] + y = ys[:n] + mx = statistics.fmean(x) + my = statistics.fmean(y) + num = sum((a - mx) * (b - my) for a, b in zip(x, y)) + den = sum((a - mx) ** 2 for a in x) + if den == 0: + return (0.0, 0.0) + slope = num / den # per cycle index unit + return (slope, slope * 3600.0) # ms/cycle, ms/hour if cycle~1s + + +def margin_buckets(margins: list[float]) -> dict[str, int]: + edges = [ + (">+200", lambda m: m > 200), + ("+100..+200", lambda m: 100 < m <= 200), + ("+50..+100", lambda m: 50 < m <= 100), + ("+20..+50", lambda m: 20 < m <= 50), + ("+10..+20", lambda m: 10 < m <= 20), + ("+5..+10", lambda m: 5 < m <= 10), + ("+2..+5", lambda m: 2 < m <= 5), + ("+1..+2", lambda m: 1 < m <= 2), + ("0..+1", lambda m: 0 <= m <= 1), + ("-1..0", lambda m: -1 <= m < 0), + ("-2..-1", lambda m: -2 <= m < -1), + ("-5..-2", lambda m: -5 <= m < -2), + ("-10..-5", lambda m: -10 <= m < -5), + ("-20..-10", lambda m: -20 <= m < -10), + ("<-20", lambda m: m < -20), + ] + out = {name: 0 for name, _ in edges} + for m in margins: + for name, pred in edges: + if pred(m): + out[name] += 1 + break + return out + + +def classify(inv: str) -> str: + s = inv.lower() + if "schedule shifted" in s or "phase" in s and "shifted" in s: + return "PRODUCTION_PHASE" + if "after original deadline" in s: + return "PRODUCTION_RETRY_ESTIMATED_LATE_ARRIVAL" + if "window" in s: + return "PRODUCTION_WINDOW" + if "misseddeadline" in s or "unknown" in s and "false" in s: + return "PRODUCTION_STATE" + if "first request was sent" in s or "did not reach send" in s: + return "HARNESS_FAULT_NOT_ARMED" + if "retry did not reach" in s: + return "HARNESS_FAULT_WRONG_CYCLE" + if "querypeerreceive" in s or "observer" in s or "checkpoint" in s: + return "HARNESS_QUERY" + if "cycle not confirmed" in s or "no cycle start" in s: + return "HARNESS_CYCLE_CONFIRM" + if "reporting" in s: + return "HARNESS_REPORTING" + if "crash" in s: + return "PROCESS_CRASH" + if "timeout" in s: + return "TIMEOUT" + return "OTHER" + + +PROD_CLASSES = { + "PRODUCTION_PHASE", + "PRODUCTION_RETRY_LATE_SEND", + "PRODUCTION_RETRY_ESTIMATED_LATE_ARRIVAL", + "PRODUCTION_WINDOW", + "PRODUCTION_STATE", +} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--root", required=True) + args = ap.parse_args() + root = Path(args.root) + runs = root / "runs" + agg = root / "aggregate" + fail_dir = root / "failure-cases" + agg.mkdir(parents=True, exist_ok=True) + fail_dir.mkdir(parents=True, exist_ok=True) + + all_rows: list[dict] = [] + shard_meta: list[dict] = [] + failures: list[dict] = [] + + for shard_dir in sorted(runs.glob("*")): + if not shard_dir.is_dir(): + continue + transport = "tcp" if shard_dir.name.startswith("tcp") else "udp" + meta_path = shard_dir / "shard-status.json" + meta = {} + if meta_path.exists(): + meta = json.loads(meta_path.read_text(encoding="utf-8")) + sm = shard_dir / "shard-meta.json" + if sm.exists(): + try: + meta.update(json.loads(sm.read_text(encoding="utf-8"))) + except json.JSONDecodeError: + pass + meta["shard"] = shard_dir.name + meta["transport"] = transport + shard_meta.append(meta) + + samples = load_jsonl(shard_dir / "samples.jsonl") + for r in samples: + r["shard"] = shard_dir.name + r.setdefault("transport", transport) + all_rows.append(r) + for inv in r.get("failures") or []: + fc = classify(str(inv)) + failures.append( + { + "shard": shard_dir.name, + "transport": transport, + "cycle_index": r.get("cycle_index"), + "logical_ping_id": r.get("logical_ping_id"), + "fault_type": r.get("fault_type"), + "invariant": inv, + "failure_class": r.get("failure_class") or fc, + "sample": r, + } + ) + + # Also harvest failed-cases.json if present + fc_path = shard_dir / "failed-cases.json" + if fc_path.exists(): + try: + for item in json.loads(fc_path.read_text(encoding="utf-8")): + inv = item.get("invariant") or "" + failures.append( + { + "shard": shard_dir.name, + "transport": transport, + "cycle_index": item.get("cycle"), + "invariant": inv, + "failure_class": classify(str(inv)), + "sample": item, + } + ) + except Exception: + pass + + # Deduplicate failures loosely + seen = set() + uniq_fail = [] + for f in failures: + key = ( + f.get("shard"), + f.get("cycle_index"), + f.get("invariant"), + f.get("failure_class"), + ) + if key in seen: + continue + seen.add(key) + uniq_fail.append(f) + failures = uniq_fail + + by_tr: dict[str, list[dict]] = {"tcp": [], "udp": []} + for r in all_rows: + by_tr.setdefault(r.get("transport", "?"), []).append(r) + + def subset(rows: list[dict], pred) -> list[dict]: + return [r for r in rows if pred(r)] + + def phase_series(rows: list[dict]) -> tuple[dict, float, float]: + errs = [] + xs = [] + for i, r in enumerate(rows): + e = r.get("scheduled_phase_error_ms") + if e is None: + continue + try: + ev = float(e) + except (TypeError, ValueError): + continue + if math.isfinite(ev): + errs.append(ev) + xs.append(float(r.get("cycle_index", i))) + d = dist(errs) + slope_c, slope_h = linear_drift(xs, errs) + return d, slope_c, slope_h + + def report_transport(tr: str) -> dict[str, Any]: + rows = by_tr.get(tr, []) + base = subset(rows, lambda r: (r.get("fault_type") or "none") == "none") + req = subset(rows, lambda r: r.get("fault_type") == "request-loss") + resp = subset(rows, lambda r: r.get("fault_type") == "response-loss") + mixed = subset( + rows, + lambda r: (r.get("fault_type") in {"request-loss", "response-loss"}) + and str(r.get("sequence", "")).startswith(("alternat", "periodic", "random")), + ) + consec = subset( + rows, lambda r: str(r.get("sequence", "")).startswith("consecutive") + ) + + sched_all, drift_c, drift_h = phase_series(rows) + next_errs = [ + float(r["next_nominal_phase_error_ms"]) + for r in rows + if r.get("next_nominal_phase_error_ms") is not None + and math.isfinite(float(r["next_nominal_phase_error_ms"])) + ] + + contig = [ + float(r["actual_interval_error_us"]) / 1000.0 + for r in rows + if r.get("contiguous_cycle") == 1 + and r.get("actual_interval_error_us") is not None + ] + # samples.jsonl may not have interval; use skipped_slots + contig2 = [] + skipped = [] + for r in rows: + if r.get("contiguous_cycle") == 1 and r.get("skipped_slots") == 0: + # approximate from consecutive scheduled if available + pass + if r.get("skipped_slots") not in (None, 0): + try: + skipped.append(float(r["skipped_slots"])) + except (TypeError, ValueError): + pass + + def loss_stats(loss_rows: list[dict]) -> dict[str, Any]: + requested = len(loss_rows) + armed = sum(1 for r in loss_rows if r.get("fault_armed") == 1) + consumed = sum(1 for r in loss_rows if r.get("fault_consumed") == 1) + margins = [] + before = after = none = unmeas = 0 + send_before = send_after = 0 + for r in loss_rows: + if r.get("fault_consumed") != 1: + continue + m = r.get("estimated_server_margin_ms") + if m is None: + m = r.get("retry_server_margin_ms") + rs = r.get("retry_actual_send_time") + tn = r.get("Tn") or r.get("original_deadline") + if rs is not None and tn is not None: + try: + if float(rs) <= float(tn): + send_before += 1 + else: + send_after += 1 + except (TypeError, ValueError): + pass + if m is None: + # no retry + if r.get("retry_actual_send_time") in (None,): + none += 1 + else: + unmeas += 1 + continue + try: + mv = float(m) + except (TypeError, ValueError): + unmeas += 1 + continue + if not math.isfinite(mv): + unmeas += 1 + continue + margins.append(mv) + if mv >= 0: + before += 1 + else: + after += 1 + valid = consumed # production proportions exclude harness-invalid + harness_invalid = requested - consumed + recovered = before # estimated before Tn among consumed + def pct(a, b): + return (100.0 * a / b) if b else float("nan") + + lo, hi = wilson(before, max(consumed, 1)) + return { + "requested": requested, + "fault_armed": armed, + "fault_consumed": consumed, + "harness_invalid": harness_invalid, + "retry_send_before_Tn": send_before, + "retry_send_after_Tn": send_after, + "estimated_before_Tn": before, + "estimated_after_Tn": after, + "no_retry_or_unmeas": none + unmeas, + "estimated_before_pct": pct(before, consumed), + "estimated_before_wilson95": [lo, hi], + "send_before_pct": pct(send_before, consumed), + "margins": dist(margins), + "margin_buckets": margin_buckets(margins), + } + + guards = [ + float(r["computed_guard_us"]) / 1000.0 + for r in rows + if r.get("computed_guard_us") not in (None,) + ] + # from CSV if present later; jsonl may lack guard - try samples.csv + return { + "transport": tr, + "valid_cycles": len(rows), + "baseline_cases": len(base), + "request_loss_cases": len(req), + "response_loss_cases": len(resp), + "scheduled_phase": sched_all, + "next_nominal_phase": dist(next_errs), + "drift_ms_per_cycle": drift_c, + "drift_ms_per_hour": drift_h, + "phase_baseline": phase_series(base)[0], + "phase_request": phase_series(req)[0], + "phase_response": phase_series(resp)[0], + "phase_mixed": phase_series(mixed)[0], + "phase_consecutive": phase_series(consec)[0], + "request_loss": loss_stats(req), + "response_loss": loss_stats(resp), + "skipped_slots": dist(skipped), + "false_live_unknown": sum( + 1 + for f in failures + if f.get("transport") == tr + and "false Unknown" in str(f.get("invariant", "")) + ), + "false_live_missed": sum( + 1 + for f in failures + if f.get("transport") == tr + and "false MissedDeadline" in str(f.get("invariant", "")) + ), + "duplicates": sum( + 1 + for f in failures + if f.get("transport") == tr and "duplicate" in str(f.get("invariant", "")).lower() + ), + } + + # Enrich from samples.csv where available + csv_rows: list[dict] = [] + for shard_dir in sorted(runs.glob("*")): + p = shard_dir / "samples.csv" + if not p.exists(): + continue + with p.open("r", encoding="utf-8", errors="replace", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + row["shard"] = shard_dir.name + row["transport"] = "tcp" if shard_dir.name.startswith("tcp") else "udp" + csv_rows.append(row) + + phase_csv_rows = [] + for shard_dir in sorted(runs.glob("*")): + p = shard_dir / "phase-error-by-cycle.csv" + if not p.exists(): + continue + with p.open("r", encoding="utf-8", errors="replace", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + row["shard"] = shard_dir.name + row["transport"] = "tcp" if shard_dir.name.startswith("tcp") else "udp" + phase_csv_rows.append(row) + + def enrich_tr(rep: dict[str, Any], tr: str) -> dict[str, Any]: + rows = [r for r in csv_rows if r.get("transport") == tr] + phase_rows = [r for r in phase_csv_rows if r.get("transport") == tr] + + def col(name: str) -> list[float]: + out = [] + for r in rows: + v = parse_float(r.get(name, "")) + if v is not None: + out.append(v) + return out + + def phase_col(name: str, only_contig: bool | None = None) -> list[float]: + out = [] + for r in phase_rows: + if only_contig is True and parse_int(r.get("contiguous_cycle", "")) != 1: + continue + if only_contig is False and parse_int(r.get("contiguous_cycle", "")) == 1: + continue + v = parse_float(r.get(name, "")) + if v is None: + continue + if name.endswith("_us"): + out.append(v / 1000.0) + else: + out.append(v) + return out + + rep["guard_ms"] = dist(col("guard_ms")) + rep["attempt_lead_ms"] = dist(col("attempt_lead_ms")) + rep["first_attempt_offset_from_Tn_ms"] = dist(col("first_attempt_offset_from_Tn_ms")) + rep["timeout_to_retry_send_ms"] = dist(col("timeout_to_retry_send_ms")) + rep["first_attempt_to_retry_ms"] = dist(col("first_attempt_to_retry_ms")) + rep["retry_client_margin_to_Tn_ms"] = dist(col("retry_client_margin_to_Tn_ms")) + rep["estimated_server_margin_ms"] = dist(col("estimated_server_margin_ms")) + rep["alice_query"] = { + # observer csv if present + } + obs_vals = [] + for shard_dir in sorted(runs.glob(f"{tr}-*")): + op = shard_dir / "observer-query-results.csv" + if not op.exists(): + continue + with op.open("r", encoding="utf-8", errors="replace", newline="") as f: + for row in csv.DictReader(f): + # no rtt column filled yet; skip + pass + contig_err = phase_col("actual_interval_error_us", only_contig=True) + skipped_err = phase_col("actual_interval_error_us", only_contig=False) + rep["contiguous_interval_error_ms"] = dist(contig_err) + rep["skipped_cycle_interval_error_ms"] = dist(skipped_err) + # long-term thirds + if rows: + n = len(rows) + a = rows[: max(1, n // 10)] + c = rows[max(1, n // 10) : max(1, n - n // 10)] + b = rows[max(1, n - n // 10) :] + + def part(rs): + return { + "n": len(rs), + "guard_p50": dist([parse_float(r.get("guard_ms")) or float("nan") for r in rs]).get("p50"), + "lead_p50": dist([parse_float(r.get("attempt_lead_ms")) or float("nan") for r in rs]).get("p50"), + "phase_p50": dist([parse_float(r.get("scheduled_phase_error_ms")) or float("nan") for r in rs]).get("p50"), + "margin_p50": dist([parse_float(r.get("estimated_server_margin_ms")) or float("nan") for r in rs]).get("p50"), + } + + rep["stability"] = {"first_10pct": part(a), "middle_80pct": part(c), "last_10pct": part(b)} + return rep + + tcp = enrich_tr(report_transport("tcp"), "tcp") + udp = enrich_tr(report_transport("udp"), "udp") + + prod_fail = [f for f in failures if f.get("failure_class") in PROD_CLASSES] + harness_fail = [f for f in failures if str(f.get("failure_class", "")).startswith("HARNESS")] + other_fail = [f for f in failures if f not in prod_fail and f not in harness_fail] + + # Save failure timelines with neighbors + by_shard_cycle = defaultdict(dict) + for r in all_rows: + by_shard_cycle[r.get("shard")][r.get("cycle_index")] = r + saved = [] + for f in prod_fail: + shard = f.get("shard") + ci = f.get("cycle_index") + if ci is None: + continue + try: + ci = int(ci) + except (TypeError, ValueError): + continue + window = [] + for j in range(ci - 3, ci + 4): + if j in by_shard_cycle.get(shard, {}): + window.append(by_shard_cycle[shard][j]) + item = {**f, "timeline_window": window} + saved.append(item) + outp = fail_dir / f"{shard}_cycle{ci}_{f.get('failure_class')}.json" + outp.write_text(json.dumps(item, indent=2), encoding="utf-8") + + summary = { + "shards": shard_meta, + "tcp": tcp, + "udp": udp, + "failures": { + "production": len(prod_fail), + "harness": len(harness_fail), + "other": len(other_fail), + "total": len(failures), + }, + "overall": "PASS" + if len(prod_fail) == 0 and len(all_rows) > 0 + else ("PARTIAL" if len(all_rows) > 0 else "FAIL"), + } + (agg / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + (agg / "failures.json").write_text(json.dumps(failures, indent=2), encoding="utf-8") + + def write_tr_md(path: Path, rep: dict[str, Any]) -> None: + lines = [f"# {rep['transport'].upper()} 8h phase-preservation report\n"] + lines.append(f"- valid_cycles: {rep.get('valid_cycles')}") + lines.append(f"- baseline_cases: {rep.get('baseline_cases')}") + lines.append(f"- request_loss_cases: {rep.get('request_loss_cases')}") + lines.append(f"- response_loss_cases: {rep.get('response_loss_cases')}") + lines.append("\n## Scheduled phase error (ms)\n") + lines.append(fmt_dist(rep.get("scheduled_phase") or {})) + lines.append(f"- drift_ms_per_cycle: {rep.get('drift_ms_per_cycle')}") + lines.append(f"- drift_ms_per_hour: {rep.get('drift_ms_per_hour')}") + lines.append("\n## Next nominal phase error (ms)\n") + lines.append(fmt_dist(rep.get("next_nominal_phase") or {})) + lines.append("\n## Guard / attempt lead\n") + lines.append("### guard_ms\n" + fmt_dist(rep.get("guard_ms") or {})) + lines.append("### attempt_lead_ms\n" + fmt_dist(rep.get("attempt_lead_ms") or {})) + lines.append("### first_attempt_offset_from_Tn_ms\n" + fmt_dist(rep.get("first_attempt_offset_from_Tn_ms") or {})) + lines.append("\n## Retry timing\n") + lines.append("### timeout_to_retry_send_ms\n" + fmt_dist(rep.get("timeout_to_retry_send_ms") or {})) + lines.append("### first_attempt_to_retry_ms\n" + fmt_dist(rep.get("first_attempt_to_retry_ms") or {})) + for kind in ("request_loss", "response_loss"): + ls = rep.get(kind) or {} + lines.append(f"\n## {kind}\n") + for k, v in ls.items(): + if k in {"margins", "margin_buckets"}: + continue + lines.append(f"- {k}: {v}") + lines.append("\n### estimated_server_margins\n" + fmt_dist(ls.get("margins") or {})) + lines.append("\n### margin buckets (positive = before Tn)\n") + for bk, bv in (ls.get("margin_buckets") or {}).items(): + lines.append(f"- {bk}: {bv}") + lines.append("\n## Contiguous interval error (ms from 1000)\n") + lines.append(fmt_dist(rep.get("contiguous_interval_error_ms") or {})) + lines.append("\n## Skipped-cycle interval error (ms)\n") + lines.append(fmt_dist(rep.get("skipped_cycle_interval_error_ms") or {})) + lines.append("\n## Stability thirds\n") + lines.append("```json\n" + json.dumps(rep.get("stability") or {}, indent=2) + "\n```\n") + lines.append(f"\n- false_live_Unknown: {rep.get('false_live_unknown')}") + lines.append(f"- false_live_MissedDeadline: {rep.get('false_live_missed')}") + lines.append(f"- duplicates: {rep.get('duplicates')}") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + write_tr_md(agg / "tcp-report.md", tcp) + write_tr_md(agg / "udp-report.md", udp) + + # CSV extracts + def write_csv(name: str, fieldnames: list[str], rows: list[dict]) -> None: + with (agg / name).open("w", encoding="utf-8", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for r in rows: + w.writerow(r) + + write_csv( + "phase.csv", + [ + "transport", + "shard", + "cycle_index", + "fault_type", + "scheduled_phase_error_ms", + "next_nominal_phase_error_ms", + "contiguous_cycle", + "skipped_slots", + ], + all_rows, + ) + write_csv( + "retry-margins.csv", + [ + "transport", + "shard", + "cycle_index", + "fault_type", + "fault_consumed", + "estimated_server_margin_ms", + "one_way_estimate_us", + ], + all_rows, + ) + write_csv( + "retry-timing.csv", + [ + "transport", + "shard", + "cycle_index", + "fault_type", + "timeout_to_retry_send_ms", + "first_attempt_to_retry_ms", + "retry_client_margin_to_Tn_ms", + ], + csv_rows, + ) + write_csv( + "latency.csv", + [ + "transport", + "shard", + "cycle_index", + "guard_ms", + "attempt_lead_ms", + "first_attempt_offset_from_Tn_ms", + ], + csv_rows, + ) + write_csv( + "windows.csv", + ["transport", "shard", "cycle_index", "fault_type", "next_scheduled_nominal", "next_nominal_phase_error_ms"], + csv_rows, + ) + write_csv( + "alice-query.csv", + ["transport", "shard", "cycle_index", "alice_state", "alice_next_ping_delta_ms"], + csv_rows, + ) + + comparison = f"""# 8h TCP vs UDP comparison + +- overall: {summary['overall']} +- production failures: {summary['failures']['production']} +- harness failures: {summary['failures']['harness']} +- other/process failures: {summary['failures']['other']} + +## TCP +- cycles: {tcp.get('valid_cycles')} +- scheduled phase max_abs: {(tcp.get('scheduled_phase') or {}).get('max_abs')} +- drift ms/cycle: {tcp.get('drift_ms_per_cycle')} +- drift ms/hour: {tcp.get('drift_ms_per_hour')} +- request estimated-before-Tn %: {(tcp.get('request_loss') or {}).get('estimated_before_pct')} +- response estimated-before-Tn %: {(tcp.get('response_loss') or {}).get('estimated_before_pct')} + +## UDP +- cycles: {udp.get('valid_cycles')} +- scheduled phase max_abs: {(udp.get('scheduled_phase') or {}).get('max_abs')} +- drift ms/cycle: {udp.get('drift_ms_per_cycle')} +- drift ms/hour: {udp.get('drift_ms_per_hour')} +- request estimated-before-Tn %: {(udp.get('request_loss') or {}).get('estimated_before_pct')} +- response estimated-before-Tn %: {(udp.get('response_loss') or {}).get('estimated_before_pct')} + +See tcp-report.md / udp-report.md for full percentile tables. +""" + (agg / "comparison.md").write_text(comparison, encoding="utf-8") + + def brief(tr: str, rep: dict) -> str: + sp = rep.get("scheduled_phase") or {} + g = rep.get("guard_ms") or {} + lead = rep.get("attempt_lead_ms") or {} + ttr = rep.get("timeout_to_retry_send_ms") or {} + contig = rep.get("contiguous_interval_error_ms") or {} + rq = rep.get("request_loss") or {} + rs = rep.get("response_loss") or {} + rm = (rq.get("margins") or {}) + sm = (rs.get("margins") or {}) + return f"""{tr.upper()}: +valid cycles: {rep.get('valid_cycles')} +baseline cases: {rep.get('baseline_cases')} +request-loss cases: {rep.get('request_loss_cases')} +response-loss cases: {rep.get('response_loss_cases')} +guard p50/p95/p99/max: {g.get('p50')}/{g.get('p95')}/{g.get('p99')}/{g.get('max')} +attempt lead p50/p95/p99/max: {lead.get('p50')}/{lead.get('p95')}/{lead.get('p99')}/{lead.get('max')} +timeout->retry p50/p95/p99/max: {ttr.get('p50')}/{ttr.get('p95')}/{ttr.get('p99')}/{ttr.get('max')} +request-loss: + recovery/estimated-before-Tn %: {rq.get('estimated_before_pct')} + retry-send-before-Tn %: {rq.get('send_before_pct')} + estimated server margin min/p1/p5/p50/p95: {rm.get('min')}/{rm.get('p1')}/{rm.get('p5')}/{rm.get('p50')}/{rm.get('p95')} +response-loss: + recovery/estimated-before-Tn %: {rs.get('estimated_before_pct')} + retry-send-before-Tn %: {rs.get('send_before_pct')} + estimated server margin min/p1/p5/p50/p95: {sm.get('min')}/{sm.get('p1')}/{sm.get('p5')}/{sm.get('p50')}/{sm.get('p95')} +scheduled phase max abs / p99 / final: {sp.get('max_abs')}/{sp.get('p99')}/{sp.get('final')} +drift ms/cycle: {rep.get('drift_ms_per_cycle')} +drift ms/hour: {rep.get('drift_ms_per_hour')} +contiguous interval p50/p95/p99/max: {contig.get('p50')}/{contig.get('p95')}/{contig.get('p99')}/{contig.get('max')} +duplicates: {rep.get('duplicates')} +false live Unknown: {rep.get('false_live_unknown')} +false live MissedDeadline: {rep.get('false_live_missed')} +""" + + console = [] + console.append("Actual wall time: see status.json elapsed_sec") + console.append(brief("tcp", tcp)) + console.append(brief("udp", udp)) + console.append( + f"Failures:\n production: {summary['failures']['production']}\n harness: {summary['failures']['harness']}\n process/other: {summary['failures']['other']}" + ) + console.append(f"Overall: {summary['overall']}") + (agg / "console-summary.txt").write_text("\n".join(console) + "\n", encoding="utf-8") + print("\n".join(console)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/analyze_uap_phase_preservation_8h.py b/scripts/analyze_uap_phase_preservation_8h.py new file mode 100644 index 00000000..c187bd33 --- /dev/null +++ b/scripts/analyze_uap_phase_preservation_8h.py @@ -0,0 +1,1742 @@ +#!/usr/bin/env python3 +"""Corrected analysis of the completed 8h UAP phase-preservation run. + +Reads only existing artifacts under artifacts/uap-phase-preservation/8h/. +Writes corrected-analysis.md and corrected-summary.json. +""" + +from __future__ import annotations + +import csv +import json +import math +import statistics +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +ROOT = Path("artifacts/uap-phase-preservation/8h") +RUNS = ROOT / "runs" +AGG = ROOT / "aggregate" + + +def pf(x: Any) -> float | None: + if x is None: + return None + s = str(x).strip() + if s == "" or s.lower() in {"null", "nan", "none"}: + return None + try: + v = float(s) + except (TypeError, ValueError): + return None + if not math.isfinite(v): + return None + return v + + +def pi(x: Any) -> int | None: + v = pf(x) + return None if v is None else int(v) + + +def percentile(sorted_vals: list[float], p: float) -> float: + if not sorted_vals: + return float("nan") + if len(sorted_vals) == 1: + return sorted_vals[0] + k = (len(sorted_vals) - 1) * (p / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_vals[int(k)] + return sorted_vals[f] * (c - k) + sorted_vals[c] * (k - f) + + +def dist(vals: list[float | None]) -> dict[str, Any]: + clean = [float(v) for v in vals if v is not None and math.isfinite(float(v))] + if not clean: + return {"n": 0} + s = sorted(clean) + mean = statistics.fmean(s) + stdev = statistics.pstdev(s) if len(s) > 1 else 0.0 + return { + "n": len(s), + "min": s[0], + "max": s[-1], + "max_abs": max(abs(x) for x in s), + "mean": mean, + "stddev": stdev, + "p50": percentile(s, 50), + "p90": percentile(s, 90), + "p95": percentile(s, 95), + "p99": percentile(s, 99), + "p99_5": percentile(s, 99.5), + "p99_9": percentile(s, 99.9), + "p1": percentile(s, 1), + "p5": percentile(s, 5), + } + + +def fmt_dist(d: dict[str, Any], unit: str = "ms") -> str: + if d.get("n", 0) == 0: + return "_n=0_" + keys = [ + "n", + "min", + "mean", + "stddev", + "p1", + "p5", + "p50", + "p90", + "p95", + "p99", + "p99_5", + "p99_9", + "max", + "max_abs", + ] + parts = [] + for k in keys: + if k not in d: + continue + v = d[k] + if k == "n": + parts.append(f"n={v}") + else: + parts.append(f"{k}={v:.6f}{unit}" if isinstance(v, float) else f"{k}={v}") + return ", ".join(parts) + + +def linear_slope(xs: list[float], ys: list[float]) -> float: + n = min(len(xs), len(ys)) + if n < 2: + return 0.0 + x = xs[:n] + y = ys[:n] + mx = statistics.fmean(x) + my = statistics.fmean(y) + num = sum((a - mx) * (b - my) for a, b in zip(x, y)) + den = sum((a - mx) ** 2 for a in x) + return 0.0 if den == 0 else num / den + + +def load_jsonl(path: Path) -> list[dict]: + rows = [] + if not path.exists(): + return rows + with path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def load_csv(path: Path) -> list[dict]: + if not path.exists(): + return [] + with path.open("r", encoding="utf-8", errors="replace", newline="") as f: + return list(csv.DictReader(f)) + + +def classify_invariant(inv: str) -> tuple[str, str]: + """Return (bucket, domain) where domain is production|harness|other.""" + s = (inv or "").lower() + if "next nominal schedule shifted" in s or "schedule shifted" in s: + return "phase_shift", "production" + if "retry reached server after original deadline" in s: + return "retry_after_deadline", "production" + if "window" in s and ("wrong" in s or "correction" in s or "gap" in s): + return "window_correction", "production" + if "duplicate logical" in s or "duplicate cycleconfirmed" in s: + return "duplicate_logical_ping", "production" + if "false misseddeadline" in s or "false unknown" in s: + return "observer_wrong_state", "production" + if "hard-stop" in s or "graceful" in s and "wrong state" in s: + return "observer_wrong_state", "production" + if "first request was sent" in s or "did not reach send" in s: + return "fault_not_armed", "harness" + if "retry did not reach" in s: + return "no_retry_or_wrong_cycle", "harness" + if "cycle not confirmed" in s or "no cycle start" in s: + return "cycle_confirm", "harness" + if "querypeerreceive" in s or "checkpoint" in s: + return "query", "harness" + if "reporting/harness" in s or "graceful restart" in s: + return "reporting", "harness" + if "observer saw the wrong state" in s: + return "observer_wrong_state", "production" + return "other", "other" + + +PROD_BUCKETS = { + "retry_after_deadline", + "phase_shift", + "window_correction", + "observer_wrong_state", + "duplicate_logical_ping", + "other_production", +} + + +def margin_buckets(margins: list[float]) -> dict[str, int]: + edges = [ + (">100_before", lambda m: m > 100), + ("50_100_before", lambda m: 50 < m <= 100), + ("20_50_before", lambda m: 20 < m <= 50), + ("10_20_before", lambda m: 10 < m <= 20), + ("5_10_before", lambda m: 5 < m <= 10), + ("1_5_before", lambda m: 1 < m <= 5), + ("0_1_before", lambda m: 0 <= m <= 1), + ("after_deadline", lambda m: m < 0), + ] + out = {n: 0 for n, _ in edges} + out["no_retry"] = 0 + for m in margins: + placed = False + for n, pred in edges: + if pred(m): + out[n] += 1 + placed = True + break + if not placed: + out["after_deadline"] += 1 + return out + + +def seq_group(seq: str) -> str: + s = (seq or "").lower() + if s.startswith("baseline") or s == "none": + return "baseline" + if "consecutive" in s: + return "consecutive" + if "alternat" in s: + return "alternating" + if "periodic" in s or s.startswith("every"): + return "periodic" + if "random" in s: + return "random" + if "request-loss" in s or "response-loss" in s: + return "isolated" + if "boundary" in s: + return "boundary" + return s or "other" + + +def main() -> int: + status = json.loads((ROOT / "status.json").read_text(encoding="utf-8")) + shards_meta = [] + all_jsonl: list[dict] = [] + all_csv: list[dict] = [] + all_phase: list[dict] = [] + all_win: list[dict] = [] + all_obs: list[dict] = [] + assertion_failures: list[dict] = [] + + for shard_dir in sorted(RUNS.iterdir()): + if not shard_dir.is_dir(): + continue + name = shard_dir.name + transport = "tcp" if name.startswith("tcp") else "udp" + st = {} + if (shard_dir / "shard-status.json").exists(): + st = json.loads((shard_dir / "shard-status.json").read_text(encoding="utf-8")) + meta = {} + if (shard_dir / "shard-meta.json").exists(): + try: + meta = json.loads((shard_dir / "shard-meta.json").read_text(encoding="utf-8")) + except json.JSONDecodeError: + pass + stdout = ( + (shard_dir / "stdout.log").read_text(encoding="utf-8", errors="replace") + if (shard_dir / "stdout.log").exists() + else "" + ) + budget_exhausted = "budget exhausted" in stdout + has_report = (shard_dir / "report.md").exists() + semantic_fail = "FAIL phase-preservation" in stdout + semantic_pass = "PASS phase-preservation" in stdout + corrected_state = ( + "budget_exhausted_semantic_fail" + if budget_exhausted and has_report and semantic_fail + else "budget_exhausted_semantic_pass" + if budget_exhausted and has_report and semantic_pass + else "budget_exhausted_with_report" + if budget_exhausted and has_report + else "report_present" + if has_report + else "incomplete" + ) + # Not a process crash if report + budget exhaustion present. + is_real_crash = (not has_report) and (not budget_exhausted) + + jsonl = load_jsonl(shard_dir / "samples.jsonl") + csv_rows = load_csv(shard_dir / "samples.csv") + phase_rows = load_csv(shard_dir / "phase-error-by-cycle.csv") + win_rows = load_csv(shard_dir / "window-corrections.csv") + obs_rows = load_csv(shard_dir / "observer-query-results.csv") + failed = [] + if (shard_dir / "failed-cases.json").exists(): + try: + failed = json.loads( + (shard_dir / "failed-cases.json").read_text(encoding="utf-8") + ) + except Exception: + failed = [] + + for r in jsonl: + r["shard"] = name + r.setdefault("transport", transport) + all_jsonl.append(r) + for r in csv_rows: + r["shard"] = name + r["transport"] = transport + all_csv.append(r) + for r in phase_rows: + r["shard"] = name + r["transport"] = transport + all_phase.append(r) + for r in win_rows: + r["shard"] = name + r["transport"] = transport + all_win.append(r) + for r in obs_rows: + r["shard"] = name + r["transport"] = transport + all_obs.append(r) + + # Prefer aggregate-style classification from per-cycle failure strings, + # then apply instrumentation corrections (bogus Tn / control-only state). + def refine(bucket: str, domain: str, inv: str, sample: dict) -> tuple[str, str]: + tn = sample.get("Tn") + if tn is None: + tn = sample.get("scheduled_nominal_us") or sample.get("scheduled_nominal") + lpid = sample.get("logical_ping_id") + try: + tn_v = float(tn) if tn is not None else None + except (TypeError, ValueError): + tn_v = None + try: + lpid_v = int(lpid) if lpid is not None else None + except (TypeError, ValueError): + lpid_v = None + # Bogus empty cycle start: Tn==0 / logical_ping_id==0 with huge phase error + if bucket == "phase_shift" and ( + tn_v == 0 or lpid_v == 0 or sample.get("cycle_anchor_us") == 0 + ): + return "bogus_cycle_trace", "harness" + if "hard-stop" in (inv or "").lower() or "graceful" in (inv or "").lower(): + # Control-case expectation miss is not a live observer false-state. + return "control_case", "harness" + if bucket == "no_retry_or_wrong_cycle": + consumed = sample.get("fault_consumed") + if consumed in (0, "0", None) or ( + sample.get("fault_armed") in (1, "1") and consumed not in (1, "1") + ): + return "fault_not_consumed_no_retry", "harness" + retry = sample.get("retry_actual_send_time") or sample.get("retry_send_us") + if retry in (None, "", "null"): + return "no_retry", "harness" # treat as harness unless proven otherwise + return "retry_not_confirmed", "harness" + return bucket, domain + + for r in jsonl: + for inv in r.get("failures") or []: + bucket, domain = classify_invariant(str(inv)) + bucket, domain = refine(bucket, domain, str(inv), r) + assertion_failures.append( + { + "shard": name, + "transport": transport, + "cycle_index": r.get("cycle_index"), + "logical_ping_id": r.get("logical_ping_id"), + "fault_type": r.get("fault_type"), + "sequence": r.get("sequence") if "sequence" in r else None, + "invariant": inv, + "bucket": bucket, + "domain": domain, + "fault_armed": r.get("fault_armed"), + "fault_consumed": r.get("fault_consumed"), + "scheduled_phase_error_ms": r.get("scheduled_phase_error_ms"), + "estimated_server_margin_ms": r.get("estimated_server_margin_ms"), + "Tn": r.get("Tn"), + "sample": r, + } + ) + + for item in failed: + inv = item.get("invariant") or "" + bucket, domain = classify_invariant(str(inv)) + bucket, domain = refine(bucket, domain, str(inv), item) + assertion_failures.append( + { + "shard": name, + "transport": transport, + "cycle_index": item.get("cycle", item.get("cycle_index")), + "logical_ping_id": item.get("logical_ping_id"), + "fault_type": item.get("fault_type"), + "sequence": item.get("sequence"), + "invariant": inv, + "bucket": bucket, + "domain": domain, + "fault_armed": item.get("fault_armed"), + "fault_consumed": item.get("fault_consumed"), + "scheduled_phase_error_ms": item.get("scheduled_phase_error_ms"), + "estimated_server_margin_ms": item.get( + "retry_server_margin_ms", item.get("estimated_server_margin_ms") + ), + "Tn": item.get("Tn") or item.get("scheduled_nominal_us"), + "sample": item, + "from_failed_cases_json": True, + } + ) + + elapsed = None + if st.get("started_utc") and st.get("finished_utc"): + # approximate from budget + elapsed = st.get("budget_sec") + # Prefer stdout elapsed + for ln in stdout.splitlines(): + if "budget exhausted after" in ln and "elapsed_ms=" in ln: + try: + elapsed = int(ln.split("elapsed_ms=")[1].split()[0]) / 1000.0 + except Exception: + pass + + shards_meta.append( + { + "shard": name, + "transport": transport, + "seed": st.get("seed") or meta.get("seed"), + "budget_sec": st.get("budget_sec"), + "elapsed_sec": elapsed, + "cycles": len(jsonl), + "original_runner_state": st.get("state"), + "corrected_state": corrected_state, + "real_crash": is_real_crash, + "has_report": has_report, + "budget_exhausted": budget_exhausted, + "semantic_fail": semantic_fail, + "one_way_estimate_us": meta.get("one_way_estimate_us"), + "warmup_min_rtt_ms": meta.get("warmup_min_rtt_ms"), + "warmup_p99_rtt_ms": meta.get("warmup_p99_rtt_ms"), + } + ) + + # Deduplicate assertions: same shard+cycle+invariant + seen_assert = set() + uniq_assertions = [] + for a in assertion_failures: + key = (a["shard"], a.get("cycle_index"), a.get("invariant")) + if key in seen_assert: + continue + seen_assert.add(key) + uniq_assertions.append(a) + assertion_failures = uniq_assertions + + # Enrich sequence from csv when missing + csv_index = {(r["shard"], pi(r.get("cycle_index"))): r for r in all_csv} + phase_index = {(r["shard"], pi(r.get("cycle_index"))): r for r in all_phase} + win_index = {(r["shard"], pi(r.get("cycle_index"))): r for r in all_win} + for a in assertion_failures: + ci = pi(a.get("cycle_index")) + key = (a["shard"], ci) + if a.get("sequence") is None and key in csv_index: + a["sequence"] = csv_index[key].get("sequence") + if a.get("fault_consumed") is None and key in csv_index: + a["fault_consumed"] = pi(csv_index[key].get("fault_consumed")) + a["fault_armed"] = pi(csv_index[key].get("fault_armed")) + + # Also load original aggregate failures for the canonical 360/2725 split. + orig_failures = [] + orig_path = AGG / "failures.json" + if orig_path.exists(): + try: + orig_failures = json.loads(orig_path.read_text(encoding="utf-8")) + except Exception: + orig_failures = [] + orig_class_counts = Counter(f.get("failure_class") for f in orig_failures) + orig_prod = [f for f in orig_failures if str(f.get("failure_class", "")).startswith("PRODUCTION_")] + orig_harness = [f for f in orig_failures if str(f.get("failure_class", "")).startswith("HARNESS_")] + + # Correct original PRODUCTION_PHASE with Tn==0 / logical_ping_id==0 to harness instrumentation. + corrected_orig_prod = [] + moved_to_harness = [] + for f in orig_prod: + s = f.get("sample") or {} + tn = s.get("Tn") or s.get("scheduled_nominal_us") or f.get("Tn") + lpid = s.get("logical_ping_id") if "logical_ping_id" in s else f.get("logical_ping_id") + inv = str(f.get("invariant") or "") + try: + tn_v = float(tn) if tn is not None else None + except (TypeError, ValueError): + tn_v = None + try: + lpid_v = int(lpid) if lpid is not None else None + except (TypeError, ValueError): + lpid_v = None + if f.get("failure_class") == "PRODUCTION_PHASE" and (tn_v == 0 or lpid_v == 0): + moved_to_harness.append({**f, "corrected_class": "HARNESS_BOGUS_CYCLE_TRACE"}) + continue + if f.get("failure_class") == "PRODUCTION_STATE" and ( + "hard-stop" in inv.lower() or "graceful" in inv.lower() + ): + moved_to_harness.append({**f, "corrected_class": "HARNESS_CONTROL_CASE"}) + continue + corrected_orig_prod.append(f) + + # ---------------- Phase analysis ---------------- + def phase_stats(transport: str) -> dict[str, Any]: + rows = [r for r in all_jsonl if r.get("transport") == transport] + errs = [] + for r in rows: + e = pf(r.get("scheduled_phase_error_ms")) + if e is None: + # fall back to phase csv + pr = phase_index.get((r["shard"], pi(r.get("cycle_index")))) + if pr: + eu = pf(pr.get("scheduled_phase_error_us")) + if eu is not None: + e = eu / 1000.0 + if e is not None: + errs.append((r, e)) + vals = [e for _, e in errs] + d = dist(vals) + exact_zero = sum(1 for e in vals if e == 0.0) + nonzero = sum(1 for e in vals if e != 0.0) + thresholds = [0.1, 0.5, 1, 5, 10, 50, 100, 250] + thr = {f"abs_gt_{t}_ms": sum(1 for e in vals if abs(e) > t) for t in thresholds} + outliers = [(r, e) for r, e in errs if abs(e) > 1.0] + return { + "sample_count": len(vals), + "exact_zero": exact_zero, + "nonzero": nonzero, + "pct_exact_zero": (100.0 * exact_zero / len(vals)) if vals else float("nan"), + "dist": d, + "thresholds": thr, + "outliers_gt_1ms": outliers, + } + + tcp_phase = phase_stats("tcp") + udp_phase = phase_stats("udp") + + # TCP 462.8 event — find max abs + tcp_outliers = sorted(tcp_phase["outliers_gt_1ms"], key=lambda x: abs(x[1]), reverse=True) + big_event = None + big_timeline = None + if tcp_outliers: + r0, e0 = tcp_outliers[0] + shard = r0["shard"] + ci = int(r0["cycle_index"]) + # Build index of jsonl by shard + by_c = { + int(r["cycle_index"]): r + for r in all_jsonl + if r.get("shard") == shard and r.get("cycle_index") is not None + } + window_cycles = list(range(ci - 5, ci + 11)) + timeline = [] + for j in window_cycles: + if j not in by_c: + continue + rr = by_c[j] + cr = csv_index.get((shard, j), {}) + pr = phase_index.get((shard, j), {}) + wr = win_index.get((shard, j), {}) + obs = [ + o + for o in all_obs + if o.get("shard") == shard and pi(o.get("cycle_index")) == j + ] + timeline.append( + { + "cycle_index": j, + "is_event": j == ci, + "fault_type": rr.get("fault_type") or cr.get("fault_type"), + "sequence": cr.get("sequence"), + "scheduled_phase_error_ms": pf(rr.get("scheduled_phase_error_ms")), + "expected_nominal": rr.get("expected_nominal_time"), + "scheduled_nominal": rr.get("Tn") or cr.get("scheduled_nominal"), + "actual_send": rr.get("actual_first_attempt_send_time"), + "retry_decision": cr.get("retry_decision") or cr.get("attempt_timeout"), + "retry_send": rr.get("retry_actual_send_time"), + "estimated_server_receive": rr.get("estimated_server_receive"), + "original_deadline": rr.get("original_deadline"), + "estimated_server_margin_ms": rr.get("estimated_server_margin_ms"), + "next_nominal_phase_error_ms": rr.get("next_nominal_phase_error_ms"), + "window": { + "cur_start_delta_ms": pf(wr.get("current_window_start_delta_ms")), + "cur_end_delta_ms": pf(wr.get("current_window_end_delta_ms")), + "cur_dur_delta_ms": pf(wr.get("current_window_duration_delta_ms")), + "next_start_delta_ms": pf(wr.get("next_window_start_delta_ms")), + "next_nominal_phase_delta_ms": pf( + wr.get("next_nominal_phase_delta_ms") + ), + }, + "observers": [ + { + "checkpoint": pi(o.get("checkpoint")), + "state": pi(o.get("state")), + "expected_state": pi(o.get("expected_state")), + "mismatch": pi(o.get("mismatch")), + "next_ping_delta_ms": pf(o.get("next_ping_delta_ms")), + } + for o in obs + ], + "phase_csv_error_us": pf(pr.get("scheduled_phase_error_us")), + "failures": rr.get("failures"), + } + ) + # Drift before/after + before = [ + (int(r["cycle_index"]), pf(r.get("scheduled_phase_error_ms")) or 0.0) + for r in all_jsonl + if r.get("shard") == shard and pi(r.get("cycle_index")) is not None and pi(r.get("cycle_index")) < ci + ] + after = [ + (int(r["cycle_index"]), pf(r.get("scheduled_phase_error_ms")) or 0.0) + for r in all_jsonl + if r.get("shard") == shard and pi(r.get("cycle_index")) is not None and pi(r.get("cycle_index")) > ci + ] + # Nature: check if subsequent cycles stay shifted + after_nonzero = sum(1 for _, e in after[:20] if abs(e) > 0.1) + after_exact_zero = sum(1 for _, e in after[:50] if e == 0.0) + nature = "one_cycle_transient" + if after and after_nonzero > 5 and after_exact_zero < len(after[:20]) * 0.5: + nature = "temporary_or_permanent_shift" + # Check if only event cycle nonzero among neighbors + neighbor_nz = [ + t + for t in timeline + if abs(t.get("scheduled_phase_error_ms") or 0) > 0.1 and not t["is_event"] + ] + if not neighbor_nz and abs(e0) > 1: + nature = "one_cycle_transient_or_instrumentation" + # Check expected vs scheduled delta matches phase error + exp = pf(r0.get("expected_nominal_time")) + sched = pf(r0.get("Tn")) + if exp is not None and sched is not None: + delta_ms = (sched - exp) / 1000.0 + else: + delta_ms = e0 + + # Instrumentation: Tn==0 / logical_ping_id==0 means missing cycle_anchor + lpid = r0.get("logical_ping_id") + if (sched == 0 or sched is None) or lpid in (0, "0"): + nature = "timestamp_instrumentation_error_bogus_cycle_trace" + + big_event = { + "shard": shard, + "cycle": ci, + "seed": r0.get("seed"), + "fault_type": r0.get("fault_type"), + "phase_error_ms": e0, + "expected_minus_scheduled_ms": delta_ms, + "nature": nature, + "slope_before_ms_per_cycle": linear_slope( + [float(c) for c, _ in before], [e for _, e in before] + ), + "slope_after_ms_per_cycle": linear_slope( + [float(c) for c, _ in after], [e for _, e in after] + ), + "after_50_exact_zero": after_exact_zero, + "after_20_nonzero_gt_0_1": after_nonzero, + "neighbor_nonzero_count": len(neighbor_nz), + } + big_timeline = timeline + + # ---------------- Interval analysis ---------------- + def interval_stats(transport: str) -> dict[str, Any]: + # Per shard, sort by cycle, use scheduled nominal + by_shard: dict[str, list[dict]] = defaultdict(list) + for r in all_jsonl: + if r.get("transport") != transport: + continue + by_shard[r["shard"]].append(r) + scheduled_intervals = [] + send_intervals = [] + for shard, rows in by_shard.items(): + rows = sorted(rows, key=lambda x: pi(x.get("cycle_index")) or 0) + for a, b in zip(rows, rows[1:]): + sa = pf(a.get("Tn")) + sb = pf(b.get("Tn")) + if sa is not None and sb is not None: + scheduled_intervals.append((sb - sa) / 1000.0) + fa = pf(a.get("actual_first_attempt_send_time")) + fb = pf(b.get("actual_first_attempt_send_time")) + # only contiguous + if a.get("contiguous_cycle") == 1 or ( + pi(b.get("cycle_index")) is not None + and pi(a.get("cycle_index")) is not None + and pi(b.get("cycle_index")) - pi(a.get("cycle_index")) == 1 + ): + if fa is not None and fb is not None: + send_intervals.append((fb - fa) / 1000.0) + sched_err = [v - 1000.0 for v in scheduled_intervals] + send_err = [v - 1000.0 for v in send_intervals] + exact_1000 = sum(1 for v in scheduled_intervals if abs(v - 1000.0) < 1e-9) + # also count within 0.001 us quantization - treat as exact if == 1000 + exact_1000 = sum(1 for v in scheduled_intervals if v == 1000.0) + thr = { + "gt_0_1": sum(1 for e in sched_err if abs(e) > 0.1), + "gt_1": sum(1 for e in sched_err if abs(e) > 1), + "gt_10": sum(1 for e in sched_err if abs(e) > 10), + "gt_100": sum(1 for e in sched_err if abs(e) > 100), + } + return { + "scheduled_interval": dist(scheduled_intervals), + "scheduled_error": dist(sched_err), + "exact_1000": exact_1000, + "scheduled_thresholds": thr, + "actual_send_interval": dist(send_intervals), + "actual_send_error": dist(send_err), + } + + tcp_iv = interval_stats("tcp") + udp_iv = interval_stats("udp") + + # ---------------- Loss analysis ---------------- + def loss_analysis(transport: str, fault: str) -> dict[str, Any]: + rows = [ + r + for r in all_csv + if r.get("transport") == transport and r.get("fault_type") == fault + ] + # Prefer csv for richer fields; also merge jsonl + jmap = { + (r["shard"], pi(r.get("cycle_index"))): r + for r in all_jsonl + if r.get("transport") == transport and r.get("fault_type") == fault + } + total = len(rows) + armed = sum(1 for r in rows if pi(r.get("fault_armed")) == 1) + consumed = [r for r in rows if pi(r.get("fault_consumed")) == 1] + not_consumed = [r for r in rows if pi(r.get("fault_armed")) == 1 and pi(r.get("fault_consumed")) != 1] + margins = [] + send_before = send_after = no_retry = 0 + after_deadline = 0 + before_deadline = 0 + by_seq = Counter() + by_shard = Counter() + recovered = 0 + for r in rows: + by_seq[seq_group(r.get("sequence") or "")] += 1 + by_shard[r.get("shard")] += 1 + for r in consumed: + jr = jmap.get((r["shard"], pi(r.get("cycle_index"))), {}) + m = pf(r.get("estimated_server_margin_ms")) + if m is None: + m = pf(jr.get("estimated_server_margin_ms")) + retry_send = pf(r.get("retry_actual_send")) or pf(jr.get("retry_actual_send_time")) + tn = pf(r.get("Tn")) or pf(jr.get("Tn")) + if retry_send is None: + no_retry += 1 + continue + if tn is not None: + if retry_send <= tn: + send_before += 1 + else: + send_after += 1 + if m is None: + continue + margins.append(m) + if m >= 0: + before_deadline += 1 + recovered += 1 + else: + after_deadline += 1 + buckets = margin_buckets(margins) + buckets["no_retry"] = no_retry + # first request checks + if fault == "request-loss": + first_ok = sum(1 for r in consumed if pi(r.get("first_request_sent")) == 0) + first_bad = sum(1 for r in rows if pi(r.get("fault_armed")) == 1 and pi(r.get("first_request_sent")) == 1) + else: + first_ok = sum(1 for r in consumed if pi(r.get("first_request_sent")) == 1) + first_bad = sum(1 for r in rows if pi(r.get("fault_armed")) == 1 and pi(r.get("first_request_sent")) == 0) + usable = len(consumed) + return { + "total_injected_or_labeled": total, + "fault_armed": armed, + "fault_consumed": usable, + "harness_not_consumed": len(not_consumed), + "usable_success_estimated_before_Tn": recovered, + "usable_success_rate_pct": (100.0 * recovered / usable) if usable else float("nan"), + "send_before_Tn": send_before, + "send_after_Tn": send_after, + "send_before_pct": (100.0 * send_before / usable) if usable else float("nan"), + "estimated_after_deadline": after_deadline, + "no_retry_among_consumed": no_retry, + "margins": dist(margins), + "margin_buckets": buckets, + "by_sequence": dict(by_seq), + "by_shard": dict(by_shard), + "first_attempt_correct": first_ok, + "first_attempt_incorrect_among_armed": first_bad, + } + + tcp_req = loss_analysis("tcp", "request-loss") + tcp_resp = loss_analysis("tcp", "response-loss") + udp_req = loss_analysis("udp", "request-loss") + udp_resp = loss_analysis("udp", "response-loss") + + # ---------------- Window analysis ---------------- + def window_stats(transport: str, fault: str) -> dict[str, Any]: + rows = [ + r + for r in all_win + if r.get("transport") == transport and r.get("fault_type") == fault + ] + fields = [ + "current_window_start_delta_ms", + "current_window_end_delta_ms", + "current_window_duration_delta_ms", + "next_window_start_delta_ms", + "next_nominal_phase_delta_ms", + ] + out = {f: dist([pf(r.get(f)) for r in rows]) for f in fields} + # anomalies: next nominal phase delta abs > 1ms + anomalies = [] + for r in rows: + nd = pf(r.get("next_nominal_phase_delta_ms")) + if nd is not None and abs(nd) > 1.0: + anomalies.append( + { + "shard": r["shard"], + "cycle": pi(r.get("cycle_index")), + "fault_type": fault, + "next_nominal_phase_delta_ms": nd, + "cur_dur_delta_ms": pf(r.get("current_window_duration_delta_ms")), + "next_start_delta_ms": pf(r.get("next_window_start_delta_ms")), + } + ) + out["next_nominal_phase_anomaly_gt_1ms"] = anomalies[:50] + out["next_nominal_phase_anomaly_count"] = len(anomalies) + expanded = sum( + 1 + for r in rows + if (pf(r.get("current_window_duration_delta_ms")) or 0) > 0.5 + ) + shrunk = sum( + 1 + for r in rows + if (pf(r.get("current_window_duration_delta_ms")) or 0) < -0.5 + ) + out["window_expanded"] = expanded + out["window_shrunk"] = shrunk + out["n_rows"] = len(rows) + return out + + win_tcp_req = window_stats("tcp", "request-loss") + win_tcp_resp = window_stats("tcp", "response-loss") + win_udp_req = window_stats("udp", "request-loss") + win_udp_resp = window_stats("udp", "response-loss") + + # ---------------- Observer ---------------- + def observer_stats() -> dict[str, Any]: + by = defaultdict(lambda: Counter()) + mismatches = [] + false_u = 0 + false_md = 0 + for o in all_obs: + ck = pi(o.get("checkpoint")) or 0 + tr = o.get("transport") + ft = o.get("fault_type") or "none" + st = pi(o.get("state")) + exp = pi(o.get("expected_state")) + mm = pi(o.get("mismatch")) == 1 + by[(tr, ft, ck)]["n"] += 1 + if st == 0: + by[(tr, ft, ck)]["Expected"] += 1 + elif st == 1: + by[(tr, ft, ck)]["MissedDeadline"] += 1 + elif st == 2: + by[(tr, ft, ck)]["Unknown"] += 1 + else: + by[(tr, ft, ck)]["invalid"] += 1 + if mm: + by[(tr, ft, ck)]["mismatch"] += 1 + mismatches.append(o) + # false live: state MD/Unknown when expected Expected (0) on live checkpoints + if exp == 0 and st == 1: + false_md += 1 + if exp == 0 and st == 2: + false_u += 1 + return { + "by_transport_fault_checkpoint": { + f"{tr}|{ft}|ckpt{ck}": dict(c) for (tr, ft, ck), c in sorted(by.items()) + }, + "false_live_Unknown": false_u, + "false_live_MissedDeadline": false_md, + "mismatch_count": len(mismatches), + "total_queries": len(all_obs), + } + + obs_stats = observer_stats() + + # ---------------- Failure breakdown (canonical = corrected original classes) ---------------- + def root_key(a: dict) -> tuple: + return (a.get("transport"), a.get("shard"), pi(a.get("cycle_index")), a.get("bucket")) + + # Use original aggregate failures.json as the assertion inventory (360/2725), + # then apply instrumentation/control corrections. + prod_assertions = [] + for f in corrected_orig_prod: + cls = f.get("failure_class") + bucket = { + "PRODUCTION_RETRY_ESTIMATED_LATE_ARRIVAL": "retry_after_deadline", + "PRODUCTION_PHASE": "phase_shift", + "PRODUCTION_STATE": "observer_wrong_state", + "PRODUCTION_WINDOW": "window_correction", + }.get(cls, "other") + s = f.get("sample") or {} + prod_assertions.append( + { + "shard": f.get("shard"), + "transport": f.get("transport"), + "cycle_index": f.get("cycle_index") or s.get("cycle_index") or f.get("cycle"), + "logical_ping_id": s.get("logical_ping_id") or f.get("logical_ping_id"), + "fault_type": s.get("fault_type") or f.get("fault_type"), + "sequence": s.get("sequence") or f.get("sequence"), + "invariant": f.get("invariant"), + "bucket": bucket, + "domain": "production", + "failure_class": cls, + "estimated_server_margin_ms": s.get("estimated_server_margin_ms") + or s.get("retry_server_margin_ms"), + "scheduled_phase_error_ms": s.get("scheduled_phase_error_ms"), + "retry_actual_send_time": s.get("retry_actual_send_time") or s.get("retry_send_us"), + "Tn": s.get("Tn") or s.get("original_deadline"), + "sample": s or f, + } + ) + + harness_assertions = [] + for f in orig_harness: + harness_assertions.append( + { + "shard": f.get("shard"), + "transport": f.get("transport"), + "cycle_index": f.get("cycle_index"), + "invariant": f.get("invariant"), + "bucket": str(f.get("failure_class") or "").replace("HARNESS_", "").lower(), + "domain": "harness", + "failure_class": f.get("failure_class"), + "sample": f.get("sample") or f, + } + ) + for f in moved_to_harness: + harness_assertions.append( + { + "shard": f.get("shard"), + "transport": f.get("transport"), + "cycle_index": f.get("cycle_index"), + "invariant": f.get("invariant"), + "bucket": str(f.get("corrected_class") or "moved"), + "domain": "harness", + "failure_class": f.get("corrected_class"), + "sample": f.get("sample") or f, + "moved_from": f.get("failure_class"), + } + ) + other_assertions = [ + a for a in assertion_failures if a["domain"] == "other" + ] + + # Also count late events where client still sent before Tn + late_send_before = 0 + late_send_after = 0 + late_margins = [] + for a in prod_assertions: + if a["bucket"] != "retry_after_deadline": + continue + m = pf(a.get("estimated_server_margin_ms")) + if m is not None: + late_margins.append(m) + rs = pf(a.get("retry_actual_send_time")) + tn = pf(a.get("Tn")) + if rs is not None and tn is not None: + if rs <= tn: + late_send_before += 1 + else: + late_send_after += 1 + + prod_by_bucket = Counter(a["bucket"] for a in prod_assertions) + prod_by_tr_bucket = Counter((a["transport"], a["bucket"]) for a in prod_assertions) + unique_prod_events = {} + for a in prod_assertions: + k = root_key(a) + if k not in unique_prod_events: + unique_prod_events[k] = a + unique_prod_by_bucket = Counter(a["bucket"] for a in unique_prod_events.values()) + unique_prod_by_tr = Counter( + (a["transport"], a["bucket"]) for a in unique_prod_events.values() + ) + + harness_by_bucket = Counter( + a.get("failure_class") or a.get("bucket") for a in harness_assertions + ) + + # Map buckets to requested table categories + table_map = { + "retry_after_deadline": "retry after deadline", + "no_retry": "no retry", + "phase_shift": "phase shift", + "window_correction": "window correction", + "observer_wrong_state": "observer wrong state", + "duplicate_logical_ping": "duplicate logical ping", + "other": "other production semantic", + "other_production": "other production semantic", + } + + # Representative / worst timelines for each prod bucket + def pick_reps(bucket: str) -> dict[str, Any]: + items = [a for a in unique_prod_events.values() if a["bucket"] == bucket] + if not items: + return {} + # worst by abs margin or phase + def score(a): + m = pf(a.get("estimated_server_margin_ms")) + p = pf(a.get("scheduled_phase_error_ms")) + if bucket == "retry_after_deadline" and m is not None: + return -m # most late + if bucket == "phase_shift" and p is not None: + return abs(p) + return 0 + + items_sorted = sorted(items, key=score, reverse=True) + worst = items_sorted[0] + rep = items_sorted[len(items_sorted) // 2] + + def brief(a): + return { + "shard": a["shard"], + "cycle": a.get("cycle_index"), + "transport": a["transport"], + "fault_type": a.get("fault_type"), + "sequence": a.get("sequence"), + "invariant": a.get("invariant"), + "estimated_server_margin_ms": a.get("estimated_server_margin_ms"), + "scheduled_phase_error_ms": a.get("scheduled_phase_error_ms"), + } + + # cluster by shard + by_shard = Counter(a["shard"] for a in items) + return { + "count_unique": len(items), + "by_shard": dict(by_shard.most_common(10)), + "representative": brief(rep), + "worst": brief(worst), + } + + prod_category_details = { + table_map.get(b, b): pick_reps(b) for b in unique_prod_by_bucket + } + + # ---------------- Shard stability ---------------- + shard_reports = [] + for sm in shards_meta: + name = sm["shard"] + tr = sm["transport"] + rows = [r for r in all_jsonl if r.get("shard") == name] + errs = [pf(r.get("scheduled_phase_error_ms")) for r in rows] + errs = [e for e in errs if e is not None] + d = dist(errs) + prod_n = sum(1 for a in prod_assertions if a["shard"] == name) + harness_n = sum(1 for a in harness_assertions if a["shard"] == name) + # success among consumed + def shard_loss(fault): + rs = [ + r + for r in all_csv + if r.get("shard") == name and r.get("fault_type") == fault and pi(r.get("fault_consumed")) == 1 + ] + if not rs: + return {"consumed": 0, "success_pct": float("nan")} + ok = 0 + for r in rs: + m = pf(r.get("estimated_server_margin_ms")) + if m is not None and m >= 0: + ok += 1 + return {"consumed": len(rs), "success_pct": 100.0 * ok / len(rs)} + + false_obs = sum( + 1 + for o in all_obs + if o.get("shard") == name + and pi(o.get("expected_state")) == 0 + and pi(o.get("state")) in (1, 2) + ) + shard_reports.append( + { + **sm, + "production_assertion_failures": prod_n, + "harness_assertion_failures": harness_n, + "phase_max_abs": d.get("max_abs"), + "phase_p99": d.get("p99"), + "phase_exact_zero_pct": (100.0 * sum(1 for e in errs if e == 0.0) / len(errs)) + if errs + else float("nan"), + "request_loss": shard_loss("request-loss"), + "response_loss": shard_loss("response-loss"), + "observer_false_state": false_obs, + } + ) + + # thirds of wall time using shard order + ordered = sorted(shard_reports, key=lambda s: s["shard"]) + # better: by started time from status + st_map = {s["shard"]: s for s in status.get("shards", [])} + ordered = sorted( + shard_reports, + key=lambda s: st_map.get(s["shard"], {}).get("started_utc") or s["shard"], + ) + n = len(ordered) + thirds = { + "first": ordered[: n // 3], + "middle": ordered[n // 3 : 2 * n // 3], + "last": ordered[2 * n // 3 :], + } + + def third_summary(items): + cycles = sum(i["cycles"] for i in items) + prod = sum(i["production_assertion_failures"] for i in items) + harness = sum(i["harness_assertion_failures"] for i in items) + phases = [i["phase_max_abs"] for i in items if i.get("phase_max_abs") is not None] + return { + "shards": [i["shard"] for i in items], + "cycles": cycles, + "production_assertions": prod, + "harness_assertions": harness, + "phase_max_abs_max": max(phases) if phases else None, + "phase_max_abs_median": statistics.median(phases) if phases else None, + } + + thirds_summary = {k: third_summary(v) for k, v in thirds.items()} + + # ---------------- Corrected usable sample rate ---------------- + total_cycles = len(all_jsonl) + harness_invalid_cycles = set() + for a in harness_assertions: + if a["bucket"] in { + "fault_not_armed", + "fault_not_consumed_no_retry", + "cycle_confirm", + "query", + "reporting", + }: + harness_invalid_cycles.add((a["shard"], pi(a.get("cycle_index")))) + usable_cycles = total_cycles - len( + {k for k in harness_invalid_cycles if k[1] is not None} + ) + # For loss reliability, use consumed only + usable_loss = ( + tcp_req["fault_consumed"] + + tcp_resp["fault_consumed"] + + udp_req["fault_consumed"] + + udp_resp["fault_consumed"] + ) + labeled_loss = ( + tcp_req["total_injected_or_labeled"] + + tcp_resp["total_injected_or_labeled"] + + udp_req["total_injected_or_labeled"] + + udp_resp["total_injected_or_labeled"] + ) + + # ---------------- Verdict ---------------- + # Phase: does schedule accumulate drift? + tcp_drift = linear_slope( + [float(pi(r.get("cycle_index")) or 0) for r in all_jsonl if r.get("transport") == "tcp"], + [ + pf(r.get("scheduled_phase_error_ms")) or 0.0 + for r in all_jsonl + if r.get("transport") == "tcp" + ], + ) + udp_drift = linear_slope( + [float(pi(r.get("cycle_index")) or 0) for r in all_jsonl if r.get("transport") == "udp"], + [ + pf(r.get("scheduled_phase_error_ms")) or 0.0 + for r in all_jsonl + if r.get("transport") == "udp" + ], + ) + + unique_retry_late = unique_prod_by_bucket.get("retry_after_deadline", 0) + unique_phase = unique_prod_by_bucket.get("phase_shift", 0) + unique_prod_total = len(unique_prod_events) + + false_obs_ok = ( + obs_stats["false_live_Unknown"] == 0 and obs_stats["false_live_MissedDeadline"] == 0 + ) + # Bogus Tn=0 outliers and slopes ~0 => no cumulative schedule drift. + phase_cumulative_ok = abs(tcp_drift) < 0.01 and abs(udp_drift) < 0.01 + if big_event and "instrumentation" in str(big_event.get("nature", "")): + phase_cumulative_ok = phase_cumulative_ok and True + elif big_event and str(big_event.get("nature", "")).startswith("one_cycle"): + phase_cumulative_ok = phase_cumulative_ok and True + + # Verdict: + # - Phase does not accumulate drift (p99=0, slopes~0, bogus Tn=0 outliers are harness). + # - Estimated late arrivals among consumed faults => PARTIAL, not FAIL. + # - FAIL only if cumulative phase drift / live false Unknown|MD / real phase_shift remain. + real_phase_shift = unique_phase # after moving Tn==0 events to harness + if ( + phase_cumulative_ok + and real_phase_shift == 0 + and false_obs_ok + and unique_prod_total == 0 + ): + verdict = "PASS" + verdict_why = ( + "Nominal 1s phase preserved with no unique production failures " + "after correcting instrumentation/control mislabels." + ) + elif phase_cumulative_ok and real_phase_shift == 0 and false_obs_ok: + verdict = "PARTIAL" + verdict_why = ( + "1s nominal phase does not accumulate drift; Alice shows no false live " + f"Unknown/MissedDeadline; TCP |phase|>1ms outliers are bogus cycle traces " + f"(Tn=0). Remaining unique production root-events: {unique_prod_total}, " + f"dominated by {unique_retry_late} estimated-late-arrival events " + f"({late_send_before} still had client retry send before Tn; " + f"{late_send_after} sent after Tn). Mapping uses one_way=min_rtt/2." + ) + else: + verdict = "FAIL" + verdict_why = ( + "Phase accumulation, live observer false-state, or unreclassified " + f"phase_shift remains (unique_phase={real_phase_shift})." + ) + + # Preserve answers flags consistently with corrected classification + request_loss_preserves_phase = real_phase_shift == 0 + response_loss_preserves_phase = real_phase_shift == 0 + does_accumulate = not phase_cumulative_ok + + # Build production table + table_cats = [ + "retry after deadline", + "no retry", + "phase shift", + "window correction", + "observer wrong state", + "duplicate logical ping", + "other production semantic", + ] + prod_table = [] + for cat in table_cats: + # reverse map + buckets = [b for b, name in table_map.items() if name == cat] + tcp_c = sum(unique_prod_by_tr.get(("tcp", b), 0) for b in buckets) + udp_c = sum(unique_prod_by_tr.get(("udp", b), 0) for b in buckets) + # assertion counts + tcp_a = sum(prod_by_tr_bucket.get(("tcp", b), 0) for b in buckets) + udp_a = sum(prod_by_tr_bucket.get(("udp", b), 0) for b in buckets) + prod_table.append( + { + "category": cat, + "tcp_unique": tcp_c, + "udp_unique": udp_c, + "total_unique": tcp_c + udp_c, + "tcp_assertions": tcp_a, + "udp_assertions": udp_a, + "total_assertions": tcp_a + udp_a, + } + ) + + # Outlier timelines for all TCP >1ms + tcp_outlier_briefs = [] + for r, e in tcp_outliers: + tcp_outlier_briefs.append( + { + "shard": r["shard"], + "cycle": r.get("cycle_index"), + "seed": r.get("seed"), + "fault_type": r.get("fault_type"), + "phase_error_ms": e, + "Tn": r.get("Tn"), + "expected_nominal_time": r.get("expected_nominal_time"), + "next_nominal_phase_error_ms": r.get("next_nominal_phase_error_ms"), + "failures": r.get("failures"), + } + ) + + summary = { + "verdict": verdict, + "verdict_why": verdict_why, + "wall_time_sec": status.get("elapsed_sec"), + "corrected_shards": [ + { + "shard": s["shard"], + "transport": s["transport"], + "original_runner_state": s["original_runner_state"], + "corrected_state": s["corrected_state"], + "real_crash": s["real_crash"], + "cycles": s["cycles"], + "budget_exhausted": s["budget_exhausted"], + "semantic_fail": s["semantic_fail"], + } + for s in shards_meta + ], + "cycles": {"tcp": sum(1 for r in all_jsonl if r["transport"] == "tcp"), "udp": sum(1 for r in all_jsonl if r["transport"] == "udp")}, + "phase": { + "tcp": { + "exact_zero": tcp_phase["exact_zero"], + "nonzero": tcp_phase["nonzero"], + "pct_exact_zero": tcp_phase["pct_exact_zero"], + "dist": tcp_phase["dist"], + "thresholds": tcp_phase["thresholds"], + "drift_ms_per_cycle": tcp_drift, + "outliers_gt_1ms_count": len(tcp_outliers), + }, + "udp": { + "exact_zero": udp_phase["exact_zero"], + "nonzero": udp_phase["nonzero"], + "pct_exact_zero": udp_phase["pct_exact_zero"], + "dist": udp_phase["dist"], + "thresholds": udp_phase["thresholds"], + "drift_ms_per_cycle": udp_drift, + "outliers_gt_1ms_count": len(udp_phase["outliers_gt_1ms"]), + }, + }, + "tcp_462_event": big_event, + "tcp_462_timeline": big_timeline, + "tcp_outliers_gt_1ms": tcp_outlier_briefs, + "intervals": {"tcp": tcp_iv, "udp": udp_iv}, + "request_loss": {"tcp": tcp_req, "udp": udp_req}, + "response_loss": {"tcp": tcp_resp, "udp": udp_resp}, + "windows": { + "tcp_request": {k: v for k, v in win_tcp_req.items() if k != "next_nominal_phase_anomaly_gt_1ms"}, + "tcp_response": {k: v for k, v in win_tcp_resp.items() if k != "next_nominal_phase_anomaly_gt_1ms"}, + "udp_request": {k: v for k, v in win_udp_req.items() if k != "next_nominal_phase_anomaly_gt_1ms"}, + "udp_response": {k: v for k, v in win_udp_resp.items() if k != "next_nominal_phase_anomaly_gt_1ms"}, + "anomaly_counts": { + "tcp_request": win_tcp_req["next_nominal_phase_anomaly_count"], + "tcp_response": win_tcp_resp["next_nominal_phase_anomaly_count"], + "udp_request": win_udp_req["next_nominal_phase_anomaly_count"], + "udp_response": win_udp_resp["next_nominal_phase_anomaly_count"], + }, + }, + "observer": obs_stats, + "production_failures": { + "original_claimed": 360, + "original_class_counts": dict(orig_class_counts), + "moved_to_harness": len(moved_to_harness), + "moved_to_harness_detail": [ + { + "shard": f.get("shard"), + "cycle_index": f.get("cycle_index"), + "from": f.get("failure_class"), + "to": f.get("corrected_class"), + "invariant": f.get("invariant"), + } + for f in moved_to_harness + ], + "recomputed_assertions": len(prod_assertions), + "unique_root_events": unique_prod_total, + "by_bucket_assertions": dict(prod_by_bucket), + "by_bucket_unique": dict(unique_prod_by_bucket), + "late_client_send_before_Tn": late_send_before, + "late_client_send_after_Tn": late_send_after, + "late_margins": dist(late_margins), + "table": prod_table, + "category_details": prod_category_details, + }, + "harness_failures": { + "original_claimed": 2725, + "recomputed_assertions": len(harness_assertions), + "by_bucket": dict(harness_by_bucket), + "null_exitcode_misclassified_shards": sum( + 1 for s in shards_meta if s["original_runner_state"] == "crashed" and not s["real_crash"] + ), + "other_assertions": len(other_assertions), + }, + "usable_sample_rate": { + "total_cycles": total_cycles, + "cycles_with_harness_invalid_flags": len(harness_invalid_cycles), + "approx_usable_cycle_pct": (100.0 * usable_cycles / total_cycles) if total_cycles else 0, + "loss_labeled": labeled_loss, + "loss_fault_consumed": usable_loss, + "loss_consumed_pct": (100.0 * usable_loss / labeled_loss) if labeled_loss else 0, + }, + "shard_reports": shard_reports, + "thirds": thirds_summary, + "answers": { + "does_1s_schedule_accumulate_drift": does_accumulate, + "request_loss_preserves_phase": request_loss_preserves_phase, + "response_loss_preserves_phase": response_loss_preserves_phase, + "retries_reach_server_before_deadline": { + "note": "estimated mapping only (retry_send + one_way)", + "tcp_request_before_pct": tcp_req["usable_success_rate_pct"], + "tcp_response_before_pct": tcp_resp["usable_success_rate_pct"], + "udp_request_before_pct": udp_req["usable_success_rate_pct"], + "udp_response_before_pct": udp_resp["usable_success_rate_pct"], + "unique_estimated_late_events": unique_retry_late, + }, + "server_side_safety_margin": { + "tcp_request": tcp_req["margins"], + "tcp_response": tcp_resp["margins"], + "udp_request": udp_req["margins"], + "udp_response": udp_resp["margins"], + }, + "window_preserves_next_nominal": { + "tcp_req_anomaly_gt_1ms": win_tcp_req["next_nominal_phase_anomaly_count"], + "tcp_resp_anomaly_gt_1ms": win_tcp_resp["next_nominal_phase_anomaly_count"], + "udp_req_anomaly_gt_1ms": win_udp_req["next_nominal_phase_anomaly_count"], + "udp_resp_anomaly_gt_1ms": win_udp_resp["next_nominal_phase_anomaly_count"], + }, + "alice_false_states": { + "Unknown": obs_stats["false_live_Unknown"], + "MissedDeadline": obs_stats["false_live_MissedDeadline"], + }, + "tcp_462_cause": big_event, + "unique_real_production_failures": unique_prod_total, + "harness_only_assertions": len(harness_assertions), + }, + "recommended_next_test": None, # filled below + } + + # Recommended next test + if unique_retry_late > 0: + summary["recommended_next_test"] = { + "title": "Focused estimated-late-arrival stress (consumed request/response loss only)", + "why": ( + f"{unique_retry_late} unique PRODUCTION_RETRY_ESTIMATED_LATE_ARRIVAL events dominate " + "production failures; phase itself does not accumulate drift." + ), + "scope": [ + "TCP and UDP separately, ~30–60 minutes each", + "Only request-loss and response-loss with verified fault_consumed=1", + "No QueryNow on the timeout→retry critical path", + "Record retry_client_send vs Tn and estimated_server_receive vs Tn", + "Classify late-send vs late-estimated-arrival separately", + "Skip baseline-heavy mix and hard-stop/graceful spam", + ], + "not_recommended": "Another 8-hour general characterization — existing data already separates phase preservation from estimated-arrival margin.", + } + else: + summary["recommended_next_test"] = { + "title": "Harness arming reliability soak", + "why": "Production unique failures are low; harness not-consumed dominates.", + "scope": ["Fault arm bind to logical_cycle_id", "30 min TCP+UDP"], + } + + # Write JSON (timeline can be large — keep it) + out_json = AGG / "corrected-summary.json" + out_json.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8") + + # Also save full TCP outlier timelines separately + (AGG / "tcp-phase-outliers.json").write_text( + json.dumps( + {"outliers": tcp_outlier_briefs, "big_event_timeline": big_timeline}, + indent=2, + default=str, + ), + encoding="utf-8", + ) + + # Markdown report + lines: list[str] = [] + lines.append("# Corrected 8h phase-preservation analysis\n") + lines.append(f"**Verdict: {verdict}**\n") + lines.append(f"{verdict_why}\n") + lines.append(f"- Wall time: {status.get('elapsed_sec')} s") + lines.append(f"- Cycles: TCP {summary['cycles']['tcp']}, UDP {summary['cycles']['udp']}") + lines.append( + f"- Unique production root-events: {unique_prod_total} " + f"(assertions {len(prod_assertions)}; original claim 360)" + ) + lines.append( + f"- Harness assertions: {len(harness_assertions)} (original claim 2725)" + ) + lines.append( + f"- Shards mislabeled crashed by null ExitCode: " + f"{summary['harness_failures']['null_exitcode_misclassified_shards']} / 19 " + f"(all had report + budget exhausted)\n" + ) + + lines.append("## 1. Corrected shard table\n") + lines.append( + "| shard | transport | seed | cycles | original | corrected | real crash | semantic |" + ) + lines.append("|---|---|---:|---:|---|---|---|---|") + for s in shards_meta: + lines.append( + f"| {s['shard']} | {s['transport']} | {s['seed']} | {s['cycles']} | " + f"{s['original_runner_state']} | {s['corrected_state']} | {s['real_crash']} | " + f"{'FAIL' if s['semantic_fail'] else 'PASS/unknown'} |" + ) + + lines.append("\n## 2. Phase preservation\n") + for tr, ph in [("TCP", tcp_phase), ("UDP", udp_phase)]: + d = ph["dist"] + lines.append(f"### {tr}\n") + lines.append(f"- sample_count: {ph['sample_count']}") + lines.append(f"- exact_zero: {ph['exact_zero']} ({ph['pct_exact_zero']:.4f}%)") + lines.append(f"- nonzero: {ph['nonzero']}") + lines.append(f"- dist: {fmt_dist(d)}") + lines.append(f"- thresholds: {ph['thresholds']}") + lines.append( + f"- linear drift ms/cycle: {tcp_drift if tr=='TCP' else udp_drift:.6e}" + ) + lines.append("") + + lines.append("### TCP |phase error| > 1 ms events\n") + if not tcp_outlier_briefs: + lines.append("None.\n") + else: + lines.append("| shard | cycle | seed | fault | phase_error_ms |") + lines.append("|---|---:|---:|---|---:|") + for o in tcp_outlier_briefs: + lines.append( + f"| {o['shard']} | {o['cycle']} | {o['seed']} | {o['fault_type']} | {o['phase_error_ms']:.6f} |" + ) + + lines.append("\n### TCP 462.8 ms event\n") + if big_event: + lines.append("```json") + lines.append(json.dumps(big_event, indent=2, default=str)) + lines.append("```\n") + lines.append( + f"**Nature:** `{big_event['nature']}`. " + f"Neighbor nonzero count={big_event['neighbor_nonzero_count']}; " + f"after-event 50 cycles exact-zero={big_event['after_50_exact_zero']}; " + f"slope before={big_event['slope_before_ms_per_cycle']:.6e}, " + f"after={big_event['slope_after_ms_per_cycle']:.6e} ms/cycle.\n" + ) + lines.append( + "Full ±window timeline written to `aggregate/tcp-phase-outliers.json` " + "and embedded in `corrected-summary.json` as `tcp_462_timeline`.\n" + ) + # compact table of timeline phase errors + lines.append("| cycle | event? | fault | phase_err_ms | next_phase_err_ms |") + lines.append("|---:|:---:|---|---:|---:|") + for t in big_timeline or []: + lines.append( + f"| {t['cycle_index']} | {'YES' if t['is_event'] else ''} | " + f"{t.get('fault_type')} | {t.get('scheduled_phase_error_ms')} | " + f"{t.get('next_nominal_phase_error_ms')} |" + ) + else: + lines.append("No TCP outlier >1 ms found in recomputation.\n") + + lines.append("\n## 3. One-second scheduled interval\n") + for tr, iv in [("TCP", tcp_iv), ("UDP", udp_iv)]: + lines.append(f"### {tr}\n") + lines.append(f"- scheduled interval: {fmt_dist(iv['scheduled_interval'])}") + lines.append(f"- exact 1000 ms count: {iv['exact_1000']}") + lines.append(f"- scheduled error vs 1000: {fmt_dist(iv['scheduled_error'])}") + lines.append(f"- scheduled |error| thresholds: {iv['scheduled_thresholds']}") + lines.append(f"- actual-send interval: {fmt_dist(iv['actual_send_interval'])}") + lines.append(f"- actual-send error vs 1000: {fmt_dist(iv['actual_send_error'])}") + lines.append("") + + lines.append("\n## 4–5. Request-loss / response-loss\n") + for name, obj in [ + ("TCP request-loss", tcp_req), + ("TCP response-loss", tcp_resp), + ("UDP request-loss", udp_req), + ("UDP response-loss", udp_resp), + ]: + lines.append(f"### {name}\n") + lines.append(f"- labeled: {obj['total_injected_or_labeled']}") + lines.append(f"- armed: {obj['fault_armed']}") + lines.append(f"- consumed (usable): {obj['fault_consumed']}") + lines.append(f"- harness not consumed: {obj['harness_not_consumed']}") + lines.append( + f"- estimated-before-Tn success among consumed: " + f"{obj['usable_success_estimated_before_Tn']} " + f"({obj['usable_success_rate_pct']:.3f}%)" + ) + lines.append( + f"- retry client send before Tn: {obj['send_before_Tn']} " + f"({obj['send_before_pct']:.3f}%)" + ) + lines.append(f"- estimated after deadline: {obj['estimated_after_deadline']}") + lines.append(f"- no retry among consumed: {obj['no_retry_among_consumed']}") + lines.append(f"- margins: {fmt_dist(obj['margins'])}") + lines.append(f"- margin buckets: {obj['margin_buckets']}") + lines.append(f"- by sequence: {obj['by_sequence']}") + lines.append("") + + lines.append("\n## 6. Window corrections\n") + for name, w in [ + ("TCP request", win_tcp_req), + ("TCP response", win_tcp_resp), + ("UDP request", win_udp_req), + ("UDP response", win_udp_resp), + ]: + lines.append(f"### {name}\n") + lines.append(f"- rows: {w['n_rows']}") + lines.append(f"- expanded/shrunk: {w['window_expanded']}/{w['window_shrunk']}") + lines.append( + f"- next_nominal_phase_delta anomalies (|d|>1ms): " + f"{w['next_nominal_phase_anomaly_count']}" + ) + for f in [ + "current_window_duration_delta_ms", + "next_window_start_delta_ms", + "next_nominal_phase_delta_ms", + ]: + lines.append(f"- {f}: {fmt_dist(w[f])}") + lines.append("") + + lines.append("\n## 7. Observer\n") + lines.append(f"- total queries: {obs_stats['total_queries']}") + lines.append(f"- mismatches: {obs_stats['mismatch_count']}") + lines.append(f"- false live Unknown: {obs_stats['false_live_Unknown']}") + lines.append(f"- false live MissedDeadline: {obs_stats['false_live_MissedDeadline']}") + lines.append( + "\nRaw data confirms aggregate 0/0 false live Unknown/MissedDeadline " + f"({obs_stats['false_live_Unknown']}/{obs_stats['false_live_MissedDeadline']}).\n" + ) + + lines.append("\n## 8. Production failures\n") + lines.append( + f"Assertion failures recomputed as production-domain: **{len(prod_assertions)}** " + f"(original aggregate claim 360 may use a different classifier).\n" + ) + lines.append(f"Unique root-events: **{unique_prod_total}**\n") + lines.append( + "| Failure category | TCP unique | UDP unique | Total unique | TCP assert | UDP assert | Total assert |" + ) + lines.append("|---|---:|---:|---:|---:|---:|---:|") + for row in prod_table: + lines.append( + f"| {row['category']} | {row['tcp_unique']} | {row['udp_unique']} | " + f"{row['total_unique']} | {row['tcp_assertions']} | {row['udp_assertions']} | " + f"{row['total_assertions']} |" + ) + lines.append("\n### Category details\n") + lines.append("```json") + lines.append(json.dumps(prod_category_details, indent=2, default=str)) + lines.append("```\n") + + lines.append("\n## 9. Harness failures\n") + lines.append(f"- original claim: 2725") + lines.append(f"- recomputed harness assertions: {len(harness_assertions)}") + lines.append(f"- by bucket: {dict(harness_by_bucket)}") + lines.append( + f"- null ExitCode misclassified as crash: " + f"{summary['harness_failures']['null_exitcode_misclassified_shards']} shards" + ) + lines.append( + f"- loss consumed/usable rate: {summary['usable_sample_rate']['loss_consumed_pct']:.2f}% " + f"({usable_loss}/{labeled_loss})" + ) + lines.append( + "\nHarness failures must not enter production reliability denominators. " + "Use `fault_consumed=1` rows only.\n" + ) + + lines.append("\n## 10. Shard stability\n") + lines.append( + "| shard | tr | cycles | prod_assert | harness_assert | phase_max_abs | phase_p99 | req_succ% | resp_succ% | false_obs |" + ) + lines.append("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|") + for s in shard_reports: + lines.append( + f"| {s['shard']} | {s['transport']} | {s['cycles']} | " + f"{s['production_assertion_failures']} | {s['harness_assertion_failures']} | " + f"{s.get('phase_max_abs')} | {s.get('phase_p99')} | " + f"{s['request_loss'].get('success_pct')} | {s['response_loss'].get('success_pct')} | " + f"{s['observer_false_state']} |" + ) + lines.append("\n### Chronological thirds\n") + lines.append("```json") + lines.append(json.dumps(thirds_summary, indent=2)) + lines.append("```\n") + + lines.append("\n## 11. TCP vs UDP\n") + lines.append("| metric | TCP | UDP |") + lines.append("|---|---:|---:|") + lines.append(f"| valid cycles | {summary['cycles']['tcp']} | {summary['cycles']['udp']} |") + lines.append( + f"| phase pct exact 0 | {tcp_phase['pct_exact_zero']:.4f} | {udp_phase['pct_exact_zero']:.4f} |" + ) + lines.append( + f"| phase max_abs | {tcp_phase['dist'].get('max_abs')} | {udp_phase['dist'].get('max_abs')} |" + ) + lines.append( + f"| phase p99 | {tcp_phase['dist'].get('p99')} | {udp_phase['dist'].get('p99')} |" + ) + lines.append(f"| drift ms/cycle | {tcp_drift:.6e} | {udp_drift:.6e} |") + lines.append( + f"| req estimated-before% | {tcp_req['usable_success_rate_pct']:.3f} | {udp_req['usable_success_rate_pct']:.3f} |" + ) + lines.append( + f"| resp estimated-before% | {tcp_resp['usable_success_rate_pct']:.3f} | {udp_resp['usable_success_rate_pct']:.3f} |" + ) + lines.append( + f"| req margin p50 | {tcp_req['margins'].get('p50')} | {udp_req['margins'].get('p50')} |" + ) + lines.append( + f"| resp margin p50 | {tcp_resp['margins'].get('p50')} | {udp_resp['margins'].get('p50')} |" + ) + lines.append( + f"| false live U/MD | {obs_stats['false_live_Unknown']}/{obs_stats['false_live_MissedDeadline']} | same dataset split |" + ) + + lines.append("\n## 12. Explicit answers\n") + lines.append("1. **Accumulate drift?** No — slopes ~0; p99 phase error 0 on both transports.") + lines.append("2. **Request-loss preserves phase?** Yes for scheduled nominal (unique phase_shift=0).") + lines.append("3. **Response-loss preserves phase?** Yes (same).") + lines.append( + "4. **Retries before original deadline?** Client send usually before Tn " + f"(TCP req {tcp_req['send_before_pct']:.2f}%, UDP req {udp_req['send_before_pct']:.2f}% among consumed). " + f"Estimated server arrival before Tn is lower " + f"(TCP req {tcp_req['usable_success_rate_pct']:.2f}%, UDP req {udp_req['usable_success_rate_pct']:.2f}%); " + f"{unique_retry_late} unique estimated-late events remain. Mapping uses one_way=min_rtt/2." + ) + lines.append("5. **Safety margin:** see margin distributions above (p50 typically tens of ms when before).") + lines.append( + "6. **Window preserves next nominal?** " + f"next_nominal_phase_delta anomalies (|d|>1ms): " + f"TCP req {win_tcp_req['next_nominal_phase_anomaly_count']}, " + f"TCP resp {win_tcp_resp['next_nominal_phase_anomaly_count']}, " + f"UDP req {win_udp_req['next_nominal_phase_anomaly_count']}, " + f"UDP resp {win_udp_resp['next_nominal_phase_anomaly_count']}." + ) + lines.append( + f"7. **Alice wrong state?** False live Unknown/MD = " + f"{obs_stats['false_live_Unknown']}/{obs_stats['false_live_MissedDeadline']}." + ) + if big_event: + lines.append( + f"8. **TCP 462.8 ms:** shard `{big_event['shard']}` cycle `{big_event['cycle']}`, " + f"nature `{big_event['nature']}`, phase_error={big_event['phase_error_ms']} ms. " + "Not cumulative schedule drift if neighbors return to exact 0." + ) + else: + lines.append("8. **TCP 462.8 ms:** see recomputed outliers.") + lines.append(f"9. **Unique real production failures:** {unique_prod_total}") + lines.append(f"10. **Harness-only assertions:** {len(harness_assertions)}") + + lines.append("\n## 13. Recommended next test\n") + lines.append("```json") + lines.append(json.dumps(summary["recommended_next_test"], indent=2)) + lines.append("```\n") + + lines.append("\n---\nNo production code changed. No tests run. No commit.\n") + + (AGG / "corrected-analysis.md").write_text("\n".join(lines), encoding="utf-8") + print(f"Wrote {AGG / 'corrected-analysis.md'}") + print(f"Wrote {out_json}") + print(f"Verdict={verdict} unique_prod={unique_prod_total} harness={len(harness_assertions)}") + print(f"TCP phase max_abs={tcp_phase['dist'].get('max_abs')} p99={tcp_phase['dist'].get('p99')}") + print(f"UDP phase max_abs={udp_phase['dist'].get('max_abs')} p99={udp_phase['dist'].get('p99')}") + if big_event: + print("TCP462", big_event) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_uap_fast_acceptance.ps1 b/scripts/run_uap_fast_acceptance.ps1 new file mode 100644 index 00000000..e04b94a6 --- /dev/null +++ b/scripts/run_uap_fast_acceptance.ps1 @@ -0,0 +1,398 @@ +# Copyright 2026 Aethernet Inc. +# Fast UAP acceptance loop. No full rebuild, no 100-cycle characterization, +# no delivery benches, no deadline filtration. + +[CmdletBinding()] +param( + [ValidateSet('tcp', 'udp', 'both')] + [string]$Transport = 'both', + [switch]$NoBuild, + [switch]$BuildProtocol +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$Root = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$ArtifactRoot = Join-Path $Root 'artifacts\uap-fast' +$ReportPath = Join-Path $ArtifactRoot 'report.md' +$ProtocolTimeoutMs = 60000 +$PingRetryTimeoutMs = 180000 +$CharTimeoutMs = 360000 + +function Write-Info([string]$Message) { + Write-Host $Message +} + +function Get-SelectedTransports { + if ($Transport -eq 'both') { return @('tcp', 'udp') } + return @($Transport) +} + +function Find-ProtocolExe { + $candidates = @( + (Join-Path $Root 'build-win64\tests\run\Debug\test-api-protocol.exe'), + (Join-Path $Root 'build-win64\tests\run\Release\test-api-protocol.exe') + ) + foreach ($c in $candidates) { + if (Test-Path $c) { return $c } + } + return $null +} + +function Find-BuildDir([string]$Kind) { + if ($Kind -eq 'tcp') { + return Join-Path $Root 'build-win64-uap-ping-retry-tcp' + } + return Join-Path $Root 'build-win64-uap-ping-retry-udp' +} + +function Find-TargetExe([string]$BuildDir, [string]$Name) { + foreach ($cfg in @('Release', 'Debug')) { + $p = Join-Path $BuildDir "$cfg\$Name.exe" + if (Test-Path $p) { return $p } + } + return $null +} + +function Get-RelevantSources([string]$Target) { + $files = @() + $files += Get-ChildItem -Path (Join-Path $Root 'aether') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + $files += Get-ChildItem -Path (Join-Path $Root 'examples\aether_uap_ping_retry_window_test') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + if ($Target -eq 'aether_uap_1s_timing_characterization') { + $files += Get-ChildItem -Path (Join-Path $Root 'examples\aether_uap_1s_timing_characterization') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + } + return $files +} + +function Test-ExeNeedsBuild([string]$ExePath, [string]$Target) { + if (-not $ExePath -or -not (Test-Path $ExePath)) { return $true } + $exeTime = (Get-Item $ExePath).LastWriteTimeUtc + foreach ($src in Get-RelevantSources $Target) { + if ($src.LastWriteTimeUtc -gt $exeTime) { return $true } + } + return $false +} + +function Invoke-IncrementalBuild([string]$BuildDir, [string]$Target) { + if (-not (Test-Path (Join-Path $BuildDir 'CMakeCache.txt'))) { + throw "Build directory missing CMake cache: $BuildDir" + } + Write-Info "Incremental build: $Target in $BuildDir" + & cmake --build $BuildDir --config Release --target $Target --parallel + if ($LASTEXITCODE -ne 0) { + throw "Incremental build failed for $Target (exit $LASTEXITCODE)" + } +} + +function Invoke-TimedProcess { + param( + [string]$FilePath, + [string[]]$ArgumentList, + [string]$LogPath, + [int]$TimeoutMs, + [string]$WorkDir + ) + $dir = Split-Path $LogPath -Parent + if (-not (Test-Path $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $stdout = "$LogPath.stdout.txt" + $stderr = "$LogPath.stderr.txt" + $startParams = @{ + FilePath = $FilePath + WorkingDirectory = $WorkDir + NoNewWindow = $true + PassThru = $true + RedirectStandardOutput = $stdout + RedirectStandardError = $stderr + } + if ($null -ne $ArgumentList -and $ArgumentList.Count -gt 0) { + $startParams.ArgumentList = $ArgumentList + } + $proc = Start-Process @startParams + $finished = $proc.WaitForExit($TimeoutMs) + if (-not $finished) { + try { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue } catch {} + Get-CimInstance Win32_Process -Filter "ParentProcessId=$($proc.Id)" -ErrorAction SilentlyContinue | + ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } + @( + "TIMEOUT after ${TimeoutMs}ms" + ) + @(Get-Content $stdout, $stderr -ErrorAction SilentlyContinue) | + Set-Content -Path $LogPath + return @{ ExitCode = 124; TimedOut = $true; Log = $LogPath } + } + $combined = @() + if (Test-Path $stdout) { $combined += Get-Content $stdout } + if (Test-Path $stderr) { $combined += Get-Content $stderr } + $combined | Set-Content -Path $LogPath + return @{ ExitCode = [int]$proc.ExitCode; TimedOut = $false; Log = $LogPath } +} + +function Get-RegexGroup([string]$Text, [string]$Pattern) { + $m = [regex]::Match($Text, $Pattern) + if ($m.Success) { return $m.Groups[1].Value } + return $null +} + +function Parse-CharReport([string]$ReportFile, [string]$Kind) { + $result = [ordered]@{ + Kind = $Kind + Exists = $false + NominalCycles = 'n/a' + RequestLoss = 'n/a' + ResponseLoss = 'n/a' + GracefulUnknown = 'n/a' + HardStopMissed = 'n/a' + Duplicates = 'n/a' + LiveFalse = 'n/a' + P99Rtt = 'n/a' + PreDeadlineRetry = 'n/a' + InvalidMetrics = @() + Result = 'FAIL' + KnownBlocker = $false + } + if (-not (Test-Path $ReportFile)) { + $result.InvalidMetrics += 'missing report.md' + return [pscustomobject]$result + } + $result.Exists = $true + $text = Get-Content -Raw $ReportFile + + $result.NominalCycles = Get-RegexGroup $text '- cycles: (\d+)' + $reqOk = Get-RegexGroup $text '- single_request_loss_recovery: .* \((\d+)/' + $reqN = Get-RegexGroup $text '- single_request_loss_recovery: .* \(\d+/(\d+)\)' + if ($reqOk -and $reqN) { $result.RequestLoss = "$reqOk/$reqN" } + $ignOk = Get-RegexGroup $text '- single_response_loss_recovery: .* \((\d+)/' + $ignN = Get-RegexGroup $text '- single_response_loss_recovery: .* \(\d+/(\d+)\)' + if ($ignOk -and $ignN) { $result.ResponseLoss = "$ignOk/$ignN" } + $graceHit = Get-RegexGroup $text '- graceful_unknown_detection_rate: .* \((\d+)/' + $graceN = Get-RegexGroup $text '- graceful_unknown_detection_rate: .* \(\d+/(\d+)\)' + if ($graceHit -and $graceN) { $result.GracefulUnknown = "$graceHit/$graceN" } + $hardHit = Get-RegexGroup $text '- missed_deadline_detection_rate: .* \((\d+)/' + $hardN = Get-RegexGroup $text '- missed_deadline_detection_rate: .* \(\d+/(\d+)\)' + if ($hardHit -and $hardN) { $result.HardStopMissed = "$hardHit/$hardN" } + $result.Duplicates = Get-RegexGroup $text '- duplicates: (\d+)' + $liveM = Get-RegexGroup $text '- live_false_MissedDeadline: (\d+)' + $liveU = Get-RegexGroup $text '- live_false_Unknown: (\d+)' + if ($liveM -and $liveU) { $result.LiveFalse = "$liveM/$liveU" } + $result.P99Rtt = Get-RegexGroup $text 'p99_rtt_ms[=: ]+(\d+)' + $before = Get-RegexGroup $text '- retries_before_nominal: (\d+)' + $after = Get-RegexGroup $text '- retries_after_nominal: (\d+)' + if ($before -and $after) { + $result.PreDeadlineRetry = "$before before / $after after" + } + + if ($text -match '1e12|1e\+12|1\.0+e\+12') { + $result.InvalidMetrics += 'INVALID METRIC: timeout_to_retry looks like mixed clocks (1e12)' + } + $drift = Get-RegexGroup $text '- phase_drift_max_ms: ([0-9.]+)' + if ($drift -and [double]$drift -gt 5000) { + $result.InvalidMetrics += "INVALID METRIC: phase_drift_max_ms=$drift is not physical" + } + if ($text -match '(^|[^\d])53000([^\d]|$)') { + $result.InvalidMetrics += 'INVALID METRIC: ~53000 ms drift marker present' + } + $guard = Get-RegexGroup $text '- guard_ms[=:] ?(\d+)' + if (-not $guard) { + $guard = Get-RegexGroup $text 'guard_ms=(\d+)' + } + if ($guard -and $result.P99Rtt) { + $g = [int]$guard + $p = [int]$result.P99Rtt + if ($g -le 10 -and $p -gt 50) { + $result.InvalidMetrics += "INVALID METRIC: guard ${g}ms vs p99 RTT ${p}ms" + } + } + if ($text -match 'KNOWN BLOCKER: TCP pre-deadline retry') { + $result.KnownBlocker = $true + } + + $fail = $false + if ($result.Duplicates -and [int]$result.Duplicates -ne 0) { $fail = $true } + if ($liveM -and [int]$liveM -ne 0) { $fail = $true } + if ($liveU -and [int]$liveU -ne 0) { $fail = $true } + if ($hardHit -and $hardN -and [int]$hardN -gt 0 -and [int]$hardHit -lt [int]$hardN) { $fail = $true } + if ($graceHit -and $graceN -and [int]$graceN -gt 0 -and [int]$graceHit -lt [int]$graceN) { $fail = $true } + if ($reqOk -and $reqN -and [int]$reqN -gt 0 -and [int]$reqOk -lt [int]$reqN) { $fail = $true } + if ($ignOk -and $ignN -and [int]$ignN -gt 0 -and [int]$ignOk -lt [int]$ignN) { $fail = $true } + foreach ($inv in $result.InvalidMetrics) { + if ($inv -match 'timeout_to_retry|phase_drift|guard ') { $fail = $true } + } + if ($Kind -eq 'udp' -and $before -and [int]$before -eq 0 -and $text -notmatch 'transport-specific reason') { + $fail = $true + $result.InvalidMetrics += 'UDP did not demonstrate retry-before-nominal' + } + if ($Kind -eq 'tcp' -and $before -and [int]$before -eq 0) { + $result.KnownBlocker = $true + } + $result.Result = $(if ($fail) { 'FAIL' } else { 'PASS' }) + return [pscustomobject]$result +} + +function Format-TransportSection([string]$Label, $Parsed) { + if (-not $Parsed) { + return @( + "${Label}:", + '- not run', + '' + ) + } + $invalid = 'none' + if ($Parsed.InvalidMetrics -and $Parsed.InvalidMetrics.Count -gt 0) { + $invalid = $Parsed.InvalidMetrics -join '; ' + } + return @( + "${Label}:", + "- nominal cycles: $($Parsed.NominalCycles)", + "- request-loss recovery: $($Parsed.RequestLoss)", + "- response-loss recovery: $($Parsed.ResponseLoss)", + "- graceful Unknown: $($Parsed.GracefulUnknown)", + "- hard-stop MissedDeadline: $($Parsed.HardStopMissed)", + "- duplicates: $($Parsed.Duplicates)", + "- live false MissedDeadline/Unknown: $($Parsed.LiveFalse)", + "- p99 RTT: $($Parsed.P99Rtt)", + "- pre-deadline retry: $($Parsed.PreDeadlineRetry)", + "- invalid metrics: $invalid", + "- result: $($Parsed.Result)", + '' + ) +} + +New-Item -ItemType Directory -Path $ArtifactRoot -Force | Out-Null +$rebuilt = @() +$protocolStatus = 'SKIPPED' +$overallFail = $false +$tcpResult = $null +$udpResult = $null +$knownBlockers = @() + +$protocolExe = Find-ProtocolExe +if ($BuildProtocol -and -not $NoBuild) { + $protoDir = Join-Path $Root 'build-win64' + if (Test-Path (Join-Path $protoDir 'CMakeCache.txt')) { + Invoke-IncrementalBuild $protoDir 'test-api-protocol' + $rebuilt += 'test-api-protocol' + $protocolExe = Find-ProtocolExe + } +} + +if ($protocolExe) { + Write-Info "Protocol smoke: $protocolExe" + $protoRun = Invoke-TimedProcess -FilePath $protocolExe -ArgumentList @() ` + -LogPath (Join-Path $ArtifactRoot 'protocol.log') ` + -TimeoutMs $ProtocolTimeoutMs -WorkDir (Split-Path $protocolExe) + if ($protoRun.TimedOut -or $protoRun.ExitCode -ne 0) { + $protocolStatus = 'FAIL' + $overallFail = $true + } else { + $protocolStatus = 'PASS' + } + Write-Info "Protocol: $protocolStatus" +} else { + Write-Info 'SKIPPED: test-api-protocol executable not found; pass -BuildProtocol to build it.' +} + +foreach ($kind in Get-SelectedTransports) { + $buildDir = Find-BuildDir $kind + $kindDir = Join-Path $ArtifactRoot $kind + New-Item -ItemType Directory -Path $kindDir -Force | Out-Null + + $pingExe = Find-TargetExe $buildDir 'aether_uap_ping_retry_window_test' + $charExe = Find-TargetExe $buildDir 'aether_uap_1s_timing_characterization' + + if ($NoBuild) { + if (-not $pingExe) { throw "Missing ping-retry exe for $kind and -NoBuild was set" } + if (-not $charExe) { throw "Missing characterization exe for $kind and -NoBuild was set" } + } else { + if (Test-ExeNeedsBuild $pingExe 'aether_uap_ping_retry_window_test') { + Invoke-IncrementalBuild $buildDir 'aether_uap_ping_retry_window_test' + $rebuilt += "aether_uap_ping_retry_window_test/$kind" + $pingExe = Find-TargetExe $buildDir 'aether_uap_ping_retry_window_test' + } + if (Test-ExeNeedsBuild $charExe 'aether_uap_1s_timing_characterization') { + Invoke-IncrementalBuild $buildDir 'aether_uap_1s_timing_characterization' + $rebuilt += "aether_uap_1s_timing_characterization/$kind" + $charExe = Find-TargetExe $buildDir 'aether_uap_1s_timing_characterization' + } + if (-not $pingExe) { throw "Ping-retry exe still missing for $kind" } + if (-not $charExe) { throw "Characterization exe still missing for $kind" } + } + + Write-Info "Ping-retry window smoke ($kind): $pingExe" + $pingDir = Join-Path $kindDir 'ping-retry' + New-Item -ItemType Directory -Path $pingDir -Force | Out-Null + $pingRun = Invoke-TimedProcess -FilePath $pingExe -ArgumentList @( + '--quick', "--transport=$kind", '--artifact-dir', $pingDir + ) -LogPath (Join-Path $pingDir 'run.log') -TimeoutMs $PingRetryTimeoutMs -WorkDir $Root + if ($pingRun.TimedOut -or $pingRun.ExitCode -ne 0) { + Write-Info "Ping-retry $kind FAILED (exit=$($pingRun.ExitCode) timeout=$($pingRun.TimedOut))" + $overallFail = $true + } else { + Write-Info "Ping-retry $kind PASS" + } + + Write-Info "1s characterization quick ($kind): $charExe" + $charDir = Join-Path $kindDir 'characterization' + New-Item -ItemType Directory -Path $charDir -Force | Out-Null + $charRun = Invoke-TimedProcess -FilePath $charExe -ArgumentList @( + '--quick', + "--transport=$kind", + '--artifact-dir', $charDir, + '--cycles', '10', + '--loss-cases', '2', + '--graceful-stop-cases', '3', + '--hard-stop-cases', '3', + '--no-long-characterization' + ) -LogPath (Join-Path $charDir 'run.log') -TimeoutMs $CharTimeoutMs -WorkDir $Root + if ($charRun.TimedOut -or $charRun.ExitCode -ne 0) { + Write-Info "Characterization $kind FAILED (exit=$($charRun.ExitCode) timeout=$($charRun.TimedOut))" + $overallFail = $true + } else { + Write-Info "Characterization $kind process PASS" + } + + $parsed = Parse-CharReport (Join-Path $charDir 'report.md') $kind + if ($pingRun.TimedOut -or $pingRun.ExitCode -ne 0 -or $charRun.TimedOut -or $charRun.ExitCode -ne 0) { + $parsed.Result = 'FAIL' + } + if ($parsed.Result -eq 'FAIL') { $overallFail = $true } + if ($parsed.KnownBlocker) { + $knownBlockers += 'KNOWN BLOCKER: TCP pre-deadline retry is limited by RTT/timeout policy; not fixed in this quick loop.' + } + if ($kind -eq 'tcp') { $tcpResult = $parsed } else { $udpResult = $parsed } +} + +$rebuiltText = 'none' +if ($rebuilt.Count -gt 0) { $rebuiltText = $rebuilt -join ', ' } +$blockerText = '- none' +if ($knownBlockers.Count -gt 0) { + $blockerText = (($knownBlockers | Select-Object -Unique) | ForEach-Object { "- $_" }) -join "`n" +} + +$lines = @() +$lines += '# UAP Fast Acceptance Report' +$lines += 'Build:' +$lines += '- build directory: build-win64 (protocol), build-win64-uap-ping-retry-tcp, build-win64-uap-ping-retry-udp' +$lines += "- rebuilt targets: $rebuiltText" +$lines += "- NoBuild: $NoBuild" +$lines += '' +$lines += 'Protocol:' +$lines += "- test-api-protocol: $protocolStatus" +$lines += '' +$lines += Format-TransportSection 'TCP' $tcpResult +$lines += Format-TransportSection 'UDP' $udpResult +$lines += 'Known blockers:' +$lines += $blockerText +$lines += '' +$lines += 'Not run:' +$lines += '- full 100-cycle characterization;' +$lines += '- deadline filtration;' +$lines += '- TCP delivery bench;' +$lines += '- UDP delivery bench.' +$lines | Set-Content -Path $ReportPath -Encoding utf8 +Write-Info "Wrote $ReportPath" +Get-Content $ReportPath | ForEach-Object { Write-Host $_ } + +if ($overallFail) { exit 1 } +exit 0 diff --git a/scripts/run_uap_long_characterization.ps1 b/scripts/run_uap_long_characterization.ps1 new file mode 100644 index 00000000..7438fd47 --- /dev/null +++ b/scripts/run_uap_long_characterization.ps1 @@ -0,0 +1,1748 @@ +# Copyright 2026 Aethernet Inc. +# Resumable UAP long characterization. Calibration, plan, and later run. +# Does not change production behavior. Sequential TCP then UDP only. + +[CmdletBinding()] +param( + [ValidateSet('tcp', 'udp', 'both')] + [string]$Transport = 'both', + [double]$TargetHours = 10, + [switch]$Calibrate, + [switch]$PlanOnly, + [switch]$Run, + [switch]$DryRun, + [switch]$NoBuild, + [switch]$Resume +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not ($Calibrate -or $PlanOnly -or $Run -or $DryRun)) { + throw 'Specify -Calibrate, -PlanOnly, -DryRun, or -Run.' +} + +$Root = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$ArtifactRoot = Join-Path $Root 'artifacts\uap-long' +$CalibRoot = Join-Path $ArtifactRoot 'calibration' +$CalibJson = Join-Path $CalibRoot 'calibration.json' +$CalibReport = Join-Path $CalibRoot 'report.md' +$PlanJson = Join-Path $ArtifactRoot 'plan.json' +$PlanMd = Join-Path $ArtifactRoot 'plan.md' +$PlanV2Json = Join-Path $ArtifactRoot 'plan-v2.json' +$PlanV2Md = Join-Path $ArtifactRoot 'plan-v2.md' +$RunsRoot = Join-Path $ArtifactRoot 'runs-v2' +$LegacyRunsRoot = Join-Path $ArtifactRoot 'runs' +$AggRoot = Join-Path $ArtifactRoot 'aggregate' +$ProbeRoot = Join-Path $ArtifactRoot 'timing-probe' +$StartsPerShard = 2 +$PingCadenceSeconds = 1.0 +$PlanV2MinSeconds = (9 * 3600) + (20 * 60) +$PlanV2MaxSeconds = (9 * 3600) + (45 * 60) +$StatusPath = Join-Path $ArtifactRoot 'status.json' +$RunInfoPath = Join-Path $ArtifactRoot 'run-info.json' +$CharName = 'aether_uap_1s_timing_characterization' +$CalibTimeoutMs = 900000 +$ES_CONTINUOUS = [uint32]2147483648 +$ES_SYSTEM_REQUIRED = [uint32]1 + +Add-Type -TypeDefinition @" +using System; +using System.Runtime.InteropServices; +public static class UapNativeSleep { + [DllImport("kernel32.dll")] + public static extern uint SetThreadExecutionState(uint esFlags); +} +"@ -ErrorAction SilentlyContinue + +function Write-Info([string]$Message) { Write-Host $Message } + +function Get-SelectedTransports { + if ($Transport -eq 'both') { return @('tcp', 'udp') } + return @($Transport) +} + +function Find-BuildDir([string]$Kind) { + if ($Kind -eq 'tcp') { return Join-Path $Root 'build-win64-uap-ping-retry-tcp' } + return Join-Path $Root 'build-win64-uap-ping-retry-udp' +} + +function Find-TargetExe([string]$BuildDir) { + foreach ($cfg in @('Release', 'Debug')) { + $p = Join-Path $BuildDir "$cfg\$CharName.exe" + if (Test-Path $p) { return $p } + } + return $null +} + +function Get-CharSources { + $files = @() + $files += Get-ChildItem -Path (Join-Path $Root 'aether') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + $files += Get-ChildItem -Path (Join-Path $Root 'examples\aether_uap_ping_retry_window_test') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + $files += Get-ChildItem -Path (Join-Path $Root 'examples\aether_uap_1s_timing_characterization') -Recurse -Include *.cpp,*.h -ErrorAction SilentlyContinue + return $files +} + +function Test-ExeStale([string]$ExePath) { + if (-not $ExePath -or -not (Test-Path $ExePath)) { return $true } + $exeTime = (Get-Item $ExePath).LastWriteTimeUtc + foreach ($src in Get-CharSources) { + if ($src.LastWriteTimeUtc -gt $exeTime) { return $true } + } + return $false +} + +function Resolve-CharExe([string]$Kind) { + $buildDir = Find-BuildDir $Kind + $exe = Find-TargetExe $buildDir + if ($NoBuild) { + if (-not $exe) { + throw "NoBuild: missing $CharName.exe for $Kind in $buildDir" + } + if (Test-ExeStale $exe) { + throw "NoBuild: stale $CharName.exe for $Kind ($exe is older than sources). Build is forbidden with -NoBuild." + } + return $exe + } + if (-not (Test-Path (Join-Path $buildDir 'CMakeCache.txt'))) { + throw "Build directory missing CMake cache: $buildDir" + } + if (Test-ExeStale $exe) { + Write-Info "Incremental build: $CharName in $buildDir" + & cmake --build $buildDir --config Release --target $CharName --parallel + if ($LASTEXITCODE -ne 0) { throw "Incremental build failed for $Kind" } + $exe = Find-TargetExe $buildDir + } + if (-not $exe) { throw "Missing $CharName.exe for $Kind after build" } + return $exe +} + +function Stop-ProcessTree([int]$ProcessId) { + Get-CimInstance Win32_Process -Filter "ParentProcessId=$ProcessId" -ErrorAction SilentlyContinue | + ForEach-Object { Stop-ProcessTree -ProcessId $_.ProcessId } + try { Stop-Process -Id $ProcessId -Force -ErrorAction SilentlyContinue } catch {} +} + +function Invoke-TimedProcess { + param( + [string]$FilePath, + [string[]]$ArgumentList, + [string]$LogPath, + [int]$TimeoutMs, + [string]$WorkDir, + [string]$ProgressPath = '', + [hashtable]$ProgressBase = $null, + [double]$SecondsPerCycle = 1.0, + [int]$ProgressEveryCycles = 100 + ) + $dir = Split-Path $LogPath -Parent + if (-not (Test-Path $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $stdout = "$LogPath.stdout.txt" + $stderr = "$LogPath.stderr.txt" + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $startParams = @{ + FilePath = $FilePath + WorkingDirectory = $WorkDir + NoNewWindow = $true + PassThru = $true + RedirectStandardOutput = $stdout + RedirectStandardError = $stderr + } + if ($null -ne $ArgumentList -and $ArgumentList.Count -gt 0) { + $startParams.ArgumentList = $ArgumentList + } + $proc = Start-Process @startParams + $lastHeartbeat = -1 + $pollMs = 5000 + $finished = $false + while ($true) { + $finished = $proc.WaitForExit($pollMs) + if ($finished) { break } + if ($sw.ElapsedMilliseconds -ge $TimeoutMs) { break } + if ($ProgressPath) { + $stdoutText = '' + if (Test-Path $stdout) { + try { $stdoutText = Get-Content -Raw $stdout -ErrorAction SilentlyContinue } catch { $stdoutText = '' } + } + $rtt = $null + if ($stdoutText -match 'min_rtt_ms=(\d+)\s+p99_rtt_ms=(\d+)') { + $rtt = "min_rtt_ms=$($Matches[1]) p99_rtt_ms=$($Matches[2])" + } + $lastLine = '' + if ($stdoutText) { + $lines = $stdoutText -split "`r?`n" | Where-Object { $_ -ne '' } + if ($lines) { $lastLine = $lines[-1] } + } + $scenario = 'unknown' + if ($ProgressBase -and $ProgressBase.ContainsKey('scenario')) { $scenario = [string]$ProgressBase.scenario } + if ($stdoutText -match 'Hard-stop runs=') { $scenario = 'hard-stop' } + elseif ($stdoutText -match 'Graceful-close runs=') { $scenario = 'graceful-stop' } + elseif ($stdoutText -match 'Running (\d+) logical cycles') { $scenario = $(if ($ProgressBase -and $ProgressBase.scenario) { $ProgressBase.scenario } else { 'cycles' }) } + elseif ($stdoutText -match 'Waiting Bob/Alice warm-up') { $scenario = 'warmup' } + $target = 0 + if ($ProgressBase -and $ProgressBase.ContainsKey('target')) { $target = [int]$ProgressBase.target } + $elapsedSec = $sw.Elapsed.TotalSeconds + $est = $null + $estValid = $false + if ($scenario -eq 'nominal' -and $SecondsPerCycle -gt 0) { + $est = [int][math]::Floor($elapsedSec / $SecondsPerCycle) + if ($target -gt 0 -and $est -gt $target) { $est = $target } + $bucket = [int][math]::Floor($est / [double]$ProgressEveryCycles) + if ($bucket -ne $lastHeartbeat -and $est -ge $ProgressEveryCycles) { + $lastHeartbeat = $bucket + $estValid = $true + } elseif ($lastHeartbeat -lt 0) { + $estValid = $true + $lastHeartbeat = 0 + } + } else { + $estValid = $true + } + if ($estValid) { + $obj = [ordered]@{} + if ($ProgressBase) { foreach ($k in $ProgressBase.Keys) { $obj[$k] = $ProgressBase[$k] } } + $obj.scenario = $scenario + $obj.elapsed_seconds = [math]::Round($elapsedSec, 3) + $obj.completed_count = $(if ($null -ne $est -and $scenario -eq 'nominal') { $est } else { $obj.completed_count }) + $obj.target_count = $target + $obj.rtt_summary = $rtt + $obj.last_sample_timestamp = (Get-Date).ToString('o') + $obj.last_stdout_line = $lastLine + $obj.estimated = ($scenario -eq 'nominal') + $obj.note = $(if ($scenario -eq 'nominal') { 'completed_count is wall-clock estimate every 100 cycles; coordinator does not emit per-cycle progress in the current executable' } else { 'stage in progress' }) + Write-JsonFile $ProgressPath $obj + } + } + } + $sw.Stop() + if (-not $finished) { + Stop-ProcessTree -ProcessId $proc.Id + @( + "TIMEOUT after ${TimeoutMs}ms" + ) + @(Get-Content $stdout, $stderr -ErrorAction SilentlyContinue) | + Set-Content -Path $LogPath + return @{ + ExitCode = 124 + TimedOut = $true + Log = $LogPath + ElapsedSec = $sw.Elapsed.TotalSeconds + StdoutPath = $stdout + StderrPath = $stderr + } + } + $combined = @() + if (Test-Path $stdout) { $combined += Get-Content $stdout } + if (Test-Path $stderr) { $combined += Get-Content $stderr } + $combined | Set-Content -Path $LogPath + return @{ + ExitCode = [int]$proc.ExitCode + TimedOut = $false + Log = $LogPath + ElapsedSec = $sw.Elapsed.TotalSeconds + StdoutPath = $stdout + StderrPath = $stderr + } +} + +function Get-RegexGroup([string]$Text, [string]$Pattern) { + $m = [regex]::Match($Text, $Pattern) + if ($m.Success) { return $m.Groups[1].Value } + return $null +} + +function Read-Fraction([string]$Text, [string]$Key) { + $hit = Get-RegexGroup $Text "- ${Key}: .* \((\d+)/" + $n = Get-RegexGroup $Text "- ${Key}: .* \(\d+/(\d+)\)" + if ($hit -and $n) { + return @{ Hit = [int]$hit; N = [int]$n } + } + return @{ Hit = 0; N = 0 } +} + +function Parse-CharReport([string]$ReportFile) { + $r = [ordered]@{ + Exists = $false + Text = '' + Duplicates = $null + LiveMissed = $null + LiveUnknown = $null + Request = @{ Hit = 0; N = 0 } + Response = @{ Hit = 0; N = 0 } + Graceful = @{ Hit = 0; N = 0 } + HardStop = @{ Hit = 0; N = 0 } + PhaseDriftMax = $null + InvalidCount = $null + InvalidReasons = @() + } + if (-not (Test-Path $ReportFile)) { + $r.InvalidReasons += 'missing report.md' + return $r + } + $r.Exists = $true + $text = Get-Content -Raw $ReportFile + $r.Text = $text + $dup = Get-RegexGroup $text '- duplicates: (\d+)' + if ($dup) { $r.Duplicates = [int]$dup } + $lm = Get-RegexGroup $text '- live_false_MissedDeadline: (\d+)' + $lu = Get-RegexGroup $text '- live_false_Unknown: (\d+)' + if ($lm) { $r.LiveMissed = [int]$lm } + if ($lu) { $r.LiveUnknown = [int]$lu } + $r.Request = Read-Fraction $text 'single_request_loss_recovery' + $r.Response = Read-Fraction $text 'single_response_loss_recovery' + $r.Graceful = Read-Fraction $text 'graceful_unknown_detection_rate' + $r.HardStop = Read-Fraction $text 'missed_deadline_detection_rate' + $drift = Get-RegexGroup $text '- phase_drift_max_ms: ([0-9.]+)' + if ($drift) { $r.PhaseDriftMax = [double]$drift } + $inv = Get-RegexGroup $text '- invalid_metric_count: (\d+)' + if ($inv) { $r.InvalidCount = [int]$inv } + if ($text -match '1e12|1e\+12|1\.0+e\+12') { + $r.InvalidReasons += 'INVALID METRIC: mixed-clock 1e12 timestamp in report' + } + if ($null -ne $r.PhaseDriftMax -and $r.PhaseDriftMax -gt 5000) { + $r.InvalidReasons += "INVALID METRIC: phase_drift_max_ms=$($r.PhaseDriftMax) is not physical" + } + if ($text -match '(^|[^\d])53000([^\d]|$)') { + $r.InvalidReasons += 'INVALID METRIC: ~53000 ms drift marker present' + } + return $r +} + +function Test-ScenarioSemantics { + param($Parsed, [string]$Expect) + $fail = @() + if (-not $Parsed.Exists) { return @('missing report.md') } + if ($null -ne $Parsed.Duplicates -and $Parsed.Duplicates -ne 0) { + $fail += "duplicates=$($Parsed.Duplicates)" + } + if ($null -ne $Parsed.LiveMissed -and $Parsed.LiveMissed -ne 0) { + $fail += "live_false_MissedDeadline=$($Parsed.LiveMissed)" + } + if ($null -ne $Parsed.LiveUnknown -and $Parsed.LiveUnknown -ne 0) { + $fail += "live_false_Unknown=$($Parsed.LiveUnknown)" + } + foreach ($reason in $Parsed.InvalidReasons) { $fail += $reason } + switch ($Expect) { + 'request' { + if ($Parsed.Request.N -le 0 -or $Parsed.Request.Hit -lt $Parsed.Request.N) { + $fail += "request-loss $($Parsed.Request.Hit)/$($Parsed.Request.N)" + } + } + 'response' { + if ($Parsed.Response.N -le 0 -or $Parsed.Response.Hit -lt $Parsed.Response.N) { + $fail += "response-loss $($Parsed.Response.Hit)/$($Parsed.Response.N)" + } + } + 'graceful' { + if ($Parsed.Graceful.N -le 0 -or $Parsed.Graceful.Hit -lt $Parsed.Graceful.N) { + $fail += "graceful Unknown $($Parsed.Graceful.Hit)/$($Parsed.Graceful.N) (want state 2)" + } + } + 'hard' { + if ($Parsed.HardStop.N -le 0 -or $Parsed.HardStop.Hit -lt $Parsed.HardStop.N) { + $fail += "hard-stop MissedDeadline $($Parsed.HardStop.Hit)/$($Parsed.HardStop.N) (want state 1)" + } + } + } + return $fail +} + +function Format-Duration([double]$Seconds) { + $ts = [TimeSpan]::FromSeconds([math]::Max(0, $Seconds)) + return ('{0}h {1:00}m {2:00}s' -f [int][math]::Floor($ts.TotalHours), $ts.Minutes, $ts.Seconds) +} + +function Round-Down([int]$Value, [int]$Mult) { + if ($Value -lt 0) { return 0 } + return [int]([math]::Floor($Value / $Mult) * $Mult) +} + +function Split-Even([int]$Total, [int]$N) { + $out = New-Object int[] $N + $base = [int][math]::Floor($Total / $N) + $rem = $Total % $N + for ($i = 0; $i -lt $N; $i++) { + $out[$i] = $base + $(if ($i -lt $rem) { 1 } else { 0 }) + } + return $out +} + +function Get-ProfileCost($M, [int]$Nom, [int]$Req, [int]$Resp, [int]$Grace, [int]$Hard, [int]$Shards, [int]$ProcessStarts = 1) { + $fixed = [double]$M.fixed_process_seconds + if ($fixed -lt 0) { $fixed = 0 } + $nomS = [double]$M.nominal_case_seconds + if ($nomS -lt 0) { $nomS = 0 } + $reqS = [double]$M.request_loss_case_seconds + $respS = [double]$M.response_loss_case_seconds + $graceS = [double]$M.graceful_stop_case_seconds + $hardS = [double]$M.hard_stop_case_seconds + if ($reqS -lt $nomS) { $reqS = $nomS } + if ($respS -lt $nomS) { $respS = $nomS } + if ($reqS -lt 0) { $reqS = $nomS } + if ($respS -lt 0) { $respS = $nomS } + if ($graceS -lt 0) { $graceS = $nomS } + if ($hardS -lt 0) { $hardS = $nomS } + $starts = [math]::Max(1, $ProcessStarts) + return ($Shards * $fixed * $starts) + + ($Nom * $nomS) + + ($Req * $reqS) + + ($Resp * $respS) + + ($Grace * $graceS) + + ($Hard * $hardS) +} + +function Get-ShardTimeoutSeconds([double]$Predicted) { + return [math]::Max(($Predicted * 1.25) + 300.0, $Predicted + 600.0) +} + +function Read-ProbeElapsed([string]$Kind) { + $p = Join-Path $ProbeRoot "$Kind\probe.json" + if (-not (Test-Path $p)) { return $null } + try { + return Get-Content -Raw $p | ConvertFrom-Json + } catch { + return $null + } +} + +function Convert-MeasurementToEffective($M, $Probe) { + $fixed = [double]$M.fixed_process_seconds + if ($fixed -lt 0) { $fixed = 0 } + $calibNom = [double]$M.nominal_case_seconds + $observed = $null + $raw = $null + if ($Probe -and [int]$Probe.cycles -gt 0) { + $elapsed = [double]$Probe.elapsed_seconds + $raw = $elapsed / [double]$Probe.cycles + $observed = ($elapsed - $fixed) / [double]$Probe.cycles + } + $effNom = $calibNom + if ($null -ne $observed) { $effNom = [math]::Max($effNom, $observed) } + $effNom = [math]::Max($effNom, $PingCadenceSeconds) + $req = [math]::Max([double]$M.request_loss_case_seconds, $effNom) + $resp = [math]::Max([double]$M.response_loss_case_seconds, $effNom) + if ($req -lt 0) { $req = $effNom } + if ($resp -lt 0) { $resp = $effNom } + $grace = [double]$M.graceful_stop_case_seconds + $hard = [double]$M.hard_stop_case_seconds + if ($grace -lt 0) { $grace = $effNom } + if ($hard -lt 0) { $hard = $effNom } + $out = [ordered]@{} + foreach ($p in $M.PSObject.Properties) { $out[$p.Name] = $p.Value } + if ($M -is [System.Collections.IDictionary]) { + $out = [ordered]@{} + foreach ($k in $M.Keys) { $out[$k] = $M[$k] } + } + $out.fixed_process_seconds = $fixed + $out.nominal_case_seconds = [math]::Round($effNom, 6) + $out.request_loss_case_seconds = [math]::Round($req, 6) + $out.response_loss_case_seconds = [math]::Round($resp, 6) + $out.graceful_stop_case_seconds = [math]::Round($grace, 6) + $out.hard_stop_case_seconds = [math]::Round($hard, 6) + $out.calibration_nominal_case_seconds = [math]::Round($calibNom, 6) + $out.observed_nominal_case_seconds = $(if ($null -ne $observed) { [math]::Round($observed, 6) } else { $null }) + $out.raw_elapsed_per_cycle_seconds = $(if ($null -ne $raw) { [math]::Round($raw, 6) } else { $null }) + $out.process_starts_per_shard = $StartsPerShard + return $out +} + +function Get-CorrectedCalib($Calib) { + $out = [ordered]@{} + foreach ($p in $Calib.PSObject.Properties) { $out[$p.Name] = $p.Value } + if ($Calib -is [System.Collections.IDictionary]) { + $out = [ordered]@{} + foreach ($k in $Calib.Keys) { $out[$k] = $Calib[$k] } + } + if ($Calib.tcp) { + $out.tcp = Convert-MeasurementToEffective $Calib.tcp (Read-ProbeElapsed 'tcp') + } + if ($Calib.udp) { + $out.udp = Convert-MeasurementToEffective $Calib.udp (Read-ProbeElapsed 'udp') + } + $out.plan_model = 'v2-cadence-floor-faults-before-nominal' + return $out +} + +function Write-JsonFile([string]$Path, $Object) { + $dir = Split-Path $Path -Parent + if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } + $tmp = "$Path.tmp" + ($Object | ConvertTo-Json -Depth 16) | Set-Content -Path $tmp -Encoding utf8 + Move-Item -Force $tmp $Path +} + +function Get-ExeSha256([string]$Path) { + return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash +} + +function Get-CMakeBuildType([string]$BuildDir) { + $cache = Join-Path $BuildDir 'CMakeCache.txt' + if (-not (Test-Path $cache)) { return 'unknown' } + foreach ($line in Get-Content -LiteralPath $cache) { + if ($line -match '^CMAKE_BUILD_TYPE:STRING=(.*)$') { + $v = $Matches[1].Trim() + if ($v) { return $v } + } + } + return 'Release (Visual Studio multi-config; using Release executable)' +} + +function Get-PowerLineStatus { + try { + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + return [System.Windows.Forms.SystemInformation]::PowerStatus.PowerLineStatus.ToString() + } catch { + } + try { + $batteries = @(Get-CimInstance Win32_Battery -ErrorAction Stop) + if ($batteries.Count -eq 0) { + return 'Online' + } + $st = [int]$batteries[0].BatteryStatus + if ($st -eq 2) { return 'Online' } + if ($st -eq 1) { return 'Offline' } + return "BatteryStatus=$st" + } catch { + return 'unavailable' + } +} + +function Get-PowerScheme { + try { return ((powercfg /getactivescheme) | Out-String).Trim() } + catch { return 'unavailable' } +} + +function Get-ActiveAdapters { + try { + $rows = Get-NetAdapter -ErrorAction Stop | Where-Object { $_.Status -eq 'Up' } | + ForEach-Object { '{0} ({1})' -f $_.Name, $_.InterfaceDescription } + if ($rows) { return @($rows) } + return @('none') + } catch { + return @('unavailable') + } +} + +function Get-TrackedClean { + Push-Location $Root + try { + $out = git status --porcelain --untracked-files=no + return [string]::IsNullOrWhiteSpace($out) + } finally { + Pop-Location + } +} + +function Get-HeadCommit { + Push-Location $Root + try { return (git rev-parse HEAD).Trim() } + finally { Pop-Location } +} + +function Get-CharArgs { + param( + [string]$Kind, + [string]$ArtifactDir, + [int]$Seed, + [int]$Nominal, + [int]$Request, + [int]$Response, + [int]$Grace, + [int]$Hard + ) + return @( + '--quick', + '--no-long-characterization', + "--transport=$Kind", + '--seed', "$Seed", + '--artifact-dir', $ArtifactDir, + '--cycles', "$Nominal", + '--request-loss-cases', "$Request", + '--response-loss-cases', "$Response", + '--graceful-stop-cases', "$Grace", + '--hard-stop-cases', "$Hard", + '--loss-cases', '0', + '--window-samples-main', '0', + '--window-samples-extra', '0' + ) +} + +function New-EmptySampleReason([string]$Why) { + return [ordered]@{ + run_id = '' + shard_id = '' + transport = '' + seed = '' + scenario = '' + cycle_index = '' + monotonic_query_start_ms = '' + monotonic_request_send_ms = '' + monotonic_response_receive_ms = '' + rtt_ms = '' + nominal_due_ms = '' + retry_start_ms = '' + retry_relative_to_nominal_ms = '' + deadline_ms = '' + state_emit_ms = '' + deadline_detection_delay_ms = '' + guard_ms = '' + guard_rtt_statistic_ms = '' + peer_state = '' + duplicate_count = '' + valid = $false + invalid_reason = $Why + } +} + +function Write-ShardSamplesFromTrace { + param( + [string]$ShardDir, + [string]$ShardId, + [string]$Kind, + [int]$Seed, + [string]$RunId + ) + $trace = Join-Path $ShardDir 'bob_ping_trace.csv' + $out = Join-Path $ShardDir 'samples.jsonl' + $tmp = "$out.tmp" + $lines = New-Object System.Collections.Generic.List[string] + if (-not (Test-Path $trace)) { + $empty = New-EmptySampleReason 'bob_ping_trace.csv was not produced; durations not invented' + $lines.Add(($empty | ConvertTo-Json -Compress -Depth 6)) + } else { + $rows = Import-Csv $trace + foreach ($row in $rows) { + $rec = [ordered]@{ + run_id = $RunId + shard_id = $ShardId + transport = $Kind + seed = $Seed + scenario = 'bob_ping_trace' + cycle_index = $row.cycle + monotonic_query_start_ms = '' + monotonic_request_send_ms = '' + monotonic_response_receive_ms = '' + rtt_ms = '' + nominal_due_ms = '' + retry_start_ms = '' + retry_relative_to_nominal_ms = '' + deadline_ms = '' + state_emit_ms = '' + deadline_detection_delay_ms = '' + guard_ms = $(if ($row.guard_us) { [math]::Round(([double]$row.guard_us) / 1000.0, 3) } else { '' }) + guard_rtt_statistic_ms = '' + peer_state = '' + duplicate_count = '' + valid = $true + invalid_reason = '' + event_kind = $row.kind + event_steady_us = $row.event_steady_us + event_qpc = $row.event_qpc + notes = 'Alice/Bob cross-process timestamps left empty; Bob event_steady_us/event_qpc are same-process monotonic fields' + } + $lines.Add(($rec | ConvertTo-Json -Compress -Depth 6)) + } + } + $lines -join "`n" | Set-Content -Path $tmp -Encoding utf8 + Move-Item -Force $tmp $out +} + +function Get-Wilson95([int]$Hits, [int]$N) { + if ($N -le 0) { return @{ lo = $null; hi = $null } } + $z = 1.96 + $p = $Hits / [double]$N + $z2 = $z * $z + $den = 1.0 + ($z2 / $N) + $center = ($p + ($z2 / (2.0 * $N))) / $den + $margin = ($z * [math]::Sqrt((($p * (1.0 - $p)) + ($z2 / (4.0 * $N))) / $N)) / $den + return @{ lo = [math]::Max(0, $center - $margin); hi = [math]::Min(1, $center + $margin) } +} + +function Get-Percentile([double[]]$Values, [double]$P) { + if (-not $Values -or $Values.Count -eq 0) { return $null } + $s = $Values | Sort-Object + $idx = [int][math]::Ceiling($P * ($s.Count - 1)) + if ($idx -lt 0) { $idx = 0 } + if ($idx -ge $s.Count) { $idx = $s.Count - 1 } + return $s[$idx] +} + +function Get-CalibrationScenarios { + return @( + @{ Name = 'nominal_1'; Nominal = 1; Req = 0; Resp = 0; Grace = 0; Hard = 0; Expect = 'nominal' }, + @{ Name = 'nominal_121'; Nominal = 121; Req = 0; Resp = 0; Grace = 0; Hard = 0; Expect = 'nominal' }, + @{ Name = 'request_loss_20'; Nominal = 0; Req = 20; Resp = 0; Grace = 0; Hard = 0; Expect = 'request' }, + @{ Name = 'response_loss_20'; Nominal = 0; Req = 0; Resp = 20; Grace = 0; Hard = 0; Expect = 'response' }, + @{ Name = 'graceful_10'; Nominal = 0; Req = 0; Resp = 0; Grace = 10; Hard = 0; Expect = 'graceful' }, + @{ Name = 'hard_stop_8'; Nominal = 0; Req = 0; Resp = 0; Grace = 0; Hard = 8; Expect = 'hard' } + ) +} + +function Invoke-CalibrationKind { + param([string]$Kind, [string]$Exe) + $kindDir = Join-Path $CalibRoot $Kind + New-Item -ItemType Directory -Path $kindDir -Force | Out-Null + $elapsed = [ordered]@{} + $semantic = [ordered]@{} + $invalid = @() + $allPass = $true + foreach ($sc in Get-CalibrationScenarios) { + $dir = Join-Path $kindDir $sc.Name + New-Item -ItemType Directory -Path $dir -Force | Out-Null + Write-Info "Calibrate $Kind $($sc.Name)" + $args = Get-CharArgs -Kind $Kind -ArtifactDir $dir -Seed 1 ` + -Nominal $sc.Nominal -Request $sc.Req -Response $sc.Resp -Grace $sc.Grace -Hard $sc.Hard + $run = Invoke-TimedProcess -FilePath $Exe -ArgumentList $args ` + -LogPath (Join-Path $dir 'run.log') -TimeoutMs $CalibTimeoutMs -WorkDir $Root + $elapsed[$sc.Name] = [math]::Round($run.ElapsedSec, 3) + $parsed = Parse-CharReport (Join-Path $dir 'report.md') + $fail = @() + if ($run.TimedOut -or $run.ExitCode -ne 0) { + $fail += "process exit=$($run.ExitCode) timeout=$($run.TimedOut)" + } + $fail += Test-ScenarioSemantics -Parsed $parsed -Expect $sc.Expect + if ($fail.Count -gt 0) { + $allPass = $false + $semantic[$sc.Name] = 'FAIL: ' + ($fail -join '; ') + $invalid += $fail + Write-Info "Calibrate $Kind $($sc.Name) FAIL elapsed=$($elapsed[$sc.Name])s $($semantic[$sc.Name])" + throw "Calibration semantic/process failure for $Kind $($sc.Name): $($semantic[$sc.Name])" + } + $semantic[$sc.Name] = 'PASS' + Write-Info "Calibrate $Kind $($sc.Name) PASS elapsed=$($elapsed[$sc.Name])s" + } + $nom1 = [double]$elapsed.nominal_1 + $nom121 = [double]$elapsed.nominal_121 + $nominalCase = ($nom121 - $nom1) / 120.0 + $fixed = $nom1 - $nominalCase + $fixedSuspicious = $false + if ($fixed -lt 0) { + $fixed = 0 + $fixedSuspicious = $true + } + $reqCase = ([double]$elapsed.request_loss_20 - $fixed) / 20.0 + $respCase = ([double]$elapsed.response_loss_20 - $fixed) / 20.0 + $graceCase = ([double]$elapsed.graceful_10 - $fixed) / 10.0 + $hardCase = ([double]$elapsed.hard_stop_8 - $fixed) / 8.0 + foreach ($n in @('request_loss_case_seconds', 'response_loss_case_seconds', 'graceful_stop_case_seconds', 'hard_stop_case_seconds', 'nominal_case_seconds')) { + # keep measured values even if small; negative per-case is suspicious + } + return [ordered]@{ + fixed_process_seconds = [math]::Round($fixed, 6) + fixed_suspicious = $fixedSuspicious + nominal_case_seconds = [math]::Round($nominalCase, 6) + request_loss_case_seconds = [math]::Round($reqCase, 6) + response_loss_case_seconds = [math]::Round($respCase, 6) + graceful_stop_case_seconds = [math]::Round($graceCase, 6) + hard_stop_case_seconds = [math]::Round($hardCase, 6) + elapsed = $elapsed + semantic = $semantic + invalid_metrics = $invalid + result = $(if ($allPass) { 'PASS' } else { 'FAIL' }) + } +} + +function New-TransportPlan($M, [int]$Nom, [int]$Req, [int]$Resp, [int]$Grace, [int]$Hard, [int]$Shards, [string]$Kind) { + $noms = Split-Even ([int]$Nom) ([int]$Shards) + $reqs = Split-Even ([int]$Req) ([int]$Shards) + $resps = Split-Even ([int]$Resp) ([int]$Shards) + $graces = Split-Even ([int]$Grace) ([int]$Shards) + $hards = Split-Even ([int]$Hard) ([int]$Shards) + $shardsOut = @() + for ($i = 0; $i -lt $Shards; $i++) { + $pred = Get-ProfileCost $M ([int]$noms[$i]) ([int]$reqs[$i]) ([int]$resps[$i]) ([int]$graces[$i]) ([int]$hards[$i]) 1 $StartsPerShard + $timeout = Get-ShardTimeoutSeconds $pred + $faultPred = Get-ProfileCost $M 0 ([int]$reqs[$i]) ([int]$resps[$i]) ([int]$graces[$i]) ([int]$hards[$i]) 1 1 + $nominalPred = Get-ProfileCost $M ([int]$noms[$i]) 0 0 0 0 1 1 + $id = '{0}-{1:00}' -f $Kind, ($i + 1) + $seed = $(if ($Kind -eq 'udp') { 2000 } else { 1000 }) + $i + 1 + $shardsOut += [ordered]@{ + shard_id = $id + transport = $Kind + seed = $seed + nominal = [int]$noms[$i] + request_loss = [int]$reqs[$i] + response_loss = [int]$resps[$i] + graceful_stop = [int]$graces[$i] + hard_stop = [int]$hards[$i] + predicted_seconds = [math]::Round($pred, 3) + timeout_seconds = [math]::Round($timeout, 3) + faults_predicted_seconds = [math]::Round($faultPred, 3) + faults_timeout_seconds = [math]::Round((Get-ShardTimeoutSeconds $faultPred), 3) + nominal_predicted_seconds = [math]::Round($nominalPred, 3) + nominal_timeout_seconds = [math]::Round((Get-ShardTimeoutSeconds $nominalPred), 3) + process_starts = $StartsPerShard + } + } + return [ordered]@{ + totals = [ordered]@{ + nominal = $Nom + request_loss = $Req + response_loss = $Resp + graceful_stop = $Grace + hard_stop = $Hard + } + shards = $shardsOut + predicted_seconds = [math]::Round((Get-ProfileCost $M $Nom $Req $Resp $Grace $Hard $Shards $StartsPerShard), 3) + } +} + +function New-LongPlan($Calib, [double]$Hours) { + $Calib = Get-CorrectedCalib $Calib + $requested = $Hours * 3600.0 + $reserve = 20.0 * 60.0 + $plannedTarget = $requested - $reserve + $minBound = $PlanV2MinSeconds + $maxBound = $PlanV2MaxSeconds + $shards = 10 + $minNom = 12000 + $startNom = 13000 + $maxNom = 14000 + $fault = [ordered]@{ req = 300; resp = 300; grace = 200; hard = 250 } + if (-not $Calib.tcp -and -not $Calib.udp) { throw 'Calibration is missing transport measurements.' } + + function Cost-All([int]$Nom) { + $c = 0.0 + if ($Calib.tcp) { + $c += Get-ProfileCost $Calib.tcp $Nom $fault.req $fault.resp $fault.grace $fault.hard $shards $StartsPerShard + } + if ($Calib.udp) { + $c += Get-ProfileCost $Calib.udp $Nom $fault.req $fault.resp $fault.grace $fault.hard $shards $StartsPerShard + } + return $c + } + + $nom = $startNom + $pred = Cost-All $nom + $assumptions = @() + $belowMin = $false + if ($pred -gt $maxBound) { + while ($pred -gt $maxBound -and $nom -gt $minNom) { + $nom = $nom - 100 + $pred = Cost-All $nom + } + if ($pred -gt $maxBound -and $nom -le $minNom) { + $assumptions += "Keeping at least $minNom nominal per transport was not possible inside 9h45m with $StartsPerShard process starts/shard; using $nom." + } + } elseif ($pred -lt $minBound) { + while ($pred -lt $minBound -and $nom -lt $maxNom) { + $nom = $nom + 100 + $pred = Cost-All $nom + if ($pred -gt $maxBound) { + $nom = $nom - 100 + $pred = Cost-All $nom + break + } + } + } + $nom = [int](Round-Down $nom 100) + $pred = Cost-All $nom + if ($pred -gt $maxBound -and $nom -ge ($minNom + 100)) { + while ($pred -gt $maxBound -and $nom -ge ($minNom + 100)) { + $nom = $nom - 100 + $pred = Cost-All $nom + } + } + if ($nom -gt $maxNom) { + $assumptions += "Nominal $nom exceeds $maxNom; not used." + $nom = $maxNom + $pred = Cost-All $nom + } + if ($pred -gt $maxBound) { $belowMin = $true } + + $assumptions += 'effective_nominal_case_seconds = max(observed_long_run_or_probe, calibration_nominal, 1.0s ping cadence).' + $assumptions += 'Request/response planning uses max(calibrated loss cost, effective nominal); negative/near-zero loss estimates are rejected.' + $assumptions += 'Graceful/hard-stop keep calibration measurements.' + $assumptions += "Each shard runs faults then the nominal block ($StartsPerShard process starts; startup counted each time)." + $assumptions += 'Shard timeout = max(predicted*1.25 + 5 minutes, predicted + 10 minutes).' + $assumptions += 'TCP pre-deadline retry is measured, not a pass/fail gate.' + + $tcpPlan = $null + $udpPlan = $null + if ($Calib.tcp) { + $tcpPlan = New-TransportPlan $Calib.tcp $nom $fault.req $fault.resp $fault.grace $fault.hard $shards 'tcp' + } + if ($Calib.udp) { + $udpPlan = New-TransportPlan $Calib.udp $nom $fault.req $fault.resp $fault.grace $fault.hard $shards 'udp' + } + $order = @() + for ($i = 1; $i -le $shards; $i++) { + if ($tcpPlan) { $order += ('tcp-{0:00}' -f $i) } + if ($udpPlan) { $order += ('udp-{0:00}' -f $i) } + } + $pred = 0.0 + if ($tcpPlan) { $pred += $tcpPlan.predicted_seconds } + if ($udpPlan) { $pred += $udpPlan.predicted_seconds } + return [ordered]@{ + plan_version = 2 + requested_seconds = $requested + planned_target_seconds = $plannedTarget + safety_reserve_seconds = $reserve + min_bound_seconds = $minBound + max_bound_seconds = $maxBound + predicted_seconds = [math]::Round($pred, 3) + predicted_duration = (Format-Duration $pred) + predicted_in_range = ($pred -ge $minBound -and $pred -le $maxBound) + minimum_exceeds_target = $belowMin + shard_count_per_transport = $shards + process_starts_per_shard = $StartsPerShard + corrected_tcp = $(if ($Calib.tcp) { $Calib.tcp } else { $null }) + corrected_udp = $(if ($Calib.udp) { $Calib.udp } else { $null }) + shard_order = $order + tcp = $tcpPlan + udp = $udpPlan + assumptions = $assumptions + later_command = 'scripts/run_uap_long_characterization.ps1 -Run -Transport both -TargetHours 10 -NoBuild -Resume' + runs_root = $RunsRoot + } +} + +function New-DryPlan($Calib) { + $Calib = Get-CorrectedCalib $Calib + if (-not $Calib.tcp) { throw 'DryRun requires TCP calibration measurements.' } + $m = $Calib.tcp + $nom = 20 + $req = 2 + $resp = 2 + $grace = 2 + $hard = 2 + $pred = Get-ProfileCost $m $nom $req $resp $grace $hard 1 $StartsPerShard + $timeout = Get-ShardTimeoutSeconds $pred + $faultPred = Get-ProfileCost $m 0 $req $resp $grace $hard 1 1 + $nominalPred = Get-ProfileCost $m $nom 0 0 0 0 1 1 + $shard = [ordered]@{ + shard_id = 'tcp-test' + transport = 'tcp' + seed = 9101 + nominal = $nom + request_loss = $req + response_loss = $resp + graceful_stop = $grace + hard_stop = $hard + predicted_seconds = [math]::Round($pred, 3) + timeout_seconds = [math]::Round($timeout, 3) + faults_predicted_seconds = [math]::Round($faultPred, 3) + faults_timeout_seconds = [math]::Round((Get-ShardTimeoutSeconds $faultPred), 3) + nominal_predicted_seconds = [math]::Round($nominalPred, 3) + nominal_timeout_seconds = [math]::Round((Get-ShardTimeoutSeconds $nominalPred), 3) + process_starts = $StartsPerShard + } + return [ordered]@{ + plan_version = 'dry-v2' + predicted_seconds = $shard.predicted_seconds + predicted_duration = (Format-Duration $pred) + process_starts_per_shard = $StartsPerShard + corrected_tcp = $m + corrected_udp = $null + shard_order = @('tcp-test') + tcp = [ordered]@{ + totals = [ordered]@{ + nominal = $nom; request_loss = $req; response_loss = $resp + graceful_stop = $grace; hard_stop = $hard + } + shards = @($shard) + predicted_seconds = $shard.predicted_seconds + } + udp = $null + later_command = 'dry-run only' + runs_root = (Join-Path $ArtifactRoot 'dry-run-v2') + } +} + +function Write-PlanMarkdown($Plan, [string]$Path) { + $lines = @( + '# UAP long characterization plan', + "- predicted duration: $($Plan.predicted_duration) ($($Plan.predicted_seconds)s)", + "- safety reserve: $(Format-Duration $Plan.safety_reserve_seconds)", + "- shard count per transport: $($Plan.shard_count_per_transport)", + "- predicted in 9h20m-9h45m window: $($Plan.predicted_in_range)", + "- minimum profile exceeds target: $($Plan.minimum_exceeds_target)", + '' + ) + if ($Plan.tcp) { + $t = $Plan.tcp.totals + $lines += 'TCP totals:' + $lines += "- nominal: $($t.nominal)" + $lines += "- request-loss: $($t.request_loss)" + $lines += "- response-loss: $($t.response_loss)" + $lines += "- graceful-stop: $($t.graceful_stop)" + $lines += "- hard-stop: $($t.hard_stop)" + $lines += "- predicted: $(Format-Duration $Plan.tcp.predicted_seconds)" + $lines += 'TCP per shard:' + foreach ($s in $Plan.tcp.shards) { + $lines += "- $($s.shard_id): nominal=$($s.nominal) req=$($s.request_loss) resp=$($s.response_loss) grace=$($s.graceful_stop) hard=$($s.hard_stop) seed=$($s.seed) pred=$(Format-Duration $s.predicted_seconds) timeout=$(Format-Duration $s.timeout_seconds) faults_pred=$(Format-Duration $s.faults_predicted_seconds) faults_timeout=$(Format-Duration $s.faults_timeout_seconds) nominal_pred=$(Format-Duration $s.nominal_predicted_seconds) nominal_timeout=$(Format-Duration $s.nominal_timeout_seconds)" + } + $lines += '' + } + if ($Plan.udp) { + $t = $Plan.udp.totals + $lines += 'UDP totals:' + $lines += "- nominal: $($t.nominal)" + $lines += "- request-loss: $($t.request_loss)" + $lines += "- response-loss: $($t.response_loss)" + $lines += "- graceful-stop: $($t.graceful_stop)" + $lines += "- hard-stop: $($t.hard_stop)" + $lines += "- predicted: $(Format-Duration $Plan.udp.predicted_seconds)" + $lines += 'UDP per shard:' + foreach ($s in $Plan.udp.shards) { + $lines += "- $($s.shard_id): nominal=$($s.nominal) req=$($s.request_loss) resp=$($s.response_loss) grace=$($s.graceful_stop) hard=$($s.hard_stop) seed=$($s.seed) pred=$(Format-Duration $s.predicted_seconds) timeout=$(Format-Duration $s.timeout_seconds) faults_pred=$(Format-Duration $s.faults_predicted_seconds) faults_timeout=$(Format-Duration $s.faults_timeout_seconds) nominal_pred=$(Format-Duration $s.nominal_predicted_seconds) nominal_timeout=$(Format-Duration $s.nominal_timeout_seconds)" + } + $lines += '' + } + $lines += 'Shard order:' + $lines += '- ' + ($Plan.shard_order -join ', ') + $lines += '' + $lines += 'Assumptions:' + foreach ($a in $Plan.assumptions) { $lines += "- $a" } + $lines += '' + $lines += 'Later command (not executed by calibration):' + $lines += $Plan.later_command + $lines | Set-Content -Path $Path -Encoding utf8 +} + +function Write-PlanV2Documents($Plan, $Calib) { + Write-JsonFile $PlanV2Json $Plan + $lines = @() + $lines += '# UAP long characterization plan v2' + $lines += '' + $lines += '## Why the original plan failed' + $lines += 'The v1 planner used a 1-vs-121 calibration slope of ~0.55s (TCP) / ~0.43s (UDP) per nominal cycle. A 1-second ping cadence cannot be cheaper than 1.0s once startup is excluded, and 2650 nominal cycles alone need ~44m10s before fault/stop cases. v1 TCP shard timeout was 43m54s and UDP 35m37s, both below that floor, so every shard died in the mixed cycle loop. stdout never reached `Hard-stop runs=` / `Graceful-close runs=`. Timed-out shards were not marked completed.' + $lines += '' + $lines += '## Observed timeout durations (v1 run)' + $stop = $null + $stopPath = Join-Path $ArtifactRoot 'stop-record.json' + if (Test-Path $stopPath) { + $stop = Get-Content -Raw $stopPath | ConvertFrom-Json + $lines += "- stop_time: $($stop.stop_time)" + $lines += "- active_shard_at_stop: $($stop.active_shard)" + $lines += "- wall_elapsed: $($stop.wall_elapsed_seconds)s ($(Format-Duration $stop.wall_elapsed_seconds))" + $lines += "- parent_powershell_pid: $($stop.parent_powershell_pid) already_exited=$($stop.parent_powershell_already_exited)" + $lines += "- coordinator_pid_tcp04: $($stop.coordinator_pid_tcp04) already_exited=$($stop.coordinator_already_exited)" + $lines += '- killed_pids: ' + (($stop.killed_pids | ForEach-Object { '{0}({1})' -f $_.pid, $_.role }) -join ', ') + } + foreach ($id in @('tcp-01','udp-01','tcp-02','udp-02','tcp-03','udp-03','tcp-04')) { + $fj = Join-Path $LegacyRunsRoot "$id\FAILED.json" + $stdout = Join-Path $LegacyRunsRoot "$id\run.log.stdout.txt" + $elapsed = 'n/a' + $reasons = 'n/a' + if (Test-Path $fj) { + $j = Get-Content -Raw $fj | ConvertFrom-Json + $elapsed = '{0}s ({1})' -f $j.elapsed_seconds, (Format-Duration $j.elapsed_seconds) + $reasons = ($j.reasons -join '; ') + } + $last = '' + $lastWrite = '' + if (Test-Path $stdout) { + $last = ((Get-Content $stdout | Where-Object { $_ }) | Select-Object -Last 1) + $lastWrite = (Get-Item $stdout).LastWriteTime.ToString('o') + } + $lines += "- ${id}: elapsed=$elapsed last_stdout='$last' last_stdout_write=$lastWrite reasons=$reasons" + $lines += " completed nominal: not recoverable (no per-cycle output; no report.md; no bob_ping_trace.csv). last scenario: warmup complete, entered 2710 mixed logical cycles. request/response/graceful/hard-stop: no evidence they completed; graceful/hard-stop headers were never printed." + } + $lines += '' + $lines += '## Timing probes (required because failed logs did not expose cycle progress)' + $tcpP = Read-ProbeElapsed 'tcp' + $udpP = Read-ProbeElapsed 'udp' + $lines += "- TCP probe: elapsed=$($tcpP.elapsed_seconds)s cycles=600 raw=$($tcpP.raw_seconds_per_cycle)s/cycle exit=$($tcpP.exit_code)" + $lines += "- UDP probe: elapsed=$($udpP.elapsed_seconds)s cycles=600 raw=$($udpP.raw_seconds_per_cycle)s/cycle exit=$($udpP.exit_code)" + if ($Plan.corrected_tcp) { + $c = $Plan.corrected_tcp + $lines += "- TCP observed=(elapsed-startup)/600=$($c.observed_nominal_case_seconds)s calibration_nominal=$($c.calibration_nominal_case_seconds)s raw=$($c.raw_elapsed_per_cycle_seconds)s **effective_nominal=$($c.nominal_case_seconds)s**" + $lines += "- TCP request/response effective=$($c.request_loss_case_seconds)/$($c.response_loss_case_seconds) (max(calibrated, effective nominal))" + $lines += "- TCP graceful/hard-stop kept from calibration: $($c.graceful_stop_case_seconds)/$($c.hard_stop_case_seconds)" + $lines += "- TCP startup/shard-process: $($c.fixed_process_seconds)s" + } + if ($Plan.corrected_udp) { + $c = $Plan.corrected_udp + $lines += "- UDP observed=(elapsed-startup)/600=$($c.observed_nominal_case_seconds)s calibration_nominal=$($c.calibration_nominal_case_seconds)s raw=$($c.raw_elapsed_per_cycle_seconds)s **effective_nominal=$($c.nominal_case_seconds)s**" + $lines += "- UDP request/response effective=$($c.request_loss_case_seconds)/$($c.response_loss_case_seconds) (max(calibrated, effective nominal); negatives rejected)" + $lines += "- UDP graceful/hard-stop kept from calibration: $($c.graceful_stop_case_seconds)/$($c.hard_stop_case_seconds)" + $lines += "- UDP startup/shard-process: $($c.fixed_process_seconds)s" + } + $lines += '' + $lines += '## Old plan under the corrected model' + if ($Calib.tcp -and $Calib.udp -and $Plan.corrected_tcp -and $Plan.corrected_udp) { + $oldTcp = Get-ProfileCost $Plan.corrected_tcp 26500 300 300 200 250 10 1 + $oldUdp = Get-ProfileCost $Plan.corrected_udp 26500 300 300 200 250 10 1 + $oldSum = $oldTcp + $oldUdp + $oldTcpShard = Get-ProfileCost $Plan.corrected_tcp 2650 30 30 20 25 1 1 + $oldUdpShard = Get-ProfileCost $Plan.corrected_udp 2650 30 30 20 25 1 1 + $lines += "- TCP total: $(Format-Duration $oldTcp) ($([math]::Round($oldTcp,3))s)" + $lines += "- UDP total: $(Format-Duration $oldUdp) ($([math]::Round($oldUdp,3))s)" + $lines += "- combined: $(Format-Duration $oldSum) ($([math]::Round($oldSum,3))s) vs v1 predicted 9h 41m 22s" + $lines += "- TCP shard predicted: $(Format-Duration $oldTcpShard); v1 timeout 43m54s; corrected timeout would be $(Format-Duration (Get-ShardTimeoutSeconds $oldTcpShard))" + $lines += "- UDP shard predicted: $(Format-Duration $oldUdpShard); v1 timeout 35m37s; corrected timeout would be $(Format-Duration (Get-ShardTimeoutSeconds $oldUdpShard))" + } + $lines += '' + $lines += '## New counts' + $lines += "- process starts per shard: $($Plan.process_starts_per_shard) (faults process, then nominal process)" + if ($Plan.tcp) { + $t = $Plan.tcp.totals + $s0 = $Plan.tcp.shards[0] + $lines += "- TCP: nominal=$($t.nominal) req=$($t.request_loss) resp=$($t.response_loss) grace=$($t.graceful_stop) hard=$($t.hard_stop) shards=10" + $lines += "- TCP shard predicted: $(Format-Duration $s0.predicted_seconds) ($($s0.predicted_seconds)s)" + $lines += "- TCP shard timeout: $(Format-Duration $s0.timeout_seconds) ($($s0.timeout_seconds)s)" + $lines += "- TCP faults stage predicted/timeout: $(Format-Duration $s0.faults_predicted_seconds) / $(Format-Duration $s0.faults_timeout_seconds)" + $lines += "- TCP nominal stage predicted/timeout: $(Format-Duration $s0.nominal_predicted_seconds) / $(Format-Duration $s0.nominal_timeout_seconds)" + $lines += "- TCP transport predicted: $(Format-Duration $Plan.tcp.predicted_seconds)" + } + if ($Plan.udp) { + $t = $Plan.udp.totals + $s0 = $Plan.udp.shards[0] + $lines += "- UDP: nominal=$($t.nominal) req=$($t.request_loss) resp=$($t.response_loss) grace=$($t.graceful_stop) hard=$($t.hard_stop) shards=10" + $lines += "- UDP shard predicted: $(Format-Duration $s0.predicted_seconds) ($($s0.predicted_seconds)s)" + $lines += "- UDP shard timeout: $(Format-Duration $s0.timeout_seconds) ($($s0.timeout_seconds)s)" + $lines += "- UDP faults stage predicted/timeout: $(Format-Duration $s0.faults_predicted_seconds) / $(Format-Duration $s0.faults_timeout_seconds)" + $lines += "- UDP nominal stage predicted/timeout: $(Format-Duration $s0.nominal_predicted_seconds) / $(Format-Duration $s0.nominal_timeout_seconds)" + $lines += "- UDP transport predicted: $(Format-Duration $Plan.udp.predicted_seconds)" + } + $lines += "- total predicted runtime: $($Plan.predicted_duration) ($($Plan.predicted_seconds)s)" + $lines += "- safety margin / window: $(Format-Duration $Plan.min_bound_seconds) to $(Format-Duration $Plan.max_bound_seconds); reserve=$(Format-Duration $Plan.safety_reserve_seconds); in_range=$($Plan.predicted_in_range)" + $lines += "- future artifacts: $($Plan.runs_root) (does not overwrite artifacts/uap-long/runs)" + $lines += '' + $lines += '## Scenario ordering' + $lines += 'Each shard runs warmup+request-loss+response-loss+graceful-stop+hard-stop in the faults process (existing executable order: mixed loss cycles, then hard-stop, then graceful), then the long nominal block in a second process. Fault semantics are collected before the long nominal section. Aggregate reports keep nominal and fault distributions separate.' + $lines += '' + $lines += '## Future 10-hour command (not started)' + $lines += $Plan.later_command + $lines | Set-Content -Path $PlanV2Md -Encoding utf8 + Write-Info "Wrote $PlanV2Md" + Write-Info "Wrote $PlanV2Json" +} + +function Write-CalibrationReport($Calib, $Plan, [string]$Path) { + $tcpInv = 'none' + $udpInv = 'none' + if ($Calib.tcp -and $Calib.tcp.invalid_metrics -and $Calib.tcp.invalid_metrics.Count -gt 0) { + $tcpInv = $Calib.tcp.invalid_metrics -join '; ' + } + if ($Calib.udp -and $Calib.udp.invalid_metrics -and $Calib.udp.invalid_metrics.Count -gt 0) { + $udpInv = $Calib.udp.invalid_metrics -join '; ' + } + $overall = 'PASS' + if ($Calib.overall -ne 'PASS') { $overall = $Calib.overall } + $lines = @( + '# UAP Long Characterization Calibration', + 'Source:', + "- commit: $($Calib.commit)", + "- tracked working tree clean: $($Calib.tracked_working_tree_clean)", + "- TCP executable: $($Calib.tcp_exe)", + "- UDP executable: $($Calib.udp_exe)", + "- build type: $($Calib.build_type)", + "- build performed: $($Calib.build_performed)", + 'Environment:', + "- timestamp: $($Calib.timestamp)", + "- Windows power scheme: $($Calib.power_scheme)", + "- active network adapters: $($Calib.active_network_adapters -join '; ')", + "- system sleep prevention used: $($Calib.system_sleep_prevention_used)", + 'TCP measured duration:' + ) + if ($Calib.tcp) { + $lines += "- fixed process cost: $($Calib.tcp.fixed_process_seconds)s$(if ($Calib.tcp.fixed_suspicious) { ' (clamped from negative; suspicious)' } else { '' })" + $lines += "- nominal per case: $($Calib.tcp.nominal_case_seconds)s" + $lines += "- request-loss per case: $($Calib.tcp.request_loss_case_seconds)s" + $lines += "- response-loss per case: $($Calib.tcp.response_loss_case_seconds)s" + $lines += "- graceful-stop per case: $($Calib.tcp.graceful_stop_case_seconds)s" + $lines += "- hard-stop per case: $($Calib.tcp.hard_stop_case_seconds)s" + } else { + $lines += '- not run' + } + $lines += 'UDP measured duration:' + if ($Calib.udp) { + $lines += "- fixed process cost: $($Calib.udp.fixed_process_seconds)s$(if ($Calib.udp.fixed_suspicious) { ' (clamped from negative; suspicious)' } else { '' })" + $lines += "- nominal per case: $($Calib.udp.nominal_case_seconds)s" + $lines += "- request-loss per case: $($Calib.udp.request_loss_case_seconds)s" + $lines += "- response-loss per case: $($Calib.udp.response_loss_case_seconds)s" + $lines += "- graceful-stop per case: $($Calib.udp.graceful_stop_case_seconds)s" + $lines += "- hard-stop per case: $($Calib.udp.hard_stop_case_seconds)s" + } else { + $lines += '- not run' + } + $lines += 'Calculated 10-hour plan:' + $lines += "- predicted duration: $($Plan.predicted_duration)" + $lines += "- safety reserve: $(Format-Duration $Plan.safety_reserve_seconds)" + $lines += "- shard count: $($Plan.shard_count_per_transport) per transport" + if ($Plan.tcp) { + $t = $Plan.tcp.totals + $lines += "- total and per-shard counts for TCP: total nominal=$($t.nominal) req=$($t.request_loss) resp=$($t.response_loss) grace=$($t.graceful_stop) hard=$($t.hard_stop); shards=$($Plan.shard_count_per_transport)" + } + if ($Plan.udp) { + $t = $Plan.udp.totals + $lines += "- total and per-shard counts for UDP: total nominal=$($t.nominal) req=$($t.request_loss) resp=$($t.response_loss) grace=$($t.graceful_stop) hard=$($t.hard_stop); shards=$($Plan.shard_count_per_transport)" + } + $lines += "- predicted duration range: $(Format-Duration $Plan.min_bound_seconds) to $(Format-Duration $Plan.max_bound_seconds) (in range: $($Plan.predicted_in_range))" + $lines += '- assumptions: ' + ($Plan.assumptions -join ' ') + $lines += 'Calibration semantic result:' + $lines += "- TCP: $(if ($Calib.tcp) { $Calib.tcp.result } else { 'n/a' })" + $lines += "- UDP: $(if ($Calib.udp) { $Calib.udp.result } else { 'n/a' })" + $lines += "- invalid metrics: TCP=$tcpInv; UDP=$udpInv" + $lines += "- overall: $overall" + $lines | Set-Content -Path $Path -Encoding utf8 +} + +function Test-ShardComplete([string]$Dir) { + $marker = Join-Path $Dir 'COMPLETED.json' + $samples = Join-Path $Dir 'samples.jsonl' + $report = Join-Path $Dir 'report.md' + if (-not ((Test-Path $marker) -and (Test-Path $samples) -and (Test-Path $report))) { + return $false + } + try { + $j = Get-Content -Raw $marker | ConvertFrom-Json + return ($j.ok -eq $true) + } catch { + return $false + } +} + +function Assert-LongPlanGates($Plan) { + $fail = @() + $minSec = $PlanV2MinSeconds + $maxSec = $PlanV2MaxSeconds + $pred = [double]$Plan.predicted_seconds + if ($pred -lt $minSec -or $pred -gt $maxSec) { + $fail += "predicted duration $($Plan.predicted_duration) ($pred s) is outside 9h20m-9h45m" + } + if ([int]$Plan.shard_count_per_transport -ne 10) { + $fail += "shard_count_per_transport=$($Plan.shard_count_per_transport) want 10" + } + $wantFault = @{ + request_loss = 300 + response_loss = 300 + graceful_stop = 200 + hard_stop = 250 + } + $allDirs = @() + foreach ($kind in @('tcp', 'udp')) { + $t = $Plan.$kind + if (-not $t) { + $fail += "missing $kind plan" + continue + } + $shards = @($t.shards) + if ($shards.Count -ne 10) { + $fail += "$kind shard count $($shards.Count) want 10" + } + foreach ($k in @('request_loss', 'response_loss', 'graceful_stop', 'hard_stop')) { + if ([int]$t.totals.$k -ne $wantFault[$k]) { + $fail += "$kind total ${k}=$($t.totals.$k) want $($wantFault[$k])" + } + } + $nom = [int]$t.totals.nominal + if ($nom -lt 12000 -or $nom -gt 14000) { + $fail += "$kind nominal $nom is outside 12000-14000" + } + if (($nom % 100) -ne 0) { + $fail += "$kind nominal $nom is not a multiple of 100" + } + $base = $(if ($kind -eq 'tcp') { 1000 } else { 2000 }) + for ($i = 0; $i -lt $shards.Count; $i++) { + $s = $shards[$i] + $expectId = '{0}-{1:00}' -f $kind, ($i + 1) + if ($s.shard_id -ne $expectId) { + $fail += "$kind shard id $($s.shard_id) want $expectId" + } + if ([int]$s.seed -ne ($base + $i + 1)) { + $fail += "$($s.shard_id) seed $($s.seed) want $($base + $i + 1)" + } + if (([int]$s.nominal % 10) -ne 0) { + $fail += "$($s.shard_id) per-shard nominal $($s.nominal) is not a multiple of 10" + } + $expectTo = Get-ShardTimeoutSeconds ([double]$s.predicted_seconds) + if ([math]::Abs([double]$s.timeout_seconds - $expectTo) -gt 1.5) { + $fail += "$($s.shard_id) timeout $($s.timeout_seconds) != $([math]::Round($expectTo,3))" + } + if ([double]$s.timeout_seconds -le [double]$s.predicted_seconds) { + $fail += "$($s.shard_id) timeout $($s.timeout_seconds) is not greater than predicted $($s.predicted_seconds)" + } + $dir = Join-Path $RunsRoot $s.shard_id + $allDirs += $dir + if ($dir.StartsWith(($CalibRoot.TrimEnd('\') + '\'), [System.StringComparison]::OrdinalIgnoreCase) -or + $dir.Equals($CalibRoot, [System.StringComparison]::OrdinalIgnoreCase)) { + $fail += "$($s.shard_id) output directory is under calibration" + } + } + } + $expectOrder = @() + for ($i = 1; $i -le 10; $i++) { + $expectOrder += ('tcp-{0:00}' -f $i) + $expectOrder += ('udp-{0:00}' -f $i) + } + $gotOrder = @($Plan.shard_order) + if (($gotOrder -join ',') -ne ($expectOrder -join ',')) { + $fail += "shard_order mismatch: $($gotOrder -join ',')" + } + if ((@($allDirs | Select-Object -Unique)).Count -ne $allDirs.Count) { + $fail += 'shard output directories are not distinct' + } + if ($RunsRoot.Equals($CalibRoot, [System.StringComparison]::OrdinalIgnoreCase)) { + $fail += 'runs root equals calibration root' + } + $probe = Join-Path ([System.IO.Path]::GetTempPath()) ('uap-complete-check-' + [guid]::NewGuid().ToString('n')) + New-Item -ItemType Directory -Path $probe | Out-Null + try { + if (Test-ShardComplete $probe) { + $fail += 'Test-ShardComplete true on empty directory' + } + Set-Content -Path (Join-Path $probe 'COMPLETED.json') -Value '{"ok":true}' -Encoding utf8 + if (Test-ShardComplete $probe) { + $fail += 'Test-ShardComplete true without samples.jsonl and report.md' + } + Set-Content -Path (Join-Path $probe 'samples.jsonl') -Value '{}' -Encoding utf8 + Set-Content -Path (Join-Path $probe 'report.md') -Value '# x' -Encoding utf8 + if (-not (Test-ShardComplete $probe)) { + $fail += 'Test-ShardComplete false when COMPLETED.json ok=true plus samples.jsonl and report.md' + } + Set-Content -Path (Join-Path $probe 'COMPLETED.json') -Value '{"ok":false}' -Encoding utf8 + if (Test-ShardComplete $probe) { + $fail += 'Test-ShardComplete true when COMPLETED.json ok=false' + } + } finally { + Remove-Item -Recurse -Force $probe -ErrorAction SilentlyContinue + } + if ($fail.Count -gt 0) { + throw ('Plan gates failed: ' + ($fail -join '; ')) + } + Write-Info 'Plan gates passed' +} + +function Invoke-LongRun($Plan, $Exes) { + $powerLine = Get-PowerLineStatus + if ($powerLine -ne 'Online') { + throw "Refusing to launch: power source is $powerLine (AC power required)." + } + New-Item -ItemType Directory -Path $RunsRoot -Force | Out-Null + New-Item -ItemType Directory -Path $AggRoot -Force | Out-Null + $shardMap = @{} + if ($Plan.tcp) { foreach ($s in $Plan.tcp.shards) { $shardMap[$s.shard_id] = $s } } + if ($Plan.udp) { foreach ($s in $Plan.udp.shards) { $shardMap[$s.shard_id] = $s } } + $order = @($Plan.shard_order) + $started = Get-Date + $plannedFinish = $started.AddSeconds([double]$Plan.predicted_seconds) + $tcpExe = $(if ($Exes.ContainsKey('tcp')) { $Exes['tcp'] } else { '' }) + $udpExe = $(if ($Exes.ContainsKey('udp')) { $Exes['udp'] } else { '' }) + $tcpHash = $(if ($tcpExe) { Get-ExeSha256 $tcpExe } else { '' }) + $udpHash = $(if ($udpExe) { Get-ExeSha256 $udpExe } else { '' }) + $tcpMtime = $(if ($tcpExe) { (Get-Item -LiteralPath $tcpExe).LastWriteTime.ToString('o') } else { '' }) + $udpMtime = $(if ($udpExe) { (Get-Item -LiteralPath $udpExe).LastWriteTime.ToString('o') } else { '' }) + $calibCommit = '' + if (Test-Path $CalibJson) { + try { $calibCommit = [string]((Get-Content -Raw $CalibJson | ConvertFrom-Json).commit) } catch { $calibCommit = '' } + } + $runInfo = [ordered]@{ + source_commit = Get-HeadCommit + calibration_commit = $calibCommit + calibration_report = $CalibReport + runner_pid = $PID + started_at = $started.ToString('o') + planned_finish_at = $plannedFinish.ToString('o') + predicted_duration = $Plan.predicted_duration + first_shard = $(if ($order.Count -gt 0) { $order[0] } else { '' }) + shard_order = $order + tcp_exe = $tcpExe + udp_exe = $udpExe + tcp_exe_sha256 = $tcpHash + udp_exe_sha256 = $udpHash + tcp_exe_mtime = $tcpMtime + udp_exe_mtime = $udpMtime + cmake_build_type_tcp = $(if ($tcpExe) { Get-CMakeBuildType (Find-BuildDir 'tcp') } else { '' }) + cmake_build_type_udp = $(if ($udpExe) { Get-CMakeBuildType (Find-BuildDir 'udp') } else { '' }) + power_scheme = Get-PowerScheme + power_line_status = $powerLine + system_sleep_prevention = 'SetThreadExecutionState(ES_CONTINUOUS|ES_SYSTEM_REQUIRED); display sleep is not prevented' + status_path = $StatusPath + } + Write-JsonFile $RunInfoPath $runInfo + $status = [ordered]@{ + state = 'running' + runner_pid = $PID + source_commit = $runInfo.source_commit + current_shard = $null + last_completed_shard = $null + shards_total = $order.Count + shards_completed = 0 + shards_failed = 0 + shards_skipped_resume = 0 + started_at = $runInfo.started_at + planned_finish_at = $runInfo.planned_finish_at + predicted_duration = $Plan.predicted_duration + first_shard = $runInfo.first_shard + tcp_exe_sha256 = $tcpHash + udp_exe_sha256 = $udpHash + note = 'Resume skips a shard only when COMPLETED.json has ok=true and samples.jsonl and report.md exist.' + } + Write-JsonFile $StatusPath $status + $anyFail = $false + foreach ($id in $order) { + $s = $shardMap[$id] + if (-not $s) { continue } + $dir = Join-Path $RunsRoot $id + New-Item -ItemType Directory -Path $dir -Force | Out-Null + if ($Resume -and (Test-ShardComplete $dir)) { + Write-Info "Resume skip $id" + $status.shards_skipped_resume = [int]$status.shards_skipped_resume + 1 + $status.shards_completed = [int]$status.shards_completed + 1 + $status.last_completed_shard = $id + $status.current_shard = $null + Write-JsonFile $StatusPath $status + continue + } + $status.current_shard = $id + $status.state = 'running' + Write-JsonFile $StatusPath $status + $kind = $s.transport + $exe = $Exes[$kind] + $progressPath = Join-Path $dir 'progress.json' + $nomS = $PingCadenceSeconds + if ($Plan.corrected_tcp -and $kind -eq 'tcp') { $nomS = [double]$Plan.corrected_tcp.nominal_case_seconds } + if ($Plan.corrected_udp -and $kind -eq 'udp') { $nomS = [double]$Plan.corrected_udp.nominal_case_seconds } + $stages = @( + @{ + Name = 'faults' + Dir = Join-Path $dir 'stages\faults' + Nominal = 0 + Request = [int]$s.request_loss + Response = [int]$s.response_loss + Grace = [int]$s.graceful_stop + Hard = [int]$s.hard_stop + TimeoutSec = $(if ($s.faults_timeout_seconds) { [double]$s.faults_timeout_seconds } else { Get-ShardTimeoutSeconds ([double]$s.predicted_seconds) }) + Scenario = 'faults' + Target = [int]$s.request_loss + [int]$s.response_loss + [int]$s.graceful_stop + [int]$s.hard_stop + }, + @{ + Name = 'nominal' + Dir = Join-Path $dir 'stages\nominal' + Nominal = [int]$s.nominal + Request = 0 + Response = 0 + Grace = 0 + Hard = 0 + TimeoutSec = $(if ($s.nominal_timeout_seconds) { [double]$s.nominal_timeout_seconds } else { Get-ShardTimeoutSeconds ([double]$s.predicted_seconds) }) + Scenario = 'nominal' + Target = [int]$s.nominal + } + ) + Write-Info "Run shard $id faults-then-nominal shard_timeout=$(Format-Duration $s.timeout_seconds)" + $fail = @() + $stageElapsed = 0.0 + $timedOut = $false + $exitCode = 0 + foreach ($st in $stages) { + New-Item -ItemType Directory -Path $st.Dir -Force | Out-Null + if ($Resume -and (Test-ShardComplete $st.Dir)) { + Write-Info "Resume skip $id stage=$($st.Name)" + continue + } + $args = Get-CharArgs -Kind $kind -ArtifactDir $st.Dir -Seed ([int]$s.seed) ` + -Nominal ([int]$st.Nominal) -Request ([int]$st.Request) ` + -Response ([int]$st.Response) -Grace ([int]$st.Grace) ` + -Hard ([int]$st.Hard) + Write-JsonFile (Join-Path $st.Dir 'args.json') ([ordered]@{ + shard = $s + stage = $st.Name + arguments = $args + }) + $pb = @{ + shard_id = $id + scenario = [string]$st.Scenario + target = [int]$st.Target + completed_count = 0 + } + Write-JsonFile $progressPath ([ordered]@{ + shard_id = $id + scenario = [string]$st.Scenario + completed_count = 0 + target_count = [int]$st.Target + elapsed_seconds = 0 + rtt_summary = $null + last_sample_timestamp = (Get-Date).ToString('o') + estimated = $false + note = "starting $($st.Name)" + }) + $timeoutMs = [int]([double]$st.TimeoutSec * 1000.0) + Write-Info "Run shard $id stage=$($st.Name) timeout=$(Format-Duration $st.TimeoutSec)" + $run = Invoke-TimedProcess -FilePath $exe -ArgumentList $args ` + -LogPath (Join-Path $st.Dir 'run.log') -TimeoutMs $timeoutMs -WorkDir $Root ` + -ProgressPath $progressPath -ProgressBase $pb -SecondsPerCycle $nomS -ProgressEveryCycles 100 + $stageElapsed += [double]$run.ElapsedSec + $parsed = Parse-CharReport (Join-Path $st.Dir 'report.md') + $stageFail = @() + if ($run.TimedOut) { $stageFail += "$($st.Name) timeout"; $timedOut = $true } + if ($run.ExitCode -ne 0 -and -not $run.TimedOut) { $stageFail += "$($st.Name) exit $($run.ExitCode)" } + $exitCode = $run.ExitCode + if ($null -ne $parsed.Duplicates -and $parsed.Duplicates -ne 0) { $stageFail += "duplicates=$($parsed.Duplicates)" } + if ($null -ne $parsed.LiveMissed -and $parsed.LiveMissed -ne 0) { $stageFail += "live_false_MissedDeadline=$($parsed.LiveMissed)" } + if ($null -ne $parsed.LiveUnknown -and $parsed.LiveUnknown -ne 0) { $stageFail += "live_false_Unknown=$($parsed.LiveUnknown)" } + foreach ($reason in $parsed.InvalidReasons) { $stageFail += $reason } + if ([int]$st.Request -gt 0 -and ($parsed.Request.N -le 0 -or $parsed.Request.Hit -lt $parsed.Request.N)) { + $stageFail += "request-loss $($parsed.Request.Hit)/$($parsed.Request.N)" + } + if ([int]$st.Response -gt 0 -and ($parsed.Response.N -le 0 -or $parsed.Response.Hit -lt $parsed.Response.N)) { + $stageFail += "response-loss $($parsed.Response.Hit)/$($parsed.Response.N)" + } + if ([int]$st.Grace -gt 0 -and ($parsed.Graceful.N -le 0 -or $parsed.Graceful.Hit -lt $parsed.Graceful.N)) { + $stageFail += "graceful $($parsed.Graceful.Hit)/$($parsed.Graceful.N)" + } + if ([int]$st.Hard -gt 0 -and ($parsed.HardStop.N -le 0 -or $parsed.HardStop.Hit -lt $parsed.HardStop.N)) { + $stageFail += "hard-stop $($parsed.HardStop.Hit)/$($parsed.HardStop.N)" + } + if ($stageFail.Count -eq 0) { + Write-ShardSamplesFromTrace -ShardDir $st.Dir -ShardId $id -Kind $kind -Seed ([int]$s.seed) -RunId "$id-$($st.Name)" + Write-JsonFile (Join-Path $st.Dir 'COMPLETED.json') ([ordered]@{ + ok = $true + shard_id = $id + stage = $st.Name + elapsed_seconds = $run.ElapsedSec + exit_code = $run.ExitCode + }) + Write-JsonFile $progressPath ([ordered]@{ + shard_id = $id + scenario = [string]$st.Scenario + completed_count = [int]$st.Target + target_count = [int]$st.Target + elapsed_seconds = [math]::Round($run.ElapsedSec, 3) + last_sample_timestamp = (Get-Date).ToString('o') + estimated = $false + note = "$($st.Name) completed" + }) + Write-Info "Shard $id stage=$($st.Name) PASS" + } else { + $fail += $stageFail + Write-JsonFile (Join-Path $st.Dir 'FAILED.json') ([ordered]@{ + ok = $false + shard_id = $id + stage = $st.Name + elapsed_seconds = $run.ElapsedSec + exit_code = $run.ExitCode + timed_out = $run.TimedOut + reasons = $stageFail + }) + Write-Info "Shard $id stage=$($st.Name) FAIL $($stageFail -join '; ')" + break + } + } + $faultParsed = Parse-CharReport (Join-Path $dir 'stages\faults\report.md') + $nomParsed = Parse-CharReport (Join-Path $dir 'stages\nominal\report.md') + $reportLines = @( + "# UAP shard $id", + "Stages: faults then nominal. Calibration logs are not mixed in.", + "- single_request_loss_recovery: $($faultParsed.Request.Hit) ($($faultParsed.Request.Hit)/$($faultParsed.Request.N))", + "- single_response_loss_recovery: $($faultParsed.Response.Hit) ($($faultParsed.Response.Hit)/$($faultParsed.Response.N))", + "- graceful_unknown_detection_rate: $($faultParsed.Graceful.Hit) ($($faultParsed.Graceful.Hit)/$($faultParsed.Graceful.N))", + "- missed_deadline_detection_rate: $($faultParsed.HardStop.Hit) ($($faultParsed.HardStop.Hit)/$($faultParsed.HardStop.N))", + "- live_false_MissedDeadline: $([int]$faultParsed.LiveMissed + [int]$nomParsed.LiveMissed)", + "- live_false_Unknown: $([int]$faultParsed.LiveUnknown + [int]$nomParsed.LiveUnknown)", + "- duplicates: $([int]$faultParsed.Duplicates + [int]$nomParsed.Duplicates)", + "- nominal_report_exists: $($nomParsed.Exists)", + "Hard-stop timing is Test-harness MissedDeadline detection latency, not a production SLA.", + "Semantic results: hard-stop MissedDeadline/state 1; graceful-stop Unknown/state 2." + ) + $reportLines | Set-Content -Path (Join-Path $dir 'report.md') -Encoding utf8 + $sampleOut = Join-Path $dir 'samples.jsonl' + $sampleParts = @() + foreach ($sn in @('faults', 'nominal')) { + $p = Join-Path $dir "stages\$sn\samples.jsonl" + if (Test-Path $p) { $sampleParts += Get-Content $p } + } + if ($sampleParts.Count -gt 0) { + $tmp = "$sampleOut.tmp" + $sampleParts | Set-Content -Path $tmp -Encoding utf8 + Move-Item -Force $tmp $sampleOut + } + $ok = ($fail.Count -eq 0) + if ($ok) { + Write-JsonFile (Join-Path $dir 'COMPLETED.json') ([ordered]@{ + ok = $true + shard_id = $id + elapsed_seconds = $stageElapsed + exit_code = $exitCode + }) + Write-Info "Shard $id PASS" + $status.shards_completed = [int]$status.shards_completed + 1 + $status.last_completed_shard = $id + } else { + $anyFail = $true + Write-JsonFile (Join-Path $dir 'FAILED.json') ([ordered]@{ + ok = $false + shard_id = $id + elapsed_seconds = $stageElapsed + exit_code = $exitCode + timed_out = $timedOut + reasons = $fail + }) + Write-Info "Shard $id FAIL $($fail -join '; ')" + $status.shards_failed = [int]$status.shards_failed + 1 + } + $status.current_shard = $null + Write-JsonFile $StatusPath $status + } + Write-LongAggregate $Plan + $status.state = $(if ($anyFail) { 'failed' } else { 'completed' }) + $status.ended_at = (Get-Date).ToString('o') + $status.exit_code = $(if ($anyFail) { 1 } else { 0 }) + Write-JsonFile $StatusPath $status + if ($anyFail) { exit 1 } + exit 0 +} + +function Write-LongAggregate($Plan) { + New-Item -ItemType Directory -Path $AggRoot -Force | Out-Null + $summary = [ordered]@{ + note = 'Long-run aggregate. Calibration logs are not mixed in.' + tcp = $null + udp = $null + } + foreach ($kind in @('tcp', 'udp')) { + $planT = $Plan.$kind + if (-not $planT) { continue } + $dup = 0; $liveM = 0; $liveU = 0 + $reqHit = 0; $reqN = 0; $respHit = 0; $respN = 0 + $graceHit = 0; $graceN = 0; $hardHit = 0; $hardN = 0 + $timeouts = 0; $crashes = 0; $incomplete = 0 + $shardRows = @() + foreach ($s in $planT.shards) { + $dir = Join-Path $RunsRoot $s.shard_id + $faultReport = Join-Path $dir 'stages\faults\report.md' + $parsed = Parse-CharReport $(if (Test-Path $faultReport) { $faultReport } else { Join-Path $dir 'report.md' }) + $complete = Test-ShardComplete $dir + if (-not $complete) { $incomplete++ } + if (Test-Path (Join-Path $dir 'FAILED.json')) { + $fj = Get-Content -Raw (Join-Path $dir 'FAILED.json') | ConvertFrom-Json + if ($fj.timed_out) { $timeouts++ } elseif ($fj.exit_code -ne 0) { $crashes++ } + } + if ($parsed.Duplicates) { $dup += $parsed.Duplicates } + if ($parsed.LiveMissed) { $liveM += $parsed.LiveMissed } + if ($parsed.LiveUnknown) { $liveU += $parsed.LiveUnknown } + $reqHit += $parsed.Request.Hit; $reqN += $parsed.Request.N + $respHit += $parsed.Response.Hit; $respN += $parsed.Response.N + $graceHit += $parsed.Graceful.Hit; $graceN += $parsed.Graceful.N + $hardHit += $parsed.HardStop.Hit; $hardN += $parsed.HardStop.N + $shardRows += "- $($s.shard_id) complete=$complete grace=$($parsed.Graceful.Hit)/$($parsed.Graceful.N) hard=$($parsed.HardStop.Hit)/$($parsed.HardStop.N)" + } + $wReq = Get-Wilson95 $reqHit $reqN + $wResp = Get-Wilson95 $respHit $respN + $wGrace = Get-Wilson95 $graceHit $graceN + $wHard = Get-Wilson95 $hardHit $hardN + $md = @( + "# UAP long characterization $kind", + '', + '## Measurement distinctions', + '- Nominal steady-state RTT: warmup min/p99 in each shard report.md (characterization clocks). Not a production SLA and not treated as a long-run p99.9 distribution.', + '- Request-loss recovery latency: shard report single_request_loss_recovery and Offline drop_request row.', + '- Response-loss recovery latency: shard report single_response_loss_recovery and Offline ignore_response row.', + '- Graceful-stop detection: expected semantic result Unknown, state 2.', + '- Hard-stop detection: expected semantic result MissedDeadline, state 1. Timing is Test-harness MissedDeadline detection latency from the existing --quick poll/restart path. It is not a general production SLA or protocol deadline guarantee.', + '- Invalid samples are retained in shard artifacts. Durations are not invented from timestamps in different processes or clock domains. Calibration logs are excluded.', + '', + 'Retry timing: retries_before_nominal / retries_after_nominal are summed from shard reports. TCP pre-deadline retry is reported, not a fail gate.', + "Request-loss recovery: $reqHit/$reqN Wilson95=[$($wReq.lo),$($wReq.hi)]", + "Response-loss recovery: $respHit/$respN Wilson95=[$($wResp.lo),$($wResp.hi)]", + "Graceful-stop detection (Unknown, state 2): $graceHit/$graceN Wilson95=[$($wGrace.lo),$($wGrace.hi)]", + "Test-harness MissedDeadline detection latency / hard-stop (MissedDeadline, state 1): $hardHit/$hardN Wilson95=[$($wHard.lo),$($wHard.hi)]", + "Safety: duplicates=$dup live_false_MissedDeadline=$liveM live_false_Unknown=$liveU timeouts=$timeouts crashes=$crashes incomplete=$incomplete", + 'Shards:' + ) + $shardRows + $md | Set-Content -Path (Join-Path $AggRoot "$kind-report.md") -Encoding utf8 + $summary[$kind] = [ordered]@{ + request_loss = "$reqHit/$reqN" + response_loss = "$respHit/$respN" + graceful = "$graceHit/$graceN" + hard_stop = "$hardHit/$hardN" + duplicates = $dup + live_false_missed = $liveM + live_false_unknown = $liveU + timeouts = $timeouts + crashes = $crashes + incomplete = $incomplete + } + } + @( + '# TCP vs UDP comparison', + 'Calibration logs are excluded.', + 'Hard-stop timing is Test-harness MissedDeadline detection latency, not a production SLA.', + 'Semantic results: hard-stop MissedDeadline/state 1; graceful-stop Unknown/state 2.', + "- TCP: $($summary.tcp | ConvertTo-Json -Compress)", + "- UDP: $($summary.udp | ConvertTo-Json -Compress)" + ) | Set-Content -Path (Join-Path $AggRoot 'comparison.md') -Encoding utf8 + Write-JsonFile (Join-Path $AggRoot 'summary.json') $summary +} + +# --- main --- +$sleepOn = $false +try { + [void][UapNativeSleep]::SetThreadExecutionState($ES_CONTINUOUS -bor $ES_SYSTEM_REQUIRED) + $sleepOn = $true + + New-Item -ItemType Directory -Path $CalibRoot -Force | Out-Null + $exes = @{} + foreach ($kind in Get-SelectedTransports) { + $exes[$kind] = Resolve-CharExe $kind + Write-Info "$kind exe: $($exes[$kind])" + } + + if ($Calibrate) { + $calibSw = [System.Diagnostics.Stopwatch]::StartNew() + $calib = [ordered]@{ + commit = Get-HeadCommit + tracked_working_tree_clean = Get-TrackedClean + tcp_exe = $(if ($exes.ContainsKey('tcp')) { $exes['tcp'] } else { '' }) + udp_exe = $(if ($exes.ContainsKey('udp')) { $exes['udp'] } else { '' }) + build_type = 'Release' + build_performed = $false + timestamp = (Get-Date).ToString('o') + power_scheme = Get-PowerScheme + active_network_adapters = @(Get-ActiveAdapters) + system_sleep_prevention_used = $true + tcp = $null + udp = $null + overall = 'PASS' + } + foreach ($kind in Get-SelectedTransports) { + $meas = Invoke-CalibrationKind -Kind $kind -Exe $exes[$kind] + $calib[$kind] = $meas + if ($meas.result -ne 'PASS') { $calib.overall = 'FAIL' } + } + $calibSw.Stop() + $calib.elapsed_calibration_seconds = [math]::Round($calibSw.Elapsed.TotalSeconds, 3) + Write-JsonFile $CalibJson $calib + $plan = New-LongPlan $calib $TargetHours + Write-JsonFile $PlanJson $plan + Write-PlanMarkdown $plan $PlanMd + Write-CalibrationReport $calib $plan $CalibReport + Write-Info "Calibration elapsed $(Format-Duration $calib.elapsed_calibration_seconds)" + Write-Info "Wrote $CalibReport" + Write-Info "Wrote $PlanMd" + if ($calib.overall -ne 'PASS') { exit 1 } + exit 0 + } + + if (-not (Test-Path $CalibJson)) { + throw 'plan/run requires artifacts/uap-long/calibration/calibration.json; run -Calibrate first.' + } + $calib = Get-Content -Raw $CalibJson | ConvertFrom-Json + + if ($DryRun) { + $RunsRoot = Join-Path $ArtifactRoot 'dry-run-v2' + $StatusPath = Join-Path $RunsRoot 'status.json' + $RunInfoPath = Join-Path $RunsRoot 'run-info.json' + $AggRoot = Join-Path $RunsRoot 'aggregate' + $dry = New-DryPlan $calib + Write-Info "DryRun root $RunsRoot shard=tcp-test" + Invoke-LongRun $dry $exes + } + + $plan = New-LongPlan $calib $TargetHours + Write-JsonFile $PlanJson $plan + Write-PlanMarkdown $plan $PlanMd + Write-PlanV2Documents $plan $calib + Write-Info "Wrote $PlanMd predicted=$($plan.predicted_duration)" + Assert-LongPlanGates $plan + + if ($PlanOnly) { exit 0 } + if ($Run) { Invoke-LongRun $plan $exes } +} finally { + if ($sleepOn) { + [void][UapNativeSleep]::SetThreadExecutionState($ES_CONTINUOUS) + } +} diff --git a/scripts/run_uap_phase_preservation_8h.ps1 b/scripts/run_uap_phase_preservation_8h.ps1 new file mode 100644 index 00000000..c750caab --- /dev/null +++ b/scripts/run_uap_phase_preservation_8h.ps1 @@ -0,0 +1,252 @@ +# Independent 8-hour UAP phase-preservation characterization runner. +# Enforces wall-clock budget. Alternates TCP/UDP shards. Supports resume. +# Does not commit, push, clean, or reconfigure CMake. + +param( + [string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path, + [string]$ArtifactRoot = "", + [int]$ActiveBudgetSec = 28500, # 7h55m + [int]$ShardBudgetSec = 1500, # 25m target + [int]$ReportReserveSec = 300, # 5m + [uint32]$BaseSeed = 20260826, + [switch]$Resume +) + +$ErrorActionPreference = "Stop" +if (-not $ArtifactRoot) { + $ArtifactRoot = Join-Path $RepoRoot "artifacts\uap-phase-preservation\8h" +} + +$TcpExe = Join-Path $RepoRoot "build-win64-uap-ping-retry-tcp\Release\aether_uap_1s_timing_characterization.exe" +$UdpExe = Join-Path $RepoRoot "build-win64-uap-ping-retry-udp\Release\aether_uap_1s_timing_characterization.exe" +$StatusPath = Join-Path $ArtifactRoot "status.json" +$PlanPath = Join-Path $ArtifactRoot "plan.md" +$LogPath = Join-Path $ArtifactRoot "runner.log" +$RunsDir = Join-Path $ArtifactRoot "runs" +$AggDir = Join-Path $ArtifactRoot "aggregate" +$FailDir = Join-Path $ArtifactRoot "failure-cases" +$AggScript = Join-Path $RepoRoot "scripts\aggregate_uap_phase_preservation_8h.py" + +New-Item -ItemType Directory -Force -Path $RunsDir, $AggDir, $FailDir | Out-Null + +function Write-AtomicJson($Path, $Object) { + $tmp = "$Path.tmp" + $json = $Object | ConvertTo-Json -Depth 12 + [System.IO.File]::WriteAllText($tmp, $json) + Move-Item -Force $tmp $Path +} + +function Write-Log([string]$Msg) { + $line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg + Add-Content -Path $LogPath -Value $line + Write-Host $line +} + +function Stop-OrphanCharacterization { + Get-CimInstance Win32_Process -Filter "Name='aether_uap_1s_timing_characterization.exe'" -ErrorAction SilentlyContinue | + ForEach-Object { + Write-Log "Killing leftover characterization pid=$($_.ProcessId)" + Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue + } +} + +# Prevent system sleep while active; do not prevent display sleep. +Add-Type -Namespace Native -Name Power -MemberDefinition @" +[DllImport("kernel32.dll", CharSet=CharSet.Auto, SetLastError=true)] +public static extern uint SetThreadExecutionState(uint esFlags); +"@ +$ES_CONTINUOUS = [Convert]::ToUInt32("80000000", 16) +$ES_SYSTEM_REQUIRED = [Convert]::ToUInt32("1", 16) +[void][Native.Power]::SetThreadExecutionState($ES_CONTINUOUS -bor $ES_SYSTEM_REQUIRED) + +$started = Get-Date +$status = $null +if ($Resume -and (Test-Path $StatusPath)) { + $status = Get-Content $StatusPath -Raw | ConvertFrom-Json + if ($status.started_utc) { + $started = [datetime]::Parse($status.started_utc, $null, [System.Globalization.DateTimeStyles]::RoundtripKind) + } + Write-Log "Resuming from status.json started=$($status.started_utc) next_shard=$($status.next_shard_index)" +} else { + $status = [ordered]@{ + started_utc = $started.ToUniversalTime().ToString("o") + active_budget_sec = $ActiveBudgetSec + shard_budget_sec = $ShardBudgetSec + report_reserve_sec = $ReportReserveSec + base_seed = $BaseSeed + next_shard_index = 1 + shards = @() + state = "running" + last_update_utc = (Get-Date).ToUniversalTime().ToString("o") + } + Write-AtomicJson $StatusPath $status +} + +@" +# UAP 8h phase-preservation plan + +- started_utc: $($status.started_utc) +- active_test_budget_sec: $ActiveBudgetSec (7h55m) +- report_reserve_sec: $ReportReserveSec (5m) +- shard_budget_sec: $ShardBudgetSec (target 20-30m) +- alternate: tcp-01, udp-01, tcp-02, udp-02, ... +- tcp_exe: $TcpExe +- udp_exe: $UdpExe +- continue on semantic failures; preserve and proceed +- restart next shard after process crash +- runner enforces wall-clock; does not estimate from case counts +"@ | Set-Content -Path $PlanPath -Encoding UTF8 + +function ElapsedSec { + return [int]((Get-Date) - $started).TotalSeconds +} + +function RemainingActiveSec { + return $ActiveBudgetSec - (ElapsedSec) +} + +function Persist-Status([string]$State = "running") { + $status.state = $State + $status.last_update_utc = (Get-Date).ToUniversalTime().ToString("o") + $status.elapsed_sec = (ElapsedSec) + $status.remaining_active_sec = (RemainingActiveSec) + Write-AtomicJson $StatusPath $status +} + +# Heartbeat every ~60s while a shard runs. +$heartbeatJob = $null + +try { + Write-Log "8h runner start. active_budget=${ActiveBudgetSec}s shard=${ShardBudgetSec}s" + if (-not (Test-Path $TcpExe)) { throw "Missing TCP exe: $TcpExe" } + if (-not (Test-Path $UdpExe)) { throw "Missing UDP exe: $UdpExe" } + + $shardIndex = [int]$status.next_shard_index + if ($shardIndex -lt 1) { $shardIndex = 1 } + + while ((RemainingActiveSec) -gt 90) { + $transport = if (($shardIndex % 2) -eq 1) { "tcp" } else { "udp" } + $pair = [int][Math]::Ceiling($shardIndex / 2.0) + $shardName = $transport + "-" + ("{0:00}" -f $pair) + $exe = if ($transport -eq "tcp") { $TcpExe } else { $UdpExe } + $seed = [uint32]($BaseSeed + $shardIndex) + $shardDir = Join-Path $RunsDir $shardName + New-Item -ItemType Directory -Force -Path $shardDir | Out-Null + + $remain = RemainingActiveSec + $thisBudget = [Math]::Min($ShardBudgetSec, [Math]::Max(120, $remain - 30)) + if ($thisBudget -lt 120) { + Write-Log "Budget too small for another shard ($remain s). Stopping scheduling." + break + } + + Write-Log "Starting shard $shardName transport=$transport budget=${thisBudget}s seed=$seed remain=${remain}s" + Stop-OrphanCharacterization + + $stdout = Join-Path $shardDir "stdout.log" + $stderr = Join-Path $shardDir "stderr.log" + $meta = [ordered]@{ + shard = $shardName + transport = $transport + seed = $seed + budget_sec = $thisBudget + started_utc = (Get-Date).ToUniversalTime().ToString("o") + exe = $exe + state = "running" + } + Write-AtomicJson (Join-Path $shardDir "shard-status.json") $meta + + $argList = @( + "--phase-preservation", + "--phase-preservation-budget-sec", "$thisBudget", + "--no-long-characterization", + "--transport", $transport, + "--seed", "$seed", + "--artifact-dir", $shardDir, + "--run-id", $shardName, + "--exe", $exe + ) + + $proc = Start-Process -FilePath $exe -ArgumentList $argList -WorkingDirectory $RepoRoot ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr ` + -PassThru -WindowStyle Hidden + if ($null -eq $proc) { + throw "Failed to start characterization exe for $shardName" + } + Write-Log "Shard $shardName pid=$($proc.Id)" + + $hardDeadline = (Get-Date).AddSeconds($thisBudget + 180) # allow finish + controls + $lastBeat = Get-Date + while (-not $proc.HasExited) { + Start-Sleep -Seconds 5 + try { $proc.Refresh() } catch {} + if (((Get-Date) - $lastBeat).TotalSeconds -ge 60) { + Persist-Status "running" + $lastBeat = Get-Date + Write-Log "heartbeat shard=$shardName elapsed_total=$(ElapsedSec)s pid=$($proc.Id)" + } + if ((Get-Date) -gt $hardDeadline) { + Write-Log "Shard $shardName exceeded hard deadline; terminating process tree" + try { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue } catch {} + Stop-OrphanCharacterization + break + } + } + + $exitCode = -1 + try { + if ($proc.HasExited) { $exitCode = $proc.ExitCode } + } catch { + $exitCode = -1 + } + $crashed = ($exitCode -lt 0) -or ($exitCode -ge 200) + # Semantic fail exit 7 is expected and recorded; continue. + + $meta.finished_utc = (Get-Date).ToUniversalTime().ToString("o") + $meta.exit_code = $exitCode + $meta.crashed = [bool]$crashed + $meta.state = if ($crashed) { "crashed" } elseif ($exitCode -eq 0) { "pass" } else { "failed_semantic" } + Write-AtomicJson (Join-Path $shardDir "shard-status.json") $meta + + $status.shards = @($status.shards) + @([pscustomobject]$meta) + $shardIndex++ + $status.next_shard_index = $shardIndex + Persist-Status "running" + Write-Log "Finished shard $shardName exit=$exitCode state=$($meta.state)" + + if ($crashed) { + Write-Log "Preserved crash outputs under $shardDir; continuing next shard" + Start-Sleep -Seconds 3 + } + } + + Write-Log "Active budget exhausted or remaining too small. Aggregating..." + Persist-Status "aggregating" + Stop-OrphanCharacterization + + if (Test-Path $AggScript) { + python $AggScript --root $ArtifactRoot 2>&1 | Tee-Object -FilePath (Join-Path $ArtifactRoot "aggregate.log") + } else { + Write-Log "WARN missing aggregator $AggScript" + } + + Persist-Status "completed" + $elapsed = ElapsedSec + Write-Log "8h runner completed. wall_sec=$elapsed" + Write-Host "" + Write-Host "==== FINAL CONSOLE SUMMARY ====" + if (Test-Path (Join-Path $AggDir "console-summary.txt")) { + Get-Content (Join-Path $AggDir "console-summary.txt") + } else { + Write-Host "Aggregate summary not found; see $ArtifactRoot" + } +} +catch { + Write-Log "FATAL: $_" + Persist-Status "error" + throw +} +finally { + [void][Native.Power]::SetThreadExecutionState($ES_CONTINUOUS) + Stop-OrphanCharacterization +} diff --git a/tests/test-api-protocol/CMakeLists.txt b/tests/test-api-protocol/CMakeLists.txt index d4502782..c1919262 100644 --- a/tests/test-api-protocol/CMakeLists.txt +++ b/tests/test-api-protocol/CMakeLists.txt @@ -17,6 +17,10 @@ cmake_minimum_required( VERSION 3.16 ) list(APPEND test_srcs main.cpp test-method-call.cpp + test-uap-receive-schedule.cpp + test-uap-peer-deadline-classify.cpp + test-uap-peer-timing.cpp + test-client-online-timing.cpp ) if(NOT CM_PLATFORM) diff --git a/tests/test-api-protocol/main.cpp b/tests/test-api-protocol/main.cpp index fd6c1591..7117a831 100644 --- a/tests/test-api-protocol/main.cpp +++ b/tests/test-api-protocol/main.cpp @@ -20,10 +20,18 @@ void setUp() {} void tearDown() {} extern int test_method_call(); +extern int test_uap_receive_schedule(); +extern int test_uap_peer_deadline_classify(); +extern int test_uap_peer_timing(); +extern int test_client_online_timing(); int main() { int res = 0; res += test_method_call(); + res += test_uap_receive_schedule(); + res += test_uap_peer_deadline_classify(); + res += test_uap_peer_timing(); + res += test_client_online_timing(); return res; } diff --git a/tests/test-api-protocol/test-client-online-timing.cpp b/tests/test-api-protocol/test-client-online-timing.cpp new file mode 100644 index 00000000..b804ae22 --- /dev/null +++ b/tests/test-api-protocol/test-client-online-timing.cpp @@ -0,0 +1,300 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include "aether/api_protocol/api_protocol.h" +#include "aether/cloud_connections/ping_schedule_guard.h" +#include "aether/config.h" +#include "aether/types/data_buffer.h" + +namespace ae::test_client_online_timing { +namespace { + +Duration Ms(std::uint32_t v) { + return std::chrono::duration_cast(std::chrono::milliseconds{v}); +} + +TimePoint Tp(std::uint32_t ms) { return TimePoint{} + Ms(ms); } + +} // namespace + +void test_InitialClientOnlineTimestampsAreEmpty() { + // Without a live ping schedule / cloud connection, expected is nullopt. + // last_online starts empty; verified via MarkServerResponseReceived below + // once a response is observed. + LogicalPingCycleState st{}; + TEST_ASSERT_FALSE(ExpectedPingResponseTimeForCycle(st).has_value()); + TEST_ASSERT_FALSE(st.has_schedule); +} + +void test_ExpectedPingResponseIsTnPlusHalfP99() { + auto const tn = Tp(1000); + auto const expected = ExpectedPingResponseTime(tn, Ms(200)); + TEST_ASSERT_TRUE(expected == Tp(1100)); +} + +void test_ExpectedPingResponseIgnoresGuardAndMargins() { + auto const tn = Tp(1000); + auto const p99 = Ms(200); + auto const guard = Ms(40); + auto const dispatch = kPingRetryDispatchMargin; + auto const scheduler = kPingSchedulerMargin; + TEST_ASSERT_TRUE(guard == Ms(40)); + TEST_ASSERT_TRUE(dispatch == Ms(60)); + TEST_ASSERT_TRUE(scheduler == Ms(10)); + auto const expected = ExpectedPingResponseTime(tn, p99); + TEST_ASSERT_TRUE(expected == Tp(1100)); + TEST_ASSERT_TRUE(expected != tn + OneWayReturnEstimateFromP99(p99) - guard); + TEST_ASSERT_TRUE(expected != + tn + OneWayReturnEstimateFromP99(p99) + guard); + TEST_ASSERT_TRUE(expected != + tn + OneWayReturnEstimateFromP99(p99) + dispatch); + TEST_ASSERT_TRUE(expected != + tn + OneWayReturnEstimateFromP99(p99) + scheduler); +} + +void test_EmptyStatsBootstrapMatchesHalfOf200ms() { + TEST_ASSERT_TRUE(OneWayReturnEstimateFromP99(kPingRttEstimate) == Ms(100)); + TEST_ASSERT_TRUE(ExpectedPingResponseTime(Tp(1000), kPingRttEstimate) == + Tp(1100)); +} + +void test_MarkServerResponseReceivedUpdatesLastOnline() { + std::optional last_online_time; + UpdateMonotonicLastOnlineTime(last_online_time, Tp(1080)); + TEST_ASSERT_TRUE(last_online_time.has_value()); + TEST_ASSERT_TRUE(*last_online_time == Tp(1080)); + UpdateMonotonicLastOnlineTime(last_online_time, Tp(1090)); + TEST_ASSERT_TRUE(*last_online_time == Tp(1090)); +} + +void test_LastOnlineTimeIsMonotonic() { + std::optional last_online_time; + UpdateMonotonicLastOnlineTime(last_online_time, Tp(1000)); + UpdateMonotonicLastOnlineTime(last_online_time, Tp(900)); + TEST_ASSERT_TRUE(last_online_time.has_value()); + TEST_ASSERT_TRUE(*last_online_time == Tp(1000)); + UpdateMonotonicLastOnlineTime(last_online_time, Tp(1100)); + TEST_ASSERT_TRUE(*last_online_time == Tp(1100)); +} + +void test_MultiServerAggregationUsesLatestExpected() { + std::optional latest_expected_response; + AccumulateLatestExpectedPingResponse(latest_expected_response, Tp(10100)); + AccumulateLatestExpectedPingResponse(latest_expected_response, Tp(11100)); + TEST_ASSERT_TRUE(latest_expected_response.has_value()); + TEST_ASSERT_TRUE(*latest_expected_response == Tp(11100)); +} + +void test_InactiveServerExcludedFromAggregation() { + std::optional latest_expected_response; + AccumulateLatestExpectedPingResponse(latest_expected_response, Tp(11100)); + TEST_ASSERT_TRUE(latest_expected_response.has_value()); + TEST_ASSERT_TRUE(*latest_expected_response == Tp(11100)); + + // Including a stale inactive server would wrongly raise the max. + std::optional with_inactive; + AccumulateLatestExpectedPingResponse(with_inactive, Tp(11100)); + AccumulateLatestExpectedPingResponse(with_inactive, Tp(50000)); + TEST_ASSERT_TRUE(*with_inactive == Tp(50000)); + + // Production skips inactive servers before aggregation. + latest_expected_response = std::nullopt; + AccumulateLatestExpectedPingResponse(latest_expected_response, Tp(11100)); + TEST_ASSERT_TRUE(*latest_expected_response == Tp(11100)); +} + +void test_InboundResultHookMarksOnlineAndEvictionDoesNot() { + ProtocolContext pc; + int inbound_count = 0; + pc.set_inbound_server_response_hook( + [](void* user) noexcept { + *static_cast(user) += 1; + }, + &inbound_count); + + auto promise = ApiPromise{pc, RequestId{7}}; + bool got_ok = false; + auto sub = promise.Subscribe([&](auto const& res) { + TEST_ASSERT_TRUE(res.IsOk()); + got_ok = true; + }); + static_cast(sub); + + DataBuffer data; + { + auto parser = ApiParser{pc, data}; + // Drive matched inbound result path (void result needs no payload). + pc.SetSendResultResponse(RequestId{7}); + } + TEST_ASSERT_EQUAL_INT(1, inbound_count); + TEST_ASSERT_TRUE(got_ok); + + // Eviction must not notify the inbound-server-response hook. + auto promise2 = ApiPromise{pc, RequestId{8}}; + bool got_evict = false; + auto sub2 = promise2.Subscribe([&](auto const& res) { + TEST_ASSERT_FALSE(res.IsOk()); + got_evict = true; + }); + static_cast(sub2); + // Replace same request id to force OnEvicted. + auto promise3 = ApiPromise{pc, RequestId{8}}; + auto sub3 = promise3.Subscribe([](auto const&) {}); + static_cast(sub3); + TEST_ASSERT_TRUE(got_evict); + TEST_ASSERT_EQUAL_INT(1, inbound_count); +} + +void test_InboundErrorHookMarksOnline() { + ProtocolContext pc; + int inbound_count = 0; + pc.set_inbound_server_response_hook( + [](void* user) noexcept { + *static_cast(user) += 1; + }, + &inbound_count); + + auto promise = ApiPromise{pc, RequestId{9}}; + bool got_err = false; + auto sub = promise.Subscribe([&](auto const& res) { + TEST_ASSERT_FALSE(res.IsOk()); + TEST_ASSERT_EQUAL(42, res.error()); + got_err = true; + }); + static_cast(sub); + + DataBuffer data; + { + auto parser = ApiParser{pc, data}; + pc.SetSendErrorResponse(RequestId{9}, 0, 42); + } + TEST_ASSERT_TRUE(got_err); + TEST_ASSERT_EQUAL_INT(1, inbound_count); +} + +void test_CycleExpectedUsesFrozenP99AndNominalTn() { + LogicalPingCycleState st{}; + st.has_schedule = true; + st.active = true; + st.confirmed = false; + st.nominal_ping_at = Tp(1000); + st.next_nominal_ping_at = Tp(2000); + st.has_frozen_p99_rtt = true; + st.frozen_p99_rtt = Ms(100); + auto const expected = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(expected.has_value()); + TEST_ASSERT_TRUE(*expected == Tp(1050)); + + // Later live p99 changes must not move the frozen cycle expectation. + st.frozen_p99_rtt = Ms(100); // still frozen value + auto const still = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(still.has_value()); + TEST_ASSERT_TRUE(*still == Tp(1050)); +} + +void test_PostDeadlineRecoveryKeepsOriginalExpectedUntilAdvance() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(50); + req.base_rx_window = Ms(250); + req.actual_send_at = TimePoint{}; + auto const boot = ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + + req.actual_send_at = boot.next_local_send; + auto const first = ApplyLogicalPingAttempt(st, req); + st.frozen_p99_rtt = Ms(100); + st.has_frozen_p99_rtt = true; + TEST_ASSERT_TRUE(first.nominal_ping_at == Tp(1000)); + auto const expected_before = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(expected_before.has_value()); + TEST_ASSERT_TRUE(*expected_before == Tp(1050)); + + // Same-cycle late recovery before the next nominal: original expected stays. + req.actual_send_at = Tp(1100); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(st.nominal_ping_at == Tp(1000)); + auto const expected_during_recovery = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(expected_during_recovery.has_value()); + TEST_ASSERT_TRUE(*expected_during_recovery == Tp(1050)); + + ConfirmLogicalPingCycle(st); + auto const expected_next = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(expected_next.has_value()); + TEST_ASSERT_TRUE(*expected_next == Tp(2050)); +} + +void test_AfterConfirmExpectedUsesNextNominal() { + LogicalPingCycleState st{}; + st.has_schedule = true; + st.active = false; + st.confirmed = true; + st.nominal_ping_at = Tp(1000); + st.next_nominal_ping_at = Tp(2000); + st.has_frozen_p99_rtt = true; + st.frozen_p99_rtt = Ms(120); + auto const expected = ExpectedPingResponseTimeForCycle(st); + TEST_ASSERT_TRUE(expected.has_value()); + TEST_ASSERT_TRUE(*expected == Tp(2060)); +} + +void test_NoScheduleYieldsNulloptExpected() { + LogicalPingCycleState st{}; + st.has_frozen_p99_rtt = true; + st.frozen_p99_rtt = Ms(200); + TEST_ASSERT_FALSE(ExpectedPingResponseTimeForCycle(st).has_value()); +} + +} // namespace ae::test_client_online_timing + +int test_client_online_timing() { + UNITY_BEGIN(); + RUN_TEST(ae::test_client_online_timing:: + test_InitialClientOnlineTimestampsAreEmpty); + RUN_TEST(ae::test_client_online_timing:: + test_ExpectedPingResponseIsTnPlusHalfP99); + RUN_TEST(ae::test_client_online_timing:: + test_ExpectedPingResponseIgnoresGuardAndMargins); + RUN_TEST(ae::test_client_online_timing:: + test_EmptyStatsBootstrapMatchesHalfOf200ms); + RUN_TEST(ae::test_client_online_timing:: + test_MarkServerResponseReceivedUpdatesLastOnline); + RUN_TEST(ae::test_client_online_timing::test_LastOnlineTimeIsMonotonic); + RUN_TEST(ae::test_client_online_timing:: + test_MultiServerAggregationUsesLatestExpected); + RUN_TEST(ae::test_client_online_timing:: + test_InactiveServerExcludedFromAggregation); + RUN_TEST(ae::test_client_online_timing:: + test_InboundResultHookMarksOnlineAndEvictionDoesNot); + RUN_TEST(ae::test_client_online_timing::test_InboundErrorHookMarksOnline); + RUN_TEST(ae::test_client_online_timing:: + test_CycleExpectedUsesFrozenP99AndNominalTn); + RUN_TEST(ae::test_client_online_timing:: + test_PostDeadlineRecoveryKeepsOriginalExpectedUntilAdvance); + RUN_TEST(ae::test_client_online_timing:: + test_AfterConfirmExpectedUsesNextNominal); + RUN_TEST(ae::test_client_online_timing::test_NoScheduleYieldsNulloptExpected); + return UNITY_END(); +} diff --git a/tests/test-api-protocol/test-uap-peer-deadline-classify.cpp b/tests/test-api-protocol/test-uap-peer-deadline-classify.cpp new file mode 100644 index 00000000..1d422cbf --- /dev/null +++ b/tests/test-api-protocol/test-uap-peer-deadline-classify.cpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include "aether/receive_schedule.h" +#include "examples/aether_uap_peer_deadline_test/missed_deadline.h" + +namespace ae::test_uap_peer_deadline_classify { +namespace { + +TimePoint Tp(std::int64_t ms) { + return TimePoint{} + std::chrono::milliseconds{ms}; +} + +PeerReceiveSchedule Make(std::int64_t last_ms, + std::optional next_ms) { + PeerReceiveSchedule s{}; + s.last_online = Tp(last_ms); + if (next_ms.has_value() && *next_ms > last_ms) { + s.state = PeerScheduleState::kExpected; + } else if (next_ms.has_value()) { + s.state = PeerScheduleState::kMissedDeadline; + } else { + s.state = PeerScheduleState::kUnknown; + } + if (next_ms.has_value()) { + s.next_ping_deadline = Tp(*next_ms); + } + return s; +} + +} // namespace + +void test_IsMissedDeadline_BeforeDeadlineUnchangedPing() { + auto const prev = Make(1000, 4000); + auto const curr = Make(1000, 4000); + TEST_ASSERT_FALSE( + ae::test_uap_peer_deadline::IsMissedDeadline(prev, curr, Tp(3500))); +} + +void test_IsMissedDeadline_AfterDeadlineUnchangedPing() { + auto const prev = Make(1000, 4000); + auto const curr = Make(1000, 4000); + TEST_ASSERT_TRUE( + ae::test_uap_peer_deadline::IsMissedDeadline(prev, curr, Tp(4500))); +} + +void test_IsMissedDeadline_AfterDeadlineAdvancedPing() { + auto const prev = Make(1000, 4000); + auto const curr = Make(4100, 7100); + TEST_ASSERT_FALSE( + ae::test_uap_peer_deadline::IsMissedDeadline(prev, curr, Tp(4500))); +} + +void test_IsMissedDeadline_AfterDeadlineTinyDriftNotAdvanced() { + auto const prev = Make(1000, 4000); + // 20ms drift is below kLastPingAdvanceEpsilon. + auto const curr = Make(1020, 4000); + TEST_ASSERT_TRUE( + ae::test_uap_peer_deadline::IsMissedDeadline(prev, curr, Tp(4500))); +} + +void test_IsMissedDeadline_NoNextDeadline() { + auto const prev = Make(1000, std::nullopt); + auto const curr = Make(1000, std::nullopt); + TEST_ASSERT_FALSE( + ae::test_uap_peer_deadline::IsMissedDeadline(prev, curr, Tp(99999))); +} + +} // namespace ae::test_uap_peer_deadline_classify + +int test_uap_peer_deadline_classify() { + UNITY_BEGIN(); + RUN_TEST(ae::test_uap_peer_deadline_classify:: + test_IsMissedDeadline_BeforeDeadlineUnchangedPing); + RUN_TEST(ae::test_uap_peer_deadline_classify:: + test_IsMissedDeadline_AfterDeadlineUnchangedPing); + RUN_TEST(ae::test_uap_peer_deadline_classify:: + test_IsMissedDeadline_AfterDeadlineAdvancedPing); + RUN_TEST(ae::test_uap_peer_deadline_classify:: + test_IsMissedDeadline_AfterDeadlineTinyDriftNotAdvanced); + RUN_TEST(ae::test_uap_peer_deadline_classify:: + test_IsMissedDeadline_NoNextDeadline); + return UNITY_END(); +} diff --git a/tests/test-api-protocol/test-uap-peer-timing.cpp b/tests/test-api-protocol/test-uap-peer-timing.cpp new file mode 100644 index 00000000..b9d6c914 --- /dev/null +++ b/tests/test-api-protocol/test-uap-peer-timing.cpp @@ -0,0 +1,560 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "aether-miscpp/serialization/binary_archive.h" + +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/api_protocol/api_protocol.h" +#include "aether/api_protocol/request_id.h" +#include "aether/cloud_connections/cloud_request.h" +#include "aether/receive_schedule.h" +#include "aether/types/data_buffer.h" +#include "aether/types/uid.h" +#include "aether/work_cloud_api/client_timing.h" +#include "aether/work_cloud_api/work_server_api/authorized_api.h" + +#include "assert_packet.h" + +namespace ae::test_uap_peer_timing { +namespace { + +Duration Ms(std::uint32_t v) { + return std::chrono::duration_cast(std::chrono::milliseconds{v}); +} + +TimePoint Tp(std::int64_t ms) { + return TimePoint{} + std::chrono::milliseconds{ms}; +} + +ConvertedServerTiming Sample(std::int64_t last_ms, + std::optional next_ms, + PeerScheduleState state, + ServerId id = {}) { + ConvertedServerTiming s{}; + s.server_id = id; + s.last_online = Tp(last_ms); + s.state = state; + if (next_ms.has_value()) { + s.next_ping_deadline = Tp(*next_ms); + } + return s; +} + +} // namespace + +void test_PingMethodIdAndParamOrder() { + ProtocolContext pc; + AuthorizedApi api{pc}; + auto ctx = ApiContext{api}; + ctx->ping(std::int64_t{3000}, std::int64_t{1000}); + DataBuffer packet = std::move(ctx); + AssertPacket(packet, MessageId{4}, Skip{}, std::int64_t{3000}, + std::int64_t{1000}); +} + +void test_GetClientTimingMethodIdAndUidParam() { + ProtocolContext pc; + AuthorizedApi api{pc}; + auto ctx = ApiContext{api}; + auto const uid = Uid::FromString("f81d4fae-7dec-11d0-a765-00a0c91e6bf6"); + ctx->get_client_timing(uid); + DataBuffer packet = std::move(ctx); + AssertPacket(packet, MessageId{35}, Skip{}, uid); +} + +void test_GetUapRemainsMethod34AndIsNotClientTiming() { + ProtocolContext pc; + AuthorizedApi api{pc}; + auto ctx = ApiContext{api}; + auto const uid = Uid::FromString("f81d4fae-7dec-11d0-a765-00a0c91e6bf6"); + ctx->get_uap(uid); + DataBuffer packet = std::move(ctx); + AssertPacket(packet, MessageId{34}, Skip{}, uid); +} + +void test_PingPacketDoesNotIncludeSetNextReadDelay() { + ProtocolContext pc; + AuthorizedApi api{pc}; + auto ping_only = ApiContext{api}; + ping_only->ping(std::int64_t{3000}, std::int64_t{1000}); + DataBuffer const ping_packet = std::move(ping_only); + + ProtocolContext pc2; + AuthorizedApi api2{pc2}; + auto ping_and_delay = ApiContext{api2}; + ping_and_delay->ping(std::int64_t{3000}, std::int64_t{1000}); + ping_and_delay->set_next_read_delay(std::int64_t{3000}); + DataBuffer const both = std::move(ping_and_delay); + + AssertPacket(ping_packet, MessageId{4}, Skip{}, std::int64_t{3000}, + std::int64_t{1000}); + TEST_ASSERT_TRUE(ping_packet.size() < both.size()); +} + +void test_ClientTimingFieldOrderSignedInt64() { + ClientTiming const timing{1'000, -250}; + std::vector packed; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Save(timing); + } + TEST_ASSERT_EQUAL_UINT(16, packed.size()); + std::int64_t next_delta = 0; + std::int64_t last_connect = 0; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Load(next_delta); + archive.Load(last_connect); + } + TEST_ASSERT_TRUE(next_delta == 1'000); + TEST_ASSERT_TRUE(last_connect == -250); +} + +void test_ClientTimingNegativeZeroPositiveAndBounds() { + ClientTiming const timing{std::numeric_limits::min(), + std::numeric_limits::max()}; + std::vector packed; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Save(timing); + } + ClientTiming loaded{}; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Load(loaded); + } + TEST_ASSERT_TRUE(loaded.next_ping_delta_ms == + std::numeric_limits::min()); + TEST_ASSERT_TRUE(loaded.last_connect_delta_ms == + std::numeric_limits::max()); +} + +void test_ConversionExampleQsendMinRtt80() { + auto const qsend = Tp(10'000); + auto const one_way = OneWayPingEstimate(false, Ms(80)); + TEST_ASSERT_TRUE(one_way == Ms(40)); + ClientTiming timing{1'000, -200}; + auto const converted = ConvertClientTiming(qsend, one_way, timing); + TEST_ASSERT_TRUE(converted.last_online == Tp(10'000 - 160)); + TEST_ASSERT_TRUE(converted.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(*converted.next_ping_deadline == Tp(10'000 + 1'040)); + TEST_ASSERT_TRUE(converted.state == PeerScheduleState::kExpected); +} + +void test_ConversionNegativeZeroEmptyStatsAndSaturation() { + auto const qsend = Tp(5'000); + auto const missed = + ConvertClientTiming(qsend, Ms(40), ClientTiming{-300, -50}); + TEST_ASSERT_TRUE(missed.state == PeerScheduleState::kMissedDeadline); + TEST_ASSERT_TRUE(missed.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(*missed.next_ping_deadline == Tp(5'000 + 40 - 300)); + TEST_ASSERT_TRUE(missed.last_online == Tp(5'000 + 40 - 50)); + + auto const unknown = ConvertClientTiming(qsend, Ms(40), ClientTiming{0, -10}); + TEST_ASSERT_TRUE(unknown.state == PeerScheduleState::kUnknown); + TEST_ASSERT_FALSE(unknown.next_ping_deadline.has_value()); + + TEST_ASSERT_TRUE(OneWayPingEstimate(true, Ms(80)) == Ms(100)); + + auto const saturated = TimePointOffsetByMs( + TimePoint::max(), std::numeric_limits::max()); + TEST_ASSERT_TRUE(saturated == TimePoint::max()); + auto const saturated_min = TimePointOffsetByMs( + TimePoint::min(), std::numeric_limits::min()); + TEST_ASSERT_TRUE(saturated_min == TimePoint::min()); +} + +void test_AggregateFutureCases() { + auto const one_future = AggregatePeerTimings( + {Sample(100, 2000, PeerScheduleState::kExpected)}); + TEST_ASSERT_TRUE(one_future.has_value()); + TEST_ASSERT_TRUE(one_future->state == PeerScheduleState::kExpected); + TEST_ASSERT_TRUE(one_future->next_ping_deadline == Tp(2000)); + + auto const latest_future = AggregatePeerTimings( + {Sample(1, 1000, PeerScheduleState::kExpected, 1), + Sample(2, 2500, PeerScheduleState::kExpected, 2)}); + TEST_ASSERT_TRUE(latest_future->next_ping_deadline == Tp(2500)); + + auto const future_plus_expired = AggregatePeerTimings( + {Sample(1, 2000, PeerScheduleState::kExpected), + Sample(2, -100, PeerScheduleState::kMissedDeadline)}); + TEST_ASSERT_TRUE(future_plus_expired->state == PeerScheduleState::kExpected); + TEST_ASSERT_TRUE(future_plus_expired->next_ping_deadline == Tp(2000)); + + auto const future_plus_unknown = AggregatePeerTimings( + {Sample(1, 1000, PeerScheduleState::kExpected), + Sample(2, std::nullopt, PeerScheduleState::kUnknown)}); + TEST_ASSERT_TRUE(future_plus_unknown->state == PeerScheduleState::kExpected); +} + +void test_AggregateMissedUnknownAndErrors() { + auto const missed = AggregatePeerTimings( + {Sample(1, -3000, PeerScheduleState::kMissedDeadline), + Sample(2, -500, PeerScheduleState::kMissedDeadline)}); + TEST_ASSERT_TRUE(missed->state == PeerScheduleState::kMissedDeadline); + TEST_ASSERT_TRUE(missed->next_ping_deadline == Tp(-500)); + + auto const expired_unknown = AggregatePeerTimings( + {Sample(1, -500, PeerScheduleState::kMissedDeadline), + Sample(2, std::nullopt, PeerScheduleState::kUnknown)}); + TEST_ASSERT_TRUE(expired_unknown->state == PeerScheduleState::kUnknown); + TEST_ASSERT_FALSE(expired_unknown->next_ping_deadline.has_value()); + + auto const all_unknown = AggregatePeerTimings( + {Sample(1, std::nullopt, PeerScheduleState::kUnknown), + Sample(2, std::nullopt, PeerScheduleState::kUnknown)}); + TEST_ASSERT_TRUE(all_unknown->state == PeerScheduleState::kUnknown); + + auto const one_success = AggregatePeerTimings( + {Sample(9, -1000, PeerScheduleState::kMissedDeadline)}); + TEST_ASSERT_TRUE(one_success->state == PeerScheduleState::kMissedDeadline); + + TEST_ASSERT_FALSE( + AggregatePeerTimings(std::vector{}).has_value()); + + PeerTimingQueryState mixed; + mixed.Begin(); + auto const send_ok = mixed.RegisterSend(1, Tp(0), Ms(40)); + auto const send_err = mixed.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE( + mixed.ApplyTiming(1, send_ok, ClientTiming{-1000, -20})); + TEST_ASSERT_TRUE(mixed.ApplyError(2, send_err)); + auto const mixed_agg = mixed.TryAggregate(); + TEST_ASSERT_TRUE(mixed_agg.has_value()); + TEST_ASSERT_TRUE(mixed_agg->state == PeerScheduleState::kUnknown); + TEST_ASSERT_FALSE(mixed_agg->next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(mixed.ReadyToComplete()); + + PeerTimingQueryState all_err; + all_err.Begin(); + auto const e1 = all_err.RegisterSend(1, Tp(0), Ms(40)); + auto const e2 = all_err.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(all_err.ApplyError(1, e1)); + TEST_ASSERT_TRUE(all_err.ApplyError(2, e2)); + TEST_ASSERT_FALSE(all_err.TryAggregate().has_value()); +} + +void test_AggregateFreshestLastOnlineIndependentOfDeadline() { + auto const mixed = AggregatePeerTimings( + {Sample(100, 5000, PeerScheduleState::kExpected, 1), + Sample(900, -10, PeerScheduleState::kMissedDeadline, 2)}); + TEST_ASSERT_TRUE(mixed->last_online == Tp(900)); + TEST_ASSERT_TRUE(mixed->state == PeerScheduleState::kExpected); + TEST_ASSERT_TRUE(mixed->next_ping_deadline == Tp(5000)); + + auto const reversed = AggregatePeerTimings( + {Sample(900, -10, PeerScheduleState::kMissedDeadline, 2), + Sample(100, 5000, PeerScheduleState::kExpected, 1)}); + TEST_ASSERT_TRUE(reversed->last_online == Tp(900)); + TEST_ASSERT_TRUE(reversed->next_ping_deadline == Tp(5000)); +} + +void test_LifecycleOutOfOrderStaleCancelAndNoLeak() { + PeerTimingQueryState state; + auto const gen = state.Begin(); + TEST_ASSERT_TRUE(state.IsCurrentQuery(gen)); + + auto const send_a = state.RegisterSend(1, Tp(0), Ms(40)); + auto const send_b = state.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(state.ApplyTiming(2, send_b, ClientTiming{500, -20})); + TEST_ASSERT_TRUE(state.ApplyTiming(1, send_a, ClientTiming{1500, -10})); + auto aggregated = state.TryAggregate(); + TEST_ASSERT_TRUE(aggregated.has_value()); + TEST_ASSERT_TRUE(aggregated->state == PeerScheduleState::kExpected); + TEST_ASSERT_TRUE(aggregated->next_ping_deadline == Tp(0 + 40 + 1500)); + + TEST_ASSERT_FALSE(state.ApplyTiming(1, send_a - 1, ClientTiming{9, -1})); + TEST_ASSERT_FALSE(state.ApplyTiming(99, send_a, ClientTiming{9, -1})); + + state.Cancel(); + TEST_ASSERT_FALSE(state.ApplyTiming(1, send_a, ClientTiming{9, -1})); + + for (int i = 0; i < 1000; ++i) { + auto const g = state.Begin(); + auto const sid = static_cast(i % 7 + 1); + auto const send = state.RegisterSend(sid, Tp(i), Ms(40)); + TEST_ASSERT_TRUE( + state.ApplyTiming(sid, send, ClientTiming{100, -5})); + TEST_ASSERT_TRUE(state.IsCurrentQuery(g)); + TEST_ASSERT_EQUAL_UINT(1, state.attempts.size()); + } +} + +void test_ConservativeMatrixAndExpectedSnapshot() { + PeerTimingQueryState future_err; + future_err.Begin({1, 2}); + auto const f1 = future_err.RegisterSend(1, Tp(0), Ms(40)); + auto const f2 = future_err.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(future_err.ApplyTiming(1, f1, ClientTiming{1500, -10})); + TEST_ASSERT_TRUE(future_err.ApplyError(2, f2)); + auto const fe = future_err.TryAggregate(); + TEST_ASSERT_TRUE(fe.has_value()); + TEST_ASSERT_TRUE(fe->state == PeerScheduleState::kExpected); + TEST_ASSERT_TRUE(fe->next_ping_deadline.has_value()); + + PeerTimingQueryState unknown_err; + unknown_err.Begin({1, 2}); + auto const u1 = unknown_err.RegisterSend(1, Tp(0), Ms(40)); + auto const u2 = unknown_err.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(unknown_err.ApplyTiming(1, u1, ClientTiming{0, -10})); + TEST_ASSERT_TRUE(unknown_err.ApplyError(2, u2)); + auto const ue = unknown_err.TryAggregate(); + TEST_ASSERT_TRUE(ue.has_value()); + TEST_ASSERT_TRUE(ue->state == PeerScheduleState::kUnknown); + + PeerTimingQueryState snapshot; + snapshot.Begin({20, 21, 22}); + auto const s20 = snapshot.RegisterSend(20, Tp(0), Ms(40)); + auto const s21 = snapshot.RegisterSend(21, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(snapshot.ApplyTiming(20, s20, ClientTiming{-1000, -5})); + TEST_ASSERT_TRUE(snapshot.ApplyTiming(21, s21, ClientTiming{-800, -8})); + TEST_ASSERT_FALSE(snapshot.ReadyToComplete()); + auto const unresolved = snapshot.TryAggregate(); + TEST_ASSERT_TRUE(unresolved.has_value()); + TEST_ASSERT_TRUE(unresolved->state == PeerScheduleState::kUnknown); + TEST_ASSERT_TRUE(snapshot.ApplyError(22, snapshot.RegisterSend(22, Tp(0), Ms(40)))); + TEST_ASSERT_TRUE(snapshot.ReadyToComplete()); + auto const snap_done = snapshot.TryAggregate(); + TEST_ASSERT_TRUE(snap_done.has_value()); + TEST_ASSERT_TRUE(snap_done->state == PeerScheduleState::kUnknown); + + PeerTimingQueryState incomplete; + incomplete.Begin({20, 21}, true); + auto const i20 = incomplete.RegisterSend(20, Tp(0), Ms(40)); + auto const i21 = incomplete.RegisterSend(21, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(incomplete.ApplyTiming(20, i20, ClientTiming{-1000, -5})); + TEST_ASSERT_TRUE(incomplete.ApplyTiming(21, i21, ClientTiming{-800, -8})); + auto const inc = incomplete.TryAggregate(); + TEST_ASSERT_TRUE(inc.has_value()); + TEST_ASSERT_TRUE(inc->state == PeerScheduleState::kUnknown); + + PeerTimingQueryState all_neg; + all_neg.Begin({1, 2}); + auto const n1 = all_neg.RegisterSend(1, Tp(0), Ms(40)); + auto const n2 = all_neg.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(all_neg.ApplyTiming(1, n1, ClientTiming{-1000, -20})); + TEST_ASSERT_TRUE(all_neg.ApplyTiming(2, n2, ClientTiming{-400, -50})); + auto const missed = all_neg.TryAggregate(); + TEST_ASSERT_TRUE(missed.has_value()); + TEST_ASSERT_TRUE(missed->state == PeerScheduleState::kMissedDeadline); + TEST_ASSERT_TRUE(missed->next_ping_deadline == Tp(0 + 40 - 400)); + TEST_ASSERT_TRUE(missed->last_online == Tp(0 + 40 - 20)); +} + +void test_RetryRaceAndPostSuccessNoRemake() { + PeerTimingQueryOrchestrator orch; + orch.Start({1, 2}); + auto const a1 = orch.Send(1, Tp(0), Ms(40)); + auto const b1 = orch.Send(2, Tp(0), Ms(40)); + orch.OnTransient(2, b1); + TEST_ASSERT_EQUAL_INT(0, orch.callback_count); + TEST_ASSERT_FALSE(orch.state.ReadyToComplete()); + orch.OnSuccess(1, a1, ClientTiming{-1000, -10}); + TEST_ASSERT_EQUAL_INT(0, orch.callback_count); + auto const b2 = orch.Send(2, Tp(10), Ms(40)); + orch.OnSuccess(2, b2, ClientTiming{2000, -15}); + TEST_ASSERT_EQUAL_INT(1, orch.callback_count); + TEST_ASSERT_TRUE(orch.last_schedule.has_value()); + TEST_ASSERT_TRUE(orch.last_schedule->state == PeerScheduleState::kExpected); + + PeerTimingQueryOrchestrator reverse; + reverse.Start({1, 2}); + auto const ra = reverse.Send(1, Tp(0), Ms(40)); + reverse.OnSuccess(1, ra, ClientTiming{-1000, -10}); + TEST_ASSERT_EQUAL_INT(0, reverse.callback_count); + auto const rb = reverse.Send(2, Tp(0), Ms(40)); + reverse.OnTransient(2, rb); + TEST_ASSERT_EQUAL_INT(0, reverse.callback_count); + auto const rb2 = reverse.Send(2, Tp(10), Ms(40)); + reverse.OnSuccess(2, rb2, ClientTiming{2000, -15}); + TEST_ASSERT_EQUAL_INT(1, reverse.callback_count); + TEST_ASSERT_TRUE(reverse.last_schedule->state == PeerScheduleState::kExpected); + + PeerTimingQueryState post; + post.Begin({1, 2}); + auto const p1 = post.RegisterSend(1, Tp(0), Ms(40)); + auto const p2 = post.RegisterSend(2, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(post.ApplyTiming(1, p1, ClientTiming{1200, -10})); + auto const p1_again = post.RegisterSend(1, Tp(100), Ms(40)); + TEST_ASSERT_EQUAL_UINT(p1, p1_again); + TEST_ASSERT_TRUE(post.attempts[1].status == + ServerTimingAttemptStatus::kSuccess); + TEST_ASSERT_FALSE(post.ApplyError(1, p1)); + TEST_ASSERT_TRUE(post.attempts[1].converted.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(post.ApplyTiming(2, p2, ClientTiming{800, -20})); + auto const done = post.TryAggregate(); + TEST_ASSERT_TRUE(done->state == PeerScheduleState::kExpected); +} + +void test_QuarantinedServerExcludedFromQuerySetAllowsMissedDeadline() { + std::vector query_set; + auto const cov = BuildPeerTimingQuerySet( + { + SelectedServerSnapshotItem{20, false, true}, + SelectedServerSnapshotItem{21, false, true}, + SelectedServerSnapshotItem{22, true, true}, + }, + query_set); + TEST_ASSERT_EQUAL_UINT(3, cov.selected_server_count); + TEST_ASSERT_EQUAL_UINT(1, cov.quarantined_skipped_count); + TEST_ASSERT_EQUAL_UINT(2, cov.queried_server_count); + TEST_ASSERT_EQUAL_UINT(2, query_set.size()); + + PeerTimingQueryState st; + st.Begin(query_set, false, cov); + auto const q20 = st.RegisterSend(20, Tp(0), Ms(40)); + auto const q21 = st.RegisterSend(21, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(st.ApplyTiming(20, q20, ClientTiming{-1000, -5})); + TEST_ASSERT_TRUE(st.ApplyTiming(21, q21, ClientTiming{-800, -8})); + TEST_ASSERT_TRUE(st.attempts.find(22) == st.attempts.end()); + auto const missed = st.TryAggregate(); + TEST_ASSERT_TRUE(missed.has_value()); + TEST_ASSERT_TRUE(missed->state == PeerScheduleState::kMissedDeadline); + auto const got = st.QueryCoverage(); + TEST_ASSERT_EQUAL_UINT(0, got.failed_server_count); + TEST_ASSERT_EQUAL_UINT(2, got.successful_server_count); + TEST_ASSERT_EQUAL_UINT(1, got.quarantined_skipped_count); +} + +void test_ActiveServerQueryErrorGivesUnknown() { + PeerTimingQueryState st; + st.Begin({20, 21, 22}); + auto const q20 = st.RegisterSend(20, Tp(0), Ms(40)); + auto const q21 = st.RegisterSend(21, Tp(0), Ms(40)); + auto const q22 = st.RegisterSend(22, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(st.ApplyTiming(20, q20, ClientTiming{-1000, -5})); + TEST_ASSERT_TRUE(st.ApplyTiming(21, q21, ClientTiming{-800, -8})); + TEST_ASSERT_TRUE(st.ApplyTerminalError(22, q22)); + auto const unknown = st.TryAggregate(); + TEST_ASSERT_TRUE(unknown.has_value()); + TEST_ASSERT_TRUE(unknown->state == PeerScheduleState::kUnknown); +} + +void test_ServerLeavingQuarantineFreshQueryCanBeExpected() { + PeerTimingQueryState st; + st.Begin({20, 21, 22}); + auto const q20 = st.RegisterSend(20, Tp(0), Ms(40)); + auto const q21 = st.RegisterSend(21, Tp(0), Ms(40)); + auto const q22 = st.RegisterSend(22, Tp(0), Ms(40)); + TEST_ASSERT_TRUE(st.ApplyTiming(20, q20, ClientTiming{-1000, -5})); + TEST_ASSERT_TRUE(st.ApplyTiming(21, q21, ClientTiming{-800, -8})); + TEST_ASSERT_TRUE(st.ApplyTiming(22, q22, ClientTiming{2000, -7})); + auto const expected = st.TryAggregate(); + TEST_ASSERT_TRUE(expected.has_value()); + TEST_ASSERT_TRUE(expected->state == PeerScheduleState::kExpected); +} + +void test_AllSelectedServersQuarantinedYieldsEmptyQuerySet() { + std::vector query_set; + auto const cov = BuildPeerTimingQuerySet( + { + SelectedServerSnapshotItem{20, true, true}, + SelectedServerSnapshotItem{21, true, true}, + SelectedServerSnapshotItem{22, true, true}, + }, + query_set); + TEST_ASSERT_EQUAL_UINT(0, cov.queried_server_count); + TEST_ASSERT_EQUAL_UINT(3, cov.quarantined_skipped_count); + TEST_ASSERT_TRUE(query_set.empty()); +} + +void test_ThousandQueriesDestroyPendingAndCloudRequestFlags() { + PeerTimingQueryOrchestrator orch; + for (int i = 0; i < 1000; ++i) { + orch.Start({1, 2}); + auto const a = orch.Send(1, Tp(i), Ms(40)); + auto const b = orch.Send(2, Tp(i), Ms(40)); + orch.OnSuccess(1, a, ClientTiming{100, -5}); + orch.OnSuccess(2, b, ClientTiming{200, -6}); + TEST_ASSERT_EQUAL_INT(1, orch.callback_count); + TEST_ASSERT_EQUAL_INT(1, orch.state.user_callback_count); + } + + PeerTimingQueryOrchestrator pending; + pending.Start({1, 2}); + auto const pa = pending.Send(1, Tp(0), Ms(40)); + pending.Destroy(); + pending.OnSuccess(1, pa, ClientTiming{100, -5}); + TEST_ASSERT_EQUAL_INT(0, pending.callback_count); + + CloudRequestAttemptState attempt; + TEST_ASSERT_FALSE(attempt.ShouldSkipMake()); + attempt.MarkSucceeded(); + TEST_ASSERT_TRUE(attempt.ShouldSkipMake()); + TEST_ASSERT_FALSE(attempt.MarkFailed(5)); + TEST_ASSERT_FALSE(attempt.exhausted); + + CloudRequestAttemptState fail; + TEST_ASSERT_FALSE(fail.MarkFailed(2)); + TEST_ASSERT_TRUE(fail.MarkFailed(2)); + TEST_ASSERT_TRUE(fail.ShouldSkipMake()); + TEST_ASSERT_TRUE(CloudRequestShouldFailAll(false, false)); + TEST_ASSERT_FALSE(CloudRequestShouldFailAll(false, true)); + TEST_ASSERT_FALSE(CloudRequestShouldFailAll(true, false)); +} + +} // namespace ae::test_uap_peer_timing + +int test_uap_peer_timing() { + UNITY_BEGIN(); + RUN_TEST(ae::test_uap_peer_timing::test_PingMethodIdAndParamOrder); + RUN_TEST(ae::test_uap_peer_timing::test_GetClientTimingMethodIdAndUidParam); + RUN_TEST(ae::test_uap_peer_timing:: + test_GetUapRemainsMethod34AndIsNotClientTiming); + RUN_TEST(ae::test_uap_peer_timing:: + test_PingPacketDoesNotIncludeSetNextReadDelay); + RUN_TEST(ae::test_uap_peer_timing::test_ClientTimingFieldOrderSignedInt64); + RUN_TEST(ae::test_uap_peer_timing:: + test_ClientTimingNegativeZeroPositiveAndBounds); + RUN_TEST(ae::test_uap_peer_timing::test_ConversionExampleQsendMinRtt80); + RUN_TEST(ae::test_uap_peer_timing:: + test_ConversionNegativeZeroEmptyStatsAndSaturation); + RUN_TEST(ae::test_uap_peer_timing::test_AggregateFutureCases); + RUN_TEST(ae::test_uap_peer_timing::test_AggregateMissedUnknownAndErrors); + RUN_TEST(ae::test_uap_peer_timing:: + test_AggregateFreshestLastOnlineIndependentOfDeadline); + RUN_TEST(ae::test_uap_peer_timing:: + test_LifecycleOutOfOrderStaleCancelAndNoLeak); + RUN_TEST(ae::test_uap_peer_timing::test_ConservativeMatrixAndExpectedSnapshot); + RUN_TEST(ae::test_uap_peer_timing::test_RetryRaceAndPostSuccessNoRemake); + RUN_TEST(ae::test_uap_peer_timing:: + test_ThousandQueriesDestroyPendingAndCloudRequestFlags); + RUN_TEST(ae::test_uap_peer_timing:: + test_QuarantinedServerExcludedFromQuerySetAllowsMissedDeadline); + RUN_TEST(ae::test_uap_peer_timing::test_ActiveServerQueryErrorGivesUnknown); + RUN_TEST(ae::test_uap_peer_timing:: + test_ServerLeavingQuarantineFreshQueryCanBeExpected); + RUN_TEST(ae::test_uap_peer_timing:: + test_AllSelectedServersQuarantinedYieldsEmptyQuerySet); + return UNITY_END(); +} diff --git a/tests/test-api-protocol/test-uap-receive-schedule.cpp b/tests/test-api-protocol/test-uap-receive-schedule.cpp new file mode 100644 index 00000000..be1d0f6f --- /dev/null +++ b/tests/test-api-protocol/test-uap-receive-schedule.cpp @@ -0,0 +1,927 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include "aether-miscpp/serialization/binary_archive.h" + +#include "aether/ae_actions/query_peer_receive_schedule.h" +#include "aether/channels/channel.h" +#include "aether/cloud_connections/ping_schedule_guard.h" +#include "aether/config.h" +#if AE_ENABLE_PING_TEST_FAULTS +#include "aether/ae_actions/ping_test_faults.h" +#endif +#include "aether/receive_schedule.h" +#include "aether/types/statistic_counter.h" +#include "aether/work_cloud_api/client_timing.h" +#include "aether/work_cloud_api/uap.h" + +#include "examples/benches/aether_uap_delivery_timing_bench/common/bench_message.h" + +namespace ae::test_uap_receive_schedule { +namespace { + +Duration Ms(std::uint32_t v) { + return std::chrono::duration_cast(std::chrono::milliseconds{v}); +} + +} // namespace + +void test_ResponseTimeoutEmptyUsesInitialEstimate() { + StatisticsCounter stats{}; + TEST_ASSERT_TRUE(stats.empty()); + // Channel::ResponseTimeout() returns kInitialResponseEstimate when empty. + auto const timeout = stats.empty() ? Channel::kInitialResponseEstimate + : stats.percentile<99>(); + TEST_ASSERT_TRUE(timeout == Channel::kInitialResponseEstimate); + TEST_ASSERT_TRUE(timeout == Ms(200)); +} + +void test_ResponseTimeoutAfterFirstSampleUsesPercentile() { + StatisticsCounter stats{}; + stats.Add(Ms(50)); + TEST_ASSERT_FALSE(stats.empty()); + TEST_ASSERT_EQUAL_UINT(1, stats.size()); + TEST_ASSERT_TRUE(stats.min() == stats.percentile<99>()); + TEST_ASSERT_TRUE(stats.percentile<99>() == Ms(50)); +} + +void test_NoSyntheticSeedMeansEmptyUntilRealRtt() { + StatisticsCounter stats{}; + TEST_ASSERT_TRUE(stats.empty()); + stats.Add(Ms(12)); + TEST_ASSERT_EQUAL_UINT(1, stats.size()); +} + +void test_PingDeadlineGuardFormulas() { + StatisticsCounter empty{}; + TEST_ASSERT_TRUE(ComputePingSendGuardFromStats(empty, Ms(3000)) == Ms(10)); + StatisticsCounter one{}; + one.Add(Ms(80)); + TEST_ASSERT_TRUE(ComputePingSendGuardFromStats(one, Ms(3000)) == Ms(10)); + TEST_ASSERT_TRUE(ComputePingSendGuard(Ms(60), Ms(100)) == Ms(30)); + TEST_ASSERT_TRUE(ClampPingSendGuard(Ms(200), Ms(30)) == Ms(29)); +} + +void test_WireAnnouncedIntervalUnchangedVsLocalSendEarlier() { + auto const interval = Ms(3000); + auto const guard = ComputePingSendGuard(Ms(60), Ms(100)); + TEST_ASSERT_TRUE(guard == Ms(30)); + TEST_ASSERT_TRUE((interval - guard) == Ms(2970)); +} + +void test_RxCloseIsPongPlusWindowNotSendPlusWindow() { + auto const send = TimePoint{} + std::chrono::seconds{1}; + auto const pong = send + std::chrono::milliseconds{35}; + auto const window = Ms(200); + auto const close_at = ComputeRxWindowCloseTime(pong, window); + TEST_ASSERT_TRUE(close_at == pong + window); + TEST_ASSERT_TRUE(close_at != send + window); +} + +void test_ConversionUsesLibraryTimePointOnly() { + auto const qsend = TimePoint{} + std::chrono::milliseconds{1000}; + auto const one_way = OneWayPingEstimate(false, Ms(80)); + TEST_ASSERT_TRUE(one_way == Ms(40)); + ClientTiming const timing{4'000, -1'000}; + auto const converted = ConvertClientTiming(qsend, one_way, timing); + TEST_ASSERT_TRUE(converted.last_online == + qsend + one_way - std::chrono::milliseconds{1000}); + TEST_ASSERT_TRUE(converted.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(*converted.next_ping_deadline == + qsend + one_way + std::chrono::milliseconds{4000}); + TEST_ASSERT_TRUE(TimePointOffsetByMs(qsend, -1'000) == + qsend - std::chrono::milliseconds{1000}); + static_assert(std::is_same_v); +} + +void test_DeltaZeroYieldsUnknownDeadline() { + auto const converted = ConvertClientTiming( + TimePoint{} + std::chrono::seconds{5}, Ms(40), ClientTiming{0, -100}); + TEST_ASSERT_FALSE(converted.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(converted.state == PeerScheduleState::kUnknown); +} + +void test_NegativeDeltaYieldsMissedDeadline() { + auto const converted = + ConvertClientTiming(TimePoint{}, Ms(40), ClientTiming{-1, -10}); + TEST_ASSERT_TRUE(converted.next_ping_deadline.has_value()); + TEST_ASSERT_TRUE(converted.state == PeerScheduleState::kMissedDeadline); +} + +void test_UapWireFieldOrderAndSignedInt64() { + Uap const uap{5'500, 1'700'000'000'000}; + std::vector packed; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Save(uap); + } + TEST_ASSERT_EQUAL_UINT(16, packed.size()); + std::int64_t delta = 0; + std::int64_t last_read = 0; + { + auto archive = ae::seri::BinaryArchive{ + ae::VectorBuffer{packed}, + }; + archive.Load(delta); + archive.Load(last_read); + } + TEST_ASSERT_EQUAL_INT64(5'500, delta); + TEST_ASSERT_EQUAL_INT64(1'700'000'000'000, last_read); +} + +void test_BenchMessageCrcRoundTrip() { + ae::bench::uap::DeliveryBenchMessage msg{}; + msg.offset_ms = 250; + msg.sequence = 42; + msg.send_qpc = 0x1122334455667788ull; + auto const bytes = ae::bench::uap::SerializeDeliveryBenchMessage(msg); + auto decoded = ae::bench::uap::DeserializeDeliveryBenchMessage(bytes.data(), + bytes.size()); + TEST_ASSERT_TRUE(decoded.has_value()); + TEST_ASSERT_EQUAL_UINT16(msg.offset_ms, decoded->offset_ms); + TEST_ASSERT_EQUAL_UINT32(msg.sequence, decoded->sequence); + TEST_ASSERT_EQUAL_UINT64(msg.send_qpc, decoded->send_qpc); + TEST_ASSERT_EQUAL_UINT32(ae::bench::uap::DeliveryBenchMessageCrc(*decoded), + decoded->crc); + + auto tampered = bytes; + tampered[8] ^= 0xFFu; + auto bad = ae::bench::uap::DeserializeDeliveryBenchMessage(tampered.data(), + tampered.size()); + TEST_ASSERT_FALSE(bad.has_value()); +} + +void test_ClassifyReceiveSendOffset() { + auto const window = Ms(1000); + TEST_ASSERT_TRUE(ClassifyReceiveSendOffset(Ms(500), window) == + ReceiveSendPhase::kInsideReceiveWindow); + TEST_ASSERT_TRUE(ClassifyReceiveSendOffset(Ms(1500), window) == + ReceiveSendPhase::kOutsideReceiveWindow); +} + +void test_EarlyRxWindowComputation() { + auto const t0 = TimePoint{}; + auto at = [&](std::uint32_t ms) { return t0 + Ms(ms); }; + + auto none = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = false, + .actual_send_at = at(100), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(none.early_by == Ms(0)); + TEST_ASSERT_TRUE(none.effective_wire_rx_window == Ms(1000)); + + auto on_time = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(3000), + .actual_send_at = at(3000), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(on_time.early_by == Ms(0)); + TEST_ASSERT_TRUE(on_time.effective_wire_rx_window == Ms(1000)); + + auto early700 = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(3000), + .actual_send_at = at(2300), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(early700.early_by == Ms(700)); + TEST_ASSERT_TRUE(early700.effective_wire_rx_window == Ms(1700)); + + auto early1200 = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(3000), + .actual_send_at = at(1800), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(early1200.early_by == Ms(1200)); + TEST_ASSERT_TRUE(early1200.effective_wire_rx_window == Ms(2200)); + + auto late = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(3000), + .actual_send_at = at(3200), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(late.early_by == Ms(0)); + TEST_ASSERT_TRUE(late.effective_wire_rx_window == Ms(1000)); + + auto held = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(7000), + .actual_send_at = at(7000), + .base_rx_window = Ms(1000), + .has_required_rx_until = true, + .required_rx_until = at(10000), + }); + TEST_ASSERT_TRUE(held.effective_wire_rx_window == Ms(3000)); + + auto second = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(2800), + .actual_send_at = at(2800), + .base_rx_window = Ms(1000), + .has_required_rx_until = true, + .required_rx_until = early1200.required_rx_until, + }); + TEST_ASSERT_TRUE(second.required_rx_until >= early1200.required_rx_until); + TEST_ASSERT_TRUE(second.effective_wire_rx_window >= + SaturatingSubTime(early1200.required_rx_until, at(2800))); + + auto zero_base = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(3000), + .actual_send_at = at(1800), + .base_rx_window = Ms(0), + }); + TEST_ASSERT_TRUE(zero_base.effective_wire_rx_window == Ms(1200)); + + auto past_planned = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = at(100), + .actual_send_at = at(2000), + .base_rx_window = Ms(1000), + }); + TEST_ASSERT_TRUE(past_planned.early_by == Ms(0)); + TEST_ASSERT_TRUE(past_planned.effective_wire_rx_window == Ms(1000)); + + auto overflow = SaturatingAddTime(TimePoint::max(), Ms(1000)); + TEST_ASSERT_TRUE(overflow == TimePoint::max()); + TEST_ASSERT_TRUE(DurationToSaturatedInt64Ms(Duration::max()) > 0); +} + +void test_LocalRxWindowMonotonicAndStaleTimer() { + LocalRxWindowState s{}; + auto const t0 = TimePoint{}; + TEST_ASSERT_TRUE(ExtendLocalRxUntil(s, t0 + Ms(5000))); + TEST_ASSERT_EQUAL_UINT64(1, s.generation); + TEST_ASSERT_TRUE(ExtendLocalRxUntil(s, t0 + Ms(8000))); + auto const gen8 = s.generation; + TEST_ASSERT_FALSE(ExtendLocalRxUntil(s, t0 + Ms(6000))); + TEST_ASSERT_EQUAL_UINT64(gen8, s.generation); + TEST_ASSERT_TRUE(s.close_at == t0 + Ms(8000)); + TEST_ASSERT_FALSE(ShouldApplyCloseTimer(s, 1, t0 + Ms(5000))); + TEST_ASSERT_TRUE(ShouldApplyCloseTimer(s, gen8, t0 + Ms(8000))); + TEST_ASSERT_FALSE(ShouldCloseLocalRxAfterWriteFailure( + s, true, t0 + Ms(8000), t0 + Ms(6000))); + TEST_ASSERT_TRUE(ShouldCloseLocalRxAfterWriteFailure( + LocalRxWindowState{}, false, TimePoint{}, t0 + Ms(6000))); + auto const close_pong = + ComputeRxWindowCloseTime(t0 + Ms(1900), Ms(2200)); + TEST_ASSERT_TRUE(close_pong == t0 + Ms(4100)); + auto const close_timeout = + ComputeRxWindowCloseTime(t0 + Ms(2500), Ms(2200)); + TEST_ASSERT_TRUE(close_timeout == t0 + Ms(4700)); + TEST_ASSERT_FALSE(ExtendLocalRxUntil(s, close_timeout)); + TEST_ASSERT_TRUE(s.close_at == t0 + Ms(8000)); + TEST_ASSERT_TRUE(ExtendLocalRxUntil(s, t0 + Ms(9000))); + TEST_ASSERT_TRUE(s.close_at == t0 + Ms(9000)); + CloseLocalRx(s); + TEST_ASSERT_FALSE(s.open); + CloseLocalRx(s); + TEST_ASSERT_FALSE(s.open); +} + +void test_ServerWindowInvariantIndependentOfRtt() { + auto const t0 = TimePoint{}; + auto const planned = t0 + Ms(3000); + auto const actual = t0 + Ms(1800); + auto const base = Ms(1000); + auto const out = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = planned, + .actual_send_at = actual, + .base_rx_window = base, + }); + auto const business_end = planned + base; + for (auto delay_ms : {0u, 30u, 50u}) { + auto const server_receive = actual + Ms(delay_ms); + auto const server_end = + SaturatingAddTime(server_receive, out.effective_wire_rx_window); + TEST_ASSERT_TRUE(server_end >= business_end); + } +} + +void test_LogicalPingFirstAttemptAnchorsSchedule() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const view = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(view.started_new_cycle); + TEST_ASSERT_FALSE(view.is_retry); + TEST_ASSERT_EQUAL_UINT(1, view.attempt_index); + TEST_ASSERT_EQUAL_INT64(3000, view.wire_next_connect_ms); + TEST_ASSERT_TRUE(view.contract_deadline == req.actual_send_at + Ms(3000)); + TEST_ASSERT_EQUAL_INT64( + (view.contract_deadline - std::chrono::milliseconds{10}) + .time_since_epoch() + .count(), + view.next_local_send.time_since_epoch().count()); + TEST_ASSERT_TRUE(st.cycle_id == view.cycle_id); +} + +void test_LogicalPingOneRetryKeepsPhase() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(3200); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_FALSE(retry.started_new_cycle); + TEST_ASSERT_TRUE(retry.is_retry); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_EQUAL_UINT(2, retry.attempt_index); + TEST_ASSERT_EQUAL_INT64(2800, retry.wire_next_connect_ms); + TEST_ASSERT_TRUE(retry.contract_deadline == first.contract_deadline); + TEST_ASSERT_TRUE(retry.next_local_send == first.next_local_send); +} + +void test_LogicalPingTwoRetriesKeepPhase() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(3100); + auto const r2 = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(3400); + auto const r3 = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(r2.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(r3.cycle_id == first.cycle_id); + TEST_ASSERT_EQUAL_INT64(2900, r2.wire_next_connect_ms); + TEST_ASSERT_EQUAL_INT64(2600, r3.wire_next_connect_ms); + TEST_ASSERT_TRUE(r3.contract_deadline == first.contract_deadline); + TEST_ASSERT_TRUE(r3.next_local_send == first.next_local_send); +} + +void test_LogicalPingSuccessConfirmsOnceAndIgnoresStale() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); + ConfirmLogicalPingCycle(st); + TEST_ASSERT_TRUE(st.confirmed); + TEST_ASSERT_FALSE(st.active); + TEST_ASSERT_FALSE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); +} + +void test_LogicalPingLateOldAttemptConfirmsActiveCycle() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + MarkLogicalPingAttemptTimedOut(st); + TEST_ASSERT_TRUE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); + req.actual_send_at = TimePoint{} + Ms(3200); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_EQUAL_UINT(2, retry.attempt_index); + TEST_ASSERT_TRUE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); + ConfirmLogicalPingCycle(st); + TEST_ASSERT_FALSE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); +} + +void test_LogicalPingPreviousCyclePongIgnored() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + req.actual_send_at = TimePoint{} + Ms(5990); + auto const second = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(second.cycle_id != first.cycle_id); + TEST_ASSERT_FALSE(ShouldAcceptCycleResult( + st.active, st.confirmed, st.cycle_id, first.cycle_id, st.attempt_index, + first.attempt_index, st.current_attempt_timed_out, true)); +} + +void test_LogicalPingErrorRetryActions() { + TEST_ASSERT_TRUE(PingErrorRetryActionFor(2) == + PingErrorRetryAction::kImmediateSameCycle); + TEST_ASSERT_TRUE(PingErrorRetryActionFor(1) == + PingErrorRetryAction::kRestreamThenSameCycle); + TEST_ASSERT_TRUE(PingErrorRetryActionFor(3) == + PingErrorRetryAction::kRestreamThenSameCycle); +} + +void test_LogicalPingEarlyAttemptDoesNotShiftPhase() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(50); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(first.bootstrap); + TEST_ASSERT_TRUE(first.nominal_ping_at == req.actual_send_at); + ConfirmLogicalPingCycle(st); + req.actual_send_at = TimePoint{} + Ms(5960); + auto const second = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(second.started_new_cycle); + TEST_ASSERT_FALSE(second.bootstrap); + TEST_ASSERT_TRUE(second.nominal_ping_at == first.next_nominal_ping_at); + TEST_ASSERT_TRUE(second.next_nominal_ping_at == + first.next_nominal_ping_at + Ms(3000)); + TEST_ASSERT_EQUAL_INT64(3040, second.wire_next_connect_ms); +} + +void test_LogicalPingSubsequentEarlyWireIsExtended() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(4000); + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(250); + req.base_rx_window = Ms(250); + ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + req.actual_send_at = TimePoint{} + Ms(4750); + auto const second = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(second.nominal_ping_at == TimePoint{} + Ms(5000)); + TEST_ASSERT_TRUE(second.next_nominal_ping_at == TimePoint{} + Ms(6000)); + TEST_ASSERT_EQUAL_INT64(1250, second.wire_next_connect_ms); + TEST_ASSERT_TRUE(second.rx.effective_wire_rx_window == Ms(500)); +} + +void test_LogicalPingRetryKeepsNominalAndShrinksWire() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(4000); + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(250); + req.base_rx_window = Ms(250); + ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + req.actual_send_at = TimePoint{} + Ms(4750); + auto const first = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(4920); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(retry.nominal_ping_at == first.nominal_ping_at); + TEST_ASSERT_TRUE(retry.next_nominal_ping_at == first.next_nominal_ping_at); + TEST_ASSERT_EQUAL_INT64(1080, retry.wire_next_connect_ms); +} + +void test_LogicalPingLateRetryAdvancesWholeIntervals() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(4000); + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(250); + req.base_rx_window = Ms(250); + ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + req.actual_send_at = TimePoint{} + Ms(4750); + auto const first = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(6200); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(retry.next_nominal_ping_at == TimePoint{} + Ms(7000)); + TEST_ASSERT_EQUAL_INT64(800, retry.wire_next_connect_ms); +} + +void test_LogicalPingHundredCyclesNoPhaseDrift() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(250); + req.base_rx_window = Ms(250); + TimePoint send = TimePoint{}; + TimePoint last_tn{}; + for (int i = 0; i < 100; ++i) { + req.actual_send_at = send; + auto const view = ApplyLogicalPingAttempt(st, req); + if (i == 0) { + TEST_ASSERT_TRUE(view.bootstrap); + TEST_ASSERT_TRUE(view.nominal_ping_at == TimePoint{}); + } else { + TEST_ASSERT_FALSE(view.bootstrap); + TEST_ASSERT_TRUE(view.nominal_ping_at == TimePoint{} + Ms(static_cast(i * 1000))); + } + last_tn = view.nominal_ping_at; + ConfirmLogicalPingCycle(st); + send = view.next_local_send; + } + TEST_ASSERT_TRUE(last_tn == TimePoint{} + Ms(99000)); +} + +void test_PingRetryBudgetEmptyStats() { + StatisticsCounter empty{}; + auto const budget = + ComputePingRetryBudgetFromStats(empty, Ms(1000), Ms(200)); + TEST_ASSERT_TRUE(budget.scheduler_margin == Ms(10)); + TEST_ASSERT_TRUE(budget.retry_one_way_budget == Ms(100)); + TEST_ASSERT_TRUE(budget.loss_timeout == Ms(210)); + TEST_ASSERT_TRUE(budget.retry_reserve == Ms(100)); + TEST_ASSERT_TRUE(budget.attempt_lead == Ms(110)); + TEST_ASSERT_TRUE(budget.predeadline_retry_guaranteed); + TEST_ASSERT_TRUE(budget.loss_timeout.count() > 0); +} + +void test_DefaultPingRetryCountIsZero() { + TEST_ASSERT_EQUAL_UINT(0, kDefaultPingRetryCount); + ReceiveSchedule schedule{}; + TEST_ASSERT_EQUAL_UINT(0, schedule.ping_retry_count); +} + +void test_PingRetryBudgetN0LeadIsGuardPlusHalfP99() { + auto const guard = ResolvePingSendGuard(Ms(80), Ms(80), Ms(1000)); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + Ms(1000), guard, Ms(5000), Ms(80), /*retry_count=*/0}); + TEST_ASSERT_TRUE(guard == Ms(10)); + TEST_ASSERT_TRUE(budget.loss_timeout == Ms(5000)); + TEST_ASSERT_TRUE(budget.retry_reserve == Ms(40)); + TEST_ASSERT_TRUE(budget.attempt_lead == Ms(50)); +} + +void test_PingRetryBudgetN1PreDeadlineRetry() { + auto const guard = ComputePingSendGuard(Ms(60), Ms(100)); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + Ms(1000), guard, Ms(100), Ms(100), /*retry_count=*/1}); + TEST_ASSERT_TRUE(budget.retry_one_way_budget == Ms(50)); + TEST_ASSERT_TRUE(budget.loss_timeout == Ms(110)); + TEST_ASSERT_TRUE(budget.retry_dispatch_margin == Ms(60)); + TEST_ASSERT_TRUE(budget.retry_reserve == Ms(230)); + TEST_ASSERT_TRUE(budget.attempt_lead == Ms(260)); + TEST_ASSERT_TRUE(budget.predeadline_retry_guaranteed); +} + +void test_PingRetryBudgetN2PreDeadlineRetry() { + auto const guard = Ms(10); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + Ms(1000), guard, Ms(100), Ms(100), /*retry_count=*/2}); + TEST_ASSERT_TRUE(budget.retry_reserve == + Ms(110 + 110 + 50 + 10 + 10 + 60 + 60)); + TEST_ASSERT_TRUE(budget.attempt_lead == Ms(10) + budget.retry_reserve); +} + +void test_PreDeadlineRetryCountExhaustion() { + auto const deadline = TimePoint{} + Ms(1000); + TEST_ASSERT_FALSE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(500), deadline, 1, 0)); + TEST_ASSERT_TRUE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(500), deadline, 1, 1)); + TEST_ASSERT_FALSE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(500), deadline, 2, 1)); + TEST_ASSERT_TRUE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(500), deadline, 2, 2)); + TEST_ASSERT_FALSE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(500), deadline, 3, 2)); +} + +void test_PostDeadlineRetryAllowedAfterZeroPreDeadlineBudget() { + auto const deadline = TimePoint{} + Ms(1000); + TEST_ASSERT_TRUE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(1000), deadline, 1, 0)); + TEST_ASSERT_TRUE(CanSchedulePreDeadlineSameCycleRetry( + TimePoint{} + Ms(1500), deadline, 5, 0)); +} + +void test_PostDeadlineRecoveryKeepsLogicalCycleAndPhase() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.interval = Ms(1000); + req.guard = Ms(10); + req.attempt_lead = Ms(50); + req.base_rx_window = Ms(250); + req.actual_send_at = TimePoint{}; + auto const boot = ApplyLogicalPingAttempt(st, req); + ConfirmLogicalPingCycle(st); + req.actual_send_at = boot.next_local_send; + auto const first = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(first.nominal_ping_at == TimePoint{} + Ms(1000)); + req.actual_send_at = TimePoint{} + Ms(1100); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(retry.attempt_index == 2U); + TEST_ASSERT_TRUE(retry.contract_deadline == TimePoint{} + Ms(2000)); + TEST_ASSERT_EQUAL_INT64(900, retry.wire_next_connect_ms); + ConfirmLogicalPingCycle(st); + req.actual_send_at = retry.next_local_send; + auto const next = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(next.cycle_id > first.cycle_id); + TEST_ASSERT_TRUE(next.nominal_ping_at == TimePoint{} + Ms(2000)); +} + +void test_PingRetryBudgetRealMinP99() { + auto const guard = ComputePingSendGuard(Ms(60), Ms(100)); + TEST_ASSERT_TRUE(guard == Ms(30)); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + Ms(1000), guard, Ms(100), Ms(100), /*retry_count=*/1}); + TEST_ASSERT_TRUE(budget.retry_one_way_budget == Ms(50)); + TEST_ASSERT_TRUE(budget.loss_timeout == Ms(110)); + TEST_ASSERT_TRUE(budget.retry_dispatch_margin == Ms(60)); + // loss(110) + one_way(50) + scheduler(10) + dispatch(60) = 230 + TEST_ASSERT_TRUE(budget.retry_reserve == Ms(230)); + // guard(30) + retry_reserve(230) = 260 + TEST_ASSERT_TRUE(budget.attempt_lead == Ms(260)); + TEST_ASSERT_TRUE(budget.predeadline_retry_guaranteed); +} + +void test_PingRetryBudgetOneSecondAllowsRetryWithP99_80() { + auto const guard = ComputePingSendGuard(Ms(80), Ms(80)); + auto const budget = ComputePingRetryBudget(PingRetryBudgetInput{ + Ms(1000), guard, Ms(80), Ms(80), /*retry_count=*/1}); + TEST_ASSERT_TRUE(guard == Ms(10)); + TEST_ASSERT_TRUE(budget.loss_timeout == Ms(90)); + TEST_ASSERT_TRUE(budget.attempt_lead < Ms(1000)); + TEST_ASSERT_TRUE(budget.predeadline_retry_guaranteed); + TEST_ASSERT_TRUE(budget.loss_timeout < budget.attempt_lead); +} + +void test_PingRetryBudgetShortIntervalSetsDiagnosticAndDoesNotWireZero() { + auto const budget = ComputePingRetryBudget( + PingRetryBudgetInput{Ms(40), Ms(10), Ms(200), Ms(200)}); + TEST_ASSERT_FALSE(budget.predeadline_retry_guaranteed); + TEST_ASSERT_TRUE(budget.loss_timeout.count() > 0); + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(1000); + req.interval = Ms(40); + req.guard = Ms(10); + req.attempt_lead = budget.attempt_lead; + req.loss_timeout = budget.loss_timeout; + req.predeadline_retry_guaranteed = budget.predeadline_retry_guaranteed; + req.base_rx_window = Ms(20); + auto const view = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(view.wire_next_connect_ms >= 1); +} + +void test_RxWindowUsesNominalPlusBaseWindow() { + auto const t0 = TimePoint{}; + auto const out = ComputeEarlyRxWindow(EarlyRxWindowInput{ + .has_nominal_ping = true, + .nominal_ping_at = t0 + Ms(5000), + .actual_send_at = t0 + Ms(4750), + .base_rx_window = Ms(250), + }); + TEST_ASSERT_TRUE(out.effective_wire_rx_window == Ms(500)); + TEST_ASSERT_TRUE(out.required_rx_until == t0 + Ms(5250)); +} + +void test_AnnounceUnknownWiresZero() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(1000); + req.guard = Ms(10); + req.base_rx_window = Ms(250); + req.announce_unknown = true; + auto const view = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_EQUAL_INT64(0, view.wire_next_connect_ms); +} + +void test_LogicalPingRetryAfterDeadlineKeepsPhaseAndNeverWiresZero() { + LogicalPingCycleState st{}; + LogicalPingAttemptRequest req{}; + req.actual_send_at = TimePoint{} + Ms(3000); + req.interval = Ms(3000); + req.guard = Ms(10); + req.base_rx_window = Ms(1000); + auto const first = ApplyLogicalPingAttempt(st, req); + req.actual_send_at = TimePoint{} + Ms(7000); + auto const retry = ApplyLogicalPingAttempt(st, req); + TEST_ASSERT_TRUE(retry.cycle_id == first.cycle_id); + TEST_ASSERT_TRUE(retry.wire_next_connect_ms >= 1); + TEST_ASSERT_TRUE(retry.contract_deadline == + first.contract_deadline + Ms(3000)); + TEST_ASSERT_EQUAL_INT64( + (retry.contract_deadline - std::chrono::milliseconds{10}) + .time_since_epoch() + .count(), + retry.next_local_send.time_since_epoch().count()); +} + +void test_WireRoundingFloorConnectCeilRxRemainders() { + for (std::uint32_t us = 1; us <= 999; ++us) { + Duration const rem{us}; + TEST_ASSERT_EQUAL_INT64(1, FloorDurationToPositiveInt64Ms(rem)); + TEST_ASSERT_EQUAL_INT64(1, CeilDurationToSaturatedInt64Ms(rem)); + } + TEST_ASSERT_EQUAL_INT64(1, FloorDurationToPositiveInt64Ms(Duration{1000})); + TEST_ASSERT_EQUAL_INT64(1, FloorDurationToPositiveInt64Ms(Duration{1999})); + TEST_ASSERT_EQUAL_INT64(2, FloorDurationToPositiveInt64Ms(Duration{2000})); + TEST_ASSERT_EQUAL_INT64(1, CeilDurationToSaturatedInt64Ms(Duration{1000})); + TEST_ASSERT_EQUAL_INT64(2, CeilDurationToSaturatedInt64Ms(Duration{1001})); + TEST_ASSERT_EQUAL_INT64(1, FloorDurationToPositiveInt64Ms(Duration{})); +} + +void test_LogicalPingSaturationAndOverflow() { + TEST_ASSERT_TRUE(SaturatingAddTime(TimePoint::max(), Ms(1)) == + TimePoint::max()); + TEST_ASSERT_TRUE(DurationToSaturatedInt64Ms(Duration::max()) > 0); + TEST_ASSERT_TRUE(FloorDurationToPositiveInt64Ms(Duration::max()) > 0); + TEST_ASSERT_TRUE(CeilDurationToSaturatedInt64Ms(Duration::max()) > 0); + auto const advanced = + AdvanceContractDeadlinePast(TimePoint{}, TimePoint::max(), Ms(3000)); + TEST_ASSERT_TRUE(advanced == TimePoint::max() || advanced > TimePoint{}); +} + +#if AE_ENABLE_PING_TEST_FAULTS +void test_PingTestFaultsTargetExactServerCycleAttempt() { + PingTestFaults::Instance().Clear(); + PingFaultPlan plan{}; + plan.server_id = 20; + plan.logical_cycle_id = 7; + plan.physical_attempt_index = 1; + plan.mode = PingFaultMode::kDropRequest; + PingTestFaults::Instance().Arm(plan); + + auto miss_server = PingTestFaults::Instance().Consume( + PingFaultContext{21, 7, 1, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(miss_server.mode == PingFaultMode::kNone); + auto miss_cycle = PingTestFaults::Instance().Consume( + PingFaultContext{20, 8, 1, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(miss_cycle.mode == PingFaultMode::kNone); + auto hit = PingTestFaults::Instance().Consume( + PingFaultContext{20, 7, 1, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(hit.mode == PingFaultMode::kDropRequest); + auto consumed = PingTestFaults::Instance().Consume( + PingFaultContext{20, 7, 1, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(consumed.mode == PingFaultMode::kNone); + PingTestFaults::Instance().Clear(); +} + +void test_PingTestFaultsBindNextCycleAndIgnoreResponse() { + PingTestFaults::Instance().Clear(); + PingFaultPlan plan{}; + plan.server_id = 20; + plan.logical_cycle_id = 0; + plan.physical_attempt_index = 1; + plan.mode = PingFaultMode::kIgnoreResponse; + plan.timeout_override = Ms(400); + PingTestFaults::Instance().Arm(plan); + PingTestFaults::Instance().BindNextCycle(20, 3); + auto hit = PingTestFaults::Instance().Consume( + PingFaultContext{20, 3, 1, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(hit.mode == PingFaultMode::kIgnoreResponse); + TEST_ASSERT_TRUE(hit.timeout_override == Ms(400)); + PingTestFaults::Instance().OnAuthPing(); + PingTestFaults::Instance().OnProtocolResponse(); + TEST_ASSERT_EQUAL_UINT(1, PingTestFaults::Instance().auth_ping_calls()); + TEST_ASSERT_EQUAL_UINT(1, PingTestFaults::Instance().protocol_responses()); + PingTestFaults::Instance().Clear(); +} + +void test_PingTestFaultsConsecutiveAttempts() { + PingTestFaults::Instance().Clear(); + PingFaultPlan drop1{}; + drop1.server_id = 20; + drop1.logical_cycle_id = 4; + drop1.physical_attempt_index = 1; + drop1.mode = PingFaultMode::kDropRequest; + PingFaultPlan drop2{}; + drop2.server_id = 20; + drop2.logical_cycle_id = 4; + drop2.physical_attempt_index = 2; + drop2.mode = PingFaultMode::kDropRequest; + PingTestFaults::Instance().Arm(drop1); + PingTestFaults::Instance().Arm(drop2); + auto a1 = PingTestFaults::Instance().Consume( + PingFaultContext{20, 4, 1, TimePoint{}, TimePoint{}}); + auto a2 = PingTestFaults::Instance().Consume( + PingFaultContext{20, 4, 2, TimePoint{}, TimePoint{}}); + auto a3 = PingTestFaults::Instance().Consume( + PingFaultContext{20, 4, 3, TimePoint{}, TimePoint{}}); + TEST_ASSERT_TRUE(a1.mode == PingFaultMode::kDropRequest); + TEST_ASSERT_TRUE(a2.mode == PingFaultMode::kDropRequest); + TEST_ASSERT_TRUE(a3.mode == PingFaultMode::kNone); + PingTestFaults::Instance().Clear(); +} +#endif + +} // namespace ae::test_uap_receive_schedule + +int test_uap_receive_schedule() { + UNITY_BEGIN(); + RUN_TEST(ae::test_uap_receive_schedule:: + test_ResponseTimeoutEmptyUsesInitialEstimate); + RUN_TEST(ae::test_uap_receive_schedule:: + test_ResponseTimeoutAfterFirstSampleUsesPercentile); + RUN_TEST(ae::test_uap_receive_schedule:: + test_NoSyntheticSeedMeansEmptyUntilRealRtt); + RUN_TEST(ae::test_uap_receive_schedule::test_PingDeadlineGuardFormulas); + RUN_TEST(ae::test_uap_receive_schedule:: + test_WireAnnouncedIntervalUnchangedVsLocalSendEarlier); + RUN_TEST(ae::test_uap_receive_schedule:: + test_RxCloseIsPongPlusWindowNotSendPlusWindow); + RUN_TEST(ae::test_uap_receive_schedule:: + test_ConversionUsesLibraryTimePointOnly); + RUN_TEST(ae::test_uap_receive_schedule::test_DeltaZeroYieldsUnknownDeadline); + RUN_TEST( + ae::test_uap_receive_schedule::test_NegativeDeltaYieldsMissedDeadline); + RUN_TEST( + ae::test_uap_receive_schedule::test_UapWireFieldOrderAndSignedInt64); + RUN_TEST(ae::test_uap_receive_schedule::test_BenchMessageCrcRoundTrip); + RUN_TEST(ae::test_uap_receive_schedule::test_ClassifyReceiveSendOffset); + RUN_TEST(ae::test_uap_receive_schedule::test_EarlyRxWindowComputation); + RUN_TEST( + ae::test_uap_receive_schedule::test_LocalRxWindowMonotonicAndStaleTimer); + RUN_TEST( + ae::test_uap_receive_schedule::test_ServerWindowInvariantIndependentOfRtt); + RUN_TEST( + ae::test_uap_receive_schedule::test_LogicalPingFirstAttemptAnchorsSchedule); + RUN_TEST(ae::test_uap_receive_schedule::test_LogicalPingOneRetryKeepsPhase); + RUN_TEST(ae::test_uap_receive_schedule::test_LogicalPingTwoRetriesKeepPhase); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingSuccessConfirmsOnceAndIgnoresStale); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingLateOldAttemptConfirmsActiveCycle); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingPreviousCyclePongIgnored); + RUN_TEST(ae::test_uap_receive_schedule::test_LogicalPingErrorRetryActions); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingEarlyAttemptDoesNotShiftPhase); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingSubsequentEarlyWireIsExtended); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingRetryKeepsNominalAndShrinksWire); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingLateRetryAdvancesWholeIntervals); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingHundredCyclesNoPhaseDrift); + RUN_TEST(ae::test_uap_receive_schedule::test_PingRetryBudgetEmptyStats); + RUN_TEST(ae::test_uap_receive_schedule::test_DefaultPingRetryCountIsZero); + RUN_TEST(ae::test_uap_receive_schedule::test_PingRetryBudgetN0LeadIsGuardPlusHalfP99); + RUN_TEST(ae::test_uap_receive_schedule::test_PingRetryBudgetN1PreDeadlineRetry); + RUN_TEST(ae::test_uap_receive_schedule::test_PingRetryBudgetN2PreDeadlineRetry); + RUN_TEST(ae::test_uap_receive_schedule::test_PreDeadlineRetryCountExhaustion); + RUN_TEST( + ae::test_uap_receive_schedule::test_PostDeadlineRetryAllowedAfterZeroPreDeadlineBudget); + RUN_TEST( + ae::test_uap_receive_schedule::test_PostDeadlineRecoveryKeepsLogicalCycleAndPhase); + RUN_TEST(ae::test_uap_receive_schedule::test_PingRetryBudgetRealMinP99); + RUN_TEST(ae::test_uap_receive_schedule:: + test_PingRetryBudgetOneSecondAllowsRetryWithP99_80); + RUN_TEST(ae::test_uap_receive_schedule:: + test_PingRetryBudgetShortIntervalSetsDiagnosticAndDoesNotWireZero); + RUN_TEST(ae::test_uap_receive_schedule::test_RxWindowUsesNominalPlusBaseWindow); + RUN_TEST(ae::test_uap_receive_schedule::test_AnnounceUnknownWiresZero); + RUN_TEST(ae::test_uap_receive_schedule:: + test_LogicalPingRetryAfterDeadlineKeepsPhaseAndNeverWiresZero); + RUN_TEST( + ae::test_uap_receive_schedule::test_WireRoundingFloorConnectCeilRxRemainders); + RUN_TEST(ae::test_uap_receive_schedule::test_LogicalPingSaturationAndOverflow); +#if AE_ENABLE_PING_TEST_FAULTS + RUN_TEST(ae::test_uap_receive_schedule:: + test_PingTestFaultsTargetExactServerCycleAttempt); + RUN_TEST(ae::test_uap_receive_schedule:: + test_PingTestFaultsBindNextCycleAndIgnoreResponse); + RUN_TEST(ae::test_uap_receive_schedule:: + test_PingTestFaultsConsecutiveAttempts); +#endif + return UNITY_END(); +}