From 193df4fd17b2f070a6480dcfbd1086e2d1e4cae7 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Fri, 14 Aug 2026 10:01:37 -0700 Subject: [PATCH] test(platform): raise FT recovery waits from 90s to 300s A force-stop/start cycle can reschedule the pipeline pod onto another Kubernetes node, forcing its storage volume to detach and reattach. Under attach/detach contention the reattach alone can exceed 90s, which intermittently timed out test_ft_input_replay_determinism even though replay determinism held (deployment_error stayed unset through every cycle). The determinism assertion runs on every 0.5s poll, so the longer wait adds infrastructure headroom without weakening the regression check. Signed-off-by: Gerd Zellweger --- python/tests/platform/test_fault_tolerance.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/tests/platform/test_fault_tolerance.py b/python/tests/platform/test_fault_tolerance.py index 398f6969715..e56a45437fb 100644 --- a/python/tests/platform/test_fault_tolerance.py +++ b/python/tests/platform/test_fault_tolerance.py @@ -79,6 +79,12 @@ # test pins 2 rather than using FELDERA_TEST_NUM_WORKERS. _WORKERS = 2 +# Waits for Running must ride out a restart that reschedules the pipeline pod +# onto another node, where the storage-volume reattach alone can take minutes +# under Kubernetes attach/detach contention. The determinism assertion runs on +# every poll, so the long timeout adds headroom without weakening the check. +_RUNNING_TIMEOUT_S = 300.0 + def _create_ft_pipeline(name: str, workers: int): """Create a single-host fault-tolerant pipeline (mirrors helper.create_pipeline @@ -116,7 +122,7 @@ def test_ft_input_replay_determinism(pipeline_name): wait_for_condition( f"running before crash (cycle {cycle})", lambda: _deployment(pipeline_name)[0] == "Running", - timeout_s=90.0, + timeout_s=_RUNNING_TIMEOUT_S, poll_interval_s=0.5, ) # Let datagen feed a partial, not-yet-checkpointed step into the log. @@ -138,6 +144,6 @@ def recovered(): wait_for_condition( f"recovered to Running after crash (cycle {cycle})", recovered, - timeout_s=90.0, + timeout_s=_RUNNING_TIMEOUT_S, poll_interval_s=0.5, )