From d4dec898c92d6bb006b5c08bfecfa914164575f1 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Wed, 5 Aug 2026 09:28:06 -0700 Subject: [PATCH] [python] Retry 503s minting GitHub Actions OIDC tokens. _github_oidc_token() sent a single unretried request to GitHub's own token-minting endpoint. A transient 503 there (seen in a feldera-qa run) killed the whole test/workload run before it ever got a pipeline handle, which also meant no support bundle existed to explain the failure. Retry the mint on 429/502/503/504 and connection errors with exponential backoff, matching how FelderaClient already treats these as transient. Signed-off-by: Ben Pfaff --- python/feldera/testutils.py | 33 +++++- .../unit/test_github_oidc_token_retry.py | 105 ++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 python/tests/unit/test_github_oidc_token_retry.py diff --git a/python/feldera/testutils.py b/python/feldera/testutils.py index b53a044a1ff..1d4b56593ea 100644 --- a/python/feldera/testutils.py +++ b/python/feldera/testutils.py @@ -4,6 +4,7 @@ import logging import math import os +import urllib.error import urllib.parse import urllib.request import platform @@ -54,6 +55,35 @@ def _get_effective_api_key(): # just before the check cannot arrive after it. _OIDC_REFRESH_MARGIN_SECONDS = 120.0 +# GitHub's own token endpoint occasionally 503s or drops the connection; these +# are as transient as the pipeline-side errors FelderaClient already retries, +# so retry the same way rather than letting one blip fail the whole run. +_OIDC_MINT_RETRYABLE_STATUS_CODES = frozenset({429, 502, 503, 504}) +_OIDC_MINT_MAX_RETRIES = 3 +_OIDC_MINT_INITIAL_BACKOFF_SECONDS = 1.0 +_OIDC_MINT_BACKOFF_MULTIPLIER = 2.0 + + +def _mint_github_oidc_token(request: urllib.request.Request) -> str: + """Issue the token-mint request, retrying transient failures.""" + backoff = _OIDC_MINT_INITIAL_BACKOFF_SECONDS + for attempt in range(_OIDC_MINT_MAX_RETRIES + 1): + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response)["value"] + except urllib.error.HTTPError as e: + if ( + e.code not in _OIDC_MINT_RETRYABLE_STATUS_CODES + or attempt == _OIDC_MINT_MAX_RETRIES + ): + raise + except urllib.error.URLError: + if attempt == _OIDC_MINT_MAX_RETRIES: + raise + time.sleep(backoff) + backoff *= _OIDC_MINT_BACKOFF_MULTIPLIER + raise AssertionError("unreachable") # loop always returns or raises + def _github_oidc_token() -> str: """A GitHub Actions ID token, re-minted shortly before it expires. @@ -75,8 +105,7 @@ def _github_oidc_token() -> str: request.add_header( "Authorization", f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}" ) - with urllib.request.urlopen(request, timeout=30) as response: - token = json.load(response)["value"] + token = _mint_github_oidc_token(request) _oidc_token_cache[audience] = ( token, diff --git a/python/tests/unit/test_github_oidc_token_retry.py b/python/tests/unit/test_github_oidc_token_retry.py new file mode 100644 index 00000000000..ccb9027b4c6 --- /dev/null +++ b/python/tests/unit/test_github_oidc_token_retry.py @@ -0,0 +1,105 @@ +"""Tests for the retry behavior when minting a GitHub Actions OIDC token.""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from unittest import mock + +import pytest + +from feldera.testutils import _OIDC_MINT_MAX_RETRIES, _mint_github_oidc_token + + +class _FakeResponse: + def __init__(self, value: str): + self._body = json.dumps({"value": value}).encode() + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self, *args, **kwargs): + return self._body + + +def _http_error(request: urllib.request.Request, code: int) -> urllib.error.HTTPError: + return urllib.error.HTTPError(request.full_url, code, "error", {}, None) + + +def _request() -> urllib.request.Request: + return urllib.request.Request("https://example.test/token") + + +class TestOidcMintRetry: + def test_retries_on_503_then_succeeds(self): + calls = [] + + def fake_urlopen(request, timeout): + calls.append(request) + if len(calls) == 1: + raise _http_error(request, 503) + return _FakeResponse("minted-token") + + with ( + mock.patch("urllib.request.urlopen", side_effect=fake_urlopen), + mock.patch("time.sleep"), + ): + token = _mint_github_oidc_token(_request()) + + assert token == "minted-token" + assert len(calls) == 2 + + def test_does_not_retry_non_transient_status(self): + calls = [] + + def fake_urlopen(request, timeout): + calls.append(request) + raise _http_error(request, 401) + + with ( + mock.patch("urllib.request.urlopen", side_effect=fake_urlopen), + mock.patch("time.sleep"), + ): + with pytest.raises(urllib.error.HTTPError) as exc_info: + _mint_github_oidc_token(_request()) + + assert exc_info.value.code == 401 + assert len(calls) == 1 + + def test_exhausts_retries_and_raises(self): + calls = [] + + def fake_urlopen(request, timeout): + calls.append(request) + raise _http_error(request, 503) + + with ( + mock.patch("urllib.request.urlopen", side_effect=fake_urlopen), + mock.patch("time.sleep"), + ): + with pytest.raises(urllib.error.HTTPError): + _mint_github_oidc_token(_request()) + + assert len(calls) == _OIDC_MINT_MAX_RETRIES + 1 + + def test_retries_on_connection_error(self): + calls = [] + + def fake_urlopen(request, timeout): + calls.append(request) + if len(calls) == 1: + raise urllib.error.URLError("connection reset") + return _FakeResponse("minted-token") + + with ( + mock.patch("urllib.request.urlopen", side_effect=fake_urlopen), + mock.patch("time.sleep"), + ): + token = _mint_github_oidc_token(_request()) + + assert token == "minted-token" + assert len(calls) == 2