From 3651e1e0c03c095571db76f7e338c349165c34b4 Mon Sep 17 00:00:00 2001 From: aipcc-bot Date: Wed, 22 Jul 2026 15:23:40 +0000 Subject: [PATCH 1/2] fix: Set FIPS cipher suites before pyarrow.flight import to prevent crash on IBM Power MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RHOAIENG-78595 # What this PR does / why we need it: The Feast offline container crashes in CrashLoopBackOff on FIPS-enabled OpenShift clusters running IBM Power (ppc64le). The existing FIPS cipher suite fix (RHOAIENG-70153) set GRPC_SSL_CIPHER_SUITES in `start_server()`, but pyarrow.flight was already imported at module level. On IBM Power, gRPC reads this env var during shared-library initialization, so the late configuration had no effect. This fix moves the FIPS cipher configuration to module level — before the pyarrow.flight import — so the env var is present when gRPC initializes its SSL context. The `_configure_grpc_fips()` call in `start_server()` is retained as a safety net. # Which issue(s) this PR fixes: Fixes RHOAIENG-78595 # Checks - [x] I've made sure the tests are passing. - [x] My PR title follows conventional commits format ## Testing Strategy - [x] Unit tests Assisted-by: Claude claude-opus-4-6 Signed-off-by: aipcc-bot Signed-off-by: Jitendra Yejare <11752425+jyejare@users.noreply.github.com> --- sdk/python/feast/offline_server.py | 79 ++++++++++++-------- sdk/python/tests/unit/test_offline_server.py | 19 +++++ 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index e130544445f..1a00735f305 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -7,37 +7,6 @@ from datetime import datetime from typing import Any, Dict, List, Optional, cast -import click -import pyarrow as pa -import pyarrow.flight as fl -from google.protobuf.json_format import Parse - -from feast import FeatureStore, FeatureView, utils -from feast.arrow_error_handler import arrow_server_error_handling_decorator -from feast.data_source import DataSource -from feast.errors import FeatureViewNotFoundException -from feast.feature_logging import FeatureServiceLoggingSource -from feast.feature_view import DUMMY_ENTITY_NAME -from feast.infra.offline_stores.offline_utils import get_offline_store_from_config -from feast.permissions.action import AuthzedAction -from feast.permissions.security_manager import assert_permissions -from feast.permissions.server.arrow import ( - AuthorizationMiddlewareFactory, - inject_user_details_decorator, -) -from feast.permissions.server.utils import ( - AuthManagerType, - ServerType, - init_auth_manager, - init_security_manager, - str_to_auth_manager_type, -) -from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto -from feast.saved_dataset import SavedDatasetStorage - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - _FIPS_CIPHER_SUITES = ":".join( [ "ECDHE-RSA-AES128-GCM-SHA256", @@ -55,14 +24,58 @@ def _is_fips_enabled() -> bool: with open("/proc/sys/crypto/fips_enabled") as f: return f.read().strip() == "1" except (FileNotFoundError, PermissionError, OSError): - logger.debug("Could not detect FIPS mode (Linux-only feature)") return False def _configure_grpc_fips() -> None: if _is_fips_enabled() and "GRPC_SSL_CIPHER_SUITES" not in os.environ: os.environ["GRPC_SSL_CIPHER_SUITES"] = _FIPS_CIPHER_SUITES - logger.info("FIPS mode detected, configured FIPS-compliant gRPC cipher suites.") + logging.getLogger(__name__).info( + "FIPS mode detected, configured FIPS-compliant gRPC cipher suites." + ) + + +# On FIPS-enabled systems (notably IBM Power ppc64le), gRPC reads +# GRPC_SSL_CIPHER_SUITES during shared-library initialization. The env var +# must be set before any gRPC-linked module (pyarrow.flight) is imported. +_configure_grpc_fips() + +import click # noqa: E402 +import pyarrow as pa # noqa: E402 +import pyarrow.flight as fl # noqa: E402 +from google.protobuf.json_format import Parse # noqa: E402 + +from feast import FeatureStore, FeatureView, utils # noqa: E402 +from feast.arrow_error_handler import ( # noqa: E402 + arrow_server_error_handling_decorator, +) +from feast.data_source import DataSource # noqa: E402 +from feast.errors import FeatureViewNotFoundException # noqa: E402 +from feast.feature_logging import FeatureServiceLoggingSource # noqa: E402 +from feast.feature_view import DUMMY_ENTITY_NAME # noqa: E402 +from feast.infra.offline_stores.offline_utils import ( # noqa: E402 + get_offline_store_from_config, +) +from feast.permissions.action import AuthzedAction # noqa: E402 +from feast.permissions.security_manager import assert_permissions # noqa: E402 +from feast.permissions.server.arrow import ( # noqa: E402 + AuthorizationMiddlewareFactory, + inject_user_details_decorator, +) +from feast.permissions.server.utils import ( # noqa: E402 + AuthManagerType, + ServerType, + init_auth_manager, + init_security_manager, + str_to_auth_manager_type, +) +from feast.protos.feast.core.DataSource_pb2 import ( # noqa: E402 + DataSource as DataSourceProto, +) +from feast.saved_dataset import SavedDatasetStorage # noqa: E402 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) class OfflineServer(fl.FlightServerBase): diff --git a/sdk/python/tests/unit/test_offline_server.py b/sdk/python/tests/unit/test_offline_server.py index 6ff93d02282..78982b6b93b 100644 --- a/sdk/python/tests/unit/test_offline_server.py +++ b/sdk/python/tests/unit/test_offline_server.py @@ -1,3 +1,4 @@ +import importlib import os from unittest.mock import MagicMock, mock_open, patch @@ -137,3 +138,21 @@ def test_configure_grpc_fips_noop_without_fips(): os.environ.pop("GRPC_SSL_CIPHER_SUITES", None) _configure_grpc_fips() assert "GRPC_SSL_CIPHER_SUITES" not in os.environ + + +def test_module_level_fips_sets_env_before_pyarrow_import(): + """GRPC_SSL_CIPHER_SUITES must be set at module load time, + before pyarrow.flight (which bundles gRPC) is imported.""" + env_backup = os.environ.pop("GRPC_SSL_CIPHER_SUITES", None) + try: + with patch("builtins.open", mock_open(read_data="1\n")): + import feast.offline_server as mod + + importlib.reload(mod) + assert "GRPC_SSL_CIPHER_SUITES" in os.environ + assert "AES128-GCM-SHA256" in os.environ["GRPC_SSL_CIPHER_SUITES"] + finally: + if env_backup is not None: + os.environ["GRPC_SSL_CIPHER_SUITES"] = env_backup + else: + os.environ.pop("GRPC_SSL_CIPHER_SUITES", None) From bf0a4714fd1c8470a8c6be85a5b1dde02249ffa9 Mon Sep 17 00:00:00 2001 From: aipcc-bot Date: Wed, 22 Jul 2026 16:49:00 +0000 Subject: [PATCH 2/2] fix: Address review feedback on FIPS cipher suite configuration # What this PR does / why we need it: Move FIPS log message below logger initialization so it is emitted at INFO level instead of being silently dropped under the default WARNING threshold. Replace importlib.reload-based import ordering test with a subprocess-based test that exercises fresh Python import from scratch, ensuring pyarrow.flight is not cached in sys.modules and the GRPC_SSL_CIPHER_SUITES ordering check is genuine. # Which issue(s) this PR fixes: Fixes RHOAIENG-78595 # Checks - [x] I've made sure the tests are passing. - [x] My PR title follows conventional commits format ## Testing Strategy - [x] Unit tests Assisted-by: Claude claude-opus-4-6 Signed-off-by: Jitendra Yejare <11752425+jyejare@users.noreply.github.com> --- sdk/python/feast/offline_server.py | 12 ++-- sdk/python/tests/unit/test_offline_server.py | 62 ++++++++++++++++---- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 1a00735f305..e82b5239767 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -27,18 +27,17 @@ def _is_fips_enabled() -> bool: return False -def _configure_grpc_fips() -> None: +def _configure_grpc_fips() -> bool: if _is_fips_enabled() and "GRPC_SSL_CIPHER_SUITES" not in os.environ: os.environ["GRPC_SSL_CIPHER_SUITES"] = _FIPS_CIPHER_SUITES - logging.getLogger(__name__).info( - "FIPS mode detected, configured FIPS-compliant gRPC cipher suites." - ) + return True + return False # On FIPS-enabled systems (notably IBM Power ppc64le), gRPC reads # GRPC_SSL_CIPHER_SUITES during shared-library initialization. The env var # must be set before any gRPC-linked module (pyarrow.flight) is imported. -_configure_grpc_fips() +_fips_configured = _configure_grpc_fips() import click # noqa: E402 import pyarrow as pa # noqa: E402 @@ -77,6 +76,9 @@ def _configure_grpc_fips() -> None: logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +if _fips_configured: + logger.info("FIPS mode detected, configured FIPS-compliant gRPC cipher suites.") + class OfflineServer(fl.FlightServerBase): def __init__( diff --git a/sdk/python/tests/unit/test_offline_server.py b/sdk/python/tests/unit/test_offline_server.py index 78982b6b93b..fa1bf85e97e 100644 --- a/sdk/python/tests/unit/test_offline_server.py +++ b/sdk/python/tests/unit/test_offline_server.py @@ -1,5 +1,7 @@ -import importlib import os +import subprocess +import sys +import textwrap from unittest.mock import MagicMock, mock_open, patch import assertpy @@ -142,17 +144,51 @@ def test_configure_grpc_fips_noop_without_fips(): def test_module_level_fips_sets_env_before_pyarrow_import(): """GRPC_SSL_CIPHER_SUITES must be set at module load time, - before pyarrow.flight (which bundles gRPC) is imported.""" - env_backup = os.environ.pop("GRPC_SSL_CIPHER_SUITES", None) - try: - with patch("builtins.open", mock_open(read_data="1\n")): - import feast.offline_server as mod - - importlib.reload(mod) + before pyarrow.flight (which bundles gRPC) is imported. + + Uses a subprocess so pyarrow.flight is not already cached in + sys.modules, which lets us verify the true import ordering. + """ + script = textwrap.dedent("""\ + import io, os, sys + + # Intercept only /proc/sys/crypto/fips_enabled to simulate FIPS + _real_open = open + def _fips_open(file, *args, **kwargs): + if str(file) == "/proc/sys/crypto/fips_enabled": + return io.StringIO("1\\n") + return _real_open(file, *args, **kwargs) + + import builtins + builtins.open = _fips_open + + # Track import order to verify env var is set before pyarrow.flight + original_import = builtins.__import__ + def tracking_import(name, *args, **kwargs): + if name == "pyarrow.flight": + assert "GRPC_SSL_CIPHER_SUITES" in os.environ, ( + "GRPC_SSL_CIPHER_SUITES not set before pyarrow.flight import" + ) + return original_import(name, *args, **kwargs) + + builtins.__import__ = tracking_import + try: + import feast.offline_server assert "GRPC_SSL_CIPHER_SUITES" in os.environ assert "AES128-GCM-SHA256" in os.environ["GRPC_SSL_CIPHER_SUITES"] - finally: - if env_backup is not None: - os.environ["GRPC_SSL_CIPHER_SUITES"] = env_backup - else: - os.environ.pop("GRPC_SSL_CIPHER_SUITES", None) + finally: + builtins.__import__ = original_import + builtins.open = _real_open + """) + env = os.environ.copy() + env.pop("GRPC_SSL_CIPHER_SUITES", None) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=env, + timeout=60, + ) + assert result.returncode == 0, ( + f"Subprocess failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + )