From d99adc2eebe1a482e5263e367f24636718d68fee Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 10 Jul 2026 09:02:38 -0400 Subject: [PATCH] fix: Make pytest config compatible with newer pytest Signed-off-by: Francisco Javier Arceo --- .secrets.baseline | 6 +- .../api/v1/featurestore_types.go | 3 +- .../api/v1alpha1/featurestore_types.go | 3 +- .../crd/bases/feast.dev_featurestores.yaml | 4 + infra/feast-operator/dist/install.yaml | 4 + pixi.lock | 7 +- .../feast/infra/online_stores/sqlite.py | 22 ++- sdk/python/feast/infra/registry/registry.py | 6 + sdk/python/pytest.ini | 4 +- sdk/python/tests/conftest.py | 150 ++++++++++++++---- .../local_feast_tests/test_e2e_local.py | 16 +- .../local_feast_tests/test_feature_service.py | 6 +- sdk/python/tests/utils/cli_repo_creator.py | 13 +- 13 files changed, 187 insertions(+), 57 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 0eefabade70..25c2be3525f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -957,7 +957,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 936 + "line_number": 937 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -989,7 +989,7 @@ "filename": "infra/feast-operator/api/v1alpha1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 650 + "line_number": 651 } ], "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go": [ @@ -1555,5 +1555,5 @@ } ] }, - "generated_at": "2026-06-26T06:19:05Z" + "generated_at": "2026-07-10T13:52:47Z" } diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index f0331ecb1b4..271ae1fe6b1 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -626,7 +626,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -653,6 +653,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "hybrid", "mongodb", "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index c165801eda7..11e201dd0d6 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -373,7 +373,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -400,6 +400,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "hybrid", "mongodb", "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 11650995e04..b840d46a3b1 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -2400,6 +2400,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -8706,6 +8707,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -14230,6 +14232,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -18736,6 +18739,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 57853e4b83f..44aab6c8af5 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -2408,6 +2408,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -8714,6 +8715,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -14238,6 +14240,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -18744,6 +18747,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef diff --git a/pixi.lock b/pixi.lock index 4a7fdda1ef8..596718ef1e6 100644 --- a/pixi.lock +++ b/pixi.lock @@ -2421,8 +2421,8 @@ packages: requires_python: '>=3.10' - pypi: ./ name: feast - version: 0.64.1.dev48+g10a1fb32b.d20260630 - sha256: 36b2ec53e444a094df5ce68594025cea29abcb5e164586f145d6dd7e5f92c1ff + version: 0.64.1.dev33+gdddcb00a4 + sha256: b96f4ccddcdc82315a4ca26d2014829d4e2ed40c7c93f2f86005e680f3418855 requires_dist: - click>=7.0.0,<9.0.0 - colorama>=0.3.9,<1 @@ -2523,6 +2523,7 @@ packages: - datasets>=3.6.0 ; extra == 'ray' - redis>=4.2.2,<8 ; extra == 'redis' - hiredis>=2.0.0,<4 ; extra == 'redis' + - scylla-driver>=3.28.0,<4 ; extra == 'scylladb' - singlestoredb<1.8.0 ; extra == 'singlestore' - snowflake-connector-python[pandas]>=3.7,<5 ; extra == 'snowflake' - sqlite-vec==0.1.6 ; extra == 'sqlite-vec' @@ -2543,7 +2544,7 @@ packages: - minio==7.2.11 ; extra == 'test' - python-keycloak==4.2.2 ; extra == 'test' - cryptography>=43.0 ; extra == 'test' - - feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,go,grpcio,hazelcast,hbase,ibis,image,k8s,mcp,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,singlestore,snowflake,spark,sqlite-vec,test,trino] ; extra == 'ci' + - feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,go,grpcio,hazelcast,hbase,ibis,image,k8s,mcp,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,scylladb,singlestore,snowflake,spark,sqlite-vec,test,trino] ; extra == 'ci' - build ; extra == 'ci' - virtualenv==20.23.0 ; extra == 'ci' - dbt-artifacts-parser ; extra == 'ci' diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 6bba852f0b8..74ddd030d57 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -16,6 +16,7 @@ import os import sqlite3 import sys +import time from datetime import date, datetime, timezone from pathlib import Path from typing import ( @@ -353,10 +354,23 @@ def teardown( tables: Sequence[FeatureView], entities: Sequence[Entity], ): - try: - os.unlink(self._get_db_path(config)) - except FileNotFoundError: - pass + if self._conn is not None: + try: + self._conn.close() + finally: + self._conn = None + + db_path = self._get_db_path(config) + for attempt in range(10): + try: + os.unlink(db_path) + return + except FileNotFoundError: + return + except PermissionError: + if attempt == 9: + raise + time.sleep(0.25) def retrieve_online_documents( self, diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 222777b325e..f09f05e971f 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -155,6 +155,12 @@ def get_registry_store_class_from_type(registry_store_type: str): def get_registry_store_class_from_scheme(registry_path: str): uri = urlparse(registry_path) + if uri.scheme == "" or ( + len(uri.scheme) == 1 and registry_path[1:3] in (":\\", ":/") + ): + registry_store_type = REGISTRY_STORE_CLASS_FOR_SCHEME["file"] + return get_registry_store_class_from_type(registry_store_type) + if uri.scheme not in REGISTRY_STORE_CLASS_FOR_SCHEME: raise Exception( f"Registry path {registry_path} has unsupported scheme {uri.scheme}. " diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index d5ad19660b7..837c2112be2 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -3,8 +3,8 @@ asyncio_mode = auto env = IS_TEST=True filterwarnings = - error::_pytest.warning_types.PytestConfigWarning - error::_pytest.warning_types.PytestUnhandledCoroutineWarning + error::pytest.PytestConfigWarning + error:.*was never awaited.*:RuntimeWarning ignore::DeprecationWarning:pyspark.sql.pandas.*: ignore::DeprecationWarning:pyspark.sql.connect.*: ignore::DeprecationWarning:httpx.*: diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 8302e313a2d..83c3a2778e6 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import importlib import logging import multiprocessing import os @@ -20,7 +21,7 @@ from multiprocessing import Process from sys import platform from textwrap import dedent -from typing import Any, Dict, List, Tuple, no_type_check +from typing import Any, Dict, List, Optional, Tuple, no_type_check from unittest import mock import pandas as pd @@ -36,34 +37,90 @@ create_document_dataset, create_image_dataset, ) -from tests.universal.feature_repos.integration_test_repo_config import ( # noqa: E402 - IntegrationTestRepoConfig, -) -from tests.universal.feature_repos.repo_configuration import ( # noqa: E402 - AVAILABLE_OFFLINE_STORES, - AVAILABLE_ONLINE_STORES, - OFFLINE_STORE_TO_PROVIDER_CONFIG, - Environment, - TestData, - construct_test_environment, - construct_universal_feature_views, - construct_universal_test_data, -) -from tests.universal.feature_repos.universal.data_sources.file import ( # noqa: E402 - FileDataSourceCreator, -) -from tests.universal.feature_repos.universal.entities import ( # noqa: E402 - customer, - driver, - location, -) -from tests.utils.auth_permissions_util import default_store from tests.utils.http_server import check_port_open, free_port # noqa: E402 -from tests.utils.ssl_certifcates_util import ( - combine_trust_stores, - create_ca_trust_store, - generate_self_signed_cert, -) + +IntegrationTestRepoConfig: Any = None +Environment = Any +TestData = Any +AVAILABLE_OFFLINE_STORES: Any = None +AVAILABLE_ONLINE_STORES: Any = None +OFFLINE_STORE_TO_PROVIDER_CONFIG: Any = None +construct_test_environment: Any = None +construct_universal_feature_views: Any = None +construct_universal_test_data: Any = None +FileDataSourceCreator: Any = None +customer: Any = None +driver: Any = None +location: Any = None +_universal_deps_missing_reason: Optional[str] = None + + +def _load_universal_feature_repo_deps() -> bool: + global IntegrationTestRepoConfig + global Environment + global TestData + global AVAILABLE_OFFLINE_STORES + global AVAILABLE_ONLINE_STORES + global OFFLINE_STORE_TO_PROVIDER_CONFIG + global construct_test_environment + global construct_universal_feature_views + global construct_universal_test_data + global FileDataSourceCreator + global customer + global driver + global location + global _universal_deps_missing_reason + + if IntegrationTestRepoConfig is not None: + return True + + try: + integration_config = importlib.import_module( + "tests.universal.feature_repos.integration_test_repo_config" + ) + repo_configuration = importlib.import_module( + "tests.universal.feature_repos.repo_configuration" + ) + file_data_sources = importlib.import_module( + "tests.universal.feature_repos.universal.data_sources.file" + ) + entities = importlib.import_module( + "tests.universal.feature_repos.universal.entities" + ) + except ModuleNotFoundError as e: + _universal_deps_missing_reason = ( + f"Optional integration test dependency is not installed: {e.name}" + ) + return False + + IntegrationTestRepoConfig = integration_config.IntegrationTestRepoConfig + Environment = repo_configuration.Environment + TestData = repo_configuration.TestData + AVAILABLE_OFFLINE_STORES = repo_configuration.AVAILABLE_OFFLINE_STORES + AVAILABLE_ONLINE_STORES = repo_configuration.AVAILABLE_ONLINE_STORES + OFFLINE_STORE_TO_PROVIDER_CONFIG = ( + repo_configuration.OFFLINE_STORE_TO_PROVIDER_CONFIG + ) + construct_test_environment = repo_configuration.construct_test_environment + construct_universal_feature_views = ( + repo_configuration.construct_universal_feature_views + ) + construct_universal_test_data = repo_configuration.construct_universal_test_data + FileDataSourceCreator = file_data_sources.FileDataSourceCreator + customer = entities.customer + driver = entities.driver + location = entities.location + _universal_deps_missing_reason = None + return True + + +def _skip_missing_universal_feature_repo_deps() -> None: + if not _load_universal_feature_repo_deps(): + pytest.skip( + _universal_deps_missing_reason + or "Optional integration test dependencies are not installed" + ) + logger = logging.getLogger(__name__) @@ -85,7 +142,7 @@ def pytest_configure(config): - if platform in ["darwin", "windows"]: + if platform == "darwin" or platform.startswith("win"): multiprocessing.set_start_method("spawn", force=True) else: multiprocessing.set_start_method("fork") @@ -192,6 +249,7 @@ def start_test_local_server(repo_path: str, port: int): @pytest.fixture def environment(request, worker_id): + _skip_missing_universal_feature_repo_deps() e = construct_test_environment( request.param, worker_id=worker_id, @@ -211,6 +269,7 @@ def environment(request, worker_id): @pytest.fixture def vectordb_environment(request, worker_id): + _skip_missing_universal_feature_repo_deps() e = construct_test_environment( request.param, worker_id=worker_id, @@ -251,6 +310,23 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): parameter should point to the same Python object (hence, we use _config_cache dict to store those objects). """ if "environment" in metafunc.fixturenames: + if not _load_universal_feature_repo_deps(): + metafunc.parametrize( + "environment", + [ + pytest.param( + None, + marks=pytest.mark.skip( + reason=_universal_deps_missing_reason + or "Optional integration test dependencies are not installed" + ), + ) + ], + indirect=True, + ids=["missing_optional_integration_deps"], + ) + return + markers = {m.name: m for m in metafunc.definition.own_markers} offline_stores = None if "universal_offline_stores" in markers: @@ -371,6 +447,7 @@ def feature_server_endpoint(environment): @pytest.fixture def universal_data_sources(environment) -> TestData: + _skip_missing_universal_feature_repo_deps() return construct_universal_test_data(environment) @@ -394,6 +471,7 @@ def feature_store_for_online_retrieval( Returns a feature store that is ready for online retrieval, along with entity rows and feature refs that can be used to query for online features. """ + _skip_missing_universal_feature_repo_deps() fs = environment.feature_store entities, datasets, data_sources = universal_data_sources feature_views = construct_universal_feature_views(data_sources) @@ -478,6 +556,11 @@ def server_port(): @pytest.fixture def feature_store(temp_dir, auth_config, applied_permissions): + try: + from tests.utils.auth_permissions_util import default_store + except ModuleNotFoundError as e: + pytest.skip(f"Optional auth test dependency is not installed: {e.name}") + print(f"Creating store at {temp_dir}") return default_store(str(temp_dir), auth_config, applied_permissions) @@ -542,6 +625,15 @@ def auth_config(request, is_integration_test): @pytest.fixture(scope="module") def tls_mode(request): + try: + from tests.utils.ssl_certifcates_util import ( + combine_trust_stores, + create_ca_trust_store, + generate_self_signed_cert, + ) + except ModuleNotFoundError as e: + pytest.skip(f"Optional TLS test dependency is not installed: {e.name}") + is_tls_mode = request.param[0] output_combined_truststore_path = "" diff --git a/sdk/python/tests/integration/local_feast_tests/test_e2e_local.py b/sdk/python/tests/integration/local_feast_tests/test_e2e_local.py index cee295775b1..878b6336aa0 100644 --- a/sdk/python/tests/integration/local_feast_tests/test_e2e_local.py +++ b/sdk/python/tests/integration/local_feast_tests/test_e2e_local.py @@ -45,15 +45,17 @@ def test_e2e_local() -> None: driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + driver_stats_path_posix = Path(driver_stats_path).as_posix() global_df = create_global_daily_stats_df(start_date, end_date) global_stats_path = os.path.join(data_dir, "global_stats.parquet") global_df.to_parquet(path=global_stats_path, allow_truncated_timestamps=True) + global_stats_path_posix = Path(global_stats_path).as_posix() with runner.local_repo( get_example_repo("example_feature_repo_2.py") - .replace("%PARQUET_PATH%", driver_stats_path) - .replace("%PARQUET_PATH_GLOBAL%", global_stats_path), + .replace("%PARQUET_PATH%", driver_stats_path_posix) + .replace("%PARQUET_PATH_GLOBAL%", global_stats_path_posix), "file", ) as store: _test_materialize_and_online_retrieval( @@ -62,8 +64,8 @@ def test_e2e_local() -> None: with runner.local_repo( get_example_repo("example_feature_repo_with_bfvs.py") - .replace("%PARQUET_PATH%", driver_stats_path) - .replace("%PARQUET_PATH_GLOBAL%", global_stats_path), + .replace("%PARQUET_PATH%", driver_stats_path_posix) + .replace("%PARQUET_PATH_GLOBAL%", global_stats_path_posix), "file", ) as store: _test_materialize_and_online_retrieval( @@ -72,8 +74,8 @@ def test_e2e_local() -> None: with runner.local_repo( get_example_repo("example_feature_repo_with_ttl_0.py") - .replace("%PARQUET_PATH%", driver_stats_path) - .replace("%PARQUET_PATH_GLOBAL%", global_stats_path), + .replace("%PARQUET_PATH%", driver_stats_path_posix) + .replace("%PARQUET_PATH_GLOBAL%", global_stats_path_posix), "file", ) as store: _test_materialize_and_online_retrieval( @@ -83,7 +85,7 @@ def test_e2e_local() -> None: # Test a failure case when the parquet file doesn't include a join key with runner.local_repo( get_example_repo("example_feature_repo_with_entity_join_key.py").replace( - "%PARQUET_PATH%", driver_stats_path + "%PARQUET_PATH%", driver_stats_path_posix ), "file", ) as store: diff --git a/sdk/python/tests/unit/local_feast_tests/test_feature_service.py b/sdk/python/tests/unit/local_feast_tests/test_feature_service.py index c331f2913de..400d7ae86b2 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_feature_service.py +++ b/sdk/python/tests/unit/local_feast_tests/test_feature_service.py @@ -56,15 +56,17 @@ def test_apply_with_fv_inference() -> None: driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + driver_stats_path_posix = driver_stats_path.replace("\\", "/") global_df = create_global_daily_stats_df(start_date, end_date) global_stats_path = os.path.join(data_dir, "global_stats.parquet") global_df.to_parquet(path=global_stats_path, allow_truncated_timestamps=True) + global_stats_path_posix = global_stats_path.replace("\\", "/") with runner.local_repo( get_example_repo("example_feature_repo_with_feature_service_3.py") - .replace("%PARQUET_PATH%", driver_stats_path) - .replace("%PARQUET_PATH_GLOBAL%", global_stats_path), + .replace("%PARQUET_PATH%", driver_stats_path_posix) + .replace("%PARQUET_PATH_GLOBAL%", global_stats_path_posix), "file", ) as store: assert len(store.list_feature_services()) == 2 diff --git a/sdk/python/tests/utils/cli_repo_creator.py b/sdk/python/tests/utils/cli_repo_creator.py index 03c0976e880..678973b9539 100644 --- a/sdk/python/tests/utils/cli_repo_creator.py +++ b/sdk/python/tests/utils/cli_repo_creator.py @@ -142,15 +142,18 @@ def local_repo( repo_path = Path(repo_dir_name) data_path = Path(data_dir_name) + registry_path_yaml = str(data_path / "registry.db") + online_store_path_yaml = str(data_path / "online_store.db") + repo_config = repo_path / "feature_store.yaml" if online_store == "sqlite": yaml_config = dedent( f""" project: {project_id} - registry: {data_path / "registry.db"} + registry: {registry_path_yaml} provider: local online_store: - path: {data_path / "online_store.db"} + path: {online_store_path_yaml} offline_store: type: {offline_store} entity_key_serialization_version: 3 @@ -160,10 +163,10 @@ def local_repo( yaml_config = dedent( f""" project: {project_id} - registry: {data_path / "registry.db"} + registry: {registry_path_yaml} provider: local online_store: - path: {data_path / "online_store.db"} + path: {online_store_path_yaml} type: milvus vector_enabled: true embedding_dim: 10 @@ -176,7 +179,7 @@ def local_repo( yaml_config = dedent( f""" project: {project_id} - registry: {data_path / "registry.db"} + registry: {registry_path_yaml} provider: local online_store: type: {online_store}