From 3f565357db58a22d38e2c5a00f8f49f7210b0128 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 13:29:29 -0700 Subject: [PATCH 1/7] fix: Prefer installing gopy from feast's fork as opposed to upstream Signed-off-by: Achal Shah --- Makefile | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a715cb2c4a8..f190bdd4a32 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,9 @@ install-go-proto-dependencies: install-go-ci-dependencies: # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line # ToDo: there should be a better way to install gopy - go get github.com/go-python/gopy + go get github.com/feast-dev/gopy go install golang.org/x/tools/cmd/goimports - go install github.com/go-python/gopy + go install github.com/feast-dev/gopy python -m pip install pybindgen==0.22.0 install-protoc-dependencies: diff --git a/setup.py b/setup.py index 9e61eff8817..4f10a8ba1d0 100644 --- a/setup.py +++ b/setup.py @@ -424,7 +424,7 @@ def build_extension(self, ext: Extension): destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"]) - subprocess.check_call(["go", "install", "github.com/go-python/gopy"]) + subprocess.check_call(["go", "install", "github.com/feast-dev/gopy"]) subprocess.check_call( [ "gopy", From c37c57658f90f79fe17dbeb107720b032f4cfeb7 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 14:06:11 -0700 Subject: [PATCH 2/7] use go mod tidy Signed-off-by: Achal Shah --- Makefile | 4 +--- go.mod | 2 -- go.sum | 2 -- setup.py | 3 +-- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index f190bdd4a32..6b2102fe663 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,7 @@ install-go-proto-dependencies: install-go-ci-dependencies: # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line # ToDo: there should be a better way to install gopy - go get github.com/feast-dev/gopy - go install golang.org/x/tools/cmd/goimports - go install github.com/feast-dev/gopy + go mod tidy python -m pip install pybindgen==0.22.0 install-protoc-dependencies: diff --git a/go.mod b/go.mod index f983dae3442..8eac9e7db7e 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/gonuts/commander v0.1.0 // indirect - github.com/gonuts/flag v0.1.0 // indirect github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.1 // indirect diff --git a/go.sum b/go.sum index e1f2436b296..9a72f839305 100644 --- a/go.sum +++ b/go.sum @@ -146,9 +146,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= -github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= diff --git a/setup.py b/setup.py index 4f10a8ba1d0..6c67cbd20bc 100644 --- a/setup.py +++ b/setup.py @@ -423,8 +423,7 @@ def build_extension(self, ext: Extension): ) destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"]) - subprocess.check_call(["go", "install", "github.com/feast-dev/gopy"]) + subprocess.check_call(["go", "mod", "tidy"]) subprocess.check_call( [ "gopy", From a5445aa849ac9a22b22607d5f96e0be2fc0e2eae Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 14:21:49 -0700 Subject: [PATCH 3/7] pin and pray Signed-off-by: Achal Shah --- Makefile | 4 +++- go.mod | 6 +++--- go.sum | 4 ++-- setup.py | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 6b2102fe663..5d3ed817ea9 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,9 @@ install-go-proto-dependencies: install-go-ci-dependencies: # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line # ToDo: there should be a better way to install gopy - go mod tidy + go get github.com/feast-dev/gopy + go install golang.org/x/tools/cmd/goimports + go install github.com/gofeast-dev/gopy@v0.4.0 python -m pip install pybindgen==0.22.0 install-protoc-dependencies: diff --git a/go.mod b/go.mod index 8eac9e7db7e..a91cc3d5e6a 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/feast-dev/feast go 1.17 +replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 + require ( github.com/apache/arrow/go/v8 v8.0.0-20220408212425-58fe60f59289 github.com/ghodss/yaml v1.0.0 @@ -40,12 +42,10 @@ require ( golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.10 // indirect + golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) - -replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 diff --git a/go.sum b/go.sum index 9a72f839305..e73c4a91e8f 100644 --- a/go.sum +++ b/go.sum @@ -546,8 +546,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1 h1:Z3vE1sGlC7qiyFJkkDcZms8Y3+yV8+W7HmDSmuf71tM= +golang.org/x/tools v0.1.11-0.20220413170336-afc6aad76eb1/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/setup.py b/setup.py index 6c67cbd20bc..f5f6f9f920b 100644 --- a/setup.py +++ b/setup.py @@ -423,7 +423,8 @@ def build_extension(self, ext: Extension): ) destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call(["go", "mod", "tidy"]) + subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"]) + subprocess.check_call(["go", "install", "github.com/go-python/gopy@v0.4.0"]) subprocess.check_call( [ "gopy", From 929891ddd5710ae4ad3e8e80fabbe58acc78ba74 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 14:53:35 -0700 Subject: [PATCH 4/7] fix import path Signed-off-by: Achal Shah --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5d3ed817ea9..46ad6b8cae9 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,9 @@ install-go-proto-dependencies: install-go-ci-dependencies: # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line # ToDo: there should be a better way to install gopy - go get github.com/feast-dev/gopy + go get github.com/go-python/gopy@v0.4.0 go install golang.org/x/tools/cmd/goimports - go install github.com/gofeast-dev/gopy@v0.4.0 + go install github.com/go-python/gopy@v0.4.0 python -m pip install pybindgen==0.22.0 install-protoc-dependencies: From 13aa9ba480e18faf47cd60c58f42d93d2fe45fd0 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 15:20:17 -0700 Subject: [PATCH 5/7] nix go install Signed-off-by: Achal Shah --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 46ad6b8cae9..603653786db 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ install-go-ci-dependencies: # ToDo: there should be a better way to install gopy go get github.com/go-python/gopy@v0.4.0 go install golang.org/x/tools/cmd/goimports - go install github.com/go-python/gopy@v0.4.0 + #go install github.com/go-python/gopy@v0.4.0 python -m pip install pybindgen==0.22.0 install-protoc-dependencies: From 5b107d6a9bc217c4743bb0606e35ec005ba44e01 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 16:14:24 -0700 Subject: [PATCH 6/7] pin go get and add back unpinned go install, and fix other tests Signed-off-by: Achal Shah --- Makefile | 8 +++++--- .../tests/integration/online_store/test_e2e_local.py | 4 ++-- sdk/python/tests/utils/cli_utils.py | 11 +++++++++-- setup.py | 6 ++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 603653786db..88f04aa95d8 100644 --- a/Makefile +++ b/Makefile @@ -170,11 +170,13 @@ install-go-proto-dependencies: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 install-go-ci-dependencies: - # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line - # ToDo: there should be a better way to install gopy + # TODO: currently gopy installation doesn't work w/o explicit go get in the next line + # TODO: there should be a better way to install gopy go get github.com/go-python/gopy@v0.4.0 go install golang.org/x/tools/cmd/goimports - #go install github.com/go-python/gopy@v0.4.0 + # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` + # but the following command is needed to install it for some reason. + go install github.com/go-python/gopy python -m pip install pybindgen==0.22.0 install-protoc-dependencies: diff --git a/sdk/python/tests/integration/online_store/test_e2e_local.py b/sdk/python/tests/integration/online_store/test_e2e_local.py index c938bc55128..c5b66e7ddce 100644 --- a/sdk/python/tests/integration/online_store/test_e2e_local.py +++ b/sdk/python/tests/integration/online_store/test_e2e_local.py @@ -130,7 +130,7 @@ def _test_materialize_and_online_retrieval( cwd=Path(store.repo_path), ) - assert r.returncode == 0 + assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}" _assert_online_features(store, driver_df, end_date - timedelta(days=7)) # Test `feast materialize-incremental` and online retrieval. @@ -138,7 +138,7 @@ def _test_materialize_and_online_retrieval( ["materialize-incremental", end_date.isoformat()], cwd=Path(store.repo_path), ) - assert r.returncode == 0 + assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}" _assert_online_features(store, driver_df, end_date) diff --git a/sdk/python/tests/utils/cli_utils.py b/sdk/python/tests/utils/cli_utils.py index f2478a4a5ee..e727d9f7194 100644 --- a/sdk/python/tests/utils/cli_utils.py +++ b/sdk/python/tests/utils/cli_utils.py @@ -76,6 +76,9 @@ def local_repo(self, example_repo_py: str, offline_store: str): path: {data_path / "online_store.db"} offline_store: type: {offline_store} + flags: + alpha_features: true + on_demand_transforms: true """ ) ) @@ -84,9 +87,13 @@ def local_repo(self, example_repo_py: str, offline_store: str): repo_example.write_text(example_repo_py) result = self.run(["apply"], cwd=repo_path) - assert result.returncode == 0 + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\n stderr: {result.stderr}" yield FeatureStore(repo_path=str(repo_path), config=None) result = self.run(["teardown"], cwd=repo_path) - assert result.returncode == 0 + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\n stderr: {result.stderr}" diff --git a/setup.py b/setup.py index f5f6f9f920b..626165adc89 100644 --- a/setup.py +++ b/setup.py @@ -408,6 +408,7 @@ def _is_go_ext(self, ext: Extension): ) def build_extension(self, ext: Extension): + print(f"Building extension {ext}") if not self._is_go_ext(ext): # the base class may mutate `self.compiler` compiler = copy.deepcopy(self.compiler) @@ -423,8 +424,8 @@ def build_extension(self, ext: Extension): ) destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"]) - subprocess.check_call(["go", "install", "github.com/go-python/gopy@v0.4.0"]) + subprocess.check_call(["go", "mod", "tidy"], env={"PATH": bin_path, **go_env}) + # subprocess.check_call(["go", "install", "github.com/go-python/gopy@v0.4.0"]) subprocess.check_call( [ "gopy", @@ -456,6 +457,7 @@ def copy_extensions_to_source(self): src_dir = os.path.join(self.build_lib, src_dir) # copy whole directory + print(f"Copying from {src_dir} to {dest_dir}") copy_tree(src_dir, dest_dir) From e70cf6988741fae787ef5754e046cd5ae1b5b3c3 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Jun 2022 16:47:46 -0700 Subject: [PATCH 7/7] cr Signed-off-by: Achal Shah --- sdk/python/tests/utils/cli_utils.py | 3 --- setup.py | 1 - 2 files changed, 4 deletions(-) diff --git a/sdk/python/tests/utils/cli_utils.py b/sdk/python/tests/utils/cli_utils.py index e727d9f7194..ee6ea138fbe 100644 --- a/sdk/python/tests/utils/cli_utils.py +++ b/sdk/python/tests/utils/cli_utils.py @@ -76,9 +76,6 @@ def local_repo(self, example_repo_py: str, offline_store: str): path: {data_path / "online_store.db"} offline_store: type: {offline_store} - flags: - alpha_features: true - on_demand_transforms: true """ ) ) diff --git a/setup.py b/setup.py index 626165adc89..49b2d45d03f 100644 --- a/setup.py +++ b/setup.py @@ -425,7 +425,6 @@ def build_extension(self, ext: Extension): destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) subprocess.check_call(["go", "mod", "tidy"], env={"PATH": bin_path, **go_env}) - # subprocess.check_call(["go", "install", "github.com/go-python/gopy@v0.4.0"]) subprocess.check_call( [ "gopy",