From 9e53a0b44deddf5d36cb26c51756d2b07fdb6a62 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sun, 3 Nov 2024 15:45:40 +0100 Subject: [PATCH 01/62] Create a function app, using a Docker image, to run FastAPI with the embedding service. --- README.md | 11 +++++-- infra/README.md | 31 ++++++++++++++++++ infra/containers.tf | 7 ++++ infra/functions.tf | 78 +++++++++++++++++++++++++++++++++++++++++++++ infra/main.tf | 25 +++++++++++++++ infra/storage.tf | 7 ++++ 6 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 infra/README.md create mode 100644 infra/containers.tf create mode 100644 infra/functions.tf create mode 100644 infra/main.tf create mode 100644 infra/storage.tf diff --git a/README.md b/README.md index a866473..2c8d909 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,7 @@ Then run the container: `docker-compose up` -### Bonus: caching in FastAPI! -TODO ## Cloud @@ -66,3 +64,12 @@ Deploy resources using: ### Azure Use the function app to Go to e.g.: http://*.azurewebsites.com. + +# TODO: +- pre-commit +- logging +- devops pipeline +- integration test +- create package from model and handler so that I can use it in Docker image +- caching requests for FastAPI using external database + diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..0ab5f01 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,31 @@ +Introduction + +Build and push Docker container so that it can be used in function app. + +Build: + +`docker build -f Dockerfile -t embedding_service:latest . ` + + +Login: + +`az acr login --name EmbeddingContainerRegistry ` + +Tag: + +`docker tag embedding_service:latest embeddingcontainerregistry.azurecr.io/embedding_service:latest` + +Push: + +`docker push embeddingcontainerregistry.azurecr.io/embedding_service:latest` + + + +`az login terraform` + +`terraform init` + +`terraform plan` + +`terraform apply` + diff --git a/infra/containers.tf b/infra/containers.tf new file mode 100644 index 0000000..43905ec --- /dev/null +++ b/infra/containers.tf @@ -0,0 +1,7 @@ +resource "azurerm_container_registry" "acr" { + name = "EmbeddingContainerRegistry" + resource_group_name = azurerm_resource_group.rg20embedding001.name + location = azurerm_resource_group.rg20embedding001.location + sku = "Basic" + admin_enabled = true +} diff --git a/infra/functions.tf b/infra/functions.tf new file mode 100644 index 0000000..6d34250 --- /dev/null +++ b/infra/functions.tf @@ -0,0 +1,78 @@ +# resource "azurerm_service_plan" "sp20embedding001" { +# name = "embedding-fa-sp" +# resource_group_name = azurerm_resource_group.rg20embedding001.name +# location = azurerm_resource_group.rg20embedding001.location +# os_type = "Linux" +# sku_name = "B1" # B1 is a basic SKU; change if necessary +# } +# +# resource "azurerm_linux_function_app" "fa20embedding001" { +# name = "embedding-fa" +# resource_group_name = azurerm_resource_group.rg20embedding001.name +# location = azurerm_resource_group.rg20embedding001.location +# +# storage_account_name = azurerm_storage_account.sa20faembedding001.name +# storage_account_access_key = azurerm_storage_account.sa20faembedding001.primary_access_key +# service_plan_id = azurerm_service_plan.sp20embedding001.id +# # https_only = true +# +# # identity { +# # type = "SystemAssigned" +# # } +# +# # app_settings = { +# # FUNCTIONS_WORKER_RUNTIME = "python" +# # FUNCTION_APP_EDIT_MODE = "readOnly" +# # DOCKER_REGISTRY_SERVER_USERNAME = azurerm_container_registry.acr.admin_username +# # DOCKER_REGISTRY_SERVER_URL = azurerm_container_registry.acr.login_server +# # DOCKER_REGISTRY_SERVER_PASSWORD = azurerm_container_registry.acr.admin_password +# # WEBSITES_ENABLE_APP_SERVICE_STORAGE = false +# # } +# +# site_config { +# application_stack { +# docker { +# image_name = "embedding_service" +# image_tag = "latest" +# registry_url = azurerm_container_registry.acr.login_server +# } +# } +# # always_on = false +# # http2_enabled = true +# # ftps_state = "Disabled" +# } +# } + + + +resource "azurerm_log_analytics_workspace" "la20embedding001" { + name = "la20embedding001" + location = azurerm_resource_group.rg20embedding001.location + resource_group_name = azurerm_resource_group.rg20embedding001.name + sku = "PerGB2018" + retention_in_days = 30 +} + +resource "azurerm_container_app_environment" "cae20embedding001" { + name = "cae20embedding001" + location = azurerm_resource_group.rg20embedding001.location + resource_group_name = azurerm_resource_group.rg20embedding001.name + log_analytics_workspace_id = azurerm_log_analytics_workspace.la20embedding001.id +} + +resource "azurerm_container_app" "ca20embedding001" { + name = "ca20embedding001" + container_app_environment_id = azurerm_container_app_environment.cae20embedding001.id + resource_group_name = azurerm_resource_group.rg20embedding001.name + revision_mode = "Single" + + template { + container { + name = "embedding-service-container-app" + image = "embeddingcontainerregistry.azurecr.io/embedding_service" + cpu = 0.25 + memory = "0.5Gi" + } + } +} + diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..8466a90 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,25 @@ +# We strongly recommend using the required_providers block to set the +# Azure Provider source and version being used +terraform { + required_version = "~>1.9.8" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "4.8.0" + } + } +} + +# Configure the Microsoft Azure Provider +provider "azurerm" { + features {} + subscription_id = "8cdc4695-331d-4883-97e0-2184f3577822" +} + +data "azurerm_client_config" "current" { +} +# Create a resource group +resource "azurerm_resource_group" "rg20embedding001" { + name = "rg20embedding001" + location = "North Europe" +} diff --git a/infra/storage.tf b/infra/storage.tf new file mode 100644 index 0000000..ea6e320 --- /dev/null +++ b/infra/storage.tf @@ -0,0 +1,7 @@ +resource "azurerm_storage_account" "sa20faembedding001" { + name = "sa20faembedding001" + resource_group_name = azurerm_resource_group.rg20embedding001.name + location = azurerm_resource_group.rg20embedding001.location + account_tier = "Standard" + account_replication_type = "LRS" +} From 9cd57f8d6e59e9ecf15ae25ec689e4ecf3e35a01 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sun, 3 Nov 2024 17:53:49 +0100 Subject: [PATCH 02/62] Create tests using fixtures, parametrization and using arrange act assert design pattern. --- README.md | 11 ++++++----- infra/README.md | 17 ++++++++++++++++- infra/containers.tf | 4 ++++ infra/functions.tf | 23 ++++++++++++++++++++++- infra/main.tf | 2 +- 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2c8d909..ac3c009 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,14 @@ Go to e.g.: http://0.0.0.0:81/docs ## Docker -First build the container: +First navigate to the `function_app` folder and then build the container: -`docker build -t api:latest .` +`docker build -t functionapp:latest .` Then run the container: -`docker run -p 81:81 -it api:latest` +`docker run -p 80:80 -it functionapp:latest` + ## Docker-compose @@ -56,9 +57,8 @@ Then run the container: ## Cloud ### Terraform -Deploy resources using: -`` +[Terraform information](infra/README.md) ### Azure @@ -67,6 +67,7 @@ Go to e.g.: http://*.azurewebsites.com. # TODO: - pre-commit +- exception handling - logging - devops pipeline - integration test diff --git a/infra/README.md b/infra/README.md index 0ab5f01..d0a42f1 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,13 @@ -Introduction +# Introduction +Terraform is infrastructure as code tool. The infrastructure we are making here contains a Docker registry and a Function app amongst other resources. +The Function App uses a Docker container which needs to be available first, so we build and push it in the next section. We might also choose to create a DevOps pipeline to do this for us. Build and push Docker container so that it can be used in function app. +## How to run + +### Docker + Build: `docker build -f Dockerfile -t embedding_service:latest . ` @@ -19,13 +25,22 @@ Push: `docker push embeddingcontainerregistry.azurecr.io/embedding_service:latest` +### Terraform +Login: `az login terraform` +Init: + `terraform init` +Plan: + `terraform plan` + +Apply: + `terraform apply` diff --git a/infra/containers.tf b/infra/containers.tf index 43905ec..181e36a 100644 --- a/infra/containers.tf +++ b/infra/containers.tf @@ -4,4 +4,8 @@ resource "azurerm_container_registry" "acr" { location = azurerm_resource_group.rg20embedding001.location sku = "Basic" admin_enabled = true + + identity { + type = "SystemAssigned" + } } diff --git a/infra/functions.tf b/infra/functions.tf index 6d34250..c4d4769 100644 --- a/infra/functions.tf +++ b/infra/functions.tf @@ -66,13 +66,34 @@ resource "azurerm_container_app" "ca20embedding001" { resource_group_name = azurerm_resource_group.rg20embedding001.name revision_mode = "Single" + registry { + server = azurerm_container_registry.acr.login_server + username = azurerm_container_registry.acr.admin_username + password_secret_name = "docker-io-pass" + + } + + ingress { + allow_insecure_connections = false + external_enabled = true + target_port = 80 + traffic_weight { + latest_revision = true + percentage = 100 + } + + } template { container { name = "embedding-service-container-app" - image = "embeddingcontainerregistry.azurecr.io/embedding_service" + image = "EmbeddingContainerRegistry/embedding_service:latest" cpu = 0.25 memory = "0.5Gi" } } + secret { + name = "docker-io-pass" + value = azurerm_container_registry.acr.admin_password + } } diff --git a/infra/main.tf b/infra/main.tf index 8466a90..96c9451 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -21,5 +21,5 @@ data "azurerm_client_config" "current" { # Create a resource group resource "azurerm_resource_group" "rg20embedding001" { name = "rg20embedding001" - location = "North Europe" + location = "West Europe" } From 495221c44b00a8e973ff32db8bf8f8201276626a Mon Sep 17 00:00:00 2001 From: blpasd Date: Wed, 6 Nov 2024 20:19:12 +0100 Subject: [PATCH 03/62] Add isort for import sorting configuration for Ruff in pyproject.toml. Hadolint for linting Docker files. Reformat files automatically according to the rules. --- .pre-commit-config.yaml | 21 ++++++++++++++++++ README.md | 6 +++--- app/main.py | 22 ++++++++++++++----- app/model.py | 4 +++- pyproject.toml | 4 ++++ tests/conftest.py | 8 +++---- tests/test_main.py | 48 ++++++++++++++++++++++++++++------------- 7 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..11fb85c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.2 + hooks: + - id: ruff # Linter + args: ["--fix"] # Automatically fix issues where possible + - id: ruff-format # Formatter + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 # Specify the version you want to use + hooks: + - id: check-added-large-files + - id: check-merge-conflict + + # Docker file linter + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker + name: Lint Dockerfiles + files: Dockerfile diff --git a/README.md b/README.md index ac3c009..2ef16e5 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ Personal code to showcase my abilities, such as: - Docker - Terraform - Azure subscription -- pre-commit +- Pre-commit +- Hadolint binary (when using pre-commit) # How to run @@ -65,8 +66,7 @@ Then run the container: Use the function app to Go to e.g.: http://*.azurewebsites.com. -# TODO: -- pre-commit +# TODO: - exception handling - logging - devops pipeline diff --git a/app/main.py b/app/main.py index 177b687..beafcd9 100755 --- a/app/main.py +++ b/app/main.py @@ -6,18 +6,22 @@ class TextInput(BaseModel): text: str + class EmbeddingOutput(BaseModel): - embedding: list[float] + embedding: list[float] description: str | None = None + class SimilarityOutput(BaseModel): similarity: float description: str | None = None + app = FastAPI() handler = Handler() + @app.post("/embed") async def embed_text(text_input: TextInput) -> EmbeddingOutput: """ @@ -27,11 +31,16 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ embedding = handler.embed(text_input.text) - return EmbeddingOutput(embedding=embedding, description="The list of float values representing the text embedding.") + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) @app.post("/similarity") -async def calculate_similarity(text_1: TextInput, text_2: TextInput) -> SimilarityOutput: +async def calculate_similarity( + text_1: TextInput, text_2: TextInput +) -> SimilarityOutput: """ Compute the cosine similarity between two input texts. @@ -40,10 +49,13 @@ async def calculate_similarity(text_1: TextInput, text_2: TextInput) -> Similari :return: The similarity score between the two input texts as JSON. """ similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) - return SimilarityOutput(similarity=similarity_score, description= - "Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.") + return SimilarityOutput( + similarity=similarity_score, + description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", + ) if __name__ == "__main__": import uvicorn + uvicorn.run("app:app", host="0.0.0.0", port=8080, reload=True) diff --git a/app/model.py b/app/model.py index 8dfae7c..d9a0c27 100755 --- a/app/model.py +++ b/app/model.py @@ -67,7 +67,9 @@ def forward(self, text: str) -> torch.Tensor: """ # Tokenize the input - inputs = self.tokenizer(text, return_tensors="pt", padding=True, truncation=True) + inputs = self.tokenizer( + text, return_tensors="pt", padding=True, truncation=True + ) # Forward pass return self.model(**inputs).pooler_output[0] diff --git a/pyproject.toml b/pyproject.toml index f6b0dca..37b7534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ python = "^3.12" fastapi = { version = "0.115.2", extras = ["all"] } torch = "2.3.1" transformers = "4.45.2" +pre-commit = "^4.0.1" [tool.poetry.dev-dependencies] pytest = "8.3.3" @@ -19,3 +20,6 @@ pre-commit = " 4.0.1" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.ruff.lint.isort] +case-sensitive = true \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 6c0c095..89ccc1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ import pytest + @pytest.fixture def long_string_input(): - return { - "text": "This is a very long string. " * 100 - } + return {"text": "This is a very long string. " * 100} + @pytest.fixture def mock_handler(mocker): """Mock the similarity handler.""" - return mocker.patch('model.handler.similarity') \ No newline at end of file + return mocker.patch("model.handler.similarity") diff --git a/tests/test_main.py b/tests/test_main.py index b0868b5..2b4514a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,8 +8,8 @@ # Arrange client = TestClient(app) -def test_embed_text(long_string_input): +def test_embed_text(long_string_input): # Act: Send a POST request to the /embed endpoint response = client.post("/embed", json=long_string_input) @@ -20,7 +20,10 @@ def test_embed_text(long_string_input): response_data = response.json() assert "embedding" in response_data assert "description" in response_data - assert response_data["description"] == "The list of float values representing the text embedding." + assert ( + response_data["description"] + == "The list of float values representing the text embedding." + ) # Assert the embedding values assert isinstance(response_data["embedding"], list) @@ -30,11 +33,14 @@ def test_embed_text(long_string_input): text_inputs = [ {"text": ""}, {"text": "This is a short sentence."}, - {"text": "This is a longer sentence, which contains more words and should still work correctly."}, + { + "text": "This is a longer sentence, which contains more words and should still work correctly." + }, ] -@pytest.mark.parametrize("text_input", text_inputs) -def test_embed_text(text_input): + +@pytest.mark.parametrize("text_input", text_inputs) +def test_embed_text_parametrized(text_input): # Act: Send a POST request to the /embed endpoint response = client.post("/embed", json=text_input) @@ -45,32 +51,44 @@ def test_embed_text(text_input): response_data = response.json() assert "embedding" in response_data assert "description" in response_data - assert response_data["description"] == "The list of float values representing the text embedding." + assert ( + response_data["description"] + == "The list of float values representing the text embedding." + ) # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list - class TestCalculateSimilarity(unittest.TestCase): def setUp(self): self.client = TestClient(app) - @patch('app.main.handler.similarity') # Mock the similarity function + @patch("app.main.handler.similarity") # Mock the similarity function def test_calculate_similarity(self, mock_similarity): # Arrange text_1 = TextInput(text="Dog") text_2 = TextInput(text="Cat") - mock_similarity.return_value = 0.95 # Mock the return value of the similarity function + mock_similarity.return_value = ( + 0.95 # Mock the return value of the similarity function + ) # Act - response = self.client.post("/similarity", json={"text_1": text_1.model_dump(), "text_2": text_2.model_dump()}) + response = self.client.post( + "/similarity", + json={"text_1": text_1.model_dump(), "text_2": text_2.model_dump()}, + ) # Assert self.assertEqual(response.status_code, 200) - self.assertEqual(response.json(), { - "similarity": 0.95, - "description": "Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar." - }) - mock_similarity.assert_called_once_with(text_1="Dog", text_2="Cat") # Check if the handler was called with the correct parameters + self.assertEqual( + response.json(), + { + "similarity": 0.95, + "description": "Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", + }, + ) + mock_similarity.assert_called_once_with( + text_1="Dog", text_2="Cat" + ) # Check if the handler was called with the correct parameters From 598ea95264a816a837fc8c185c855d923f11865b Mon Sep 17 00:00:00 2001 From: blpasd Date: Wed, 6 Nov 2024 20:31:34 +0100 Subject: [PATCH 04/62] Add isort for import sorting configuration for Ruff in pyproject.toml. Hadolint for linting Docker files. Reformat files automatically according to the rules. --- .pre-commit-config.yaml | 2 +- app/main.py | 3 ++- pyproject.toml | 5 +---- tests/test_main.py | 7 ++++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11fb85c..bd69943 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: v0.7.2 hooks: - id: ruff # Linter - args: ["--fix"] # Automatically fix issues where possible + args: ["--select", "I", "--fix"] # Automatically fix issues where possible - id: ruff-format # Formatter - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/app/main.py b/app/main.py index beafcd9..d4489b1 100755 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,8 @@ from fastapi import FastAPI -from app.model import Handler from pydantic import BaseModel +from app.model import Handler + class TextInput(BaseModel): text: str diff --git a/pyproject.toml b/pyproject.toml index 37b7534..3d21020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,4 @@ pre-commit = " 4.0.1" [build-system] requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -[tool.ruff.lint.isort] -case-sensitive = true \ No newline at end of file +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index 2b4514a..c27d9a2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,10 @@ import unittest -import pytest from unittest.mock import patch + +import pytest from fastapi.testclient import TestClient -from app.main import app -from app.main import TextInput + +from app.main import TextInput, app # Arrange client = TestClient(app) From 3b710fbf5f1be93bbf5245abe7ff59dc9b179998 Mon Sep 17 00:00:00 2001 From: blpasd Date: Wed, 6 Nov 2024 20:43:41 +0100 Subject: [PATCH 05/62] Add example of logging and exception handling. --- README.md | 2 -- app/main.py | 22 ++++++++++++++++------ app/model.py | 5 +++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2ef16e5..d279219 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,6 @@ Use the function app to Go to e.g.: http://*.azurewebsites.com. # TODO: -- exception handling -- logging - devops pipeline - integration test - create package from model and handler so that I can use it in Docker image diff --git a/app/main.py b/app/main.py index d4489b1..83c94da 100755 --- a/app/main.py +++ b/app/main.py @@ -1,8 +1,12 @@ -from fastapi import FastAPI +import logging + +from fastapi import FastAPI, HTTPException from pydantic import BaseModel from app.model import Handler +logger = logging.getLogger(__name__) + class TextInput(BaseModel): text: str @@ -31,11 +35,17 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :param text_input: The text to embed. :return: The embedding of the input text as JSON. """ - embedding = handler.embed(text_input.text) - return EmbeddingOutput( - embedding=embedding, - description="The list of float values representing the text embedding.", - ) + logger.info(f"Embedding text: {text_input.text}") + try: + embedding = handler.embed(text_input.text) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + except RuntimeError: + HTTPException( + status_code=404, detail="Something went wrong with creating an embedding." + ) @app.post("/similarity") diff --git a/app/model.py b/app/model.py index d9a0c27..07cf795 100755 --- a/app/model.py +++ b/app/model.py @@ -1,6 +1,10 @@ +import logging + import torch from transformers import BertModel, BertTokenizer +logger = logging.getLogger(__name__) + class Handler: """ @@ -20,6 +24,7 @@ def __init__(self, model_name: str = "bert-base-uncased") -> None: # Load the model and tokenizer with the Hugging Face Transformers library self.tokenizer = BertTokenizer.from_pretrained(model_name) self.model = BertModel.from_pretrained(model_name) + logger.info("Handler initialisation completed. ") def embed(self, text) -> list[float]: """ From 0c2ebf91ecd258cd1ed26c48c35de0675c2a9e2f Mon Sep 17 00:00:00 2001 From: blpasd Date: Mon, 11 Nov 2024 09:00:26 +0100 Subject: [PATCH 06/62] WIP: Docker compose file with Redis database and caching for FastAPI. --- README.md | 9 ++++++- app/main.py | 58 ++++++++++++++++++++++++++++++++++++++------- docker-compose.yaml | 23 ++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 docker-compose.yaml diff --git a/README.md b/README.md index d279219..61e9cac 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,11 @@ Then run the container: ## Docker-compose +Redis can be used as a backend to cache FastAPI requests. -`docker-compose up` +Start the FastAPI and Redis database using: `docker-compose up` +To rebuild, run: `docker-compose up --build` ## Cloud @@ -67,6 +69,11 @@ Use the function app to Go to e.g.: http://*.azurewebsites.com. # TODO: +- fix infra bug: [DEBUG] POST https://management.azure.com/subscriptions//resourceGroups/rg20embedding001/providers/Microsoft.App/containerApps/ca20embedding001/listSecrets?api-version=2023-05-01 (status: 500): retrying in 1s (9 left) + +I do have the correct role set for the principal. + + - devops pipeline - integration test - create package from model and handler so that I can use it in Docker image diff --git a/app/main.py b/app/main.py index 83c94da..b1aff67 100755 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,19 @@ import logging +from contextlib import asynccontextmanager +from os import environ +import redis from fastapi import FastAPI, HTTPException from pydantic import BaseModel from app.model import Handler +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) logger = logging.getLogger(__name__) +logger.info("START") class TextInput(BaseModel): @@ -22,10 +30,34 @@ class SimilarityOutput(BaseModel): description: str | None = None -app = FastAPI() +class Redis: + def __init__(self): + self.client = redis.Redis( + host=environ.get("REDIS_HOST"), + port=environ.get("REDIS_PORT"), + decode_responses=True, + ) # Directly return responses in non-binary + logger.info( + f"Redis database connection established for {environ.get("REDIS_HOST")} on port {environ.get("REDIS_PORT")}" + ) + + +redis_client = None + + +@asynccontextmanager # Makes sure redis connection is closed after application shutdown +async def lifespan(app: FastAPI): + global redis_client + redis_client = Redis().client + yield + + +# TODO: what to do when there is no Redis database? handler = Handler() +app = FastAPI(lifespan=lifespan) + @app.post("/embed") async def embed_text(text_input: TextInput) -> EmbeddingOutput: @@ -36,16 +68,26 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ logger.info(f"Embedding text: {text_input.text}") - try: - embedding = handler.embed(text_input.text) + global redis_client + cached_embedding = redis_client.get(text_input.text) + if cached_embedding: return EmbeddingOutput( - embedding=embedding, + embedding=cached_embedding, description="The list of float values representing the text embedding.", ) - except RuntimeError: - HTTPException( - status_code=404, detail="Something went wrong with creating an embedding." - ) + else: + try: + embedding = handler.embed(text_input.text) + redis_client.set(text_input.text, str(embedding)) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + except RuntimeError: + HTTPException( + status_code=404, + detail="Something went wrong with creating an embedding.", + ) @app.post("/similarity") diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..bdd7748 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,23 @@ +version: "3.8" + +services: + fastapi: + container_name: fastapi + build: + context: . + dockerfile: Dockerfile + environment: + - REDIS_HOST=localhost # Redis hostname (will be used in FastAPI to connect to Redis) + - REDIS_PORT=6379 # Default Redis port + depends_on: + - redis # FastAPI app depends on Redis being up + ports: + - "8080:8080" + network_mode: host + + redis: + image: "redis:latest" # Using the official Redis image + container_name: redis_cache + ports: + - "6379:6379" # Expose Redis on port 6379 + network_mode: host diff --git a/pyproject.toml b/pyproject.toml index 3d21020..6604573 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ fastapi = { version = "0.115.2", extras = ["all"] } torch = "2.3.1" transformers = "4.45.2" pre-commit = "^4.0.1" +redis = "^5.2.0" [tool.poetry.dev-dependencies] pytest = "8.3.3" From 4081aa65a463fa91a9dfeffbd92324d85308f5c0 Mon Sep 17 00:00:00 2001 From: blpasd Date: Mon, 11 Nov 2024 19:14:22 +0100 Subject: [PATCH 07/62] Intermediate progress --- app/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index b1aff67..6d26ae1 100755 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,4 @@ +import json import logging from contextlib import asynccontextmanager from os import environ @@ -13,7 +14,6 @@ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format ) logger = logging.getLogger(__name__) -logger.info("START") class TextInput(BaseModel): @@ -69,7 +69,8 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: """ logger.info(f"Embedding text: {text_input.text}") global redis_client - cached_embedding = redis_client.get(text_input.text) + cached_embedding = await redis_client.get(text_input.text) + cached_embedding = json.loads(cached_embedding) if cached_embedding: return EmbeddingOutput( embedding=cached_embedding, @@ -78,7 +79,7 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: else: try: embedding = handler.embed(text_input.text) - redis_client.set(text_input.text, str(embedding)) + await redis_client.set(text_input.text, json.dumps(embedding)) return EmbeddingOutput( embedding=embedding, description="The list of float values representing the text embedding.", From c119e2d8e9de9b710918fc8cd1d37ce1c819dde5 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 20:13:59 +0100 Subject: [PATCH 08/62] fixed infra bug. --- infra/README.md | 6 +++++- infra/functions.tf | 51 ++-------------------------------------------- 2 files changed, 7 insertions(+), 50 deletions(-) diff --git a/infra/README.md b/infra/README.md index d0a42f1..dc5076c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -29,7 +29,11 @@ Push: Login: -`az login terraform` +`az login` + +Navigate to the infra folder: + +`cd infra` Init: diff --git a/infra/functions.tf b/infra/functions.tf index c4d4769..5b89304 100644 --- a/infra/functions.tf +++ b/infra/functions.tf @@ -1,50 +1,3 @@ -# resource "azurerm_service_plan" "sp20embedding001" { -# name = "embedding-fa-sp" -# resource_group_name = azurerm_resource_group.rg20embedding001.name -# location = azurerm_resource_group.rg20embedding001.location -# os_type = "Linux" -# sku_name = "B1" # B1 is a basic SKU; change if necessary -# } -# -# resource "azurerm_linux_function_app" "fa20embedding001" { -# name = "embedding-fa" -# resource_group_name = azurerm_resource_group.rg20embedding001.name -# location = azurerm_resource_group.rg20embedding001.location -# -# storage_account_name = azurerm_storage_account.sa20faembedding001.name -# storage_account_access_key = azurerm_storage_account.sa20faembedding001.primary_access_key -# service_plan_id = azurerm_service_plan.sp20embedding001.id -# # https_only = true -# -# # identity { -# # type = "SystemAssigned" -# # } -# -# # app_settings = { -# # FUNCTIONS_WORKER_RUNTIME = "python" -# # FUNCTION_APP_EDIT_MODE = "readOnly" -# # DOCKER_REGISTRY_SERVER_USERNAME = azurerm_container_registry.acr.admin_username -# # DOCKER_REGISTRY_SERVER_URL = azurerm_container_registry.acr.login_server -# # DOCKER_REGISTRY_SERVER_PASSWORD = azurerm_container_registry.acr.admin_password -# # WEBSITES_ENABLE_APP_SERVICE_STORAGE = false -# # } -# -# site_config { -# application_stack { -# docker { -# image_name = "embedding_service" -# image_tag = "latest" -# registry_url = azurerm_container_registry.acr.login_server -# } -# } -# # always_on = false -# # http2_enabled = true -# # ftps_state = "Disabled" -# } -# } - - - resource "azurerm_log_analytics_workspace" "la20embedding001" { name = "la20embedding001" location = azurerm_resource_group.rg20embedding001.location @@ -76,7 +29,7 @@ resource "azurerm_container_app" "ca20embedding001" { ingress { allow_insecure_connections = false external_enabled = true - target_port = 80 + target_port = 81 traffic_weight { latest_revision = true percentage = 100 @@ -86,7 +39,7 @@ resource "azurerm_container_app" "ca20embedding001" { template { container { name = "embedding-service-container-app" - image = "EmbeddingContainerRegistry/embedding_service:latest" + image = "${azurerm_container_registry.acr.login_server}/embedding_service:latest" cpu = 0.25 memory = "0.5Gi" } From f44e8affc270005e4a95eac2c16d943d14720ee2 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 20:50:06 +0100 Subject: [PATCH 09/62] Start with github actions workflows --- .github/workflows/azure-pipeline.yaml | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/azure-pipeline.yaml diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml new file mode 100644 index 0000000..e9f5405 --- /dev/null +++ b/.github/workflows/azure-pipeline.yaml @@ -0,0 +1,80 @@ +name: Build, Push Docker Image & Deploy Infrastructure with Terraform + +on: + push: + branches: + - main + pull_request: + branches: + - main + +env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ACR_NAME: ${{ secrets.ACR_NAME }} + IMAGE_NAME: 'embedding_service' + IMAGE_TAG: 'latest' + TF_WORKSPACE: 'infra' + TF_VARS: '-var location=eucentral' # Example of passing variables to Terraform + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Azure Container Registry (ACR) + uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.ACR_NAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Build Docker image + run: | + docker build -t ${{ secrets.ACR_NAME }}.azurecr.io/$IMAGE_NAME:$IMAGE_TAG . + + - name: Push Docker image to ACR + run: | + docker push ${{ secrets.ACR_NAME }}.azurecr.io/$IMAGE_NAME:$IMAGE_TAG + + terraform-deploy: + runs-on: ubuntu-latest + needs: build-and-push + environment: production + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: '~>1.9.8' # Specify your desired Terraform version + + - name: Terraform Init + run: | + cd $TF_WORKSPACE + terraform init + + - name: Terraform Plan + run: | + cd $TF_WORKSPACE + terraform plan $TF_VARS + + - name: Terraform Apply + run: | + cd $TF_WORKSPACE + terraform apply -auto-approve $TF_VARS + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} From 94e2bf91c179cf58d0bbf0fe33522f1c43ecb642 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 20:56:48 +0100 Subject: [PATCH 10/62] Remove TF vars --- .github/workflows/azure-pipeline.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index e9f5405..c4d36fc 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -17,7 +17,6 @@ env: IMAGE_NAME: 'embedding_service' IMAGE_TAG: 'latest' TF_WORKSPACE: 'infra' - TF_VARS: '-var location=eucentral' # Example of passing variables to Terraform jobs: build-and-push: @@ -30,7 +29,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Log in to Azure Container Registry (ACR) + - name: Log in to Azure Container Registry (ACR) # TODO: pre-deploy a container registry if none exists uses: azure/docker-login@v1 with: login-server: ${{ secrets.ACR_NAME }}.azurecr.io From 6a8803314701d8f5dd620bd90318e195004f12c8 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 21:02:08 +0100 Subject: [PATCH 11/62] Building Docker file on Github is slow. Use ACR container build task for fun to see if faster. --- .github/workflows/azure-pipeline.yaml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index c4d36fc..f4aaad4 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -26,23 +26,27 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Set up Azure CLI + uses: azure/setup-azurecli@v1 - - name: Log in to Azure Container Registry (ACR) # TODO: pre-deploy a container registry if none exists - uses: azure/docker-login@v1 + - name: Login to Azure using Service Principal + uses: azure/login@v1 with: - login-server: ${{ secrets.ACR_NAME }}.azurecr.io - username: ${{ secrets.ACR_USERNAME }} - password: ${{ secrets.ACR_PASSWORD }} + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + client-secret: ${{ secrets.AZURE_CLIENT_SECRET }} - - name: Build Docker image + - name: Login to Azure Container Registry (ACR) run: | - docker build -t ${{ secrets.ACR_NAME }}.azurecr.io/$IMAGE_NAME:$IMAGE_TAG . + az acr login --name $ACR_NAME + + - name: Build Docker image using ACR Build + run: | + az acr build --registry $ACR_NAME --image $IMAGE_NAME:$IMAGE_TAG . - name: Push Docker image to ACR run: | - docker push ${{ secrets.ACR_NAME }}.azurecr.io/$IMAGE_NAME:$IMAGE_TAG + echo "Docker image $IMAGE_NAME:$IMAGE_TAG built and pushed to ACR successfully." terraform-deploy: runs-on: ubuntu-latest From 50e6b981c9b7ed60ce6a6c1332c2b166be6f633b Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 21:05:41 +0100 Subject: [PATCH 12/62] No Azure setup needed I think, just login --- .github/workflows/azure-pipeline.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index f4aaad4..c555d78 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -26,9 +26,6 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Azure CLI - uses: azure/setup-azurecli@v1 - - name: Login to Azure using Service Principal uses: azure/login@v1 with: From b516954b1b8198d4756c2f7e7d25e863b51e8b34 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 21:13:09 +0100 Subject: [PATCH 13/62] Add subscription-id --- .github/workflows/azure-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index c555d78..d5c7904 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -32,6 +32,7 @@ jobs: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} client-secret: ${{ secrets.AZURE_CLIENT_SECRET }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Azure Container Registry (ACR) run: | From e40787cbdd355e5a55f2343896b35b38a8813e1b Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 21:15:35 +0100 Subject: [PATCH 14/62] Use v2 --- .github/workflows/azure-pipeline.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index d5c7904..6faaca6 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -27,11 +27,10 @@ jobs: uses: actions/checkout@v3 - name: Login to Azure using Service Principal - uses: azure/login@v1 + uses: azure/login@v2 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} - client-secret: ${{ secrets.AZURE_CLIENT_SECRET }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Azure Container Registry (ACR) From 184182a33788fb537b41dcf41956142dcb878ea9 Mon Sep 17 00:00:00 2001 From: blpasd Date: Sat, 16 Nov 2024 21:19:25 +0100 Subject: [PATCH 15/62] permissions: id-token: write --- .github/workflows/azure-pipeline.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml index 6faaca6..0bcc864 100644 --- a/.github/workflows/azure-pipeline.yaml +++ b/.github/workflows/azure-pipeline.yaml @@ -21,13 +21,15 @@ env: jobs: build-and-push: runs-on: ubuntu-latest - + permissions: + id-token: write steps: - name: Checkout code uses: actions/checkout@v3 - name: Login to Azure using Service Principal uses: azure/login@v2 + with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} From e368252d7ff202d91568446439747cc404cbe3d3 Mon Sep 17 00:00:00 2001 From: romusters Date: Fri, 24 Jan 2025 16:21:49 +0100 Subject: [PATCH 16/62] Add ansible configuration --- ansible.yaml | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 ansible.yaml diff --git a/ansible.yaml b/ansible.yaml new file mode 100644 index 0000000..16b7700 --- /dev/null +++ b/ansible.yaml @@ -0,0 +1,128 @@ +--- +- name: Set up ML Ops Environment + hosts: localhost + become: true + + vars: + pyenv_root: "~/.pyenv" + python_version: "3.12" + + tasks: + + # Ensure system is updated + - name: Update and upgrade the system + apt: + update_cache: yes + upgrade: dist + + # Install essential packages + - name: Install essential development tools + apt: + name: + - build-essential + - curl + - git + - libssl-dev + - zlib1g-dev + - libbz2-dev + - libreadline-dev + - libsqlite3-dev + - wget + - llvm + - libncurses5-dev + - libncursesw5-dev + - xz-utils + - tk-dev + - libffi-dev + - liblzma-dev + - python3-openssl + - docker.io + - docker-compose + - unzip + state: present + + # Install Pyenv + - name: Clone pyenv repository + git: + repo: "https://github.com/pyenv/pyenv.git" + dest: "{{ pyenv_root }}" + update: no + + - name: Set up pyenv environment variables + copy: + dest: ~/.bashrc + content: | + export PYENV_ROOT="{{ pyenv_root }}" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init --path)" + owner: "{{ ansible_user_id }}" + mode: 0644 + notify: + - Reload bashrc + + - name: Install Python {{ python_version }} using pyenv + shell: | + source ~/.bashrc && \ + pyenv install {{ python_version }} && \ + pyenv global {{ python_version }} + args: + executable: /bin/bash + + # Install Poetry + - name: Install Poetry + shell: | + curl -sSL https://install.python-poetry.org | python3 - + args: + executable: /bin/bash + + # Install Terraform + - name: Download Terraform + shell: | + curl -fsSL https://releases.hashicorp.com/terraform/1.5.0/terraform_1.5.0_linux_amd64.zip -o terraform.zip + unzip terraform.zip -d /usr/local/bin/ + rm terraform.zip + args: + executable: /bin/bash + + # Install Azure CLI + - name: Install Azure CLI + shell: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + args: + executable: /bin/bash + + # Install Google Cloud CLI + - name: Install Google Cloud CLI + shell: | + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + sudo apt update && sudo apt install -y google-cloud-sdk + args: + executable: /bin/bash + + # Install Hadolint + - name: Install Hadolint + shell: | + wget -O /usr/local/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 + chmod +x /usr/local/bin/hadolint + args: + executable: /bin/bash + + # Verify Docker Installation + - name: Add user to Docker group + user: + name: "{{ ansible_user_id }}" + groups: docker + append: yes + + - name: Enable Docker service + systemd: + name: docker + enabled: yes + state: started + + handlers: + - name: Reload bashrc + shell: source ~/.bashrc + args: + executable: /bin/bash \ No newline at end of file From fa751510cd6dde1fff70ef2f08a33f606526574b Mon Sep 17 00:00:00 2001 From: romusters Date: Fri, 24 Jan 2025 17:00:48 +0100 Subject: [PATCH 17/62] Some updates. --- .gitignore | 3 +++ Dockerfile | 4 +++- README.md | 10 +++++++++- pyproject.toml | 3 +-- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2dc53ca..29349ed 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +poetry.lock + + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index e0cc9a8..38aa650 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,12 @@ EXPOSE 81 WORKDIR /code -RUN pip install --no-cache-dir poetry==1.8.4 +RUN pip install --no-cache-dir poetry==1.8.4 --trusted-host "pypi.org" --trusted-host "files.pythonhosted.org" COPY ./pyproject.toml /code/pyproject.toml + +ENV PIP_TRUSTED_HOST= RUN poetry install --no-cache COPY ./app /code/app diff --git a/README.md b/README.md index d279219..27a1cc5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ Personal code to showcase my abilities, such as: - Pre-commit - Hadolint binary (when using pre-commit) +## Ansible +Ansible can be used to install prerequisites: `ansible-playbook ansible.yaml --ask-become-pass` + # How to run @@ -30,7 +33,7 @@ To run the project locally using Python, run: `poetry run fastapi run app/main.py --host 0.0.0.0 --port 8080` -TODO: try [uv](https://github.com/astral-sh/uv) instead of Poetry. + Go to e.g.: http://0.0.0.0:81/docs @@ -66,7 +69,12 @@ Then run the container: Use the function app to Go to e.g.: http://*.azurewebsites.com. +# Remarks +Although unadvised, setting `PYTHONHTTPSVERIFY` to `false` circumpasses SSL certificate verification when behind proxy firewall. Installing the required certificates in `certifi`, local truststore or poetry configuration is preferred. + # TODO: +- create interface for Redis and Pinecone database +- try [uv](https://github.com/astral-sh/uv) instead of Poetry. - devops pipeline - integration test - create package from model and handler so that I can use it in Docker image diff --git a/pyproject.toml b/pyproject.toml index 3d21020..e2242ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,11 +11,10 @@ python = "^3.12" fastapi = { version = "0.115.2", extras = ["all"] } torch = "2.3.1" transformers = "4.45.2" -pre-commit = "^4.0.1" [tool.poetry.dev-dependencies] pytest = "8.3.3" -pre-commit = " 4.0.1" +pre-commit = "4.0.1" [build-system] requires = ["poetry-core>=1.0.0"] From fc3a8052b2e19c282840ce05b528102c812522c2 Mon Sep 17 00:00:00 2001 From: romusters Date: Fri, 24 Jan 2025 21:05:52 +0100 Subject: [PATCH 18/62] Update docker file to build when behind corporate proxy --- Dockerfile | 26 +++++++++++++++++++------- README.md | 4 ++-- docker-compose.yaml | 4 ++-- pyproject.toml | 3 ++- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index e0cc9a8..b9143fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,27 @@ -FROM python:3.12 +FROM python:3.12.6-slim-bookworm -EXPOSE 81 +WORKDIR /app -WORKDIR /code +RUN apt-get update && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir poetry==1.8.4 +RUN pip install --no-cache-dir --upgrade pip poetry --trusted-host pypi.org --trusted-host files.pythonhosted.org -COPY ./pyproject.toml /code/pyproject.toml +COPY pyproject.toml /app -RUN poetry install --no-cache +# Be careful: this should not run in docker compose environment. It is only used to circumvent corporate proxy for personal projects. +RUN poetry source add fpho https://files.pythonhosted.org && \ +poetry config certificates.fpho.cert false && \ +poetry source add pypi && \ +poetry config certificates.PyPI.cert false && \ +poetry config certificates.pypi.cert false -COPY ./app /code/app +ENV POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_CREATE=false \ + POETRY_CACHE_DIR='/var/cache/pypoetry' \ + POETRY_HOME='/usr/local' +COPY app /app +RUN poetry install --no-cache + +EXPOSE 81 CMD ["poetry", "run", "fastapi", "run", "app/main.py", "--port", "81"] \ No newline at end of file diff --git a/README.md b/README.md index 61e9cac..0d3515f 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,9 @@ Then run the container: ## Docker-compose Redis can be used as a backend to cache FastAPI requests. -Start the FastAPI and Redis database using: `docker-compose up` +Start the FastAPI and Redis database using: `docker compose up` -To rebuild, run: `docker-compose up --build` +To rebuild, run: `docker compose up --build` ## Cloud diff --git a/docker-compose.yaml b/docker-compose.yaml index bdd7748..69c3321 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,13 +10,13 @@ services: - REDIS_HOST=localhost # Redis hostname (will be used in FastAPI to connect to Redis) - REDIS_PORT=6379 # Default Redis port depends_on: - - redis # FastAPI app depends on Redis being up + - redis ports: - "8080:8080" network_mode: host redis: - image: "redis:latest" # Using the official Redis image + image: "redis:latest" container_name: redis_cache ports: - "6379:6379" # Expose Redis on port 6379 diff --git a/pyproject.toml b/pyproject.toml index 6604573..58cb1ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,9 @@ torch = "2.3.1" transformers = "4.45.2" pre-commit = "^4.0.1" redis = "^5.2.0" +requests = "2.31.0" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "8.3.3" pre-commit = " 4.0.1" From 40ebf33ce9b8a687ff95d6783ab82f7d5b24f507 Mon Sep 17 00:00:00 2001 From: romusters Date: Sat, 25 Jan 2025 17:52:30 +0100 Subject: [PATCH 19/62] Redis database for caching embeddings --- Dockerfile | 11 +++++--- README.md | 11 +++++++- app/main.py | 65 ++++++++++++++++++++++++++++----------------- docker-compose.yaml | 8 +++--- 4 files changed, 64 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index b9143fc..526ebdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /app RUN apt-get update && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir --upgrade pip poetry --trusted-host pypi.org --trusted-host files.pythonhosted.org +RUN pip install --no-cache-dir --upgrade certifi pip poetry --trusted-host pypi.org --trusted-host files.pythonhosted.org COPY pyproject.toml /app @@ -20,8 +20,13 @@ ENV POETRY_NO_INTERACTION=1 \ POETRY_CACHE_DIR='/var/cache/pypoetry' \ POETRY_HOME='/usr/local' -COPY app /app + RUN poetry install --no-cache +COPY app /app + EXPOSE 81 -CMD ["poetry", "run", "fastapi", "run", "app/main.py", "--port", "81"] \ No newline at end of file +ENV PYTHONHTTPSVERIFY=0 +RUN cat /app/certificates.crt >> /usr/local/lib/python3.12/site-packages/certifi/cacert.pem + +CMD ["poetry", "run", "fastapi", "run", "main.py", "--port", "8080"] \ No newline at end of file diff --git a/README.md b/README.md index 0d3515f..3283603 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Personal code to showcase my abilities, such as: # Prerequisites - Python 3.12 and Pip/Poetry +- Pyenv - Docker - Terraform - Azure subscription @@ -24,8 +25,16 @@ Personal code to showcase my abilities, such as: # How to run +## Pyenv +Install correct Python version using Pyenv: + +`pyenv install 3.12` ## Python + +`poetry env use $(pyenv which python)` +`poetry install` + To run the project locally using Python, run: `poetry run fastapi run app/main.py --host 0.0.0.0 --port 8080` @@ -73,7 +82,7 @@ Go to e.g.: http://*.azurewebsites.com. I do have the correct role set for the principal. - +- make Redis asynchronous - devops pipeline - integration test - create package from model and handler so that I can use it in Docker image diff --git a/app/main.py b/app/main.py index 6d26ae1..d1765be 100755 --- a/app/main.py +++ b/app/main.py @@ -40,15 +40,29 @@ def __init__(self): logger.info( f"Redis database connection established for {environ.get("REDIS_HOST")} on port {environ.get("REDIS_PORT")}" ) + def get_key(self, key: str): + """ + Retrieves a key from Redis if it exists. + :param redis_url: Redis connection URL. + :param key: The key to retrieve. + :return: The value of the key if it exists, otherwise None. + """ + # Check if the key exists + exists = self.client.exists(key) + if exists: + # Retrieve the key's value + value = self.client.get(key) + return value + return None -redis_client = None +redis_object = None @asynccontextmanager # Makes sure redis connection is closed after application shutdown async def lifespan(app: FastAPI): - global redis_client - redis_client = Redis().client + global redis_object + redis_object = Redis() yield @@ -68,18 +82,21 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ logger.info(f"Embedding text: {text_input.text}") - global redis_client - cached_embedding = await redis_client.get(text_input.text) - cached_embedding = json.loads(cached_embedding) + global redis_object + logging.info(f"text_input.text: {text_input.text}") + + cached_embedding = redis_object.get_key(text_input.text) + logging.info(f"cached_embedding: {cached_embedding}") + logging.info(f"type cached_embedding: {type(cached_embedding)}") if cached_embedding: return EmbeddingOutput( - embedding=cached_embedding, + embedding=json.loads(cached_embedding), description="The list of float values representing the text embedding.", ) else: try: embedding = handler.embed(text_input.text) - await redis_client.set(text_input.text, json.dumps(embedding)) + redis_object.client.set(text_input.text, json.dumps(embedding)) return EmbeddingOutput( embedding=embedding, description="The list of float values representing the text embedding.", @@ -91,25 +108,25 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: ) -@app.post("/similarity") -async def calculate_similarity( - text_1: TextInput, text_2: TextInput -) -> SimilarityOutput: - """ - Compute the cosine similarity between two input texts. +# @app.post("/similarity") +# async def calculate_similarity( +# text_1: TextInput, text_2: TextInput +# ) -> SimilarityOutput: +# """ +# Compute the cosine similarity between two input texts. - :param text_1: The first text. - :param text_2: The second text. - :return: The similarity score between the two input texts as JSON. - """ - similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) - return SimilarityOutput( - similarity=similarity_score, - description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", - ) +# :param text_1: The first text. +# :param text_2: The second text. +# :return: The similarity score between the two input texts as JSON. +# """ +# similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) +# return SimilarityOutput( +# similarity=similarity_score, +# description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", +# ) if __name__ == "__main__": import uvicorn - uvicorn.run("app:app", host="0.0.0.0", port=8080, reload=True) + uvicorn.run("app:app", host="0.0.0.0", reload=True) diff --git a/docker-compose.yaml b/docker-compose.yaml index 69c3321..a54f1b7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,17 +7,19 @@ services: context: . dockerfile: Dockerfile environment: - - REDIS_HOST=localhost # Redis hostname (will be used in FastAPI to connect to Redis) + - REDIS_HOST=redis # Redis hostname (will be used in FastAPI to connect to Redis) - REDIS_PORT=6379 # Default Redis port + - PYTHONHTTPSVERIFY=0 + # - REQUESTS_CA_BUNDLE="" depends_on: - redis ports: - "8080:8080" - network_mode: host + # network_mode: host redis: image: "redis:latest" container_name: redis_cache ports: - "6379:6379" # Expose Redis on port 6379 - network_mode: host + # network_mode: host From e3dfd18381160272d8e5a0649734870a2befe74d Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 12:06:30 +0100 Subject: [PATCH 20/62] Update tests using Redis and cached endpoints --- app/main.py | 50 +++++++--------- pyproject.toml | 1 + tests/conftest.py | 12 ---- tests/payload_tests.py | 1 + tests/test_main.py | 133 ++++++++++++++++++++++++++--------------- 5 files changed, 107 insertions(+), 90 deletions(-) delete mode 100644 tests/conftest.py create mode 100644 tests/payload_tests.py diff --git a/app/main.py b/app/main.py index d1765be..d6af3f8 100755 --- a/app/main.py +++ b/app/main.py @@ -56,22 +56,13 @@ def get_key(self, key: str): return value return None - -redis_object = None - -@asynccontextmanager # Makes sure redis connection is closed after application shutdown -async def lifespan(app: FastAPI): - global redis_object - redis_object = Redis() - yield - +app = FastAPI() # TODO: what to do when there is no Redis database? +database_object = Redis() handler = Handler() -app = FastAPI(lifespan=lifespan) - @app.post("/embed") async def embed_text(text_input: TextInput) -> EmbeddingOutput: @@ -82,10 +73,9 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ logger.info(f"Embedding text: {text_input.text}") - global redis_object logging.info(f"text_input.text: {text_input.text}") - cached_embedding = redis_object.get_key(text_input.text) + cached_embedding = database_object.get_key(text_input.text) logging.info(f"cached_embedding: {cached_embedding}") logging.info(f"type cached_embedding: {type(cached_embedding)}") if cached_embedding: @@ -96,7 +86,7 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: else: try: embedding = handler.embed(text_input.text) - redis_object.client.set(text_input.text, json.dumps(embedding)) + database_object.client.set(text_input.text, json.dumps(embedding)) return EmbeddingOutput( embedding=embedding, description="The list of float values representing the text embedding.", @@ -108,22 +98,22 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: ) -# @app.post("/similarity") -# async def calculate_similarity( -# text_1: TextInput, text_2: TextInput -# ) -> SimilarityOutput: -# """ -# Compute the cosine similarity between two input texts. - -# :param text_1: The first text. -# :param text_2: The second text. -# :return: The similarity score between the two input texts as JSON. -# """ -# similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) -# return SimilarityOutput( -# similarity=similarity_score, -# description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", -# ) +@app.post("/similarity") +async def calculate_similarity( + text_1: TextInput, text_2: TextInput +) -> SimilarityOutput: + """ + Compute the cosine similarity between two input texts. + + :param text_1: The first text. + :param text_2: The second text. + :return: The similarity score between the two input texts as JSON. + """ + similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) + return SimilarityOutput( + similarity=similarity_score, + description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", + ) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 58cb1ff..1987430 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ transformers = "4.45.2" pre-commit = "^4.0.1" redis = "^5.2.0" requests = "2.31.0" +parameterized = "^0.9.0" [tool.poetry.group.dev.dependencies] pytest = "8.3.3" diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 89ccc1e..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -import pytest - - -@pytest.fixture -def long_string_input(): - return {"text": "This is a very long string. " * 100} - - -@pytest.fixture -def mock_handler(mocker): - """Mock the similarity handler.""" - return mocker.patch("model.handler.similarity") diff --git a/tests/payload_tests.py b/tests/payload_tests.py new file mode 100644 index 0000000..9252c20 --- /dev/null +++ b/tests/payload_tests.py @@ -0,0 +1 @@ +long_string_input = {"text": "This is a very long string. " * 100} \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index c27d9a2..7a16a5e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,65 +1,102 @@ +import json import unittest -from unittest.mock import patch -import pytest -from fastapi.testclient import TestClient - -from app.main import TextInput, app - -# Arrange -client = TestClient(app) +from app.main import app, TextInput +from fastapi.testclient import TestClient +from parameterized import parameterized +from tests.payload_tests import long_string_input +from unittest.mock import patch, MagicMock -def test_embed_text(long_string_input): - # Act: Send a POST request to the /embed endpoint - response = client.post("/embed", json=long_string_input) - # Assert the response status code - assert response.status_code == 200 +class TestEmbedEndpoint(unittest.TestCase): - # Assert the response data structure - response_data = response.json() - assert "embedding" in response_data - assert "description" in response_data - assert ( - response_data["description"] - == "The list of float values representing the text embedding." - ) + # Arrange + def setUp(self): + self.client = TestClient(app) - # Assert the embedding values - assert isinstance(response_data["embedding"], list) - assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list + @patch("app.main.database_object") + def test_embed_text_cached(self, mock_database_object): + mock_database_object.get_key = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) -text_inputs = [ - {"text": ""}, - {"text": "This is a short sentence."}, - { - "text": "This is a longer sentence, which contains more words and should still work correctly." - }, -] + # Act: Send a POST request to the /embed endpoint + response = self.client.post("/embed", json=long_string_input) + # Assert the response status code + assert response.status_code == 200 -@pytest.mark.parametrize("text_input", text_inputs) -def test_embed_text_parametrized(text_input): - # Act: Send a POST request to the /embed endpoint - response = client.post("/embed", json=text_input) + # Assert the response data structure + response_data = response.json() + assert "embedding" in response_data + assert "description" in response_data + assert ( + response_data["description"] + == "The list of float values representing the text embedding." + ) - # Assert the response status code - assert response.status_code == 200 + # Assert the embedding values + assert isinstance(response_data["embedding"], list) + assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list + mock_database_object.get_key.assert_called_once_with(long_string_input['text']) + + + @patch("app.main.handler") + @patch("app.main.database_object") + def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): + mock_database_object.get_key = MagicMock(return_value=None) + mock_handler_object.embed = MagicMock(return_value=[1,2,3]) + + # Act: Send a POST request to the /embed endpoint + response = self.client.post("/embed", json=long_string_input) + + # Assert the response status code + assert response.status_code == 200 + + # Assert the response data structure + response_data = response.json() + assert "embedding" in response_data + assert "description" in response_data + assert ( + response_data["description"] + == "The list of float values representing the text embedding." + ) - # Assert the response data structure - response_data = response.json() - assert "embedding" in response_data - assert "description" in response_data - assert ( - response_data["description"] - == "The list of float values representing the text embedding." - ) + # Assert the embedding values + assert isinstance(response_data["embedding"], list) + assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list + mock_database_object.get_key.assert_called_once_with(long_string_input['text']) + mock_handler_object.embed.assert_called_once_with(long_string_input['text']) + + + @parameterized.expand([ + ({"text": ""}, ), + ({"text": "This is a short sentence."}, ), + ({ + "text": "This is a longer sentence, which contains more words and should still work correctly." + }, ), + ]) + @patch("app.main.database_object") + def test_embed_text_parametrized(self, text_input, mock_database_object): + mock_database_object.get_key = MagicMock(return_value=None) + # Act: Send a POST request to the /embed endpoint + response = self.client.post("/embed", json=text_input) + + # Assert the response status code + assert response.status_code == 200 + + # Assert the response data structure + response_data = response.json() + assert "embedding" in response_data + assert "description" in response_data + assert ( + response_data["description"] + == "The list of float values representing the text embedding." + ) - # Assert the embedding values - assert isinstance(response_data["embedding"], list) - assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list + # Assert the embedding values + assert isinstance(response_data["embedding"], list) + assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list class TestCalculateSimilarity(unittest.TestCase): From ecd5023c0c43565bbb1ee8716e9e5b719eda4fe0 Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 12:54:44 +0100 Subject: [PATCH 21/62] Clean up --- README.md | 5 ++--- app/main.py | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3283603..419b7c6 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,7 @@ To run the project locally using Python, run: `poetry run fastapi run app/main.py --host 0.0.0.0 --port 8080` -TODO: try [uv](https://github.com/astral-sh/uv) instead of Poetry. - -Go to e.g.: http://0.0.0.0:81/docs +Go to e.g.: http://0.0.0.0:8080/docs ## Tests @@ -87,4 +85,5 @@ I do have the correct role set for the principal. - integration test - create package from model and handler so that I can use it in Docker image - caching requests for FastAPI using external database +- try [uv](https://github.com/astral-sh/uv) instead of Poetry. diff --git a/app/main.py b/app/main.py index d6af3f8..f5d0a06 100755 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,5 @@ import json import logging -from contextlib import asynccontextmanager from os import environ import redis @@ -73,19 +72,19 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ logger.info(f"Embedding text: {text_input.text}") - logging.info(f"text_input.text: {text_input.text}") cached_embedding = database_object.get_key(text_input.text) - logging.info(f"cached_embedding: {cached_embedding}") - logging.info(f"type cached_embedding: {type(cached_embedding)}") if cached_embedding: + logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") return EmbeddingOutput( embedding=json.loads(cached_embedding), description="The list of float values representing the text embedding.", ) else: try: + logging.info(f"Generating embedding for: {text_input.text[0:10]}...") embedding = handler.embed(text_input.text) + logging.info(f"Setting embedding for: {text_input.text[0:10]}...") database_object.client.set(text_input.text, json.dumps(embedding)) return EmbeddingOutput( embedding=embedding, From 7a65e377df50aea5792ccfea3a08536e6d6ca1a1 Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 13:12:43 +0100 Subject: [PATCH 22/62] Update ansible --- .gitignore | 2 +- README.md | 2 +- ansible.yaml => ansible/ansible.yaml | 48 ++++++++++++++++++++++++++++ ansible/var.yaml | 4 +++ 4 files changed, 54 insertions(+), 2 deletions(-) rename ansible.yaml => ansible/ansible.yaml (68%) create mode 100644 ansible/var.yaml diff --git a/.gitignore b/.gitignore index 29349ed..1fd5225 100755 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ poetry.lock - +~ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 56f5d75..3151cae 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Personal code to showcase my abilities, such as: - Hadolint binary (when using pre-commit) ## Ansible -Ansible can be used to install prerequisites: `ansible-playbook ansible.yaml --ask-become-pass` +Ansible can be used to install prerequisites: `ansible-playbook ansible/ansible.yaml --ask-become-pass -e @ansible/var.yaml` # How to run diff --git a/ansible.yaml b/ansible/ansible.yaml similarity index 68% rename from ansible.yaml rename to ansible/ansible.yaml index 16b7700..7187751 100644 --- a/ansible.yaml +++ b/ansible/ansible.yaml @@ -40,6 +40,16 @@ - docker-compose - unzip state: present + + - name: Configure Git user name + command: > + git config --global user.name "{{ git_user_name }}" + when: git_user_name is defined + + - name: Configure Git user email + command: > + git config --global user.email "{{ git_user_email }}" + when: git_user_email is defined # Install Pyenv - name: Clone pyenv repository @@ -121,6 +131,44 @@ enabled: yes state: started + - name: Determine PyCharm download URL + set_fact: + pycharm_download_url: >- + https://download.jetbrains.com/python/pycharm-{{ pycharm_edition }}-{{ pycharm_version }}.tar.gz + + - name: Download PyCharm + get_url: + url: "{{ pycharm_download_url }}" + dest: "/tmp/pycharm-{{ pycharm_edition }}-{{ pycharm_version }}.tar.gz" + + - name: Extract PyCharm + unarchive: + src: "/tmp/pycharm-{{ pycharm_edition }}-{{ pycharm_version }}.tar.gz" + dest: "/opt/" + remote_src: yes + + - name: Create symbolic link for PyCharm + file: + src: "/opt/pycharm-{{ pycharm_version }}/bin/pycharm.sh" + dest: "/usr/local/bin/pycharm" + state: link + + - name: Verify PyCharm Professional license setup + debug: + msg: > + "PyCharm Professional installed. Complete license activation manually." + when: pycharm_edition == "professional" + + - name: Display PyCharm installation info + debug: + msg: > + "PyCharm {{ pycharm_edition }} edition installed in /opt/pycharm-{{ pycharm_version }}. + Launch using 'pycharm' command." + + - name: Verify Git installation + command: git --version + register: git_version + handlers: - name: Reload bashrc shell: source ~/.bashrc diff --git a/ansible/var.yaml b/ansible/var.yaml new file mode 100644 index 0000000..057fa56 --- /dev/null +++ b/ansible/var.yaml @@ -0,0 +1,4 @@ +git_user_name: "Your Name" +git_user_email: "youremail@example.com" +pycharm_version: "2023.2.1" # Replace with desired PyCharm version +pycharm_edition: "community" # Change to "community" for Community Edition From f928e39adb514fd5639a8e6574fcb228620524f5 Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 14:30:47 +0100 Subject: [PATCH 23/62] Change FastAPI project structure. --- Dockerfile | 9 +-- app/api/endpoints/__init__.py | 0 app/api/endpoints/embed.py | 72 ++++++++++++++++++++++ app/config/__init__.py | 0 app/config/settings.py | 10 ++++ app/db/__init__.py | 0 app/db/database.py | 40 +++++++++++++ app/main.py | 110 ++-------------------------------- app/schemas/__init__.py | 0 app/schemas/default.py | 15 +++++ docker-compose.yaml | 4 +- pyproject.toml | 1 + tests/test_main.py | 17 ++++-- 13 files changed, 160 insertions(+), 118 deletions(-) create mode 100644 app/api/endpoints/__init__.py create mode 100644 app/api/endpoints/embed.py create mode 100644 app/config/__init__.py create mode 100644 app/config/settings.py create mode 100644 app/db/__init__.py create mode 100644 app/db/database.py create mode 100644 app/schemas/__init__.py create mode 100644 app/schemas/default.py diff --git a/Dockerfile b/Dockerfile index bc76bdc..d47ea15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,13 +20,14 @@ ENV POETRY_NO_INTERACTION=1 \ POETRY_CACHE_DIR='/var/cache/pypoetry' \ POETRY_HOME='/usr/local' -RUN poetry install --no-cache - COPY app /app +RUN poetry install --no-cache + EXPOSE 8080 -ENV PYTHONHTTPSVERIFY=0 -# If there are certificates, add them +# Be careful: this should not run in docker compose environment. It is only used to circumvent corporate proxy for personal projects. +ENV PYTHONHTTPSVERIFY=0 +# If there are certificates, add them. RUN cat /app/certificates.crt >> /usr/local/lib/python3.12/site-packages/certifi/cacert.pem CMD ["poetry", "run", "fastapi", "run", "main.py", "--port", "8080"] \ No newline at end of file diff --git a/app/api/endpoints/__init__.py b/app/api/endpoints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/endpoints/embed.py b/app/api/endpoints/embed.py new file mode 100644 index 0000000..895b16c --- /dev/null +++ b/app/api/endpoints/embed.py @@ -0,0 +1,72 @@ +import json +import logging + +from app.db.database import Redis +from app.model import Handler +from app.schemas.default import TextInput, EmbeddingOutput, SimilarityOutput +from fastapi import HTTPException +from fastapi import APIRouter + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +embed_router = APIRouter() + +database_object = Redis() +handler = Handler() + + +@embed_router.post("/embed") +async def embed_text(text_input: TextInput) -> EmbeddingOutput: + """ + Endpoint to creating embeddings from the input text. + + :param text_input: The text to embed. + :return: The embedding of the input text as JSON. + """ + logger.info(f"Embedding text: {text_input.text}") + + cached_embedding = database_object.get_key(text_input.text) + if cached_embedding: + logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") + return EmbeddingOutput( + embedding=json.loads(cached_embedding), + description="The list of float values representing the text embedding.", + ) + else: + try: + logging.info(f"Generating embedding for: {text_input.text[0:10]}...") + embedding = handler.embed(text_input.text) + logging.info(f"Setting embedding for: {text_input.text[0:10]}...") + database_object.client.set(text_input.text, json.dumps(embedding)) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + except RuntimeError: + HTTPException( + status_code=404, + detail="Something went wrong with creating an embedding.", + ) + + +@embed_router.post("/similarity") +async def calculate_similarity( + text_1: TextInput, text_2: TextInput +) -> SimilarityOutput: + """ + Compute the cosine similarity between two input texts. + + :param text_1: The first text. + :param text_2: The second text. + :return: The similarity score between the two input texts as JSON. + """ + similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) + return SimilarityOutput( + similarity=similarity_score, + description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", + ) diff --git a/app/config/__init__.py b/app/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/config/settings.py b/app/config/settings.py new file mode 100644 index 0000000..547cdd0 --- /dev/null +++ b/app/config/settings.py @@ -0,0 +1,10 @@ +import os +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + DATABASE_URL: str = os.getenv("REDIS_HOST") + DATABASE_PORT: str = os.getenv("REDIS_PORT") + + +settings = Settings() \ No newline at end of file diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/db/database.py b/app/db/database.py new file mode 100644 index 0000000..3b9ebcf --- /dev/null +++ b/app/db/database.py @@ -0,0 +1,40 @@ +import logging +import redis + +from app.config.settings import Settings + +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) +logger = logging.getLogger(__name__) + +settings = Settings() + +class Redis: + # TODO: what to do when there is no Redis database? + def __init__(self): + self.client = redis.Redis( + host=settings.DATABASE_URL, + port=settings.DATABASE_PORT, + decode_responses=True, + ) # Directly return responses in non-binary + logger.info( + f"Redis database connection established for {settings.DATABASE_URL} on port {settings.DATABASE_PORT}" + ) + + def get_key(self, key: str): + """ + Retrieves a key from Redis if it exists. + + :param redis_url: Redis connection URL. + :param key: The key to retrieve. + :return: The value of the key if it exists, otherwise None. + """ + # Check if the key exists + exists = self.client.exists(key) + if exists: + # Retrieve the key's value + value = self.client.get(key) + return value + return None \ No newline at end of file diff --git a/app/main.py b/app/main.py index f5d0a06..457fdd7 100755 --- a/app/main.py +++ b/app/main.py @@ -1,118 +1,18 @@ -import json import logging -from os import environ -import redis -from fastapi import FastAPI, HTTPException -from pydantic import BaseModel +from app.api.endpoints.embed import embed_router +from fastapi import FastAPI -from app.model import Handler logging.basicConfig( - level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) logger = logging.getLogger(__name__) -class TextInput(BaseModel): - text: str - - -class EmbeddingOutput(BaseModel): - embedding: list[float] - description: str | None = None - - -class SimilarityOutput(BaseModel): - similarity: float - description: str | None = None - - -class Redis: - def __init__(self): - self.client = redis.Redis( - host=environ.get("REDIS_HOST"), - port=environ.get("REDIS_PORT"), - decode_responses=True, - ) # Directly return responses in non-binary - logger.info( - f"Redis database connection established for {environ.get("REDIS_HOST")} on port {environ.get("REDIS_PORT")}" - ) - def get_key(self, key: str): - """ - Retrieves a key from Redis if it exists. - - :param redis_url: Redis connection URL. - :param key: The key to retrieve. - :return: The value of the key if it exists, otherwise None. - """ - # Check if the key exists - exists = self.client.exists(key) - if exists: - # Retrieve the key's value - value = self.client.get(key) - return value - return None - app = FastAPI() - -# TODO: what to do when there is no Redis database? -database_object = Redis() - -handler = Handler() - - -@app.post("/embed") -async def embed_text(text_input: TextInput) -> EmbeddingOutput: - """ - Endpoint to creating embeddings from the input text. - - :param text_input: The text to embed. - :return: The embedding of the input text as JSON. - """ - logger.info(f"Embedding text: {text_input.text}") - - cached_embedding = database_object.get_key(text_input.text) - if cached_embedding: - logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") - return EmbeddingOutput( - embedding=json.loads(cached_embedding), - description="The list of float values representing the text embedding.", - ) - else: - try: - logging.info(f"Generating embedding for: {text_input.text[0:10]}...") - embedding = handler.embed(text_input.text) - logging.info(f"Setting embedding for: {text_input.text[0:10]}...") - database_object.client.set(text_input.text, json.dumps(embedding)) - return EmbeddingOutput( - embedding=embedding, - description="The list of float values representing the text embedding.", - ) - except RuntimeError: - HTTPException( - status_code=404, - detail="Something went wrong with creating an embedding.", - ) - - -@app.post("/similarity") -async def calculate_similarity( - text_1: TextInput, text_2: TextInput -) -> SimilarityOutput: - """ - Compute the cosine similarity between two input texts. - - :param text_1: The first text. - :param text_2: The second text. - :return: The similarity score between the two input texts as JSON. - """ - similarity_score = handler.similarity(text_1=text_1.text, text_2=text_2.text) - return SimilarityOutput( - similarity=similarity_score, - description="Cosine similarity indicating semantic similarity. A value close to 1.0 is very similar, close to 0.0 close to -1.0 means little to no similarity, is very dissimilar.", - ) +app.include_router(embed_router) if __name__ == "__main__": diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/schemas/default.py b/app/schemas/default.py new file mode 100644 index 0000000..4a72ee8 --- /dev/null +++ b/app/schemas/default.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel + + +class TextInput(BaseModel): + text: str + + +class EmbeddingOutput(BaseModel): + embedding: list[float] + description: str | None = None + + +class SimilarityOutput(BaseModel): + similarity: float + description: str | None = None \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index a54f1b7..65018b1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,16 +10,14 @@ services: - REDIS_HOST=redis # Redis hostname (will be used in FastAPI to connect to Redis) - REDIS_PORT=6379 # Default Redis port - PYTHONHTTPSVERIFY=0 - # - REQUESTS_CA_BUNDLE="" depends_on: - redis ports: - "8080:8080" - # network_mode: host + redis: image: "redis:latest" container_name: redis_cache ports: - "6379:6379" # Expose Redis on port 6379 - # network_mode: host diff --git a/pyproject.toml b/pyproject.toml index 9d03feb..df8d7ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ pre-commit = "^4.0.1" redis = "^5.2.0" requests = "2.31.0" parameterized = "^0.9.0" +pydantic-settings = "^2.7.1" [tool.poetry.group.dev.dependencies] pytest = "8.3.3" diff --git a/tests/test_main.py b/tests/test_main.py index 7a16a5e..38b4554 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,14 +1,19 @@ import json import unittest +from dotenv import load_dotenv +load_dotenv() # This will load variables from .env -from app.main import app, TextInput +from app.main import app +from app.schemas.default import TextInput from fastapi.testclient import TestClient from parameterized import parameterized from tests.payload_tests import long_string_input from unittest.mock import patch, MagicMock + + class TestEmbedEndpoint(unittest.TestCase): # Arrange @@ -16,7 +21,7 @@ def setUp(self): self.client = TestClient(app) - @patch("app.main.database_object") + @patch("app.api.endpoints.embed.database_object") def test_embed_text_cached(self, mock_database_object): mock_database_object.get_key = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) @@ -41,8 +46,8 @@ def test_embed_text_cached(self, mock_database_object): mock_database_object.get_key.assert_called_once_with(long_string_input['text']) - @patch("app.main.handler") - @patch("app.main.database_object") + @patch("app.api.endpoints.embed.handler") + @patch("app.api.endpoints.embed.database_object") def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): mock_database_object.get_key = MagicMock(return_value=None) mock_handler_object.embed = MagicMock(return_value=[1,2,3]) @@ -76,7 +81,7 @@ def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): "text": "This is a longer sentence, which contains more words and should still work correctly." }, ), ]) - @patch("app.main.database_object") + @patch("app.api.endpoints.embed.database_object") def test_embed_text_parametrized(self, text_input, mock_database_object): mock_database_object.get_key = MagicMock(return_value=None) # Act: Send a POST request to the /embed endpoint @@ -103,7 +108,7 @@ class TestCalculateSimilarity(unittest.TestCase): def setUp(self): self.client = TestClient(app) - @patch("app.main.handler.similarity") # Mock the similarity function + @patch("app.api.endpoints.embed.handler.similarity") # Mock the similarity function def test_calculate_similarity(self, mock_similarity): # Arrange text_1 = TextInput(text="Dog") From a7820c7e975e66cb44df59984fd8bce2043115c0 Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 17:13:02 +0100 Subject: [PATCH 24/62] Database interface for Redis and Pinecone. Code is independent of database choice. --- Dockerfile | 6 +-- README.md | 6 ++- app/api/endpoints/embed.py | 10 ++-- app/config/settings.py | 7 +-- app/db/database_interface.py | 36 +++++++++++++ app/db/database_interface_factory.py | 22 ++++++++ app/db/pinecone_database.py | 50 +++++++++++++++++++ app/db/{database.py => redis_database.py} | 30 ++++++++--- docker-compose-pinecone.yaml | 27 ++++++++++ ...-compose.yaml => docker-compose-redis.yaml | 10 ++-- pyproject.toml | 1 + tests/test_main.py | 10 ++-- 12 files changed, 187 insertions(+), 28 deletions(-) create mode 100644 app/db/database_interface.py create mode 100644 app/db/database_interface_factory.py create mode 100644 app/db/pinecone_database.py rename app/db/{database.py => redis_database.py} (55%) create mode 100644 docker-compose-pinecone.yaml rename docker-compose.yaml => docker-compose-redis.yaml (58%) diff --git a/Dockerfile b/Dockerfile index d47ea15..98392a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,14 @@ ENV POETRY_NO_INTERACTION=1 \ POETRY_CACHE_DIR='/var/cache/pypoetry' \ POETRY_HOME='/usr/local' -COPY app /app - RUN poetry install --no-cache -EXPOSE 8080 +COPY app /app + # Be careful: this should not run in docker compose environment. It is only used to circumvent corporate proxy for personal projects. ENV PYTHONHTTPSVERIFY=0 # If there are certificates, add them. RUN cat /app/certificates.crt >> /usr/local/lib/python3.12/site-packages/certifi/cacert.pem +EXPOSE 8080 CMD ["poetry", "run", "fastapi", "run", "main.py", "--port", "8080"] \ No newline at end of file diff --git a/README.md b/README.md index 3151cae..08ae3f9 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,12 @@ Redis can be used as a backend to cache FastAPI requests. Start the FastAPI and Redis database using: `docker compose up` -To rebuild, run: `docker compose up --build` +To rebuild, run e.g.: `docker compose --env-file .env --file docker-compose-pinecone.yaml up --build` +> **_NOTE:_** **The code works independent on choice of database**. +Two databases are currently supported. +Additional databases could be added by implementing the interface. +The corresponding docker compose files are: `-redis` and `-pinecone`. ## Cloud diff --git a/app/api/endpoints/embed.py b/app/api/endpoints/embed.py index 895b16c..bd15972 100644 --- a/app/api/endpoints/embed.py +++ b/app/api/endpoints/embed.py @@ -1,7 +1,8 @@ import json import logging -from app.db.database import Redis +from app.config.settings import Settings +from app.db.database_interface_factory import DatabaseFactory from app.model import Handler from app.schemas.default import TextInput, EmbeddingOutput, SimilarityOutput from fastapi import HTTPException @@ -16,7 +17,10 @@ embed_router = APIRouter() -database_object = Redis() +settings = Settings() +database_object = DatabaseFactory.get_database(settings.DATABASE_KIND) # Switch between databases easily using an interface. +database_object.connect() + handler = Handler() @@ -30,7 +34,7 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: """ logger.info(f"Embedding text: {text_input.text}") - cached_embedding = database_object.get_key(text_input.text) + cached_embedding = database_object.get(text_input.text) if cached_embedding: logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") return EmbeddingOutput( diff --git a/app/config/settings.py b/app/config/settings.py index 547cdd0..5d6fa6f 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -3,8 +3,9 @@ class Settings(BaseSettings): - DATABASE_URL: str = os.getenv("REDIS_HOST") - DATABASE_PORT: str = os.getenv("REDIS_PORT") - + DATABASE_URL: str = os.getenv("DATABASE_HOST") + DATABASE_PORT: str = os.getenv("DATABASE_PORT") + DATABASE_KIND: str = os.getenv("DATABASE_KIND") + DATABASE_API_KEY: str = os.getenv("DATABASE_API_KEY", "dummy-api-key") settings = Settings() \ No newline at end of file diff --git a/app/db/database_interface.py b/app/db/database_interface.py new file mode 100644 index 0000000..cb7e34b --- /dev/null +++ b/app/db/database_interface.py @@ -0,0 +1,36 @@ +import logging +from abc import ABC, abstractmethod + +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) +logger = logging.getLogger(__name__) + + +class DatabaseInterface(ABC): + @abstractmethod + def connect(self): + pass + + @abstractmethod + def set(self, key, value): + pass + + @abstractmethod + def get(self, key): + pass + + # TODO + # @abstractmethod + # def close(self): + # pass + + # def __enter__(self): + # # Opening the connection + # self.connect() + # return self + + # def __exit__(self, exc_type, exc_val, exc_tb): + # # Closing the connection, even on error + # self.close() diff --git a/app/db/database_interface_factory.py b/app/db/database_interface_factory.py new file mode 100644 index 0000000..7898459 --- /dev/null +++ b/app/db/database_interface_factory.py @@ -0,0 +1,22 @@ +import logging + +from app.db.redis_database import RedisDatabase +from app.db.pinecone_database import PineconeDatabase + + +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) +logger = logging.getLogger(__name__) + +class DatabaseFactory: + + @staticmethod + def get_database(db_type, **kwargs): + if db_type == 'redis': + return RedisDatabase(**kwargs) + elif db_type == 'pinecone': + return PineconeDatabase(**kwargs) + else: + raise ValueError(f"Unsupported database type: {db_type}") diff --git a/app/db/pinecone_database.py b/app/db/pinecone_database.py new file mode 100644 index 0000000..6d0104b --- /dev/null +++ b/app/db/pinecone_database.py @@ -0,0 +1,50 @@ +import logging +from pinecone import Pinecone +from app.config.settings import Settings +from app.db.database_interface import DatabaseInterface +from typing import List + +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) +logger = logging.getLogger(__name__) + +settings = Settings() + +class PineconeDatabase(DatabaseInterface): + def __init__(self): + self.DATABASE_URL = settings.DATABASE_URL + self.PORT = settings.DATABASE_PORT + self.API_KEY = settings.DATABASE_API_KEY + self.client = None + + + def connect(self): + try: + self.client = Pinecone(api_key=self.API_KEY, host=self.DATABASE_URL) + logger.info("Pinecone initialized successfully.") + except Exception as e: + logger.error(f"Failed to initialize Pinecone: {e}") + + + + def get(self, key: str): + """ + Retrieves a key from Redis if it exists. + + :param redis_url: Redis connection URL. + :param key: The key to retrieve. + :return: The value of the key if it exists, otherwise None. + """ + index = self.client.Index(host=self.DATABASE_URL) + # Check if the key exists + fetch_response = index.fetch(key) + if fetch_response: + return fetch_response + return None + + + def set(self, key: str, value: List[float]): + upsert_response = self.client.Index.upsert(vectors=[(key, value)]) + logging.info(f"The key {key[0:10]}... was upserted with response {upsert_response}") \ No newline at end of file diff --git a/app/db/database.py b/app/db/redis_database.py similarity index 55% rename from app/db/database.py rename to app/db/redis_database.py index 3b9ebcf..dd8f195 100644 --- a/app/db/database.py +++ b/app/db/redis_database.py @@ -1,7 +1,10 @@ import logging import redis +from typing import List from app.config.settings import Settings +from app.db.database_interface import DatabaseInterface + logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) @@ -11,19 +14,30 @@ settings = Settings() -class Redis: - # TODO: what to do when there is no Redis database? +class RedisDatabase(DatabaseInterface): def __init__(self): + self.host = settings.DATABASE_URL + self.port = settings.DATABASE_PORT + self.client = None + + def connect(self): self.client = redis.Redis( - host=settings.DATABASE_URL, - port=settings.DATABASE_PORT, + host=self.host, + port=self.port, decode_responses=True, ) # Directly return responses in non-binary - logger.info( - f"Redis database connection established for {settings.DATABASE_URL} on port {settings.DATABASE_PORT}" - ) + if not self.client.ping(): + raise ConnectionError("Could not connect to Redis!") + else: + logger.info( + f"Redis database connection established for {settings.DATABASE_URL} on port {settings.DATABASE_PORT}" + ) + return super().connect() + + def set(self, key: str, value: List[float]): + self.client.set(key=key, value=value) - def get_key(self, key: str): + def get(self, key: str): """ Retrieves a key from Redis if it exists. diff --git a/docker-compose-pinecone.yaml b/docker-compose-pinecone.yaml new file mode 100644 index 0000000..d6d85bb --- /dev/null +++ b/docker-compose-pinecone.yaml @@ -0,0 +1,27 @@ +version: "3.8" + +services: + fastapi: + container_name: fastapi + build: + context: . + dockerfile: Dockerfile + environment: + - DATABASE_KIND=pinecone + - DATABASE_HOST=localhost + - DATABASE_API_KEY="dummy-api-key" + - DATABASE_PORT=5080 + - PYTHONHTTPSVERIFY=0 + depends_on: + - pinecone + ports: + - "8080:8080" + + pinecone: + image: ghcr.io/pinecone-io/pinecone-local:latest + environment: + DATABASE_PORT: 5080 + DATABASE_HOST: localhost + ports: + - "5080-6000:5080-6000" + platform: linux/amd64 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose-redis.yaml similarity index 58% rename from docker-compose.yaml rename to docker-compose-redis.yaml index 65018b1..fcf3dc2 100644 --- a/docker-compose.yaml +++ b/docker-compose-redis.yaml @@ -7,17 +7,17 @@ services: context: . dockerfile: Dockerfile environment: - - REDIS_HOST=redis # Redis hostname (will be used in FastAPI to connect to Redis) - - REDIS_PORT=6379 # Default Redis port + - DATABASE_KIND=redis + - DATABASE_HOST=redis + - DATABASE_PORT=6379 - PYTHONHTTPSVERIFY=0 depends_on: - - redis + - redis ports: - "8080:8080" - redis: image: "redis:latest" container_name: redis_cache ports: - - "6379:6379" # Expose Redis on port 6379 + - "6379:6379" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index df8d7ed..7d13619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ redis = "^5.2.0" requests = "2.31.0" parameterized = "^0.9.0" pydantic-settings = "^2.7.1" +pinecone = "^5.4.2" [tool.poetry.group.dev.dependencies] pytest = "8.3.3" diff --git a/tests/test_main.py b/tests/test_main.py index 38b4554..fe1a541 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -23,7 +23,7 @@ def setUp(self): @patch("app.api.endpoints.embed.database_object") def test_embed_text_cached(self, mock_database_object): - mock_database_object.get_key = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) + mock_database_object.get = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) # Act: Send a POST request to the /embed endpoint response = self.client.post("/embed", json=long_string_input) @@ -43,13 +43,13 @@ def test_embed_text_cached(self, mock_database_object): # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list - mock_database_object.get_key.assert_called_once_with(long_string_input['text']) + mock_database_object.get.assert_called_once_with(long_string_input['text']) @patch("app.api.endpoints.embed.handler") @patch("app.api.endpoints.embed.database_object") def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): - mock_database_object.get_key = MagicMock(return_value=None) + mock_database_object.get = MagicMock(return_value=None) mock_handler_object.embed = MagicMock(return_value=[1,2,3]) # Act: Send a POST request to the /embed endpoint @@ -70,7 +70,7 @@ def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list - mock_database_object.get_key.assert_called_once_with(long_string_input['text']) + mock_database_object.get.assert_called_once_with(long_string_input['text']) mock_handler_object.embed.assert_called_once_with(long_string_input['text']) @@ -83,7 +83,7 @@ def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): ]) @patch("app.api.endpoints.embed.database_object") def test_embed_text_parametrized(self, text_input, mock_database_object): - mock_database_object.get_key = MagicMock(return_value=None) + mock_database_object.get = MagicMock(return_value=None) # Act: Send a POST request to the /embed endpoint response = self.client.post("/embed", json=text_input) From 0439893d5c00b5a98fce848dd0ae88111a09bff2 Mon Sep 17 00:00:00 2001 From: romusters Date: Mon, 27 Jan 2025 17:14:20 +0100 Subject: [PATCH 25/62] Example env files --- .env.example | 2 ++ tests/.env.example | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 .env.example create mode 100644 tests/.env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..84a420d --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +DATABASE_URL=... +DATABASE_API_KEY=... \ No newline at end of file diff --git a/tests/.env.example b/tests/.env.example new file mode 100644 index 0000000..0f788f9 --- /dev/null +++ b/tests/.env.example @@ -0,0 +1,3 @@ +DATABASE_HOST="test" +DATABASE_PORT="test" +DATABASE_API_KEY="test" \ No newline at end of file From e56322f4737cfe2fba8e720f18db0bb0f1073811 Mon Sep 17 00:00:00 2001 From: romusters Date: Tue, 28 Jan 2025 10:16:56 +0100 Subject: [PATCH 26/62] Pinecone database support is limited. Might be proxy. --- app/db/pinecone_database.py | 27 ++++++++++++++++++++++++--- docker-compose-pinecone.yaml | 6 ++---- docker-compose-redis.yaml | 2 -- pyproject.toml | 2 +- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/app/db/pinecone_database.py b/app/db/pinecone_database.py index 6d0104b..2117069 100644 --- a/app/db/pinecone_database.py +++ b/app/db/pinecone_database.py @@ -1,9 +1,12 @@ import logging -from pinecone import Pinecone +import time + from app.config.settings import Settings from app.db.database_interface import DatabaseInterface +from pinecone.grpc import PineconeGRPC as Pinecone from typing import List + logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format @@ -14,19 +17,34 @@ class PineconeDatabase(DatabaseInterface): def __init__(self): - self.DATABASE_URL = settings.DATABASE_URL + self.DATABASE_URL = f"{settings.DATABASE_URL}:{settings.DATABASE_PORT}" self.PORT = settings.DATABASE_PORT self.API_KEY = settings.DATABASE_API_KEY self.client = None + self.index_name = 'default' def connect(self): try: self.client = Pinecone(api_key=self.API_KEY, host=self.DATABASE_URL) + logger.info(f"Pinecone api_key: {self.API_KEY}, host: {self.DATABASE_URL} .") logger.info("Pinecone initialized successfully.") except Exception as e: logger.error(f"Failed to initialize Pinecone: {e}") + def check_index(self, index_name: str = 'default') -> None: + + if not self.client.has_index(index_name): + self.client.create_index( + name=index_name, + dimension=2, + metric="cosine", + ) + + # Wait for the index to be ready + while not self.client.describe_index(self.index_name).status['ready']: + time.sleep(1) + def get(self, key: str): @@ -37,6 +55,7 @@ def get(self, key: str): :param key: The key to retrieve. :return: The value of the key if it exists, otherwise None. """ + self.check_index() index = self.client.Index(host=self.DATABASE_URL) # Check if the key exists fetch_response = index.fetch(key) @@ -46,5 +65,7 @@ def get(self, key: str): def set(self, key: str, value: List[float]): - upsert_response = self.client.Index.upsert(vectors=[(key, value)]) + self.check_index() + index = self.client.Index(host=self.DATABASE_URL) + upsert_response = index.upsert(vectors=[(key, value)]) logging.info(f"The key {key[0:10]}... was upserted with response {upsert_response}") \ No newline at end of file diff --git a/docker-compose-pinecone.yaml b/docker-compose-pinecone.yaml index d6d85bb..724983f 100644 --- a/docker-compose-pinecone.yaml +++ b/docker-compose-pinecone.yaml @@ -1,5 +1,3 @@ -version: "3.8" - services: fastapi: container_name: fastapi @@ -8,8 +6,8 @@ services: dockerfile: Dockerfile environment: - DATABASE_KIND=pinecone - - DATABASE_HOST=localhost - - DATABASE_API_KEY="dummy-api-key" + - DATABASE_HOST=http://localhost + - DATABASE_API_KEY="pclocal" - DATABASE_PORT=5080 - PYTHONHTTPSVERIFY=0 depends_on: diff --git a/docker-compose-redis.yaml b/docker-compose-redis.yaml index fcf3dc2..0130458 100644 --- a/docker-compose-redis.yaml +++ b/docker-compose-redis.yaml @@ -1,5 +1,3 @@ -version: "3.8" - services: fastapi: container_name: fastapi diff --git a/pyproject.toml b/pyproject.toml index 7d13619..8f31680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ redis = "^5.2.0" requests = "2.31.0" parameterized = "^0.9.0" pydantic-settings = "^2.7.1" -pinecone = "^5.4.2" +pinecone = {extras = ["grpc"], version = "^5.4.2"} [tool.poetry.group.dev.dependencies] pytest = "8.3.3" From 4dc159c539512ab7b7520fd7c9c220497c37062d Mon Sep 17 00:00:00 2001 From: romusters Date: Tue, 28 Jan 2025 10:18:51 +0100 Subject: [PATCH 27/62] Remove two todos :D --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 08ae3f9..070bcc3 100644 --- a/README.md +++ b/README.md @@ -91,10 +91,8 @@ Although unadvised, setting `PYTHONHTTPSVERIFY` to `false` circumpasses SSL cert I do have the correct role set for the principal. - make Redis asynchronous -- create interface for Redis and Pinecone database - devops pipeline - integration test - create package from model and handler so that I can use it in Docker image -- caching requests for FastAPI using external database - try [uv](https://github.com/astral-sh/uv) instead of Poetry. From 1d4f4c1ab731fac06ca4f967b5878773bac0f704 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 28 Jan 2025 12:01:38 +0100 Subject: [PATCH 28/62] Update ansible config --- ansible/ansible.yaml | 128 +++++++++++++++++++++++++++++++------------ ansible/git.yaml | 14 +++++ 2 files changed, 106 insertions(+), 36 deletions(-) create mode 100644 ansible/git.yaml diff --git a/ansible/ansible.yaml b/ansible/ansible.yaml index 7187751..d4cf070 100644 --- a/ansible/ansible.yaml +++ b/ansible/ansible.yaml @@ -36,20 +36,13 @@ - libffi-dev - liblzma-dev - python3-openssl - - docker.io - - docker-compose - unzip state: present - - name: Configure Git user name - command: > - git config --global user.name "{{ git_user_name }}" - when: git_user_name is defined - - - name: Configure Git user email - command: > - git config --global user.email "{{ git_user_email }}" - when: git_user_email is defined + - name: Verify Git installation + command: git --version + register: git_version + # Install Pyenv - name: Clone pyenv repository @@ -73,7 +66,7 @@ - name: Install Python {{ python_version }} using pyenv shell: | source ~/.bashrc && \ - pyenv install {{ python_version }} && \ + pyenv install -s {{ python_version }} && \ pyenv global {{ python_version }} args: executable: /bin/bash @@ -84,15 +77,31 @@ curl -sSL https://install.python-poetry.org | python3 - args: executable: /bin/bash - - # Install Terraform + - name: Download Terraform - shell: | - curl -fsSL https://releases.hashicorp.com/terraform/1.5.0/terraform_1.5.0_linux_amd64.zip -o terraform.zip - unzip terraform.zip -d /usr/local/bin/ - rm terraform.zip - args: - executable: /bin/bash + get_url: + url: https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_linux_amd64.zip + dest: /tmp/terraform.zip + + - name: Unzip Terraform + unarchive: + src: /tmp/terraform.zip + dest: /usr/local/bin/ + remote_src: yes + + - name: Cleanup Terraform zip file + file: + path: /tmp/terraform.zip + state: absent + + - name: Verify Terraform installation + shell: terraform -version + register: terraform_version_output + changed_when: false + + - name: Show Terraform version + debug: + msg: "{{ terraform_version_output.stdout }}" # Install Azure CLI - name: Install Azure CLI @@ -118,18 +127,29 @@ args: executable: /bin/bash - # Verify Docker Installation - - name: Add user to Docker group - user: - name: "{{ ansible_user_id }}" - groups: docker - append: yes + - name: Add Docker GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Add Docker repository + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release | lower }} stable" + state: present + + - name: Install Docker + apt: + name: docker-ce + state: latest - - name: Enable Docker service - systemd: - name: docker - enabled: yes - state: started + - name: Verify Docker installation + command: docker --version + register: docker_version_output + changed_when: false + + - name: Show Docker version + debug: + msg: "{{ docker_version_output.stdout }}" - name: Determine PyCharm download URL set_fact: @@ -149,7 +169,7 @@ - name: Create symbolic link for PyCharm file: - src: "/opt/pycharm-{{ pycharm_version }}/bin/pycharm.sh" + src: "/opt/pycharm-{{ pycharm_edition }}-{{ pycharm_version }}/bin/pycharm.sh" dest: "/usr/local/bin/pycharm" state: link @@ -164,13 +184,49 @@ msg: > "PyCharm {{ pycharm_edition }} edition installed in /opt/pycharm-{{ pycharm_version }}. Launch using 'pycharm' command." + + - name: Create a desktop shortcut for PyCharm (Optional) + copy: + dest: "/usr/share/applications/pycharm.desktop" + content: | + [Desktop Entry] + Version=1.0 + Name=PyCharm Community Edition + Comment=The Python IDE for Professional Developers + Exec=/opt/pycharm-community-{{ pycharm_version }}/bin/pycharm.sh %f + Icon=/opt/pycharm-community-{{ pycharm_version }}/bin/pycharm.png + Terminal=false + Type=Application + Categories=Development;IDE; + StartupWMClass=jetbrains-pycharm + mode: '0644' + + - name: Verify PyCharm installation + command: pycharm --version + register: pycharm_version_output + changed_when: false + + - name: Create src directory + file: + path: ~/virtualenvs + state: directory + mode: '0755' + + - name: Create bin directory + file: + path: ~/repositories + state: directory + mode: '0755' + + - name: Create config directory + file: + path: ~/data + state: directory + mode: '0755' - - name: Verify Git installation - command: git --version - register: git_version handlers: - name: Reload bashrc shell: source ~/.bashrc args: - executable: /bin/bash \ No newline at end of file + executable: /bin/bash diff --git a/ansible/git.yaml b/ansible/git.yaml new file mode 100644 index 0000000..aee4954 --- /dev/null +++ b/ansible/git.yaml @@ -0,0 +1,14 @@ +- name: Configure Git + hosts: localhost + tasks: + - name: Set Git username + git_config: + name: user.name + value: "{{ git_user_name }}" + scope: global + + - name: Set Git email + git_config: + name: user.email + value: "{{ git_user_email }}" + scope: global From d8bc916baaa6f73c0fd8f278d4112f88fce1924e Mon Sep 17 00:00:00 2001 From: romusters Date: Tue, 28 Jan 2025 17:02:38 +0100 Subject: [PATCH 29/62] Add provision of Google Cloud Run using Docker container and Artifact Registry storing the Fast API app Docker image. --- .gitignore | 2 +- infra/README.md | 50 +++++----------- infra/azure/README.md | 50 ++++++++++++++++ infra/{ => azure}/containers.tf | 0 infra/{ => azure}/functions.tf | 0 infra/{ => azure}/main.tf | 0 infra/{ => azure}/storage.tf | 0 infra/gcp/main.tf | 103 ++++++++++++++++++++++++++++++++ 8 files changed, 168 insertions(+), 37 deletions(-) create mode 100644 infra/azure/README.md rename infra/{ => azure}/containers.tf (100%) rename infra/{ => azure}/functions.tf (100%) rename infra/{ => azure}/main.tf (100%) rename infra/{ => azure}/storage.tf (100%) create mode 100644 infra/gcp/main.tf diff --git a/.gitignore b/.gitignore index 1fd5225..3240620 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ poetry.lock ~ - +.terraform # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/infra/README.md b/infra/README.md index dc5076c..0a4b444 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,50 +1,28 @@ -# Introduction +# Google -Terraform is infrastructure as code tool. The infrastructure we are making here contains a Docker registry and a Function app amongst other resources. -The Function App uses a Docker container which needs to be available first, so we build and push it in the next section. We might also choose to create a DevOps pipeline to do this for us. -Build and push Docker container so that it can be used in function app. +To deploy the infra on Google, first authenticate: -## How to run +`gcloud auth application-default login` -### Docker +Set the project id: -Build: +`gcloud config set project ` -`docker build -f Dockerfile -t embedding_service:latest . ` +Enable gcloud service 'artifactregistry.googleapis.com': +`gcloud services enable artifactregistry.googleapis.com` -Login: +`terraform init` -`az acr login --name EmbeddingContainerRegistry ` +`terraform plan` -Tag: - -`docker tag embedding_service:latest embeddingcontainerregistry.azurecr.io/embedding_service:latest` - -Push: - -`docker push embeddingcontainerregistry.azurecr.io/embedding_service:latest` - -### Terraform - -Login: - -`az login` - -Navigate to the infra folder: - -`cd infra` - -Init: - -`terraform init` - -Plan: +`terraform apply` -`terraform plan` +## Push Docker container -Apply: +`gcloud auth configure-docker europe-west4-docker.pkg.dev` -`terraform apply` +## Github Actions +The infrastructure can also be deployed using Github Actions \ No newline at end of file diff --git a/infra/azure/README.md b/infra/azure/README.md new file mode 100644 index 0000000..dc5076c --- /dev/null +++ b/infra/azure/README.md @@ -0,0 +1,50 @@ +# Introduction + +Terraform is infrastructure as code tool. The infrastructure we are making here contains a Docker registry and a Function app amongst other resources. +The Function App uses a Docker container which needs to be available first, so we build and push it in the next section. We might also choose to create a DevOps pipeline to do this for us. +Build and push Docker container so that it can be used in function app. + +## How to run + +### Docker + +Build: + +`docker build -f Dockerfile -t embedding_service:latest . ` + + +Login: + +`az acr login --name EmbeddingContainerRegistry ` + +Tag: + +`docker tag embedding_service:latest embeddingcontainerregistry.azurecr.io/embedding_service:latest` + +Push: + +`docker push embeddingcontainerregistry.azurecr.io/embedding_service:latest` + +### Terraform + +Login: + +`az login` + +Navigate to the infra folder: + +`cd infra` + +Init: + +`terraform init` + +Plan: + +`terraform plan` + + +Apply: + +`terraform apply` + diff --git a/infra/containers.tf b/infra/azure/containers.tf similarity index 100% rename from infra/containers.tf rename to infra/azure/containers.tf diff --git a/infra/functions.tf b/infra/azure/functions.tf similarity index 100% rename from infra/functions.tf rename to infra/azure/functions.tf diff --git a/infra/main.tf b/infra/azure/main.tf similarity index 100% rename from infra/main.tf rename to infra/azure/main.tf diff --git a/infra/storage.tf b/infra/azure/storage.tf similarity index 100% rename from infra/storage.tf rename to infra/azure/storage.tf diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf new file mode 100644 index 0000000..8b196bb --- /dev/null +++ b/infra/gcp/main.tf @@ -0,0 +1,103 @@ +variable "project_id" { + description = "project id" + type = string + default = "fastapi-449213" +} + +variable "project" { + description = "project" + type = string + default = "fastapi" +} + + +variable "region" { + description = "The GCP region" + type = string + default = "europe-west4" +} + + +provider "google" { + project = var.project_id + region = var.region +} + + +# Enable necessary APIs +resource "google_project_service" "container_registry" { + for_each = toset([ + "container.googleapis.com", + "run.googleapis.com", + "artifactregistry.googleapis.com" + ]) + project = var.project_id + service = each.key +} + +resource "google_project_service" "artifact_registry_api" { + service = "artifactregistry.googleapis.com" + project = var.project_id +} + +# Container Registry: Images are stored in Artifact Registry +resource "google_artifact_registry_repository" "container_registry" { + repository_id = "fastapi-docker-repo" + format = "DOCKER" + location = var.region + description = "Docker repository for FastAPI images" +} + +# IAM Binding for Artifact Registry +resource "google_artifact_registry_repository_iam_binding" "artifact_registry_binding" { + repository = google_artifact_registry_repository.container_registry.name + role = "roles/artifactregistry.writer" + members = ["serviceAccount:${google_service_account.cloud_run_service_account.email}"] +} + +# Service Account for Cloud Run +resource "google_service_account" "cloud_run_service_account" { + account_id = "cloud-run-service-account" + display_name = "Cloud Run Service Account" +} + +# Grant necessary roles to the Service Account +resource "google_project_iam_binding" "cloud_run_iam" { + project = var.project_id + role = "roles/run.admin" + members = ["serviceAccount:${google_service_account.cloud_run_service_account.email}"] +} + +# Deploy FastAPI Docker app to Cloud Run +resource "google_cloud_run_service" "fastapi_service" { + name = "fastapi-service" + location = var.region + + template { + spec { + containers { + image = "${google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.container_registry.name}/fastapi:latest" + resources { + limits = { + memory = "512Mi" + cpu = "1" + } + } + } + service_account_name = google_service_account.cloud_run_service_account.email + } + } + + traffic { + percent = 100 + latest_revision = true + } +} + +# Grant permissions to Cloud Run Invoker +resource "google_cloud_run_service_iam_binding" "invoker_permission" { + service = google_cloud_run_service.fastapi_service.name + location = var.region + role = "roles/run.invoker" + members = ["allUsers"] # Allows public access; modify as needed +} \ No newline at end of file From d08920f850ac235f8cbb27ce123580e4bfa1e310 Mon Sep 17 00:00:00 2001 From: romusters Date: Tue, 28 Jan 2025 17:23:12 +0100 Subject: [PATCH 30/62] Make caching an option. Increase memory of Cloud run instance. --- app/api/endpoints/embed.py | 54 ++++++++++++++++++++++---------------- app/config/settings.py | 7 ++--- infra/gcp/main.tf | 23 +++++++++++++++- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/app/api/endpoints/embed.py b/app/api/endpoints/embed.py index bd15972..fa3172a 100644 --- a/app/api/endpoints/embed.py +++ b/app/api/endpoints/embed.py @@ -18,8 +18,10 @@ embed_router = APIRouter() settings = Settings() -database_object = DatabaseFactory.get_database(settings.DATABASE_KIND) # Switch between databases easily using an interface. -database_object.connect() + +if settings.CACHE_ENABLED: + database_object = DatabaseFactory.get_database(settings.DATABASE_KIND) # Switch between databases easily using an interface. + database_object.connect() handler = Handler() @@ -33,30 +35,36 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :return: The embedding of the input text as JSON. """ logger.info(f"Embedding text: {text_input.text}") - - cached_embedding = database_object.get(text_input.text) - if cached_embedding: - logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") - return EmbeddingOutput( - embedding=json.loads(cached_embedding), - description="The list of float values representing the text embedding.", - ) - else: - try: - logging.info(f"Generating embedding for: {text_input.text[0:10]}...") - embedding = handler.embed(text_input.text) - logging.info(f"Setting embedding for: {text_input.text[0:10]}...") - database_object.client.set(text_input.text, json.dumps(embedding)) + if settings.CACHE_ENABLED: + cached_embedding = database_object.get(text_input.text) + if cached_embedding: + logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") return EmbeddingOutput( - embedding=embedding, + embedding=json.loads(cached_embedding), description="The list of float values representing the text embedding.", ) - except RuntimeError: - HTTPException( - status_code=404, - detail="Something went wrong with creating an embedding.", - ) - + else: + try: + logging.info(f"Generating embedding for: {text_input.text[0:10]}...") + embedding = handler.embed(text_input.text) + logging.info(f"Setting embedding for: {text_input.text[0:10]}...") + database_object.client.set(text_input.text, json.dumps(embedding)) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + except RuntimeError: + HTTPException( + status_code=404, + detail="Something went wrong with creating an embedding.", + ) + else: + logging.info(f"Generating embedding for: {text_input.text[0:10]}...") + embedding = handler.embed(text_input.text) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) @embed_router.post("/similarity") async def calculate_similarity( diff --git a/app/config/settings.py b/app/config/settings.py index 5d6fa6f..1f8ac34 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -3,9 +3,10 @@ class Settings(BaseSettings): - DATABASE_URL: str = os.getenv("DATABASE_HOST") - DATABASE_PORT: str = os.getenv("DATABASE_PORT") - DATABASE_KIND: str = os.getenv("DATABASE_KIND") + DATABASE_URL: str = os.getenv("DATABASE_HOST", "dummy") + DATABASE_PORT: str = os.getenv("DATABASE_PORT", "dummy") + DATABASE_KIND: str = os.getenv("DATABASE_KIND", "dummy") DATABASE_API_KEY: str = os.getenv("DATABASE_API_KEY", "dummy-api-key") + CACHE_ENABLED: bool = os.getenv("CACHE_ENABLED", "dummy") settings = Settings() \ No newline at end of file diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 8b196bb..e861017 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -76,13 +76,34 @@ resource "google_cloud_run_service" "fastapi_service" { template { spec { containers { + image = "${google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.container_registry.name}/fastapi:latest" resources { limits = { - memory = "512Mi" + memory = "2048Mi" cpu = "1" } } + env { + name = "DATABASE_URL" + value = "value1" + } + env { + name = "DATABASE_PORT" + value = "value2" + } + env { + name = "DATABASE_KIND" + value = "value3" + } + env { + name = "DATABASE_API_KEY" + value = "value3" + } + env { + name = "CACHE_ENABLED" + value = false + } } service_account_name = google_service_account.cloud_run_service_account.email } From 8ba69b6749aedd968aa93108702b9f16e78e1a9d Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 29 Jan 2025 10:53:20 +0100 Subject: [PATCH 31/62] Update ansible --- README.md | 24 ++++++++++++------------ ansible/ansible.yaml | 34 +++++++++++++++++++++++++++++++--- ansible/git.yaml | 1 + 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 070bcc3..45cb9af 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,16 @@ Personal code to showcase my abilities, such as: 1. Software Engineering in Python: - 2. Code (e.g. design patterns) - - 3. Tests (unit and integration) - - 4. Project tooling (pre-commit, Ruff) - + 2. Code (e.g. design patterns) + + 3. Tests (unit and integration) + + 4. Project tooling (pre-commit, Ruff) + 2. Docker 3. Terraform 4. Cloud Engineering -5. DevOps +5. DevOps # Prerequisites @@ -66,10 +66,10 @@ Start the FastAPI and Redis database using: `docker compose up` To rebuild, run e.g.: `docker compose --env-file .env --file docker-compose-pinecone.yaml up --build` -> **_NOTE:_** **The code works independent on choice of database**. -Two databases are currently supported. -Additional databases could be added by implementing the interface. -The corresponding docker compose files are: `-redis` and `-pinecone`. +> **_NOTE:_** **The code works independent on choice of database**. +Two databases are currently supported. +Additional databases could be added by implementing the interface. +The corresponding docker compose files are: `-redis` and `-pinecone`. ## Cloud @@ -79,7 +79,7 @@ The corresponding docker compose files are: `-redis` and `-pinecone`. ### Azure -Use the function app to +Use the function app to Go to e.g.: http://*.azurewebsites.com. # Remarks diff --git a/ansible/ansible.yaml b/ansible/ansible.yaml index d4cf070..3009c0d 100644 --- a/ansible/ansible.yaml +++ b/ansible/ansible.yaml @@ -137,16 +137,38 @@ repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release | lower }} stable" state: present - - name: Install Docker + - name: install docker apt: - name: docker-ce + name: "{{item}}" state: latest + update_cache: yes + loop: + - docker-ce + - docker-ce-cli + - containerd.io + + - name: check docker is active + service: + name: docker + state: started + enabled: yes - name: Verify Docker installation command: docker --version register: docker_version_output changed_when: false + - name: Ensure group "docker" exists + ansible.builtin.group: + name: docker + state: present + + - name: Add user to Docker group + user: + name: "blpasd" + groups: docker + append: true + - name: Show Docker version debug: msg: "{{ docker_version_output.stdout }}" @@ -224,7 +246,13 @@ state: directory mode: '0755' - + - name: Reboot for Docker to work + ansible.builtin.debug: + msg: + - "Reboot for Docker to work" + - "https://docs.docker.com/engine/install/ubuntu/" + - "https://stackoverflow.com/questions/75713844/how-to-resolve-failed-to-create-nat-chain-docker-as-reboot-not-working" + - " sudo journalctl -u docker" handlers: - name: Reload bashrc shell: source ~/.bashrc diff --git a/ansible/git.yaml b/ansible/git.yaml index aee4954..c6fe0bc 100644 --- a/ansible/git.yaml +++ b/ansible/git.yaml @@ -12,3 +12,4 @@ name: user.email value: "{{ git_user_email }}" scope: global + From 311f52aa0d9f0403f9cfa64ebff0541b6b3e46e5 Mon Sep 17 00:00:00 2001 From: R Date: Wed, 29 Jan 2025 12:16:47 +0100 Subject: [PATCH 32/62] Update ansible again --- ansible/ansible.yaml | 112 +++++++++++++++++++++++++------------------ ansible/var.yaml | 1 + 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/ansible/ansible.yaml b/ansible/ansible.yaml index 3009c0d..8af176a 100644 --- a/ansible/ansible.yaml +++ b/ansible/ansible.yaml @@ -110,7 +110,7 @@ args: executable: /bin/bash - # Install Google Cloud CLI + # TODO: install using apt_key - name: Install Google Cloud CLI shell: | echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list @@ -127,52 +127,6 @@ args: executable: /bin/bash - - name: Add Docker GPG key - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - - - name: Add Docker repository - apt_repository: - repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release | lower }} stable" - state: present - - - name: install docker - apt: - name: "{{item}}" - state: latest - update_cache: yes - loop: - - docker-ce - - docker-ce-cli - - containerd.io - - - name: check docker is active - service: - name: docker - state: started - enabled: yes - - - name: Verify Docker installation - command: docker --version - register: docker_version_output - changed_when: false - - - name: Ensure group "docker" exists - ansible.builtin.group: - name: docker - state: present - - - name: Add user to Docker group - user: - name: "blpasd" - groups: docker - append: true - - - name: Show Docker version - debug: - msg: "{{ docker_version_output.stdout }}" - - name: Determine PyCharm download URL set_fact: pycharm_download_url: >- @@ -246,6 +200,69 @@ state: directory mode: '0755' + - name: Install GitHub CLI + apt: + name: gh + state: latest + + - name: Verify GitHub CLI installation + command: gh --version + register: gh_version_output + changed_when: false + + - name: Show GitHub CLI version + debug: + msg: "GitHub CLI version: {{ gh_version_output.stdout }}" + + - name: Add Docker GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Add Docker repository + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release | lower }} stable" + state: present + + - name: install docker + apt: + name: "{{item}}" + state: latest + update_cache: yes + loop: + - docker-ce + - docker-ce-cli + - containerd.io + + - name: Verify Docker installation + command: docker --version + register: docker_version_output + changed_when: false + + - name: Show Docker version + debug: + msg: "{{ docker_version_output.stdout }}" + + - name: Ensure group "docker" exists + ansible.builtin.group: + name: docker + state: present + + - name: Add user to Docker group + user: + name: "blpasd" + groups: docker + append: true + + + - name: check docker is active + service: + name: docker + state: started + enabled: yes + + + - name: Reboot for Docker to work ansible.builtin.debug: msg: @@ -253,6 +270,7 @@ - "https://docs.docker.com/engine/install/ubuntu/" - "https://stackoverflow.com/questions/75713844/how-to-resolve-failed-to-create-nat-chain-docker-as-reboot-not-working" - " sudo journalctl -u docker" + handlers: - name: Reload bashrc shell: source ~/.bashrc diff --git a/ansible/var.yaml b/ansible/var.yaml index 057fa56..c22bc7e 100644 --- a/ansible/var.yaml +++ b/ansible/var.yaml @@ -2,3 +2,4 @@ git_user_name: "Your Name" git_user_email: "youremail@example.com" pycharm_version: "2023.2.1" # Replace with desired PyCharm version pycharm_edition: "community" # Change to "community" for Community Edition +terraform_version: "1.10.5" From 6861f4816bccf53bb9b6b72a0a527f73e3d11436 Mon Sep 17 00:00:00 2001 From: romusters Date: Wed, 29 Jan 2025 14:55:21 +0100 Subject: [PATCH 33/62] GCP Workload Identity Federation and Github configuration done. Update the --- ...-pipeline.yaml => azure-pipeline.yaml.bak} | 0 .github/workflows/docker.yaml | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+) rename .github/workflows/{azure-pipeline.yaml => azure-pipeline.yaml.bak} (100%) create mode 100644 .github/workflows/docker.yaml diff --git a/.github/workflows/azure-pipeline.yaml b/.github/workflows/azure-pipeline.yaml.bak similarity index 100% rename from .github/workflows/azure-pipeline.yaml rename to .github/workflows/azure-pipeline.yaml.bak diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..c5c621b --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,43 @@ +name: Build and Push to Google Cloud + +on: + push: + branches: + - main + +env: + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + ARTIFACT_REPO_NAME: europe-west4-docker.pkg.dev + REPO_NAME: fastapi-docker-repo + IMAGE_NAME: fastapi + TAG: latest + +jobs: + build-and-push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Authenticate with GCP + uses: 'google-github-actions/auth@v2' + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + + - name: Configure Docker to use Google Artifact Registry + run: | + gcloud auth configure-docker $ARTIFACT_REPO_NAME + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and Push Docker image + run: | + docker buildx build \ + --push \ + --tag + $ARTIFACT_REPO_NAME/$PROJECT_ID/$REPO_NAME/$IMAGE_NAME:$TAG . \ No newline at end of file From d5c77800fb40163c49463b3e2337acb51d25c1f2 Mon Sep 17 00:00:00 2001 From: romusters Date: Wed, 29 Jan 2025 15:59:29 +0100 Subject: [PATCH 34/62] Update permissions for token --- .github/workflows/docker.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c5c621b..6c51623 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -16,7 +16,9 @@ jobs: build-and-push: name: Build and Push Docker Image runs-on: ubuntu-latest - + permissions: + contents: 'read' + id-token: 'write' steps: - name: Checkout repository uses: actions/checkout@v4 From 7811108589556fd661e5f7ad1e63ea0d3079cd86 Mon Sep 17 00:00:00 2001 From: romusters Date: Wed, 29 Jan 2025 16:00:43 +0100 Subject: [PATCH 35/62] Missing '\' --- .github/workflows/docker.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 6c51623..414a2e9 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,5 +41,4 @@ jobs: run: | docker buildx build \ --push \ - --tag - $ARTIFACT_REPO_NAME/$PROJECT_ID/$REPO_NAME/$IMAGE_NAME:$TAG . \ No newline at end of file + --tag $ARTIFACT_REPO_NAME/$PROJECT_ID/$REPO_NAME/$IMAGE_NAME:$TAG . \ No newline at end of file From 48d1bc5cd0e189c19aee6cdc0d341686b9a2651e Mon Sep 17 00:00:00 2001 From: romusters Date: Wed, 29 Jan 2025 16:06:52 +0100 Subject: [PATCH 36/62] Add dummy certificate file. --- app/certificates.crt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/certificates.crt diff --git a/app/certificates.crt b/app/certificates.crt new file mode 100644 index 0000000..e69de29 From 160af5e890b5435bfcc0cb2b3d5066f2f795bd3c Mon Sep 17 00:00:00 2001 From: romusters Date: Wed, 29 Jan 2025 17:36:56 +0100 Subject: [PATCH 37/62] Some notes to self. --- .github/workflows/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..bf148b1 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,14 @@ +# Explanation + +To be able to push Docker images to the Artifact Registry, I used this blog: + +https://medium.com/@carstensavage/integrate-workload-identity-federation-with-github-actions-google-cloud-1893306f75c5 + + +The workload identity provider name can be found using: + +`gcloud iam workload-identity-pools providers list --project=PROJECT_ID --location=global --workload-identity-pool=WORKLOAD_IDENTITY_POOL_NAME` + +The Service account email is found in the GCP Service Account section. + +Make sure the principal has the following roles: Artifact Registry Administrator and Storage Admin. \ No newline at end of file From cb08d016b887376c3721df9a882c5877b574877b Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 11:25:37 +0100 Subject: [PATCH 38/62] Github Action te deploy cloud resources on GCP. Updates workflow to do Docker stuff first and then deploy. Update TODOs in README. --- .github/workflows/deploy-gcp.yaml | 56 +++++++++++++++++++ .../{docker.yaml => docker-gcp.yaml} | 0 README.md | 5 +- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy-gcp.yaml rename .github/workflows/{docker.yaml => docker-gcp.yaml} (100%) diff --git a/.github/workflows/deploy-gcp.yaml b/.github/workflows/deploy-gcp.yaml new file mode 100644 index 0000000..ed95af1 --- /dev/null +++ b/.github/workflows/deploy-gcp.yaml @@ -0,0 +1,56 @@ +name: Terraform Deployment + +on: + push: + branches: + - main + workflow_dispatch: + branches: + - main + workflow_run: + workflows: ["Build and Push to Google Cloud"] + types: + - completed + +permissions: + contents: read + id-token: 'write' + +env: + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + ARTIFACT_REPO_NAME: europe-west4-docker.pkg.dev + REPO_NAME: fastapi-docker-repo + IMAGE_NAME: fastapi + TAG: latest + +jobs: + build-and-push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Authenticate with GCP + uses: 'google-github-actions/auth@v2' + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.10.5 + - name: Terraform init and validate + run: | + terraform -chdir=infra/gcp init + + - name: Terraform plan + run: | + terraform -chdir=infra/gcp plan + + - name: Terraform apply + run: | + terraform -chdir=infra/gcp apply --auto-approve \ No newline at end of file diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker-gcp.yaml similarity index 100% rename from .github/workflows/docker.yaml rename to .github/workflows/docker-gcp.yaml diff --git a/README.md b/README.md index 45cb9af..5251814 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,10 @@ Although unadvised, setting `PYTHONHTTPSVERIFY` to `false` circumpasses SSL cert - fix infra bug: [DEBUG] POST https://management.azure.com/subscriptions//resourceGroups/rg20embedding001/providers/Microsoft.App/containerApps/ca20embedding001/listSecrets?api-version=2023-05-01 (status: 500): retrying in 1s (9 left) I do have the correct role set for the principal. - +- create config file which stores Artifact repo name and is used by Github Action as well as terraform +- create dummy database to disable cache +- deploy using terraform +- test using pipeline - make Redis asynchronous - devops pipeline - integration test From 316200844d1ff6457de3dce534e552b312b92cc2 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 11:33:02 +0100 Subject: [PATCH 39/62] Indentation --- .github/workflows/deploy-gcp.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-gcp.yaml b/.github/workflows/deploy-gcp.yaml index ed95af1..1c310f3 100644 --- a/.github/workflows/deploy-gcp.yaml +++ b/.github/workflows/deploy-gcp.yaml @@ -43,14 +43,15 @@ jobs: uses: hashicorp/setup-terraform@v1 with: terraform_version: 1.10.5 + - name: Terraform init and validate run: | terraform -chdir=infra/gcp init - name: Terraform plan - run: | - terraform -chdir=infra/gcp plan + run: | + terraform -chdir=infra/gcp plan - name: Terraform apply - run: | - terraform -chdir=infra/gcp apply --auto-approve \ No newline at end of file + run: | + terraform -chdir=infra/gcp apply --auto-approve \ No newline at end of file From e3ee5a595fd1a8395b46eac4d888d257cbb8f3d7 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 11:44:18 +0100 Subject: [PATCH 40/62] Update when workflows are triggered. Bump hashicorp/setup-terraform version to v3. --- .github/workflows/deploy-gcp.yaml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-gcp.yaml b/.github/workflows/deploy-gcp.yaml index 1c310f3..55fa00d 100644 --- a/.github/workflows/deploy-gcp.yaml +++ b/.github/workflows/deploy-gcp.yaml @@ -1,12 +1,6 @@ -name: Terraform Deployment +name: Deploy cloud resources on GCP using Terraform on: - push: - branches: - - main - workflow_dispatch: - branches: - - main workflow_run: workflows: ["Build and Push to Google Cloud"] types: @@ -24,8 +18,8 @@ env: TAG: latest jobs: - build-and-push: - name: Build and Push Docker Image + deploy-gcp-resources: + name: Deploy cloud resources on GCP using Terraform runs-on: ubuntu-latest steps: @@ -40,10 +34,10 @@ jobs: service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} - name: Setup Terraform - uses: hashicorp/setup-terraform@v1 + uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.10.5 - + - name: Terraform init and validate run: | terraform -chdir=infra/gcp init From 0f3ec872ef073fe0bdac392b2a32b8ad219d5bdd Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 14:26:03 +0100 Subject: [PATCH 41/62] Update artifact block type to existing. --- infra/gcp/main.tf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index e861017..a5a3096 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -23,9 +23,17 @@ provider "google" { region = var.region } +# Container Registry: Images are stored in Artifact Registry +# The registry already exists +data "google_artifact_registry_repository" "container_registry" { + repository_id = "fastapi-docker-repo" + format = "DOCKER" + location = var.region + description = "Docker repository for FastAPI images" +} # Enable necessary APIs -resource "google_project_service" "container_registry" { +resource "google_project_service" "container_registry" { for_each = toset([ "container.googleapis.com", "run.googleapis.com", @@ -40,14 +48,6 @@ resource "google_project_service" "artifact_registry_api" { project = var.project_id } -# Container Registry: Images are stored in Artifact Registry -resource "google_artifact_registry_repository" "container_registry" { - repository_id = "fastapi-docker-repo" - format = "DOCKER" - location = var.region - description = "Docker repository for FastAPI images" -} - # IAM Binding for Artifact Registry resource "google_artifact_registry_repository_iam_binding" "artifact_registry_binding" { repository = google_artifact_registry_repository.container_registry.name From ab2435385f2d3e8ef8b49ed31ccce8bace92b9d4 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 14:37:53 +0100 Subject: [PATCH 42/62] Improve Docker caching. Update existing artifact resource. --- Dockerfile | 3 ++- infra/gcp/main.tf | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98392a4..1bbf9dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.12.6-slim-bookworm WORKDIR /app -RUN apt-get update && apt-get clean && rm -rf /var/lib/apt/lists/* +# Adding a package such as 'curl' allows better caching of this layer +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir --upgrade certifi pip poetry --trusted-host pypi.org --trusted-host files.pythonhosted.org diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index a5a3096..6c81ae7 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -27,9 +27,9 @@ provider "google" { # The registry already exists data "google_artifact_registry_repository" "container_registry" { repository_id = "fastapi-docker-repo" - format = "DOCKER" +# format = "DOCKER" location = var.region - description = "Docker repository for FastAPI images" +# description = "Docker repository for FastAPI images" } # Enable necessary APIs @@ -50,7 +50,7 @@ resource "google_project_service" "artifact_registry_api" { # IAM Binding for Artifact Registry resource "google_artifact_registry_repository_iam_binding" "artifact_registry_binding" { - repository = google_artifact_registry_repository.container_registry.name + repository = data.google_artifact_registry_repository.container_registry.name role = "roles/artifactregistry.writer" members = ["serviceAccount:${google_service_account.cloud_run_service_account.email}"] } @@ -77,7 +77,7 @@ resource "google_cloud_run_service" "fastapi_service" { spec { containers { - image = "${google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.container_registry.name}/fastapi:latest" + image = "${data.google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.container_registry.name}/fastapi:latest" resources { limits = { memory = "2048Mi" From f07946a9e77aaee46c52d46eb1b8a87d9928683f Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 14:49:28 +0100 Subject: [PATCH 43/62] Forgot a prefixing with 'data' --- infra/gcp/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 6c81ae7..a2d2733 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -77,7 +77,7 @@ resource "google_cloud_run_service" "fastapi_service" { spec { containers { - image = "${data.google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.container_registry.name}/fastapi:latest" + image = "${data.google_artifact_registry_repository.container_registry.location}-docker.pkg.dev/${var.project_id}/${data.google_artifact_registry_repository.container_registry.name}/fastapi:latest" resources { limits = { memory = "2048Mi" From 2ec8ea50600d165019041368da0ce966f3a5e993 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 15:25:50 +0100 Subject: [PATCH 44/62] Fix --- .github/workflows/deploy-gcp.yaml | 3 +++ infra/gcp/main.tf | 9 ++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-gcp.yaml b/.github/workflows/deploy-gcp.yaml index 55fa00d..95fe1e1 100644 --- a/.github/workflows/deploy-gcp.yaml +++ b/.github/workflows/deploy-gcp.yaml @@ -1,6 +1,9 @@ name: Deploy cloud resources on GCP using Terraform on: + push: + branches: + - main workflow_run: workflows: ["Build and Push to Google Cloud"] types: diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index a2d2733..31dfba4 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -52,20 +52,19 @@ resource "google_project_service" "artifact_registry_api" { resource "google_artifact_registry_repository_iam_binding" "artifact_registry_binding" { repository = data.google_artifact_registry_repository.container_registry.name role = "roles/artifactregistry.writer" - members = ["serviceAccount:${google_service_account.cloud_run_service_account.email}"] + members = ["serviceAccount:${data.google_service_account.cloud_run_service_account.email}"] } # Service Account for Cloud Run -resource "google_service_account" "cloud_run_service_account" { +data "google_service_account" "cloud_run_service_account" { account_id = "cloud-run-service-account" - display_name = "Cloud Run Service Account" } # Grant necessary roles to the Service Account resource "google_project_iam_binding" "cloud_run_iam" { project = var.project_id role = "roles/run.admin" - members = ["serviceAccount:${google_service_account.cloud_run_service_account.email}"] + members = ["serviceAccount:${data.google_service_account.cloud_run_service_account.email}"] } # Deploy FastAPI Docker app to Cloud Run @@ -105,7 +104,7 @@ resource "google_cloud_run_service" "fastapi_service" { value = false } } - service_account_name = google_service_account.cloud_run_service_account.email + service_account_name = data.google_service_account.cloud_run_service_account.email } } From 10b32d203391c2d2d2e885223c006046a7c09b8f Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 15:58:58 +0100 Subject: [PATCH 45/62] Add terraform backend to hold terraform state --- infra/gcp/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 31dfba4..fb95d48 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -23,6 +23,13 @@ provider "google" { region = var.region } +terraform { + backend "gcs" { + bucket = "terraform-state-fast-api-example" + prefix = "terraform/state" # Path within the bucket to store the state file + } +} + # Container Registry: Images are stored in Artifact Registry # The registry already exists data "google_artifact_registry_repository" "container_registry" { From 18caa34d82d31eb696e6ed61c3fe020e7a6f7b49 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 16:32:52 +0100 Subject: [PATCH 46/62] Forgot about pre-commit, better add it as an action. --- Dockerfile | 4 +-- app/api/endpoints/embed.py | 20 ++++++------ app/config/settings.py | 4 ++- app/db/database_interface_factory.py | 9 +++--- app/db/pinecone_database.py | 28 ++++++++--------- app/db/redis_database.py | 7 +++-- app/main.py | 4 +-- app/schemas/default.py | 2 +- tests/payload_tests.py | 2 +- tests/test_main.py | 47 ++++++++++++++-------------- 10 files changed, 66 insertions(+), 61 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1bbf9dd..0e9a1ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,9 @@ FROM python:3.12.6-slim-bookworm WORKDIR /app # Adding a package such as 'curl' allows better caching of this layer -RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +RUN apt-get update && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir --upgrade certifi pip poetry --trusted-host pypi.org --trusted-host files.pythonhosted.org +RUN pip install --no-cache-dir certifi==2024.12.14 pip==25.0 poetry==2.0.1 --trusted-host pypi.org --trusted-host files.pythonhosted.org COPY pyproject.toml /app diff --git a/app/api/endpoints/embed.py b/app/api/endpoints/embed.py index fa3172a..0fa44d4 100644 --- a/app/api/endpoints/embed.py +++ b/app/api/endpoints/embed.py @@ -1,13 +1,12 @@ import json import logging +from fastapi import APIRouter, HTTPException + from app.config.settings import Settings from app.db.database_interface_factory import DatabaseFactory from app.model import Handler -from app.schemas.default import TextInput, EmbeddingOutput, SimilarityOutput -from fastapi import HTTPException -from fastapi import APIRouter - +from app.schemas.default import EmbeddingOutput, SimilarityOutput, TextInput logging.basicConfig( level=logging.INFO, @@ -20,7 +19,9 @@ settings = Settings() if settings.CACHE_ENABLED: - database_object = DatabaseFactory.get_database(settings.DATABASE_KIND) # Switch between databases easily using an interface. + database_object = DatabaseFactory.get_database( + settings.DATABASE_KIND + ) # Switch between databases easily using an interface. database_object.connect() handler = Handler() @@ -36,7 +37,7 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: """ logger.info(f"Embedding text: {text_input.text}") if settings.CACHE_ENABLED: - cached_embedding = database_object.get(text_input.text) + cached_embedding = database_object.get(text_input.text) if cached_embedding: logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") return EmbeddingOutput( @@ -62,9 +63,10 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: logging.info(f"Generating embedding for: {text_input.text[0:10]}...") embedding = handler.embed(text_input.text) return EmbeddingOutput( - embedding=embedding, - description="The list of float values representing the text embedding.", - ) + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + @embed_router.post("/similarity") async def calculate_similarity( diff --git a/app/config/settings.py b/app/config/settings.py index 1f8ac34..7ce4478 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -1,4 +1,5 @@ import os + from pydantic_settings import BaseSettings @@ -9,4 +10,5 @@ class Settings(BaseSettings): DATABASE_API_KEY: str = os.getenv("DATABASE_API_KEY", "dummy-api-key") CACHE_ENABLED: bool = os.getenv("CACHE_ENABLED", "dummy") -settings = Settings() \ No newline at end of file + +settings = Settings() diff --git a/app/db/database_interface_factory.py b/app/db/database_interface_factory.py index 7898459..03dbffa 100644 --- a/app/db/database_interface_factory.py +++ b/app/db/database_interface_factory.py @@ -1,8 +1,7 @@ import logging -from app.db.redis_database import RedisDatabase from app.db.pinecone_database import PineconeDatabase - +from app.db.redis_database import RedisDatabase logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) @@ -10,13 +9,13 @@ ) logger = logging.getLogger(__name__) -class DatabaseFactory: +class DatabaseFactory: @staticmethod def get_database(db_type, **kwargs): - if db_type == 'redis': + if db_type == "redis": return RedisDatabase(**kwargs) - elif db_type == 'pinecone': + elif db_type == "pinecone": return PineconeDatabase(**kwargs) else: raise ValueError(f"Unsupported database type: {db_type}") diff --git a/app/db/pinecone_database.py b/app/db/pinecone_database.py index 2117069..52491d4 100644 --- a/app/db/pinecone_database.py +++ b/app/db/pinecone_database.py @@ -1,11 +1,11 @@ import logging import time +from typing import List -from app.config.settings import Settings -from app.db.database_interface import DatabaseInterface from pinecone.grpc import PineconeGRPC as Pinecone -from typing import List +from app.config.settings import Settings +from app.db.database_interface import DatabaseInterface logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) @@ -15,26 +15,27 @@ settings = Settings() + class PineconeDatabase(DatabaseInterface): def __init__(self): self.DATABASE_URL = f"{settings.DATABASE_URL}:{settings.DATABASE_PORT}" self.PORT = settings.DATABASE_PORT self.API_KEY = settings.DATABASE_API_KEY self.client = None - self.index_name = 'default' - + self.index_name = "default" def connect(self): try: self.client = Pinecone(api_key=self.API_KEY, host=self.DATABASE_URL) - logger.info(f"Pinecone api_key: {self.API_KEY}, host: {self.DATABASE_URL} .") + logger.info( + f"Pinecone api_key: {self.API_KEY}, host: {self.DATABASE_URL} ." + ) logger.info("Pinecone initialized successfully.") except Exception as e: logger.error(f"Failed to initialize Pinecone: {e}") - def check_index(self, index_name: str = 'default') -> None: - - if not self.client.has_index(index_name): + def check_index(self, index_name: str = "default") -> None: + if not self.client.has_index(index_name): self.client.create_index( name=index_name, dimension=2, @@ -42,11 +43,9 @@ def check_index(self, index_name: str = 'default') -> None: ) # Wait for the index to be ready - while not self.client.describe_index(self.index_name).status['ready']: + while not self.client.describe_index(self.index_name).status["ready"]: time.sleep(1) - - def get(self, key: str): """ Retrieves a key from Redis if it exists. @@ -62,10 +61,11 @@ def get(self, key: str): if fetch_response: return fetch_response return None - def set(self, key: str, value: List[float]): self.check_index() index = self.client.Index(host=self.DATABASE_URL) upsert_response = index.upsert(vectors=[(key, value)]) - logging.info(f"The key {key[0:10]}... was upserted with response {upsert_response}") \ No newline at end of file + logging.info( + f"The key {key[0:10]}... was upserted with response {upsert_response}" + ) diff --git a/app/db/redis_database.py b/app/db/redis_database.py index dd8f195..15120ea 100644 --- a/app/db/redis_database.py +++ b/app/db/redis_database.py @@ -1,11 +1,11 @@ import logging -import redis from typing import List +import redis + from app.config.settings import Settings from app.db.database_interface import DatabaseInterface - logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format @@ -14,6 +14,7 @@ settings = Settings() + class RedisDatabase(DatabaseInterface): def __init__(self): self.host = settings.DATABASE_URL @@ -51,4 +52,4 @@ def get(self, key: str): # Retrieve the key's value value = self.client.get(key) return value - return None \ No newline at end of file + return None diff --git a/app/main.py b/app/main.py index 457fdd7..c8cd2e8 100755 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,11 @@ import logging -from app.api.endpoints.embed import embed_router from fastapi import FastAPI +from app.api.endpoints.embed import embed_router logging.basicConfig( - level=logging.INFO, + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) logger = logging.getLogger(__name__) diff --git a/app/schemas/default.py b/app/schemas/default.py index 4a72ee8..df2a462 100644 --- a/app/schemas/default.py +++ b/app/schemas/default.py @@ -12,4 +12,4 @@ class EmbeddingOutput(BaseModel): class SimilarityOutput(BaseModel): similarity: float - description: str | None = None \ No newline at end of file + description: str | None = None diff --git a/tests/payload_tests.py b/tests/payload_tests.py index 9252c20..27d60fe 100644 --- a/tests/payload_tests.py +++ b/tests/payload_tests.py @@ -1 +1 @@ -long_string_input = {"text": "This is a very long string. " * 100} \ No newline at end of file +long_string_input = {"text": "This is a very long string. " * 100} diff --git a/tests/test_main.py b/tests/test_main.py index fe1a541..df30363 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,25 +2,24 @@ import unittest from dotenv import load_dotenv + load_dotenv() # This will load variables from .env -from app.main import app -from app.schemas.default import TextInput +from unittest.mock import MagicMock, patch + from fastapi.testclient import TestClient from parameterized import parameterized -from tests.payload_tests import long_string_input -from unittest.mock import patch, MagicMock - - +from app.main import app +from app.schemas.default import TextInput +from tests.payload_tests import long_string_input -class TestEmbedEndpoint(unittest.TestCase): +class TestEmbedEndpoint(unittest.TestCase): # Arrange def setUp(self): self.client = TestClient(app) - @patch("app.api.endpoints.embed.database_object") def test_embed_text_cached(self, mock_database_object): mock_database_object.get = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) @@ -43,15 +42,14 @@ def test_embed_text_cached(self, mock_database_object): # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list - mock_database_object.get.assert_called_once_with(long_string_input['text']) - + mock_database_object.get.assert_called_once_with(long_string_input["text"]) @patch("app.api.endpoints.embed.handler") @patch("app.api.endpoints.embed.database_object") def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): mock_database_object.get = MagicMock(return_value=None) - mock_handler_object.embed = MagicMock(return_value=[1,2,3]) - + mock_handler_object.embed = MagicMock(return_value=[1, 2, 3]) + # Act: Send a POST request to the /embed endpoint response = self.client.post("/embed", json=long_string_input) @@ -70,17 +68,20 @@ def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list - mock_database_object.get.assert_called_once_with(long_string_input['text']) - mock_handler_object.embed.assert_called_once_with(long_string_input['text']) - - - @parameterized.expand([ - ({"text": ""}, ), - ({"text": "This is a short sentence."}, ), - ({ - "text": "This is a longer sentence, which contains more words and should still work correctly." - }, ), - ]) + mock_database_object.get.assert_called_once_with(long_string_input["text"]) + mock_handler_object.embed.assert_called_once_with(long_string_input["text"]) + + @parameterized.expand( + [ + ({"text": ""},), + ({"text": "This is a short sentence."},), + ( + { + "text": "This is a longer sentence, which contains more words and should still work correctly." + }, + ), + ] + ) @patch("app.api.endpoints.embed.database_object") def test_embed_text_parametrized(self, text_input, mock_database_object): mock_database_object.get = MagicMock(return_value=None) From 4f93b919ad145399915be974119e5c95af2cc755 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:37:24 +0100 Subject: [PATCH 47/62] Create dummy database to help in testing and simplyfying code. --- app/api/endpoints/embed.py | 56 ++++++++++++---------------- app/config/settings.py | 10 ++--- app/db/database_interface_factory.py | 3 ++ app/db/dummy_database.py | 27 ++++++++++++++ app/db/redis_database.py | 2 + app/model.py | 1 + tests/.env | 5 +++ tests/test_main.py | 7 +++- 8 files changed, 72 insertions(+), 39 deletions(-) create mode 100644 app/db/dummy_database.py create mode 100644 tests/.env diff --git a/app/api/endpoints/embed.py b/app/api/endpoints/embed.py index 0fa44d4..7770236 100644 --- a/app/api/endpoints/embed.py +++ b/app/api/endpoints/embed.py @@ -18,11 +18,11 @@ settings = Settings() -if settings.CACHE_ENABLED: - database_object = DatabaseFactory.get_database( - settings.DATABASE_KIND - ) # Switch between databases easily using an interface. - database_object.connect() + +database_object = DatabaseFactory.get_database( + settings.DATABASE_KIND, +) # Switch between databases easily using an interface. +database_object.connect() handler = Handler() @@ -35,37 +35,27 @@ async def embed_text(text_input: TextInput) -> EmbeddingOutput: :param text_input: The text to embed. :return: The embedding of the input text as JSON. """ - logger.info(f"Embedding text: {text_input.text}") - if settings.CACHE_ENABLED: - cached_embedding = database_object.get(text_input.text) - if cached_embedding: - logging.info(f"Retrieving cached embedding for: {text_input.text[0:10]}...") - return EmbeddingOutput( - embedding=json.loads(cached_embedding), - description="The list of float values representing the text embedding.", - ) - else: - try: - logging.info(f"Generating embedding for: {text_input.text[0:10]}...") - embedding = handler.embed(text_input.text) - logging.info(f"Setting embedding for: {text_input.text[0:10]}...") - database_object.client.set(text_input.text, json.dumps(embedding)) - return EmbeddingOutput( - embedding=embedding, - description="The list of float values representing the text embedding.", - ) - except RuntimeError: - HTTPException( - status_code=404, - detail="Something went wrong with creating an embedding.", - ) - else: - logging.info(f"Generating embedding for: {text_input.text[0:10]}...") - embedding = handler.embed(text_input.text) + logger.info(f"Embedding text: {text_input.text[0:10]}") + embedding = database_object.get(text_input.text) + if embedding: return EmbeddingOutput( - embedding=embedding, + embedding=json.loads(embedding), description="The list of float values representing the text embedding.", ) + else: + try: + embedding = handler.embed(text_input.text) + if settings.CACHE_ENABLED: + database_object.client.set(text_input.text, json.dumps(embedding)) + return EmbeddingOutput( + embedding=embedding, + description="The list of float values representing the text embedding.", + ) + except RuntimeError: + HTTPException( + status_code=404, + detail="Something went wrong with creating an embedding.", + ) @embed_router.post("/similarity") diff --git a/app/config/settings.py b/app/config/settings.py index 7ce4478..656b8c9 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -4,11 +4,11 @@ class Settings(BaseSettings): - DATABASE_URL: str = os.getenv("DATABASE_HOST", "dummy") - DATABASE_PORT: str = os.getenv("DATABASE_PORT", "dummy") - DATABASE_KIND: str = os.getenv("DATABASE_KIND", "dummy") - DATABASE_API_KEY: str = os.getenv("DATABASE_API_KEY", "dummy-api-key") - CACHE_ENABLED: bool = os.getenv("CACHE_ENABLED", "dummy") + DATABASE_URL: str = os.getenv("DATABASE_HOST") + DATABASE_PORT: str = os.getenv("DATABASE_PORT") + DATABASE_KIND: str = os.getenv("DATABASE_KIND") + DATABASE_API_KEY: str = os.getenv("DATABASE_API_KEY") + CACHE_ENABLED: bool = os.getenv("CACHE_ENABLED") settings = Settings() diff --git a/app/db/database_interface_factory.py b/app/db/database_interface_factory.py index 03dbffa..8115c5a 100644 --- a/app/db/database_interface_factory.py +++ b/app/db/database_interface_factory.py @@ -1,5 +1,6 @@ import logging +from app.db.dummy_database import DummyDatabase from app.db.pinecone_database import PineconeDatabase from app.db.redis_database import RedisDatabase @@ -17,5 +18,7 @@ def get_database(db_type, **kwargs): return RedisDatabase(**kwargs) elif db_type == "pinecone": return PineconeDatabase(**kwargs) + elif db_type == "dummy": + return DummyDatabase(**kwargs) else: raise ValueError(f"Unsupported database type: {db_type}") diff --git a/app/db/dummy_database.py b/app/db/dummy_database.py new file mode 100644 index 0000000..9aa45a5 --- /dev/null +++ b/app/db/dummy_database.py @@ -0,0 +1,27 @@ +import logging +from typing import List + +from app.config.settings import Settings +from app.db.database_interface import DatabaseInterface + +logging.basicConfig( + level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", # Log format +) +logger = logging.getLogger(__name__) + +settings = Settings() + + +class DummyDatabase(DatabaseInterface): + def __init__(self): + return + + def connect(self): + return + + def set(self, key: str, value: List[float]): + return + + def get(self, key: str): + return diff --git a/app/db/redis_database.py b/app/db/redis_database.py index 15120ea..4b94c53 100644 --- a/app/db/redis_database.py +++ b/app/db/redis_database.py @@ -36,6 +36,7 @@ def connect(self): return super().connect() def set(self, key: str, value: List[float]): + logging.info(f"Setting embedding for: {key[0:10]}...") self.client.set(key=key, value=value) def get(self, key: str): @@ -49,6 +50,7 @@ def get(self, key: str): # Check if the key exists exists = self.client.exists(key) if exists: + logging.info(f"Retrieving cached embedding for: {key[0:10]}...") # Retrieve the key's value value = self.client.get(key) return value diff --git a/app/model.py b/app/model.py index 07cf795..c1cbe0d 100755 --- a/app/model.py +++ b/app/model.py @@ -37,6 +37,7 @@ def embed(self, text) -> list[float]: """ # Forward pass + logging.info(f"Generating embedding for: {text[0:10]}...") tensors = self.forward(text) return tensors.tolist() diff --git a/tests/.env b/tests/.env new file mode 100644 index 0000000..bac2637 --- /dev/null +++ b/tests/.env @@ -0,0 +1,5 @@ +DATABASE_URL= 'test' +DATABASE_PORT='test' +DATABASE_KIND='dummy' +DATABASE_API_KEY='test' +CACHE_ENABLED=True # Redis hostname (will be used in FastAPI to connect to Redis) \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index df30363..47f00bb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,5 @@ import json +import os import unittest from dotenv import load_dotenv @@ -10,6 +11,7 @@ from fastapi.testclient import TestClient from parameterized import parameterized +from app.config.settings import Settings from app.main import app from app.schemas.default import TextInput from tests.payload_tests import long_string_input @@ -20,8 +22,9 @@ class TestEmbedEndpoint(unittest.TestCase): def setUp(self): self.client = TestClient(app) + @patch("app.api.endpoints.embed.handler") @patch("app.api.endpoints.embed.database_object") - def test_embed_text_cached(self, mock_database_object): + def test_embed_text_cached(self, mock_database_object, mock_handler_object): mock_database_object.get = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) # Act: Send a POST request to the /embed endpoint @@ -42,7 +45,9 @@ def test_embed_text_cached(self, mock_database_object): # Assert the embedding values assert isinstance(response_data["embedding"], list) assert len(response_data["embedding"]) > 0 # Ensure it's a non-empty list + assert response_data["embedding"] == [0.1, 0.2, 0.3] mock_database_object.get.assert_called_once_with(long_string_input["text"]) + mock_handler_object.embed.assert_not_called() @patch("app.api.endpoints.embed.handler") @patch("app.api.endpoints.embed.database_object") From fe513ac7891c3348d2b73e155d6f97916bb4da7c Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:45:00 +0100 Subject: [PATCH 48/62] Add pre-commit and pytest to Github ACtions --- .github/workflows/deploy-gcp.yaml | 3 --- .github/workflows/docker-gcp.yaml | 7 ++++--- .github/workflows/python.yaml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/python.yaml diff --git a/.github/workflows/deploy-gcp.yaml b/.github/workflows/deploy-gcp.yaml index 95fe1e1..55fa00d 100644 --- a/.github/workflows/deploy-gcp.yaml +++ b/.github/workflows/deploy-gcp.yaml @@ -1,9 +1,6 @@ name: Deploy cloud resources on GCP using Terraform on: - push: - branches: - - main workflow_run: workflows: ["Build and Push to Google Cloud"] types: diff --git a/.github/workflows/docker-gcp.yaml b/.github/workflows/docker-gcp.yaml index 414a2e9..e0fef8d 100644 --- a/.github/workflows/docker-gcp.yaml +++ b/.github/workflows/docker-gcp.yaml @@ -1,9 +1,10 @@ name: Build and Push to Google Cloud on: - push: - branches: - - main + workflow_run: + workflows: ["Pre-commit and test Python and Docker files"] + types: + - completed env: PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml new file mode 100644 index 0000000..6dbe34e --- /dev/null +++ b/.github/workflows/python.yaml @@ -0,0 +1,30 @@ +name: Pre-commit and test Python and Docker files + +on: + push: + branches: + - main + +jobs: + runs-on: ubuntu-latest + pre-commit: + steps: + - uses: actions/checkout@v3 # TODO: only checkout once + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.1 + + test: + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: | + pip install poetry + poetry install --no-cache + - name: Test with pytest + run: | + pip install pytest pytest-cov + pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file From 494dfcd3fb7c6439805e8440944b6c65de2ee158 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:46:41 +0100 Subject: [PATCH 49/62] Change runs on location in pipeline --- .github/workflows/python.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 6dbe34e..4433f2b 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -6,8 +6,9 @@ on: - main jobs: - runs-on: ubuntu-latest + pre-commit: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 # TODO: only checkout once - uses: actions/setup-python@v3 From 7ea1838f3743045800788c0ce59d765f5403322a Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:47:16 +0100 Subject: [PATCH 50/62] Another runs-on --- .github/workflows/python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 4433f2b..e17a4af 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -15,6 +15,7 @@ jobs: - uses: pre-commit/action@v3.0.1 test: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python From 877a6e5e296cd4941684a805bcd09ab6af908c10 Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:54:22 +0100 Subject: [PATCH 51/62] Add packages --- .github/workflows/python.yaml | 2 +- pyproject.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index e17a4af..b2c134f 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -28,5 +28,5 @@ jobs: poetry install --no-cache - name: Test with pytest run: | - pip install pytest pytest-cov + poetry install --with dev pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8f31680..bd7ac68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ pinecone = {extras = ["grpc"], version = "^5.4.2"} [tool.poetry.group.dev.dependencies] pytest = "8.3.3" pre-commit = "4.0.1" +python-dotenv = "^1.0.1" +pytest-cov = "^6.0.0" [build-system] requires = ["poetry-core>=1.0.0"] From 302fc3027e65965a5abe04435f0af9f06f06150c Mon Sep 17 00:00:00 2001 From: romusters Date: Thu, 30 Jan 2025 17:57:53 +0100 Subject: [PATCH 52/62] Poetry run --- .github/workflows/python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index b2c134f..72a68b5 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -29,4 +29,4 @@ jobs: - name: Test with pytest run: | poetry install --with dev - pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file + poetry run pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file From 9d49e46b862442a6231d7b957a3384b015d4f2b4 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 19:33:30 +0100 Subject: [PATCH 53/62] Build package using UV in Github runner. eploy artifactory for Python. --- .github/workflows/docker-gcp.yaml | 16 +++++++- {app => embedding_api}/__init__.py | 0 .../api/endpoints/__init__.py | 0 {app => embedding_api}/api/endpoints/embed.py | 0 {app => embedding_api}/certificates.crt | 0 {app => embedding_api}/config/__init__.py | 0 {app => embedding_api}/config/settings.py | 0 {app => embedding_api}/db/__init__.py | 0 .../db/database_interface.py | 0 .../db/database_interface_factory.py | 0 {app => embedding_api}/db/dummy_database.py | 0 .../db/pinecone_database.py | 0 {app => embedding_api}/db/redis_database.py | 0 {app => embedding_api}/main.py | 0 {app => embedding_api}/model.py | 0 {app => embedding_api}/schemas/__init__.py | 0 {app => embedding_api}/schemas/default.py | 0 infra/gcp/main.tf | 8 ++++ pyproject.toml | 41 +++++++++++-------- 19 files changed, 47 insertions(+), 18 deletions(-) rename {app => embedding_api}/__init__.py (100%) rename {app => embedding_api}/api/endpoints/__init__.py (100%) rename {app => embedding_api}/api/endpoints/embed.py (100%) rename {app => embedding_api}/certificates.crt (100%) rename {app => embedding_api}/config/__init__.py (100%) rename {app => embedding_api}/config/settings.py (100%) rename {app => embedding_api}/db/__init__.py (100%) rename {app => embedding_api}/db/database_interface.py (100%) rename {app => embedding_api}/db/database_interface_factory.py (100%) rename {app => embedding_api}/db/dummy_database.py (100%) rename {app => embedding_api}/db/pinecone_database.py (100%) rename {app => embedding_api}/db/redis_database.py (100%) rename {app => embedding_api}/main.py (100%) rename {app => embedding_api}/model.py (100%) rename {app => embedding_api}/schemas/__init__.py (100%) rename {app => embedding_api}/schemas/default.py (100%) diff --git a/.github/workflows/docker-gcp.yaml b/.github/workflows/docker-gcp.yaml index e0fef8d..9326c52 100644 --- a/.github/workflows/docker-gcp.yaml +++ b/.github/workflows/docker-gcp.yaml @@ -9,7 +9,8 @@ on: env: PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} ARTIFACT_REPO_NAME: europe-west4-docker.pkg.dev - REPO_NAME: fastapi-docker-repo + DOCKER_REGISTRY_NAME: fastapi-docker-repo + PYTHON_REGISTRY_NAME: handler-python-package IMAGE_NAME: fastapi TAG: latest @@ -31,6 +32,19 @@ jobs: workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + - name: Install UV + run: | + pip install uv + + - name: Build package + run: | + uv build + + - name: Set Up UV for Publishing + run: | + export UV_PYPI_REGISTRY=https://${{ ARTIFACT_REPO_NAME }}/${{ secrets.GCP_PROJECT_ID }}/${{ env.PYTHON_REGISTRY_NAME }}/ + uv publish + - name: Configure Docker to use Google Artifact Registry run: | gcloud auth configure-docker $ARTIFACT_REPO_NAME diff --git a/app/__init__.py b/embedding_api/__init__.py similarity index 100% rename from app/__init__.py rename to embedding_api/__init__.py diff --git a/app/api/endpoints/__init__.py b/embedding_api/api/endpoints/__init__.py similarity index 100% rename from app/api/endpoints/__init__.py rename to embedding_api/api/endpoints/__init__.py diff --git a/app/api/endpoints/embed.py b/embedding_api/api/endpoints/embed.py similarity index 100% rename from app/api/endpoints/embed.py rename to embedding_api/api/endpoints/embed.py diff --git a/app/certificates.crt b/embedding_api/certificates.crt similarity index 100% rename from app/certificates.crt rename to embedding_api/certificates.crt diff --git a/app/config/__init__.py b/embedding_api/config/__init__.py similarity index 100% rename from app/config/__init__.py rename to embedding_api/config/__init__.py diff --git a/app/config/settings.py b/embedding_api/config/settings.py similarity index 100% rename from app/config/settings.py rename to embedding_api/config/settings.py diff --git a/app/db/__init__.py b/embedding_api/db/__init__.py similarity index 100% rename from app/db/__init__.py rename to embedding_api/db/__init__.py diff --git a/app/db/database_interface.py b/embedding_api/db/database_interface.py similarity index 100% rename from app/db/database_interface.py rename to embedding_api/db/database_interface.py diff --git a/app/db/database_interface_factory.py b/embedding_api/db/database_interface_factory.py similarity index 100% rename from app/db/database_interface_factory.py rename to embedding_api/db/database_interface_factory.py diff --git a/app/db/dummy_database.py b/embedding_api/db/dummy_database.py similarity index 100% rename from app/db/dummy_database.py rename to embedding_api/db/dummy_database.py diff --git a/app/db/pinecone_database.py b/embedding_api/db/pinecone_database.py similarity index 100% rename from app/db/pinecone_database.py rename to embedding_api/db/pinecone_database.py diff --git a/app/db/redis_database.py b/embedding_api/db/redis_database.py similarity index 100% rename from app/db/redis_database.py rename to embedding_api/db/redis_database.py diff --git a/app/main.py b/embedding_api/main.py similarity index 100% rename from app/main.py rename to embedding_api/main.py diff --git a/app/model.py b/embedding_api/model.py similarity index 100% rename from app/model.py rename to embedding_api/model.py diff --git a/app/schemas/__init__.py b/embedding_api/schemas/__init__.py similarity index 100% rename from app/schemas/__init__.py rename to embedding_api/schemas/__init__.py diff --git a/app/schemas/default.py b/embedding_api/schemas/default.py similarity index 100% rename from app/schemas/default.py rename to embedding_api/schemas/default.py diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index fb95d48..5d6b5a9 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -50,6 +50,14 @@ resource "google_project_service" "container_registry" { service = each.key } +resource "google_artifact_registry_repository" "my-repo" { + location = var.region + repository_id = "python-package" + description = "example docker repository" + format = "PYTHON" +} + + resource "google_project_service" "artifact_registry_api" { service = "artifactregistry.googleapis.com" project = var.project_id diff --git a/pyproject.toml b/pyproject.toml index bd7ac68..5755819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,36 @@ -[tool.poetry] -name = "portfolio-embedding-endpoint" +[project] +name = "embedding-api" version = "0.1.0" description = "Personal project to showcase my abilities" -authors = ["R. M. "] +authors = [{"name"="R"}] license = "MIT" package-mode = false +requires-python = ">=3.11,<4.0" +packages = [ + { include = "embedding_api"}, +] -[tool.poetry.dependencies] -python = "^3.12" -fastapi = { version = "0.115.2", extras = ["all"] } -torch = "2.3.1" -transformers = "4.45.2" -pre-commit = "^4.0.1" -redis = "^5.2.0" -requests = "2.31.0" -parameterized = "^0.9.0" -pydantic-settings = "^2.7.1" -pinecone = {extras = ["grpc"], version = "^5.4.2"} -[tool.poetry.group.dev.dependencies] -pytest = "8.3.3" +dependencies = [ + "fastapi[all] (==0.115.8)", + "torch (==2.3.1)", + "transformers (==4.45.2)", + "redis (>=5.2.0,<6.0.0)", + "pydantic-settings (>=2.7.1,<3.0.0)", + "pinecone[grpc] (>=5.4.2,<6.0.0)" +] + +#[dev-dependencies] +#"pre-commit (>=4.0.1,<5.0.0)", +#"parameterized (>=0.9.0,<1.0.0)" + + +[tool.poetry.dependencies] +pytest = "8.4.2" pre-commit = "4.0.1" python-dotenv = "^1.0.1" pytest-cov = "^6.0.0" [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=2.0.1"] build-backend = "poetry.core.masonry.api" \ No newline at end of file From cde3f13976e341a0a4d9080e62003339078c3683 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 19:50:33 +0100 Subject: [PATCH 54/62] Fix env var. Create env with uv. Add Python version. --- .github/workflows/docker-gcp.yaml | 2 +- .github/workflows/python.yaml | 7 +++---- .python-version | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/docker-gcp.yaml b/.github/workflows/docker-gcp.yaml index 9326c52..737e13e 100644 --- a/.github/workflows/docker-gcp.yaml +++ b/.github/workflows/docker-gcp.yaml @@ -42,7 +42,7 @@ jobs: - name: Set Up UV for Publishing run: | - export UV_PYPI_REGISTRY=https://${{ ARTIFACT_REPO_NAME }}/${{ secrets.GCP_PROJECT_ID }}/${{ env.PYTHON_REGISTRY_NAME }}/ + export UV_PYPI_REGISTRY=https://${{ env.ARTIFACT_REPO_NAME }}/${{ secrets.GCP_PROJECT_ID }}/${{ env.PYTHON_REGISTRY_NAME }}/ uv publish - name: Configure Docker to use Google Artifact Registry diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 72a68b5..28d1c15 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -24,9 +24,8 @@ jobs: python-version: '3.12' - name: Install dependencies run: | - pip install poetry - poetry install --no-cache + pip install uv + uv sync - name: Test with pytest run: | - poetry install --with dev - poetry run pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file + uv run pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 From aef98fdcb29f19c8c6ffc4a66fb26b1514b3937d Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:10:08 +0100 Subject: [PATCH 55/62] Pre-commit. Fix dev eependencies --- embedding_api/api/endpoints/embed.py | 3 +- embedding_api/db/pinecone_database.py | 3 +- embedding_api/db/redis_database.py | 1 - embedding_api/main.py | 3 +- pyproject.toml | 18 +- tests/test_main.py | 6 +- uv.lock | 1306 +++++++++++++++++++++++++ 7 files changed, 1321 insertions(+), 19 deletions(-) create mode 100644 uv.lock diff --git a/embedding_api/api/endpoints/embed.py b/embedding_api/api/endpoints/embed.py index 7770236..4045ac6 100644 --- a/embedding_api/api/endpoints/embed.py +++ b/embedding_api/api/endpoints/embed.py @@ -1,12 +1,11 @@ import json import logging -from fastapi import APIRouter, HTTPException - from app.config.settings import Settings from app.db.database_interface_factory import DatabaseFactory from app.model import Handler from app.schemas.default import EmbeddingOutput, SimilarityOutput, TextInput +from fastapi import APIRouter, HTTPException logging.basicConfig( level=logging.INFO, diff --git a/embedding_api/db/pinecone_database.py b/embedding_api/db/pinecone_database.py index 52491d4..c8ccdb6 100644 --- a/embedding_api/db/pinecone_database.py +++ b/embedding_api/db/pinecone_database.py @@ -2,10 +2,9 @@ import time from typing import List -from pinecone.grpc import PineconeGRPC as Pinecone - from app.config.settings import Settings from app.db.database_interface import DatabaseInterface +from pinecone.grpc import PineconeGRPC as Pinecone logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/embedding_api/db/redis_database.py b/embedding_api/db/redis_database.py index 4b94c53..7f4f489 100644 --- a/embedding_api/db/redis_database.py +++ b/embedding_api/db/redis_database.py @@ -2,7 +2,6 @@ from typing import List import redis - from app.config.settings import Settings from app.db.database_interface import DatabaseInterface diff --git a/embedding_api/main.py b/embedding_api/main.py index c8cd2e8..86b55ab 100755 --- a/embedding_api/main.py +++ b/embedding_api/main.py @@ -1,8 +1,7 @@ import logging -from fastapi import FastAPI - from app.api.endpoints.embed import embed_router +from fastapi import FastAPI logging.basicConfig( level=logging.INFO, diff --git a/pyproject.toml b/pyproject.toml index 5755819..48b9221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Personal project to showcase my abilities" authors = [{"name"="R"}] license = "MIT" package-mode = false -requires-python = ">=3.11,<4.0" +requires-python = "==3.12.8" packages = [ { include = "embedding_api"}, ] @@ -20,16 +20,16 @@ dependencies = [ "pinecone[grpc] (>=5.4.2,<6.0.0)" ] -#[dev-dependencies] -#"pre-commit (>=4.0.1,<5.0.0)", -#"parameterized (>=0.9.0,<1.0.0)" +[tool.uv] +dev-dependencies = [ + "parameterized (>=0.9.0,<1.0.0)", + "pytest(==8.3.4)", + "pre-commit (==4.0.1)", + "python-dotenv (==1.0.1)", + "pytest-cov (==6.0.0)" +] -[tool.poetry.dependencies] -pytest = "8.4.2" -pre-commit = "4.0.1" -python-dotenv = "^1.0.1" -pytest-cov = "^6.0.0" [build-system] requires = ["poetry-core>=2.0.1"] diff --git a/tests/test_main.py b/tests/test_main.py index 47f00bb..2ca96c4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,12 +8,12 @@ from unittest.mock import MagicMock, patch -from fastapi.testclient import TestClient -from parameterized import parameterized - from app.config.settings import Settings from app.main import app from app.schemas.default import TextInput +from fastapi.testclient import TestClient +from parameterized import parameterized + from tests.payload_tests import long_string_input diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..37e355a --- /dev/null +++ b/uv.lock @@ -0,0 +1,1306 @@ +version = 1 +revision = 1 +requires-python = "==3.12.8" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, +] + +[[package]] +name = "anyio" +version = "4.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/73/199a98fc2dae33535d6b8e8e6ec01f8c1d76c9adb096c6b7d64823038cde/anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a", size = 181126 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/eb/e7f063ad1fec6b3178a3cd82d1a3c4de82cccf283fc42746168188e1cdd5/anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a", size = 96041 }, +] + +[[package]] +name = "certifi" +version = "2025.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393 }, +] + +[[package]] +name = "cfgv" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249 }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/9a/dd1e1cdceb841925b7798369a09279bd1cf183cef0f9ddf15a3a6502ee45/charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", size = 196105 }, + { url = "https://files.pythonhosted.org/packages/d3/8c/90bfabf8c4809ecb648f39794cf2a84ff2e7d2a6cf159fe68d9a26160467/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", size = 140404 }, + { url = "https://files.pythonhosted.org/packages/ad/8f/e410d57c721945ea3b4f1a04b74f70ce8fa800d393d72899f0a40526401f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", size = 150423 }, + { url = "https://files.pythonhosted.org/packages/f0/b8/e6825e25deb691ff98cf5c9072ee0605dc2acfca98af70c2d1b1bc75190d/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", size = 143184 }, + { url = "https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", size = 145268 }, + { url = "https://files.pythonhosted.org/packages/74/94/8a5277664f27c3c438546f3eb53b33f5b19568eb7424736bdc440a88a31f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616", size = 147601 }, + { url = "https://files.pythonhosted.org/packages/7c/5f/6d352c51ee763623a98e31194823518e09bfa48be2a7e8383cf691bbb3d0/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", size = 141098 }, + { url = "https://files.pythonhosted.org/packages/78/d4/f5704cb629ba5ab16d1d3d741396aec6dc3ca2b67757c45b0599bb010478/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", size = 149520 }, + { url = "https://files.pythonhosted.org/packages/c5/96/64120b1d02b81785f222b976c0fb79a35875457fa9bb40827678e54d1bc8/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", size = 152852 }, + { url = "https://files.pythonhosted.org/packages/84/c9/98e3732278a99f47d487fd3468bc60b882920cef29d1fa6ca460a1fdf4e6/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", size = 150488 }, + { url = "https://files.pythonhosted.org/packages/13/0e/9c8d4cb99c98c1007cc11eda969ebfe837bbbd0acdb4736d228ccaabcd22/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", size = 146192 }, + { url = "https://files.pythonhosted.org/packages/b2/21/2b6b5b860781a0b49427309cb8670785aa543fb2178de875b87b9cc97746/charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", size = 95550 }, + { url = "https://files.pythonhosted.org/packages/21/5b/1b390b03b1d16c7e382b561c5329f83cc06623916aab983e8ab9239c7d5c/charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", size = 102785 }, + { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "coverage" +version = "7.6.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/d6/2b53ab3ee99f2262e6f0b8369a43f6d66658eab45510331c0b3d5c8c4272/coverage-7.6.12.tar.gz", hash = "sha256:48cfc4641d95d34766ad41d9573cc0f22a48aa88d22657a1fe01dca0dbae4de2", size = 805941 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/7f/4af2ed1d06ce6bee7eafc03b2ef748b14132b0bdae04388e451e4b2c529b/coverage-7.6.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b172f8e030e8ef247b3104902cc671e20df80163b60a203653150d2fc204d1ad", size = 208645 }, + { url = "https://files.pythonhosted.org/packages/dc/60/d19df912989117caa95123524d26fc973f56dc14aecdec5ccd7d0084e131/coverage-7.6.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:641dfe0ab73deb7069fb972d4d9725bf11c239c309ce694dd50b1473c0f641c3", size = 208898 }, + { url = "https://files.pythonhosted.org/packages/bd/10/fecabcf438ba676f706bf90186ccf6ff9f6158cc494286965c76e58742fa/coverage-7.6.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e549f54ac5f301e8e04c569dfdb907f7be71b06b88b5063ce9d6953d2d58574", size = 242987 }, + { url = "https://files.pythonhosted.org/packages/4c/53/4e208440389e8ea936f5f2b0762dcd4cb03281a7722def8e2bf9dc9c3d68/coverage-7.6.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959244a17184515f8c52dcb65fb662808767c0bd233c1d8a166e7cf74c9ea985", size = 239881 }, + { url = "https://files.pythonhosted.org/packages/c4/47/2ba744af8d2f0caa1f17e7746147e34dfc5f811fb65fc153153722d58835/coverage-7.6.12-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bda1c5f347550c359f841d6614fb8ca42ae5cb0b74d39f8a1e204815ebe25750", size = 242142 }, + { url = "https://files.pythonhosted.org/packages/e9/90/df726af8ee74d92ee7e3bf113bf101ea4315d71508952bd21abc3fae471e/coverage-7.6.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ceeb90c3eda1f2d8c4c578c14167dbd8c674ecd7d38e45647543f19839dd6ea", size = 241437 }, + { url = "https://files.pythonhosted.org/packages/f6/af/995263fd04ae5f9cf12521150295bf03b6ba940d0aea97953bb4a6db3e2b/coverage-7.6.12-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f16f44025c06792e0fb09571ae454bcc7a3ec75eeb3c36b025eccf501b1a4c3", size = 239724 }, + { url = "https://files.pythonhosted.org/packages/1c/8e/5bb04f0318805e190984c6ce106b4c3968a9562a400180e549855d8211bd/coverage-7.6.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b076e625396e787448d27a411aefff867db2bffac8ed04e8f7056b07024eed5a", size = 241329 }, + { url = "https://files.pythonhosted.org/packages/9e/9d/fa04d9e6c3f6459f4e0b231925277cfc33d72dfab7fa19c312c03e59da99/coverage-7.6.12-cp312-cp312-win32.whl", hash = "sha256:00b2086892cf06c7c2d74983c9595dc511acca00665480b3ddff749ec4fb2a95", size = 211289 }, + { url = "https://files.pythonhosted.org/packages/53/40/53c7ffe3c0c3fff4d708bc99e65f3d78c129110d6629736faf2dbd60ad57/coverage-7.6.12-cp312-cp312-win_amd64.whl", hash = "sha256:7ae6eabf519bc7871ce117fb18bf14e0e343eeb96c377667e3e5dd12095e0288", size = 212079 }, + { url = "https://files.pythonhosted.org/packages/fb/b2/f655700e1024dec98b10ebaafd0cedbc25e40e4abe62a3c8e2ceef4f8f0a/coverage-7.6.12-py3-none-any.whl", hash = "sha256:eb8668cfbc279a536c633137deeb9435d2962caec279c3f8cf8b91fff6ff8953", size = 200552 }, +] + +[[package]] +name = "distlib" +version = "0.3.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/dd/1bec4c5ddb504ca60fc29472f3d27e8d4da1257a854e1d96742f15c1d02d/distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403", size = 613923 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973 }, +] + +[[package]] +name = "dnspython" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/4a/263763cb2ba3816dd94b08ad3a33d5fdae34ecb856678773cc40a3605829/dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1", size = 345197 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, +] + +[[package]] +name = "email-validator" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/ce/13508a1ec3f8bb981ae4ca79ea40384becc868bfae97fd1c942bb3a001b1/email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7", size = 48967 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521 }, +] + +[[package]] +name = "embedding-api" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "fastapi", extra = ["all"] }, + { name = "pinecone", extra = ["grpc"] }, + { name = "pydantic-settings" }, + { name = "redis" }, + { name = "torch" }, + { name = "transformers" }, +] + +[package.dev-dependencies] +dev = [ + { name = "parameterized" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "python-dotenv" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", extras = ["all"], specifier = "==0.115.8" }, + { name = "pinecone", extras = ["grpc"], specifier = ">=5.4.2,<6.0.0" }, + { name = "pydantic-settings", specifier = ">=2.7.1,<3.0.0" }, + { name = "redis", specifier = ">=5.2.0,<6.0.0" }, + { name = "torch", specifier = "==2.3.1" }, + { name = "transformers", specifier = "==4.45.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "parameterized", specifier = ">=0.9.0,<1.0.0" }, + { name = "pre-commit", specifier = "==4.0.1" }, + { name = "pytest", specifier = "==8.3.4" }, + { name = "pytest-cov", specifier = "==6.0.0" }, + { name = "python-dotenv", specifier = "==1.0.1" }, +] + +[[package]] +name = "fastapi" +version = "0.115.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/b2/5a5dc4affdb6661dea100324e19a7721d5dc524b464fe8e366c093fd7d87/fastapi-0.115.8.tar.gz", hash = "sha256:0ce9111231720190473e222cdf0f07f7206ad7e53ea02beb1d2dc36e2f0741e9", size = 295403 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/7d/2d6ce181d7a5f51dedb8c06206cbf0ec026a99bf145edd309f9e17c3282f/fastapi-0.115.8-py3-none-any.whl", hash = "sha256:753a96dd7e036b34eeef8babdfcfe3f28ff79648f86551eb36bfc1b0bf4a8cbf", size = 94814 }, +] + +[package.optional-dependencies] +all = [ + { name = "email-validator" }, + { name = "fastapi-cli", extra = ["standard"] }, + { name = "httpx" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "orjson" }, + { name = "pydantic-extra-types" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "pyyaml" }, + { name = "ujson" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "fastapi-cli" +version = "0.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich-toolkit" }, + { name = "typer" }, + { name = "uvicorn", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/73/82a5831fbbf8ed75905bacf5b2d9d3dfd6f04d6968b29fe6f72a5ae9ceb1/fastapi_cli-0.0.7.tar.gz", hash = "sha256:02b3b65956f526412515907a0793c9094abd4bfb5457b389f645b0ea6ba3605e", size = 16753 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/e6/5daefc851b514ce2287d8f5d358ae4341089185f78f3217a69d0ce3a390c/fastapi_cli-0.0.7-py3-none-any.whl", hash = "sha256:d549368ff584b2804336c61f192d86ddea080c11255f375959627911944804f4", size = 10705 }, +] + +[package.optional-dependencies] +standard = [ + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "filelock" +version = "3.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9c/0b15fb47b464e1b663b1acd1253a062aa5feecb07d4e597daea542ebd2b5/filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e", size = 18027 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/ec/00d68c4ddfedfe64159999e5f8a98fb8442729a63e2077eb9dcd89623d27/filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338", size = 16164 }, +] + +[[package]] +name = "fsspec" +version = "2025.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/79/68612ed99700e6413de42895aa725463e821a6b3be75c87fcce1b4af4c70/fsspec-2025.2.0.tar.gz", hash = "sha256:1c24b16eaa0a1798afa0337aa0db9b256718ab2a89c425371f5628d22c3b6afd", size = 292283 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/94/758680531a00d06e471ef649e4ec2ed6bf185356a7f9fbfbb7368a40bd49/fsspec-2025.2.0-py3-none-any.whl", hash = "sha256:9de2ad9ce1f85e1931858535bc882543171d197001a0a5eb2ddc04f1781ab95b", size = 184484 }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.67.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/e1/fbffb85a624f1404133b5bb624834e77e0f549e2b8548146fe18c56e1411/googleapis_common_protos-1.67.0.tar.gz", hash = "sha256:21398025365f138be356d5923e9168737d94d46a72aefee4a6110a1f23463c86", size = 57344 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/30/2bd0eb03a7dee7727cd2ec643d1e992979e62d5e7443507381cce0455132/googleapis_common_protos-1.67.0-py2.py3-none-any.whl", hash = "sha256:579de760800d13616f51cf8be00c876f00a9f146d3e6510e19d1f4111758b741", size = 164985 }, +] + +[[package]] +name = "grpcio" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/e1/4b21b5017c33f3600dcc32b802bb48fe44a4d36d6c066f52650c7c2690fa/grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56", size = 12788932 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/a4/ddbda79dd176211b518f0f3795af78b38727a31ad32bc149d6a7b910a731/grpcio-1.70.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:ef4c14508299b1406c32bdbb9fb7b47612ab979b04cf2b27686ea31882387cff", size = 5198135 }, + { url = "https://files.pythonhosted.org/packages/30/5c/60eb8a063ea4cb8d7670af8fac3f2033230fc4b75f62669d67c66ac4e4b0/grpcio-1.70.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:aa47688a65643afd8b166928a1da6247d3f46a2784d301e48ca1cc394d2ffb40", size = 11447529 }, + { url = "https://files.pythonhosted.org/packages/fb/b9/1bf8ab66729f13b44e8f42c9de56417d3ee6ab2929591cfee78dce749b57/grpcio-1.70.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:880bfb43b1bb8905701b926274eafce5c70a105bc6b99e25f62e98ad59cb278e", size = 5664484 }, + { url = "https://files.pythonhosted.org/packages/d1/06/2f377d6906289bee066d96e9bdb91e5e96d605d173df9bb9856095cccb57/grpcio-1.70.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e654c4b17d07eab259d392e12b149c3a134ec52b11ecdc6a515b39aceeec898", size = 6303739 }, + { url = "https://files.pythonhosted.org/packages/ae/50/64c94cfc4db8d9ed07da71427a936b5a2bd2b27c66269b42fbda82c7c7a4/grpcio-1.70.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2394e3381071045a706ee2eeb6e08962dd87e8999b90ac15c55f56fa5a8c9597", size = 5910417 }, + { url = "https://files.pythonhosted.org/packages/53/89/8795dfc3db4389c15554eb1765e14cba8b4c88cc80ff828d02f5572965af/grpcio-1.70.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b3c76701428d2df01964bc6479422f20e62fcbc0a37d82ebd58050b86926ef8c", size = 6626797 }, + { url = "https://files.pythonhosted.org/packages/9c/b2/6a97ac91042a2c59d18244c479ee3894e7fb6f8c3a90619bb5a7757fa30c/grpcio-1.70.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac073fe1c4cd856ebcf49e9ed6240f4f84d7a4e6ee95baa5d66ea05d3dd0df7f", size = 6190055 }, + { url = "https://files.pythonhosted.org/packages/86/2b/28db55c8c4d156053a8c6f4683e559cd0a6636f55a860f87afba1ac49a51/grpcio-1.70.0-cp312-cp312-win32.whl", hash = "sha256:cd24d2d9d380fbbee7a5ac86afe9787813f285e684b0271599f95a51bce33528", size = 3600214 }, + { url = "https://files.pythonhosted.org/packages/17/c3/a7a225645a965029ed432e5b5e9ed959a574e62100afab553eef58be0e37/grpcio-1.70.0-cp312-cp312-win_amd64.whl", hash = "sha256:0495c86a55a04a874c7627fd33e5beaee771917d92c0e6d9d797628ac40e7655", size = 4292538 }, +] + +[[package]] +name = "h11" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/38/3af3d3633a34a3316095b39c8e8fb4853a28a536e55d347bd8d8e9a14b03/h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", size = 100418 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 }, +] + +[[package]] +name = "httpcore" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/41/d7d0a89eb493922c37d343b607bc1b5da7f5be7e383740b4753ad8943e90/httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c", size = 85196 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/f5/72347bc88306acb359581ac4d52f23c0ef445b57157adedb9aee0cd689d2/httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd", size = 78551 }, +] + +[[package]] +name = "httptools" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/9a/ce5e1f7e131522e6d3426e8e7a490b3a01f39a6696602e1c4f33f9e94277/httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c", size = 240639 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/0e/d0b71465c66b9185f90a091ab36389a7352985fe857e352801c39d6127c8/httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2", size = 200683 }, + { url = "https://files.pythonhosted.org/packages/e2/b8/412a9bb28d0a8988de3296e01efa0bd62068b33856cdda47fe1b5e890954/httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44", size = 104337 }, + { url = "https://files.pythonhosted.org/packages/9b/01/6fb20be3196ffdc8eeec4e653bc2a275eca7f36634c86302242c4fbb2760/httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1", size = 508796 }, + { url = "https://files.pythonhosted.org/packages/f7/d8/b644c44acc1368938317d76ac991c9bba1166311880bcc0ac297cb9d6bd7/httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2", size = 510837 }, + { url = "https://files.pythonhosted.org/packages/52/d8/254d16a31d543073a0e57f1c329ca7378d8924e7e292eda72d0064987486/httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81", size = 485289 }, + { url = "https://files.pythonhosted.org/packages/5f/3c/4aee161b4b7a971660b8be71a92c24d6c64372c1ab3ae7f366b3680df20f/httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f", size = 489779 }, + { url = "https://files.pythonhosted.org/packages/12/b7/5cae71a8868e555f3f67a50ee7f673ce36eac970f029c0c5e9d584352961/httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970", size = 88634 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + +[[package]] +name = "huggingface-hub" +version = "0.29.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/37/797d6476f13e5ef6af5fc48a5d641d32b39c37e166ccf40c3714c5854a85/huggingface_hub-0.29.1.tar.gz", hash = "sha256:9524eae42077b8ff4fc459ceb7a514eca1c1232b775276b009709fe2a084f250", size = 389776 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/05/75b90de9093de0aadafc868bb2fa7c57651fd8f45384adf39bd77f63980d/huggingface_hub-0.29.1-py3-none-any.whl", hash = "sha256:352f69caf16566c7b6de84b54a822f6238e17ddd8ae3da4f8f2272aea5b198d5", size = 468049 }, +] + +[[package]] +name = "identify" +version = "2.6.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/d1/524aa3350f78bcd714d148ade6133d67d6b7de2cdbae7d99039c024c9a25/identify-2.6.7.tar.gz", hash = "sha256:3fa266b42eba321ee0b2bb0936a6a6b9e36a1351cbb69055b3082f4193035684", size = 99260 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/00/1fd4a117c6c93f2dcc5b7edaeaf53ea45332ef966429be566ca16c2beb94/identify-2.6.7-py2.py3-none-any.whl", hash = "sha256:155931cb617a401807b09ecec6635d6c692d180090a1cedca8ef7d58ba5b6aa0", size = 99097 }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, +] + +[[package]] +name = "intel-openmp" +version = "2021.4.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/18/527f247d673ff84c38e0b353b6901539b99e83066cd505be42ad341ab16d/intel_openmp-2021.4.0-py2.py3-none-win32.whl", hash = "sha256:6e863d8fd3d7e8ef389d52cf97a50fe2afe1a19247e8c0d168ce021546f96fc9", size = 1860605 }, + { url = "https://files.pythonhosted.org/packages/6f/21/b590c0cc3888b24f2ac9898c41d852d7454a1695fbad34bee85dba6dc408/intel_openmp-2021.4.0-py2.py3-none-win_amd64.whl", hash = "sha256:eef4c8bcc8acefd7f5cd3b9384dbf73d59e2c99fc56545712ded913f43c4a94f", size = 3516906 }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234 }, +] + +[[package]] +name = "jinja2" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/92/b3130cbbf5591acf9ade8708c365f3238046ac7cb8ccba6e81abccb0ccff/jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb", size = 244674 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/0f/2ba5fbcd631e3e88689309dbe978c5769e883e4b84ebfe7da30b43275c5a/jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb", size = 134596 }, +] + +[[package]] +name = "lz4" +version = "4.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/bc/b2e79af05be82841706ddd7d78059e5f78e6ca5828f92034394b54e303b7/lz4-4.4.3.tar.gz", hash = "sha256:91ed5b71f9179bf3dbfe85d92b52d4b53de2e559aa4daa3b7de18e0dd24ad77d", size = 171848 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/40/9a6db39950ba872c3b75ccf4826288a46b109ded1d20508d6044cc36e33c/lz4-4.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:43461e439ef71d49bb0ee3a1719494cd952a58d205496698e0cde866f22006bc", size = 220484 }, + { url = "https://files.pythonhosted.org/packages/b7/25/edd77ac155e167f0d183f0a30be1665ab581f77108ca6e19d628cd381e42/lz4-4.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ae50a175fb7b900f7aa42575f4fe99c32ca0ff57e5a8c1fd25e1243e67409db", size = 189473 }, + { url = "https://files.pythonhosted.org/packages/55/59/80673123358c0e0b2b773b74ac3d14717e35cfcceac5243b61f88e08b883/lz4-4.4.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38df5929ffefa9dda120ba1790a2e94fda81916c5aaa1ee652f4b1e515ebb9ed", size = 1264959 }, + { url = "https://files.pythonhosted.org/packages/ea/69/24a3d8609f9a05d93b407d93842d35e953bebf625cb4d128a9105c983d59/lz4-4.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b45914f25d916324531d0259072b402c5f99b67c6e9ac8cbc3d49935aeb1d97", size = 1184842 }, + { url = "https://files.pythonhosted.org/packages/88/6e/680d0fc3dbec31aaffcad23d2e429b2974253ffda4636ea8a7e2cce5461c/lz4-4.4.3-cp312-cp312-win32.whl", hash = "sha256:848c5b040d2cfe35097b1d65d1095d83a3f86374ce879e189533f61405d8763b", size = 88157 }, + { url = "https://files.pythonhosted.org/packages/d4/c9/8fcaf3445d3dc2973861b1a1a27090e23952807facabcf092a587ff77754/lz4-4.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:b1d179bdefd9ddb8d11d7de7825e73fb957511b722a8cb484e417885c210e68c", size = 99833 }, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348 }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149 }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118 }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993 }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178 }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319 }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352 }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097 }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, +] + +[[package]] +name = "mkl" +version = "2021.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "intel-openmp" }, + { name = "tbb" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/c6/892fe3bc91e811b78e4f85653864f2d92541d5e5c306b0cb3c2311e9ca64/mkl-2021.4.0-py2.py3-none-win32.whl", hash = "sha256:439c640b269a5668134e3dcbcea4350459c4a8bc46469669b2d67e07e3d330e8", size = 129048357 }, + { url = "https://files.pythonhosted.org/packages/fe/1c/5f6dbf18e8b73e0a5472466f0ea8d48ce9efae39bd2ff38cebf8dce61259/mkl-2021.4.0-py2.py3-none-win_amd64.whl", hash = "sha256:ceef3cafce4c009dd25f65d7ad0d833a0fbadc3d8903991ec92351fe5de1e718", size = 228499609 }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198 }, +] + +[[package]] +name = "networkx" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263 }, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 }, +] + +[[package]] +name = "numpy" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/90/8956572f5c4ae52201fdec7ba2044b2c882832dcec7d5d0922c9e9acf2de/numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020", size = 20262700 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ec/43628dcf98466e087812142eec6d1c1a6c6bdfdad30a0aa07b872dc01f6f/numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d", size = 20929458 }, + { url = "https://files.pythonhosted.org/packages/9b/c0/2f4225073e99a5c12350954949ed19b5d4a738f541d33e6f7439e33e98e4/numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95", size = 14115299 }, + { url = "https://files.pythonhosted.org/packages/ca/fa/d2c5575d9c734a7376cc1592fae50257ec95d061b27ee3dbdb0b3b551eb2/numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea", size = 5145723 }, + { url = "https://files.pythonhosted.org/packages/eb/dc/023dad5b268a7895e58e791f28dc1c60eb7b6c06fcbc2af8538ad069d5f3/numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532", size = 6678797 }, + { url = "https://files.pythonhosted.org/packages/3f/19/bcd641ccf19ac25abb6fb1dcd7744840c11f9d62519d7057b6ab2096eb60/numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e", size = 14067362 }, + { url = "https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe", size = 16116679 }, + { url = "https://files.pythonhosted.org/packages/d0/a1/e90f7aa66512be3150cb9d27f3d9995db330ad1b2046474a13b7040dfd92/numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021", size = 15264272 }, + { url = "https://files.pythonhosted.org/packages/dc/b6/50bd027cca494de4fa1fc7bf1662983d0ba5f256fa0ece2c376b5eb9b3f0/numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8", size = 17880549 }, + { url = "https://files.pythonhosted.org/packages/96/30/f7bf4acb5f8db10a96f73896bdeed7a63373137b131ca18bd3dab889db3b/numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe", size = 6293394 }, + { url = "https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d", size = 12626357 }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.1.3.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/6d/121efd7382d5b0284239f4ab1fc1590d86d34ed4a4a2fdb13b30ca8e5740/nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:ee53ccca76a6fc08fb9701aa95b6ceb242cdaab118c3bb152af4e579af792728", size = 410594774 }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.1.105" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/00/6b218edd739ecfc60524e585ba8e6b00554dd908de2c9c66c1af3e44e18d/nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:e54fde3983165c624cb79254ae9818a456eb6e87a7fd4d56a2352c24ee542d7e", size = 14109015 }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.1.105" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/9f/c64c03f49d6fbc56196664d05dba14e3a561038a81a638eeb47f4d4cfd48/nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:339b385f50c309763ca65456ec75e17bbefcbbf2893f462cb8b90584cd27a1c2", size = 23671734 }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.1.105" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/d5/c68b1d2cdfcc59e72e8a5949a37ddb22ae6cade80cd4a57a84d4c8b55472/nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:6e258468ddf5796e25f1dc591a31029fa317d97a0a94ed93468fc86301d61e40", size = 823596 }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "8.9.2.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/74/a2e2be7fb83aaedec84f391f082cf765dfb635e7caa9b49065f73e4835d8/nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl", hash = "sha256:5ccb288774fdfb07a7e7025ffec286971c06d8d7b4fb162525334616d7629ff9", size = 731725872 }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.0.2.54" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/94/eb540db023ce1d162e7bea9f8f5aa781d57c65aed513c33ee9a5123ead4d/nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl", hash = "sha256:794e3948a1aa71fd817c3775866943936774d1c14e7628c74f6f7417224cdf56", size = 121635161 }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.2.106" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/31/4890b1c9abc496303412947fc7dcea3d14861720642b49e8ceed89636705/nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:9d264c5036dde4e64f1de8c50ae753237c12e0b1348738169cd0f8a536c0e1e0", size = 56467784 }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.4.5.107" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/1d/8de1e5c67099015c834315e333911273a8c6aaba78923dd1d1e25fc5f217/nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd", size = 124161928 }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.1.0.106" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/5b/cfaeebf25cd9fdec14338ccb16f6b2c4c7fa9163aefcf057d86b9cc248bb/nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c", size = 195958278 }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.20.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/2a/0a131f572aa09f741c30ccd45a8e56316e8be8dfc7bc19bf0ab7cfef7b19/nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:057f6bf9685f75215d0c53bf3ac4a10b3e6578351de307abad9e18a99182af56", size = 176249402 }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.61" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/f8/9d85593582bd99b8d7c65634d2304780aefade049b2b94d96e44084be90b/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:45fd79f2ae20bd67e8bc411055939049873bfd8fac70ff13bd4865e0b9bdab17", size = 39243473 }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.1.105" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/d3/8057f0587683ed2fcd4dbfbdfdfa807b9160b809976099d36b8f60d08f03/nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:dc21cf308ca5691e7c04d962e213f8a4aa9bbfa23d95412f452254c2caeb09e5", size = 99138 }, +] + +[[package]] +name = "orjson" +version = "3.10.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/5dea21763eeff8c1590076918a446ea3d6140743e0e36f58f369928ed0f4/orjson-3.10.15.tar.gz", hash = "sha256:05ca7fe452a2e9d8d9d706a2984c95b9c2ebc5db417ce0b7a49b91d50642a23e", size = 5282482 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/85/22fe737188905a71afcc4bf7cc4c79cd7f5bbe9ed1fe0aac4ce4c33edc30/orjson-3.10.15-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d11c0714fc85bfcf36ada1179400862da3288fc785c30e8297844c867d7505a", size = 249504 }, + { url = "https://files.pythonhosted.org/packages/48/b7/2622b29f3afebe938a0a9037e184660379797d5fd5234e5998345d7a5b43/orjson-3.10.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dba5a1e85d554e3897fa9fe6fbcff2ed32d55008973ec9a2b992bd9a65d2352d", size = 125080 }, + { url = "https://files.pythonhosted.org/packages/ce/8f/0b72a48f4403d0b88b2a41450c535b3e8989e8a2d7800659a967efc7c115/orjson-3.10.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7723ad949a0ea502df656948ddd8b392780a5beaa4c3b5f97e525191b102fff0", size = 150121 }, + { url = "https://files.pythonhosted.org/packages/06/ec/acb1a20cd49edb2000be5a0404cd43e3c8aad219f376ac8c60b870518c03/orjson-3.10.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6fd9bc64421e9fe9bd88039e7ce8e58d4fead67ca88e3a4014b143cec7684fd4", size = 139796 }, + { url = "https://files.pythonhosted.org/packages/33/e1/f7840a2ea852114b23a52a1c0b2bea0a1ea22236efbcdb876402d799c423/orjson-3.10.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dadba0e7b6594216c214ef7894c4bd5f08d7c0135f4dd0145600be4fbcc16767", size = 154636 }, + { url = "https://files.pythonhosted.org/packages/fa/da/31543337febd043b8fa80a3b67de627669b88c7b128d9ad4cc2ece005b7a/orjson-3.10.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48f59114fe318f33bbaee8ebeda696d8ccc94c9e90bc27dbe72153094e26f41", size = 130621 }, + { url = "https://files.pythonhosted.org/packages/ed/78/66115dc9afbc22496530d2139f2f4455698be444c7c2475cb48f657cefc9/orjson-3.10.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:035fb83585e0f15e076759b6fedaf0abb460d1765b6a36f48018a52858443514", size = 138516 }, + { url = "https://files.pythonhosted.org/packages/22/84/cd4f5fb5427ffcf823140957a47503076184cb1ce15bcc1165125c26c46c/orjson-3.10.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d13b7fe322d75bf84464b075eafd8e7dd9eae05649aa2a5354cfa32f43c59f17", size = 130762 }, + { url = "https://files.pythonhosted.org/packages/93/1f/67596b711ba9f56dd75d73b60089c5c92057f1130bb3a25a0f53fb9a583b/orjson-3.10.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7066b74f9f259849629e0d04db6609db4cf5b973248f455ba5d3bd58a4daaa5b", size = 414700 }, + { url = "https://files.pythonhosted.org/packages/7c/0c/6a3b3271b46443d90efb713c3e4fe83fa8cd71cda0d11a0f69a03f437c6e/orjson-3.10.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:88dc3f65a026bd3175eb157fea994fca6ac7c4c8579fc5a86fc2114ad05705b7", size = 141077 }, + { url = "https://files.pythonhosted.org/packages/3b/9b/33c58e0bfc788995eccd0d525ecd6b84b40d7ed182dd0751cd4c1322ac62/orjson-3.10.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b342567e5465bd99faa559507fe45e33fc76b9fb868a63f1642c6bc0735ad02a", size = 129898 }, + { url = "https://files.pythonhosted.org/packages/01/c1/d577ecd2e9fa393366a1ea0a9267f6510d86e6c4bb1cdfb9877104cac44c/orjson-3.10.15-cp312-cp312-win32.whl", hash = "sha256:0a4f27ea5617828e6b58922fdbec67b0aa4bb844e2d363b9244c47fa2180e665", size = 142566 }, + { url = "https://files.pythonhosted.org/packages/ed/eb/a85317ee1732d1034b92d56f89f1de4d7bf7904f5c8fb9dcdd5b1c83917f/orjson-3.10.15-cp312-cp312-win_amd64.whl", hash = "sha256:ef5b87e7aa9545ddadd2309efe6824bd3dd64ac101c15dae0f2f597911d46eaa", size = 133732 }, +] + +[[package]] +name = "packaging" +version = "24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, +] + +[[package]] +name = "parameterized" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/49/00c0c0cc24ff4266025a53e41336b79adaa5a4ebfad214f433d623f9865e/parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1", size = 24351 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2f/804f58f0b856ab3bf21617cccf5b39206e6c4c94c2cd227bde125ea6105f/parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b", size = 20475 }, +] + +[[package]] +name = "pinecone" +version = "5.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "pinecone-plugin-inference" }, + { name = "pinecone-plugin-interface" }, + { name = "python-dateutil" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/4e/3376f99662f56e7462a4c444edc19e0cbb20676f03b8f70f56a964f34de4/pinecone-5.4.2.tar.gz", hash = "sha256:23e8aaa73b400bb11a3b626c4129284fb170f19025b82f65bd89cbb0dab2b873", size = 191780 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/a4/f7214bf02bb2edb29778e35fa6e73e2d188c403e6d9c2b6945f660a776b3/pinecone-5.4.2-py3-none-any.whl", hash = "sha256:1fad082c66a50a229b58cda0c3a1fa0083532dc9de8303015fe4071cb25c19a8", size = 427295 }, +] + +[package.optional-dependencies] +grpc = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "lz4" }, + { name = "protobuf" }, + { name = "protoc-gen-openapiv2" }, +] + +[[package]] +name = "pinecone-plugin-inference" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pinecone-plugin-interface" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/82/09f6fb3c9d3b005c5b110d323a98f848f57babb1394ebea9f72e26f68242/pinecone_plugin_inference-3.1.0.tar.gz", hash = "sha256:eff826178e1fe448577be2ff3d8dbb072befbbdc2d888e214624523a1c37cd8d", size = 49315 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/45/4ae4e38439919584c2d34b6bef5d0ef8d068030871dd4da911d174840ee6/pinecone_plugin_inference-3.1.0-py3-none-any.whl", hash = "sha256:96e861527bd41e90d58b7e76abd4e713d9af28f63e76a51864dfb9cf7180e3df", size = 87477 }, +] + +[[package]] +name = "pinecone-plugin-interface" +version = "0.0.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/fb/e8a4063264953ead9e2b24d9b390152c60f042c951c47f4592e9996e57ff/pinecone_plugin_interface-0.0.7.tar.gz", hash = "sha256:b8e6675e41847333aa13923cc44daa3f85676d7157324682dc1640588a982846", size = 3370 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/1d/a21fdfcd6d022cb64cef5c2a29ee6691c6c103c4566b41646b080b7536a5/pinecone_plugin_interface-0.0.7-py3-none-any.whl", hash = "sha256:875857ad9c9fc8bbc074dbe780d187a2afd21f5bfe0f3b08601924a61ef1bba8", size = 6249 }, +] + +[[package]] +name = "platformdirs" +version = "4.3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/fc/128cc9cb8f03208bdbf93d3aa862e16d376844a14f9a0ce5cf4507372de4/platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907", size = 21302 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb", size = 18439 }, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, +] + +[[package]] +name = "pre-commit" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c8/e22c292035f1bac8b9f5237a2622305bc0304e776080b246f3df57c4ff9f/pre_commit-4.0.1.tar.gz", hash = "sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2", size = 191678 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/8f/496e10d51edd6671ebe0432e33ff800aa86775d2d147ce7d43389324a525/pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878", size = 218713 }, +] + +[[package]] +name = "protobuf" +version = "4.25.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/48/d5/cccc7e82bbda9909ced3e7a441a24205ea07fea4ce23a772743c0c7611fa/protobuf-4.25.6.tar.gz", hash = "sha256:f8cfbae7c5afd0d0eaccbe73267339bff605a2315860bb1ba08eb66670a9a91f", size = 380631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/41/0ff3559d9a0fbdb37c9452f2b84e61f7784d8d7b9850182c7ef493f523ee/protobuf-4.25.6-cp310-abi3-win32.whl", hash = "sha256:61df6b5786e2b49fc0055f636c1e8f0aff263808bb724b95b164685ac1bcc13a", size = 392454 }, + { url = "https://files.pythonhosted.org/packages/79/84/c700d6c3f3be770495b08a1c035e330497a31420e4a39a24c22c02cefc6c/protobuf-4.25.6-cp310-abi3-win_amd64.whl", hash = "sha256:b8f837bfb77513fe0e2f263250f423217a173b6d85135be4d81e96a4653bcd3c", size = 413443 }, + { url = "https://files.pythonhosted.org/packages/b7/03/361e87cc824452376c2abcef0eabd18da78a7439479ec6541cf29076a4dc/protobuf-4.25.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6d4381f2417606d7e01750e2729fe6fbcda3f9883aa0c32b51d23012bded6c91", size = 394246 }, + { url = "https://files.pythonhosted.org/packages/64/d5/7dbeb69b74fa88f297c6d8f11b7c9cef0c2e2fb1fdf155c2ca5775cfa998/protobuf-4.25.6-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:5dd800da412ba7f6f26d2c08868a5023ce624e1fdb28bccca2dc957191e81fb5", size = 293714 }, + { url = "https://files.pythonhosted.org/packages/d4/f0/6d5c100f6b18d973e86646aa5fc09bc12ee88a28684a56fd95511bceee68/protobuf-4.25.6-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4434ff8bb5576f9e0c78f47c41cdf3a152c0b44de475784cd3fd170aef16205a", size = 294634 }, + { url = "https://files.pythonhosted.org/packages/71/eb/be11a1244d0e58ee04c17a1f939b100199063e26ecca8262c04827fe0bf5/protobuf-4.25.6-py3-none-any.whl", hash = "sha256:07972021c8e30b870cfc0863409d033af940213e0e7f64e27fe017b929d2c9f7", size = 156466 }, +] + +[[package]] +name = "protoc-gen-openapiv2" +version = "0.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/d2/84fecd8df61640226c726c12ad7ddd2a7666a7cd7f898b9a5b72e3a66d44/protoc-gen-openapiv2-0.0.1.tar.gz", hash = "sha256:6f79188d842c13177c9c0558845442c340b43011bf67dfef1dfc3bc067506409", size = 7323 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/ac/bd8961859d8f3f81530465d2ce9b165627e961c00348939009bac2700cc6/protoc_gen_openapiv2-0.0.1-py3-none-any.whl", hash = "sha256:18090c8be3877c438e7da0f7eb7cace45a9a210306bca4707708dbad367857be", size = 7883 }, +] + +[[package]] +name = "pydantic" +version = "2.10.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/ae/d5220c5c52b158b1de7ca89fc5edb72f304a70a4c540c84c8844bf4008de/pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236", size = 761681 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/3c/8cc1cc84deffa6e25d2d0c688ebb80635dfdbf1dbea3e30c541c8cf4d860/pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584", size = 431696 }, +] + +[[package]] +name = "pydantic-core" +version = "2.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/01/f3e5ac5e7c25833db5eb555f7b7ab24cd6f8c322d3a3ad2d67a952dc0abc/pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39", size = 413443 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/74/51c8a5482ca447871c93e142d9d4a92ead74de6c8dc5e66733e22c9bba89/pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0", size = 1893127 }, + { url = "https://files.pythonhosted.org/packages/d3/f3/c97e80721735868313c58b89d2de85fa80fe8dfeeed84dc51598b92a135e/pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef", size = 1811340 }, + { url = "https://files.pythonhosted.org/packages/9e/91/840ec1375e686dbae1bd80a9e46c26a1e0083e1186abc610efa3d9a36180/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7", size = 1822900 }, + { url = "https://files.pythonhosted.org/packages/f6/31/4240bc96025035500c18adc149aa6ffdf1a0062a4b525c932065ceb4d868/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934", size = 1869177 }, + { url = "https://files.pythonhosted.org/packages/fa/20/02fbaadb7808be578317015c462655c317a77a7c8f0ef274bc016a784c54/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6", size = 2038046 }, + { url = "https://files.pythonhosted.org/packages/06/86/7f306b904e6c9eccf0668248b3f272090e49c275bc488a7b88b0823444a4/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c", size = 2685386 }, + { url = "https://files.pythonhosted.org/packages/8d/f0/49129b27c43396581a635d8710dae54a791b17dfc50c70164866bbf865e3/pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2", size = 1997060 }, + { url = "https://files.pythonhosted.org/packages/0d/0f/943b4af7cd416c477fd40b187036c4f89b416a33d3cc0ab7b82708a667aa/pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4", size = 2004870 }, + { url = "https://files.pythonhosted.org/packages/35/40/aea70b5b1a63911c53a4c8117c0a828d6790483f858041f47bab0b779f44/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3", size = 1999822 }, + { url = "https://files.pythonhosted.org/packages/f2/b3/807b94fd337d58effc5498fd1a7a4d9d59af4133e83e32ae39a96fddec9d/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4", size = 2130364 }, + { url = "https://files.pythonhosted.org/packages/fc/df/791c827cd4ee6efd59248dca9369fb35e80a9484462c33c6649a8d02b565/pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57", size = 2158303 }, + { url = "https://files.pythonhosted.org/packages/9b/67/4e197c300976af185b7cef4c02203e175fb127e414125916bf1128b639a9/pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc", size = 1834064 }, + { url = "https://files.pythonhosted.org/packages/1f/ea/cd7209a889163b8dcca139fe32b9687dd05249161a3edda62860430457a5/pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9", size = 1989046 }, + { url = "https://files.pythonhosted.org/packages/bc/49/c54baab2f4658c26ac633d798dab66b4c3a9bbf47cff5284e9c182f4137a/pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b", size = 1885092 }, +] + +[[package]] +name = "pydantic-extra-types" +version = "2.10.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/ed/69f3f3de12c02ebd58b2f66ffb73d0f5a1b10b322227897499753cebe818/pydantic_extra_types-2.10.2.tar.gz", hash = "sha256:934d59ab7a02ff788759c3a97bc896f5cfdc91e62e4f88ea4669067a73f14b98", size = 86893 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/da/86bc9addde8a24348ac15f8f7dcb853f78e9573c7667800dd9bc60558678/pydantic_extra_types-2.10.2-py3-none-any.whl", hash = "sha256:9eccd55a2b7935cea25f0a67f6ff763d55d80c41d86b887d88915412ccf5b7fa", size = 35473 }, +] + +[[package]] +name = "pydantic-settings" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/7b/c58a586cd7d9ac66d2ee4ba60ca2d241fa837c02bca9bea80a9a8c3d22a9/pydantic_settings-2.7.1.tar.gz", hash = "sha256:10c9caad35e64bfb3c2fbf70a078c0e25cc92499782e5200747f942a065dec93", size = 79920 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, +] + +[[package]] +name = "pygments" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, +] + +[[package]] +name = "pytest" +version = "8.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/35/30e0d83068951d90a01852cb1cef56e5d8a09d20c7f511634cc2f7e0372a/pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761", size = 1445919 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/92/76a1c94d3afee238333bc0a42b82935dd8f9cf8ce9e336ff87ee14d9e1cf/pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6", size = 343083 }, +] + +[[package]] +name = "pytest-cov" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/45/9b538de8cef30e17c7b45ef42f538a94889ed6a16f2387a6c89e73220651/pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0", size = 66945 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/3b/48e79f2cd6a61dbbd4807b4ed46cb564b4fd50a76166b1c4ea5c1d9e2371/pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35", size = 22949 }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "python-dotenv" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/57/e84d88dfe0aec03b7a2d4327012c1627ab5f03652216c63d49846d7a6c58/python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", size = 39115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 }, +] + +[[package]] +name = "python-multipart" +version = "0.0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, +] + +[[package]] +name = "redis" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502 }, +] + +[[package]] +name = "regex" +version = "2024.11.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/30/9a87ce8336b172cc232a0db89a3af97929d06c11ceaa19d97d84fa90a8f8/regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a", size = 483781 }, + { url = "https://files.pythonhosted.org/packages/01/e8/00008ad4ff4be8b1844786ba6636035f7ef926db5686e4c0f98093612add/regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9", size = 288455 }, + { url = "https://files.pythonhosted.org/packages/60/85/cebcc0aff603ea0a201667b203f13ba75d9fc8668fab917ac5b2de3967bc/regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2", size = 284759 }, + { url = "https://files.pythonhosted.org/packages/94/2b/701a4b0585cb05472a4da28ee28fdfe155f3638f5e1ec92306d924e5faf0/regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4", size = 794976 }, + { url = "https://files.pythonhosted.org/packages/4b/bf/fa87e563bf5fee75db8915f7352e1887b1249126a1be4813837f5dbec965/regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577", size = 833077 }, + { url = "https://files.pythonhosted.org/packages/a1/56/7295e6bad94b047f4d0834e4779491b81216583c00c288252ef625c01d23/regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3", size = 823160 }, + { url = "https://files.pythonhosted.org/packages/fb/13/e3b075031a738c9598c51cfbc4c7879e26729c53aa9cca59211c44235314/regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e", size = 796896 }, + { url = "https://files.pythonhosted.org/packages/24/56/0b3f1b66d592be6efec23a795b37732682520b47c53da5a32c33ed7d84e3/regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe", size = 783997 }, + { url = "https://files.pythonhosted.org/packages/f9/a1/eb378dada8b91c0e4c5f08ffb56f25fcae47bf52ad18f9b2f33b83e6d498/regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e", size = 781725 }, + { url = "https://files.pythonhosted.org/packages/83/f2/033e7dec0cfd6dda93390089864732a3409246ffe8b042e9554afa9bff4e/regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29", size = 789481 }, + { url = "https://files.pythonhosted.org/packages/83/23/15d4552ea28990a74e7696780c438aadd73a20318c47e527b47a4a5a596d/regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39", size = 852896 }, + { url = "https://files.pythonhosted.org/packages/e3/39/ed4416bc90deedbfdada2568b2cb0bc1fdb98efe11f5378d9892b2a88f8f/regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51", size = 860138 }, + { url = "https://files.pythonhosted.org/packages/93/2d/dd56bb76bd8e95bbce684326302f287455b56242a4f9c61f1bc76e28360e/regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad", size = 787692 }, + { url = "https://files.pythonhosted.org/packages/0b/55/31877a249ab7a5156758246b9c59539abbeba22461b7d8adc9e8475ff73e/regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54", size = 262135 }, + { url = "https://files.pythonhosted.org/packages/38/ec/ad2d7de49a600cdb8dd78434a1aeffe28b9d6fc42eb36afab4a27ad23384/regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b", size = 273567 }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, +] + +[[package]] +name = "rich" +version = "13.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 }, +] + +[[package]] +name = "rich-toolkit" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/8a/71cfbf6bf6257ea785d1f030c22468f763eea1b3e5417620f2ba9abd6dca/rich_toolkit-0.13.2.tar.gz", hash = "sha256:fea92557530de7c28f121cbed572ad93d9e0ddc60c3ca643f1b831f2f56b95d3", size = 72288 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/1b/1c2f43af46456050b27810a7a013af8a7e12bc545a0cdc00eb0df55eb769/rich_toolkit-0.13.2-py3-none-any.whl", hash = "sha256:f3f6c583e5283298a2f7dbd3c65aca18b7f818ad96174113ab5bec0b0e35ed61", size = 13566 }, +] + +[[package]] +name = "safetensors" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/4f/2ef9ef1766f8c194b01b67a63a444d2e557c8fe1d82faf3ebd85f370a917/safetensors-0.5.2.tar.gz", hash = "sha256:cb4a8d98ba12fa016f4241932b1fc5e702e5143f5374bba0bbcf7ddc1c4cf2b8", size = 66957 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/d1/017e31e75e274492a11a456a9e7c171f8f7911fe50735b4ec6ff37221220/safetensors-0.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:45b6092997ceb8aa3801693781a71a99909ab9cc776fbc3fa9322d29b1d3bef2", size = 427067 }, + { url = "https://files.pythonhosted.org/packages/24/84/e9d3ff57ae50dd0028f301c9ee064e5087fe8b00e55696677a0413c377a7/safetensors-0.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6d0d6a8ee2215a440e1296b843edf44fd377b055ba350eaba74655a2fe2c4bae", size = 408856 }, + { url = "https://files.pythonhosted.org/packages/f1/1d/fe95f5dd73db16757b11915e8a5106337663182d0381811c81993e0014a9/safetensors-0.5.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86016d40bcaa3bcc9a56cd74d97e654b5f4f4abe42b038c71e4f00a089c4526c", size = 450088 }, + { url = "https://files.pythonhosted.org/packages/cf/21/e527961b12d5ab528c6e47b92d5f57f33563c28a972750b238b871924e49/safetensors-0.5.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:990833f70a5f9c7d3fc82c94507f03179930ff7d00941c287f73b6fcbf67f19e", size = 458966 }, + { url = "https://files.pythonhosted.org/packages/a5/8b/1a037d7a57f86837c0b41905040369aea7d8ca1ec4b2a77592372b2ec380/safetensors-0.5.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dfa7c2f3fe55db34eba90c29df94bcdac4821043fc391cb5d082d9922013869", size = 509915 }, + { url = "https://files.pythonhosted.org/packages/61/3d/03dd5cfd33839df0ee3f4581a20bd09c40246d169c0e4518f20b21d5f077/safetensors-0.5.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46ff2116150ae70a4e9c490d2ab6b6e1b1b93f25e520e540abe1b81b48560c3a", size = 527664 }, + { url = "https://files.pythonhosted.org/packages/c5/dc/8952caafa9a10a3c0f40fa86bacf3190ae7f55fa5eef87415b97b29cb97f/safetensors-0.5.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab696dfdc060caffb61dbe4066b86419107a24c804a4e373ba59be699ebd8d5", size = 461978 }, + { url = "https://files.pythonhosted.org/packages/60/da/82de1fcf1194e3dbefd4faa92dc98b33c06bed5d67890e0962dd98e18287/safetensors-0.5.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975", size = 491253 }, + { url = "https://files.pythonhosted.org/packages/5a/9a/d90e273c25f90c3ba1b0196a972003786f04c39e302fbd6649325b1272bb/safetensors-0.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a00e737948791b94dad83cf0eafc09a02c4d8c2171a239e8c8572fe04e25960e", size = 628644 }, + { url = "https://files.pythonhosted.org/packages/70/3c/acb23e05aa34b4f5edd2e7f393f8e6480fbccd10601ab42cd03a57d4ab5f/safetensors-0.5.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d3a06fae62418ec8e5c635b61a8086032c9e281f16c63c3af46a6efbab33156f", size = 721648 }, + { url = "https://files.pythonhosted.org/packages/71/45/eaa3dba5253a7c6931230dc961641455710ab231f8a89cb3c4c2af70f8c8/safetensors-0.5.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1506e4c2eda1431099cebe9abf6c76853e95d0b7a95addceaa74c6019c65d8cf", size = 659588 }, + { url = "https://files.pythonhosted.org/packages/b0/71/2f9851164f821064d43b481ddbea0149c2d676c4f4e077b178e7eeaa6660/safetensors-0.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c5b5d9da594f638a259fca766046f44c97244cc7ab8bef161b3e80d04becc76", size = 632533 }, + { url = "https://files.pythonhosted.org/packages/00/f1/5680e2ef61d9c61454fad82c344f0e40b8741a9dbd1e31484f0d31a9b1c3/safetensors-0.5.2-cp38-abi3-win32.whl", hash = "sha256:fe55c039d97090d1f85277d402954dd6ad27f63034fa81985a9cc59655ac3ee2", size = 291167 }, + { url = "https://files.pythonhosted.org/packages/86/ca/aa489392ec6fb59223ffce825461e1f811a3affd417121a2088be7a5758b/safetensors-0.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:78abdddd03a406646107f973c7843276e7b64e5e32623529dc17f3d94a20f589", size = 303756 }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, +] + +[[package]] +name = "starlette" +version = "0.45.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/fb/2984a686808b89a6781526129a4b51266f678b2d2b97ab2d325e56116df8/starlette-0.45.3.tar.gz", hash = "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f", size = 2574076 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/61/f2b52e107b1fc8944b33ef56bf6ac4ebbe16d91b94d2b87ce013bf63fb84/starlette-0.45.3-py3-none-any.whl", hash = "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d", size = 71507 }, +] + +[[package]] +name = "sympy" +version = "1.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/8a/5a7fd6284fa8caac23a26c9ddf9c30485a48169344b4bd3b0f02fef1890f/sympy-1.13.3.tar.gz", hash = "sha256:b27fd2c6530e0ab39e275fc9b683895367e51d5da91baa8d3d64db2565fec4d9", size = 7533196 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/ff/c87e0622b1dadea79d2fb0b25ade9ed98954c9033722eb707053d310d4f3/sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73", size = 6189483 }, +] + +[[package]] +name = "tbb" +version = "2021.13.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/8a/5062b00c378c051e26507e5eca8d3b5c91ed63f8a2139f6f0f422be84b02/tbb-2021.13.1-py3-none-win32.whl", hash = "sha256:00f5e5a70051650ddd0ab6247c0549521968339ec21002e475cd23b1cbf46d66", size = 248994 }, + { url = "https://files.pythonhosted.org/packages/9b/24/84ce997e8ae6296168a74d0d9c4dde572d90fb23fd7c0b219c30ff71e00e/tbb-2021.13.1-py3-none-win_amd64.whl", hash = "sha256:cbf024b2463fdab3ebe3fa6ff453026358e6b903839c80d647e08ad6d0796ee9", size = 286908 }, +] + +[[package]] +name = "tokenizers" +version = "0.20.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/25/b1681c1c30ea3ea6e584ae3fffd552430b12faa599b558c4c4783f56d7ff/tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539", size = 340513 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/00/92a08af2a6b0c88c50f1ab47d7189e695722ad9714b0ee78ea5e1e2e1def/tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f", size = 2667951 }, + { url = "https://files.pythonhosted.org/packages/ec/9a/e17a352f0bffbf415cf7d73756f5c73a3219225fc5957bc2f39d52c61684/tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73", size = 2555167 }, + { url = "https://files.pythonhosted.org/packages/27/37/d108df55daf4f0fcf1f58554692ff71687c273d870a34693066f0847be96/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64", size = 2898389 }, + { url = "https://files.pythonhosted.org/packages/b2/27/32f29da16d28f59472fa7fb38e7782069748c7e9ab9854522db20341624c/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64", size = 2795866 }, + { url = "https://files.pythonhosted.org/packages/29/4e/8a9a3c89e128c4a40f247b501c10279d2d7ade685953407c4d94c8c0f7a7/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d", size = 3085446 }, + { url = "https://files.pythonhosted.org/packages/b4/3b/a2a7962c496ebcd95860ca99e423254f760f382cd4bd376f8895783afaf5/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f", size = 3094378 }, + { url = "https://files.pythonhosted.org/packages/1f/f4/a8a33f0192a1629a3bd0afcad17d4d221bbf9276da4b95d226364208d5eb/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f", size = 3385755 }, + { url = "https://files.pythonhosted.org/packages/9e/65/c83cb3545a65a9eaa2e13b22c93d5e00bd7624b354a44adbdc93d5d9bd91/tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad", size = 2997679 }, + { url = "https://files.pythonhosted.org/packages/55/e9/a80d4e592307688a67c7c59ab77e03687b6a8bd92eb5db763a2c80f93f57/tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5", size = 8989296 }, + { url = "https://files.pythonhosted.org/packages/90/af/60c957af8d2244321124e893828f1a4817cde1a2d08d09d423b73f19bd2f/tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2", size = 9303621 }, + { url = "https://files.pythonhosted.org/packages/be/a9/96172310ee141009646d63a1ca267c099c462d747fe5ef7e33f74e27a683/tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c", size = 2188979 }, + { url = "https://files.pythonhosted.org/packages/bd/68/61d85ae7ae96dde7d0974ff3538db75d5cdc29be2e4329cd7fc51a283e22/tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2", size = 2380725 }, +] + +[[package]] +name = "torch" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "mkl", marker = "sys_platform == 'win32'" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/82/68ccd49add4d21937f087871350905ffc709f32c92bf95334e7abf442147/torch-2.3.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:a486c0b1976a118805fc7c9641d02df7afbb0c21e6b555d3bb985c9f9601b61a", size = 779079866 }, + { url = "https://files.pythonhosted.org/packages/1b/a1/e8b286b85f19dd701a4b853c0554898b1fa69cea552c7d1ec39bc86f59aa/torch-2.3.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:224259821fe3e4c6f7edf1528e4fe4ac779c77addaa74215eb0b63a5c474d66c", size = 86853451 }, + { url = "https://files.pythonhosted.org/packages/af/77/cf6ceb000f8a064c7b373fb3471d85bcc39917d175af82fead4a2857c669/torch-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:e5fdccbf6f1334b2203a61a0e03821d5845f1421defe311dabeae2fc8fbeac2d", size = 159727172 }, + { url = "https://files.pythonhosted.org/packages/49/b6/1a2e3d43d4bc4ad7a4575b3745d707a68d5ed00ba263b205b6281bdd0921/torch-2.3.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:3c333dc2ebc189561514eda06e81df22bf8fb64e2384746b2cb9f04f96d1d4c8", size = 60978559 }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + +[[package]] +name = "transformers" +version = "4.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/4c/3862b2dd6cdf83b187897bd351da0f7fb74d0df642b03c6f5d06353a3ca0/transformers-4.45.2.tar.gz", hash = "sha256:72bc390f6b203892561f05f86bbfaa0e234aab8e927a83e62b9d92ea7e3ae101", size = 8478357 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9d/030cc1b3e88172967e22ee1d012e0d5e0384eb70d2a098d1669d549aea29/transformers-4.45.2-py3-none-any.whl", hash = "sha256:c551b33660cfc815bae1f9f097ecfd1e65be623f13c6ee0dda372bd881460210", size = 9881312 }, +] + +[[package]] +name = "typer" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/ce/dca7b219718afd37a0068f4f2530a727c2b74a8b6e8e0c0080a4c0de4fcd/typer-0.15.1.tar.gz", hash = "sha256:a0588c0a7fa68a1978a069818657778f86abe6ff5ea6abf472f940a08bfe4f0a", size = 99789 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/cc/0a838ba5ca64dc832aa43f727bd586309846b0ffb2ce52422543e6075e8a/typer-0.15.1-py3-none-any.whl", hash = "sha256:7994fb7b8155b64d3402518560648446072864beefd44aa2dc36972a5972e847", size = 44908 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "ujson" +version = "5.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/00/3110fd566786bfa542adb7932d62035e0c0ef662a8ff6544b6643b3d6fd7/ujson-5.10.0.tar.gz", hash = "sha256:b3cd8f3c5d8c7738257f1018880444f7b7d9b66232c64649f562d7ba86ad4bc1", size = 7154885 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/a6/fd3f8bbd80842267e2d06c3583279555e8354c5986c952385199d57a5b6c/ujson-5.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:98ba15d8cbc481ce55695beee9f063189dce91a4b08bc1d03e7f0152cd4bbdd5", size = 55642 }, + { url = "https://files.pythonhosted.org/packages/a8/47/dd03fd2b5ae727e16d5d18919b383959c6d269c7b948a380fdd879518640/ujson-5.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9d2edbf1556e4f56e50fab7d8ff993dbad7f54bac68eacdd27a8f55f433578e", size = 51807 }, + { url = "https://files.pythonhosted.org/packages/25/23/079a4cc6fd7e2655a473ed9e776ddbb7144e27f04e8fc484a0fb45fe6f71/ujson-5.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6627029ae4f52d0e1a2451768c2c37c0c814ffc04f796eb36244cf16b8e57043", size = 51972 }, + { url = "https://files.pythonhosted.org/packages/04/81/668707e5f2177791869b624be4c06fb2473bf97ee33296b18d1cf3092af7/ujson-5.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ccb77b3e40b151e20519c6ae6d89bfe3f4c14e8e210d910287f778368bb3d1", size = 53686 }, + { url = "https://files.pythonhosted.org/packages/bd/50/056d518a386d80aaf4505ccf3cee1c40d312a46901ed494d5711dd939bc3/ujson-5.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3caf9cd64abfeb11a3b661329085c5e167abbe15256b3b68cb5d914ba7396f3", size = 58591 }, + { url = "https://files.pythonhosted.org/packages/fc/d6/aeaf3e2d6fb1f4cfb6bf25f454d60490ed8146ddc0600fae44bfe7eb5a72/ujson-5.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6e32abdce572e3a8c3d02c886c704a38a1b015a1fb858004e03d20ca7cecbb21", size = 997853 }, + { url = "https://files.pythonhosted.org/packages/f8/d5/1f2a5d2699f447f7d990334ca96e90065ea7f99b142ce96e85f26d7e78e2/ujson-5.10.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a65b6af4d903103ee7b6f4f5b85f1bfd0c90ba4eeac6421aae436c9988aa64a2", size = 1140689 }, + { url = "https://files.pythonhosted.org/packages/f2/2c/6990f4ccb41ed93744aaaa3786394bca0875503f97690622f3cafc0adfde/ujson-5.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:604a046d966457b6cdcacc5aa2ec5314f0e8c42bae52842c1e6fa02ea4bda42e", size = 1043576 }, + { url = "https://files.pythonhosted.org/packages/14/f5/a2368463dbb09fbdbf6a696062d0c0f62e4ae6fa65f38f829611da2e8fdd/ujson-5.10.0-cp312-cp312-win32.whl", hash = "sha256:6dea1c8b4fc921bf78a8ff00bbd2bfe166345f5536c510671bccececb187c80e", size = 38764 }, + { url = "https://files.pythonhosted.org/packages/59/2d/691f741ffd72b6c84438a93749ac57bf1a3f217ac4b0ea4fd0e96119e118/ujson-5.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:38665e7d8290188b1e0d57d584eb8110951a9591363316dd41cf8686ab1d0abc", size = 42211 }, +] + +[[package]] +name = "urllib3" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", size = 307268 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 }, +] + +[[package]] +name = "uvicorn" +version = "0.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", size = 62315 }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284 }, + { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349 }, + { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089 }, + { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770 }, + { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321 }, + { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022 }, +] + +[[package]] +name = "virtualenv" +version = "20.29.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/88/dacc875dd54a8acadb4bcbfd4e3e86df8be75527116c91d8f9784f5e9cab/virtualenv-20.29.2.tar.gz", hash = "sha256:fdaabebf6d03b5ba83ae0a02cfe96f48a716f4fae556461d180825866f75b728", size = 4320272 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/fa/849483d56773ae29740ae70043ad88e068f98a6401aa819b5d6bee604683/virtualenv-20.29.2-py3-none-any.whl", hash = "sha256:febddfc3d1ea571bdb1dc0f98d7b45d24def7428214d4fb73cc486c9568cce6a", size = 4301478 }, +] + +[[package]] +name = "watchfiles" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/26/c705fc77d0a9ecdb9b66f1e2976d95b81df3cae518967431e7dbf9b5e219/watchfiles-1.0.4.tar.gz", hash = "sha256:6ba473efd11062d73e4f00c2b730255f9c1bdd73cd5f9fe5b5da8dbd4a717205", size = 94625 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/1a/8f4d9a1461709756ace48c98f07772bc6d4519b1e48b5fa24a4061216256/watchfiles-1.0.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:229e6ec880eca20e0ba2f7e2249c85bae1999d330161f45c78d160832e026ee2", size = 391345 }, + { url = "https://files.pythonhosted.org/packages/bc/d2/6750b7b3527b1cdaa33731438432e7238a6c6c40a9924049e4cebfa40805/watchfiles-1.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5717021b199e8353782dce03bd8a8f64438832b84e2885c4a645f9723bf656d9", size = 381515 }, + { url = "https://files.pythonhosted.org/packages/4e/17/80500e42363deef1e4b4818729ed939aaddc56f82f4e72b2508729dd3c6b/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0799ae68dfa95136dde7c472525700bd48777875a4abb2ee454e3ab18e9fc712", size = 449767 }, + { url = "https://files.pythonhosted.org/packages/10/37/1427fa4cfa09adbe04b1e97bced19a29a3462cc64c78630787b613a23f18/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43b168bba889886b62edb0397cab5b6490ffb656ee2fcb22dec8bfeb371a9e12", size = 455677 }, + { url = "https://files.pythonhosted.org/packages/c5/7a/39e9397f3a19cb549a7d380412fd9e507d4854eddc0700bfad10ef6d4dba/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb2c46e275fbb9f0c92e7654b231543c7bbfa1df07cdc4b99fa73bedfde5c844", size = 482219 }, + { url = "https://files.pythonhosted.org/packages/45/2d/7113931a77e2ea4436cad0c1690c09a40a7f31d366f79c6f0a5bc7a4f6d5/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:857f5fc3aa027ff5e57047da93f96e908a35fe602d24f5e5d8ce64bf1f2fc733", size = 518830 }, + { url = "https://files.pythonhosted.org/packages/f9/1b/50733b1980fa81ef3c70388a546481ae5fa4c2080040100cd7bf3bf7b321/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55ccfd27c497b228581e2838d4386301227fc0cb47f5a12923ec2fe4f97b95af", size = 497997 }, + { url = "https://files.pythonhosted.org/packages/2b/b4/9396cc61b948ef18943e7c85ecfa64cf940c88977d882da57147f62b34b1/watchfiles-1.0.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c11ea22304d17d4385067588123658e9f23159225a27b983f343fcffc3e796a", size = 452249 }, + { url = "https://files.pythonhosted.org/packages/fb/69/0c65a5a29e057ad0dc691c2fa6c23b2983c7dabaa190ba553b29ac84c3cc/watchfiles-1.0.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:74cb3ca19a740be4caa18f238298b9d472c850f7b2ed89f396c00a4c97e2d9ff", size = 614412 }, + { url = "https://files.pythonhosted.org/packages/7f/b9/319fcba6eba5fad34327d7ce16a6b163b39741016b1996f4a3c96b8dd0e1/watchfiles-1.0.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c7cce76c138a91e720d1df54014a047e680b652336e1b73b8e3ff3158e05061e", size = 611982 }, + { url = "https://files.pythonhosted.org/packages/f1/47/143c92418e30cb9348a4387bfa149c8e0e404a7c5b0585d46d2f7031b4b9/watchfiles-1.0.4-cp312-cp312-win32.whl", hash = "sha256:b045c800d55bc7e2cadd47f45a97c7b29f70f08a7c2fa13241905010a5493f94", size = 271822 }, + { url = "https://files.pythonhosted.org/packages/ea/94/b0165481bff99a64b29e46e07ac2e0df9f7a957ef13bec4ceab8515f44e3/watchfiles-1.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:c2acfa49dd0ad0bf2a9c0bb9a985af02e89345a7189be1efc6baa085e0f72d7c", size = 285441 }, + { url = "https://files.pythonhosted.org/packages/11/de/09fe56317d582742d7ca8c2ca7b52a85927ebb50678d9b0fa8194658f536/watchfiles-1.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:22bb55a7c9e564e763ea06c7acea24fc5d2ee5dfc5dafc5cfbedfe58505e9f90", size = 277141 }, +] + +[[package]] +name = "websockets" +version = "15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/7a/8bc4d15af7ff30f7ba34f9a172063bfcee9f5001d7cef04bee800a658f33/websockets-15.0.tar.gz", hash = "sha256:ca36151289a15b39d8d683fd8b7abbe26fc50be311066c5f8dcf3cb8cee107ab", size = 175574 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/1e/92c4547d7b2a93f848aedaf37e9054111bc00dc11bff4385ca3f80dbb412/websockets-15.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cccc18077acd34c8072578394ec79563664b1c205f7a86a62e94fafc7b59001f", size = 174709 }, + { url = "https://files.pythonhosted.org/packages/9f/37/eae4830a28061ba552516d84478686b637cd9e57d6a90b45ad69e89cb0af/websockets-15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4c22992e24f12de340ca5f824121a5b3e1a37ad4360b4e1aaf15e9d1c42582d", size = 172372 }, + { url = "https://files.pythonhosted.org/packages/46/2f/b409f8b8aa9328d5a47f7a301a43319d540d70cf036d1e6443675978a988/websockets-15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1206432cc6c644f6fc03374b264c5ff805d980311563202ed7fef91a38906276", size = 172607 }, + { url = "https://files.pythonhosted.org/packages/d6/81/d7e2e4542d4b4df849b0110df1b1f94f2647b71ab4b65d672090931ad2bb/websockets-15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3cc75ef3e17490042c47e0523aee1bcc4eacd2482796107fd59dd1100a44bc", size = 182422 }, + { url = "https://files.pythonhosted.org/packages/b6/91/3b303160938d123eea97f58be363f7dbec76e8c59d587e07b5bc257dd584/websockets-15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b89504227a5311610e4be16071465885a0a3d6b0e82e305ef46d9b064ce5fb72", size = 181362 }, + { url = "https://files.pythonhosted.org/packages/f2/8b/df6807f1ca339c567aba9a7ab03bfdb9a833f625e8d2b4fc7529e4c701de/websockets-15.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56e3efe356416bc67a8e093607315951d76910f03d2b3ad49c4ade9207bf710d", size = 181787 }, + { url = "https://files.pythonhosted.org/packages/21/37/e6d3d5ebb0ebcaf98ae84904205c9dcaf3e0fe93e65000b9f08631ed7309/websockets-15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab", size = 182058 }, + { url = "https://files.pythonhosted.org/packages/c9/df/6aca296f2be4c638ad20908bb3d7c94ce7afc8d9b4b2b0780d1fc59b359c/websockets-15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:aea01f40995fa0945c020228ab919b8dfc93fc8a9f2d3d705ab5b793f32d9e99", size = 181434 }, + { url = "https://files.pythonhosted.org/packages/88/f1/75717a982bab39bbe63c83f9df0e7753e5c98bab907eb4fb5d97fe5c8c11/websockets-15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9f8e33747b1332db11cf7fcf4a9512bef9748cb5eb4d3f7fbc8c30d75dc6ffc", size = 181431 }, + { url = "https://files.pythonhosted.org/packages/e7/15/cee9e63ed9ac5bfc1a3ae8fc6c02c41745023c21eed622eef142d8fdd749/websockets-15.0-cp312-cp312-win32.whl", hash = "sha256:32e02a2d83f4954aa8c17e03fe8ec6962432c39aca4be7e8ee346b05a3476904", size = 175678 }, + { url = "https://files.pythonhosted.org/packages/4e/00/993974c60f40faabb725d4dbae8b072ef73b4c4454bd261d3b1d34ace41f/websockets-15.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa", size = 176119 }, + { url = "https://files.pythonhosted.org/packages/e8/b2/31eec524b53f01cd8343f10a8e429730c52c1849941d1f530f8253b6d934/websockets-15.0-py3-none-any.whl", hash = "sha256:51ffd53c53c4442415b613497a34ba0aa7b99ac07f1e4a62db5dcd640ae6c3c3", size = 169023 }, +] From 5298e2d3a114c803e5dacd3b34b462238886f254 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:12:56 +0100 Subject: [PATCH 56/62] Set Python version. --- .github/workflows/python.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 28d1c15..4cb44b9 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -18,10 +18,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + + - name: Read Python version from .version file + id: python_version + run: echo "PYTHON_VERSION=$(cat .version)" >> $GITHUB_ENV + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ env.PYTHON_VERSION }} + - name: Install dependencies run: | pip install uv From f0dad8455b42e9ef8b7af1e9d92c0a970011e5a8 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:15:01 +0100 Subject: [PATCH 57/62] Change file name for python version file. --- .python-version | 1 - .version | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 .python-version create mode 100644 .version diff --git a/.python-version b/.python-version deleted file mode 100644 index e4fba21..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/.version b/.version new file mode 100644 index 0000000..04e2079 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +3.12.8 From aa86059a71f8b577f3913877c838bfcadb88fb4f Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:20:13 +0100 Subject: [PATCH 58/62] Fix tests --- embedding_api/api/endpoints/embed.py | 8 ++++---- embedding_api/db/database_interface_factory.py | 6 +++--- embedding_api/db/dummy_database.py | 4 ++-- embedding_api/db/pinecone_database.py | 4 ++-- embedding_api/db/redis_database.py | 4 ++-- embedding_api/main.py | 2 +- tests/test_main.py | 18 +++++++++--------- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/embedding_api/api/endpoints/embed.py b/embedding_api/api/endpoints/embed.py index 4045ac6..de3dcdd 100644 --- a/embedding_api/api/endpoints/embed.py +++ b/embedding_api/api/endpoints/embed.py @@ -1,10 +1,10 @@ import json import logging -from app.config.settings import Settings -from app.db.database_interface_factory import DatabaseFactory -from app.model import Handler -from app.schemas.default import EmbeddingOutput, SimilarityOutput, TextInput +from embedding_api.config.settings import Settings +from embedding_api.db.database_interface_factory import DatabaseFactory +from embedding_api.model import Handler +from embedding_api.schemas.default import EmbeddingOutput, SimilarityOutput, TextInput from fastapi import APIRouter, HTTPException logging.basicConfig( diff --git a/embedding_api/db/database_interface_factory.py b/embedding_api/db/database_interface_factory.py index 8115c5a..3a9faee 100644 --- a/embedding_api/db/database_interface_factory.py +++ b/embedding_api/db/database_interface_factory.py @@ -1,8 +1,8 @@ import logging -from app.db.dummy_database import DummyDatabase -from app.db.pinecone_database import PineconeDatabase -from app.db.redis_database import RedisDatabase +from embedding_api.db.dummy_database import DummyDatabase +from embedding_api.db.pinecone_database import PineconeDatabase +from embedding_api.db.redis_database import RedisDatabase logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/embedding_api/db/dummy_database.py b/embedding_api/db/dummy_database.py index 9aa45a5..b7ce9b8 100644 --- a/embedding_api/db/dummy_database.py +++ b/embedding_api/db/dummy_database.py @@ -1,8 +1,8 @@ import logging from typing import List -from app.config.settings import Settings -from app.db.database_interface import DatabaseInterface +from embedding_api.config.settings import Settings +from embedding_api.db.database_interface import DatabaseInterface logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/embedding_api/db/pinecone_database.py b/embedding_api/db/pinecone_database.py index c8ccdb6..2f540a0 100644 --- a/embedding_api/db/pinecone_database.py +++ b/embedding_api/db/pinecone_database.py @@ -2,8 +2,8 @@ import time from typing import List -from app.config.settings import Settings -from app.db.database_interface import DatabaseInterface +from embedding_api.config.settings import Settings +from embedding_api.db.database_interface import DatabaseInterface from pinecone.grpc import PineconeGRPC as Pinecone logging.basicConfig( diff --git a/embedding_api/db/redis_database.py b/embedding_api/db/redis_database.py index 7f4f489..e4d1ff1 100644 --- a/embedding_api/db/redis_database.py +++ b/embedding_api/db/redis_database.py @@ -2,8 +2,8 @@ from typing import List import redis -from app.config.settings import Settings -from app.db.database_interface import DatabaseInterface +from embedding_api.config.settings import Settings +from embedding_api.db.database_interface import DatabaseInterface logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/embedding_api/main.py b/embedding_api/main.py index 86b55ab..db7ac39 100755 --- a/embedding_api/main.py +++ b/embedding_api/main.py @@ -1,6 +1,6 @@ import logging -from app.api.endpoints.embed import embed_router +from embedding_api.api.endpoints.embed import embed_router from fastapi import FastAPI logging.basicConfig( diff --git a/tests/test_main.py b/tests/test_main.py index 2ca96c4..8f10694 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,9 +8,9 @@ from unittest.mock import MagicMock, patch -from app.config.settings import Settings -from app.main import app -from app.schemas.default import TextInput +from embedding_api.config.settings import Settings +from embedding_api.main import app +from embedding_api.schemas.default import TextInput from fastapi.testclient import TestClient from parameterized import parameterized @@ -22,8 +22,8 @@ class TestEmbedEndpoint(unittest.TestCase): def setUp(self): self.client = TestClient(app) - @patch("app.api.endpoints.embed.handler") - @patch("app.api.endpoints.embed.database_object") + @patch("embedding_api.api.endpoints.embed.handler") + @patch("embedding_api.api.endpoints.embed.database_object") def test_embed_text_cached(self, mock_database_object, mock_handler_object): mock_database_object.get = MagicMock(return_value=json.dumps([0.1, 0.2, 0.3])) @@ -49,8 +49,8 @@ def test_embed_text_cached(self, mock_database_object, mock_handler_object): mock_database_object.get.assert_called_once_with(long_string_input["text"]) mock_handler_object.embed.assert_not_called() - @patch("app.api.endpoints.embed.handler") - @patch("app.api.endpoints.embed.database_object") + @patch("embedding_api.api.endpoints.embed.handler") + @patch("embedding_api.api.endpoints.embed.database_object") def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): mock_database_object.get = MagicMock(return_value=None) mock_handler_object.embed = MagicMock(return_value=[1, 2, 3]) @@ -87,7 +87,7 @@ def test_embed_text_not_cached(self, mock_database_object, mock_handler_object): ), ] ) - @patch("app.api.endpoints.embed.database_object") + @patch("embedding_api.api.endpoints.embed.database_object") def test_embed_text_parametrized(self, text_input, mock_database_object): mock_database_object.get = MagicMock(return_value=None) # Act: Send a POST request to the /embed endpoint @@ -114,7 +114,7 @@ class TestCalculateSimilarity(unittest.TestCase): def setUp(self): self.client = TestClient(app) - @patch("app.api.endpoints.embed.handler.similarity") # Mock the similarity function + @patch("embedding_api.api.endpoints.embed.handler.similarity") # Mock the similarity function def test_calculate_similarity(self, mock_similarity): # Arrange text_1 = TextInput(text="Dog") From 49aed254cf88b7723b3611bc6bace9d5c2dbdb14 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:21:46 +0100 Subject: [PATCH 59/62] pre-commit fix --- embedding_api/api/endpoints/embed.py | 3 ++- embedding_api/db/pinecone_database.py | 3 ++- embedding_api/db/redis_database.py | 1 + embedding_api/main.py | 3 ++- tests/test_main.py | 10 ++++++---- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/embedding_api/api/endpoints/embed.py b/embedding_api/api/endpoints/embed.py index de3dcdd..5598cc5 100644 --- a/embedding_api/api/endpoints/embed.py +++ b/embedding_api/api/endpoints/embed.py @@ -1,11 +1,12 @@ import json import logging +from fastapi import APIRouter, HTTPException + from embedding_api.config.settings import Settings from embedding_api.db.database_interface_factory import DatabaseFactory from embedding_api.model import Handler from embedding_api.schemas.default import EmbeddingOutput, SimilarityOutput, TextInput -from fastapi import APIRouter, HTTPException logging.basicConfig( level=logging.INFO, diff --git a/embedding_api/db/pinecone_database.py b/embedding_api/db/pinecone_database.py index 2f540a0..699e60b 100644 --- a/embedding_api/db/pinecone_database.py +++ b/embedding_api/db/pinecone_database.py @@ -2,9 +2,10 @@ import time from typing import List +from pinecone.grpc import PineconeGRPC as Pinecone + from embedding_api.config.settings import Settings from embedding_api.db.database_interface import DatabaseInterface -from pinecone.grpc import PineconeGRPC as Pinecone logging.basicConfig( level=logging.INFO, # Set the minimum logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/embedding_api/db/redis_database.py b/embedding_api/db/redis_database.py index e4d1ff1..e98c2e9 100644 --- a/embedding_api/db/redis_database.py +++ b/embedding_api/db/redis_database.py @@ -2,6 +2,7 @@ from typing import List import redis + from embedding_api.config.settings import Settings from embedding_api.db.database_interface import DatabaseInterface diff --git a/embedding_api/main.py b/embedding_api/main.py index db7ac39..2520f19 100755 --- a/embedding_api/main.py +++ b/embedding_api/main.py @@ -1,8 +1,9 @@ import logging -from embedding_api.api.endpoints.embed import embed_router from fastapi import FastAPI +from embedding_api.api.endpoints.embed import embed_router + logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", diff --git a/tests/test_main.py b/tests/test_main.py index 8f10694..8b9a7d3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,12 +8,12 @@ from unittest.mock import MagicMock, patch -from embedding_api.config.settings import Settings -from embedding_api.main import app -from embedding_api.schemas.default import TextInput from fastapi.testclient import TestClient from parameterized import parameterized +from embedding_api.config.settings import Settings +from embedding_api.main import app +from embedding_api.schemas.default import TextInput from tests.payload_tests import long_string_input @@ -114,7 +114,9 @@ class TestCalculateSimilarity(unittest.TestCase): def setUp(self): self.client = TestClient(app) - @patch("embedding_api.api.endpoints.embed.handler.similarity") # Mock the similarity function + @patch( + "embedding_api.api.endpoints.embed.handler.similarity" + ) # Mock the similarity function def test_calculate_similarity(self, mock_similarity): # Arrange text_1 = TextInput(text="Dog") From e8cccf7b07eb89f9e2333d868f77cf942b3b9453 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:27:29 +0100 Subject: [PATCH 60/62] Change infra resource name and update gitignore --- .gitignore | 2 ++ infra/gcp/main.tf | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3240620..8880a5b 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.vscode + poetry.lock ~ .terraform diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 5d6b5a9..788b890 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -50,12 +50,11 @@ resource "google_project_service" "container_registry" { service = each.key } -resource "google_artifact_registry_repository" "my-repo" { +resource "google_artifact_registry_repository" "python_registry" { location = var.region repository_id = "python-package" - description = "example docker repository" + description = "python registry" format = "PYTHON" -} resource "google_project_service" "artifact_registry_api" { From 4f3fcf11229e1251dba8b178d66c74a9ca1d0e24 Mon Sep 17 00:00:00 2001 From: blpasd Date: Thu, 20 Feb 2025 20:33:23 +0100 Subject: [PATCH 61/62] Closing brakcet --- infra/gcp/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/gcp/main.tf b/infra/gcp/main.tf index 788b890..62720fe 100644 --- a/infra/gcp/main.tf +++ b/infra/gcp/main.tf @@ -55,7 +55,7 @@ resource "google_artifact_registry_repository" "python_registry" { repository_id = "python-package" description = "python registry" format = "PYTHON" - +} resource "google_project_service" "artifact_registry_api" { service = "artifactregistry.googleapis.com" From e31d18d647a5d60ea3c5f2864e58d9500c28fd65 Mon Sep 17 00:00:00 2001 From: "robert.musters" Date: Sat, 28 Mar 2026 11:25:51 +0100 Subject: [PATCH 62/62] feat: retrieve crossref papers as pdf using unpaywall --- data-pipeline/Dockerfile | 14 + data-pipeline/QUICKSTART.md | 183 ++++++ data-pipeline/UNPAYWALL_JOB.md | 549 ++++++++++++++++++ data-pipeline/main.py | 41 ++ data-pipeline/manage_job.sh | 108 ++++ data-pipeline/unpaywall_ingester.py | 382 ++++++++++++ .../data-pipeline/terraform.tfvars.example | 30 + infra/gcp/data-pipeline/unpaywall_job.tf | 164 ++++++ tests/test_unpaywall_ingester.py | 193 ++++++ 9 files changed, 1664 insertions(+) create mode 100644 data-pipeline/Dockerfile create mode 100644 data-pipeline/QUICKSTART.md create mode 100644 data-pipeline/UNPAYWALL_JOB.md create mode 100644 data-pipeline/main.py create mode 100755 data-pipeline/manage_job.sh create mode 100644 data-pipeline/unpaywall_ingester.py create mode 100644 infra/gcp/data-pipeline/terraform.tfvars.example create mode 100644 infra/gcp/data-pipeline/unpaywall_job.tf create mode 100644 tests/test_unpaywall_ingester.py diff --git a/data-pipeline/Dockerfile b/data-pipeline/Dockerfile new file mode 100644 index 0000000..74ecaac --- /dev/null +++ b/data-pipeline/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + pip install --no-cache-dir requests google-cloud-storage + +# Copy application code +COPY data-pipeline/ . + +# Set entrypoint +ENTRYPOINT ["python", "main.py"] diff --git a/data-pipeline/QUICKSTART.md b/data-pipeline/QUICKSTART.md new file mode 100644 index 0000000..9409500 --- /dev/null +++ b/data-pipeline/QUICKSTART.md @@ -0,0 +1,183 @@ +# Quick Start: Daily CrossRef + Unpaywall Cloud Run Job + +Automatically fetches papers published each day from CrossRef, checks Unpaywall for open access status, and stores them in Cloud Storage. + +## 1. Prepare Configuration + +```bash +cd infra/ +cp terraform.tfvars.example terraform.tfvars + +# Edit terraform.tfvars - set project_id, gcs_bucket, unpaywall_email +# Configure days_back if needed (default: 1 = yesterday) +``` + +## 2. Build & Push Docker Image + +```bash +# Authenticate Docker with Artifact Registry +gcloud auth configure-docker europe-west4-docker.pkg.dev + +# Build the image +docker build -f data-pipeline/Dockerfile \ + -t europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest . + +# Push to Artifact Registry +docker push europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest +``` + +## 3. Deploy to Cloud Run + +### Option A: Using Terraform (Recommended) + +```bash +cd infra/ +terraform init +terraform plan +terraform apply +``` + +### Option B: Using gcloud CLI + +```bash +# Set variables +PROJECT_ID="fastapi-449213" +REGION="europe-west4" +GCS_BUCKET="data-pipeline-job" +UNPAYWALL_EMAIL="romusters@gmail.com" +CONTAINER_IMAGE="europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest" +DAYS_BACK=1 + +# 1. Create service account +gcloud iam service-accounts create unpaywall-ingestion-job \ + --display-name="Unpaywall Ingestion Job Service Account" \ + --project=$PROJECT_ID + +# 2. Grant Cloud Storage access +gcloud storage buckets add-iam-policy-binding gs://$GCS_BUCKET \ + --member=serviceAccount:unpaywall-ingestion-job@$PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/storage.objectCreator + +# 3. Create Cloud Run Job +gcloud run jobs create unpaywall-ingestion-job \ + --image=$CONTAINER_IMAGE \ + --region=$REGION \ + --set-env-vars=GCS_BUCKET=$GCS_BUCKET,UNPAYWALL_EMAIL=$UNPAYWALL_EMAIL,DAYS_BACK=$DAYS_BACK \ + --service-account=unpaywall-ingestion-job@$PROJECT_ID.iam.gserviceaccount.com \ + --task-timeout=3600s \ + --memory=2Gi \ + --cpu=2 \ + --project=$PROJECT_ID + +# 4. Create Cloud Scheduler to trigger job daily at 2 AM UTC +gcloud scheduler jobs create app-engine unpaywall-ingestion-trigger \ + --schedule="0 2 * * *" \ + --http-method=POST \ + --uri=https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT_ID/jobs/unpaywall-ingestion-job:run \ + --oidc-service-account-email=unpaywall-ingestion-job@$PROJECT_ID.iam.gserviceaccount.com \ + --location=$REGION \ + --project=$PROJECT_ID +``` + +### Option C: Run Container Locally (for testing) + +```bash +# Set environment variables +export GCS_BUCKET="your-bucket-name" +export UNPAYWALL_EMAIL="your-email@example.com" +export DAYS_BACK=1 + +# Option 1: Run with local image (if you built it) +docker run --rm \ + -e GCS_BUCKET=$GCS_BUCKET \ + -e UNPAYWALL_EMAIL=$UNPAYWALL_EMAIL \ + -e DAYS_BACK=$DAYS_BACK \ + -v ~/.config/gcloud:/root/.config/gcloud \ + europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest + +# Option 2: Pull and run from Artifact Registry +docker pull europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest + +docker run --rm \ + -e GCS_BUCKET=$GCS_BUCKET \ + -e UNPAYWALL_EMAIL=$UNPAYWALL_EMAIL \ + -e DAYS_BACK=$DAYS_BACK \ + -v ~/.config/gcloud:/root/.config/gcloud \ + europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest + +# Option 3: Run with credentials file (for CI/CD) +docker run --rm \ + -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-key.json \ + -e GCS_BUCKET=$GCS_BUCKET \ + -e UNPAYWALL_EMAIL=$UNPAYWALL_EMAIL \ + -e DAYS_BACK=$DAYS_BACK \ + -v /path/to/gcp-key.json:/tmp/gcp-key.json:ro \ + europe-west4-docker.pkg.dev/fastapi-449213/portfolio/unpaywall-ingestion:latest +``` + +**Notes for local testing:** +- `-v ~/.config/gcloud:/root/.config/gcloud` - Mounts gcloud credentials into container +- `--rm` - Removes container after execution +- You need to be authenticated with GCP: `gcloud auth application-default login` +- Make sure your GCP user has Cloud Storage write permissions + +```bash +# Check job status +gcloud run jobs describe unpaywall-ingestion-job --region europe-west4 + +# Check scheduler (runs daily at 2 AM UTC) +gcloud scheduler jobs describe unpaywall-ingestion-trigger --location europe-west4 +``` + +## 5. View Results (Next Day) + +```bash +# After job runs, check papers stored +gsutil ls gs://YOUR_BUCKET/unpaywall/papers/$(date +%Y%m%d)/ + +# View ingestion statistics +gsutil cat gs://YOUR_BUCKET/unpaywall/logs/$(date +%Y%m%d)/ingestion_log.json +``` + +## Manual Execution + +```bash +# Run today's papers immediately +gcloud run jobs execute unpaywall-ingestion-job \ + --region europe-west4 + +# Run with custom days back +gcloud run jobs execute unpaywall-ingestion-job \ + --region europe-west4 \ + --set-env-vars="DAYS_BACK=7" + +# View logs +gcloud logging read "resource.type=cloud_run_job" --limit 50 +``` + +## What Gets Stored? + +- ~1,500 papers found from CrossRef daily +- ~450 papers with open access (30% typical) +- Stored as JSON in: `gs://bucket/unpaywall/papers/YYYYMMDD/` +- Statistics saved in: `gs://bucket/unpaywall/logs/YYYYMMDD/ingestion_log.json` + +## Files Reference + +| File | Purpose | +|------|---------| +| `data-pipeline/unpaywall_ingester.py` | Core ingestion logic (CrossRef + Unpaywall) | +| `data-pipeline/main.py` | Cloud Run entry point | +| `data-pipeline/Dockerfile` | Container definition | +| `infra/unpaywall_job.tf` | Terraform infrastructure | +| `data-pipeline/UNPAYWALL_JOB.md` | Full documentation | + +## Key Configuration + +In `terraform.tfvars`: +- `project_id` - Your GCP project +- `gcs_bucket` - Cloud Storage bucket +- `unpaywall_email` - Email registered with Unpaywall +- `days_back` - How many days to look back (1-30, default: 1) +- `schedule_frequency` - Cron schedule (default: `0 2 * * *` = daily 2 AM UTC) + diff --git a/data-pipeline/UNPAYWALL_JOB.md b/data-pipeline/UNPAYWALL_JOB.md new file mode 100644 index 0000000..c3f6eaf --- /dev/null +++ b/data-pipeline/UNPAYWALL_JOB.md @@ -0,0 +1,549 @@ +# Unpaywall + CrossRef Ingestion Cloud Run Job + +This document describes the Cloud Run job for ingesting open access papers from CrossRef and Unpaywall APIs, storing them in Google Cloud Storage. + +## Overview + +The job automatically fetches papers published each day from CrossRef API and checks Unpaywall for open access status. + +**Workflow:** +- Queries CrossRef for papers published on a specific date +- For each paper, checks Unpaywall API for open access status +- Stores only OA papers in JSON format +- Logs ingestion statistics + +``` +Cloud Scheduler (Daily) + ↓ +Cloud Run Job + ↓ +CrossRef API (papers from date) + ↓ +Unpaywall API (check OA status) + ↓ +Cloud Storage (gs://bucket/unpaywall/papers/) +``` + +## Setup + +### Prerequisites + +1. **Google Cloud Project** with: + - Cloud Run API enabled + - Cloud Scheduler API enabled + - Cloud Storage bucket created + - Service Account with appropriate permissions + +2. **Container Image** built and pushed to Artifact Registry or Container Registry + +3. **Unpaywall Email**: Register at https://unpaywall.org/products/api to get API access + +### Deployment + +#### 1. Build and Push Docker Image + +```bash +# From the repository root +docker build -f data-pipeline/Dockerfile -t gcr.io/PROJECT_ID/unpaywall-ingestion:latest . +docker push gcr.io/PROJECT_ID/unpaywall-ingestion:latest +``` + +#### 2. Deploy with Terraform + +```bash +cd infra/ + +# Create terraform.tfvars +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your project details + +# Deploy +terraform init +terraform plan +terraform apply +``` + +## Usage + +### Running Manually + +#### Ingest from Yesterday's CrossRef Papers + +```bash +gcloud run jobs execute unpaywall-ingestion-job \ + --region us-central1 +``` + +#### Ingest Last 7 Days + +```bash +gcloud run jobs execute unpaywall-ingestion-job \ + --region us-central1 \ + --set-env-vars="DAYS_BACK=7" +``` + +### Scheduling + +The job runs automatically via Cloud Scheduler based on the configured cron schedule. Default: Daily at 2 AM UTC. + +**To modify the schedule:** + +```bash +# Edit schedule_frequency in terraform.tfvars +terraform apply +``` + +## Environment Variables + +| Variable | Description | Required | Example | +|----------|-------------|----------|---------| +| `GCS_BUCKET` | Cloud Storage bucket name | Yes | `my-papers-bucket` | +| `UNPAYWALL_EMAIL` | Email for Unpaywall API | Yes | `user@example.com` | +| `DAYS_BACK` | Days to look back (1-30) | No | `1` | + +## Cloud Storage Structure + +Papers are organized by ingestion date: + +``` +gs://your-bucket/ +├── unpaywall/ +│ ├── papers/ +│ │ └── 20240115/ +│ │ ├── 10_1234_test1.json +│ │ ├── 10_1234_test2.json +│ │ └── ... +│ └── logs/ +│ └── 20240115/ +│ └── ingestion_log.json +``` + +### Paper JSON Format + +```json +{ + "doi": "10.1234/example", + "title": "Example Paper Title", + "authors": [...], + "year": 2024, + "is_oa": true, + "oa_status": "gold", + "oa_locations": [...], + "best_oa_location": {...} +} +``` + +### Ingestion Log Format + +```json +{ + "source": "crossref", + "days_back": 1, + "total_papers_found": 1500, + "successful_oa": 450, + "failed": 2, + "timestamp": "2024-01-15T02:00:00", + "failed_dois": ["10.1234/failed1"] +} +``` + +## Monitoring + +### View Job Logs + +```bash +gcloud run jobs describe unpaywall-ingestion-job --region us-central1 +gcloud logging read "resource.type=cloud_run_job" --limit 50 +``` + +### Check Ingestion Statistics + +Papers and logs are stored in Cloud Storage: + +```bash +# List papers from today +gsutil ls gs://your-bucket/unpaywall/papers/$(date +%Y%m%d)/ + +# View ingestion log +gsutil cat gs://your-bucket/unpaywall/logs/$(date +%Y%m%d)/ingestion_log.json +``` + +## API Rate Limiting + +**CrossRef:** 50 requests/second (per IP) +**Unpaywall:** Generally lenient; includes email in user-agent + +Current implementation includes: +- Batch processing with logging every 100 papers +- Retry logic (1 retry) for transient failures +- Configurable timeouts (15s for CrossRef, 10s for Unpaywall) + +**Tips:** +- CrossRef returns 1000 papers max per request +- Job processes ~10-20 papers/second depending on API latency +- For 1 week of papers (~10,000 papers): ~10-15 minutes + +## Cost Optimization + +- **Job timeout**: Set to 3600s (1 hour) by default. Adjust based on volume +- **CPU/Memory**: 2 CPU, 2GB memory. Adjust via Terraform if needed +- **Storage**: Papers stored as JSON (~2-5 KB per paper typical) + +Example cost estimate (daily): +- ~1,500 papers/day from CrossRef +- ~450 OA papers (30% typical OA rate) +- ~1-2 MB stored/day +- Cloud Storage: ~$0.02/month for storage +- Cloud Run: ~0.01 credits/day + +## Troubleshooting + +### Job Fails with "GCS_BUCKET not found" + +Check the bucket exists and the service account has write access: + +```bash +gsutil iam ch serviceAccount:unpaywall-ingestion-job@PROJECT_ID.iam.gserviceaccount.com:objectCreator gs://your-bucket +``` + +### High Failure Rate + +- Check CrossRef/Unpaywall API status +- Review logs in Cloud Logging +- Monitor API response times + +### Job Timeout + +Increase timeout or reduce days_back: + +```bash +# In terraform.tfvars +job_timeout = 7200 # 2 hours +terraform apply +``` + +## Development + +### Local Testing + +```bash +# Install dependencies +pip install -r requirements.txt +pip install requests google-cloud-storage + +# Test CrossRef fetch +python -c " +from data_pipeline.unpaywall_ingester import UnpaywallIngester + +ingester = UnpaywallIngester( + gcs_bucket='test-bucket', + email='your-email@example.com' +) + +# Fetch papers from today +papers = ingester._fetch_crossref_papers('2024-01-15', limit=10) +for paper in papers: + print(f\"Found: {paper['doi']} - {paper['title']}\") +" +``` + +### Run Tests + +```bash +pytest tests/test_unpaywall_ingester.py -v +``` + +## API Reference + +#### `ingest_daily_from_crossref(days_back: int = 1, limit_per_day: int = 10000, date_prefix: str = None) -> dict` + +Ingest papers from CrossRef published in the last N days, check Unpaywall for OA. + +**Parameters:** +- `days_back`: Number of days to look back (1 = yesterday) +- `limit_per_day`: Maximum papers per day from CrossRef +- `date_prefix`: Storage date prefix (default: YYYYMMDD) + +**Returns:** Statistics dictionary + +**Example:** +```python +stats = ingester.ingest_daily_from_crossref(days_back=1) +# Returns: { +# "source": "crossref", +# "days_back": 1, +# "total_papers_found": 1500, +# "successful_oa": 450, +# "failed": 2, +# ... +# } +``` + +#### `_fetch_crossref_papers(date: str, limit: int = 10000) -> list[dict]` + +Fetch papers from CrossRef published on specific date. + +**Parameters:** +- `date`: Date string (YYYY-MM-DD format) +- `limit`: Maximum results + +## Links + +- [CrossRef API Documentation](https://github.com/CrossRef/rest-api-doc) +- [Unpaywall API Documentation](https://unpaywall.org/products/api) +- [Cloud Run Documentation](https://cloud.google.com/run/docs) +- [Cloud Scheduler Documentation](https://cloud.google.com/scheduler/docs) +- [Google Cloud Storage Documentation](https://cloud.google.com/storage/docs) + +## Setup + +### Prerequisites + +1. **Google Cloud Project** with: + - Cloud Run API enabled + - Cloud Scheduler API enabled + - Cloud Storage bucket created + - Service Account with appropriate permissions + +2. **Container Image** built and pushed to Artifact Registry or Container Registry + +3. **Unpaywall Email**: Register at https://unpaywall.org/products/api to get API access + +### Deployment + +#### 1. Build and Push Docker Image + +```bash +# From the repository root +docker build -f data-pipeline/Dockerfile -t gcr.io/PROJECT_ID/unpaywall-ingestion:latest . +docker push gcr.io/PROJECT_ID/unpaywall-ingestion:latest +``` + +#### 2. Deploy with Terraform + +```bash +cd infra/ + +# Create terraform.tfvars +cat > terraform.tfvars << EOF +project_id = "YOUR_PROJECT_ID" +region = "us-central1" +gcs_bucket = "your-bucket-name" +unpaywall_email = "your-email@example.com" +container_image = "gcr.io/YOUR_PROJECT_ID/unpaywall-ingestion:latest" +schedule_frequency = "0 2 * * *" # Daily at 2 AM UTC +EOF + +# Deploy +terraform init +terraform plan +terraform apply +``` + +## Usage + +### Running Manually + +#### Ingest Specific DOIs + +```bash +gcloud run jobs execute unpaywall-ingestion-job \ + --region us-central1 \ + --set-env-vars="DOIS=10.1234/test1,10.1234/test2" +``` + +#### Ingest by Query + +```bash +gcloud run jobs execute unpaywall-ingestion-job \ + --region us-central1 \ + --set-env-vars="QUERY=machine learning" +``` + +### Scheduling + +The job runs automatically via Cloud Scheduler based on the configured cron schedule. Default: Daily at 2 AM UTC. + +**To modify the schedule:** + +```bash +# Edit the schedule_frequency variable in terraform.tfvars +terraform apply +``` + +## Environment Variables + +| Variable | Description | Required | Example | +|----------|-------------|----------|---------| +| `GCS_BUCKET` | Cloud Storage bucket name | Yes | `my-papers-bucket` | +| `UNPAYWALL_EMAIL` | Email for Unpaywall API | Yes | `user@example.com` | +| `DOIS` | Comma-separated list of DOIs | No | `10.1234/test1,10.1234/test2` | +| `QUERY` | Search query (not fully supported) | No | `machine learning` | + +## Cloud Storage Structure + +Papers are organized by ingestion date: + +``` +gs://your-bucket/ +├── unpaywall/ +│ ├── papers/ +│ │ └── 20240115/ +│ │ ├── 10_1234_test1.json +│ │ ├── 10_1234_test2.json +│ │ └── ... +│ └── logs/ +│ └── 20240115/ +│ └── ingestion_log.json +``` + +### Paper JSON Format + +```json +{ + "doi": "10.1234/example", + "title": "Example Paper Title", + "authors": [...], + "year": 2024, + "is_oa": true, + "oa_status": "gold", + "oa_locations": [...], + "best_oa_location": {...} +} +``` + +### Ingestion Log Format + +```json +{ + "total_requested": 100, + "successful": 98, + "failed": 2, + "timestamp": "2024-01-15T02:00:00", + "failed_dois": ["10.1234/failed1", "10.1234/failed2"] +} +``` + +## Monitoring + +### View Job Logs + +```bash +gcloud run jobs describe unpaywall-ingestion-job --region us-central1 +gcloud logging read "resource.type=cloud_run_job" --limit 50 +``` + +### Check Ingestion Statistics + +Papers and logs are stored in Cloud Storage: + +```bash +gsutil ls gs://your-bucket/unpaywall/papers/ +gsutil cat gs://your-bucket/unpaywall/logs/20240115/ingestion_log.json +``` + +## API Rate Limiting + +The Unpaywall API is rate-limited. Current implementation includes: +- 10-second timeout per request +- Batch processing with logging every 10 papers +- Retry logic (1 retry) for transient failures + +**Tips:** +- Space out bulk ingestions to avoid hitting rate limits +- Monitor failed DOIs in the ingestion log +- Stagger multiple job executions if needed + +## Cost Optimization + +- **Job timeout**: Set to 3600s (1 hour) by default. Adjust based on volume +- **CPU/Memory**: 2 CPU, 2GB memory. Adjust via Terraform if needed +- **Storage**: Papers stored as JSON (~1-5 KB per paper typical) + +Example cost estimate: +- 10,000 papers/month × 2 KB = ~20 GB storage +- Cloud Storage: ~$0.02/month for storage +- Cloud Run: ~$0.00 for 3600 execution seconds/month + +## Troubleshooting + +### Job Fails with "GCS_BUCKET not found" + +Check the bucket exists and the service account has write access: + +```bash +gsutil iam ch serviceAccount:unpaywall-ingestion-job@PROJECT_ID.iam.gserviceaccount.com:objectCreator gs://your-bucket +``` + +### High Failure Rate + +- Check Unpaywall API status +- Verify DOIs are valid +- Review logs in Cloud Logging + +### Job Timeout + +Reduce the number of DOIs per execution or increase timeout: + +```bash +# In terraform.tfvars +job_timeout = 7200 # 2 hours +terraform apply +``` + +## Development + +### Local Testing + +```bash +# Install dependencies +pip install -r requirements.txt +pip install requests google-cloud-storage + +# Test the ingester +python -c " +from data_pipeline.unpaywall_ingester import UnpaywallIngester + +ingester = UnpaywallIngester( + gcs_bucket='test-bucket', + email='your-email@example.com' +) + +# Test fetch (requires GCP credentials for Cloud Storage) +result = ingester._fetch_paper('10.1371/journal.pbio.1001535') +print(result) +" +``` + +### Run Tests + +```bash +pytest tests/test_unpaywall_ingester.py -v +``` + +## API Reference + +### UnpaywallIngester + +#### `ingest_papers(dois: list[str], date_prefix: str = None) -> dict` + +Ingest papers by DOI. + +**Parameters:** +- `dois`: List of DOI strings +- `date_prefix`: Storage date prefix (default: YYYYMMDD) + +**Returns:** Statistics dictionary with `successful`, `failed`, `timestamp`, and `failed_dois` + +#### `ingest_by_query(query: str, limit: int = 1000, date_prefix: str = None) -> dict` + +Ingest papers by search query (limited support). + +**Note:** Unpaywall API doesn't support full-text search. This method is a placeholder for custom search implementations. + +## Links + +- [Unpaywall API Documentation](https://unpaywall.org/products/api) +- [Cloud Run Documentation](https://cloud.google.com/run/docs) +- [Cloud Scheduler Documentation](https://cloud.google.com/scheduler/docs) +- [Google Cloud Storage Documentation](https://cloud.google.com/storage/docs) diff --git a/data-pipeline/main.py b/data-pipeline/main.py new file mode 100644 index 0000000..206a4b6 --- /dev/null +++ b/data-pipeline/main.py @@ -0,0 +1,41 @@ +"""Cloud Run job entry point for daily CrossRef ingestion.""" + +import os +import sys + +from unpaywall_ingester import UnpaywallIngester + + +def main(): + """Main entry point for Cloud Run job.""" + # Read environment variables + gcs_bucket = os.getenv("GCS_BUCKET") + email = os.getenv("UNPAYWALL_EMAIL") + days_back = int(os.getenv("DAYS_BACK", "1")) + download_pdfs = os.getenv("DOWNLOAD_PDFS", "true").lower() == "true" + + # Validate required variables + if not gcs_bucket: + raise ValueError("GCS_BUCKET environment variable is required") + if not email: + raise ValueError("UNPAYWALL_EMAIL environment variable is required") + + # Initialize ingester + ingester = UnpaywallIngester(gcs_bucket=gcs_bucket, email=email) + + # Ingest from CrossRef + print(f"Starting daily CrossRef ingestion (looking back {days_back} day(s))") + if download_pdfs: + print("⚠️ PDF download enabled - this may take longer") + + stats = ingester.ingest_daily_from_crossref( + days_back=days_back, + download_pdfs=download_pdfs, + ) + + print(f"Ingestion complete. Stats: {stats}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data-pipeline/manage_job.sh b/data-pipeline/manage_job.sh new file mode 100755 index 0000000..02ba95b --- /dev/null +++ b/data-pipeline/manage_job.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Helper script for managing the Unpaywall ingestion job + +set -e + +PROJECT_ID="${1:-}" +REGION="${2:-us-central1}" +ACTION="${3:-}" + +if [ -z "$PROJECT_ID" ]; then + echo "Usage: $0 [region] " + echo "" + echo "Actions:" + echo " build-image Build and push Docker image to Container Registry" + echo " deploy Deploy/update job with Terraform" + echo " run [dois|query] Manually trigger the job" + echo " logs Show recent job logs" + echo " status Show job status" + echo " delete Delete the job and scheduler" + exit 1 +fi + +case "$ACTION" in + build-image) + echo "Building Docker image..." + docker build -f data-pipeline/Dockerfile \ + -t gcr.io/$PROJECT_ID/unpaywall-ingestion:latest \ + -t gcr.io/$PROJECT_ID/unpaywall-ingestion:$(date +%s) \ + . + echo "Pushing image to Container Registry..." + docker push gcr.io/$PROJECT_ID/unpaywall-ingestion:latest + echo "Done! Update container_image in terraform.tfvars" + ;; + + deploy) + echo "Deploying with Terraform..." + cd infra + terraform init + terraform plan -var="project_id=$PROJECT_ID" -var="region=$REGION" + terraform apply -var="project_id=$PROJECT_ID" -var="region=$REGION" + cd - + echo "Deployment complete!" + ;; + + run) + echo "Triggering job execution..." + if [ "$3" == "dois" ]; then + read -p "Enter comma-separated DOIs: " dois + gcloud run jobs execute unpaywall-ingestion-job \ + --project=$PROJECT_ID \ + --region=$REGION \ + --set-env-vars="DOIS=$dois" + elif [ "$3" == "query" ]; then + read -p "Enter search query: " query + gcloud run jobs execute unpaywall-ingestion-job \ + --project=$PROJECT_ID \ + --region=$REGION \ + --set-env-vars="QUERY=$query" + else + gcloud run jobs execute unpaywall-ingestion-job \ + --project=$PROJECT_ID \ + --region=$REGION + fi + echo "Job triggered!" + ;; + + logs) + echo "Fetching recent logs..." + gcloud logging read \ + "resource.type=cloud_run_job AND resource.labels.job_name=unpaywall-ingestion-job" \ + --project=$PROJECT_ID \ + --limit=50 \ + --format="table(timestamp,jsonPayload.message)" \ + --sort-by=timestamp.reverse + ;; + + status) + echo "Job status:" + gcloud run jobs describe unpaywall-ingestion-job \ + --project=$PROJECT_ID \ + --region=$REGION \ + --format="table(name,status)" + echo "" + echo "Scheduler status:" + gcloud scheduler jobs describe unpaywall-ingestion-trigger \ + --project=$PROJECT_ID \ + --location=$REGION \ + --format="table(name,schedule,state)" + ;; + + delete) + read -p "Are you sure? This will delete the job and scheduler. (y/N) " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + cd infra + terraform destroy -var="project_id=$PROJECT_ID" -var="region=$REGION" + cd - + echo "Deleted!" + else + echo "Aborted." + fi + ;; + + *) + echo "Unknown action: $ACTION" + exit 1 + ;; +esac diff --git a/data-pipeline/unpaywall_ingester.py b/data-pipeline/unpaywall_ingester.py new file mode 100644 index 0000000..b451280 --- /dev/null +++ b/data-pipeline/unpaywall_ingester.py @@ -0,0 +1,382 @@ +"""Ingestion job for papers from Unpaywall and CrossRef APIs.""" + +import json +import logging +from datetime import datetime, timedelta +from typing import Optional +from urllib.parse import urlencode + +import requests +import urllib3 +from google.cloud import storage + +# Suppress InsecureRequestWarning from urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class UnpaywallIngester: + """Ingests open access papers from Unpaywall API and CrossRef, stores in Cloud Storage.""" + + UNPAYWALL_BASE_URL = "https://api.unpaywall.org/v2" + CROSSREF_BASE_URL = "https://api.crossref.org/v1" + BATCH_SIZE = 100 + + def __init__( + self, + gcs_bucket: str, + email: str, + gcs_project_id: Optional[str] = None, + ): + """ + Initialize the Unpaywall ingester. + + Args: + gcs_bucket: Google Cloud Storage bucket name + email: Email for Unpaywall API (required by API) + gcs_project_id: GCP project ID (optional, uses default if not provided) + """ + self.gcs_bucket = gcs_bucket + self.email = email + self.storage_client = storage.Client(project=gcs_project_id) + self.bucket = self.storage_client.bucket(gcs_bucket) + + + def ingest_daily_from_crossref( + self, + days_back: int = 1, + limit_per_day: int = 1000, + download_pdfs: bool = False, + date_prefix: Optional[str] = None, + ) -> dict: + """ + Ingest papers from CrossRef published in the last N days, check Unpaywall for OA. + + Args: + days_back: Number of days to look back (1 = yesterday, 2 = last 2 days, etc.) + limit_per_day: Maximum papers per day from CrossRef + download_pdfs: Whether to download PDF files from Unpaywall + date_prefix: Optional custom date prefix for storage path + + Returns: + Dictionary with ingestion statistics + """ + if not date_prefix: + date_prefix = datetime.utcnow().strftime("%Y%m%d") + + stats = { + "source": "crossref", + "days_back": days_back, + "download_pdfs": download_pdfs, + "total_papers_found": 0, + "successful_oa": 0, + "pdfs_downloaded": 0, + "failed": 0, + "timestamp": datetime.utcnow().isoformat(), + "failed_dois": [], + } + + for day_offset in range(days_back): + target_date = datetime.utcnow() - timedelta(days=day_offset + 1) + date_str = target_date.strftime("%Y-%m-%d") + + logger.info(f"Fetching {limit_per_day} papers from {date_str}") + + papers = self._fetch_crossref_papers(date_str, limit_per_day) + logger.info(f"Found {len(papers)} papers from {date_str}") + + for paper in papers: + stats["total_papers_found"] += 1 + + try: + doi = paper.get("doi") + if not doi: + logger.warning("Paper without DOI, skipping") + continue + + oa_data = self._fetch_paper_unpaywall(doi) + if oa_data and oa_data.get("is_oa"): + self._store_paper(oa_data, date_prefix) + stats["successful_oa"] += 1 + + # Download PDF if requested + if download_pdfs: + if self._download_pdf(oa_data, date_prefix): + stats["pdfs_downloaded"] += 1 + else: + logger.debug(f"Paper not OA or not indexed: {doi}") + + except Exception as e: + logger.error(f"Failed to process paper: {str(e)}") + stats["failed"] += 1 + stats["failed_dois"].append(paper.get("doi")) + + if stats["total_papers_found"] % 100 == 0: + logger.info( + f"Processed {stats['total_papers_found']} papers " + f"({stats['successful_oa']} OA, {stats['pdfs_downloaded']} PDFs)" + ) + + self._store_ingestion_log(stats, date_prefix) + return stats + + def _fetch_paper(self, doi: str) -> Optional[dict]: + """ + Fetch paper metadata from Unpaywall API. + + Args: + doi: Digital Object Identifier + + Returns: + Paper metadata dictionary or None if not found + """ + try: + url = f"{self.UNPAYWALL_BASE_URL}/{doi}" + params = {"email": self.email} + response = requests.get(url, params=params, timeout=10, verify=False) + if response.status_code == 404: + logger.debug(f"DOI not found in Unpaywall: {doi}") + return None + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + logger.error(f"API error fetching DOI {doi}: {str(e)}") + return None + + def _fetch_paper_unpaywall(self, doi: str) -> Optional[dict]: + """Alias for _fetch_paper for clarity.""" + return self._fetch_paper(doi) + + def _fetch_crossref_papers(self, date: str, limit: int = 10) -> list[dict]: + """ + Fetch papers published on a specific date from CrossRef API. + + Args: + date: Date string in YYYY-MM-DD format + limit: Maximum number of results + + Returns: + List of paper metadata dictionaries + """ + try: + papers = [] + url = f"{self.CROSSREF_BASE_URL}/works" + + # Query papers from specific date + params = { + "filter": f"from-pub-date:{date},until-pub-date:{date}", + "rows": min(1000, limit), # CrossRef max is 1000 per request + "sort": "published", + "order": "asc", + "select": "DOI,title,author,published-online,type", + } + + offset = 0 + while len(papers) < limit: + params["offset"] = offset + response = requests.get(url, params=params, timeout=15, verify=False) + response.raise_for_status() + + data = response.json() + items = data.get("message", {}).get("items", []) + + if not items: + break + + for item in items: + if len(papers) >= limit: + break + + doi = item.get("DOI") + if doi: + papers.append({ + "doi": doi, + "title": item.get("title", [""])[0] if item.get("title") else "", + "authors": item.get("author", []), + "published": item.get("published-online", {}).get("date-parts", [[None]])[0], + "type": item.get("type"), + }) + + offset += len(items) + logger.debug(f"Fetched {len(papers)} papers from CrossRef") + + logger.info(f"Retrieved {len(papers)} papers from CrossRef for {date}") + return papers + + except requests.exceptions.RequestException as e: + logger.error(f"CrossRef API error: {str(e)}") + return [] + + def _store_paper(self, paper: dict, date_prefix: str) -> None: + """ + Store paper metadata in Cloud Storage. + + Args: + paper: Paper metadata dictionary + date_prefix: Date prefix for storage path (YYYYMMDD format) + """ + doi = paper.get("doi", "unknown").replace("/", "_") + blob_name = f"unpaywall/papers/{date_prefix}/{doi}.json" + + blob = self.bucket.blob(blob_name) + blob.upload_from_string( + json.dumps(paper, indent=2), + content_type="application/json", + ) + logger.debug(f"Stored paper: {blob_name}") + + def _download_pdf(self, paper: dict, date_prefix: str) -> Optional[str]: + """ + Download PDF from Unpaywall best OA location with validation. + + Args: + paper: Paper metadata dictionary from Unpaywall + date_prefix: Date prefix for storage path + + Returns: + GCS blob name if successful, None otherwise + """ + try: + doi = paper.get("doi", "unknown").replace("/", "_") + best_oa_location = paper.get("best_oa_location") + + if not best_oa_location: + logger.debug(f"No OA location for {doi}") + return None + + pdf_url = best_oa_location.get("url_for_pdf") or best_oa_location.get("url") + if not pdf_url: + logger.debug(f"No PDF URL available for {doi}") + return None + + # Download PDF with retry logic + logger.info(f"Downloading PDF for {doi} from {pdf_url}") + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "application/pdf" + } + + max_retries = 2 + for attempt in range(max_retries + 1): + try: + response = requests.get( + pdf_url, + timeout=30, + verify=False, + headers=headers, + allow_redirects=True, + stream=True + ) + response.raise_for_status() + break + except requests.exceptions.RequestException as e: + if attempt < max_retries: + logger.warning(f"Download attempt {attempt + 1} failed for {doi}, retrying...") + continue + raise + + # Read content with size limit (50MB max) + max_size = 50 * 1024 * 1024 + content = b"" + for chunk in response.iter_content(chunk_size=8192): + content += chunk + if len(content) > max_size: + logger.warning(f"PDF too large for {doi}: exceeded {max_size} bytes") + return None + + # Comprehensive PDF validation + if not self._is_valid_pdf(content, doi): + return None + + # Store in Cloud Storage + blob_name = f"unpaywall/pdfs/{date_prefix}/{doi}.pdf" + blob = self.bucket.blob(blob_name) + blob.upload_from_string( + content, + content_type="application/pdf", + ) + logger.info(f"Stored PDF: {blob_name} ({len(content)} bytes)") + return blob_name + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to download PDF for {doi}: {str(e)}") + return None + except Exception as e: + logger.error(f"Error storing PDF for {doi}: {str(e)}") + return None + + def _is_valid_pdf(self, content: bytes, doi: str) -> bool: + """ + Validate that content is a valid PDF file. + + Args: + content: File content bytes + doi: DOI for logging + + Returns: + True if valid PDF, False otherwise + """ + if len(content) < 100: + logger.warning(f"PDF too small for {doi}: {len(content)} bytes") + return False + + # Check PDF magic bytes/header + if not content.startswith(b"%PDF"): + logger.warning(f"Invalid PDF header for {doi}: {content[:20]}") + return False + + # Check for PDF footer (should end with %%EOF) + if not (b"%%EOF" in content[-20:] or b"%EOF" in content[-20:]): + logger.warning(f"Missing PDF footer for {doi}") + return False + + # Check for common corruption patterns (basic check) + # If file contains too much non-ASCII content early on, it's likely corrupted + try: + # Try to find the xref table (valid PDFs have this) + if b"xref" not in content: + logger.warning(f"Missing xref table in PDF for {doi}") + return False + except Exception as e: + logger.warning(f"PDF validation error for {doi}: {str(e)}") + return False + + return True + + def _store_ingestion_log(self, stats: dict, date_prefix: str) -> None: + """ + Store ingestion statistics to Cloud Storage. + + Args: + stats: Ingestion statistics dictionary + date_prefix: Date prefix for storage path + """ + blob_name = f"unpaywall/logs/{date_prefix}/ingestion_log.json" + blob = self.bucket.blob(blob_name) + blob.upload_from_string( + json.dumps(stats, indent=2), + content_type="application/json", + ) + logger.info(f"Ingestion log stored: {blob_name}") + + +def run_job( + gcs_bucket: str, + email: str, +) -> None: + """ + Main job entry point for Cloud Run. + + Args: + gcs_bucket: Google Cloud Storage bucket name + email: Email for Unpaywall API + """ + logger.info("Starting Unpaywall ingestion job") + + ingester = UnpaywallIngester(gcs_bucket=gcs_bucket, email=email) + + stats = ingester.ingest_daily_from_crossref(days_back=1) + + logger.info(f"Job completed. Stats: {stats}") diff --git a/infra/gcp/data-pipeline/terraform.tfvars.example b/infra/gcp/data-pipeline/terraform.tfvars.example new file mode 100644 index 0000000..57d0bc0 --- /dev/null +++ b/infra/gcp/data-pipeline/terraform.tfvars.example @@ -0,0 +1,30 @@ +# Example Terraform Configuration for Unpaywall Ingestion Job +# Copy this to infra/terraform.tfvars and customize for your environment + +project_id = "my-gcp-project" +region = "us-central1" + +# Cloud Storage bucket where papers will be stored +gcs_bucket = "my-papers-bucket" + +# Email registered with Unpaywall API (https://unpaywall.org/products/api) +unpaywall_email = "your-email@example.com" + +# Container image (must be pushed to registry first) +container_image = "gcr.io/my-gcp-project/unpaywall-ingestion:latest" + +# Number of days to look back (1-30) +# 1 = yesterday's papers, 7 = last week, etc. +days_back = 1 + +# Cron schedule: Daily at 2 AM UTC +# Format: "minute hour day month dayofweek" +# Examples: +# "0 2 * * *" - Daily at 2 AM +# "0 */6 * * *" - Every 6 hours +# "0 2 * * 0" - Weekly on Sunday at 2 AM +schedule_frequency = "0 2 * * *" + +# Job timeout in seconds (default: 3600 = 1 hour) +job_timeout = 3600 + diff --git a/infra/gcp/data-pipeline/unpaywall_job.tf b/infra/gcp/data-pipeline/unpaywall_job.tf new file mode 100644 index 0000000..ba75bee --- /dev/null +++ b/infra/gcp/data-pipeline/unpaywall_job.tf @@ -0,0 +1,164 @@ +# Cloud Run Job for Unpaywall paper ingestion +# This Terraform configuration creates a scheduled Cloud Run job that ingests +# papers from Unpaywall and stores them in Google Cloud Storage. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "gcs_bucket" { + description = "Google Cloud Storage bucket for storing papers" + type = string +} + +variable "unpaywall_email" { + description = "Email for Unpaywall API" + type = string + sensitive = true +} + +variable "container_image" { + description = "Container image URI for the Cloud Run job" + type = string +} + +variable "schedule_frequency" { + description = "Cron schedule for job execution (e.g., '0 2 * * *' for 2 AM daily)" + type = string + default = "0 2 * * *" # Daily at 2 AM UTC +} + +variable "job_timeout" { + description = "Job timeout in seconds" + type = number + default = 3600 # 1 hour +} + +variable "days_back" { + description = "Number of days to look back when fetching from CrossRef (1-30)" + type = number + default = 1 + + validation { + condition = var.days_back > 0 && var.days_back <= 30 + error_message = "days_back must be between 1 and 30" + } +} + +# Service account for the Cloud Run job +resource "google_service_account" "unpaywall_job" { + account_id = "unpaywall-ingestion-job" + display_name = "Unpaywall Ingestion Job Service Account" +} + +# Grant Cloud Storage write access +resource "google_storage_bucket_iam_member" "job_storage_access" { + bucket = var.gcs_bucket + role = "roles/storage.objectCreator" + member = google_service_account.unpaywall_job.member +} + +# Cloud Run Job +resource "google_cloud_run_v2_job" "unpaywall_ingestion" { + name = "unpaywall-ingestion-job" + location = var.region + + deletion_protection = false + + template { + parallelism = 1 + task_count = 1 + timeout = "${var.job_timeout}s" + service_account = google_service_account.unpaywall_job.email + execution_environment = "GEN2" + max_retries = 1 + + containers { + image = var.container_image + + env { + name = "GCS_BUCKET" + value = var.gcs_bucket + } + + env { + name = "UNPAYWALL_EMAIL" + value = var.unpaywall_email + } + + env { + name = "DAYS_BACK" + value = tostring(var.days_back) + } + + resources { + limits = { + cpu = "2" + memory = "2Gi" + } + } + } + } + + depends_on = [ + google_storage_bucket_iam_member.job_storage_access, + ] +} + +# Cloud Scheduler to trigger the job on a schedule +resource "google_cloud_scheduler_job" "unpaywall_trigger" { + name = "unpaywall-ingestion-trigger" + description = "Triggers Unpaywall ingestion job on a schedule" + schedule = var.schedule_frequency + time_zone = "UTC" + attempt_deadline = "320s" + region = var.region + + retry_config { + retry_count = 1 + } + + http_target { + http_method = "POST" + uri = "https://${var.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${var.project_id}/jobs/${google_cloud_run_v2_job.unpaywall_ingestion.name}:run" + + headers = { + "Content-Type" = "application/json" + } + + auth_header { + service_account_email = google_service_account.unpaywall_job.email + } + + body = jsonencode({}) + } + + depends_on = [google_cloud_run_v2_job.unpaywall_ingestion] +} + +# Output the job name for reference +output "job_name" { + value = google_cloud_run_v2_job.unpaywall_ingestion.name + description = "Name of the Cloud Run job" +} + +output "scheduler_job_name" { + value = google_cloud_scheduler_job.unpaywall_trigger.name + description = "Name of the Cloud Scheduler job" +} diff --git a/tests/test_unpaywall_ingester.py b/tests/test_unpaywall_ingester.py new file mode 100644 index 0000000..7731e5d --- /dev/null +++ b/tests/test_unpaywall_ingester.py @@ -0,0 +1,193 @@ +"""Unit tests for Unpaywall ingester.""" + +import json +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from data_pipeline.unpaywall_ingester import UnpaywallIngester + + +@pytest.fixture +def mock_storage(): + """Mock Google Cloud Storage client.""" + with patch("data_pipeline.unpaywall_ingester.storage") as mock: + yield mock + + +@pytest.fixture +def ingester(mock_storage): + """Create an ingester instance with mocked storage.""" + mock_storage.Client.return_value = MagicMock() + return UnpaywallIngester( + gcs_bucket="test-bucket", + email="test@example.com", + ) + + +class TestUnpaywallIngester: + """Test cases for UnpaywallIngester.""" + + def test_initialization(self, ingester): + """Test ingester initialization.""" + assert ingester.gcs_bucket == "test-bucket" + assert ingester.email == "test@example.com" + + @patch("data_pipeline.unpaywall_ingester.requests.get") + def test_fetch_paper_success(self, mock_get, ingester): + """Test successful paper fetch.""" + mock_response = Mock() + mock_response.json.return_value = { + "doi": "10.1234/test", + "is_oa": True, + "title": "Test Paper", + } + mock_get.return_value = mock_response + + result = ingester._fetch_paper("10.1234/test") + + assert result["is_oa"] is True + assert result["title"] == "Test Paper" + mock_get.assert_called_once() + + @patch("data_pipeline.unpaywall_ingester.requests.get") + def test_fetch_paper_failure(self, mock_get, ingester): + """Test paper fetch with API error.""" + mock_get.side_effect = Exception("API Error") + + result = ingester._fetch_paper("10.1234/test") + + assert result is None + + def test_store_paper(self, ingester): + """Test storing paper metadata.""" + paper = { + "doi": "10.1234/test", + "title": "Test Paper", + "is_oa": True, + } + + ingester._store_paper(paper, "20240101") + + # Verify blob upload was called + ingester.bucket.blob.assert_called_once() + call_args = ingester.bucket.blob.call_args + assert "unpaywall/papers/20240101" in call_args[0][0] + + def test_ingest_papers_with_valid_dois(self, ingester): + """Test ingesting multiple papers.""" + dois = ["10.1234/test1", "10.1234/test2"] + + with patch.object( + ingester, "_fetch_paper" + ) as mock_fetch, patch.object(ingester, "_store_paper") as mock_store: + mock_fetch.side_effect = [ + {"doi": "10.1234/test1", "is_oa": True}, + {"doi": "10.1234/test2", "is_oa": True}, + ] + + stats = ingester.ingest_papers(dois) + + assert stats["total_requested"] == 2 + assert stats["successful"] == 2 + assert mock_store.call_count == 2 + + def test_ingest_papers_with_mixed_results(self, ingester): + """Test ingesting papers with some failures.""" + dois = ["10.1234/test1", "10.1234/test2"] + + with patch.object( + ingester, "_fetch_paper" + ) as mock_fetch, patch.object(ingester, "_store_paper"): + mock_fetch.side_effect = [ + {"doi": "10.1234/test1", "is_oa": True}, + None, # Failed fetch + ] + + stats = ingester.ingest_papers(dois) + + assert stats["total_requested"] == 2 + assert stats["successful"] == 1 + assert stats["failed"] == 1 + + def test_custom_date_prefix(self, ingester): + """Test using custom date prefix for storage.""" + dois = ["10.1234/test"] + + with patch.object( + ingester, "_fetch_paper" + ) as mock_fetch, patch.object(ingester, "_store_paper") as mock_store: + mock_fetch.return_value = {"doi": "10.1234/test", "is_oa": True} + + ingester.ingest_papers(dois, date_prefix="20250101") + + mock_store.assert_called_once() + call_args = mock_store.call_args + assert call_args[0][1] == "20250101" + + def test_ingest_daily_from_crossref(self, ingester): + """Test ingesting papers from CrossRef.""" + with patch.object( + ingester, "_fetch_crossref_papers" + ) as mock_crossref, patch.object( + ingester, "_fetch_paper_unpaywall" + ) as mock_unpaywall, patch.object(ingester, "_store_paper") as mock_store: + mock_crossref.return_value = [ + {"doi": "10.1234/test1", "title": "Paper 1"}, + {"doi": "10.1234/test2", "title": "Paper 2"}, + ] + mock_unpaywall.side_effect = [ + {"doi": "10.1234/test1", "is_oa": True}, + {"doi": "10.1234/test2", "is_oa": False}, + ] + + stats = ingester.ingest_daily_from_crossref(days_back=1) + + assert stats["total_papers_found"] == 2 + assert stats["successful_oa"] == 1 + + @patch("data_pipeline.unpaywall_ingester.requests.get") + def test_fetch_crossref_papers(self, mock_get, ingester): + """Test fetching papers from CrossRef.""" + mock_response = Mock() + mock_response.json.return_value = { + "message": { + "items": [ + { + "DOI": "10.1234/test1", + "title": ["Test Paper 1"], + "author": [{"given": "John", "family": "Doe"}], + "published-online": {"date-parts": [[2024, 1, 15]]}, + "type": "journal-article", + } + ] + } + } + mock_get.return_value = mock_response + + papers = ingester._fetch_crossref_papers("2024-01-15") + + assert len(papers) >= 1 + assert papers[0]["doi"] == "10.1234/test1" + assert "test paper 1" in papers[0]["title"].lower() + + @patch("data_pipeline.unpaywall_ingester.requests.get") + def test_fetch_crossref_papers_empty(self, mock_get, ingester): + """Test CrossRef fetch with no results.""" + mock_response = Mock() + mock_response.json.return_value = {"message": {"items": []}} + mock_get.return_value = mock_response + + papers = ingester._fetch_crossref_papers("2024-01-15") + + assert len(papers) == 0 + + def test_fetch_paper_unpaywall_alias(self, ingester): + """Test that _fetch_paper_unpaywall is an alias for _fetch_paper.""" + with patch.object(ingester, "_fetch_paper") as mock_fetch: + mock_fetch.return_value = {"doi": "10.1234/test", "is_oa": True} + + result = ingester._fetch_paper_unpaywall("10.1234/test") + + assert result["is_oa"] is True + mock_fetch.assert_called_once_with("10.1234/test")