From d3ec875ebe5ab237a9bc4b38658b56c90125156c Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Thu, 31 Aug 2023 15:48:13 -0700 Subject: [PATCH 01/28] add rules evaluation --- Makefile | 9 +- lekko_client/evaluation/evaluation.py | 60 +++ lekko_client/evaluation/rules.py | 189 ++++++++++ .../v1beta1/configuration_service_pb2.py | 1 - .../v1beta1/configuration_service_pb2_grpc.py | 2 +- .../gen/lekko/feature/v1beta1/feature_pb2.py | 38 ++ .../gen/lekko/feature/v1beta1/feature_pb2.pyi | 199 ++++++++++ .../lekko/feature/v1beta1/feature_pb2_grpc.py | 4 + .../gen/lekko/feature/v1beta1/static_pb2.py | 48 +++ .../gen/lekko/feature/v1beta1/static_pb2.pyi | 351 ++++++++++++++++++ .../lekko/feature/v1beta1/static_pb2_grpc.py | 4 + .../gen/lekko/rules/v1beta2/rules_pb2.py | 44 +++ .../gen/lekko/rules/v1beta2/rules_pb2.pyi | 199 ++++++++++ .../gen/lekko/rules/v1beta2/rules_pb2_grpc.py | 4 + .../gen/lekko/rules/v1beta3/rules_pb2.py | 38 ++ .../gen/lekko/rules/v1beta3/rules_pb2.pyi | 257 +++++++++++++ .../gen/lekko/rules/v1beta3/rules_pb2_grpc.py | 4 + pyproject.toml | 1 + 18 files changed, 1447 insertions(+), 5 deletions(-) create mode 100644 lekko_client/evaluation/evaluation.py create mode 100644 lekko_client/evaluation/rules.py create mode 100644 lekko_client/gen/lekko/feature/v1beta1/feature_pb2.py create mode 100644 lekko_client/gen/lekko/feature/v1beta1/feature_pb2.pyi create mode 100644 lekko_client/gen/lekko/feature/v1beta1/feature_pb2_grpc.py create mode 100644 lekko_client/gen/lekko/feature/v1beta1/static_pb2.py create mode 100644 lekko_client/gen/lekko/feature/v1beta1/static_pb2.pyi create mode 100644 lekko_client/gen/lekko/feature/v1beta1/static_pb2_grpc.py create mode 100644 lekko_client/gen/lekko/rules/v1beta2/rules_pb2.py create mode 100644 lekko_client/gen/lekko/rules/v1beta2/rules_pb2.pyi create mode 100644 lekko_client/gen/lekko/rules/v1beta2/rules_pb2_grpc.py create mode 100644 lekko_client/gen/lekko/rules/v1beta3/rules_pb2.py create mode 100644 lekko_client/gen/lekko/rules/v1beta3/rules_pb2.pyi create mode 100644 lekko_client/gen/lekko/rules/v1beta3/rules_pb2_grpc.py diff --git a/Makefile b/Makefile index 70111c5..28aa012 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,9 @@ fmt: venv .PHONY: bufgen bufgen: - buf generate buf.build/lekkodev/sdk - sed -i'.bak' -e 's/^from lekko.client.v1beta1/from ./' lekko_client/gen/lekko/client/v1beta1/*.py - rm lekko_client/gen/lekko/client/v1beta1/*.bak + buf generate buf.build/lekkodev/sdk --type lekko.client.v1beta1 + buf generate buf.build/lekkodev/cli --type lekko.rules.v1beta3 --type lekko.feature.v1beta1 + grep -rl "from lekko.\|import lekko.\|type: lekko." ./lekko_client/gen --include \*.py --include \*.pyi | xargs sed -i'.bak' -E -e 's/ lekko\./ lekko_client.gen.lekko./' + rm -f lekko_client/gen/lekko/client/v1beta1/*.bak + rm -f lekko_client/gen/lekko/feature/v1beta1/*.bak + rm -f lekko_client/gen/lekko/rules/v1beta3/*.bak diff --git a/lekko_client/evaluation/evaluation.py b/lekko_client/evaluation/evaluation.py new file mode 100644 index 0000000..f4b9367 --- /dev/null +++ b/lekko_client/evaluation/evaluation.py @@ -0,0 +1,60 @@ +from dataclasses import dataclass +from typing import List, Optional + +from google.protobuf.any_pb2 import Any + +from lekko_client.evaluation.rules import ClientContext, evaluate_rule +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Constraint, Feature + + +@dataclass +class EvaluationResult: + value: Any + # Stores the path of the tree node that returned the final value + # after successful evaluation. + path: List[int] + + +@dataclass +class TraverseResult: + value: Optional[Any] + passes: bool + path: List[int] + + +def evaluate(config: Feature, namespace: str, context: ClientContext = None) -> EvaluationResult: + if not config.tree: + raise ValueError("config tree is empty") + + for i, constraint in enumerate(config.tree.constraints): + child_result = traverse(constraint, namespace, config.key, context) + if child_result.passes: + if child_result.value: + return EvaluationResult(value=child_result.value, path=[i, *child_result.path]) + break + return EvaluationResult(value=_get_any(config.tree.default, config.tree.default_new), path=[]) + + +def traverse(override: Constraint, namespace: str, config_name: str, context: ClientContext = None) -> TraverseResult: + if not override: + return TraverseResult(None, False, []) + passes = evaluate_rule(override.rule_ast_new, namespace, config_name, context) + if not passes: + return TraverseResult(None, False, []) + + for i, constraint in enumerate(override.constraints): + child_result = traverse(constraint, namespace, config_name, context) + if child_result.passes: + if child_result.value: + return TraverseResult(child_result.value, True, [i, *child_result.path]) + break + return TraverseResult(_get_any(override.value, override.value_new), True, []) + + +def _get_any(val: Optional[Any], val_new: Optional[LekkoAny]) -> Any: + if val_new and val_new.type_url: + return Any(type_url=val_new.type_url, value=val_new.value) + if val: + return val + raise ValueError("config value not found") diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py new file mode 100644 index 0000000..acd78f1 --- /dev/null +++ b/lekko_client/evaluation/rules.py @@ -0,0 +1,189 @@ +import struct +from typing import Dict, Optional + +from google.protobuf.struct_pb2 import Value +from xxhash import xxh32 + +from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import ( + Value as LekkoValue, +) +from lekko_client.gen.lekko.rules.v1beta3.rules_pb2 import ( + CallExpression, + ComparisonOperator, + LogicalOperator, + Rule, +) + +ClientContext = Optional[Dict[str, LekkoValue]] + + +def evaluate_rule(rule: Rule, namespace: str, config_name: str, context: ClientContext = None) -> bool: + if not rule: + raise ValueError("empty rule") + + rule_type = rule.WhichOneof("rule") + if not rule_type: + raise ValueError("empty rule") + + rule_value = getattr(rule, rule_type) + + if rule_type == "bool_const": + return rule_value + elif rule_type == "not": + return not evaluate_rule(rule_value, namespace, config_name, context) + elif rule_type == "logical_expression": + if not rule_value.rules: + raise ValueError("no rules found in logical expression") + + logical_operator = rule_value.logical_operator + return ( + all(evaluate_rule(r, namespace, config_name, context) for r in rule_value.rules) + if logical_operator == LogicalOperator.LOGICAL_OPERATOR_AND + else any(evaluate_rule(r, namespace, config_name, context) for r in rule_value.rules) + ) + elif rule_type == "atom": + context_key = rule_value.context_key + context_value = context.get(context_key) if context else None + + if rule_value.comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_PRESENT: + return context_value is not None + + if context_value is None: + return False + + if rule_value.comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_EQUALS: + return evaluate_equals(rule_value.comparison_value, context_value) + elif rule_value.comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_NOT_EQUALS: + return not evaluate_equals(rule_value.comparison_value, context_value) + elif rule_value.comparison_operator in ( + ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS, + ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS, + ): + return evaluate_number_comparator( + rule_value.comparison_operator, rule_value.comparison_value, context_value + ) + elif rule_value.comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_CONTAINED_WITHIN: + return evaluate_contained_within(rule_value.comparison_value, context_value) + elif rule_value.comparison_operator in ( + ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH, + ComparisonOperator.COMPARISON_OPERATOR_ENDS_WITH, + ComparisonOperator.COMPARISON_OPERATOR_CONTAINS, + ): + return evaluate_string_comparator( + rule_value.comparison_operator, rule_value.comparison_value, context_value + ) + else: + raise ValueError("unknown comparison operator") + elif rule_type == "call_expression": + if rule_value.WhichOneof("function") == "bucket": + return evaluate_bucket(rule_value.bucket, namespace, config_name, context) + else: + raise ValueError("unknown function type") + else: + raise ValueError("unknown rule type") + + +def evaluate_equals(rule_value: Value, context_value: LekkoValue) -> bool: + rule_kind = rule_value.WhichOneof("kind") or "" + context_kind = context_value.WhichOneof("kind") or "" + if rule_kind not in ["bool_value", "string_value", "number_value"]: + raise ValueError("unsupported type for equals operator") + + if rule_kind == "number_value": + if context_kind not in ["double_value", "int_value"]: + raise ValueError("type mismatch") + elif rule_kind != context_kind: + raise ValueError("type mismatch") + + return getattr(rule_value, rule_kind) == getattr(context_value, context_kind) + + +def evaluate_string_comparator( + comparison_operator: ComparisonOperator, rule_value: Value, context_value: LekkoValue +) -> bool: + rule_str = get_string(rule_value) + context_str = get_string(context_value) + + if comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH: + return context_str.startswith(rule_str) + elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_ENDS_WITH: + return context_str.endswith(rule_str) + elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_CONTAINS: + return rule_str in context_str + else: + raise ValueError("unexpected string comparison operator") + + +def get_string(value: Value | LekkoValue) -> str: + if not value: + raise ValueError("value is undefined") + + if value.WhichOneof("kind") == "string_value": + return value.string_value + else: + raise ValueError("value is not a string") + + +def evaluate_number_comparator( + comparison_operator: ComparisonOperator, rule_value: Value, context_value: LekkoValue +) -> bool: + rule_num = get_number(rule_value) + context_num = get_number(context_value) + + if comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN: + return context_num < rule_num + elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS: + return context_num <= rule_num + elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN: + return context_num > rule_num + elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: + return context_num >= rule_num + else: + raise ValueError("unexpected numerical comparison operator") + + +def get_number(value: Value | LekkoValue) -> float: + value_kind = value.WhichOneof("kind") + if value_kind in ["number_value", "int_value", "double_value"]: + return float(getattr(value, value_kind)) + else: + raise ValueError("value is not a number") + + +def evaluate_contained_within(rule_value: Value, context_value: LekkoValue) -> bool: + if rule_value.WhichOneof("kind") != "list_value": + raise ValueError("type mismatch: expecting list for operator contained within") + + # TODO: this will throw if there's a type mismatch, which means that all items in rule list must be of same type + # This is consistent with other language SDKs, but we should consider just returning False on type mismatch + return any(evaluate_equals(list_elem_val, context_value) for list_elem_val in rule_value.list_value.values) + + +def evaluate_bucket(bucket_f: CallExpression.Bucket, namespace: str, config_name: str, context: ClientContext): + ctx_key = bucket_f.context_key + value = context.get(ctx_key) if context else None + if not value: + # If key is missing in context map, evaluate to false - move to next rule + return False + + bytes_buffer: bytes = b"" + + value_kind = value.WhichOneof("kind") + if not value_kind: + return False + + value_val = getattr(value, value_kind) + if value_kind == "string_value": + bytes_buffer = bytes(value_val, "utf-8") + elif value_kind == "int_value": + bytes_buffer = value_val.to_bytes(8, byteorder="big") + elif value_kind == "double_value": + bytes_buffer = struct.pack(">d", value_val) + else: + raise ValueError("unsupported value type for bucket") + + bytes_frags = [bytes(namespace, "utf-8"), bytes(config_name, "utf-8"), bytes(ctx_key, "utf-8"), bytes_buffer] + result = xxh32(b"".join(bytes_frags), 0).intdigest() + return result % 100000 <= bucket_f.threshold diff --git a/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2.py b/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2.py index 34c05d7..20f6ded 100644 --- a/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2.py +++ b/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2.py @@ -20,7 +20,6 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.client.v1beta1.configuration_service_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _GETBOOLVALUEREQUEST_CONTEXTENTRY._options = None _GETBOOLVALUEREQUEST_CONTEXTENTRY._serialized_options = b'8\001' diff --git a/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2_grpc.py b/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2_grpc.py index dea8334..5f3d05c 100644 --- a/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2_grpc.py +++ b/lekko_client/gen/lekko/client/v1beta1/configuration_service_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -from . import configuration_service_pb2 as lekko_dot_client_dot_v1beta1_dot_configuration__service__pb2 +from lekko_client.gen.lekko.client.v1beta1 import configuration_service_pb2 as lekko_dot_client_dot_v1beta1_dot_configuration__service__pb2 class ConfigurationServiceStub(object): diff --git a/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.py b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.py new file mode 100644 index 0000000..ad461ec --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lekko/feature/v1beta1/feature.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 +from lekko_client.gen.lekko.rules.v1beta2 import rules_pb2 as lekko_dot_rules_dot_v1beta2_dot_rules__pb2 +from lekko_client.gen.lekko.rules.v1beta3 import rules_pb2 as lekko_dot_rules_dot_v1beta3_dot_rules__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#lekko/feature/v1beta1/feature.proto\x12\x15lekko.feature.v1beta1\x1a\x19google/protobuf/any.proto\x1a\x1flekko/rules/v1beta2/rules.proto\x1a\x1flekko/rules/v1beta3/rules.proto\"\xa6\x01\n\x07\x46\x65\x61ture\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12/\n\x04tree\x18\x03 \x01(\x0b\x32\x1b.lekko.feature.v1beta1.TreeR\x04tree\x12\x36\n\x04type\x18\x04 \x01(\x0e\x32\".lekko.feature.v1beta1.FeatureTypeR\x04type\"\xb8\x01\n\x04Tree\x12.\n\x07\x64\x65\x66\x61ult\x18\x01 \x01(\x0b\x32\x14.google.protobuf.AnyR\x07\x64\x65\x66\x61ult\x12\x43\n\x0b\x63onstraints\x18\x02 \x03(\x0b\x32!.lekko.feature.v1beta1.ConstraintR\x0b\x63onstraints\x12;\n\x0b\x64\x65\x66\x61ult_new\x18\x03 \x01(\x0b\x32\x1a.lekko.feature.v1beta1.AnyR\ndefaultNew\"\xc1\x02\n\nConstraint\x12\x12\n\x04rule\x18\x01 \x01(\tR\x04rule\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05value\x12\x43\n\x0b\x63onstraints\x18\x03 \x03(\x0b\x32!.lekko.feature.v1beta1.ConstraintR\x0b\x63onstraints\x12\x38\n\x08rule_ast\x18\x04 \x01(\x0b\x32\x19.lekko.rules.v1beta2.RuleB\x02\x18\x01R\x07ruleAst\x12;\n\x0crule_ast_new\x18\x05 \x01(\x0b\x32\x19.lekko.rules.v1beta3.RuleR\nruleAstNew\x12\x37\n\tvalue_new\x18\x06 \x01(\x0b\x32\x1a.lekko.feature.v1beta1.AnyR\x08valueNew\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value*\xb8\x01\n\x0b\x46\x65\x61tureType\x12\x1c\n\x18\x46\x45\x41TURE_TYPE_UNSPECIFIED\x10\x00\x12\x15\n\x11\x46\x45\x41TURE_TYPE_BOOL\x10\x01\x12\x14\n\x10\x46\x45\x41TURE_TYPE_INT\x10\x02\x12\x16\n\x12\x46\x45\x41TURE_TYPE_FLOAT\x10\x03\x12\x17\n\x13\x46\x45\x41TURE_TYPE_STRING\x10\x04\x12\x15\n\x11\x46\x45\x41TURE_TYPE_JSON\x10\x05\x12\x16\n\x12\x46\x45\x41TURE_TYPE_PROTO\x10\x06\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.feature.v1beta1.feature_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _CONSTRAINT.fields_by_name['rule_ast']._options = None + _CONSTRAINT.fields_by_name['rule_ast']._serialized_options = b'\030\001' + _globals['_FEATURETYPE']._serialized_start=892 + _globals['_FEATURETYPE']._serialized_end=1076 + _globals['_FEATURE']._serialized_start=156 + _globals['_FEATURE']._serialized_end=322 + _globals['_TREE']._serialized_start=325 + _globals['_TREE']._serialized_end=509 + _globals['_CONSTRAINT']._serialized_start=512 + _globals['_CONSTRAINT']._serialized_end=833 + _globals['_ANY']._serialized_start=835 + _globals['_ANY']._serialized_end=889 +# @@protoc_insertion_point(module_scope) diff --git a/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.pyi b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.pyi new file mode 100644 index 0000000..e0658de --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2.pyi @@ -0,0 +1,199 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Lekko Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.any_pb2 +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import lekko_client.gen.lekko.rules.v1beta2.rules_pb2 +import lekko_client.gen.lekko.rules.v1beta3.rules_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _FeatureType: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _FeatureTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_FeatureType.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + FEATURE_TYPE_UNSPECIFIED: _FeatureType.ValueType # 0 + FEATURE_TYPE_BOOL: _FeatureType.ValueType # 1 + FEATURE_TYPE_INT: _FeatureType.ValueType # 2 + FEATURE_TYPE_FLOAT: _FeatureType.ValueType # 3 + FEATURE_TYPE_STRING: _FeatureType.ValueType # 4 + FEATURE_TYPE_JSON: _FeatureType.ValueType # 5 + FEATURE_TYPE_PROTO: _FeatureType.ValueType # 6 + +class FeatureType(_FeatureType, metaclass=_FeatureTypeEnumTypeWrapper): + """Enumerates the canonical types that lekko supports""" + +FEATURE_TYPE_UNSPECIFIED: FeatureType.ValueType # 0 +FEATURE_TYPE_BOOL: FeatureType.ValueType # 1 +FEATURE_TYPE_INT: FeatureType.ValueType # 2 +FEATURE_TYPE_FLOAT: FeatureType.ValueType # 3 +FEATURE_TYPE_STRING: FeatureType.ValueType # 4 +FEATURE_TYPE_JSON: FeatureType.ValueType # 5 +FEATURE_TYPE_PROTO: FeatureType.ValueType # 6 +global___FeatureType = FeatureType + +@typing_extensions.final +class Feature(google.protobuf.message.Message): + """A prototype of the wrapper type that will be used to house all feature flags for + the 'homegrown' feature flagging solution: + User-defined proto defintions, and a tree-based constraints system. + A real-life example of this in practice is visualized here: + https://lucid.app/lucidchart/f735298f-db2c-4207-8d14-28b375a25871/edit?view_items=bV8G0U69AJNc&invitationId=inv_d057a3b1-21d6-4290-9aea-5eb1c556a8ef# + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + TREE_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + key: builtins.str + description: builtins.str + @property + def tree(self) -> global___Tree: ... + type: global___FeatureType.ValueType + def __init__( + self, + *, + key: builtins.str = ..., + description: builtins.str = ..., + tree: global___Tree | None = ..., + type: global___FeatureType.ValueType = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["tree", b"tree"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "key", b"key", "tree", b"tree", "type", b"type"]) -> None: ... + +global___Feature = Feature + +@typing_extensions.final +class Tree(google.protobuf.message.Message): + """When the rules evaluator is traversing the tree, it will keep a local variable + 'value' that is updated along the way and is finally returned. It is initially + set to the default value of the root node. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DEFAULT_FIELD_NUMBER: builtins.int + CONSTRAINTS_FIELD_NUMBER: builtins.int + DEFAULT_NEW_FIELD_NUMBER: builtins.int + @property + def default(self) -> google.protobuf.any_pb2.Any: + """The default value to fall back to. If there are no constraints/rules + defined, this is what gets returned. + """ + @property + def constraints(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Constraint]: ... + @property + def default_new(self) -> global___Any: ... + def __init__( + self, + *, + default: google.protobuf.any_pb2.Any | None = ..., + constraints: collections.abc.Iterable[global___Constraint] | None = ..., + default_new: global___Any | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["default", b"default", "default_new", b"default_new"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["constraints", b"constraints", "default", b"default", "default_new", b"default_new"]) -> None: ... + +global___Tree = Tree + +@typing_extensions.final +class Constraint(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RULE_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + CONSTRAINTS_FIELD_NUMBER: builtins.int + RULE_AST_FIELD_NUMBER: builtins.int + RULE_AST_NEW_FIELD_NUMBER: builtins.int + VALUE_NEW_FIELD_NUMBER: builtins.int + rule: builtins.str + """RulesLang string. Purely for readability. All edits to ruleslang + are made through rule_ast_new instead. + """ + @property + def value(self) -> google.protobuf.any_pb2.Any: + """This can be empty. If non-empty, and the above rule evaluated to true, + then the rules engine should set its return value to this value. + """ + @property + def constraints(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Constraint]: + """If this list is empty, or none of the rules pass, + return the most recent concrete value we traversed. + """ + @property + def rule_ast(self) -> lekko_client.gen.lekko.rules.v1beta2.rules_pb2.Rule: + """Rules AST used for rules evaluation. It is a strict derivative of the + string rule above. + Deprecated: use rule_ast_new instead. + """ + @property + def rule_ast_new(self) -> lekko_client.gen.lekko.rules.v1beta3.rules_pb2.Rule: + """Rules AST used for rules evaluation. It is an n-ary tree.""" + @property + def value_new(self) -> global___Any: ... + def __init__( + self, + *, + rule: builtins.str = ..., + value: google.protobuf.any_pb2.Any | None = ..., + constraints: collections.abc.Iterable[global___Constraint] | None = ..., + rule_ast: lekko_client.gen.lekko.rules.v1beta2.rules_pb2.Rule | None = ..., + rule_ast_new: lekko_client.gen.lekko.rules.v1beta3.rules_pb2.Rule | None = ..., + value_new: global___Any | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["rule_ast", b"rule_ast", "rule_ast_new", b"rule_ast_new", "value", b"value", "value_new", b"value_new"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["constraints", b"constraints", "rule", b"rule", "rule_ast", b"rule_ast", "rule_ast_new", b"rule_ast_new", "value", b"value", "value_new", b"value_new"]) -> None: ... + +global___Constraint = Constraint + +@typing_extensions.final +class Any(google.protobuf.message.Message): + """New custom any type which allows us to manage dynamic types and values + ourselves in application code. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TYPE_URL_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + type_url: builtins.str + value: builtins.bytes + def __init__( + self, + *, + type_url: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["type_url", b"type_url", "value", b"value"]) -> None: ... + +global___Any = Any diff --git a/lekko_client/gen/lekko/feature/v1beta1/feature_pb2_grpc.py b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2_grpc.py new file mode 100644 index 0000000..2daafff --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/feature_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/lekko_client/gen/lekko/feature/v1beta1/static_pb2.py b/lekko_client/gen/lekko/feature/v1beta1/static_pb2.py new file mode 100644 index 0000000..c73b4cb --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/static_pb2.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lekko/feature/v1beta1/static.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from lekko_client.gen.lekko.feature.v1beta1 import feature_pb2 as lekko_dot_feature_dot_v1beta1_dot_feature__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\"lekko/feature/v1beta1/static.proto\x12\x15lekko.feature.v1beta1\x1a#lekko/feature/v1beta1/feature.proto\"\x9c\x02\n\rStaticFeature\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x36\n\x04type\x18\x02 \x01(\x0e\x32\".lekko.feature.v1beta1.FeatureTypeR\x04type\x12@\n\x07imports\x18\x03 \x03(\x0b\x32&.lekko.feature.v1beta1.ImportStatementR\x07imports\x12>\n\x07\x66\x65\x61ture\x18\x04 \x01(\x0b\x32$.lekko.feature.v1beta1.FeatureStructR\x07\x66\x65\x61ture\x12?\n\x0b\x66\x65\x61ture_old\x18\x05 \x01(\x0b\x32\x1e.lekko.feature.v1beta1.FeatureR\nfeatureOld\"\xd5\x01\n\rFeatureStruct\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12\x39\n\x07\x64\x65\x66\x61ult\x18\x03 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarExprR\x07\x64\x65\x66\x61ult\x12\x32\n\x05rules\x18\x04 \x01(\x0b\x32\x1c.lekko.feature.v1beta1.RulesR\x05rules\"o\n\x05Rules\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x31\n\x05rules\x18\x02 \x03(\x0b\x32\x1b.lekko.feature.v1beta1.RuleR\x05rules\"\x90\x01\n\x04Rule\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x1c\n\tcondition\x18\x02 \x01(\tR\tcondition\x12\x35\n\x05value\x18\x03 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarExprR\x05value\"\xea\x01\n\x0fImportStatement\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x32\n\x03lhs\x18\x02 \x01(\x0b\x32 .lekko.feature.v1beta1.IdentExprR\x03lhs\x12\x1a\n\x08operator\x18\x03 \x01(\tR\x08operator\x12\x1d\n\nline_break\x18\x04 \x01(\x08R\tlineBreak\x12\x33\n\x03rhs\x18\x05 \x01(\x0b\x32!.lekko.feature.v1beta1.ImportExprR\x03rhs\"\x87\x01\n\nImportExpr\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x30\n\x03\x64ot\x18\x02 \x01(\x0b\x32\x1e.lekko.feature.v1beta1.DotExprR\x03\x64ot\x12\x12\n\x04\x61rgs\x18\x03 \x03(\tR\x04\x61rgs\"`\n\x07\x44otExpr\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x0c\n\x01x\x18\x02 \x01(\tR\x01x\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\"_\n\x08StarExpr\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x1e\n\nexpression\x18\x02 \x01(\tR\nexpression\"V\n\tIdentExpr\x12\x33\n\x04meta\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.StarMetaR\x04meta\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"e\n\x08StarMeta\x12;\n\x08\x63omments\x18\x01 \x01(\x0b\x32\x1f.lekko.feature.v1beta1.CommentsR\x08\x63omments\x12\x1c\n\tmultiline\x18\x02 \x01(\x08R\tmultiline\"\xb0\x01\n\x08\x43omments\x12\x36\n\x06\x62\x65\x66ore\x18\x01 \x03(\x0b\x32\x1e.lekko.feature.v1beta1.CommentR\x06\x62\x65\x66ore\x12\x36\n\x06suffix\x18\x02 \x03(\x0b\x32\x1e.lekko.feature.v1beta1.CommentR\x06suffix\x12\x34\n\x05\x61\x66ter\x18\x03 \x03(\x0b\x32\x1e.lekko.feature.v1beta1.CommentR\x05\x61\x66ter\"\x1f\n\x07\x43omment\x12\x14\n\x05token\x18\x01 \x01(\tR\x05tokenb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.feature.v1beta1.static_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_STATICFEATURE']._serialized_start=99 + _globals['_STATICFEATURE']._serialized_end=383 + _globals['_FEATURESTRUCT']._serialized_start=386 + _globals['_FEATURESTRUCT']._serialized_end=599 + _globals['_RULES']._serialized_start=601 + _globals['_RULES']._serialized_end=712 + _globals['_RULE']._serialized_start=715 + _globals['_RULE']._serialized_end=859 + _globals['_IMPORTSTATEMENT']._serialized_start=862 + _globals['_IMPORTSTATEMENT']._serialized_end=1096 + _globals['_IMPORTEXPR']._serialized_start=1099 + _globals['_IMPORTEXPR']._serialized_end=1234 + _globals['_DOTEXPR']._serialized_start=1236 + _globals['_DOTEXPR']._serialized_end=1332 + _globals['_STAREXPR']._serialized_start=1334 + _globals['_STAREXPR']._serialized_end=1429 + _globals['_IDENTEXPR']._serialized_start=1431 + _globals['_IDENTEXPR']._serialized_end=1517 + _globals['_STARMETA']._serialized_start=1519 + _globals['_STARMETA']._serialized_end=1620 + _globals['_COMMENTS']._serialized_start=1623 + _globals['_COMMENTS']._serialized_end=1799 + _globals['_COMMENT']._serialized_start=1801 + _globals['_COMMENT']._serialized_end=1832 +# @@protoc_insertion_point(module_scope) diff --git a/lekko_client/gen/lekko/feature/v1beta1/static_pb2.pyi b/lekko_client/gen/lekko/feature/v1beta1/static_pb2.pyi new file mode 100644 index 0000000..0ffd883 --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/static_pb2.pyi @@ -0,0 +1,351 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Lekko Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import lekko_client.gen.lekko.feature.v1beta1.feature_pb2 +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class StaticFeature(google.protobuf.message.Message): + """Represents a statically parsed feature. + This model is meant to contain all sorts of data that is available + to us via inspecting the starlark file, but may not be available + post-compilation. E.g. what import statements were defined, and what + comments were written surrounding certain expressions. + The goal is to add to this model any information we wish to surface + to the UI or that is needed for static mutation of a feature. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + IMPORTS_FIELD_NUMBER: builtins.int + FEATURE_FIELD_NUMBER: builtins.int + FEATURE_OLD_FIELD_NUMBER: builtins.int + key: builtins.str + type: lekko_client.gen.lekko.feature.v1beta1.feature_pb2.FeatureType.ValueType + @property + def imports(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ImportStatement]: ... + @property + def feature(self) -> global___FeatureStruct: + """Experimental protobuf message that is a representation + of the statically parsed starlark AST. + """ + @property + def feature_old(self) -> lekko_client.gen.lekko.feature.v1beta1.feature_pb2.Feature: + """For backwards compatibility. Eventually, the materialized + model will be deprecated in favor of a statically parsed model + """ + def __init__( + self, + *, + key: builtins.str = ..., + type: lekko_client.gen.lekko.feature.v1beta1.feature_pb2.FeatureType.ValueType = ..., + imports: collections.abc.Iterable[global___ImportStatement] | None = ..., + feature: global___FeatureStruct | None = ..., + feature_old: lekko_client.gen.lekko.feature.v1beta1.feature_pb2.Feature | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["feature", b"feature", "feature_old", b"feature_old"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["feature", b"feature", "feature_old", b"feature_old", "imports", b"imports", "key", b"key", "type", b"type"]) -> None: ... + +global___StaticFeature = StaticFeature + +@typing_extensions.final +class FeatureStruct(google.protobuf.message.Message): + """Represents everything stored in the feature struct in starlark. + i.e. `feature(...)` + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + DEFAULT_FIELD_NUMBER: builtins.int + RULES_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + description: builtins.str + @property + def default(self) -> global___StarExpr: ... + @property + def rules(self) -> global___Rules: ... + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + description: builtins.str = ..., + default: global___StarExpr | None = ..., + rules: global___Rules | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["default", b"default", "meta", b"meta", "rules", b"rules"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["default", b"default", "description", b"description", "meta", b"meta", "rules", b"rules"]) -> None: ... + +global___FeatureStruct = FeatureStruct + +@typing_extensions.final +class Rules(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + RULES_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + @property + def rules(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Rule]: ... + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + rules: collections.abc.Iterable[global___Rule] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["meta", b"meta"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["meta", b"meta", "rules", b"rules"]) -> None: ... + +global___Rules = Rules + +@typing_extensions.final +class Rule(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + CONDITION_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + condition: builtins.str + @property + def value(self) -> global___StarExpr: ... + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + condition: builtins.str = ..., + value: global___StarExpr | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["meta", b"meta", "value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["condition", b"condition", "meta", b"meta", "value", b"value"]) -> None: ... + +global___Rule = Rule + +@typing_extensions.final +class ImportStatement(google.protobuf.message.Message): + """An assignment expression in starlark, e.g. `x = 1`.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + LHS_FIELD_NUMBER: builtins.int + OPERATOR_FIELD_NUMBER: builtins.int + LINE_BREAK_FIELD_NUMBER: builtins.int + RHS_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + @property + def lhs(self) -> global___IdentExpr: + """the variable that the import is assigned to""" + operator: builtins.str + """e.g. `=`""" + line_break: builtins.bool + """Line break between operator and RHS""" + @property + def rhs(self) -> global___ImportExpr: ... + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + lhs: global___IdentExpr | None = ..., + operator: builtins.str = ..., + line_break: builtins.bool = ..., + rhs: global___ImportExpr | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["lhs", b"lhs", "meta", b"meta", "rhs", b"rhs"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["lhs", b"lhs", "line_break", b"line_break", "meta", b"meta", "operator", b"operator", "rhs", b"rhs"]) -> None: ... + +global___ImportStatement = ImportStatement + +@typing_extensions.final +class ImportExpr(google.protobuf.message.Message): + """an import expression, e.g. `proto.package("google.protobuf")`""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + DOT_FIELD_NUMBER: builtins.int + ARGS_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + @property + def dot(self) -> global___DotExpr: ... + @property + def args(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + dot: global___DotExpr | None = ..., + args: collections.abc.Iterable[builtins.str] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["dot", b"dot", "meta", b"meta"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["args", b"args", "dot", b"dot", "meta", b"meta"]) -> None: ... + +global___ImportExpr = ImportExpr + +@typing_extensions.final +class DotExpr(google.protobuf.message.Message): + """a dot expression, e.g. `proto.package`""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + X_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + x: builtins.str + """The token that appears before the dot in the dot expression, e.g. `proto`""" + name: builtins.str + """The token that appears after the dot in the dot expression, e.g. `package`""" + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + x: builtins.str = ..., + name: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["meta", b"meta"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["meta", b"meta", "name", b"name", "x", b"x"]) -> None: ... + +global___DotExpr = DotExpr + +@typing_extensions.final +class StarExpr(google.protobuf.message.Message): + """A single starlark expression. May not be fully decomposed. + Stores the string representation of the expression in the metadata. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + EXPRESSION_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + expression: builtins.str + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + expression: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["meta", b"meta"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["expression", b"expression", "meta", b"meta"]) -> None: ... + +global___StarExpr = StarExpr + +@typing_extensions.final +class IdentExpr(google.protobuf.message.Message): + """A single token in starlark. E.g. True, False, or a variable name.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + META_FIELD_NUMBER: builtins.int + TOKEN_FIELD_NUMBER: builtins.int + @property + def meta(self) -> global___StarMeta: ... + token: builtins.str + def __init__( + self, + *, + meta: global___StarMeta | None = ..., + token: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["meta", b"meta"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["meta", b"meta", "token", b"token"]) -> None: ... + +global___IdentExpr = IdentExpr + +@typing_extensions.final +class StarMeta(google.protobuf.message.Message): + """Metadata commonly associated with any starlark expression""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + COMMENTS_FIELD_NUMBER: builtins.int + MULTILINE_FIELD_NUMBER: builtins.int + @property + def comments(self) -> global___Comments: ... + multiline: builtins.bool + def __init__( + self, + *, + comments: global___Comments | None = ..., + multiline: builtins.bool = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["comments", b"comments"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["comments", b"comments", "multiline", b"multiline"]) -> None: ... + +global___StarMeta = StarMeta + +@typing_extensions.final +class Comments(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BEFORE_FIELD_NUMBER: builtins.int + SUFFIX_FIELD_NUMBER: builtins.int + AFTER_FIELD_NUMBER: builtins.int + @property + def before(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Comment]: ... + @property + def suffix(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Comment]: ... + @property + def after(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Comment]: ... + def __init__( + self, + *, + before: collections.abc.Iterable[global___Comment] | None = ..., + suffix: collections.abc.Iterable[global___Comment] | None = ..., + after: collections.abc.Iterable[global___Comment] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["after", b"after", "before", b"before", "suffix", b"suffix"]) -> None: ... + +global___Comments = Comments + +@typing_extensions.final +class Comment(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TOKEN_FIELD_NUMBER: builtins.int + token: builtins.str + def __init__( + self, + *, + token: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["token", b"token"]) -> None: ... + +global___Comment = Comment diff --git a/lekko_client/gen/lekko/feature/v1beta1/static_pb2_grpc.py b/lekko_client/gen/lekko/feature/v1beta1/static_pb2_grpc.py new file mode 100644 index 0000000..2daafff --- /dev/null +++ b/lekko_client/gen/lekko/feature/v1beta1/static_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.py b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.py new file mode 100644 index 0000000..a1dcad0 --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lekko/rules/v1beta2/rules.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1flekko/rules/v1beta2/rules.proto\x12\x13lekko.rules.v1beta2\x1a\x1cgoogle/protobuf/struct.proto\"\xec\x01\n\x04Rule\x12/\n\x04\x61tom\x18\x01 \x01(\x0b\x32\x19.lekko.rules.v1beta2.AtomH\x00R\x04\x61tom\x12-\n\x03not\x18\x02 \x01(\x0b\x32\x19.lekko.rules.v1beta2.RuleH\x00R\x03not\x12W\n\x12logical_expression\x18\x03 \x01(\x0b\x32&.lekko.rules.v1beta2.LogicalExpressionH\x00R\x11logicalExpression\x12\x1f\n\nbool_const\x18\x04 \x01(\x08H\x00R\tboolConst:\x02\x18\x01\x42\x06\n\x04rule\"\xde\x01\n\x11LogicalExpression\x12\x38\n\nfirst_rule\x18\x01 \x01(\x0b\x32\x19.lekko.rules.v1beta2.RuleR\tfirstRule\x12:\n\x0bsecond_rule\x18\x02 \x01(\x0b\x32\x19.lekko.rules.v1beta2.RuleR\nsecondRule\x12O\n\x10logical_operator\x18\x03 \x01(\x0e\x32$.lekko.rules.v1beta2.LogicalOperatorR\x0flogicalOperator:\x02\x18\x01\"\xc8\x01\n\x04\x41tom\x12\x1f\n\x0b\x63ontext_key\x18\x01 \x01(\tR\ncontextKey\x12\x41\n\x10\x63omparison_value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.ValueR\x0f\x63omparisonValue\x12X\n\x13\x63omparison_operator\x18\x03 \x01(\x0e\x32\'.lekko.rules.v1beta2.ComparisonOperatorR\x12\x63omparisonOperator:\x02\x18\x01*\xb8\x03\n\x12\x43omparisonOperator\x12#\n\x1f\x43OMPARISON_OPERATOR_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x43OMPARISON_OPERATOR_EQUALS\x10\x01\x12!\n\x1d\x43OMPARISON_OPERATOR_LESS_THAN\x10\x02\x12+\n\'COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS\x10\x03\x12$\n COMPARISON_OPERATOR_GREATER_THAN\x10\x04\x12.\n*COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS\x10\x05\x12(\n$COMPARISON_OPERATOR_CONTAINED_WITHIN\x10\x06\x12#\n\x1f\x43OMPARISON_OPERATOR_STARTS_WITH\x10\x07\x12!\n\x1d\x43OMPARISON_OPERATOR_ENDS_WITH\x10\x08\x12 \n\x1c\x43OMPARISON_OPERATOR_CONTAINS\x10\t\x12\x1f\n\x1b\x43OMPARISON_OPERATOR_PRESENT\x10\n\x1a\x02\x18\x01*j\n\x0fLogicalOperator\x12 \n\x1cLOGICAL_OPERATOR_UNSPECIFIED\x10\x00\x12\x18\n\x14LOGICAL_OPERATOR_AND\x10\x01\x12\x17\n\x13LOGICAL_OPERATOR_OR\x10\x02\x1a\x02\x18\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.rules.v1beta2.rules_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _COMPARISONOPERATOR._options = None + _COMPARISONOPERATOR._serialized_options = b'\030\001' + _LOGICALOPERATOR._options = None + _LOGICALOPERATOR._serialized_options = b'\030\001' + _RULE._options = None + _RULE._serialized_options = b'\030\001' + _LOGICALEXPRESSION._options = None + _LOGICALEXPRESSION._serialized_options = b'\030\001' + _ATOM._options = None + _ATOM._serialized_options = b'\030\001' + _globals['_COMPARISONOPERATOR']._serialized_start=754 + _globals['_COMPARISONOPERATOR']._serialized_end=1194 + _globals['_LOGICALOPERATOR']._serialized_start=1196 + _globals['_LOGICALOPERATOR']._serialized_end=1302 + _globals['_RULE']._serialized_start=87 + _globals['_RULE']._serialized_end=323 + _globals['_LOGICALEXPRESSION']._serialized_start=326 + _globals['_LOGICALEXPRESSION']._serialized_end=548 + _globals['_ATOM']._serialized_start=551 + _globals['_ATOM']._serialized_end=751 +# @@protoc_insertion_point(module_scope) diff --git a/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.pyi b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.pyi new file mode 100644 index 0000000..251abbb --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2.pyi @@ -0,0 +1,199 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Lekko Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.struct_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _ComparisonOperator: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ComparisonOperatorEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ComparisonOperator.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + COMPARISON_OPERATOR_UNSPECIFIED: _ComparisonOperator.ValueType # 0 + COMPARISON_OPERATOR_EQUALS: _ComparisonOperator.ValueType # 1 + """== only applies to number, string and bool values.""" + COMPARISON_OPERATOR_LESS_THAN: _ComparisonOperator.ValueType # 2 + """> < >= <= only applies to number values.""" + COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS: _ComparisonOperator.ValueType # 3 + COMPARISON_OPERATOR_GREATER_THAN: _ComparisonOperator.ValueType # 4 + COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: _ComparisonOperator.ValueType # 5 + COMPARISON_OPERATOR_CONTAINED_WITHIN: _ComparisonOperator.ValueType # 6 + """Contained within only applies to list values. Elements + of the list must be primitive (i.e. number, string or bool) + """ + COMPARISON_OPERATOR_STARTS_WITH: _ComparisonOperator.ValueType # 7 + """Starts with and ends with only apply to string values.""" + COMPARISON_OPERATOR_ENDS_WITH: _ComparisonOperator.ValueType # 8 + COMPARISON_OPERATOR_CONTAINS: _ComparisonOperator.ValueType # 9 + """Contains only applies to string values, and for now is strict equality. + If we support things like regex or case insensitive matches, they will + be separate operators. + """ + COMPARISON_OPERATOR_PRESENT: _ComparisonOperator.ValueType # 10 + """Present is the only operator that doesn't require a comparison value.""" + +class ComparisonOperator(_ComparisonOperator, metaclass=_ComparisonOperatorEnumTypeWrapper): ... + +COMPARISON_OPERATOR_UNSPECIFIED: ComparisonOperator.ValueType # 0 +COMPARISON_OPERATOR_EQUALS: ComparisonOperator.ValueType # 1 +"""== only applies to number, string and bool values.""" +COMPARISON_OPERATOR_LESS_THAN: ComparisonOperator.ValueType # 2 +"""> < >= <= only applies to number values.""" +COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS: ComparisonOperator.ValueType # 3 +COMPARISON_OPERATOR_GREATER_THAN: ComparisonOperator.ValueType # 4 +COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: ComparisonOperator.ValueType # 5 +COMPARISON_OPERATOR_CONTAINED_WITHIN: ComparisonOperator.ValueType # 6 +"""Contained within only applies to list values. Elements +of the list must be primitive (i.e. number, string or bool) +""" +COMPARISON_OPERATOR_STARTS_WITH: ComparisonOperator.ValueType # 7 +"""Starts with and ends with only apply to string values.""" +COMPARISON_OPERATOR_ENDS_WITH: ComparisonOperator.ValueType # 8 +COMPARISON_OPERATOR_CONTAINS: ComparisonOperator.ValueType # 9 +"""Contains only applies to string values, and for now is strict equality. +If we support things like regex or case insensitive matches, they will +be separate operators. +""" +COMPARISON_OPERATOR_PRESENT: ComparisonOperator.ValueType # 10 +"""Present is the only operator that doesn't require a comparison value.""" +global___ComparisonOperator = ComparisonOperator + +class _LogicalOperator: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _LogicalOperatorEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_LogicalOperator.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + LOGICAL_OPERATOR_UNSPECIFIED: _LogicalOperator.ValueType # 0 + LOGICAL_OPERATOR_AND: _LogicalOperator.ValueType # 1 + LOGICAL_OPERATOR_OR: _LogicalOperator.ValueType # 2 + +class LogicalOperator(_LogicalOperator, metaclass=_LogicalOperatorEnumTypeWrapper): ... + +LOGICAL_OPERATOR_UNSPECIFIED: LogicalOperator.ValueType # 0 +LOGICAL_OPERATOR_AND: LogicalOperator.ValueType # 1 +LOGICAL_OPERATOR_OR: LogicalOperator.ValueType # 2 +global___LogicalOperator = LogicalOperator + +@typing_extensions.final +class Rule(google.protobuf.message.Message): + """A Rule is a top level object that recursively defines an AST represented + by ruleslang. A rule is always one of 4 things: + 1. Atom -> This is a leaf node in the tree that returns true or false + 2. Not -> This negates the result of the underlying Rule. + 3. LogicalExpression -> This rule links two rules through an "and" or an "or". + 4. BoolConst -> true or false. This will be used for higher level short-circuits. + Parenthases and other logical constructs can all be represented by the correct + construction of this rule tree. + + !(A && B && C) || D can be represented by LogExp ( Not ( LogExp ( LogExp ( Atom(A) && Atom(B) ) && Atom (C))) || Atom(D)) + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ATOM_FIELD_NUMBER: builtins.int + NOT_FIELD_NUMBER: builtins.int + LOGICAL_EXPRESSION_FIELD_NUMBER: builtins.int + BOOL_CONST_FIELD_NUMBER: builtins.int + @property + def atom(self) -> global___Atom: ... + @property + def logical_expression(self) -> global___LogicalExpression: ... + bool_const: builtins.bool + def __init__( + self, + *, + atom: global___Atom | None = ..., + logical_expression: global___LogicalExpression | None = ..., + bool_const: builtins.bool = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["atom", b"atom", "bool_const", b"bool_const", "logical_expression", b"logical_expression", "not", b"not", "rule", b"rule"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["atom", b"atom", "bool_const", b"bool_const", "logical_expression", b"logical_expression", "not", b"not", "rule", b"rule"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["rule", b"rule"]) -> typing_extensions.Literal["atom", "not", "logical_expression", "bool_const"] | None: ... + +global___Rule = Rule + +@typing_extensions.final +class LogicalExpression(google.protobuf.message.Message): + """LogicalExpression operator applies a logical operator like "and" or "or" to two rules. + They are evaluated in the order expressed by the field numbers and field names, with "first_rule" first. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_RULE_FIELD_NUMBER: builtins.int + SECOND_RULE_FIELD_NUMBER: builtins.int + LOGICAL_OPERATOR_FIELD_NUMBER: builtins.int + @property + def first_rule(self) -> global___Rule: ... + @property + def second_rule(self) -> global___Rule: ... + logical_operator: global___LogicalOperator.ValueType + def __init__( + self, + *, + first_rule: global___Rule | None = ..., + second_rule: global___Rule | None = ..., + logical_operator: global___LogicalOperator.ValueType = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["first_rule", b"first_rule", "second_rule", b"second_rule"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["first_rule", b"first_rule", "logical_operator", b"logical_operator", "second_rule", b"second_rule"]) -> None: ... + +global___LogicalExpression = LogicalExpression + +@typing_extensions.final +class Atom(google.protobuf.message.Message): + """An atom is a fragment of ruleslang that can result in a true or false. + An atom always has a comparison operator and a context key, and can optionally + have a comparison value. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONTEXT_KEY_FIELD_NUMBER: builtins.int + COMPARISON_VALUE_FIELD_NUMBER: builtins.int + COMPARISON_OPERATOR_FIELD_NUMBER: builtins.int + context_key: builtins.str + @property + def comparison_value(self) -> google.protobuf.struct_pb2.Value: + """For the "PRESENT" operator, the comparison value should be null.""" + comparison_operator: global___ComparisonOperator.ValueType + """For operators, context is on the left, comparison value on the right.""" + def __init__( + self, + *, + context_key: builtins.str = ..., + comparison_value: google.protobuf.struct_pb2.Value | None = ..., + comparison_operator: global___ComparisonOperator.ValueType = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["comparison_value", b"comparison_value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["comparison_operator", b"comparison_operator", "comparison_value", b"comparison_value", "context_key", b"context_key"]) -> None: ... + +global___Atom = Atom diff --git a/lekko_client/gen/lekko/rules/v1beta2/rules_pb2_grpc.py b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2_grpc.py new file mode 100644 index 0000000..2daafff --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta2/rules_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.py b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.py new file mode 100644 index 0000000..5b306eb --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lekko/rules/v1beta3/rules.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1flekko/rules/v1beta3/rules.proto\x12\x13lekko.rules.v1beta3\x1a\x1cgoogle/protobuf/struct.proto\"\xb8\x02\n\x04Rule\x12/\n\x04\x61tom\x18\x01 \x01(\x0b\x32\x19.lekko.rules.v1beta3.AtomH\x00R\x04\x61tom\x12-\n\x03not\x18\x02 \x01(\x0b\x32\x19.lekko.rules.v1beta3.RuleH\x00R\x03not\x12W\n\x12logical_expression\x18\x03 \x01(\x0b\x32&.lekko.rules.v1beta3.LogicalExpressionH\x00R\x11logicalExpression\x12\x1f\n\nbool_const\x18\x04 \x01(\x08H\x00R\tboolConst\x12N\n\x0f\x63\x61ll_expression\x18\x05 \x01(\x0b\x32#.lekko.rules.v1beta3.CallExpressionH\x00R\x0e\x63\x61llExpressionB\x06\n\x04rule\"\x95\x01\n\x11LogicalExpression\x12/\n\x05rules\x18\x01 \x03(\x0b\x32\x19.lekko.rules.v1beta3.RuleR\x05rules\x12O\n\x10logical_operator\x18\x03 \x01(\x0e\x32$.lekko.rules.v1beta3.LogicalOperatorR\x0flogicalOperator\"\xc4\x01\n\x04\x41tom\x12\x1f\n\x0b\x63ontext_key\x18\x01 \x01(\tR\ncontextKey\x12\x41\n\x10\x63omparison_value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.ValueR\x0f\x63omparisonValue\x12X\n\x13\x63omparison_operator\x18\x03 \x01(\x0e\x32\'.lekko.rules.v1beta3.ComparisonOperatorR\x12\x63omparisonOperator\"\xab\x01\n\x0e\x43\x61llExpression\x12\x44\n\x06\x62ucket\x18\x01 \x01(\x0b\x32*.lekko.rules.v1beta3.CallExpression.BucketH\x00R\x06\x62ucket\x1aG\n\x06\x42ucket\x12\x1f\n\x0b\x63ontext_key\x18\x01 \x01(\tR\ncontextKey\x12\x1c\n\tthreshold\x18\x02 \x01(\rR\tthresholdB\n\n\x08\x66unction*\xd8\x03\n\x12\x43omparisonOperator\x12#\n\x1f\x43OMPARISON_OPERATOR_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x43OMPARISON_OPERATOR_EQUALS\x10\x01\x12!\n\x1d\x43OMPARISON_OPERATOR_LESS_THAN\x10\x02\x12+\n\'COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS\x10\x03\x12$\n COMPARISON_OPERATOR_GREATER_THAN\x10\x04\x12.\n*COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS\x10\x05\x12(\n$COMPARISON_OPERATOR_CONTAINED_WITHIN\x10\x06\x12#\n\x1f\x43OMPARISON_OPERATOR_STARTS_WITH\x10\x07\x12!\n\x1d\x43OMPARISON_OPERATOR_ENDS_WITH\x10\x08\x12 \n\x1c\x43OMPARISON_OPERATOR_CONTAINS\x10\t\x12\x1f\n\x1b\x43OMPARISON_OPERATOR_PRESENT\x10\n\x12\"\n\x1e\x43OMPARISON_OPERATOR_NOT_EQUALS\x10\x0b*f\n\x0fLogicalOperator\x12 \n\x1cLOGICAL_OPERATOR_UNSPECIFIED\x10\x00\x12\x18\n\x14LOGICAL_OPERATOR_AND\x10\x01\x12\x17\n\x13LOGICAL_OPERATOR_OR\x10\x02\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.rules.v1beta3.rules_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_COMPARISONOPERATOR']._serialized_start=927 + _globals['_COMPARISONOPERATOR']._serialized_end=1399 + _globals['_LOGICALOPERATOR']._serialized_start=1401 + _globals['_LOGICALOPERATOR']._serialized_end=1503 + _globals['_RULE']._serialized_start=87 + _globals['_RULE']._serialized_end=399 + _globals['_LOGICALEXPRESSION']._serialized_start=402 + _globals['_LOGICALEXPRESSION']._serialized_end=551 + _globals['_ATOM']._serialized_start=554 + _globals['_ATOM']._serialized_end=750 + _globals['_CALLEXPRESSION']._serialized_start=753 + _globals['_CALLEXPRESSION']._serialized_end=924 + _globals['_CALLEXPRESSION_BUCKET']._serialized_start=841 + _globals['_CALLEXPRESSION_BUCKET']._serialized_end=912 +# @@protoc_insertion_point(module_scope) diff --git a/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.pyi b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.pyi new file mode 100644 index 0000000..35aac56 --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2.pyi @@ -0,0 +1,257 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Lekko Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.struct_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _ComparisonOperator: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ComparisonOperatorEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ComparisonOperator.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + COMPARISON_OPERATOR_UNSPECIFIED: _ComparisonOperator.ValueType # 0 + COMPARISON_OPERATOR_EQUALS: _ComparisonOperator.ValueType # 1 + """== only applies to number, string and bool values.""" + COMPARISON_OPERATOR_LESS_THAN: _ComparisonOperator.ValueType # 2 + """> < >= <= only applies to number values.""" + COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS: _ComparisonOperator.ValueType # 3 + COMPARISON_OPERATOR_GREATER_THAN: _ComparisonOperator.ValueType # 4 + COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: _ComparisonOperator.ValueType # 5 + COMPARISON_OPERATOR_CONTAINED_WITHIN: _ComparisonOperator.ValueType # 6 + """Contained within only applies to list values. Elements + of the list must be primitive (i.e. number, string or bool) + """ + COMPARISON_OPERATOR_STARTS_WITH: _ComparisonOperator.ValueType # 7 + """Starts with and ends with only apply to string values.""" + COMPARISON_OPERATOR_ENDS_WITH: _ComparisonOperator.ValueType # 8 + COMPARISON_OPERATOR_CONTAINS: _ComparisonOperator.ValueType # 9 + """Contains only applies to string values, and for now is strict equality. + If we support things like regex or case insensitive matches, they will + be separate operators. + """ + COMPARISON_OPERATOR_PRESENT: _ComparisonOperator.ValueType # 10 + """Present is the only operator that doesn't require a comparison value.""" + COMPARISON_OPERATOR_NOT_EQUALS: _ComparisonOperator.ValueType # 11 + """!= only applies to number, string and bool values.""" + +class ComparisonOperator(_ComparisonOperator, metaclass=_ComparisonOperatorEnumTypeWrapper): ... + +COMPARISON_OPERATOR_UNSPECIFIED: ComparisonOperator.ValueType # 0 +COMPARISON_OPERATOR_EQUALS: ComparisonOperator.ValueType # 1 +"""== only applies to number, string and bool values.""" +COMPARISON_OPERATOR_LESS_THAN: ComparisonOperator.ValueType # 2 +"""> < >= <= only applies to number values.""" +COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS: ComparisonOperator.ValueType # 3 +COMPARISON_OPERATOR_GREATER_THAN: ComparisonOperator.ValueType # 4 +COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: ComparisonOperator.ValueType # 5 +COMPARISON_OPERATOR_CONTAINED_WITHIN: ComparisonOperator.ValueType # 6 +"""Contained within only applies to list values. Elements +of the list must be primitive (i.e. number, string or bool) +""" +COMPARISON_OPERATOR_STARTS_WITH: ComparisonOperator.ValueType # 7 +"""Starts with and ends with only apply to string values.""" +COMPARISON_OPERATOR_ENDS_WITH: ComparisonOperator.ValueType # 8 +COMPARISON_OPERATOR_CONTAINS: ComparisonOperator.ValueType # 9 +"""Contains only applies to string values, and for now is strict equality. +If we support things like regex or case insensitive matches, they will +be separate operators. +""" +COMPARISON_OPERATOR_PRESENT: ComparisonOperator.ValueType # 10 +"""Present is the only operator that doesn't require a comparison value.""" +COMPARISON_OPERATOR_NOT_EQUALS: ComparisonOperator.ValueType # 11 +"""!= only applies to number, string and bool values.""" +global___ComparisonOperator = ComparisonOperator + +class _LogicalOperator: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _LogicalOperatorEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_LogicalOperator.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + LOGICAL_OPERATOR_UNSPECIFIED: _LogicalOperator.ValueType # 0 + LOGICAL_OPERATOR_AND: _LogicalOperator.ValueType # 1 + LOGICAL_OPERATOR_OR: _LogicalOperator.ValueType # 2 + +class LogicalOperator(_LogicalOperator, metaclass=_LogicalOperatorEnumTypeWrapper): ... + +LOGICAL_OPERATOR_UNSPECIFIED: LogicalOperator.ValueType # 0 +LOGICAL_OPERATOR_AND: LogicalOperator.ValueType # 1 +LOGICAL_OPERATOR_OR: LogicalOperator.ValueType # 2 +global___LogicalOperator = LogicalOperator + +@typing_extensions.final +class Rule(google.protobuf.message.Message): + """A Rule is a top level object that recursively defines an AST represented + by ruleslang. A rule is always one of 4 things: + 1. Atom -> This is a leaf node in the tree that returns true or false + 2. Not -> This negates the result of the underlying Rule. + 3. LogicalExpression -> This rule links at least two rules through an "and" or an "or". + 4. BoolConst -> true or false. This will be used for higher level short-circuits. + 5. CallExpression -> This rule is a function call that returns true or false. + Parentheses and other logical constructs can all be represented by the correct + construction of this rule tree. + + !(A && B && C) || D can be represented by LogExp ( Not ( LogExp ( Atom(A) && Atom(B) && Atom(C) )) || Atom(D)) + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ATOM_FIELD_NUMBER: builtins.int + NOT_FIELD_NUMBER: builtins.int + LOGICAL_EXPRESSION_FIELD_NUMBER: builtins.int + BOOL_CONST_FIELD_NUMBER: builtins.int + CALL_EXPRESSION_FIELD_NUMBER: builtins.int + @property + def atom(self) -> global___Atom: ... + @property + def logical_expression(self) -> global___LogicalExpression: ... + bool_const: builtins.bool + @property + def call_expression(self) -> global___CallExpression: ... + def __init__( + self, + *, + atom: global___Atom | None = ..., + logical_expression: global___LogicalExpression | None = ..., + bool_const: builtins.bool = ..., + call_expression: global___CallExpression | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["atom", b"atom", "bool_const", b"bool_const", "call_expression", b"call_expression", "logical_expression", b"logical_expression", "not", b"not", "rule", b"rule"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["atom", b"atom", "bool_const", b"bool_const", "call_expression", b"call_expression", "logical_expression", b"logical_expression", "not", b"not", "rule", b"rule"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["rule", b"rule"]) -> typing_extensions.Literal["atom", "not", "logical_expression", "bool_const", "call_expression"] | None: ... + +global___Rule = Rule + +@typing_extensions.final +class LogicalExpression(google.protobuf.message.Message): + """LogicalExpression operator applies a logical operator like "and" or "or" to n rules. + They are evaluated in the order expressed by the repeated field. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RULES_FIELD_NUMBER: builtins.int + LOGICAL_OPERATOR_FIELD_NUMBER: builtins.int + @property + def rules(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Rule]: ... + logical_operator: global___LogicalOperator.ValueType + def __init__( + self, + *, + rules: collections.abc.Iterable[global___Rule] | None = ..., + logical_operator: global___LogicalOperator.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["logical_operator", b"logical_operator", "rules", b"rules"]) -> None: ... + +global___LogicalExpression = LogicalExpression + +@typing_extensions.final +class Atom(google.protobuf.message.Message): + """An atom is a fragment of ruleslang that can result in a true or false. + An atom always has a comparison operator and a context key, and can optionally + have a comparison value. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONTEXT_KEY_FIELD_NUMBER: builtins.int + COMPARISON_VALUE_FIELD_NUMBER: builtins.int + COMPARISON_OPERATOR_FIELD_NUMBER: builtins.int + context_key: builtins.str + @property + def comparison_value(self) -> google.protobuf.struct_pb2.Value: + """For the "PRESENT" operator, the comparison value should be null.""" + comparison_operator: global___ComparisonOperator.ValueType + """For operators, context is on the left, comparison value on the right.""" + def __init__( + self, + *, + context_key: builtins.str = ..., + comparison_value: google.protobuf.struct_pb2.Value | None = ..., + comparison_operator: global___ComparisonOperator.ValueType = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["comparison_value", b"comparison_value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["comparison_operator", b"comparison_operator", "comparison_value", b"comparison_value", "context_key", b"context_key"]) -> None: ... + +global___Atom = Atom + +@typing_extensions.final +class CallExpression(google.protobuf.message.Message): + """CallExpression represents a function call, e.g. f(a, b, c). + Each function has a specific signature, so a CallExpression is + expressed as one of the different supported functions. + Example signature + message Example { + uint32 x = 1; + string y = 2; + } + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class Bucket(google.protobuf.message.Message): + """Bucketing function for percentage-based context evaluation""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONTEXT_KEY_FIELD_NUMBER: builtins.int + THRESHOLD_FIELD_NUMBER: builtins.int + context_key: builtins.str + threshold: builtins.int + """Threshold for dividing buckets. + Stored as an integer in the range [0, 100000] instead of a double + to avoid potential precision issues while supporting up to 3 + decimal places to users. + e.g. threshold = 75125 -> 75.125% + """ + def __init__( + self, + *, + context_key: builtins.str = ..., + threshold: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["context_key", b"context_key", "threshold", b"threshold"]) -> None: ... + + BUCKET_FIELD_NUMBER: builtins.int + @property + def bucket(self) -> global___CallExpression.Bucket: ... + def __init__( + self, + *, + bucket: global___CallExpression.Bucket | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["bucket", b"bucket", "function", b"function"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["bucket", b"bucket", "function", b"function"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["function", b"function"]) -> typing_extensions.Literal["bucket"] | None: ... + +global___CallExpression = CallExpression diff --git a/lekko_client/gen/lekko/rules/v1beta3/rules_pb2_grpc.py b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2_grpc.py new file mode 100644 index 0000000..2daafff --- /dev/null +++ b/lekko_client/gen/lekko/rules/v1beta3/rules_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/pyproject.toml b/pyproject.toml index 23316b6..d77a4ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ 'grpcio ~= 1.32', 'grpcio-tools ~= 1.32', 'grpc-interceptor ~= 0.15', + 'xxhash ~= 3.0', ] [project.optional-dependencies] From 708f253a7cd068f19e5db2cc04698864912726fe Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Fri, 1 Sep 2023 13:33:41 -0700 Subject: [PATCH 02/28] add tests --- lekko_client/evaluation/rules.py | 6 +-- tests/conftest.py | 11 ++++ tests/evaulation/test_evaluation.py | 31 ++++++++++++ tests/evaulation/test_rules.py | 76 ++++++++++++++++++++++++++++ tests/fixtures/rules.proto.bin | Bin 0 -> 2148 bytes 5 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/evaulation/test_evaluation.py create mode 100644 tests/evaulation/test_rules.py create mode 100644 tests/fixtures/rules.proto.bin diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py index acd78f1..881d563 100644 --- a/lekko_client/evaluation/rules.py +++ b/lekko_client/evaluation/rules.py @@ -1,5 +1,5 @@ import struct -from typing import Dict, Optional +from typing import Dict, Optional, Union from google.protobuf.struct_pb2 import Value from xxhash import xxh32 @@ -116,7 +116,7 @@ def evaluate_string_comparator( raise ValueError("unexpected string comparison operator") -def get_string(value: Value | LekkoValue) -> str: +def get_string(value: Union[Value, LekkoValue]) -> str: if not value: raise ValueError("value is undefined") @@ -144,7 +144,7 @@ def evaluate_number_comparator( raise ValueError("unexpected numerical comparison operator") -def get_number(value: Value | LekkoValue) -> float: +def get_number(value: Union[Value, LekkoValue]) -> float: value_kind = value.WhichOneof("kind") if value_kind in ["number_value", "int_value", "double_value"]: return float(getattr(value, value_kind)) diff --git a/tests/conftest.py b/tests/conftest.py index f29db1f..5a6a3ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ from lekko_client import helpers from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import DESCRIPTOR +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature @pytest.fixture @@ -98,3 +99,13 @@ def test_server(test_channel, test_thread): @pytest.fixture def test_server_no_interceptor(test_channel_no_interceptor, test_thread): return MockServer(test_channel_no_interceptor, test_thread) + + +@pytest.fixture +def test_complex_rule_feature(): + filename = "tests/fixtures/rules.proto.bin" + feature = Feature() + with open(filename, "rb") as f: + feature.ParseFromString(f.read()) + + return feature diff --git a/tests/evaulation/test_evaluation.py b/tests/evaulation/test_evaluation.py new file mode 100644 index 0000000..0e78444 --- /dev/null +++ b/tests/evaulation/test_evaluation.py @@ -0,0 +1,31 @@ +import pytest +from google.protobuf.wrappers_pb2 import Int64Value + +from lekko_client.evaluation.evaluation import evaluate +from lekko_client.helpers import convert_context + + +@pytest.mark.parametrize( + "context,expected", + [ + ({"a": 1}, 0), + ({"a": 1, "c": 3}, 1), + ({"c": 3}, 2), + ({"f": 3}, 0), + ({"f": 5}, 5), + ({"h": 4}, 7), + ({"i": 4}, 8), + ({"p": "a foo bar"}, 15), + ({"q": "hello world"}, 16), + ({"r": "a foo bar"}, 17), + ({"s": 2}, 18), + ({"t": "anything"}, 19), + ({"u": 11}, 20), + ], +) +def test_evaluation(test_complex_rule_feature, context, expected): + client_context = convert_context(context) + result = evaluate(test_complex_rule_feature, "default", client_context) + inner_result = Int64Value() + assert result.value.Unpack(inner_result) + assert inner_result.value == expected diff --git a/tests/evaulation/test_rules.py b/tests/evaulation/test_rules.py new file mode 100644 index 0000000..f10fd57 --- /dev/null +++ b/tests/evaulation/test_rules.py @@ -0,0 +1,76 @@ +import pytest + +from lekko_client.evaluation.rules import evaluate_rule +from lekko_client.gen.lekko.rules.v1beta3.rules_pb2 import CallExpression, Rule +from lekko_client.helpers import convert_context + + +@pytest.mark.parametrize( + "context_val,namespace,config_name,expected", + [ + (1, "ns_1", "feature_1", False), + (2, "ns_1", "feature_1", False), + (3, "ns_1", "feature_1", True), + (4, "ns_1", "feature_1", False), + (5, "ns_1", "feature_1", True), + (101, "ns_1", "feature_1", True), + (102, "ns_1", "feature_1", True), + (103, "ns_1", "feature_1", False), + (104, "ns_1", "feature_1", False), + (105, "ns_1", "feature_1", True), + (1, "ns_2", "feature_2", False), + (2, "ns_2", "feature_2", True), + (3, "ns_2", "feature_2", False), + (4, "ns_2", "feature_2", False), + (5, "ns_2", "feature_2", True), + (101, "ns_2", "feature_2", True), + (102, "ns_2", "feature_2", True), + (103, "ns_2", "feature_2", False), + (104, "ns_2", "feature_2", True), + (105, "ns_2", "feature_2", True), + (3.1415, "ns_1", "feature_1", False), + (2.7182, "ns_1", "feature_1", False), + (1.6180, "ns_1", "feature_1", True), + (6.6261, "ns_1", "feature_1", True), + (6.0221, "ns_1", "feature_1", False), + (2.9979, "ns_1", "feature_1", True), + (6.6730, "ns_1", "feature_1", False), + (1.3807, "ns_1", "feature_1", True), + (1.4142, "ns_1", "feature_1", True), + (2.0000, "ns_1", "feature_1", False), + (3.1415, "ns_2", "feature_2", True), + (2.7182, "ns_2", "feature_2", False), + (1.6180, "ns_2", "feature_2", True), + (6.6261, "ns_2", "feature_2", False), + (6.0221, "ns_2", "feature_2", False), + (2.9979, "ns_2", "feature_2", False), + (6.6730, "ns_2", "feature_2", False), + (1.3807, "ns_2", "feature_2", False), + (1.4142, "ns_2", "feature_2", True), + (2.0000, "ns_2", "feature_2", False), + ("hello", "ns_1", "feature_1", False), + ("world", "ns_1", "feature_1", False), + ("i", "ns_1", "feature_1", True), + ("am", "ns_1", "feature_1", True), + ("a", "ns_1", "feature_1", True), + ("unit", "ns_1", "feature_1", False), + ("test", "ns_1", "feature_1", True), + ("case", "ns_1", "feature_1", True), + ("for", "ns_1", "feature_1", False), + ("bucket", "ns_1", "feature_1", False), + ("hello", "ns_2", "feature_2", True), + ("world", "ns_2", "feature_2", False), + ("i", "ns_2", "feature_2", True), + ("am", "ns_2", "feature_2", True), + ("a", "ns_2", "feature_2", True), + ("unit", "ns_2", "feature_2", False), + ("test", "ns_2", "feature_2", True), + ("case", "ns_2", "feature_2", False), + ("for", "ns_2", "feature_2", False), + ("bucket", "ns_2", "feature_2", False), + ], +) +def test_bucket(context_val, namespace, config_name, expected): + rule = Rule(call_expression=CallExpression(bucket=CallExpression.Bucket(context_key="key", threshold=50000))) + context = convert_context({"key": context_val}) + assert evaluate_rule(rule, namespace, config_name, context) == expected diff --git a/tests/fixtures/rules.proto.bin b/tests/fixtures/rules.proto.bin new file mode 100644 index 0000000000000000000000000000000000000000..31792811c562b82fa2449db609f4ce3da90c59f2 GIT binary patch literal 2148 zcmbuA&2HL26os+D{M=9iQz}`Os$R$@u@EgJ?JA@!+Dcuvi)^Yc42%uLc#MAneV9H* z->Z+%@r-bsq^i`3SL4y>J7?}(oeM^k`ZOjzmpVxrQA`*nG)|DE3C`Ij^ywbO5$jhQ z-oP;V5z?W@nCDX`T*X6v=C%ACh7n8HEOm#UgXG=Cwd1Fh?4aY|WP&4ff(6dA>~B`~BY377 zXs!veCT>ybi2LVS-a*S#*Q(xWc?T`;q1G2LmUxM8m71|8OSR0Q-KnV)j$A1t>k_$o zgk0m5qRWOvu8YV@=JG}v2@-j8AIba9$IFSL%_GV5i_CJ?%NuoOQ!+P?m;qj?`g|st zL6K=>KAAeRC7J9Y6G9%o*moK0D>dk`-GU})Au+5nhnNj=4b3gaJM1s1?<-Q Date: Sat, 2 Sep 2023 13:51:14 -0700 Subject: [PATCH 03/28] atom tests --- lekko_client/evaluation/rules.py | 2 +- tests/evaulation/test_rules.py | 383 ++++++++++++++++++++++++++++++- 2 files changed, 383 insertions(+), 2 deletions(-) diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py index 881d563..b3cb20b 100644 --- a/lekko_client/evaluation/rules.py +++ b/lekko_client/evaluation/rules.py @@ -161,7 +161,7 @@ def evaluate_contained_within(rule_value: Value, context_value: LekkoValue) -> b return any(evaluate_equals(list_elem_val, context_value) for list_elem_val in rule_value.list_value.values) -def evaluate_bucket(bucket_f: CallExpression.Bucket, namespace: str, config_name: str, context: ClientContext): +def evaluate_bucket(bucket_f: CallExpression.Bucket, namespace: str, config_name: str, context: ClientContext) -> bool: ctx_key = bucket_f.context_key value = context.get(ctx_key) if context else None if not value: diff --git a/tests/evaulation/test_rules.py b/tests/evaulation/test_rules.py index f10fd57..2b94a0d 100644 --- a/tests/evaulation/test_rules.py +++ b/tests/evaulation/test_rules.py @@ -1,7 +1,15 @@ +from typing import Any + import pytest +from google.protobuf.struct_pb2 import Struct, Value from lekko_client.evaluation.rules import evaluate_rule -from lekko_client.gen.lekko.rules.v1beta3.rules_pb2 import CallExpression, Rule +from lekko_client.gen.lekko.rules.v1beta3.rules_pb2 import ( + Atom, + CallExpression, + ComparisonOperator, + Rule, +) from lekko_client.helpers import convert_context @@ -74,3 +82,376 @@ def test_bucket(context_val, namespace, config_name, expected): rule = Rule(call_expression=CallExpression(bucket=CallExpression.Bucket(context_key="key", threshold=50000))) context = convert_context({"key": context_val}) assert evaluate_rule(rule, namespace, config_name, context) == expected + + +def test_bool_const(): + for b in [True, False]: + rule = Rule(bool_const=b) + assert evaluate_rule(rule, "ns1", "config1") == b + + +def test_present(): + rule = Rule(atom=Atom(context_key="age", comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_PRESENT)) + assert evaluate_rule(rule, "ns1", "config1") is False + assert evaluate_rule(rule, "ns1", "config1", convert_context({"age": 10})) is True + assert evaluate_rule(rule, "ns1", "config1", convert_context({"age": "not a number"})) is True + + +def convert_to_value(v: Any) -> Value: + s = Struct() + s.update({"key": v}) + return s.fields["key"] + + +@pytest.mark.parametrize( + "test_atom,test_context,expected,raises", + [ + ( + Atom( + context_key="isprod", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(True), + ), + convert_context({"isprod": True}), + True, + False, + ), + ( + Atom( + context_key="isprod", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(True), + ), + convert_context({"isprod": False}), + False, + False, + ), + ( + Atom( + context_key="isprod", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(True), + ), + convert_context({"isprod": "not a bool"}), + False, + True, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 35}), + False, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12.001}), + False, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12.001), + ), + convert_context({"age": 12.001}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": "not a number"}), + False, + True, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_NOT_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 25}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_NOT_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + False, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({}), # not present + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + convert_context({"city": "Rome"}), + True, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + convert_context({"city": "rome"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + convert_context({"city": "Paris"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + convert_context({"city": 99}), + False, + True, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + False, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 11}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN_OR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 12}), + False, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + comparison_value=convert_to_value(12), + ), + convert_context({"age": 13}), + True, + False, + ), + ( + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + comparison_value=convert_to_value(12), + ), + convert_context({"age": "not a number"}), + False, + True, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINED_WITHIN, + comparison_value=convert_to_value(["Rome", "Paris"]), + ), + convert_context({"city": "London"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINED_WITHIN, + comparison_value=convert_to_value(["Rome", "Paris"]), + ), + convert_context({"city": "Rome"}), + True, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINED_WITHIN, + comparison_value=convert_to_value(["Rome", "Paris"]), + ), + convert_context({}), # not present + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINED_WITHIN, + comparison_value=convert_to_value(["Rome", "Paris"]), + ), + convert_context({"city": "rome"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH, + comparison_value=convert_to_value("Ro"), + ), + convert_context({"city": "Rome"}), + True, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH, + comparison_value=convert_to_value("Ro"), + ), + convert_context({"city": "London"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH, + comparison_value=convert_to_value("Ro"), + ), + convert_context({"city": "rome"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_STARTS_WITH, + comparison_value=convert_to_value("Ro"), + ), + None, # not present + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_ENDS_WITH, + comparison_value=convert_to_value("me"), + ), + convert_context({"city": "Rome"}), + True, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_ENDS_WITH, + comparison_value=convert_to_value("me"), + ), + convert_context({"city": "London"}), + False, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINS, + comparison_value=convert_to_value(""), + ), + convert_context({"city": "Rome"}), + True, + False, + ), + ( + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_CONTAINS, + comparison_value=convert_to_value("foo"), + ), + convert_context({"city": "Rome"}), + False, + False, + ), + ], +) +def test_atom(test_atom, test_context, expected, raises): + rule = Rule(atom=test_atom) + if raises: + with pytest.raises(Exception): + evaluate_rule(rule, "ns1", "config1", test_context) + else: + assert evaluate_rule(rule, "ns1", "config1", test_context) == expected From 9ab59746dd63e308940bfb82b4fe0a8dae650318 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Sat, 2 Sep 2023 21:52:16 -0700 Subject: [PATCH 04/28] logical op tests --- tests/evaulation/test_rules.py | 202 +++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/tests/evaulation/test_rules.py b/tests/evaulation/test_rules.py index 2b94a0d..f423f76 100644 --- a/tests/evaulation/test_rules.py +++ b/tests/evaulation/test_rules.py @@ -8,6 +8,8 @@ Atom, CallExpression, ComparisonOperator, + LogicalExpression, + LogicalOperator, Rule, ) from lekko_client.helpers import convert_context @@ -455,3 +457,203 @@ def test_atom(test_atom, test_context, expected, raises): evaluate_rule(rule, "ns1", "config1", test_context) else: assert evaluate_rule(rule, "ns1", "config1", test_context) == expected + + +@pytest.mark.parametrize( + "test_atoms,logical_op,test_context,expected,raises", + [ + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + comparison_value=convert_to_value(10), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_OR, + convert_context({"age": 8}), + False, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN, + comparison_value=convert_to_value(10), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_OR, + convert_context({"age": 12}), + True, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_AND, + convert_context({"age": 8, "city": "Rome"}), + False, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_AND, + convert_context({"age": 3, "city": "Rome"}), + True, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_UNSPECIFIED, + convert_context({"age": 3}), + True, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_AND, + convert_context({"age": 3}), + False, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ) + ], + LogicalOperator.LOGICAL_OPERATOR_AND, + convert_context({"age": 3}), + True, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ) + ], + LogicalOperator.LOGICAL_OPERATOR_OR, + convert_context({"age": 3}), + True, + False, + ), + ([], LogicalOperator.LOGICAL_OPERATOR_AND, convert_context({}), False, True), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(8), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_AND, + convert_context({"age": 8}), + False, + False, + ), + ( + [ + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_LESS_THAN, + comparison_value=convert_to_value(5), + ), + Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ), + Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(8), + ), + ], + LogicalOperator.LOGICAL_OPERATOR_OR, + convert_context({"age": 8}), + True, + False, + ), + ], +) +def test_logical_op(test_atoms, logical_op, test_context, expected, raises): + rule = Rule( + logical_expression=LogicalExpression( + rules=[Rule(atom=atom) for atom in test_atoms], logical_operator=logical_op + ) + ) + + if raises: + with pytest.raises(Exception): + evaluate_rule(rule, "ns_1", "feature_1", test_context) + else: + assert evaluate_rule(rule, "ns_1", "feature_1", test_context) == expected From 65d4cea91a6b6aa283bb878b4bea146c7972e5a2 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Tue, 5 Sep 2023 12:56:22 -0700 Subject: [PATCH 05/28] rest of evaluation tests and pr feedback --- lekko_client/evaluation/evaluation.py | 10 +- tests/conftest.py | 193 +++++++++++++++++- tests/evaluation/test_evaluation.py | 109 ++++++++++ .../{evaulation => evaluation}/test_rules.py | 0 tests/evaulation/test_evaluation.py | 31 --- 5 files changed, 305 insertions(+), 38 deletions(-) create mode 100644 tests/evaluation/test_evaluation.py rename tests/{evaulation => evaluation}/test_rules.py (100%) delete mode 100644 tests/evaulation/test_evaluation.py diff --git a/lekko_client/evaluation/evaluation.py b/lekko_client/evaluation/evaluation.py index f4b9367..706c3e9 100644 --- a/lekko_client/evaluation/evaluation.py +++ b/lekko_client/evaluation/evaluation.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import List, Optional -from google.protobuf.any_pb2 import Any +from google.protobuf.any_pb2 import Any as ProtoAny from lekko_client.evaluation.rules import ClientContext, evaluate_rule from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny @@ -18,13 +18,13 @@ class EvaluationResult: @dataclass class TraverseResult: - value: Optional[Any] + value: Optional[ProtoAny] passes: bool path: List[int] def evaluate(config: Feature, namespace: str, context: ClientContext = None) -> EvaluationResult: - if not config.tree: + if not config.HasField("tree"): raise ValueError("config tree is empty") for i, constraint in enumerate(config.tree.constraints): @@ -52,9 +52,9 @@ def traverse(override: Constraint, namespace: str, config_name: str, context: Cl return TraverseResult(_get_any(override.value, override.value_new), True, []) -def _get_any(val: Optional[Any], val_new: Optional[LekkoAny]) -> Any: +def _get_any(val: Optional[ProtoAny], val_new: Optional[LekkoAny]) -> ProtoAny: if val_new and val_new.type_url: - return Any(type_url=val_new.type_url, value=val_new.value) + return ProtoAny(type_url=val_new.type_url, value=val_new.value) if val: return val raise ValueError("config value not found") diff --git a/tests/conftest.py b/tests/conftest.py index 5a6a3ac..bab163b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,32 @@ import concurrent.futures.thread from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from typing import List, Tuple +from typing import Any, List, Tuple from unittest import mock import grpc import grpc_testing import pytest +from google.protobuf.any_pb2 import Any as ProtoAny from google.protobuf.message import Message as ProtoMessage +from google.protobuf.struct_pb2 import Struct, Value +from google.protobuf.wrappers_pb2 import Int64Value from grpc_testing import _channel # noqa from lekko_client import helpers from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import DESCRIPTOR -from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import ( + Constraint, + Feature, + FeatureType, + Tree, +) +from lekko_client.gen.lekko.rules.v1beta3.rules_pb2 import ( + Atom, + ComparisonOperator, + Rule, +) @pytest.fixture @@ -109,3 +123,178 @@ def test_complex_rule_feature(): feature.ParseFromString(f.read()) return feature + + +@pytest.fixture +def test_feature_default_value() -> ProtoAny: + any_proto = ProtoAny() + any_proto.Pack(Int64Value(value=1)) + return any_proto + + +@pytest.fixture +def test_feature_constraint_value() -> ProtoAny: + any_proto = ProtoAny() + any_proto.Pack(Int64Value(value=2)) + return any_proto + + +def convert_to_value(v: Any) -> Value: + s = Struct() + s.update({"key": v}) + return s.fields["key"] + + +@pytest.fixture +def test_feature_no_constraints(test_feature_default_value) -> Feature: + return Feature( + key="key", + description="config description", + type=FeatureType.FEATURE_TYPE_INT, + tree=Tree( + default=test_feature_default_value, + default_new=LekkoAny(type_url=test_feature_default_value.type_url, value=test_feature_default_value.value), + ), + ) + + +@pytest.fixture +def test_feature_one_level_traversal(test_feature_default_value, test_feature_constraint_value) -> Feature: + return Feature( + key="key", + description="config description", + type=FeatureType.FEATURE_TYPE_INT, + tree=Tree( + default=test_feature_default_value, + default_new=LekkoAny(type_url=test_feature_default_value.type_url, value=test_feature_default_value.value), + constraints=[ + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(10), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, value=test_feature_constraint_value.value + ), + ), + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, value=test_feature_constraint_value.value + ), + ), + ], + ), + ) + + +@pytest.fixture +def test_feature_two_level_traversal(test_feature_default_value, test_feature_constraint_value) -> Feature: + return Feature( + key="key", + description="config description", + type=FeatureType.FEATURE_TYPE_INT, + tree=Tree( + default=test_feature_default_value, + default_new=LekkoAny(type_url=test_feature_default_value.type_url, value=test_feature_default_value.value), + constraints=[ + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(10), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, value=test_feature_constraint_value.value + ), + constraints=[ + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, + value=test_feature_constraint_value.value, + ), + ), + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Paris"), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, + value=test_feature_constraint_value.value, + ), + ), + ], + ), + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="age", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value(12), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, value=test_feature_constraint_value.value + ), + constraints=[ + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Rome"), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, + value=test_feature_constraint_value.value, + ), + ), + Constraint( + rule_ast_new=Rule( + atom=Atom( + context_key="city", + comparison_operator=ComparisonOperator.COMPARISON_OPERATOR_EQUALS, + comparison_value=convert_to_value("Paris"), + ) + ), + value=test_feature_constraint_value, + value_new=LekkoAny( + type_url=test_feature_constraint_value.type_url, + value=test_feature_constraint_value.value, + ), + ), + ], + ), + ], + ), + ) diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py new file mode 100644 index 0000000..092ffe9 --- /dev/null +++ b/tests/evaluation/test_evaluation.py @@ -0,0 +1,109 @@ +from typing import List + +import pytest +from google.protobuf.wrappers_pb2 import Int64Value + +from lekko_client.evaluation.evaluation import evaluate +from lekko_client.evaluation.rules import ClientContext +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature +from lekko_client.helpers import convert_context + + +@pytest.mark.parametrize( + "context,expected", + [ + ({"a": 1}, 0), + ({"a": 1, "c": 3}, 1), + ({"c": 3}, 2), + ({"f": 3}, 0), + ({"f": 5}, 5), + ({"h": 4}, 7), + ({"i": 4}, 8), + ({"p": "a foo bar"}, 15), + ({"q": "hello world"}, 16), + ({"r": "a foo bar"}, 17), + ({"s": 2}, 18), + ({"t": "anything"}, 19), + ({"u": 11}, 20), + ], +) +def test_complex_evaluation(test_complex_rule_feature, context, expected): + client_context = convert_context(context) + result = evaluate(test_complex_rule_feature, "default", client_context) + inner_result = Int64Value() + assert result.value.Unpack(inner_result) + assert inner_result.value == expected + + +@pytest.mark.only +def test_empty_config_tree(): + with pytest.raises(Exception): + evaluate(Feature(), "ns") + + +@pytest.mark.parametrize( + "feature_fixture_name,test_context,expected_fixture_name,expected_path", + [ + ("test_feature_no_constraints", convert_context({}), "test_feature_default_value", []), + ("test_feature_no_constraints", convert_context({"key": "anything"}), "test_feature_default_value", []), + ("test_feature_no_constraints", None, "test_feature_default_value", []), + ("test_feature_one_level_traversal", convert_context({}), "test_feature_default_value", []), + ("test_feature_one_level_traversal", convert_context({"age": 5}), "test_feature_default_value", []), + ("test_feature_one_level_traversal", convert_context({"age": 10}), "test_feature_constraint_value", [0]), + ("test_feature_one_level_traversal", convert_context({"age": 12}), "test_feature_constraint_value", [1]), + ("test_feature_two_level_traversal", convert_context({}), "test_feature_default_value", []), + ("test_feature_two_level_traversal", convert_context({"age": 10}), "test_feature_constraint_value", [0]), + ( + "test_feature_two_level_traversal", + convert_context({"age": 10, "city": "Rome"}), + "test_feature_constraint_value", + [0, 0], + ), + ( + "test_feature_two_level_traversal", + convert_context({"age": 10, "city": "Paris"}), + "test_feature_constraint_value", + [0, 1], + ), + ( + "test_feature_two_level_traversal", + convert_context({"age": 12, "city": "Milan"}), + "test_feature_constraint_value", + [1], + ), + ("test_feature_two_level_traversal", convert_context({"age": 12}), "test_feature_constraint_value", [1]), + ( + "test_feature_two_level_traversal", + convert_context({"age": 12, "city": "Rome"}), + "test_feature_constraint_value", + [1, 0], + ), + ( + "test_feature_two_level_traversal", + convert_context({"age": 12, "city": "Paris"}), + "test_feature_constraint_value", + [1, 1], + ), + ( + "test_feature_two_level_traversal", + convert_context({"age": 12, "city": "Milan"}), + "test_feature_constraint_value", + [1], + ), + ], +) +def test_eval( + feature_fixture_name: str, + test_context: ClientContext, + expected_fixture_name: str, + expected_path: List[int], + request, +): + test_feature = request.getfixturevalue(feature_fixture_name) + expected = request.getfixturevalue(expected_fixture_name) + if expected is not None: + result = evaluate(test_feature, "ns_1", test_context) + assert result.value == expected + assert result.path == expected_path + else: + raise ValueError("test case needs to either expect an error or a result") diff --git a/tests/evaulation/test_rules.py b/tests/evaluation/test_rules.py similarity index 100% rename from tests/evaulation/test_rules.py rename to tests/evaluation/test_rules.py diff --git a/tests/evaulation/test_evaluation.py b/tests/evaulation/test_evaluation.py deleted file mode 100644 index 0e78444..0000000 --- a/tests/evaulation/test_evaluation.py +++ /dev/null @@ -1,31 +0,0 @@ -import pytest -from google.protobuf.wrappers_pb2 import Int64Value - -from lekko_client.evaluation.evaluation import evaluate -from lekko_client.helpers import convert_context - - -@pytest.mark.parametrize( - "context,expected", - [ - ({"a": 1}, 0), - ({"a": 1, "c": 3}, 1), - ({"c": 3}, 2), - ({"f": 3}, 0), - ({"f": 5}, 5), - ({"h": 4}, 7), - ({"i": 4}, 8), - ({"p": "a foo bar"}, 15), - ({"q": "hello world"}, 16), - ({"r": "a foo bar"}, 17), - ({"s": 2}, 18), - ({"t": "anything"}, 19), - ({"u": 11}, 20), - ], -) -def test_evaluation(test_complex_rule_feature, context, expected): - client_context = convert_context(context) - result = evaluate(test_complex_rule_feature, "default", client_context) - inner_result = Int64Value() - assert result.value.Unpack(inner_result) - assert inner_result.value == expected From 4bc28c0b80b7b1f3997954520ee3660f3b72caa0 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Tue, 5 Sep 2023 14:10:18 -0700 Subject: [PATCH 06/28] fix --- lekko_client/evaluation/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lekko_client/evaluation/evaluation.py b/lekko_client/evaluation/evaluation.py index 706c3e9..c87e78b 100644 --- a/lekko_client/evaluation/evaluation.py +++ b/lekko_client/evaluation/evaluation.py @@ -10,7 +10,7 @@ @dataclass class EvaluationResult: - value: Any + value: ProtoAny # Stores the path of the tree node that returned the final value # after successful evaluation. path: List[int] From 8ac531ac742803d817dc4d41a74bf229075382af Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Sat, 9 Sep 2023 17:03:48 -0700 Subject: [PATCH 07/28] add new client interfaces --- Makefile | 3 +- lekko_client/__init__.py | 85 +++- lekko_client/clients/__init__.py | 4 + lekko_client/clients/cached_backend_client.py | 70 ++++ lekko_client/clients/cached_git_client.py | 110 ++++++ lekko_client/clients/client.py | 61 +++ lekko_client/clients/distribution_client.py | 153 ++++++++ .../{client.py => clients/grpc_client.py} | 114 ++---- lekko_client/constants.py | 2 + .../v1beta1/distribution_service_pb2.py | 61 +++ .../v1beta1/distribution_service_pb2.pyi | 369 ++++++++++++++++++ .../v1beta1/distribution_service_pb2_grpc.py | 246 ++++++++++++ lekko_client/helpers.py | 9 +- lekko_client/models.py | 9 + lekko_client/stores/__init__.py | 2 + lekko_client/stores/memory.py | 31 ++ lekko_client/stores/store.py | 60 +++ pyproject.toml | 3 + tests/conftest.py | 2 +- tests/test_client.py | 56 +-- 20 files changed, 1325 insertions(+), 125 deletions(-) create mode 100644 lekko_client/clients/__init__.py create mode 100644 lekko_client/clients/cached_backend_client.py create mode 100644 lekko_client/clients/cached_git_client.py create mode 100644 lekko_client/clients/client.py create mode 100644 lekko_client/clients/distribution_client.py rename lekko_client/{client.py => clients/grpc_client.py} (64%) create mode 100644 lekko_client/constants.py create mode 100644 lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.py create mode 100644 lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.pyi create mode 100644 lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2_grpc.py create mode 100644 lekko_client/models.py create mode 100644 lekko_client/stores/__init__.py create mode 100644 lekko_client/stores/memory.py create mode 100644 lekko_client/stores/store.py diff --git a/Makefile b/Makefile index 28aa012..66772b3 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,9 @@ fmt: venv .PHONY: bufgen bufgen: buf generate buf.build/lekkodev/sdk --type lekko.client.v1beta1 - buf generate buf.build/lekkodev/cli --type lekko.rules.v1beta3 --type lekko.feature.v1beta1 + buf generate buf.build/lekkodev/cli --type lekko.rules.v1beta3 --type lekko.feature.v1beta1 --type lekko.backend.v1beta1 grep -rl "from lekko.\|import lekko.\|type: lekko." ./lekko_client/gen --include \*.py --include \*.pyi | xargs sed -i'.bak' -E -e 's/ lekko\./ lekko_client.gen.lekko./' rm -f lekko_client/gen/lekko/client/v1beta1/*.bak rm -f lekko_client/gen/lekko/feature/v1beta1/*.bak rm -f lekko_client/gen/lekko/rules/v1beta3/*.bak + rm -f lekko_client/gen/lekko/backend/v1beta1/*.bak diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 0b448b0..48565da 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -1,9 +1,86 @@ """Lekko Python SDK Client""" -__version__ = "0.1.4" +from enum import Enum +from typing import Any, Dict, Optional, Type -LEKKO_API_URL = "prod.api.lekko.dev:443" -LEKKO_SIDECAR_URL = "localhost:50051" +from google.protobuf.message import Message as ProtoMessage -from lekko_client.client import APIClient, SidecarClient # noqa +from lekko_client.clients import ( + APIClient, + CachedBackendClient, + CachedGitClient, + Client, + SidecarClient, +) +from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa from lekko_client.exceptions import * # noqa +from lekko_client.stores import MemoryStore + +__version__ = "0.1.4" + +__client: Client + + +class Mode(Enum): + API = 1 + SIDECAR = 2 + CACHED_SERVER = 3 + CACHED_GIT = 4 + + +def initialize( + mode: Mode, + owner_name: str, + repo_name: str, + api_key: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, +) -> Client: + global __client + if mode == Mode.API: + __client = APIClient(owner_name, repo_name, api_key, context) + elif mode == Mode.SIDECAR: + __client = SidecarClient(owner_name, repo_name, api_key, context) + elif mode == Mode.CACHED_GIT: + __client = CachedGitClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), "", api_key, context) + elif mode == Mode.CACHED_SERVER: + __client = CachedBackendClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), api_key, context) + else: + raise ValueError("Invalid mode") + return __client + + +def get_bool(namespace: str, key: str, context: Dict[str, Any]) -> bool: + return __client.get_bool(namespace, key, context) + + +def get_int(namespace: str, key: str, context: Dict[str, Any]) -> int: + return __client.get_int(namespace, key, context) + + +def get_float(namespace: str, key: str, context: Dict[str, Any]) -> float: + return __client.get_float(namespace, key, context) + + +def get_string(namespace: str, key: str, context: Dict[str, Any]) -> str: + return __client.get_string(namespace, key, context) + + +def get_json(namespace: str, key: str, context: Dict[str, Any]) -> dict: + return __client.get_json(namespace, key, context) + + +def get_proto( + namespace: str, + key: str, + context: Dict[str, Any], +) -> ProtoMessage: + return __client.get_proto(namespace, key, context) + + +def get_proto_by_type( + namespace: str, + key: str, + context: Dict[str, Any], + proto_message_type: Type[Client.ProtoType], +) -> Client.ProtoType: + return __client.get_proto_by_type(namespace, key, context, proto_message_type) diff --git a/lekko_client/clients/__init__.py b/lekko_client/clients/__init__.py new file mode 100644 index 0000000..0730a28 --- /dev/null +++ b/lekko_client/clients/__init__.py @@ -0,0 +1,4 @@ +from lekko_client.clients.cached_backend_client import CachedBackendClient # noqa +from lekko_client.clients.cached_git_client import CachedGitClient # noqa +from lekko_client.clients.client import Client # noqa +from lekko_client.clients.grpc_client import APIClient, SidecarClient # noqa diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py new file mode 100644 index 0000000..baa0873 --- /dev/null +++ b/lekko_client/clients/cached_backend_client.py @@ -0,0 +1,70 @@ +import time +from threading import Thread +from typing import Any, Dict, Optional + +import grpc + +from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsResponse, +) +from lekko_client.stores.store import Store + + +class CachedBackendClient(CachedDistributionClient): + class RefreshThread(Thread): + def __init__(self, client: "CachedBackendClient", refresh_interval: int): + super().__init__() + self.daemon = True + self.client = client + self.refresh_interval = refresh_interval + self._enabled = True + + def stop(self): + self._enabled = False + + def run(self): + while self._enabled: + if self.client.should_update_store(): + self.client.update_store() + time.sleep(self.refresh_interval / 1000) + + def __init__( + self, + uri: str, + owner_name: str, + repo_name: str, + store: Store, + api_key: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), + ): + super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) + self.timeout = None + self.closed = False + self.update_interval = 1000 + + def initialize(self): + self.update_store() + if self.update_interval: + self.refresh_thread = CachedBackendClient.RefreshThread(self, self.update_interval) + self.refresh_thread.start() + + def get_contents(self) -> Optional[GetRepositoryContentsResponse]: + if not self._client: + return None + return self._client.GetRepositoryContents(repo_key=self.repository, session_key=self.session_key) + + def update_store(self): + self.load() + + def should_update_store(self): + if not self._client: + return + version_response = self._client.GetRepositoryVersion(repo_key=self.repository, session_key=self.session_key) + current_sha = self.store.commit_sha + return current_sha != version_response.commit_sha + + def close(self): + super().close() + self.refresh_thread.stop() diff --git a/lekko_client/clients/cached_git_client.py b/lekko_client/clients/cached_git_client.py new file mode 100644 index 0000000..f5cb6b3 --- /dev/null +++ b/lekko_client/clients/cached_git_client.py @@ -0,0 +1,110 @@ +import glob +import os +from typing import Any, Dict, List, Optional + +import grpc +import yaml +from dulwich.errors import NotGitRepository +from dulwich.object_store import tree_lookup_path +from dulwich.repo import Repo as GitRepo +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer +from watchdog.observers.api import BaseObserver + +from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + Feature as DistFeature, +) +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsResponse, + Namespace, +) +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature +from lekko_client.stores.store import Store + + +class CachedGitClient(CachedDistributionClient): + ROOT_CONFIG_METADATA_FILENAME = "lekko.root.yaml" + + class GitFileEventHandler(FileSystemEventHandler): + def __init__(self, client: "CachedGitClient") -> None: + super().__init__() + self.client = client + + def on_any_event(self, event): + self.client.load() + return super().on_any_event(event) + + def __init__( + self, + lekko_uri: str, + repository_owner: str, + repository_name: str, + store: Store, + path: str, + api_key: Optional[str], + context: Optional[Dict[str, Any]] = None, + credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), + should_watch: Optional[bool] = True, + ): + super().__init__(lekko_uri, repository_owner, repository_name, store, api_key, context, credentials) + self.watcher: Optional[BaseObserver] = None + self.path = path + self.should_watch = should_watch + + def initialize(self): + super().initialize() + self.load() + if self.should_watch: + event_handler = CachedGitClient.GitFileEventHandler(self) + self.watcher = Observer() + self.watcher.schedule(event_handler, self.path, recursive=True) + self.watcher.start() + + def get_contents(self) -> GetRepositoryContentsResponse: + try: + repo = GitRepo(self.path) + except NotGitRepository: + raise ValueError("Not a git repository") + + return GetRepositoryContentsResponse( + commit_sha=repo.head().decode("utf-8"), namespaces=self.get_namespaces(repo) + ) + + def get_namespaces(self, repo: GitRepo) -> List[Namespace]: + md_file_path = os.path.join(self.path, self.ROOT_CONFIG_METADATA_FILENAME) + with open(md_file_path) as f: + md_contents = yaml.safe_load(f) + + ns_names = md_contents.get("namespaces", []) + return [Namespace(name=ns_name, features=self.get_configs(repo, ns_name)) for ns_name in ns_names] + + def get_configs(self, repo: GitRepo, ns_name: str) -> List[DistFeature]: + proto_dir_path = os.path.join(self.path, ns_name, "gen", "proto") + if not os.path.isdir(proto_dir_path): + return [] + + features = [] + for proto_bin_file in glob.glob(os.path.join(proto_dir_path, "*.proto.bin")): + with open(proto_bin_file, "rb") as proto_bin: + _, sha = tree_lookup_path( + repo.get_object, repo[repo.head()].tree, proto_bin_file.encode() # type: ignore + ) + feature = Feature() + feature.ParseFromString(proto_bin.read()) + features.append( + DistFeature( + name=proto_bin_file.replace(".proto.bin", ""), + sha=sha.decode("utf-8"), + feature=feature, + ) + ) + return features + + def close(self): + super().close() + # if self.events_batcher: + # await self.events_batcher.close() + if self.watcher: + self.watcher.stop() + self.watcher.join() diff --git a/lekko_client/clients/client.py b/lekko_client/clients/client.py new file mode 100644 index 0000000..362b88b --- /dev/null +++ b/lekko_client/clients/client.py @@ -0,0 +1,61 @@ +import os +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Type, TypeVar + +from google.protobuf.message import Message as ProtoMessage + + +class Client(ABC): + ProtoType = TypeVar("ProtoType", bound=ProtoMessage) + + def __init__( + self, + owner_name: str, + repo_name: str, + api_key: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + ): + self.owner_name = owner_name + self.repo_name = repo_name + self.context = context or {} + + self.api_key = api_key or os.environ.get("LEKKO_API_KEY") + + @abstractmethod + def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: + ... + + @abstractmethod + def get_int(self, namespace: str, key: str, context: Dict[str, Any]) -> int: + ... + + @abstractmethod + def get_float(self, namespace: str, key: str, context: Dict[str, Any]) -> float: + ... + + @abstractmethod + def get_string(self, namespace: str, key: str, context: Dict[str, Any]) -> str: + ... + + @abstractmethod + def get_json(self, namespace: str, key: str, context: Dict[str, Any]) -> dict: + ... + + @abstractmethod + def get_proto( + self, + namsespace: str, + key: str, + context: Dict[str, Any], + ) -> ProtoMessage: + ... + + @abstractmethod + def get_proto_by_type( + self, + namsespace: str, + key: str, + context: Dict[str, Any], + proto_message_type: Type[ProtoType], + ) -> ProtoType: + ... diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py new file mode 100644 index 0000000..11990fa --- /dev/null +++ b/lekko_client/clients/distribution_client.py @@ -0,0 +1,153 @@ +import json +from abc import abstractmethod +from typing import Any, Dict, Optional, Type, TypeVar, Union + +import grpc +from google.protobuf import descriptor_pool as proto_descriptor_pool +from google.protobuf import symbol_database as proto_symbol_database +from google.protobuf.any_pb2 import Any as ProtoAny +from google.protobuf.json_format import MessageToJson +from google.protobuf.message import Message as ProtoMessage +from google.protobuf.struct_pb2 import Value +from google.protobuf.wrappers_pb2 import BoolValue, FloatValue, Int64Value, StringValue + +from lekko_client.clients.client import Client +from lekko_client.evaluation.evaluation import evaluate +from lekko_client.exceptions import MismatchedProtoType +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + DeregisterClientRequest, + GetRepositoryContentsResponse, + RegisterClientRequest, + RepositoryKey, +) +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2_grpc import ( + DistributionServiceStub, +) +from lekko_client.helpers import convert_context, get_grpc_channel +from lekko_client.stores.store import Store + + +class CachedDistributionClient(Client): + def __init__( + self, + uri: str, + owner_name: str, + repo_name: str, + store: Store, + api_key: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), + ): + from lekko_client import __version__ + + super().__init__(owner_name, repo_name, api_key, context) + self.uri = uri + self.repository = RepositoryKey(owner_name=owner_name, repo_name=repo_name) + self.store = store + self._client = None + if self.api_key: + channel = get_grpc_channel(self.uri, self.api_key, credentials) + self._client = DistributionServiceStub(channel) + register_response = self._client.RegisterClient( + RegisterClientRequest(repo_key=self.repository, sidecar_version=__version__) + ) + self.session_key = register_response.session_key + # if self.events_batcher: + # await self.events_batcher.init(self.session_key) + self.initialize() + + _TYPE_MAPPING: Dict[Type, Type[Union[BoolValue, Int64Value, StringValue, FloatValue]]] = { + bool: BoolValue, + int: Int64Value, + str: StringValue, + float: FloatValue, + } + + @abstractmethod + def initialize(self): + ... + + def load(self) -> bool: + contents = self.get_contents() + if not contents: + return False + loaded = self.store.load(contents) + return loaded + + @abstractmethod + def get_contents(self) -> Optional[GetRepositoryContentsResponse]: + ... + + def _get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: + feature_data = self.store.get(namespace, key) + result = evaluate(feature_data.feature, namespace, convert_context(context)) + return result.value + + ReturnType = TypeVar("ReturnType", str, float, int, bool) + + def _get_scalar(self, namespace: str, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: + result = self._get(namespace, key, context) + return_wrapper = self._TYPE_MAPPING[typ]() + result.Unpack(return_wrapper) + return return_wrapper.value # type:ignore + + def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: + return self._get_scalar(namespace, key, context, bool) + + def get_int(self, namespace: str, key: str, context: Dict[str, Any]) -> int: + return self._get_scalar(namespace, key, context, int) + + def get_float(self, namespace: str, key: str, context: Dict[str, Any]) -> float: + return self._get_scalar(namespace, key, context, float) + + def get_string(self, namespace: str, key: str, context: Dict[str, Any]) -> str: + return self._get_scalar(namespace, key, context, str) + + def get_json(self, namespace: str, key: str, context: Dict[str, Any]) -> Any: + result = self._get(namespace, key, context) + return_wrapper = Value() + result.Unpack(return_wrapper) + return json.loads(MessageToJson(return_wrapper)) + + def get_proto(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoMessage: + val = self._get(namespace, key, context) + db = proto_symbol_database.SymbolDatabase(pool=proto_descriptor_pool.Default()) + try: + ret_val = db.GetSymbol(val.type_url.split("/")[1])() + if val.Unpack(ret_val): + return ret_val + except (KeyError, IndexError): + pass + return val + + def get_proto_by_type( + self, + namespace: str, + key: str, + context: Dict[str, Any], + proto_message_type: Type[Client.ProtoType], + ) -> Client.ProtoType: + val = self._get(namespace, key, context) + ret_val = proto_message_type() + if val.Unpack(ret_val): + return ret_val + + raise MismatchedProtoType(f"Error unpacking from {val.type_url} to {proto_message_type.DESCRIPTOR.name}") + + # def track(self, namespace: str, key: str, result: StoredEvalResult, ctx: Optional[ClientContext] = None): + # if not self.events_batcher: + # return + # self.events_batcher.track(FlagEvaluationEvent( + # repo_key=self.repo_key, + # commit_sha=result.commit_sha, + # feature_sha=result.config_sha, + # namespace_name=namespace, + # feature_name=key, + # context_keys=to_context_keys_proto(ctx), + # result_path=result.eval_result.path, + # client_event_time=Timestamp.now() + # )) + + def close(self): + if self._client and self.session_key: + self._client.DeregisterClient(DeregisterClientRequest(session_key=self.session_key)) diff --git a/lekko_client/client.py b/lekko_client/clients/grpc_client.py similarity index 64% rename from lekko_client/client.py rename to lekko_client/clients/grpc_client.py index 412930b..7df11ed 100644 --- a/lekko_client/client.py +++ b/lekko_client/clients/grpc_client.py @@ -1,6 +1,4 @@ import json -import os -from abc import ABC, abstractmethod from typing import Any, Dict, Optional, Tuple, Type, TypeVar import grpc @@ -9,7 +7,8 @@ from google.protobuf.any_pb2 import Any as AnyProto from google.protobuf.message import Message as ProtoMessage -from lekko_client import LEKKO_API_URL, LEKKO_SIDECAR_URL +from lekko_client.clients.client import Client +from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL from lekko_client.exceptions import ( AuthenticationError, FeatureNotFound, @@ -32,63 +31,7 @@ from lekko_client.helpers import convert_context, get_grpc_channel -class Client(ABC): - ProtoType = TypeVar("ProtoType", bound=ProtoMessage) - - def __init__( - self, - owner_name: str, - repo_name: str, - namespace: str, - api_key: Optional[str] = None, - context: Optional[Dict[str, Any]] = None, - ): - self.owner_name = owner_name - self.repo_name = repo_name - self.namespace = namespace - self.context = context or {} - - self.api_key = api_key or os.environ.get("LEKKO_API_KEY") - - @abstractmethod - def get_bool(self, key: str, context: Dict[str, Any]) -> bool: - ... - - @abstractmethod - def get_int(self, key: str, context: Dict[str, Any]) -> int: - ... - - @abstractmethod - def get_float(self, key: str, context: Dict[str, Any]) -> float: - ... - - @abstractmethod - def get_string(self, key: str, context: Dict[str, Any]) -> str: - ... - - @abstractmethod - def get_json(self, key: str, context: Dict[str, Any]) -> dict: - ... - - @abstractmethod - def get_proto( - self, - key: str, - context: Dict[str, Any], - ) -> ProtoMessage: - ... - - @abstractmethod - def get_proto_by_type( - self, - key: str, - context: Dict[str, Any], - proto_message_type: Type[ProtoType], - ) -> ProtoType: - ... - - -class GRPCClient(Client): +class ConfigServiceClient(Client): _channels: Dict[Tuple[str, str], grpc.Channel] = {} class JsonBytes(bytes): @@ -110,16 +53,13 @@ def __init__( uri: str, owner_name: str, repo_name: str, - namespace: str, api_key: Optional[str] = None, context: Optional[Dict[str, Any]] = None, credentials: Optional[grpc.ChannelCredentials] = None, ): - super().__init__(owner_name, repo_name, namespace, api_key) + super().__init__(owner_name, repo_name, api_key) self.repository = RepositoryKey(owner_name=owner_name, repo_name=repo_name) - self.api_key = api_key or os.environ.get("LEKKO_API_KEY") - self.namespace = namespace self.context = context or {} self.uri = uri @@ -130,29 +70,29 @@ def __init__( self._client = ConfigurationServiceStub(channel) try: - self._client.Register(RegisterRequest(repo_key=self.repository, namespace_list=[namespace])) + self._client.Register(RegisterRequest(repo_key=self.repository, namespace_list=[])) except grpc.RpcError: # TODO:SAM - re-registering shouldn't cause errors in the future pass - def get_bool(self, key: str, context: Dict[str, Any]) -> bool: - return self._get(key, context, bool) + def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: + return self._get(namespace, key, context, bool) - def get_int(self, key: str, context: Dict[str, Any]) -> int: - return self._get(key, context, int) + def get_int(self, namespace: str, key: str, context: Dict[str, Any]) -> int: + return self._get(namespace, key, context, int) - def get_float(self, key: str, context: Dict[str, Any]) -> float: - return self._get(key, context, float) + def get_float(self, namespace: str, key: str, context: Dict[str, Any]) -> float: + return self._get(namespace, key, context, float) - def get_string(self, key: str, context: Dict[str, Any]) -> str: - return self._get(key, context, str) + def get_string(self, namespace: str, key: str, context: Dict[str, Any]) -> str: + return self._get(namespace, key, context, str) - def get_json(self, key: str, context: Dict[str, Any]) -> dict: - json_bytes = self._get(key, context, GRPCClient.JsonBytes) + def get_json(self, namespace: str, key: str, context: Dict[str, Any]) -> dict: + json_bytes = self._get(namespace, key, context, ConfigServiceClient.JsonBytes) return json.loads(json_bytes.decode("utf-8")) - def get_proto(self, key: str, context: Dict[str, Any]) -> ProtoMessage: - val = self._get_proto(key, context) + def get_proto(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoMessage: + val = self._get_proto(namespace, key, context) db = proto_symbol_database.SymbolDatabase(pool=proto_descriptor_pool.Default()) try: ret_val = db.GetSymbol(val.type_url.split("/")[1])() @@ -164,25 +104,26 @@ def get_proto(self, key: str, context: Dict[str, Any]) -> ProtoMessage: def get_proto_by_type( self, + namespace: str, key: str, context: Dict[str, Any], proto_message_type: Type[Client.ProtoType], ) -> Client.ProtoType: - val = self._get_proto(key, context) + val = self._get_proto(namespace, key, context) ret_val = proto_message_type() if val.Unpack(ret_val): return ret_val raise MismatchedProtoType(f"Error unpacking from {val.type_url} to {proto_message_type.DESCRIPTOR.name}") - def _get(self, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: + def _get(self, namespace: str, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: ctx = self.context | context fn_name, req_type = self._TYPE_MAPPING[typ] try: req = req_type( key=key, context=convert_context(ctx), - namespace=self.namespace, + namespace=namespace, repo_key=self.repository, ) response = getattr(self._client, fn_name)(req) @@ -194,13 +135,13 @@ def _get(self, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> Retu raise MismatchedType(e.details()) from e raise - def _get_proto(self, key: str, context: Dict[str, Any]) -> AnyProto: + def _get_proto(self, namespace: str, key: str, context: Dict[str, Any]) -> AnyProto: ctx = self.context | context try: req = GetProtoValueRequest( key=key, context=convert_context(ctx), - namespace=self.namespace, + namespace=namespace, repo_key=self.repository, ) response = self._client.GetProtoValue(req) @@ -218,25 +159,23 @@ def _get_proto(self, key: str, context: Dict[str, Any]) -> AnyProto: raise -class SidecarClient(GRPCClient): +class SidecarClient(ConfigServiceClient): def __init__( self, owner_name: str, repo_name: str, - namespace: str, api_key: Optional[str] = None, context: Optional[Dict[str, Any]] = None, uri: str = LEKKO_SIDECAR_URL, ): - super().__init__(uri, owner_name, repo_name, namespace, api_key, context) + super().__init__(uri, owner_name, repo_name, api_key, context) -class APIClient(GRPCClient): +class APIClient(ConfigServiceClient): def __init__( self, owner_name: str, repo_name: str, - namespace: str, api_key: Optional[str] = None, context: Optional[Dict[str, Any]] = None, uri: str = LEKKO_API_URL, @@ -246,7 +185,6 @@ def __init__( uri, owner_name, repo_name, - namespace, api_key, context, credentials, diff --git a/lekko_client/constants.py b/lekko_client/constants.py new file mode 100644 index 0000000..e62d1b1 --- /dev/null +++ b/lekko_client/constants.py @@ -0,0 +1,2 @@ +LEKKO_API_URL = "prod.api.lekko.dev:443" +LEKKO_SIDECAR_URL = "localhost:50051" diff --git a/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.py b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.py new file mode 100644 index 0000000..469c0a0 --- /dev/null +++ b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lekko/backend/v1beta1/distribution_service.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from lekko_client.gen.lekko.feature.v1beta1 import feature_pb2 as lekko_dot_feature_dot_v1beta1_dot_feature__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n0lekko/backend/v1beta1/distribution_service.proto\x12\x15lekko.backend.v1beta1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a#lekko/feature/v1beta1/feature.proto\"K\n\rRepositoryKey\x12\x1d\n\nowner_name\x18\x01 \x01(\tR\townerName\x12\x1b\n\trepo_name\x18\x02 \x01(\tR\x08repoName\"\x7f\n\x1bGetRepositoryVersionRequest\x12?\n\x08repo_key\x18\x01 \x01(\x0b\x32$.lekko.backend.v1beta1.RepositoryKeyR\x07repoKey\x12\x1f\n\x0bsession_key\x18\x02 \x01(\tR\nsessionKey\"=\n\x1cGetRepositoryVersionResponse\x12\x1d\n\ncommit_sha\x18\x01 \x01(\tR\tcommitSha\"\xca\x01\n\x1cGetRepositoryContentsRequest\x12?\n\x08repo_key\x18\x01 \x01(\x0b\x32$.lekko.backend.v1beta1.RepositoryKeyR\x07repoKey\x12%\n\x0enamespace_name\x18\x02 \x01(\tR\rnamespaceName\x12!\n\x0c\x66\x65\x61ture_name\x18\x03 \x01(\tR\x0b\x66\x65\x61tureName\x12\x1f\n\x0bsession_key\x18\x04 \x01(\tR\nsessionKey\"\x80\x01\n\x1dGetRepositoryContentsResponse\x12\x1d\n\ncommit_sha\x18\x01 \x01(\tR\tcommitSha\x12@\n\nnamespaces\x18\x02 \x03(\x0b\x32 .lekko.backend.v1beta1.NamespaceR\nnamespaces\"[\n\tNamespace\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12:\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x1e.lekko.backend.v1beta1.FeatureR\x08\x66\x65\x61tures\"i\n\x07\x46\x65\x61ture\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n\x03sha\x18\x02 \x01(\tR\x03sha\x12\x38\n\x07\x66\x65\x61ture\x18\x03 \x01(\x0b\x32\x1e.lekko.feature.v1beta1.FeatureR\x07\x66\x65\x61ture\"2\n\nContextKey\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x12\n\x04type\x18\x02 \x01(\tR\x04type\"\x8f\x03\n\x13\x46lagEvaluationEvent\x12?\n\x08repo_key\x18\x01 \x01(\x0b\x32$.lekko.backend.v1beta1.RepositoryKeyR\x07repoKey\x12\x1d\n\ncommit_sha\x18\x02 \x01(\tR\tcommitSha\x12\x1f\n\x0b\x66\x65\x61ture_sha\x18\x03 \x01(\tR\nfeatureSha\x12%\n\x0enamespace_name\x18\x04 \x01(\tR\rnamespaceName\x12!\n\x0c\x66\x65\x61ture_name\x18\x05 \x01(\tR\x0b\x66\x65\x61tureName\x12\x44\n\x0c\x63ontext_keys\x18\x06 \x03(\x0b\x32!.lekko.backend.v1beta1.ContextKeyR\x0b\x63ontextKeys\x12\x1f\n\x0bresult_path\x18\x07 \x03(\x05R\nresultPath\x12\x46\n\x11\x63lient_event_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0f\x63lientEventTime\"\x87\x01\n SendFlagEvaluationMetricsRequest\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.lekko.backend.v1beta1.FlagEvaluationEventR\x06\x65vents\x12\x1f\n\x0bsession_key\x18\x02 \x01(\tR\nsessionKey\"#\n!SendFlagEvaluationMetricsResponse\"\xdc\x01\n\x15RegisterClientRequest\x12?\n\x08repo_key\x18\x01 \x01(\x0b\x32$.lekko.backend.v1beta1.RepositoryKeyR\x07repoKey\x12%\n\x0enamespace_list\x18\x02 \x03(\tR\rnamespaceList\x12\x32\n\x15initial_bootstrap_sha\x18\x03 \x01(\tR\x13initialBootstrapSha\x12\'\n\x0fsidecar_version\x18\x04 \x01(\tR\x0esidecarVersion\"9\n\x16RegisterClientResponse\x12\x1f\n\x0bsession_key\x18\x04 \x01(\tR\nsessionKey\":\n\x17\x44\x65registerClientRequest\x12\x1f\n\x0bsession_key\x18\x01 \x01(\tR\nsessionKey\"\x1a\n\x18\x44\x65registerClientResponse\" \n\x1eGetDeveloperAccessTokenRequest\"7\n\x1fGetDeveloperAccessTokenResponse\x12\x14\n\x05token\x18\x01 \x01(\tR\x05token2\xa8\x06\n\x13\x44istributionService\x12\x81\x01\n\x14GetRepositoryVersion\x12\x32.lekko.backend.v1beta1.GetRepositoryVersionRequest\x1a\x33.lekko.backend.v1beta1.GetRepositoryVersionResponse\"\x00\x12\x84\x01\n\x15GetRepositoryContents\x12\x33.lekko.backend.v1beta1.GetRepositoryContentsRequest\x1a\x34.lekko.backend.v1beta1.GetRepositoryContentsResponse\"\x00\x12\x90\x01\n\x19SendFlagEvaluationMetrics\x12\x37.lekko.backend.v1beta1.SendFlagEvaluationMetricsRequest\x1a\x38.lekko.backend.v1beta1.SendFlagEvaluationMetricsResponse\"\x00\x12o\n\x0eRegisterClient\x12,.lekko.backend.v1beta1.RegisterClientRequest\x1a-.lekko.backend.v1beta1.RegisterClientResponse\"\x00\x12u\n\x10\x44\x65registerClient\x12..lekko.backend.v1beta1.DeregisterClientRequest\x1a/.lekko.backend.v1beta1.DeregisterClientResponse\"\x00\x12\x8a\x01\n\x17GetDeveloperAccessToken\x12\x35.lekko.backend.v1beta1.GetDeveloperAccessTokenRequest\x1a\x36.lekko.backend.v1beta1.GetDeveloperAccessTokenResponse\"\x00\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lekko.backend.v1beta1.distribution_service_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_REPOSITORYKEY']._serialized_start=145 + _globals['_REPOSITORYKEY']._serialized_end=220 + _globals['_GETREPOSITORYVERSIONREQUEST']._serialized_start=222 + _globals['_GETREPOSITORYVERSIONREQUEST']._serialized_end=349 + _globals['_GETREPOSITORYVERSIONRESPONSE']._serialized_start=351 + _globals['_GETREPOSITORYVERSIONRESPONSE']._serialized_end=412 + _globals['_GETREPOSITORYCONTENTSREQUEST']._serialized_start=415 + _globals['_GETREPOSITORYCONTENTSREQUEST']._serialized_end=617 + _globals['_GETREPOSITORYCONTENTSRESPONSE']._serialized_start=620 + _globals['_GETREPOSITORYCONTENTSRESPONSE']._serialized_end=748 + _globals['_NAMESPACE']._serialized_start=750 + _globals['_NAMESPACE']._serialized_end=841 + _globals['_FEATURE']._serialized_start=843 + _globals['_FEATURE']._serialized_end=948 + _globals['_CONTEXTKEY']._serialized_start=950 + _globals['_CONTEXTKEY']._serialized_end=1000 + _globals['_FLAGEVALUATIONEVENT']._serialized_start=1003 + _globals['_FLAGEVALUATIONEVENT']._serialized_end=1402 + _globals['_SENDFLAGEVALUATIONMETRICSREQUEST']._serialized_start=1405 + _globals['_SENDFLAGEVALUATIONMETRICSREQUEST']._serialized_end=1540 + _globals['_SENDFLAGEVALUATIONMETRICSRESPONSE']._serialized_start=1542 + _globals['_SENDFLAGEVALUATIONMETRICSRESPONSE']._serialized_end=1577 + _globals['_REGISTERCLIENTREQUEST']._serialized_start=1580 + _globals['_REGISTERCLIENTREQUEST']._serialized_end=1800 + _globals['_REGISTERCLIENTRESPONSE']._serialized_start=1802 + _globals['_REGISTERCLIENTRESPONSE']._serialized_end=1859 + _globals['_DEREGISTERCLIENTREQUEST']._serialized_start=1861 + _globals['_DEREGISTERCLIENTREQUEST']._serialized_end=1919 + _globals['_DEREGISTERCLIENTRESPONSE']._serialized_start=1921 + _globals['_DEREGISTERCLIENTRESPONSE']._serialized_end=1947 + _globals['_GETDEVELOPERACCESSTOKENREQUEST']._serialized_start=1949 + _globals['_GETDEVELOPERACCESSTOKENREQUEST']._serialized_end=1981 + _globals['_GETDEVELOPERACCESSTOKENRESPONSE']._serialized_start=1983 + _globals['_GETDEVELOPERACCESSTOKENRESPONSE']._serialized_end=2038 + _globals['_DISTRIBUTIONSERVICE']._serialized_start=2041 + _globals['_DISTRIBUTIONSERVICE']._serialized_end=2849 +# @@protoc_insertion_point(module_scope) diff --git a/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.pyi b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.pyi new file mode 100644 index 0000000..ca28072 --- /dev/null +++ b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2.pyi @@ -0,0 +1,369 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Lekko Technologies, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import google.protobuf.timestamp_pb2 +import lekko_client.gen.lekko.feature.v1beta1.feature_pb2 +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class RepositoryKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OWNER_NAME_FIELD_NUMBER: builtins.int + REPO_NAME_FIELD_NUMBER: builtins.int + owner_name: builtins.str + repo_name: builtins.str + def __init__( + self, + *, + owner_name: builtins.str = ..., + repo_name: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["owner_name", b"owner_name", "repo_name", b"repo_name"]) -> None: ... + +global___RepositoryKey = RepositoryKey + +@typing_extensions.final +class GetRepositoryVersionRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REPO_KEY_FIELD_NUMBER: builtins.int + SESSION_KEY_FIELD_NUMBER: builtins.int + @property + def repo_key(self) -> global___RepositoryKey: ... + session_key: builtins.str + def __init__( + self, + *, + repo_key: global___RepositoryKey | None = ..., + session_key: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["repo_key", b"repo_key"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["repo_key", b"repo_key", "session_key", b"session_key"]) -> None: ... + +global___GetRepositoryVersionRequest = GetRepositoryVersionRequest + +@typing_extensions.final +class GetRepositoryVersionResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + COMMIT_SHA_FIELD_NUMBER: builtins.int + commit_sha: builtins.str + def __init__( + self, + *, + commit_sha: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["commit_sha", b"commit_sha"]) -> None: ... + +global___GetRepositoryVersionResponse = GetRepositoryVersionResponse + +@typing_extensions.final +class GetRepositoryContentsRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REPO_KEY_FIELD_NUMBER: builtins.int + NAMESPACE_NAME_FIELD_NUMBER: builtins.int + FEATURE_NAME_FIELD_NUMBER: builtins.int + SESSION_KEY_FIELD_NUMBER: builtins.int + @property + def repo_key(self) -> global___RepositoryKey: ... + namespace_name: builtins.str + """optional namespace_name to filter responses by""" + feature_name: builtins.str + """optional feature_name to filter responses by""" + session_key: builtins.str + def __init__( + self, + *, + repo_key: global___RepositoryKey | None = ..., + namespace_name: builtins.str = ..., + feature_name: builtins.str = ..., + session_key: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["repo_key", b"repo_key"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["feature_name", b"feature_name", "namespace_name", b"namespace_name", "repo_key", b"repo_key", "session_key", b"session_key"]) -> None: ... + +global___GetRepositoryContentsRequest = GetRepositoryContentsRequest + +@typing_extensions.final +class GetRepositoryContentsResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + COMMIT_SHA_FIELD_NUMBER: builtins.int + NAMESPACES_FIELD_NUMBER: builtins.int + commit_sha: builtins.str + @property + def namespaces(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Namespace]: ... + def __init__( + self, + *, + commit_sha: builtins.str = ..., + namespaces: collections.abc.Iterable[global___Namespace] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["commit_sha", b"commit_sha", "namespaces", b"namespaces"]) -> None: ... + +global___GetRepositoryContentsResponse = GetRepositoryContentsResponse + +@typing_extensions.final +class Namespace(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + FEATURES_FIELD_NUMBER: builtins.int + name: builtins.str + @property + def features(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Feature]: ... + def __init__( + self, + *, + name: builtins.str = ..., + features: collections.abc.Iterable[global___Feature] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["features", b"features", "name", b"name"]) -> None: ... + +global___Namespace = Namespace + +@typing_extensions.final +class Feature(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + SHA_FIELD_NUMBER: builtins.int + FEATURE_FIELD_NUMBER: builtins.int + name: builtins.str + sha: builtins.str + """The sha of the protobuf binary according to git.""" + @property + def feature(self) -> lekko_client.gen.lekko.feature.v1beta1.feature_pb2.Feature: ... + def __init__( + self, + *, + name: builtins.str = ..., + sha: builtins.str = ..., + feature: lekko_client.gen.lekko.feature.v1beta1.feature_pb2.Feature | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["feature", b"feature"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["feature", b"feature", "name", b"name", "sha", b"sha"]) -> None: ... + +global___Feature = Feature + +@typing_extensions.final +class ContextKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + key: builtins.str + type: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + type: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "type", b"type"]) -> None: ... + +global___ContextKey = ContextKey + +@typing_extensions.final +class FlagEvaluationEvent(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REPO_KEY_FIELD_NUMBER: builtins.int + COMMIT_SHA_FIELD_NUMBER: builtins.int + FEATURE_SHA_FIELD_NUMBER: builtins.int + NAMESPACE_NAME_FIELD_NUMBER: builtins.int + FEATURE_NAME_FIELD_NUMBER: builtins.int + CONTEXT_KEYS_FIELD_NUMBER: builtins.int + RESULT_PATH_FIELD_NUMBER: builtins.int + CLIENT_EVENT_TIME_FIELD_NUMBER: builtins.int + @property + def repo_key(self) -> global___RepositoryKey: ... + commit_sha: builtins.str + feature_sha: builtins.str + namespace_name: builtins.str + feature_name: builtins.str + @property + def context_keys(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ContextKey]: + """A list of context keys (not values) that were provided at runtime.""" + @property + def result_path(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The node in the tree that contained the final return value of the feature.""" + @property + def client_event_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + repo_key: global___RepositoryKey | None = ..., + commit_sha: builtins.str = ..., + feature_sha: builtins.str = ..., + namespace_name: builtins.str = ..., + feature_name: builtins.str = ..., + context_keys: collections.abc.Iterable[global___ContextKey] | None = ..., + result_path: collections.abc.Iterable[builtins.int] | None = ..., + client_event_time: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["client_event_time", b"client_event_time", "repo_key", b"repo_key"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["client_event_time", b"client_event_time", "commit_sha", b"commit_sha", "context_keys", b"context_keys", "feature_name", b"feature_name", "feature_sha", b"feature_sha", "namespace_name", b"namespace_name", "repo_key", b"repo_key", "result_path", b"result_path"]) -> None: ... + +global___FlagEvaluationEvent = FlagEvaluationEvent + +@typing_extensions.final +class SendFlagEvaluationMetricsRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EVENTS_FIELD_NUMBER: builtins.int + SESSION_KEY_FIELD_NUMBER: builtins.int + @property + def events(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FlagEvaluationEvent]: ... + session_key: builtins.str + def __init__( + self, + *, + events: collections.abc.Iterable[global___FlagEvaluationEvent] | None = ..., + session_key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["events", b"events", "session_key", b"session_key"]) -> None: ... + +global___SendFlagEvaluationMetricsRequest = SendFlagEvaluationMetricsRequest + +@typing_extensions.final +class SendFlagEvaluationMetricsResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___SendFlagEvaluationMetricsResponse = SendFlagEvaluationMetricsResponse + +@typing_extensions.final +class RegisterClientRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REPO_KEY_FIELD_NUMBER: builtins.int + NAMESPACE_LIST_FIELD_NUMBER: builtins.int + INITIAL_BOOTSTRAP_SHA_FIELD_NUMBER: builtins.int + SIDECAR_VERSION_FIELD_NUMBER: builtins.int + @property + def repo_key(self) -> global___RepositoryKey: ... + @property + def namespace_list(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """The namespaces to register within the repo. If empty, + all namespaces will be registered. + """ + initial_bootstrap_sha: builtins.str + """If the client was initialized from a git bootstrap, + the commit sha is provided. If there was no bootstrap, this + can be an empty string. + """ + sidecar_version: builtins.str + """If the client is a lekko sidecar, provide the semver version, + or if not available, the sha of the sidecar. + """ + def __init__( + self, + *, + repo_key: global___RepositoryKey | None = ..., + namespace_list: collections.abc.Iterable[builtins.str] | None = ..., + initial_bootstrap_sha: builtins.str = ..., + sidecar_version: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["repo_key", b"repo_key"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["initial_bootstrap_sha", b"initial_bootstrap_sha", "namespace_list", b"namespace_list", "repo_key", b"repo_key", "sidecar_version", b"sidecar_version"]) -> None: ... + +global___RegisterClientRequest = RegisterClientRequest + +@typing_extensions.final +class RegisterClientResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_KEY_FIELD_NUMBER: builtins.int + session_key: builtins.str + """TODO make this field 1 if we rewrite the API.""" + def __init__( + self, + *, + session_key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_key", b"session_key"]) -> None: ... + +global___RegisterClientResponse = RegisterClientResponse + +@typing_extensions.final +class DeregisterClientRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_KEY_FIELD_NUMBER: builtins.int + session_key: builtins.str + def __init__( + self, + *, + session_key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_key", b"session_key"]) -> None: ... + +global___DeregisterClientRequest = DeregisterClientRequest + +@typing_extensions.final +class DeregisterClientResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___DeregisterClientResponse = DeregisterClientResponse + +@typing_extensions.final +class GetDeveloperAccessTokenRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___GetDeveloperAccessTokenRequest = GetDeveloperAccessTokenRequest + +@typing_extensions.final +class GetDeveloperAccessTokenResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TOKEN_FIELD_NUMBER: builtins.int + token: builtins.str + """github access token""" + def __init__( + self, + *, + token: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["token", b"token"]) -> None: ... + +global___GetDeveloperAccessTokenResponse = GetDeveloperAccessTokenResponse diff --git a/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2_grpc.py b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2_grpc.py new file mode 100644 index 0000000..019bc44 --- /dev/null +++ b/lekko_client/gen/lekko/backend/v1beta1/distribution_service_pb2_grpc.py @@ -0,0 +1,246 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from lekko_client.gen.lekko.backend.v1beta1 import distribution_service_pb2 as lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2 + + +class DistributionServiceStub(object): + """Initial implementation of a config distribution service. Clients should begin by + calling the register RPC which returns a session key, which is used in all other RPCs. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetRepositoryVersion = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/GetRepositoryVersion', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionResponse.FromString, + ) + self.GetRepositoryContents = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/GetRepositoryContents', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsResponse.FromString, + ) + self.SendFlagEvaluationMetrics = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/SendFlagEvaluationMetrics', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsResponse.FromString, + ) + self.RegisterClient = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/RegisterClient', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientResponse.FromString, + ) + self.DeregisterClient = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/DeregisterClient', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientResponse.FromString, + ) + self.GetDeveloperAccessToken = channel.unary_unary( + '/lekko.backend.v1beta1.DistributionService/GetDeveloperAccessToken', + request_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenRequest.SerializeToString, + response_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenResponse.FromString, + ) + + +class DistributionServiceServicer(object): + """Initial implementation of a config distribution service. Clients should begin by + calling the register RPC which returns a session key, which is used in all other RPCs. + """ + + def GetRepositoryVersion(self, request, context): + """Returns the latest commit sha of the repository. The client is expected to poll this + rpc to become aware of updates. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetRepositoryContents(self, request, context): + """Returns the entire state of the repository, including all feature flags. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SendFlagEvaluationMetrics(self, request, context): + """Sends metrics to the backend related to flag evaluation. This rpc can be used + to batch metrics to lekko servers. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def RegisterClient(self, request, context): + """Registers a client and returns a session key. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeregisterClient(self, request, context): + """Deregisters a client using a session key. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetDeveloperAccessToken(self, request, context): + """Returns a github access token that provides access to the contents of + some of Lekko's private repositories + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DistributionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetRepositoryVersion': grpc.unary_unary_rpc_method_handler( + servicer.GetRepositoryVersion, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionResponse.SerializeToString, + ), + 'GetRepositoryContents': grpc.unary_unary_rpc_method_handler( + servicer.GetRepositoryContents, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsResponse.SerializeToString, + ), + 'SendFlagEvaluationMetrics': grpc.unary_unary_rpc_method_handler( + servicer.SendFlagEvaluationMetrics, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsResponse.SerializeToString, + ), + 'RegisterClient': grpc.unary_unary_rpc_method_handler( + servicer.RegisterClient, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientResponse.SerializeToString, + ), + 'DeregisterClient': grpc.unary_unary_rpc_method_handler( + servicer.DeregisterClient, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientResponse.SerializeToString, + ), + 'GetDeveloperAccessToken': grpc.unary_unary_rpc_method_handler( + servicer.GetDeveloperAccessToken, + request_deserializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenRequest.FromString, + response_serializer=lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'lekko.backend.v1beta1.DistributionService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class DistributionService(object): + """Initial implementation of a config distribution service. Clients should begin by + calling the register RPC which returns a session key, which is used in all other RPCs. + """ + + @staticmethod + def GetRepositoryVersion(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/GetRepositoryVersion', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryVersionResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetRepositoryContents(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/GetRepositoryContents', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetRepositoryContentsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SendFlagEvaluationMetrics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/SendFlagEvaluationMetrics', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.SendFlagEvaluationMetricsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def RegisterClient(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/RegisterClient', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.RegisterClientResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def DeregisterClient(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/DeregisterClient', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.DeregisterClientResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetDeveloperAccessToken(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/lekko.backend.v1beta1.DistributionService/GetDeveloperAccessToken', + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenRequest.SerializeToString, + lekko_dot_backend_dot_v1beta1_dot_distribution__service__pb2.GetDeveloperAccessTokenResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/lekko_client/helpers.py b/lekko_client/helpers.py index a8bf31c..ef2fb4c 100644 --- a/lekko_client/helpers.py +++ b/lekko_client/helpers.py @@ -39,17 +39,20 @@ def intercept(self, method, request_or_iterator, call_details): return method(request_or_iterator, new_details) -_CHANNELS: Dict[Tuple[str, str], grpc.Channel] = {} +_CHANNELS: Dict[Tuple[str, Optional[str]], grpc.Channel] = {} -def get_grpc_channel(url: str, api_key: str, credentials: Optional[grpc.ChannelCredentials] = None) -> grpc.Channel: +def get_grpc_channel( + url: str, api_key: Optional[str] = None, credentials: Optional[grpc.ChannelCredentials] = None +) -> grpc.Channel: if (url, api_key) not in _CHANNELS: if credentials: channel = grpc.secure_channel(url, credentials) else: channel = grpc.insecure_channel(url) - channel = grpc.intercept_channel(channel, *[ApiKeyInterceptor(api_key)]) + if api_key: + channel = grpc.intercept_channel(channel, *[ApiKeyInterceptor(api_key)]) _CHANNELS[(url, api_key)] = channel return _CHANNELS[(url, api_key)] diff --git a/lekko_client/models.py b/lekko_client/models.py new file mode 100644 index 0000000..7b06611 --- /dev/null +++ b/lekko_client/models.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature + + +@dataclass +class FeatureData: + config_sha: str + feature: Feature diff --git a/lekko_client/stores/__init__.py b/lekko_client/stores/__init__.py new file mode 100644 index 0000000..bad8c99 --- /dev/null +++ b/lekko_client/stores/__init__.py @@ -0,0 +1,2 @@ +from lekko_client.stores.memory import MemoryStore # noqa +from lekko_client.stores.store import Store # noqa diff --git a/lekko_client/stores/memory.py b/lekko_client/stores/memory.py new file mode 100644 index 0000000..b415a35 --- /dev/null +++ b/lekko_client/stores/memory.py @@ -0,0 +1,31 @@ +from typing import Dict + +from lekko_client.models import FeatureData +from lekko_client.stores.store import Store + + +class MemoryStore(Store): + def __init__(self): + super().__init__() + self.configs: Dict[str, Dict[str, FeatureData]] = {} + + def get(self, namespace: str, config_key: str) -> FeatureData: + namespace_map = self.configs.get(namespace) + if not namespace_map: + raise Exception("namespace not found") + result = namespace_map.get(config_key) + if not result: + raise Exception("config not found") + return result + + def load_impl(self, contents) -> bool: + super().load(contents) + new_configs = {} + for ns in contents.namespaces: + namespace_map = {} + for cfg in ns.features: + if cfg.feature: + namespace_map[cfg.name] = FeatureData(cfg.sha, cfg.feature) + new_configs[ns.name] = namespace_map + self.configs = new_configs + return True diff --git a/lekko_client/stores/store.py b/lekko_client/stores/store.py new file mode 100644 index 0000000..1e1fc0a --- /dev/null +++ b/lekko_client/stores/store.py @@ -0,0 +1,60 @@ +from abc import ABC, abstractmethod +from hashlib import sha256 + +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsResponse, +) +from lekko_client.models import FeatureData + + +class Store(ABC): + def __init__(self): + self._commit_sha = "" + self._content_hash = "" + + @abstractmethod + def get(self, namespace: str, config_key: str) -> FeatureData: + ... + + def load(self, contents: GetRepositoryContentsResponse) -> bool: + if not contents: + return False + + contents = self.sort_contents(contents) + content_hash = self.hash_contents(contents) + + if not self.should_update(contents, content_hash): + return False + + if not self.load_impl(contents): + return False + + self._commit_sha = contents.commit_sha + self._content_hash = content_hash + return True + + @abstractmethod + def load_impl(self, contents: GetRepositoryContentsResponse) -> bool: + ... + + @property + def commit_sha(self) -> str: + return self._commit_sha + + @property + def content_hash(self) -> str: + return self._content_hash + + def should_update(self, contents: GetRepositoryContentsResponse, content_hash: str): + return contents.commit_sha != self.commit_sha or content_hash != self.content_hash + + @classmethod + def sort_contents(cls, contents: GetRepositoryContentsResponse): + for ns in contents.namespaces: + ns.features.sort(key=lambda cfg: cfg.name) + contents.namespaces.sort(key=lambda ns: ns.name) + return contents + + @classmethod + def hash_contents(cls, contents: GetRepositoryContentsResponse): + return sha256(contents.SerializeToString()).hexdigest() diff --git a/pyproject.toml b/pyproject.toml index d77a4ad..092e949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,9 @@ dependencies = [ 'grpcio-tools ~= 1.32', 'grpc-interceptor ~= 0.15', 'xxhash ~= 3.0', + 'pyyaml ~= 6.0', + 'dulwich ~= 0.21', + 'watchdog ~= 3.0', ] [project.optional-dependencies] diff --git a/tests/conftest.py b/tests/conftest.py index bab163b..87aa011 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ def test_thread(): def test_channel_no_interceptor(test_thread): channel = grpc_testing.channel(DESCRIPTOR.services_by_name.values(), grpc_testing.strict_real_time()) try: - with mock.patch("lekko_client.client.get_grpc_channel", return_value=channel): + with mock.patch("lekko_client.clients.grpc_client.get_grpc_channel", return_value=channel): yield channel finally: channel.close() diff --git a/tests/test_client.py b/tests/test_client.py index 1bcf7b8..a81064e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -7,13 +7,12 @@ from google.protobuf import wrappers_pb2 from google.protobuf.any_pb2 import Any -from lekko_client.client import ( - APIClient, +from lekko_client.clients import APIClient, SidecarClient +from lekko_client.exceptions import ( AuthenticationError, FeatureNotFound, MismatchedProtoType, MismatchedType, - SidecarClient, ) from lekko_client.gen.lekko.client.v1beta1 import configuration_service_pb2 as messages @@ -42,17 +41,18 @@ def test_get_scalar(test_server, proto_fn_name, response_obj, response_val, test api_key = "lekko_apikey123" context = {"ctx_key": response_val} - client = SidecarClient(owner_name, repo_name, namespace, api_key) - resp = getattr(client, test_fn_name)(feature_name, context) + client = SidecarClient(owner_name, repo_name, api_key) + resp = getattr(client, test_fn_name)(namespace, feature_name, context) assert resp == response_val completed_requests = async_requests.result() assert len(completed_requests) == 2 assert completed_requests[0].arg.repo_key.owner_name == owner_name assert completed_requests[0].arg.repo_key.repo_name == repo_name - assert completed_requests[0].arg.namespace_list == [namespace] + assert completed_requests[0].arg.namespace_list == [] assert ("apikey", api_key) in completed_requests[0].metadata assert completed_requests[1].arg.key == feature_name + assert completed_requests[1].arg.namespace == namespace req_ctx = {k: getattr(v, v.WhichOneof("kind")) for k, v in completed_requests[1].arg.context.items()} assert req_ctx == context assert ("apikey", api_key) in completed_requests[1].metadata @@ -68,8 +68,8 @@ def test_get_json(test_server): ] test_server.mock_async_responses(requests) - client = SidecarClient("owner", "repo", "namespace", "lekko_apikey123") - resp = client.get_json("val", {}) + client = SidecarClient("owner", "repo", "lekko_apikey123") + resp = client.get_json("val", "namespace", {}) assert resp == expected @@ -85,13 +85,13 @@ def test_get_proto_by_type(test_server): ] test_server.mock_async_responses(requests) - client = SidecarClient("owner", "repo", "namespace", "lekko_apikey123") - resp = client.get_proto_by_type("val", {}, wrappers_pb2.Int32Value) + client = SidecarClient("owner", "repo", "lekko_apikey123") + resp = client.get_proto_by_type("val", "namespace", {}, wrappers_pb2.Int32Value) assert resp == int_proto with pytest.raises(MismatchedProtoType): - client.get_proto_by_type("val", {}, wrappers_pb2.Int64Value) + client.get_proto_by_type("val", "namespace", {}, wrappers_pb2.Int64Value) def test_get_proto(test_server): @@ -110,23 +110,23 @@ def test_get_proto(test_server): ] test_server.mock_async_responses(requests) - client = SidecarClient("owner", "repo", "namespace", "lekko_apikey123") + client = SidecarClient("owner", "repo", "lekko_apikey123") # When the proto symbol is loaded in the db, it should Unpack correctly - resp = client.get_proto("val", {}) + resp = client.get_proto("val", "namespace", {}) assert resp == int_proto # If the proto symbol can't be found, we fall through to returning the Any proto with mock.patch("google.protobuf.symbol_database.SymbolDatabase.GetSymbol", side_effect=KeyError): - resp = client.get_proto("val", {}) + resp = client.get_proto("val", "namespace", {}) assert resp == any_proto # test get proto with value and value_v2 being returned - resp = client.get_proto("val", {}) + resp = client.get_proto("val", "namespace", {}) assert resp == int_proto # test get proto with only value_v2 being returned - resp = client.get_proto("val", {}) + resp = client.get_proto("val", "namespace", {}) assert resp == int_proto @@ -139,11 +139,11 @@ def test_missing_api_key(test_server): test_server.mock_async_responses(requests) with pytest.raises(AuthenticationError): - SidecarClient("owner", "repo", "namespace") + SidecarClient("owner", "repo") os.environ["LEKKO_API_KEY"] = "lekko_apikey123" - client = SidecarClient("owner", "repo", "namespace") - result = client.get_bool("key", {}) + client = SidecarClient("owner", "repo") + result = client.get_bool("key", "namespace", {}) assert result == expected @@ -167,12 +167,12 @@ def test_errors(test_server_no_interceptor): test_server_no_interceptor.mock_async_responses(requests) - client = SidecarClient("owner", "repo", "namespace", "lekko_apikey123") + client = SidecarClient("owner", "repo", "lekko_apikey123") with pytest.raises(FeatureNotFound): - client.get_bool("key", {}) + client.get_bool("key", "namespace", {}) with pytest.raises(MismatchedType): - client.get_bool("key", {}) + client.get_bool("key", "namespace", {}) def test_proto_errors(test_server_no_interceptor): @@ -200,15 +200,15 @@ def test_proto_errors(test_server_no_interceptor): test_server_no_interceptor.mock_async_responses(requests) - client = SidecarClient("owner", "repo", "namespace", "lekko_apikey123") + client = SidecarClient("owner", "repo", "lekko_apikey123") with pytest.raises(FeatureNotFound): - client.get_proto("key", {}) + client.get_proto("key", "namespace", {}) with pytest.raises(MismatchedType): - client.get_proto("key", {}) + client.get_proto("key", "namespace", {}) with pytest.raises(grpc.RpcError): - client.get_proto("key", {}) + client.get_proto("key", "namespace", {}) def test_get_api_client(test_server): @@ -218,7 +218,7 @@ def test_get_api_client(test_server): ] test_server.mock_async_responses(requests) - client = APIClient("owner", "repo", "namespace", "lekko_apikey123") - resp = client.get_string("val", {"conext": "hello"}) + client = APIClient("owner", "repo", "lekko_apikey123") + resp = client.get_string("val", "namespace", {"conext": "hello"}) assert resp == "feature value" From 72bdb659dd098516665d067b80e0ed707628dfe6 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Sun, 10 Sep 2023 12:39:45 -0700 Subject: [PATCH 08/28] add git path --- lekko_client/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 48565da..d259f25 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -34,6 +34,7 @@ def initialize( repo_name: str, api_key: Optional[str] = None, context: Optional[Dict[str, Any]] = None, + git_repo_path: Optional[str] = None, ) -> Client: global __client if mode == Mode.API: @@ -41,7 +42,9 @@ def initialize( elif mode == Mode.SIDECAR: __client = SidecarClient(owner_name, repo_name, api_key, context) elif mode == Mode.CACHED_GIT: - __client = CachedGitClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), "", api_key, context) + if not git_repo_path: + raise ValueError("Must provide a path to git repo") + __client = CachedGitClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), git_repo_path, api_key, context) elif mode == Mode.CACHED_SERVER: __client = CachedBackendClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), api_key, context) else: From c50e50005ebc24214d8f5b03969257db677f2d3d Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Sun, 10 Sep 2023 13:08:24 -0700 Subject: [PATCH 09/28] cachedgit bug fixes --- lekko_client/clients/cached_backend_client.py | 2 +- lekko_client/clients/cached_git_client.py | 7 ++++--- lekko_client/stores/memory.py | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index baa0873..f8e00fa 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -39,10 +39,10 @@ def __init__( context: Optional[Dict[str, Any]] = None, credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), ): - super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) self.timeout = None self.closed = False self.update_interval = 1000 + super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) def initialize(self): self.update_store() diff --git a/lekko_client/clients/cached_git_client.py b/lekko_client/clients/cached_git_client.py index f5cb6b3..2ca3d99 100644 --- a/lekko_client/clients/cached_git_client.py +++ b/lekko_client/clients/cached_git_client.py @@ -47,10 +47,10 @@ def __init__( credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), should_watch: Optional[bool] = True, ): - super().__init__(lekko_uri, repository_owner, repository_name, store, api_key, context, credentials) self.watcher: Optional[BaseObserver] = None self.path = path self.should_watch = should_watch + super().__init__(lekko_uri, repository_owner, repository_name, store, api_key, context, credentials) def initialize(self): super().initialize() @@ -86,15 +86,16 @@ def get_configs(self, repo: GitRepo, ns_name: str) -> List[DistFeature]: features = [] for proto_bin_file in glob.glob(os.path.join(proto_dir_path, "*.proto.bin")): + proto_bin_relative_filename = os.path.relpath(proto_bin_file, self.path) with open(proto_bin_file, "rb") as proto_bin: _, sha = tree_lookup_path( - repo.get_object, repo[repo.head()].tree, proto_bin_file.encode() # type: ignore + repo.get_object, repo[repo.head()].tree, proto_bin_relative_filename.encode() # type: ignore ) feature = Feature() feature.ParseFromString(proto_bin.read()) features.append( DistFeature( - name=proto_bin_file.replace(".proto.bin", ""), + name=os.path.basename(proto_bin_file).replace(".proto.bin", ""), sha=sha.decode("utf-8"), feature=feature, ) diff --git a/lekko_client/stores/memory.py b/lekko_client/stores/memory.py index b415a35..f947bfb 100644 --- a/lekko_client/stores/memory.py +++ b/lekko_client/stores/memory.py @@ -19,7 +19,6 @@ def get(self, namespace: str, config_key: str) -> FeatureData: return result def load_impl(self, contents) -> bool: - super().load(contents) new_configs = {} for ns in contents.namespaces: namespace_map = {} From ac6380d8a34e3c22c4c124c76f77ac5892700bc6 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Wed, 13 Sep 2023 09:51:52 -0700 Subject: [PATCH 10/28] fix cached server client --- lekko_client/clients/cached_backend_client.py | 6 ++++-- lekko_client/stores/store.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index f8e00fa..73533a8 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -6,7 +6,9 @@ from lekko_client.clients.distribution_client import CachedDistributionClient from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsRequest, GetRepositoryContentsResponse, + GetRepositoryVersionRequest, ) from lekko_client.stores.store import Store @@ -53,7 +55,7 @@ def initialize(self): def get_contents(self) -> Optional[GetRepositoryContentsResponse]: if not self._client: return None - return self._client.GetRepositoryContents(repo_key=self.repository, session_key=self.session_key) + return self._client.GetRepositoryContents(GetRepositoryContentsRequest(repo_key=self.repository, session_key=self.session_key)) def update_store(self): self.load() @@ -61,7 +63,7 @@ def update_store(self): def should_update_store(self): if not self._client: return - version_response = self._client.GetRepositoryVersion(repo_key=self.repository, session_key=self.session_key) + version_response = self._client.GetRepositoryVersion(GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key)) current_sha = self.store.commit_sha return current_sha != version_response.commit_sha diff --git a/lekko_client/stores/store.py b/lekko_client/stores/store.py index 1e1fc0a..cd5400c 100644 --- a/lekko_client/stores/store.py +++ b/lekko_client/stores/store.py @@ -46,7 +46,8 @@ def content_hash(self) -> str: return self._content_hash def should_update(self, contents: GetRepositoryContentsResponse, content_hash: str): - return contents.commit_sha != self.commit_sha or content_hash != self.content_hash + ret = contents.commit_sha != self.commit_sha or content_hash != self.content_hash + return ret @classmethod def sort_contents(cls, contents: GetRepositoryContentsResponse): From 3ffff9d018f18c74b65fb20a87df4cb2236619bb Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Wed, 13 Sep 2023 10:39:01 -0700 Subject: [PATCH 11/28] add event tracking --- lekko_client/clients/cached_backend_client.py | 8 +- lekko_client/clients/distribution_client.py | 76 ++++++++++++++++++- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index 73533a8..ef3d454 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -55,7 +55,9 @@ def initialize(self): def get_contents(self) -> Optional[GetRepositoryContentsResponse]: if not self._client: return None - return self._client.GetRepositoryContents(GetRepositoryContentsRequest(repo_key=self.repository, session_key=self.session_key)) + return self._client.GetRepositoryContents( + GetRepositoryContentsRequest(repo_key=self.repository, session_key=self.session_key) + ) def update_store(self): self.load() @@ -63,7 +65,9 @@ def update_store(self): def should_update_store(self): if not self._client: return - version_response = self._client.GetRepositoryVersion(GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key)) + version_response = self._client.GetRepositoryVersion( + GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key) + ) current_sha = self.store.commit_sha return current_sha != version_response.commit_sha diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 11990fa..a0490f4 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -1,6 +1,9 @@ import json +import time from abc import abstractmethod -from typing import Any, Dict, Optional, Type, TypeVar, Union +from datetime import datetime +from threading import Thread +from typing import Any, Dict, List, Optional, Type, TypeVar, Union import grpc from google.protobuf import descriptor_pool as proto_descriptor_pool @@ -9,25 +12,71 @@ from google.protobuf.json_format import MessageToJson from google.protobuf.message import Message as ProtoMessage from google.protobuf.struct_pb2 import Value +from google.protobuf.timestamp_pb2 import Timestamp from google.protobuf.wrappers_pb2 import BoolValue, FloatValue, Int64Value, StringValue from lekko_client.clients.client import Client -from lekko_client.evaluation.evaluation import evaluate +from lekko_client.evaluation.evaluation import EvaluationResult, evaluate +from lekko_client.evaluation.rules import ClientContext from lekko_client.exceptions import MismatchedProtoType from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + ContextKey, DeregisterClientRequest, + FlagEvaluationEvent, GetRepositoryContentsResponse, RegisterClientRequest, RepositoryKey, + SendFlagEvaluationMetricsRequest, ) from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2_grpc import ( DistributionServiceStub, ) +from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import ( + Value as LekkoValue, +) from lekko_client.helpers import convert_context, get_grpc_channel +from lekko_client.models import FeatureData from lekko_client.stores.store import Store class CachedDistributionClient(Client): + class EventsBatcher(Thread): + def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval: int): + super().__init__() + self.daemon = True + self.dist_client = dist_client + self.upload_interval = upload_interval + self.session_key = session_key + self.events: List[FlagEvaluationEvent] = [] + self._enabled = True + + def stop(self): + self._enabled = False + + def add_event(self, event: FlagEvaluationEvent): + self.events.append(event) + + def run(self): + # TODO: Lock + while self._enabled: + if self.events: + self.dist_client.SendFlagEvaluationMetrics( + SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) + ) + self.events = [] + time.sleep(self.upload_interval / 1000) + + @classmethod + def get_value_type(cls, val: LekkoValue) -> str: + return (val.WhichOneof("kind") or "").removesuffix("_value") + + @classmethod + def get_context_keys(cls, context: Optional[ClientContext] = None) -> List[ContextKey]: + if not context: + return [] + + return [ContextKey(key=k, type=cls.get_value_type(v)) for k, v in context.items()] + def __init__( self, uri: str, @@ -52,8 +101,9 @@ def __init__( RegisterClientRequest(repo_key=self.repository, sidecar_version=__version__) ) self.session_key = register_response.session_key - # if self.events_batcher: - # await self.events_batcher.init(self.session_key) + self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15) + self.events_batcher.start() + self.initialize() _TYPE_MAPPING: Dict[Type, Type[Union[BoolValue, Int64Value, StringValue, FloatValue]]] = { @@ -78,9 +128,27 @@ def load(self) -> bool: def get_contents(self) -> Optional[GetRepositoryContentsResponse]: ... + def track( + self, namespace: str, feature_data: FeatureData, result: EvaluationResult, context: Optional[ClientContext] + ) -> None: + timestamp = Timestamp() + timestamp.FromDatetime(datetime.utcnow()) + event = FlagEvaluationEvent( + repo_key=self.repository, + commit_sha=self.store.commit_sha, + feature_sha=feature_data.config_sha, + namespace_name=namespace, + feature_name=feature_data.feature.key, + context_keys=self.events_batcher.get_context_keys(context), + result_path=result.path, + client_event_time=timestamp, + ) + self.events_batcher.add_event(event) + def _get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: feature_data = self.store.get(namespace, key) result = evaluate(feature_data.feature, namespace, convert_context(context)) + self.track(namespace, feature_data, result, context) return result.value ReturnType = TypeVar("ReturnType", str, float, int, bool) From 8b755e585659fddcb9db3442825d1f16fb95a89b Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Wed, 13 Sep 2023 11:24:52 -0700 Subject: [PATCH 12/28] shutdown event upload thread --- lekko_client/clients/distribution_client.py | 29 +++++++-------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index a0490f4..4b71311 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -56,14 +56,17 @@ def stop(self): def add_event(self, event: FlagEvaluationEvent): self.events.append(event) + def upload_events(self): + if self.events: + self.dist_client.SendFlagEvaluationMetrics( + SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) + ) + self.events = [] + def run(self): # TODO: Lock while self._enabled: - if self.events: - self.dist_client.SendFlagEvaluationMetrics( - SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) - ) - self.events = [] + self.upload_events() time.sleep(self.upload_interval / 1000) @classmethod @@ -202,20 +205,8 @@ def get_proto_by_type( raise MismatchedProtoType(f"Error unpacking from {val.type_url} to {proto_message_type.DESCRIPTOR.name}") - # def track(self, namespace: str, key: str, result: StoredEvalResult, ctx: Optional[ClientContext] = None): - # if not self.events_batcher: - # return - # self.events_batcher.track(FlagEvaluationEvent( - # repo_key=self.repo_key, - # commit_sha=result.commit_sha, - # feature_sha=result.config_sha, - # namespace_name=namespace, - # feature_name=key, - # context_keys=to_context_keys_proto(ctx), - # result_path=result.eval_result.path, - # client_event_time=Timestamp.now() - # )) - def close(self): if self._client and self.session_key: + self.events_batcher.upload_events() + self.events_batcher.stop() self._client.DeregisterClient(DeregisterClientRequest(session_key=self.session_key)) From fcaee632b3cf9f1c5b46a09495ffe61adc5171e3 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Thu, 14 Sep 2023 11:26:46 -0700 Subject: [PATCH 13/28] better exceptions --- lekko_client/__init__.py | 6 ++-- lekko_client/clients/cached_backend_client.py | 11 +++---- lekko_client/clients/cached_git_client.py | 3 +- lekko_client/clients/distribution_client.py | 15 +++++---- lekko_client/evaluation/evaluation.py | 5 +-- lekko_client/evaluation/rules.py | 33 ++++++++++--------- lekko_client/exceptions.py | 12 +++++++ lekko_client/stores/memory.py | 5 +-- tests/evaluation/test_evaluation.py | 2 +- 9 files changed, 54 insertions(+), 38 deletions(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index d259f25..0b42448 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -5,6 +5,7 @@ from google.protobuf.message import Message as ProtoMessage +from lekko_client import exceptions from lekko_client.clients import ( APIClient, CachedBackendClient, @@ -13,7 +14,6 @@ SidecarClient, ) from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa -from lekko_client.exceptions import * # noqa from lekko_client.stores import MemoryStore __version__ = "0.1.4" @@ -43,12 +43,12 @@ def initialize( __client = SidecarClient(owner_name, repo_name, api_key, context) elif mode == Mode.CACHED_GIT: if not git_repo_path: - raise ValueError("Must provide a path to git repo") + raise exceptions.GitRepoNotFound("Must provide a path to git repo in Cached Git mode") __client = CachedGitClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), git_repo_path, api_key, context) elif mode == Mode.CACHED_SERVER: __client = CachedBackendClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), api_key, context) else: - raise ValueError("Invalid mode") + raise exceptions.LekkoError("Unknown client mode") return __client diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index ef3d454..f9cefb3 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -15,11 +15,11 @@ class CachedBackendClient(CachedDistributionClient): class RefreshThread(Thread): - def __init__(self, client: "CachedBackendClient", refresh_interval: int): + def __init__(self, client: "CachedBackendClient", refresh_interval_ms: int): super().__init__() self.daemon = True self.client = client - self.refresh_interval = refresh_interval + self.refresh_interval = refresh_interval_ms self._enabled = True def stop(self): @@ -43,14 +43,13 @@ def __init__( ): self.timeout = None self.closed = False - self.update_interval = 1000 + self.update_interval_ms = 1000 super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) def initialize(self): self.update_store() - if self.update_interval: - self.refresh_thread = CachedBackendClient.RefreshThread(self, self.update_interval) - self.refresh_thread.start() + self.refresh_thread = CachedBackendClient.RefreshThread(self, self.update_interval_ms) + self.refresh_thread.start() def get_contents(self) -> Optional[GetRepositoryContentsResponse]: if not self._client: diff --git a/lekko_client/clients/cached_git_client.py b/lekko_client/clients/cached_git_client.py index 2ca3d99..ae85263 100644 --- a/lekko_client/clients/cached_git_client.py +++ b/lekko_client/clients/cached_git_client.py @@ -12,6 +12,7 @@ from watchdog.observers.api import BaseObserver from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.exceptions import GitRepoNotFound from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( Feature as DistFeature, ) @@ -65,7 +66,7 @@ def get_contents(self) -> GetRepositoryContentsResponse: try: repo = GitRepo(self.path) except NotGitRepository: - raise ValueError("Not a git repository") + raise GitRepoNotFound(f"{self.path} is not a git repository") return GetRepositoryContentsResponse( commit_sha=repo.head().decode("utf-8"), namespaces=self.get_namespaces(repo) diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 4b71311..e0610ac 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -18,7 +18,7 @@ from lekko_client.clients.client import Client from lekko_client.evaluation.evaluation import EvaluationResult, evaluate from lekko_client.evaluation.rules import ClientContext -from lekko_client.exceptions import MismatchedProtoType +from lekko_client.exceptions import MismatchedProtoType, MismatchedType from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( ContextKey, DeregisterClientRequest, @@ -41,11 +41,11 @@ class CachedDistributionClient(Client): class EventsBatcher(Thread): - def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval: int): + def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval_ms: int): super().__init__() self.daemon = True self.dist_client = dist_client - self.upload_interval = upload_interval + self.upload_interval = upload_interval_ms self.session_key = session_key self.events: List[FlagEvaluationEvent] = [] self._enabled = True @@ -67,7 +67,7 @@ def run(self): # TODO: Lock while self._enabled: self.upload_events() - time.sleep(self.upload_interval / 1000) + time.sleep(self.upload_interval) @classmethod def get_value_type(cls, val: LekkoValue) -> str: @@ -104,7 +104,7 @@ def __init__( RegisterClientRequest(repo_key=self.repository, sidecar_version=__version__) ) self.session_key = register_response.session_key - self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15) + self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15 * 1000) self.events_batcher.start() self.initialize() @@ -159,8 +159,9 @@ def _get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: def _get_scalar(self, namespace: str, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: result = self._get(namespace, key, context) return_wrapper = self._TYPE_MAPPING[typ]() - result.Unpack(return_wrapper) - return return_wrapper.value # type:ignore + if result.Unpack(return_wrapper): + return return_wrapper.value # type:ignore + raise MismatchedType(f"Feature {key} is of type {result.type_url} and cannot be converted to {typ}") def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: return self._get_scalar(namespace, key, context, bool) diff --git a/lekko_client/evaluation/evaluation.py b/lekko_client/evaluation/evaluation.py index c87e78b..8ed42b1 100644 --- a/lekko_client/evaluation/evaluation.py +++ b/lekko_client/evaluation/evaluation.py @@ -4,6 +4,7 @@ from google.protobuf.any_pb2 import Any as ProtoAny from lekko_client.evaluation.rules import ClientContext, evaluate_rule +from lekko_client.exceptions import EvaluationError from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Constraint, Feature @@ -25,7 +26,7 @@ class TraverseResult: def evaluate(config: Feature, namespace: str, context: ClientContext = None) -> EvaluationResult: if not config.HasField("tree"): - raise ValueError("config tree is empty") + raise EvaluationError("Unable to evaluate feature: config tree is empty") for i, constraint in enumerate(config.tree.constraints): child_result = traverse(constraint, namespace, config.key, context) @@ -57,4 +58,4 @@ def _get_any(val: Optional[ProtoAny], val_new: Optional[LekkoAny]) -> ProtoAny: return ProtoAny(type_url=val_new.type_url, value=val_new.value) if val: return val - raise ValueError("config value not found") + raise EvaluationError("Constraint or default value is empty") diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py index b3cb20b..08762a3 100644 --- a/lekko_client/evaluation/rules.py +++ b/lekko_client/evaluation/rules.py @@ -4,6 +4,7 @@ from google.protobuf.struct_pb2 import Value from xxhash import xxh32 +from lekko_client.exceptions import EvaluationError from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import ( Value as LekkoValue, ) @@ -19,11 +20,11 @@ def evaluate_rule(rule: Rule, namespace: str, config_name: str, context: ClientContext = None) -> bool: if not rule: - raise ValueError("empty rule") + raise EvaluationError("Empty rule") rule_type = rule.WhichOneof("rule") if not rule_type: - raise ValueError("empty rule") + raise EvaluationError("Empty rule") rule_value = getattr(rule, rule_type) @@ -33,7 +34,7 @@ def evaluate_rule(rule: Rule, namespace: str, config_name: str, context: ClientC return not evaluate_rule(rule_value, namespace, config_name, context) elif rule_type == "logical_expression": if not rule_value.rules: - raise ValueError("no rules found in logical expression") + raise EvaluationError("No rules found in logical expression") logical_operator = rule_value.logical_operator return ( @@ -75,27 +76,27 @@ def evaluate_rule(rule: Rule, namespace: str, config_name: str, context: ClientC rule_value.comparison_operator, rule_value.comparison_value, context_value ) else: - raise ValueError("unknown comparison operator") + raise EvaluationError("Unknown comparison operator") elif rule_type == "call_expression": if rule_value.WhichOneof("function") == "bucket": return evaluate_bucket(rule_value.bucket, namespace, config_name, context) else: - raise ValueError("unknown function type") + raise EvaluationError("Unknown CallExpression function type") else: - raise ValueError("unknown rule type") + raise EvaluationError("Unknown rule type") def evaluate_equals(rule_value: Value, context_value: LekkoValue) -> bool: rule_kind = rule_value.WhichOneof("kind") or "" context_kind = context_value.WhichOneof("kind") or "" if rule_kind not in ["bool_value", "string_value", "number_value"]: - raise ValueError("unsupported type for equals operator") + raise EvaluationError("Unsupported rule type for equals operator") if rule_kind == "number_value": if context_kind not in ["double_value", "int_value"]: - raise ValueError("type mismatch") + raise EvaluationError("Type mismatch in equals operator rule") elif rule_kind != context_kind: - raise ValueError("type mismatch") + raise EvaluationError("Type mismatch in equals operator rule") return getattr(rule_value, rule_kind) == getattr(context_value, context_kind) @@ -113,17 +114,17 @@ def evaluate_string_comparator( elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_CONTAINS: return rule_str in context_str else: - raise ValueError("unexpected string comparison operator") + raise EvaluationError("Unknown string comparison operator") def get_string(value: Union[Value, LekkoValue]) -> str: if not value: - raise ValueError("value is undefined") + raise EvaluationError("String Value is undefined") if value.WhichOneof("kind") == "string_value": return value.string_value else: - raise ValueError("value is not a string") + raise EvaluationError("get_string called with non-string Value") def evaluate_number_comparator( @@ -141,7 +142,7 @@ def evaluate_number_comparator( elif comparison_operator == ComparisonOperator.COMPARISON_OPERATOR_GREATER_THAN_OR_EQUALS: return context_num >= rule_num else: - raise ValueError("unexpected numerical comparison operator") + raise EvaluationError("Unknown numerical comparison operator") def get_number(value: Union[Value, LekkoValue]) -> float: @@ -149,12 +150,12 @@ def get_number(value: Union[Value, LekkoValue]) -> float: if value_kind in ["number_value", "int_value", "double_value"]: return float(getattr(value, value_kind)) else: - raise ValueError("value is not a number") + raise EvaluationError("get_number caled with non-numeric Value") def evaluate_contained_within(rule_value: Value, context_value: LekkoValue) -> bool: if rule_value.WhichOneof("kind") != "list_value": - raise ValueError("type mismatch: expecting list for operator contained within") + raise EvaluationError("Contained within operator must use a list value") # TODO: this will throw if there's a type mismatch, which means that all items in rule list must be of same type # This is consistent with other language SDKs, but we should consider just returning False on type mismatch @@ -182,7 +183,7 @@ def evaluate_bucket(bucket_f: CallExpression.Bucket, namespace: str, config_name elif value_kind == "double_value": bytes_buffer = struct.pack(">d", value_val) else: - raise ValueError("unsupported value type for bucket") + raise EvaluationError("Unsupported value type for bucket") bytes_frags = [bytes(namespace, "utf-8"), bytes(config_name, "utf-8"), bytes(ctx_key, "utf-8"), bytes_buffer] result = xxh32(b"".join(bytes_frags), 0).intdigest() diff --git a/lekko_client/exceptions.py b/lekko_client/exceptions.py index cf867c2..8329945 100644 --- a/lekko_client/exceptions.py +++ b/lekko_client/exceptions.py @@ -10,6 +10,10 @@ class LekkoRpcError(LekkoError): pass +class NamespaceNotFound(LekkoRpcError): + pass + + class FeatureNotFound(LekkoRpcError): pass @@ -20,3 +24,11 @@ class MismatchedType(LekkoRpcError): class MismatchedProtoType(LekkoError): pass + + +class EvaluationError(LekkoError): + pass + + +class GitRepoNotFound(LekkoError): + pass diff --git a/lekko_client/stores/memory.py b/lekko_client/stores/memory.py index f947bfb..47a8ac3 100644 --- a/lekko_client/stores/memory.py +++ b/lekko_client/stores/memory.py @@ -1,5 +1,6 @@ from typing import Dict +from lekko_client.exceptions import FeatureNotFound, NamespaceNotFound from lekko_client.models import FeatureData from lekko_client.stores.store import Store @@ -12,10 +13,10 @@ def __init__(self): def get(self, namespace: str, config_key: str) -> FeatureData: namespace_map = self.configs.get(namespace) if not namespace_map: - raise Exception("namespace not found") + raise NamespaceNotFound(f"Namespace {namespace} not found") result = namespace_map.get(config_key) if not result: - raise Exception("config not found") + raise FeatureNotFound(f"Feature {config_key} not found in namespace {namespace}") return result def load_impl(self, contents) -> bool: diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py index 092ffe9..67227fc 100644 --- a/tests/evaluation/test_evaluation.py +++ b/tests/evaluation/test_evaluation.py @@ -106,4 +106,4 @@ def test_eval( assert result.value == expected assert result.path == expected_path else: - raise ValueError("test case needs to either expect an error or a result") + assert not "test case needs to either expect an error or a result" From bb7c73c1fc4c99f58fa60b6a6f1f0077e56f02f9 Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Sat, 16 Sep 2023 13:44:40 -0700 Subject: [PATCH 14/28] drop python39, improve init config --- lekko_client/__init__.py | 113 +++++++++++++----- lekko_client/clients/__init__.py | 6 +- lekko_client/clients/cached_backend_client.py | 24 +++- lekko_client/clients/cached_git_client.py | 4 +- lekko_client/clients/client.py | 4 + lekko_client/clients/distribution_client.py | 29 ++--- lekko_client/clients/grpc_client.py | 5 + lekko_client/evaluation/rules.py | 6 +- lekko_client/exceptions.py | 4 + tox.ini | 9 +- 10 files changed, 144 insertions(+), 60 deletions(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 0b42448..dd8b45f 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -1,17 +1,18 @@ """Lekko Python SDK Client""" -from enum import Enum +import functools +from dataclasses import dataclass +from threading import RLock from typing import Any, Dict, Optional, Type from google.protobuf.message import Message as ProtoMessage from lekko_client import exceptions from lekko_client.clients import ( - APIClient, CachedBackendClient, CachedGitClient, Client, - SidecarClient, + ConfigServiceClient, ) from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa from lekko_client.stores import MemoryStore @@ -19,59 +20,112 @@ __version__ = "0.1.4" __client: Client +__client_lock = RLock() -class Mode(Enum): - API = 1 - SIDECAR = 2 - CACHED_SERVER = 3 - CACHED_GIT = 4 +@dataclass(kw_only=True) +class Config: + owner_name: str + repo_name: str + api_key: Optional[str] = None + context: Optional[Dict[str, Any]] = None + lekko_uri: str = LEKKO_API_URL -def initialize( - mode: Mode, - owner_name: str, - repo_name: str, - api_key: Optional[str] = None, - context: Optional[Dict[str, Any]] = None, - git_repo_path: Optional[str] = None, -) -> Client: +@dataclass(kw_only=True) +class SidecarConfig(Config): + lekko_uri = LEKKO_SIDECAR_URL + + +@dataclass(kw_only=True) +class APIConfig(Config): + pass + + +@dataclass(kw_only=True) +class CachedServerConfig(Config): + pass + + +@dataclass(kw_only=True) +class CachedGitConfig(Config): + git_repo_path: str + + +def initialize(config: Config) -> Client: + global __client + with __client_lock: + if __client: + __client.close() + match config: + case APIConfig(_) | SidecarConfig(_): + __client = ConfigServiceClient( + config.lekko_uri, config.owner_name, config.repo_name, config.api_key, config.context + ) + case CachedGitConfig(_): + __client = CachedGitClient( + config.lekko_uri, + config.owner_name, + config.repo_name, + MemoryStore(), + config.git_repo_path, + config.api_key, + config.context, + ) + case CachedServerConfig(_): + __client = CachedBackendClient( + config.lekko_uri, config.owner_name, config.repo_name, MemoryStore(), config.api_key, config.context + ) + case _: + raise exceptions.LekkoError("Unknown client mode") + return __client + + +def set_client(client: Client): global __client - if mode == Mode.API: - __client = APIClient(owner_name, repo_name, api_key, context) - elif mode == Mode.SIDECAR: - __client = SidecarClient(owner_name, repo_name, api_key, context) - elif mode == Mode.CACHED_GIT: - if not git_repo_path: - raise exceptions.GitRepoNotFound("Must provide a path to git repo in Cached Git mode") - __client = CachedGitClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), git_repo_path, api_key, context) - elif mode == Mode.CACHED_SERVER: - __client = CachedBackendClient(LEKKO_API_URL, owner_name, repo_name, MemoryStore(), api_key, context) - else: - raise exceptions.LekkoError("Unknown client mode") - return __client + with __client_lock: + if __client: + __client.close() + __client = client + + +def __get_safe(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + with __client_lock: + if not __client: + raise exceptions.ClientNotInitialized("lekko_client.initialize() must be called prior to using API") + return func(args, kwargs) + + return wrapper +@__get_safe def get_bool(namespace: str, key: str, context: Dict[str, Any]) -> bool: return __client.get_bool(namespace, key, context) +@__get_safe def get_int(namespace: str, key: str, context: Dict[str, Any]) -> int: return __client.get_int(namespace, key, context) +@__get_safe def get_float(namespace: str, key: str, context: Dict[str, Any]) -> float: return __client.get_float(namespace, key, context) +@__get_safe def get_string(namespace: str, key: str, context: Dict[str, Any]) -> str: return __client.get_string(namespace, key, context) +@__get_safe def get_json(namespace: str, key: str, context: Dict[str, Any]) -> dict: return __client.get_json(namespace, key, context) +@__get_safe def get_proto( namespace: str, key: str, @@ -80,6 +134,7 @@ def get_proto( return __client.get_proto(namespace, key, context) +@__get_safe def get_proto_by_type( namespace: str, key: str, diff --git a/lekko_client/clients/__init__.py b/lekko_client/clients/__init__.py index 0730a28..0e73494 100644 --- a/lekko_client/clients/__init__.py +++ b/lekko_client/clients/__init__.py @@ -1,4 +1,8 @@ from lekko_client.clients.cached_backend_client import CachedBackendClient # noqa from lekko_client.clients.cached_git_client import CachedGitClient # noqa from lekko_client.clients.client import Client # noqa -from lekko_client.clients.grpc_client import APIClient, SidecarClient # noqa +from lekko_client.clients.grpc_client import ( # noqa + APIClient, + ConfigServiceClient, + SidecarClient, +) diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index f9cefb3..53ee3fe 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -1,10 +1,12 @@ import time -from threading import Thread +from threading import Event, Thread from typing import Any, Dict, Optional import grpc +from google.protobuf.any_pb2 import Any as ProtoAny from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.exceptions import ClientNotInitialized from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( GetRepositoryContentsRequest, GetRepositoryContentsResponse, @@ -41,17 +43,18 @@ def __init__( context: Optional[Dict[str, Any]] = None, credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), ): + super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) self.timeout = None self.closed = False self.update_interval_ms = 1000 - super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) + self.initialized_event = Event() def initialize(self): - self.update_store() + super().initialize() self.refresh_thread = CachedBackendClient.RefreshThread(self, self.update_interval_ms) self.refresh_thread.start() - def get_contents(self) -> Optional[GetRepositoryContentsResponse]: + def load_contents(self) -> Optional[GetRepositoryContentsResponse]: if not self._client: return None return self._client.GetRepositoryContents( @@ -60,10 +63,13 @@ def get_contents(self) -> Optional[GetRepositoryContentsResponse]: def update_store(self): self.load() + self.initialized_event.set() - def should_update_store(self): + def should_update_store(self) -> bool: if not self._client: - return + return False + if not self.initialized_event.is_set(): + return True version_response = self._client.GetRepositoryVersion( GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key) ) @@ -73,3 +79,9 @@ def should_update_store(self): def close(self): super().close() self.refresh_thread.stop() + self.initialized_event.clear() + + def get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: + if not self.initialized_event.wait(timeout=5): # Give the background thread 5 seconds to populate + raise ClientNotInitialized("Repository contents not yet loaded from server") + return super().get(namespace, key, context) diff --git a/lekko_client/clients/cached_git_client.py b/lekko_client/clients/cached_git_client.py index ae85263..8a214b3 100644 --- a/lekko_client/clients/cached_git_client.py +++ b/lekko_client/clients/cached_git_client.py @@ -48,10 +48,10 @@ def __init__( credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), should_watch: Optional[bool] = True, ): + super().__init__(lekko_uri, repository_owner, repository_name, store, api_key, context, credentials) self.watcher: Optional[BaseObserver] = None self.path = path self.should_watch = should_watch - super().__init__(lekko_uri, repository_owner, repository_name, store, api_key, context, credentials) def initialize(self): super().initialize() @@ -62,7 +62,7 @@ def initialize(self): self.watcher.schedule(event_handler, self.path, recursive=True) self.watcher.start() - def get_contents(self) -> GetRepositoryContentsResponse: + def load_contents(self) -> GetRepositoryContentsResponse: try: repo = GitRepo(self.path) except NotGitRepository: diff --git a/lekko_client/clients/client.py b/lekko_client/clients/client.py index 362b88b..c28d026 100644 --- a/lekko_client/clients/client.py +++ b/lekko_client/clients/client.py @@ -59,3 +59,7 @@ def get_proto_by_type( proto_message_type: Type[ProtoType], ) -> ProtoType: ... + + @abstractmethod + def close(self): + ... diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index e0610ac..9b15559 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -3,7 +3,7 @@ from abc import abstractmethod from datetime import datetime from threading import Thread -from typing import Any, Dict, List, Optional, Type, TypeVar, Union +from typing import Any, Dict, List, Optional, Type, TypeVar import grpc from google.protobuf import descriptor_pool as proto_descriptor_pool @@ -109,7 +109,7 @@ def __init__( self.initialize() - _TYPE_MAPPING: Dict[Type, Type[Union[BoolValue, Int64Value, StringValue, FloatValue]]] = { + _TYPE_MAPPING: Dict[Type, Type[BoolValue | Int64Value | StringValue | FloatValue]] = { bool: BoolValue, int: Int64Value, str: StringValue, @@ -121,14 +121,14 @@ def initialize(self): ... def load(self) -> bool: - contents = self.get_contents() + contents = self.load_contents() if not contents: return False loaded = self.store.load(contents) return loaded @abstractmethod - def get_contents(self) -> Optional[GetRepositoryContentsResponse]: + def load_contents(self) -> Optional[GetRepositoryContentsResponse]: ... def track( @@ -148,7 +148,7 @@ def track( ) self.events_batcher.add_event(event) - def _get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: + def get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: feature_data = self.store.get(namespace, key) result = evaluate(feature_data.feature, namespace, convert_context(context)) self.track(namespace, feature_data, result, context) @@ -156,33 +156,33 @@ def _get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: ReturnType = TypeVar("ReturnType", str, float, int, bool) - def _get_scalar(self, namespace: str, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: - result = self._get(namespace, key, context) + def get_scalar(self, namespace: str, key: str, context: Dict[str, Any], typ: Type[ReturnType]) -> ReturnType: + result = self.get(namespace, key, context) return_wrapper = self._TYPE_MAPPING[typ]() if result.Unpack(return_wrapper): return return_wrapper.value # type:ignore raise MismatchedType(f"Feature {key} is of type {result.type_url} and cannot be converted to {typ}") def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: - return self._get_scalar(namespace, key, context, bool) + return self.get_scalar(namespace, key, context, bool) def get_int(self, namespace: str, key: str, context: Dict[str, Any]) -> int: - return self._get_scalar(namespace, key, context, int) + return self.get_scalar(namespace, key, context, int) def get_float(self, namespace: str, key: str, context: Dict[str, Any]) -> float: - return self._get_scalar(namespace, key, context, float) + return self.get_scalar(namespace, key, context, float) def get_string(self, namespace: str, key: str, context: Dict[str, Any]) -> str: - return self._get_scalar(namespace, key, context, str) + return self.get_scalar(namespace, key, context, str) def get_json(self, namespace: str, key: str, context: Dict[str, Any]) -> Any: - result = self._get(namespace, key, context) + result = self.get(namespace, key, context) return_wrapper = Value() result.Unpack(return_wrapper) return json.loads(MessageToJson(return_wrapper)) def get_proto(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoMessage: - val = self._get(namespace, key, context) + val = self.get(namespace, key, context) db = proto_symbol_database.SymbolDatabase(pool=proto_descriptor_pool.Default()) try: ret_val = db.GetSymbol(val.type_url.split("/")[1])() @@ -199,7 +199,7 @@ def get_proto_by_type( context: Dict[str, Any], proto_message_type: Type[Client.ProtoType], ) -> Client.ProtoType: - val = self._get(namespace, key, context) + val = self.get(namespace, key, context) ret_val = proto_message_type() if val.Unpack(ret_val): return ret_val @@ -207,6 +207,7 @@ def get_proto_by_type( raise MismatchedProtoType(f"Error unpacking from {val.type_url} to {proto_message_type.DESCRIPTOR.name}") def close(self): + super().close() if self._client and self.session_key: self.events_batcher.upload_events() self.events_batcher.stop() diff --git a/lekko_client/clients/grpc_client.py b/lekko_client/clients/grpc_client.py index 7df11ed..03e52f9 100644 --- a/lekko_client/clients/grpc_client.py +++ b/lekko_client/clients/grpc_client.py @@ -16,6 +16,7 @@ MismatchedType, ) from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import ( + DeregisterRequest, GetBoolValueRequest, GetFloatValueRequest, GetIntValueRequest, @@ -75,6 +76,10 @@ def __init__( # TODO:SAM - re-registering shouldn't cause errors in the future pass + def close(self): + super().close() + self._client.Deregister(DeregisterRequest()) + def get_bool(self, namespace: str, key: str, context: Dict[str, Any]) -> bool: return self._get(namespace, key, context, bool) diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py index 08762a3..00a4cdd 100644 --- a/lekko_client/evaluation/rules.py +++ b/lekko_client/evaluation/rules.py @@ -1,5 +1,5 @@ import struct -from typing import Dict, Optional, Union +from typing import Dict, Optional from google.protobuf.struct_pb2 import Value from xxhash import xxh32 @@ -117,7 +117,7 @@ def evaluate_string_comparator( raise EvaluationError("Unknown string comparison operator") -def get_string(value: Union[Value, LekkoValue]) -> str: +def get_string(value: Value | LekkoValue) -> str: if not value: raise EvaluationError("String Value is undefined") @@ -145,7 +145,7 @@ def evaluate_number_comparator( raise EvaluationError("Unknown numerical comparison operator") -def get_number(value: Union[Value, LekkoValue]) -> float: +def get_number(value: Value | LekkoValue) -> float: value_kind = value.WhichOneof("kind") if value_kind in ["number_value", "int_value", "double_value"]: return float(getattr(value, value_kind)) diff --git a/lekko_client/exceptions.py b/lekko_client/exceptions.py index 8329945..b21a4bc 100644 --- a/lekko_client/exceptions.py +++ b/lekko_client/exceptions.py @@ -32,3 +32,7 @@ class EvaluationError(LekkoError): class GitRepoNotFound(LekkoError): pass + + +class ClientNotInitialized(LekkoError): + pass diff --git a/tox.ini b/tox.ini index a9834e6..bb08ef4 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = True envlist = clean - py{39,310,311} + py{310,311} lint typecheck report @@ -11,7 +11,6 @@ envlist = python = 3.11: py311 3.10: py310 - 3.9: py39 [testenv] deps = @@ -25,8 +24,8 @@ passenv = setenv = COVERAGE_FILE = {toxworkdir}/.coverage.{envname} depends: - py{39,310,311}: clean - report: py{39,310,311} + py{310,311}: clean + report: py{310,311} commands = coverage erase pytest --cov=lekko_client tests/ @@ -71,7 +70,7 @@ setenv = COVERAGE_FILE = {toxworkdir}/.coverage deps = coverage -depends = py{39,310,311} +depends = py{310,311} commands = coverage combine -q --keep coverage report From 42f617d2e82a1d5ea51d6bc3f2bb78975a98d2ac Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Tue, 26 Sep 2023 14:50:34 -0700 Subject: [PATCH 15/28] add distribution client tests --- lekko_client/clients/__init__.py | 2 +- .../{grpc_client.py => config_client.py} | 0 lekko_client/clients/distribution_client.py | 67 +++++------ lekko_client/evaluation/evaluation.py | 3 +- lekko_client/evaluation/rules.py | 4 +- lekko_client/helpers.py | 17 ++- lekko_client/models.py | 4 + lekko_client/stores/memory.py | 5 +- tests/clients/test_cached_backend_client.py | 0 .../test_config_client.py} | 0 tests/clients/test_distribution_client.py | 113 ++++++++++++++++++ tests/conftest.py | 46 ++++++- tests/evaluation/test_evaluation.py | 3 +- 13 files changed, 217 insertions(+), 47 deletions(-) rename lekko_client/clients/{grpc_client.py => config_client.py} (100%) create mode 100644 tests/clients/test_cached_backend_client.py rename tests/{test_client.py => clients/test_config_client.py} (100%) create mode 100644 tests/clients/test_distribution_client.py diff --git a/lekko_client/clients/__init__.py b/lekko_client/clients/__init__.py index 0e73494..5f947df 100644 --- a/lekko_client/clients/__init__.py +++ b/lekko_client/clients/__init__.py @@ -1,7 +1,7 @@ from lekko_client.clients.cached_backend_client import CachedBackendClient # noqa from lekko_client.clients.cached_git_client import CachedGitClient # noqa from lekko_client.clients.client import Client # noqa -from lekko_client.clients.grpc_client import ( # noqa +from lekko_client.clients.config_client import ( # noqa APIClient, ConfigServiceClient, SidecarClient, diff --git a/lekko_client/clients/grpc_client.py b/lekko_client/clients/config_client.py similarity index 100% rename from lekko_client/clients/grpc_client.py rename to lekko_client/clients/config_client.py diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 9b15559..eb74d82 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -17,10 +17,8 @@ from lekko_client.clients.client import Client from lekko_client.evaluation.evaluation import EvaluationResult, evaluate -from lekko_client.evaluation.rules import ClientContext -from lekko_client.exceptions import MismatchedProtoType, MismatchedType +from lekko_client.exceptions import LekkoRpcError, MismatchedProtoType, MismatchedType from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( - ContextKey, DeregisterClientRequest, FlagEvaluationEvent, GetRepositoryContentsResponse, @@ -31,11 +29,8 @@ from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2_grpc import ( DistributionServiceStub, ) -from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import ( - Value as LekkoValue, -) -from lekko_client.helpers import convert_context, get_grpc_channel -from lekko_client.models import FeatureData +from lekko_client.helpers import convert_context, get_context_keys, get_grpc_channel +from lekko_client.models import ClientContext, FeatureData from lekko_client.stores.store import Store @@ -69,17 +64,6 @@ def run(self): self.upload_events() time.sleep(self.upload_interval) - @classmethod - def get_value_type(cls, val: LekkoValue) -> str: - return (val.WhichOneof("kind") or "").removesuffix("_value") - - @classmethod - def get_context_keys(cls, context: Optional[ClientContext] = None) -> List[ContextKey]: - if not context: - return [] - - return [ContextKey(key=k, type=cls.get_value_type(v)) for k, v in context.items()] - def __init__( self, uri: str, @@ -90,22 +74,17 @@ def __init__( context: Optional[Dict[str, Any]] = None, credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), ): - from lekko_client import __version__ - super().__init__(owner_name, repo_name, api_key, context) self.uri = uri self.repository = RepositoryKey(owner_name=owner_name, repo_name=repo_name) self.store = store - self._client = None + self._client: Optional[DistributionServiceStub] = None + self.events_batcher = None if self.api_key: - channel = get_grpc_channel(self.uri, self.api_key, credentials) - self._client = DistributionServiceStub(channel) - register_response = self._client.RegisterClient( - RegisterClientRequest(repo_key=self.repository, sidecar_version=__version__) - ) - self.session_key = register_response.session_key - self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15 * 1000) - self.events_batcher.start() + self.initialize_client(credentials) + if self._client: + self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15 * 1000) + self.events_batcher.start() self.initialize() @@ -116,6 +95,19 @@ def __init__( float: FloatValue, } + def initialize_client(self, credentials: grpc.ChannelCredentials): + from lekko_client import __version__ + + channel = get_grpc_channel(self.uri, self.api_key, credentials) + self._client = DistributionServiceStub(channel) + try: + register_response = self._client.RegisterClient( + RegisterClientRequest(repo_key=self.repository, sidecar_version=__version__) + ) + except grpc.RpcError as e: + raise LekkoRpcError(f"Unable to register distribution service: {e}") + self.session_key = register_response.session_key + @abstractmethod def initialize(self): ... @@ -134,6 +126,9 @@ def load_contents(self) -> Optional[GetRepositoryContentsResponse]: def track( self, namespace: str, feature_data: FeatureData, result: EvaluationResult, context: Optional[ClientContext] ) -> None: + if not self.events_batcher: + return + timestamp = Timestamp() timestamp.FromDatetime(datetime.utcnow()) event = FlagEvaluationEvent( @@ -142,7 +137,7 @@ def track( feature_sha=feature_data.config_sha, namespace_name=namespace, feature_name=feature_data.feature.key, - context_keys=self.events_batcher.get_context_keys(context), + context_keys=get_context_keys(context), result_path=result.path, client_event_time=timestamp, ) @@ -150,8 +145,9 @@ def track( def get(self, namespace: str, key: str, context: Dict[str, Any]) -> ProtoAny: feature_data = self.store.get(namespace, key) - result = evaluate(feature_data.feature, namespace, convert_context(context)) - self.track(namespace, feature_data, result, context) + client_context = convert_context(context) + result = evaluate(feature_data.feature, namespace, client_context) + self.track(namespace, feature_data, result, client_context) return result.value ReturnType = TypeVar("ReturnType", str, float, int, bool) @@ -209,6 +205,7 @@ def get_proto_by_type( def close(self): super().close() if self._client and self.session_key: - self.events_batcher.upload_events() - self.events_batcher.stop() + if self.events_batcher: + self.events_batcher.upload_events() + self.events_batcher.stop() self._client.DeregisterClient(DeregisterClientRequest(session_key=self.session_key)) diff --git a/lekko_client/evaluation/evaluation.py b/lekko_client/evaluation/evaluation.py index 8ed42b1..65bfcd1 100644 --- a/lekko_client/evaluation/evaluation.py +++ b/lekko_client/evaluation/evaluation.py @@ -3,10 +3,11 @@ from google.protobuf.any_pb2 import Any as ProtoAny -from lekko_client.evaluation.rules import ClientContext, evaluate_rule +from lekko_client.evaluation.rules import evaluate_rule from lekko_client.exceptions import EvaluationError from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Constraint, Feature +from lekko_client.models import ClientContext @dataclass diff --git a/lekko_client/evaluation/rules.py b/lekko_client/evaluation/rules.py index 00a4cdd..cdd2071 100644 --- a/lekko_client/evaluation/rules.py +++ b/lekko_client/evaluation/rules.py @@ -1,5 +1,4 @@ import struct -from typing import Dict, Optional from google.protobuf.struct_pb2 import Value from xxhash import xxh32 @@ -14,8 +13,7 @@ LogicalOperator, Rule, ) - -ClientContext = Optional[Dict[str, LekkoValue]] +from lekko_client.models import ClientContext def evaluate_rule(rule: Rule, namespace: str, config_name: str, context: ClientContext = None) -> bool: diff --git a/lekko_client/helpers.py b/lekko_client/helpers.py index ef2fb4c..7b378b4 100644 --- a/lekko_client/helpers.py +++ b/lekko_client/helpers.py @@ -1,12 +1,14 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import grpc from grpc_interceptor import ClientCallDetails, ClientInterceptor +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ContextKey from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import Value +from lekko_client.models import ClientContext -def convert_context(context: dict) -> Dict[str, Value]: +def convert_context(context: dict) -> ClientContext: def convert_value(val: Any) -> Value: if isinstance(val, bool): return Value(bool_value=val) @@ -20,6 +22,17 @@ def convert_value(val: Any) -> Value: return {k: convert_value(v) for k, v in context.items()} +def get_value_type(val: Value) -> str: + return (val.WhichOneof("kind") or "").removesuffix("_value") + + +def get_context_keys(context: Optional[ClientContext] = None) -> List[ContextKey]: + if not context: + return [] + + return [ContextKey(key=k, type=get_value_type(v)) for k, v in context.items()] + + class ApiKeyInterceptor(ClientInterceptor): """A test interceptor that injects invocation metadata.""" diff --git a/lekko_client/models.py b/lekko_client/models.py index 7b06611..cee39b9 100644 --- a/lekko_client/models.py +++ b/lekko_client/models.py @@ -1,7 +1,11 @@ from dataclasses import dataclass +from typing import Dict, Optional +from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import Value from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature +ClientContext = Optional[Dict[str, Value]] + @dataclass class FeatureData: diff --git a/lekko_client/stores/memory.py b/lekko_client/stores/memory.py index 47a8ac3..f9d4d59 100644 --- a/lekko_client/stores/memory.py +++ b/lekko_client/stores/memory.py @@ -1,6 +1,9 @@ from typing import Dict from lekko_client.exceptions import FeatureNotFound, NamespaceNotFound +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsResponse, +) from lekko_client.models import FeatureData from lekko_client.stores.store import Store @@ -19,7 +22,7 @@ def get(self, namespace: str, config_key: str) -> FeatureData: raise FeatureNotFound(f"Feature {config_key} not found in namespace {namespace}") return result - def load_impl(self, contents) -> bool: + def load_impl(self, contents: GetRepositoryContentsResponse) -> bool: new_configs = {} for ns in contents.namespaces: namespace_map = {} diff --git a/tests/clients/test_cached_backend_client.py b/tests/clients/test_cached_backend_client.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_client.py b/tests/clients/test_config_client.py similarity index 100% rename from tests/test_client.py rename to tests/clients/test_config_client.py diff --git a/tests/clients/test_distribution_client.py b/tests/clients/test_distribution_client.py new file mode 100644 index 0000000..c0c4721 --- /dev/null +++ b/tests/clients/test_distribution_client.py @@ -0,0 +1,113 @@ +from unittest import mock + +import pytest +from google.protobuf import wrappers_pb2 +from google.protobuf.any_pb2 import Any as ProtoAny +from google.protobuf.message import Message as ProtoMessage +from google.protobuf.struct_pb2 import Struct + +from lekko_client.clients.config_client import AnyProto +from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.evaluation.evaluation import EvaluationResult +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + RepositoryKey, +) +from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature +from lekko_client.helpers import convert_context, get_context_keys +from lekko_client.models import FeatureData + + +@pytest.mark.parametrize( + "fn_under_test,expected", + [ + ("get_bool", wrappers_pb2.BoolValue(value=True)), + ("get_int", wrappers_pb2.Int64Value(value=10)), + ("get_float", wrappers_pb2.FloatValue(value=2.5)), + ("get_string", wrappers_pb2.StringValue(value="test_value")), + ], +) +def test_scalar( + mock_distribution_client: CachedDistributionClient, + test_feature_no_constraints: Feature, + fn_under_test: str, + expected: ProtoMessage, +): + mock_distribution_client.store.get.return_value = FeatureData("test_sha", test_feature_no_constraints) + + expected_any = ProtoAny() + expected_any.Pack(expected) + namespace = "test_namespace" + key = test_feature_no_constraints.key + ctx = {"ctx_key": "ctx_val"} + with mock.patch( + "lekko_client.clients.distribution_client.evaluate", return_value=EvaluationResult(value=expected_any, path=[1]) + ) as mock_eval: + getattr(mock_distribution_client, fn_under_test)(namespace, key, ctx) + res_unpacked = type(expected)() + assert expected_any.Unpack(res_unpacked) + assert res_unpacked == expected + + mock_eval.assert_called_once_with(test_feature_no_constraints, namespace, convert_context(ctx)) + mock_distribution_client.store.get.assert_called_once_with(namespace, key) + + # Not worth freezing time to use assert_called_once_with + (flag_evaluation,) = mock_distribution_client.events_batcher.add_event.call_args_list[0].args + assert flag_evaluation.repo_key == RepositoryKey(owner_name="owner", repo_name="repo") + assert flag_evaluation.commit_sha == "test_commit_sha" + assert flag_evaluation.feature_sha == "test_sha" + assert flag_evaluation.namespace_name == namespace + assert flag_evaluation.feature_name == key + assert flag_evaluation.context_keys == get_context_keys(convert_context(ctx)) + assert flag_evaluation.result_path == [1] + + +@pytest.mark.parametrize( + "expected", + [ + [1, 2, 3], + {"test": "dict"}, + "str", + 1.2, + True, + ], +) +def test_get_json(mock_distribution_client, test_feature_no_constraints, expected): + s = Struct() + s.update({"key": expected}) + expected_value = s.fields["key"] + any_proto = AnyProto() + any_proto.Pack(expected_value) + + mock_distribution_client.store.get.return_value = FeatureData("test_sha", test_feature_no_constraints) + + with mock.patch( + "lekko_client.clients.distribution_client.evaluate", return_value=EvaluationResult(value=any_proto, path=[1]) + ): + assert expected == mock_distribution_client.get_json("namespace", "key", {}) + + +def test_get_proto_by_type(mock_distribution_client, test_feature_no_constraints): + any_proto = AnyProto() + int_proto = wrappers_pb2.Int32Value(value=10) + any_proto.Pack(int_proto) + mock_distribution_client.store.get.return_value = FeatureData("test_sha", test_feature_no_constraints) + + with mock.patch( + "lekko_client.clients.distribution_client.evaluate", return_value=EvaluationResult(value=any_proto, path=[1]) + ): + assert int_proto == mock_distribution_client.get_proto_by_type("namespace", "key", {}, wrappers_pb2.Int32Value) + + +def test_get_proto(mock_distribution_client, test_feature_no_constraints): + any_proto = AnyProto() + int_proto = wrappers_pb2.Int32Value(value=10) + any_proto.Pack(int_proto) + mock_distribution_client.store.get.return_value = FeatureData("test_sha", test_feature_no_constraints) + + with mock.patch( + "lekko_client.clients.distribution_client.evaluate", return_value=EvaluationResult(value=any_proto, path=[1]) + ): + assert int_proto == mock_distribution_client.get_proto("namespace", "key", {}) + + with mock.patch("google.protobuf.symbol_database.SymbolDatabase.GetSymbol", side_effect=KeyError): + assert any_proto == mock_distribution_client.get_proto("namespace", "key", {}) diff --git a/tests/conftest.py b/tests/conftest.py index 87aa011..40df731 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import concurrent.futures.thread from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from typing import Any, List, Tuple +from typing import Any, List, Optional, Tuple from unittest import mock import grpc @@ -14,6 +14,13 @@ from grpc_testing import _channel # noqa from lekko_client import helpers +from lekko_client.clients.distribution_client import CachedDistributionClient +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2 import ( + GetRepositoryContentsResponse, +) +from lekko_client.gen.lekko.backend.v1beta1.distribution_service_pb2_grpc import ( + DistributionServiceStub, +) from lekko_client.gen.lekko.client.v1beta1.configuration_service_pb2 import DESCRIPTOR from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Any as LekkoAny from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import ( @@ -27,6 +34,7 @@ ComparisonOperator, Rule, ) +from lekko_client.stores.store import Store @pytest.fixture @@ -45,7 +53,7 @@ def test_thread(): def test_channel_no_interceptor(test_thread): channel = grpc_testing.channel(DESCRIPTOR.services_by_name.values(), grpc_testing.strict_real_time()) try: - with mock.patch("lekko_client.clients.grpc_client.get_grpc_channel", return_value=channel): + with mock.patch("lekko_client.clients.config_client.get_grpc_channel", return_value=channel): yield channel finally: channel.close() @@ -298,3 +306,37 @@ def test_feature_two_level_traversal(test_feature_default_value, test_feature_co ], ), ) + + +@pytest.fixture +def mock_store() -> Store: + store = mock.Mock(spec_set=Store) + type(store).commit_sha = mock.PropertyMock(return_value="test_commit_sha") + return store + + +@pytest.fixture +def mock_event_batcher(): + def mock_event_batcher_factory(self, client, session_key, interval): + return mock.Mock(spec_set=CachedDistributionClient.EventsBatcher) + + return mock_event_batcher_factory + + +@pytest.fixture +def mock_distribution_client(mock_store, mock_event_batcher) -> CachedDistributionClient: + class MockDistributionClient(CachedDistributionClient): + EventsBatcher = mock_event_batcher + + def initialize(self): + pass + + def load_contents(self) -> Optional[GetRepositoryContentsResponse]: + return mock.Mock(spec_set=GetRepositoryContentsResponse) + + def initialize_client(self, credentials: grpc.ChannelCredentials): + self._client = mock.Mock(spec_set=DistributionServiceStub) + self.session_key = "session key" + + client = MockDistributionClient("uri", "owner", "repo", mock_store, api_key="api_key") + return client diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py index 67227fc..ddd3e57 100644 --- a/tests/evaluation/test_evaluation.py +++ b/tests/evaluation/test_evaluation.py @@ -4,9 +4,9 @@ from google.protobuf.wrappers_pb2 import Int64Value from lekko_client.evaluation.evaluation import evaluate -from lekko_client.evaluation.rules import ClientContext from lekko_client.gen.lekko.feature.v1beta1.feature_pb2 import Feature from lekko_client.helpers import convert_context +from lekko_client.models import ClientContext @pytest.mark.parametrize( @@ -35,7 +35,6 @@ def test_complex_evaluation(test_complex_rule_feature, context, expected): assert inner_result.value == expected -@pytest.mark.only def test_empty_config_tree(): with pytest.raises(Exception): evaluate(Feature(), "ns") From b15dafaf6bdfdbdb62d0d4c6592ca0bb93bc345a Mon Sep 17 00:00:00 2001 From: Sam Marcellus Date: Thu, 28 Sep 2023 17:06:08 -0700 Subject: [PATCH 16/28] documentation --- README.md | 71 +++++++++++++++++++++++++++++++++++----- example.py | 36 +++++++++++++------- lekko_client/__init__.py | 17 +++++++++- 3 files changed, 104 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 23e9c3b..7bc4ce1 100644 --- a/README.md +++ b/README.md @@ -8,23 +8,78 @@ The Lekko SDK for Python ### Installation `pip install lekko_client` -### Initializing a Lekko client +### Usage + +#### Initializing a cached Lekko client + +Creates a client that fetches configs from Lekko backend and caches them in memory. Configs are kept up to date via polling. + ```python -from lekko_client import APIClient, SidecarClient +import lekko_client -client = SidecarClient( # Or APIClient +lekko_client.initialize(lekko_client.CachedServerConfig( owner_name="", repo_name="", - namespace="", api_key="", # Optional - defaults to "LEKKO_API_KEY" ENV Var - uri="", # Optional - defaults to "localhost:50051" for Sidecar - # and "prod.api.lekko.dev:443" for APIClient + lekko_uri="", # Optional - defaults to "prod.api.lekko.dev:443" context={}, # Optionally provide context dict to be merged into each get request -) +)) -str_feature = client.get_string("my_feature", {"context_key": "context_val"}) +str_feature = lekko_client.get_string("my_namespace", "my_feature", {"context_key": "context_val"}) ``` +#### Initializing a cached Lekko client in git mode + +Creates a client that reads configs from a git repository on disk and caches them in memory. Configs are kept up to date via a file watcher. + +```python +import lekko_client + +lekko_client.initialize(lekko_client.CachedGitConfig( + owner_name="", + repo_name="", + git_repo_path="", + api_key="", # Optional - defaults to "LEKKO_API_KEY" ENV Var + lekko_uri="", # Optional - defaults to "prod.api.lekko.dev:443" + context={}, # Optionally provide context dict to be merged into each get request +)) + +str_feature = lekko_client.get_string("my_namespace", "my_feature", {"context_key": "context_val"}) +``` + +#### Initializing a Lekko client with a sidecar or server backend + +Create a client that communicates with a Lekko Sidecar or the Lekko API backend + +```python +import lekko_client + +lekko_client.initialize(lekko_client.APIConfig( # Or lekko_client.SidcarConfig + owner_name="", + repo_name="", + api_key="", # Optional - defaults to "LEKKO_API_KEY" ENV Var + context={}, # Optionally provide context dict to be merged into each get request +)) + +str_feature = lekko_client.get_string("my_namespace", "my_feature", {"context_key": "context_val"}) +``` + +### Lifecycle Management +`lekko_client.initialize()` must be invoked prior to calling the `lekko_client.get_*()` functions. We recommend invoking it early in your app's lifecycle, for example when constructing your Flask app or as part of FastAPI's lifecycle context manager. + +We recommended you invoke `lekko_client.close()` during app shutdown. This will ensure all evaluation events are properly tracked and the distribution server is unregistered. + +It is also possible to do your own lifecycle management and avoid the `lekko_client.initialize()` and `lekko_client.get_*()` methods entirely. Feel free to construct any of the clients in `lekko_client.clients` manually. This could make sense for the API or Sidecar clients, which have minimal state and startup costs. For example, it may be reasonable to do something like: + +```python +from lekko_client.clients import SidecarClient + +SidecarClient( + owner_name="", + repo_name="", + api_key="", +).get_string("my_namespace", "my_feature", {"context_key": "context_val"}) +``` ## Proto Features There are two methods to retrieve a Proto Feature, with one allowing you to specify the expected proto message type. diff --git a/example.py b/example.py index e84d715..42ac8d6 100644 --- a/example.py +++ b/example.py @@ -1,15 +1,16 @@ import argparse +import dataclasses import grpc from google.protobuf.json_format import MessageToDict -from lekko_client import APIClient, SidecarClient +import lekko_client from lekko_client.exceptions import LekkoError if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--apikey", type=str) - parser.add_argument("--sidecar", action="store_true") + parser.add_argument("--mode", type=str, choices=["sidecar", "api", "cachedapi", "cachedgit"]) parser.add_argument("--owner", type=str) parser.add_argument("--repo", type=str) parser.add_argument("--namespace", type=str, default="default") @@ -20,25 +21,38 @@ choices=["bool", "int", "float", "str", "json", "proto"], default="bool", ) + parser.add_argument("--git-path", type=str, default="") parser.add_argument("--proto-type", type=str, default="") parser.add_argument("--proto-file", type=str) args = parser.parse_args() - client_cls = SidecarClient if args.sidecar else APIClient - client = client_cls(args.owner, args.repo, args.namespace, api_key=args.apikey) + # client_cls = SidecarClient if args.sidecar else APIClient + + base_config = lekko_client.Config(owner_name=args.owner, repo_name=args.repo, api_key=args.apikey) + if args.mode == "sidecar": + config = lekko_client.SidecarConfig(**dataclasses.asdict(base_config)) + elif args.mode == "api": + config = lekko_client.APIConfig(**dataclasses.asdict(base_config)) + elif args.mode == "cachedapi": + config = lekko_client.CachedServerConfig(**dataclasses.asdict(base_config)) + elif args.mode == "cachedgit": + config = lekko_client.CachedGitConfig(**dataclasses.asdict(base_config), git_repo_path=args.git_path) + else: + raise ValueError("Invalid mode") + lekko_client.initialize(config) val = None try: if args.feature_type == "bool": - val = client.get_bool(args.feature, {}) + val = lekko_client.get_bool(args.namespace, args.feature, {}) elif args.feature_type == "int": - val = client.get_int(args.feature, {}) + val = lekko_client.get_int(args.namespace, args.feature, {}) elif args.feature_type == "str": - val = client.get_string(args.feature, {}) + val = lekko_client.get_string(args.namespace, args.feature, {}) elif args.feature_type == "json": - val = client.get_json(args.feature, {}) + val = lekko_client.get_json(args.namespace, args.feature, {}) elif args.feature_type == "float": - val = client.get_float(args.feature, {}) + val = lekko_client.get_float(args.namespace, args.feature, {}) elif args.feature_type == "proto": if not args.proto_file: print( @@ -49,9 +63,9 @@ if args.proto_type: msg_type = getattr(imported_proto, args.proto_type) - val = MessageToDict(client.get_proto_by_type(args.feature, {}, msg_type)) + val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.feature, {}, msg_type)) else: - val = MessageToDict(client.get_proto(args.feature, {})) + val = MessageToDict(lekko_client.get_proto(args.namespace, args.feature, {})) print(f"Got {val} for feature {args.namespace}/{args.feature}") except LekkoError as e: print(f"Failed to get feature: {e}") diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index dd8b45f..af5455b 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -19,7 +19,7 @@ __version__ = "0.1.4" -__client: Client +__client: Optional[Client] = None __client_lock = RLock() @@ -89,6 +89,14 @@ def set_client(client: Client): __client = client +def close(): + global __client + with __client_lock: + if __client: + __client.close() + __client = None + + def __get_safe(func): @functools.wraps(func) def wrapper(*args, **kwargs): @@ -102,26 +110,31 @@ def wrapper(*args, **kwargs): @__get_safe def get_bool(namespace: str, key: str, context: Dict[str, Any]) -> bool: + assert __client return __client.get_bool(namespace, key, context) @__get_safe def get_int(namespace: str, key: str, context: Dict[str, Any]) -> int: + assert __client return __client.get_int(namespace, key, context) @__get_safe def get_float(namespace: str, key: str, context: Dict[str, Any]) -> float: + assert __client return __client.get_float(namespace, key, context) @__get_safe def get_string(namespace: str, key: str, context: Dict[str, Any]) -> str: + assert __client return __client.get_string(namespace, key, context) @__get_safe def get_json(namespace: str, key: str, context: Dict[str, Any]) -> dict: + assert __client return __client.get_json(namespace, key, context) @@ -131,6 +144,7 @@ def get_proto( key: str, context: Dict[str, Any], ) -> ProtoMessage: + assert __client return __client.get_proto(namespace, key, context) @@ -141,4 +155,5 @@ def get_proto_by_type( context: Dict[str, Any], proto_message_type: Type[Client.ProtoType], ) -> Client.ProtoType: + assert __client return __client.get_proto_by_type(namespace, key, context, proto_message_type) From dbf4b929dee6a72f084b14bef5c86ad7156a8aa2 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Thu, 2 Nov 2023 14:28:28 -0700 Subject: [PATCH 17/28] fixes --- .gitignore | 3 +++ lekko_client/__init__.py | 6 +++--- lekko_client/clients/cached_git_client.py | 2 +- lekko_client/clients/distribution_client.py | 8 ++++++-- lekko_client/stores/memory.py | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 68bc17f..566ca4a 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# VSCode +.vscode \ No newline at end of file diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index af5455b..37b6ec5 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -58,11 +58,11 @@ def initialize(config: Config) -> Client: if __client: __client.close() match config: - case APIConfig(_) | SidecarConfig(_): + case APIConfig() | SidecarConfig(): __client = ConfigServiceClient( config.lekko_uri, config.owner_name, config.repo_name, config.api_key, config.context ) - case CachedGitConfig(_): + case CachedGitConfig(): __client = CachedGitClient( config.lekko_uri, config.owner_name, @@ -103,7 +103,7 @@ def wrapper(*args, **kwargs): with __client_lock: if not __client: raise exceptions.ClientNotInitialized("lekko_client.initialize() must be called prior to using API") - return func(args, kwargs) + return func(*args, **kwargs) return wrapper diff --git a/lekko_client/clients/cached_git_client.py b/lekko_client/clients/cached_git_client.py index 8a214b3..e2977b4 100644 --- a/lekko_client/clients/cached_git_client.py +++ b/lekko_client/clients/cached_git_client.py @@ -53,7 +53,7 @@ def __init__( self.path = path self.should_watch = should_watch - def initialize(self): + def initialize(self) -> None: super().initialize() self.load() if self.should_watch: diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index eb74d82..ca7d4c5 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -61,7 +61,10 @@ def upload_events(self): def run(self): # TODO: Lock while self._enabled: - self.upload_events() + try: + self.upload_events() + except: + log.warning("failed to send config evaluation events to lekko") time.sleep(self.upload_interval) def __init__( @@ -79,6 +82,7 @@ def __init__( self.repository = RepositoryKey(owner_name=owner_name, repo_name=repo_name) self.store = store self._client: Optional[DistributionServiceStub] = None + self.session_key = '' self.events_batcher = None if self.api_key: self.initialize_client(credentials) @@ -109,7 +113,7 @@ def initialize_client(self, credentials: grpc.ChannelCredentials): self.session_key = register_response.session_key @abstractmethod - def initialize(self): + def initialize(self) -> None: ... def load(self) -> bool: diff --git a/lekko_client/stores/memory.py b/lekko_client/stores/memory.py index f9d4d59..2c4857c 100644 --- a/lekko_client/stores/memory.py +++ b/lekko_client/stores/memory.py @@ -9,7 +9,7 @@ class MemoryStore(Store): - def __init__(self): + def __init__(self) -> None: super().__init__() self.configs: Dict[str, Dict[str, FeatureData]] = {} From 426a3db8c9a8e5267ed2afdece8898b07d8483b3 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Thu, 2 Nov 2023 16:14:20 -0700 Subject: [PATCH 18/28] fix --- lekko_client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 37b6ec5..936cb89 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -72,7 +72,7 @@ def initialize(config: Config) -> Client: config.api_key, config.context, ) - case CachedServerConfig(_): + case CachedServerConfig(): __client = CachedBackendClient( config.lekko_uri, config.owner_name, config.repo_name, MemoryStore(), config.api_key, config.context ) From 2f6305e0b671fe843cca6372c169e5dbeb6ca807 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 09:49:00 -0700 Subject: [PATCH 19/28] moar fixes --- example.py | 34 +++++++++---------- lekko_client/__init__.py | 18 ++++++++-- lekko_client/clients/cached_backend_client.py | 5 +-- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/example.py b/example.py index 42ac8d6..d3f2f01 100644 --- a/example.py +++ b/example.py @@ -14,9 +14,9 @@ parser.add_argument("--owner", type=str) parser.add_argument("--repo", type=str) parser.add_argument("--namespace", type=str, default="default") - parser.add_argument("--feature", type=str, default="example") + parser.add_argument("--config", type=str, default="example") parser.add_argument( - "--feature-type", + "--config-type", type=str, choices=["bool", "int", "float", "str", "json", "proto"], default="bool", @@ -43,17 +43,17 @@ val = None try: - if args.feature_type == "bool": - val = lekko_client.get_bool(args.namespace, args.feature, {}) - elif args.feature_type == "int": - val = lekko_client.get_int(args.namespace, args.feature, {}) - elif args.feature_type == "str": - val = lekko_client.get_string(args.namespace, args.feature, {}) - elif args.feature_type == "json": - val = lekko_client.get_json(args.namespace, args.feature, {}) - elif args.feature_type == "float": - val = lekko_client.get_float(args.namespace, args.feature, {}) - elif args.feature_type == "proto": + if args.config_type == "bool": + val = lekko_client.get_bool(args.namespace, args.config, {}) + elif args.config_type == "int": + val = lekko_client.get_int(args.namespace, args.config, {}) + elif args.config_type == "str": + val = lekko_client.get_string(args.namespace, args.config, {}) + elif args.config_type == "json": + val = lekko_client.get_json(args.namespace, args.config, {}) + elif args.config_type == "float": + val = lekko_client.get_float(args.namespace, args.config, {}) + elif args.config_type == "proto": if not args.proto_file: print( "Must provide a --proto-file. Could be a path or a well-known proto like 'google/protobuf/wrappers.proto'" @@ -63,11 +63,11 @@ if args.proto_type: msg_type = getattr(imported_proto, args.proto_type) - val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.feature, {}, msg_type)) + val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.config, {}, msg_type)) else: - val = MessageToDict(lekko_client.get_proto(args.namespace, args.feature, {})) - print(f"Got {val} for feature {args.namespace}/{args.feature}") + val = MessageToDict(lekko_client.get_proto(args.namespace, args.config, {})) + print(f"Got {val} for config {args.namespace}/{args.config}") except LekkoError as e: - print(f"Failed to get feature: {e}") + print(f"Failed to get config: {e}") if e.__cause__: print(f"Caused by: {e.__cause__}") diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 936cb89..2bde0fa 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -13,6 +13,8 @@ CachedGitClient, Client, ConfigServiceClient, + APIClient, + SidecarClient, ) from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa from lekko_client.stores import MemoryStore @@ -58,9 +60,19 @@ def initialize(config: Config) -> Client: if __client: __client.close() match config: - case APIConfig() | SidecarConfig(): - __client = ConfigServiceClient( - config.lekko_uri, config.owner_name, config.repo_name, config.api_key, config.context + case APIConfig(): + __client = APIClient( + owner_name=config.owner_name, + repo_name=config.repo_name, + api_key=config.api_key, + context=config.context, + ) + case SidecarConfig(): + __client = SidecarClient( + owner_name=config.owner_name, + repo_name=config.repo_name, + api_key=config.api_key, + context=config.context, ) case CachedGitConfig(): __client = CachedGitClient( diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index 53ee3fe..b7e7ef7 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -42,12 +42,13 @@ def __init__( api_key: Optional[str] = None, context: Optional[Dict[str, Any]] = None, credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials(), + update_interval_ms: int = 1000, ): - super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) + self.update_interval_ms = update_interval_ms self.timeout = None self.closed = False - self.update_interval_ms = 1000 self.initialized_event = Event() + super().__init__(uri, owner_name, repo_name, store, api_key, context, credentials) def initialize(self): super().initialize() From 4c6a5f4a3a96c32793b688de548ee6b2a6c03501 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 10:50:25 -0700 Subject: [PATCH 20/28] fix logging --- lekko_client/__init__.py | 3 +++ lekko_client/clients/distribution_client.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 2bde0fa..73feee9 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from threading import RLock from typing import Any, Dict, Optional, Type +import logging from google.protobuf.message import Message as ProtoMessage @@ -21,6 +22,8 @@ __version__ = "0.1.4" +logging.getLogger(__name__).addHandler(logging.NullHandler()) + __client: Optional[Client] = None __client_lock = RLock() diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index ca7d4c5..83d7abf 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -4,6 +4,7 @@ from datetime import datetime from threading import Thread from typing import Any, Dict, List, Optional, Type, TypeVar +import logging import grpc from google.protobuf import descriptor_pool as proto_descriptor_pool @@ -33,6 +34,8 @@ from lekko_client.models import ClientContext, FeatureData from lekko_client.stores.store import Store +log = logging.getLogger(__name__) + class CachedDistributionClient(Client): class EventsBatcher(Thread): @@ -64,7 +67,7 @@ def run(self): try: self.upload_events() except: - log.warning("failed to send config evaluation events to lekko") + log.warning("Failed to upload config evaluation events.") time.sleep(self.upload_interval) def __init__( From f7254e7a4e335711c27d3cfe5cc71cde4933eeef Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 11:38:47 -0700 Subject: [PATCH 21/28] fix events --- example.py | 78 +++++++++++++-------- lekko_client/clients/distribution_client.py | 29 +++++--- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/example.py b/example.py index d3f2f01..fca76ae 100644 --- a/example.py +++ b/example.py @@ -1,5 +1,7 @@ import argparse import dataclasses +import time +import logging import grpc from google.protobuf.json_format import MessageToDict @@ -8,6 +10,7 @@ from lekko_client.exceptions import LekkoError if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(threadName)s - %(levelname)s - %(message)s') parser = argparse.ArgumentParser() parser.add_argument("--apikey", type=str) parser.add_argument("--mode", type=str, choices=["sidecar", "api", "cachedapi", "cachedgit"]) @@ -24,10 +27,11 @@ parser.add_argument("--git-path", type=str, default="") parser.add_argument("--proto-type", type=str, default="") parser.add_argument("--proto-file", type=str) + parser.add_argument("-f", action="store_true", help="keep reading in a loop") + parser.add_argument("-n", type=int, help="number of time to read config in a loop", default=0) + parser.add_argument("--sleep", type=float, help="time in second to sleep between reading configs", default=0.1) args = parser.parse_args() - # client_cls = SidecarClient if args.sidecar else APIClient - base_config = lekko_client.Config(owner_name=args.owner, repo_name=args.repo, api_key=args.apikey) if args.mode == "sidecar": config = lekko_client.SidecarConfig(**dataclasses.asdict(base_config)) @@ -41,33 +45,45 @@ raise ValueError("Invalid mode") lekko_client.initialize(config) - val = None - try: - if args.config_type == "bool": - val = lekko_client.get_bool(args.namespace, args.config, {}) - elif args.config_type == "int": - val = lekko_client.get_int(args.namespace, args.config, {}) - elif args.config_type == "str": - val = lekko_client.get_string(args.namespace, args.config, {}) - elif args.config_type == "json": - val = lekko_client.get_json(args.namespace, args.config, {}) - elif args.config_type == "float": - val = lekko_client.get_float(args.namespace, args.config, {}) - elif args.config_type == "proto": - if not args.proto_file: - print( - "Must provide a --proto-file. Could be a path or a well-known proto like 'google/protobuf/wrappers.proto'" - ) - else: - imported_proto = grpc.protos(args.proto_file) - - if args.proto_type: - msg_type = getattr(imported_proto, args.proto_type) - val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.config, {}, msg_type)) + count = 0 + start = time.perf_counter_ns() + while True: + count += 1 + val = None + try: + if args.config_type == "bool": + val = lekko_client.get_bool(args.namespace, args.config, {}) + elif args.config_type == "int": + val = lekko_client.get_int(args.namespace, args.config, {}) + elif args.config_type == "str": + val = lekko_client.get_string(args.namespace, args.config, {}) + elif args.config_type == "json": + val = lekko_client.get_json(args.namespace, args.config, {}) + elif args.config_type == "float": + val = lekko_client.get_float(args.namespace, args.config, {}) + elif args.config_type == "proto": + if not args.proto_file: + print( + "Must provide a --proto-file. Could be a path or a well-known proto like 'google/protobuf/wrappers.proto'" + ) else: - val = MessageToDict(lekko_client.get_proto(args.namespace, args.config, {})) - print(f"Got {val} for config {args.namespace}/{args.config}") - except LekkoError as e: - print(f"Failed to get config: {e}") - if e.__cause__: - print(f"Caused by: {e.__cause__}") + imported_proto = grpc.protos(args.proto_file) + + if args.proto_type: + msg_type = getattr(imported_proto, args.proto_type) + val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.config, {}, msg_type)) + else: + val = MessageToDict(lekko_client.get_proto(args.namespace, args.config, {})) + #print(f"Got {val} for config {args.namespace}/{args.config}") + except LekkoError as e: + print(f"Failed to get config: {e}") + if e.__cause__: + print(f"Caused by: {e.__cause__}") + time.sleep(args.sleep) + if not args.f and count > args.n: + break + + total_ms = (time.perf_counter_ns() - start) / 1_000_000 + print(f"total ms: {total_ms}") + print(f"count: {count}") + print(f"qps: {count / (total_ms / 1_000)}") diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 83d7abf..39ba6a8 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -5,6 +5,7 @@ from threading import Thread from typing import Any, Dict, List, Optional, Type, TypeVar import logging +import queue import grpc from google.protobuf import descriptor_pool as proto_descriptor_pool @@ -39,36 +40,41 @@ class CachedDistributionClient(Client): class EventsBatcher(Thread): - def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval_ms: int): + def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval_ms: int, batch_size: int): super().__init__() self.daemon = True self.dist_client = dist_client - self.upload_interval = upload_interval_ms + self.upload_interval = upload_interval_ms / 1000 + self.batch_size = batch_size self.session_key = session_key self.events: List[FlagEvaluationEvent] = [] self._enabled = True + self.queue: queue.Queue[FlagEvaluationEvent] = queue.Queue() def stop(self): self._enabled = False def add_event(self, event: FlagEvaluationEvent): - self.events.append(event) + self.queue.put(event) def upload_events(self): if self.events: self.dist_client.SendFlagEvaluationMetrics( SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) ) - self.events = [] def run(self): - # TODO: Lock + last_upload_time = time.time() while self._enabled: - try: - self.upload_events() - except: - log.warning("Failed to upload config evaluation events.") - time.sleep(self.upload_interval) + event = self.queue.get() + self.events.append(event) + now = time.time() + if (len(self.events) > self.batch_size) or (now - last_upload_time > self.upload_interval): + try: + self.upload_events() + last_upload_time = now + except: + log.warning("Failed to upload config evaluation events.") def __init__( self, @@ -90,7 +96,7 @@ def __init__( if self.api_key: self.initialize_client(credentials) if self._client: - self.events_batcher = self.EventsBatcher(self._client, self.session_key, 15 * 1000) + self.events_batcher = self.EventsBatcher(self._client, self.session_key, upload_interval_ms=5 * 1_000, batch_size=100) self.events_batcher.start() self.initialize() @@ -215,4 +221,5 @@ def close(self): if self.events_batcher: self.events_batcher.upload_events() self.events_batcher.stop() + self.events_batcher.join(timeout=5) self._client.DeregisterClient(DeregisterClientRequest(session_key=self.session_key)) From a7882e9335899d419518b7afee07736092e98a11 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 11:44:29 -0700 Subject: [PATCH 22/28] fmt --- Makefile | 2 +- lekko_client/__init__.py | 17 ++++++++--------- lekko_client/clients/distribution_client.py | 16 ++++++++++------ tox.ini | 10 +++++----- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 66772b3..27af154 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON?=python +PYTHON?=python3 .PHONY: venv venv: venv/bin/touchfile diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index 73feee9..a8d53e7 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -1,20 +1,19 @@ """Lekko Python SDK Client""" import functools +import logging from dataclasses import dataclass from threading import RLock from typing import Any, Dict, Optional, Type -import logging from google.protobuf.message import Message as ProtoMessage from lekko_client import exceptions from lekko_client.clients import ( + APIClient, CachedBackendClient, CachedGitClient, Client, - ConfigServiceClient, - APIClient, SidecarClient, ) from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa @@ -65,16 +64,16 @@ def initialize(config: Config) -> Client: match config: case APIConfig(): __client = APIClient( - owner_name=config.owner_name, - repo_name=config.repo_name, - api_key=config.api_key, + owner_name=config.owner_name, + repo_name=config.repo_name, + api_key=config.api_key, context=config.context, ) case SidecarConfig(): __client = SidecarClient( - owner_name=config.owner_name, - repo_name=config.repo_name, - api_key=config.api_key, + owner_name=config.owner_name, + repo_name=config.repo_name, + api_key=config.api_key, context=config.context, ) case CachedGitConfig(): diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 39ba6a8..02a2748 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -1,11 +1,11 @@ import json +import logging +import queue import time from abc import abstractmethod from datetime import datetime from threading import Thread from typing import Any, Dict, List, Optional, Type, TypeVar -import logging -import queue import grpc from google.protobuf import descriptor_pool as proto_descriptor_pool @@ -40,7 +40,9 @@ class CachedDistributionClient(Client): class EventsBatcher(Thread): - def __init__(self, dist_client: DistributionServiceStub, session_key: str, upload_interval_ms: int, batch_size: int): + def __init__( + self, dist_client: DistributionServiceStub, session_key: str, upload_interval_ms: int, batch_size: int + ): super().__init__() self.daemon = True self.dist_client = dist_client @@ -73,7 +75,7 @@ def run(self): try: self.upload_events() last_upload_time = now - except: + except Exception: log.warning("Failed to upload config evaluation events.") def __init__( @@ -91,12 +93,14 @@ def __init__( self.repository = RepositoryKey(owner_name=owner_name, repo_name=repo_name) self.store = store self._client: Optional[DistributionServiceStub] = None - self.session_key = '' + self.session_key = "" self.events_batcher = None if self.api_key: self.initialize_client(credentials) if self._client: - self.events_batcher = self.EventsBatcher(self._client, self.session_key, upload_interval_ms=5 * 1_000, batch_size=100) + self.events_batcher = self.EventsBatcher( + self._client, self.session_key, upload_interval_ms=5 * 1_000, batch_size=100 + ) self.events_batcher.start() self.initialize() diff --git a/tox.ini b/tox.ini index bb08ef4..5b935fa 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = True envlist = clean - py{310,311} + py{311} lint typecheck report @@ -24,15 +24,15 @@ passenv = setenv = COVERAGE_FILE = {toxworkdir}/.coverage.{envname} depends: - py{310,311}: clean - report: py{310,311} + py{311}: clean + report: py{311} commands = coverage erase pytest --cov=lekko_client tests/ [testenv:fmt] description = black -basepython = python3.10 +basepython = python3.11 deps = black isort @@ -70,7 +70,7 @@ setenv = COVERAGE_FILE = {toxworkdir}/.coverage deps = coverage -depends = py{310,311} +depends = py{311} commands = coverage combine -q --keep coverage report From 6df3fc1afbdb646b0ca0a49f87b08aefb9154c55 Mon Sep 17 00:00:00 2001 From: David Kang Date: Fri, 3 Nov 2023 13:47:23 -0700 Subject: [PATCH 23/28] Fix events thread not joining, gracefully handle GetRespositoryVersion failing, fix tests --- lekko_client/__init__.py | 2 +- lekko_client/clients/cached_backend_client.py | 12 +++++++++--- lekko_client/clients/distribution_client.py | 5 ++++- pyproject.toml | 2 +- tests/conftest.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index a8d53e7..f944e61 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -19,7 +19,7 @@ from lekko_client.constants import LEKKO_API_URL, LEKKO_SIDECAR_URL # noqa from lekko_client.stores import MemoryStore -__version__ = "0.1.4" +__version__ = "0.2.0" logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lekko_client/clients/cached_backend_client.py b/lekko_client/clients/cached_backend_client.py index b7e7ef7..2ea24a7 100644 --- a/lekko_client/clients/cached_backend_client.py +++ b/lekko_client/clients/cached_backend_client.py @@ -1,3 +1,4 @@ +import logging import time from threading import Event, Thread from typing import Any, Dict, Optional @@ -14,6 +15,8 @@ ) from lekko_client.stores.store import Store +log = logging.getLogger(__name__) + class CachedBackendClient(CachedDistributionClient): class RefreshThread(Thread): @@ -71,9 +74,12 @@ def should_update_store(self) -> bool: return False if not self.initialized_event.is_set(): return True - version_response = self._client.GetRepositoryVersion( - GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key) - ) + try: + version_response = self._client.GetRepositoryVersion( + GetRepositoryVersionRequest(repo_key=self.repository, session_key=self.session_key) + ) + except Exception: + log.warning("Failed to fetch latest repository version", exc_info=True) current_sha = self.store.commit_sha return current_sha != version_response.commit_sha diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index 02a2748..ab1b436 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -68,7 +68,10 @@ def upload_events(self): def run(self): last_upload_time = time.time() while self._enabled: - event = self.queue.get() + try: + event = self.queue.get(block=False) + except queue.Empty: + continue self.events.append(event) now = time.time() if (len(self.events) > self.batch_size) or (now - last_upload_time > self.upload_interval): diff --git a/pyproject.toml b/pyproject.toml index 092e949..6ec702f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ profile = "black" skip_glob = ["*_pb2*.py*"] [tool.bumpversion] -current_version = "0.1.4" +current_version = "0.2.0" tag = true allow_dirty = true commit = false diff --git a/tests/conftest.py b/tests/conftest.py index 40df731..e843fcf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -317,7 +317,7 @@ def mock_store() -> Store: @pytest.fixture def mock_event_batcher(): - def mock_event_batcher_factory(self, client, session_key, interval): + def mock_event_batcher_factory(self, client, session_key, upload_interval_ms, batch_size): return mock.Mock(spec_set=CachedDistributionClient.EventsBatcher) return mock_event_batcher_factory From 3897fe7b0d54a251eab4c844a13736729b5910c5 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 11:53:27 -0700 Subject: [PATCH 24/28] bump version --- lekko_client/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index f944e61..a3352d1 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -88,7 +88,12 @@ def initialize(config: Config) -> Client: ) case CachedServerConfig(): __client = CachedBackendClient( - config.lekko_uri, config.owner_name, config.repo_name, MemoryStore(), config.api_key, config.context + config.lekko_uri, + config.owner_name, + config.repo_name, + MemoryStore(), + config.api_key, + config.context, ) case _: raise exceptions.LekkoError("Unknown client mode") @@ -116,7 +121,9 @@ def __get_safe(func): def wrapper(*args, **kwargs): with __client_lock: if not __client: - raise exceptions.ClientNotInitialized("lekko_client.initialize() must be called prior to using API") + raise exceptions.ClientNotInitialized( + "lekko_client.initialize() must be called prior to using API" + ) return func(*args, **kwargs) return wrapper From 4a4864a439ff13a3b6b0f84bcce5a598243d1586 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Fri, 3 Nov 2023 15:42:24 -0700 Subject: [PATCH 25/28] fmt --- lekko_client/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lekko_client/__init__.py b/lekko_client/__init__.py index a3352d1..bc2bff9 100644 --- a/lekko_client/__init__.py +++ b/lekko_client/__init__.py @@ -121,9 +121,7 @@ def __get_safe(func): def wrapper(*args, **kwargs): with __client_lock: if not __client: - raise exceptions.ClientNotInitialized( - "lekko_client.initialize() must be called prior to using API" - ) + raise exceptions.ClientNotInitialized("lekko_client.initialize() must be called prior to using API") return func(*args, **kwargs) return wrapper From 6430793252d80cbc76eace1c9707df41e9d947b6 Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Tue, 7 Nov 2023 13:31:20 -0800 Subject: [PATCH 26/28] drain events queue when stopping --- example.py | 5 ++- lekko_client/clients/distribution_client.py | 42 ++++++++++++--------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/example.py b/example.py index fca76ae..86907c1 100644 --- a/example.py +++ b/example.py @@ -74,16 +74,17 @@ val = MessageToDict(lekko_client.get_proto_by_type(args.namespace, args.config, {}, msg_type)) else: val = MessageToDict(lekko_client.get_proto(args.namespace, args.config, {})) - #print(f"Got {val} for config {args.namespace}/{args.config}") + # print(f"Got {val} for config {args.namespace}/{args.config}") except LekkoError as e: print(f"Failed to get config: {e}") if e.__cause__: print(f"Caused by: {e.__cause__}") time.sleep(args.sleep) - if not args.f and count > args.n: + if not args.f and count >= args.n: break total_ms = (time.perf_counter_ns() - start) / 1_000_000 print(f"total ms: {total_ms}") print(f"count: {count}") print(f"qps: {count / (total_ms / 1_000)}") + lekko_client.close() diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index ab1b436..c7a691f 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -53,33 +53,42 @@ def __init__( self._enabled = True self.queue: queue.Queue[FlagEvaluationEvent] = queue.Queue() - def stop(self): + def stop(self) -> None: self._enabled = False - def add_event(self, event: FlagEvaluationEvent): + def add_event(self, event: FlagEvaluationEvent) -> None: self.queue.put(event) - def upload_events(self): - if self.events: + def _upload_events(self) -> None: + if len(self.events) > 0: self.dist_client.SendFlagEvaluationMetrics( SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) ) - - def run(self): + self.events = [] + + def _accept_event(self) -> None: + try: + self.events.append(self.queue.get(block=False)) + except queue.Empty: + pass + + def run(self) -> None: last_upload_time = time.time() while self._enabled: - try: - event = self.queue.get(block=False) - except queue.Empty: - continue - self.events.append(event) + self._accept_event() now = time.time() - if (len(self.events) > self.batch_size) or (now - last_upload_time > self.upload_interval): + if (len(self.events) >= self.batch_size) or (now - last_upload_time > self.upload_interval): try: - self.upload_events() + self._upload_events() last_upload_time = now except Exception: - log.warning("Failed to upload config evaluation events.") + log.exception("Failed to upload config evaluation events.") + + while not self.queue.empty(): + self._accept_event() + if len(self.events) >= self.batch_size: + self._upload_events() + self._upload_events() def __init__( self, @@ -102,7 +111,7 @@ def __init__( self.initialize_client(credentials) if self._client: self.events_batcher = self.EventsBatcher( - self._client, self.session_key, upload_interval_ms=5 * 1_000, batch_size=100 + self._client, self.session_key, upload_interval_ms=5 * 1_000, batch_size=1_000 ) self.events_batcher.start() @@ -226,7 +235,6 @@ def close(self): super().close() if self._client and self.session_key: if self.events_batcher: - self.events_batcher.upload_events() self.events_batcher.stop() - self.events_batcher.join(timeout=5) + self.events_batcher.join(timeout=1) self._client.DeregisterClient(DeregisterClientRequest(session_key=self.session_key)) From 32a7aa3017fa3ce6d0ee1703eb3de03dacbbf48a Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Tue, 7 Nov 2023 13:32:57 -0800 Subject: [PATCH 27/28] fmt --- lekko_client/clients/distribution_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lekko_client/clients/distribution_client.py b/lekko_client/clients/distribution_client.py index c7a691f..c3e0d44 100644 --- a/lekko_client/clients/distribution_client.py +++ b/lekko_client/clients/distribution_client.py @@ -65,7 +65,7 @@ def _upload_events(self) -> None: SendFlagEvaluationMetricsRequest(events=self.events, session_key=self.session_key) ) self.events = [] - + def _accept_event(self) -> None: try: self.events.append(self.queue.get(block=False)) @@ -83,7 +83,7 @@ def run(self) -> None: last_upload_time = now except Exception: log.exception("Failed to upload config evaluation events.") - + while not self.queue.empty(): self._accept_event() if len(self.events) >= self.batch_size: From 2910cb7b5cf30ee37232efd5a1e8daf031b564cd Mon Sep 17 00:00:00 2001 From: Sergey Passichenko Date: Tue, 7 Nov 2023 14:01:41 -0800 Subject: [PATCH 28/28] add py312 --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 5b935fa..5e573e7 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ isolated_build = True envlist = clean - py{311} + py{311,312} lint typecheck report @@ -10,7 +10,7 @@ envlist = [gh-actions] python = 3.11: py311 - 3.10: py310 + 3.12: py312 [testenv] deps = @@ -24,8 +24,8 @@ passenv = setenv = COVERAGE_FILE = {toxworkdir}/.coverage.{envname} depends: - py{311}: clean - report: py{311} + py{311,312}: clean + report: py{311,312} commands = coverage erase pytest --cov=lekko_client tests/ @@ -70,7 +70,7 @@ setenv = COVERAGE_FILE = {toxworkdir}/.coverage deps = coverage -depends = py{311} +depends = py{311,312} commands = coverage combine -q --keep coverage report