diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 61c7871c..e4a4debe 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -22,4 +22,4 @@ jobs: poetry_version: ">=0.12" pypi_token: ${{ secrets.TEST_PYPI_TOKEN }} repository_name: "testpypi" - repository_url: "https://test.pypi.org/legacy/" + repository_url: "https://test.pypi.org/legacy/" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea9de4b..0f113e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added flake8 check workflow on pull_request event [#321](https://github.com/scanapi/scanapi/pull/321) +- Allow ${{}} exprs to call methods from external python modules [#324](https://github.com/scanapi/scanapi/pull/324) + +### Changed +- Updated poetry-publish version to v1.3 [#311](https://github.com/scanapi/scanapi/pull/311) ## [2.5.0] - 2021-07-23 ### Added diff --git a/scanapi/__main__.py b/scanapi/__main__.py index 5f48e48a..66ffec86 100644 --- a/scanapi/__main__.py +++ b/scanapi/__main__.py @@ -36,6 +36,13 @@ def main(): is_flag=True, help="Run ScanAPI without generating report.", ) +@click.option( + "-n", + "--no-report", + "no_report", + is_flag=True, + help="Run ScanAPI without generating report.", +) @click.option( "-c", "--config-path", diff --git a/scanapi/evaluators/code_evaluator.py b/scanapi/evaluators/code_evaluator.py index 03c4cad2..81a5c684 100644 --- a/scanapi/evaluators/code_evaluator.py +++ b/scanapi/evaluators/code_evaluator.py @@ -8,6 +8,7 @@ import uuid # noqa: F401 from scanapi.errors import InvalidPythonCodeError +from scanapi.evaluators.rmc_evaluator import RemoteMethodCallEvaluator logger = logging.getLogger(__name__) @@ -27,7 +28,14 @@ def evaluate(cls, sequence, vars, is_a_test_case=False): code = match.group("python_code") response = vars.get("response") + rmc_match = RemoteMethodCallEvaluator.pattern.match(code.strip()) + try: + if rmc_match: + return RemoteMethodCallEvaluator.evaluate( + code, vars, is_a_test_case, rmc_match + ) + if is_a_test_case: return cls._assert_code(code, response) diff --git a/scanapi/evaluators/rmc_evaluator.py b/scanapi/evaluators/rmc_evaluator.py new file mode 100644 index 00000000..a13f4001 --- /dev/null +++ b/scanapi/evaluators/rmc_evaluator.py @@ -0,0 +1,185 @@ +import ast +import re +import operator +import inspect +import importlib +from functools import partial +from typing import Dict, Any, Union, Callable, Tuple, Mapping +from scanapi import std + + +_sentinel = object() + + +def get_module(name: str): + """Import a module dynamically.""" + if name.lower() == "std": + return std + module = importlib.import_module(name) + print(f"Loaded {module}") + return module + + +def getname(location: str, root) -> Any: + """Get an ident's value from a given namespace.""" + trail = location.split(".") + node = root + for i, name in enumerate(trail): + try: + node = getattr(node, name) + except AttributeError: + raise AttributeError(f'No such location: {root}.{".".join(trail[:i + 1])}') + return node + + +def unroll_name(name: Union[str, ast.Attribute, ast.Name]) -> str: + """Unroll a ast.Name / ast.Attribute / str object.""" + if isinstance(name, str): + return name + if isinstance(name, ast.Name): + return name.id + return unroll_name(name.value) + "." + name.attr + + +def call_against_vars(func: Callable, args: Tuple, kwargs: Dict, vars: Mapping): + """ + Call a function against the spec vars + --- + Bind func to args/kwargs; + If function has no spec, feed vars as a single positional argument. + Feed `vars` as keyword arguments to func if func's spec share some of the same + names, including a reference to `vars` itself. + If `vars` and spec share no keys whatsoever, raise exception. + """ + + bound_func = func + + if args or kwargs: + bound_func = partial(bound_func, *args or (), **kwargs or {}) + + if getattr(func, "__module__", None) == "builtins": + return bound_func(vars) + + try: + spec = inspect.getfullargspec(func) + except TypeError: + return bound_func(vars) + + try: + vars = dict(vars) + except TypeError: + raise TypeError(f"vars={vars} is not dict-like") + + if "vars" not in vars: + vars["vars"] = vars + + feed_keys = {*spec.kwonlyargs, *spec.args} + + if not feed_keys & vars.keys(): + raise RuntimeError( + f"vars={vars.keys()} contain no key that function {func} expects as parameter" + ) + + return bound_func(**{key: vars[key] for key in vars.keys() & feed_keys}) + + +class RemoteMethodCallEvaluator: + + pattern = re.compile(r"^(?P[\w.]*)\s*:\s*(?P.*)$") + + @classmethod + def evaluate( + cls, + code: str, + vars: Dict[str, Any], + is_a_test_case: bool = False, + match=None, + ): + """ + Parse a remote method call (rmc) expression, then run it against input `vars`. + + A rmc expression is made of two fields separated by a colon (:): + * a module to import the function from: (eg 'pandas') + * the name location of the function (eg 'is_nan') + + mymodule.py: + + def ok(response): + return response.status_code == 200 + + def status_is(code, response): + return code == r esponse.status_code + + We can call this module's functions with rmc expressions. + You can omit any and all arguments, but if you do provide some, + they will be bound to the function before it gets to run. + + {{ mymodule:response.ok }} + # with positional arguments + {{ mymodule:response.status_is(200) }} + # or keyword arguments + {{ mymodule:response.status_is(code=200) }} + + Beware that partial binding binds from the left on positional arguments, so + you should expect your positional arguments to be fed through the expression + rather than from `vars`, ie don't write this but the above: + + ${{ mymodule:response.status_is(200) }} + + def status_is(response, code): # response would be 200 here and a collision would happen + ... + + `vars` is fed to the function as keyword arguments; only `vars` keys found in + the function's spec are fed to the function, so you can write stuff like this: + + def analyze_response(response): + ... + + with vars = {'response': ... , 'book_id': 333} + ${{ !mymodule.analyze_response }} + + to just be able to process the vars you're interested in. + """ + + code = str(code) + + # Parse expr + match = match or cls.pattern.match(code) + if match is None: + raise ValueError("Failed to parse expr: %r" % code) + + modulename, callcode = match.groups() + modulename = modulename or "std" + + expr = ast.parse(callcode, mode="eval").body + + name = None + args = None + kwargs = None + + if isinstance(expr, ast.Call): + name = unroll_name(expr.func) + args = [ast.literal_eval(arg) for arg in expr.args] + kwargs = {kw.arg: ast.literal_eval(kw.value) for kw in expr.keywords} + elif isinstance(expr, ast.Name): + name = expr.id + elif isinstance(expr, ast.Attribute): + name = unroll_name(expr) + else: + raise ValueError( + "Failed to parse %r as an attribute name or function call." % callcode + ) + # + + # Build function + module = get_module(modulename) + func = getname(name, module) + + result = call_against_vars(func, args, kwargs, vars) + + if is_a_test_case: + if operator.truth(result): + return (True, None) + return (False, code) + + return result diff --git a/scanapi/evaluators/spec_evaluator.py b/scanapi/evaluators/spec_evaluator.py index 22a0f9b6..e84b0d3b 100644 --- a/scanapi/evaluators/spec_evaluator.py +++ b/scanapi/evaluators/spec_evaluator.py @@ -39,6 +39,9 @@ def get(self, key, default=None): def __repr__(self): return self.registry.__repr__() + def keys(self): + return self.registry.keys() + def __getitem__(self, key): if key in self: return self.registry[key] diff --git a/scanapi/reporter.py b/scanapi/reporter.py index 8b843574..2404dec5 100644 --- a/scanapi/reporter.py +++ b/scanapi/reporter.py @@ -44,6 +44,20 @@ def write(self, results): logger.info("\nThe documentation was generated successfully.") logger.info(f"It is available at {abspath(self.output_path)}") + def write_without_generating_report(self, results): + """ Part of the Reporter instance that is responsible for writing the results without + generating the scanapi-report.html. + """ + logger.info("Writing results without generating report") + for r in results: + if logger.root.level == logging.DEBUG: + continue + else: + for test in r["tests_results"]: + logger.info(f" [{test['status'].upper()}] {test['name']}") + if test["status"] == "failed": + logger.info(f"\t {test['failure']} is false") + @staticmethod def write_without_generating_report(results): """Part of the Reporter instance that is responsible for writing the diff --git a/scanapi/std.py b/scanapi/std.py new file mode 100644 index 00000000..df327c0e --- /dev/null +++ b/scanapi/std.py @@ -0,0 +1,22 @@ +""" +A compendium of off-the-shelf operations for rmc expressions. +""" + + +class response: + + # ${{ std:response.status_is(200) }} + def status_is(code: int, *, response) -> bool: + return response.status_code == code + + # ${{ std:response.status_in_range(200, 204) }} + def status_in(*codes: int, response) -> bool: + return response.status_code in codes + + # ${{ std:response.status_in_range(200, 299) }} + def status_in_range(start: int, end: int, *, response) -> bool: + return response.status_code in range(start, end) + + # ${{ std:response.ok }} + def ok(response) -> bool: + return response.status_code in range(200, 300) diff --git a/tests/unit/evaluators/test_rmc_evaluator.py b/tests/unit/evaluators/test_rmc_evaluator.py new file mode 100644 index 00000000..f41d0912 --- /dev/null +++ b/tests/unit/evaluators/test_rmc_evaluator.py @@ -0,0 +1,153 @@ +import ast +from unittest.mock import Mock + +import pytest +import scanapi +from scanapi.evaluators import rmc_evaluator as rmc + + +rmc_eval = rmc.RemoteMethodCallEvaluator.evaluate + + +class TestRemoteMethodCallEvaluator: + class TestGetName: + def test_expected_behavior(self): + assert rmc.getname("evaluators.rmc_evaluator", scanapi) == rmc + import functools + + assert ( + rmc.getname("partial.__module__", functools) + == functools.partial.__module__ + ) + + class TestWhenNameDoesntExist: + def test_should_raise_attribute_error(self): + with pytest.raises(AttributeError) as excinfo: + rmc.getname("evaluators.rmc_evaltor", scanapi) == rmc + assert str(excinfo.value) == ( + f"No such location: {scanapi}.evaluators.rmc_evaltor" + ) + + class TestUnrollName: + def test_expected_behavior(self): + assert rmc.unroll_name(ast.parse("a", mode="eval").body) == "a" + assert rmc.unroll_name(ast.parse("a.c", mode="eval").body) == "a.c" + assert rmc.unroll_name(ast.parse("a.b.c", mode="eval").body) == "a.b.c" + + class TestEvaluate: + class TestWhenCodeRightMemberIsInvalid: + def test_expected_behavior(self): + with pytest.raises(ValueError) as excinfo: + rmc_eval("a:b + c", {}) + assert str(excinfo.value) == ( + "Failed to parse 'b + c' as an attribute name or function call." + ) + + class TestOnlyBareName: + def test_expected_behavior(self): + # local path module + assert not rmc_eval( + "scanapi.std:response.ok", {"response": Mock(status_code=400)} + ) + assert rmc_eval( + "scanapi.std:response.ok", {"response": Mock(status_code=200)} + ) + + # no spec + assert rmc_eval("builtins:str", {"object": {4}}) == "{'object': {4}}" + + # spec but doesn't take keyword arguments + assert rmc_eval("builtins:list", {"iterable": (4, 5)}) == ["iterable"] + + class TestExprWithPositionalArgs: + def test_expected_behavior(self): + assert rmc_eval( + "scanapi.std:response.status_is(200)", + {"response": Mock(status_code=200)}, + ) + assert not rmc_eval( + "scanapi.std:response.status_is(200)", + {"response": Mock(status_code=400)}, + ) + + class TestExprWithKeywordArgs: + def test_expected_behavior(self): + assert rmc_eval( + "scanapi.std:response.status_is(code=200)", + {"response": Mock(status_code=200)}, + ) + assert not rmc_eval( + "scanapi.std:response.status_is(code=200)", + {"response": Mock(status_code=400)}, + ) + + class TestExprWithStdConst: + def test_expected_behavior(self): + assert not rmc_eval( + "std:response.ok", {"response": Mock(status_code=400)} + ) + assert not rmc_eval(":response.ok", {"response": Mock(status_code=400)}) + + class TestGetModule: + def test_expected_behavior(self): + assert hasattr(rmc.get_module("scanapi.std"), "response") + assert hasattr(rmc.get_module("operator"), "__add__") + assert hasattr(rmc.get_module("pathlib"), "Path") + + class TestCallAgainstVars: + def test_expected_behavior(self): + def f(a: int, b: int) -> int: + return a + b + + assert rmc.call_against_vars(f, (), {}, {"a": 3, "b": 4}) == 7 + assert rmc.call_against_vars(f, (), {"b": 4}, {"a": 3}) == 7 + assert rmc.call_against_vars(f, (3,), {}, {"b": 4}) == 7 + + class TestWhenVarsIsEmpty: + def test_should_raise_runtime_error(self): + def f(a: int, b: int) -> int: + return a + b + + # vars should never be empty, so this fails + vars = {} + vars["vars"] = vars + with pytest.raises(RuntimeError) as excinfo: + assert rmc.call_against_vars(f, (3,), {"b": 4}, vars) == 7 + + assert str(excinfo.value) == ( + f"vars={vars.keys()} contain no key that function {f} expects as parameter" + ) + + class TestWhenVarsIsNotDictLike: + def test_should_raise_type_error(self): + with pytest.raises(TypeError) as excinfo: + assert rmc.call_against_vars(lambda m: m, (), {}, 44) + assert str(excinfo.value) == "vars=44 is not dict-like" + + class TestWhenFuncIsBuiltin: + def test_should_use_vars_as_pos_arg(self): + vars = {"a": 3} + assert rmc.call_against_vars(str, (), {}, vars) == str(vars) + assert rmc.call_against_vars(list, (), {}, vars) == list(vars) + + class TestWhenVarsIsPartOfSpec: + def test_should_feed_vars_as_keyword_arg(self): + def f(vars) -> int: + return vars["a"] + vars["b"] + + assert rmc.call_against_vars(f, (), {}, {"a": 3, "b": 4}) == 7 + + class TestWhenSpecAndVarsShareNoCommonKeys: + def test_should_raise_runtime_error(self): + def f(args) -> int: + return args["a"] + args["b"] + + vars = {"a": 3, "b": 4} + vars["vars"] = vars + + with pytest.raises(RuntimeError) as excinfo: + rmc.call_against_vars(f, (), {}, vars) == 7 + + assert str(excinfo.value) == ( + f"vars={vars.keys()} contain no key that function {f} expects as parameter" + ) diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index daa529d3..e86967ab 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -143,6 +143,7 @@ def test_should_clean_and_save_preferences(self): "reporter": None, "output_path": "path/output-path", "template": None, + "no_report": False, "config_path": "path/config-path", } )