diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..acb1e62a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,46 @@ +version: 2.1 + +orbs: + python: circleci/python@1.2 + +jobs: + build-and-test: + docker: + - image: cimg/python:3.12 + steps: + - checkout + - python/install-packages: + pip-dependency-file: requirements-test.txt + pkg-manager: pip + - run: + name: Run tests + command: pytest -m "not compiled" --cov=pythonwhat + publish: + docker: + - image: cimg/python:3.9 + steps: + - checkout + - run: + command: | + python setup.py sdist bdist_wheel + pip install pipenv + pipenv install twine + pipenv run twine upload --verbose --repository pypi dist/* + +workflows: + build: + jobs: + - build-and-test: + context: org-global + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ + - publish: + context: org-global + requires: + - build-and-test + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ + branches: + ignore: /.*/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..7f0089f5 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,20 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.5 + install: + - requirements: requirements.txt + - method: pip + path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3f28e872..00000000 --- a/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -sudo: false -language: python - -python: - - "3.5" - -before_install: - - pip install -r requirements.txt - - pip install -e . - -script: pytest -m "not compiled" --cov=pythonwhat - -after_success: codecov --token=$CODECOV_TOKEN - -deploy: - provider: pypi - user: machow - password: - secure: Jk24S6GmgT7MPym5ZCE1Rm4jeKISu/cJcEsrmMa/Tl3SQClGqvdgXjfyqEbe+xyEcdx4XWzyDsKT9dST/w6ngn5HmYkw9+JcqROfhANY5z9pTq7vT2wab2EjJcKxFXgMtgmojm1galUCsRhxB45t4nF7t8oDnQvRqAIoCxMJ3XFhdAgOatGMFDzCielEE60gyuEMVMXUoYEcyDIdCeWzdKHpzXTNVPWlKWGro1omGnJ3KwOLHi9Vw+Bmi7KavNGuHOZvouMkcBDJhDBqGE/OMtL4pc17PSgCKHeSxhw5mH+RQAwHAlL1obHahFACyfHbK2q2Wt/rWPkJqsGjPeXZGcgNyzKD+ily0aORXgY7jc9WEp6asWY65LjgiPqAMTspiYuBvioqro+TzKlpfase9UFRXioUwuFvFqLZO47mEgpbrEV5NX8uvCJ0TZs8WJNQ6AecJXl2Ql94aAgSkJ0BJfv9u5m4MSz6Ntdg/I6dXz8C9WgYkuMvSzCJfTdBLbPtzczi+vFvu8sxNlC9Sf+8EB+HC8gerSnPlFBz3Fzi7kvNqRC8/nmZfXNfsLlNNASb+iCpWrIkcS488oGi3sPdd3ESfOi51UMKaFDGoRS53Br38fbLoMWg1pvMXtyvSpIn3XR2racJuIVvd/qyaSG7t//5B2JT86eNlGGWE4RStlI= - on: - tags: true - distributions: sdist bdist_wheel - repo: datacamp/pythonwhat - skip_upload_docs: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2e8d2d..cd1ab6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,145 @@ All notable changes to the pythonwhat project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 2.24.0 + +- Support Python version 3.9 +- Update dependencies to up-to-date versions + +## 2.23.2 + +- Update behaviour of has_expr() default feedback message. If the student's evaluation is too long, it is now shortened and an ellipsis is added. + +## 2.23.1 + +- Fix string formatting in has_expr(). Strings will now have quotes in error messages. Leading and trailing whitespace is no longer removed. + +## 2.23.0 + +- Update to protowhat v2 (embedding xwhats, `prepare_validation` helper, for checking bash history, autodebug) + +## 2.22.0 + +- Add support for replacing the placeholder `__focus__` with the focused code in the `expr_code` argument +- Expose bash history functionality from protowhat + +## 2.21.0 + +- Use path info in `run` + +## 2.20.2 + +- Improve checking exceptions +- Improve checking file content + +## 2.20.1 + +- Speed improvements + +## 2.20.0 + +- Expose `_debug` function +- Update protowhat + +## 2.19.0 + +- `state` is now the first argument to SCT functions (instead of a keyword argument) +- functionality is shared with protowhat: + - `Reporter`, `Chain`, `F` and logic SCTs are reused + - `State` and `Feedback` inherit + - `Dispatcher` handles parsing (decoupled from `State`) + - `Test` moved + +## 2.18.1 + +### Fixed + +- Local setup with `setup_state` now works correctly (e.g. when the PEC contains a list comprehension) + +## 2.18.0 + +- Add optional `force_diagnose` parameter to `test_exercise` to force passing the `diagnose` tests in `check_correct`. + +## 2.17.2 + +### Improved + +Documentation has been improved significantly (hopefully): + +- Every 'compound statement check' has an example with explanation in the reference now. +- Common usecases have additional examples +- Checking compound statements article has been simplified, limits to context now. +- More helpful message in case check_object or has_printout not called on root state +- More helpful message in case the zoomed in on object is not an AST +- Explain has_equal_value vs has_equal_ast +- Get rid of some articles in favor of more fleshed out reference documentation + +### Fixed + +- CI is now using the `datacamp` account on PyPi + +## 2.17.1 + +### Fixed + +- Tuples of Numpy arrays can now be checked properly. + +### Removed + +- Code in `ObjectAssignmentParser` that is not used. + +## 2.17.0 + +### Added + +- Function `has_no_error()` to check earlier on whether the student did not generate any errors. + +### Improved + +- Messaging between V1 and V2 is entirely consistent now. +- No need for `__JINJA__` prefix in custom messages specified in SCTs anymore. + +### Removed + +- No more support for `expand_message` argument in old 'node checking functions' such as `test_for_loop()`. + +## 2.16.2 + +### Added + +- Ability to check class definitions with ``check_class_def()`` (and ``check_bases()``). Tested and documented. + +## 2.16.1 + +### Changed + +- The `check_object()` only on root check is only done if `PYTHONWHAT_V2_ONLY` environment variable is set. + +### Added + +- More checks that guard against commonly made mistakes (some in v2 only, others not) + +## 2.16.0 + +### Added + +- Function documentation (with examples) for `multi()`, `check_or()`, `check_correct()` and `check_not()` + +### Changed + +- If an SCT is incorrectly coded, it will generate more easily understandable errors so the author can easily fix the issue. +- If an SCT correctly runs but does not make a lot of sense, easily understanble errors will be thrown so the author can make improvements. + +## 2.15.3 + +### Added + +- `check_object()` does not check whether the targeted object is specified in the solution process if + student and solution process are identical (as is the case in the `SingleProcessExercise`). +- `has_expr()`, the function used by `has_equal_value()`, `has_equal_output()` and `has_equal_error()` can take an `override` argument, + that causes the solution expression _not_ to run and use the value specified in `override` instead. + For more information, have a look at the 'SingleProcessExercise' article on the documentation. + ## 2.15.2 ### Removed @@ -116,9 +255,9 @@ _Small changes to follow up on `2.13.0`_ ```python # Check whether the function was called: - Ex().check_function('df.a.sum', signature = False) + Ex().check_function('df.a.sum', signature=False) # Check whether the function was called and generated the correct result: - Ex().check_function('df.a.sum', signature = False).has_equal_value() + Ex().check_function('df.a.sum', signature=False).has_equal_value() ``` This update means that you should no longer need to use two `has_equal_ast()`'s inside a `test_correct()` to allow for two @@ -157,7 +296,7 @@ _Small changes to follow up on `2.13.0`_ - You can now robustly test printouts with `Ex().has_printout(index = x)`. This function will look for the `x`'th `print()` call in the solution code, rerun that call while capturing the output, and then look for that output in the output that was generated by the student's code submission. ```python - Ex().has_printout(index = 0) + Ex().has_printout(index=0) ``` This approach is far easier and more robust than using `Ex().check_function().check_args().has_equal_value()`. @@ -239,7 +378,7 @@ _Small changes to follow up on `2.13.0`_ ```Python from pythonwhat.local import setup_state -s = setup_state(sol_code = "x = 4", stu_code = "x = 5") +s = setup_state(sol_code="x = 4", stu_code="x = 5") s.check_object('x').has_equal_value() # # pythonwhat.Test.TestFail: Check the variable `x`. Unexpected expression value: expected `4`, got `5`. @@ -249,4 +388,4 @@ s.check_object('x').has_equal_value() ### Removed -- There is no support for `keep_objs_in_env`, as nobody is using it. \ No newline at end of file +- There is no support for `keep_objs_in_env`, as nobody is using it. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..096dd503 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include requirements.txt diff --git a/README.md b/README.md index 1f69dee7..f50b0e0b 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # pythonwhat [![Build Status](https://travis-ci.org/datacamp/pythonwhat.svg?branch=master)](https://travis-ci.org/datacamp/pythonwhat) -[![codecov](https://codecov.io/gh/datacamp/pythonwhat/branch/master/graph/badge.svg)](https://codecov.io/gh/datacamp/pythonwhat) [![PyPI version](https://badge.fury.io/py/pythonwhat.svg)](https://badge.fury.io/py/pythonwhat) [![Documentation Status](https://readthedocs.org/projects/pythonwhat/badge/?version=stable)](http://pythonwhat.readthedocs.io/en/stable/?badge=stable) +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat?ref=badge_shield) Verify Python code submissions and auto-generate meaningful feedback messages. Originally developed for Python exercises on DataCamp for so-called Submission Correctness Tests, but can also be used independently. -- New to teaching on DataCamp? Check out https://authoring.datacamp.com. -- To learn what SCTs are and how they work, visit [this article](https://authoring.datacamp.com/courses/exercises/technical-details/sct.html) specifically. +- New to teaching on DataCamp? Check out https://instructor-support.datacamp.com +- To learn what SCTs are and how they work, visit [this article](https://instructor-support.datacamp.com/courses/course-development/submission-correctness-tests) specifically. - For a complete overview of all functionality inside pythonwhat and articles about what to use when, consult https://pythonwhat.readthedocs.io. ## Installation @@ -33,7 +33,7 @@ _, ctxt = prep_context() globals().update(ctxt) # initialize state with student and solution submission -from pythonwhat.local import setup_state +from pythonwhat.test_exercise import setup_state setup_state(stu_code = "x = 5", sol_code = "x = 4") Ex().check_object('x') @@ -48,15 +48,21 @@ dir(Ex()._state) # list all elements available in the state object Ex()._state.student_code # access student_code of state object ``` -To learn how to include an SCT in a DataCamp course, visit https://authoring.datacamp.com. +To learn how to include an SCT in a DataCamp course, visit https://instructor-support.datacamp.com. ## Run tests -``` -pyenv local 3.5.2 -pip install -r requirements.txt -pip install -e . +```bash +pyenv local 3.12.7 +pip3.12 install -r requirements-test.txt +pip3.12 install -e . pytest ``` +## Contributing + Bugs? Questions? Suggestions? [Create an issue](https://github.com/datacamp/pythonwhat/issues/new), or [contact us](mailto:content-engineering@datacamp.com)! + +## License + +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat?ref=badge_large) diff --git a/docs/articles/checking_compound_statements.rst b/docs/articles/checking_compound_statements.rst index 305b8051..13c7022e 100644 --- a/docs/articles/checking_compound_statements.rst +++ b/docs/articles/checking_compound_statements.rst @@ -15,7 +15,7 @@ The ``if`` statement example in the tutorial describes how different ``check_`` producing a child state, to which additional SCT functions can be chained. The ``check_if_else()`` function scanned the code for an ``if`` statement, and broke it into three parts: a ``test``, the ``body`` and the ``orelse`` part; the former two were dived into with the SCT functions ``check_test`` and ``check_body``. Notice that the naming is consistent: the ``test`` part that ``check_if_else()`` surfaces can be inspected with ``check_test()``. -The ``body`` part that ``check_if_else()`` unearths can be inspected with ``check_ifs``. +The ``body`` part that ``check_if_else()`` unearths can be inspected with ``check_body``. Similar to how ``if`` statements has a ``check_if_else`` associated with it, all other compound statements have corresponding ``check_`` functions to perform this action of looking up a statement, @@ -25,251 +25,27 @@ and chopping it up into its constituents that can be inspected with ``check_ 0: - print("x is strictly positive") - - # sct - Ex().check_if_else().multi( - check_test().has_code(/x\s+>\s+0/), # chain A - check_body().check_function('print', 0).check_args('value').has_equal_value() # chain B - ) - -It checks whether the student specified an ``if`` statement, but the check of the ``test`` part is not very robust. -The SCT would not accept a submission that contains ``0 < x`` in the test. - -To increase robustness, pythonwhat features functionality to rerun pieces of the student and solution code, -and then check the resulting value, printed output, or errors they produce. - -Consider the same solution, an example submission, and a better SCT: - -.. code:: - - # solution - x = 4 - if x > 0: - print("x is strictly positive") - - # sct - Ex().check_if_else().multi( - check_test().multi( - set_env(x = -1).has_equal_value(), # chain A1 - set_env(x = 1).has_equal_value(), # chain A2 - set_env(x = 0).has_equal_value() # chain A3 - ), - check_body().check_function('print', 0).check_args('value').has_equal_value() # chain B - ) - - # example student submission - x = 4 - if x >= 0: - print("x is strictly positive") - - -In this SCT, `chain A` has been made more advanced: - -- As explained previously, ``check_test()`` zooms on the 'test portions' of the if statement in both student (``x >= 0``) and solution code (``x > 0``) -- Instead of ``has_code()``, we are using a series of ``set_env().has_equal_value()`` calls. - ``set_env()`` will temporarily override the value of ``x`` in the student/solution process, - and ``has_equal_value()`` will re-execute the pieces of code that are zoomed in on, and compare the results. - - + chain A1 re-executes ``x >= 0`` in the student process and ``x > 0`` in the solution process when ``x`` is equal to ``-1``, and checks if the results are the same, which they are: they both evaluate to ``False``. - + chain A2 re-executes ``x >= 0`` in the student process and ``x > 0`` in the solution process when ``x`` is equal to ``1``, and checks if the results are the same, which they are: they both evaluate to ``True``. - + chain A3 re-executes ``x >= 0`` in the student process and ``x > 0`` in the solution process when ``x`` is equal to ``1``, and checks if the results are the same. - They are not: the student expression evaluates to ``False``, while the solution expressione evaluates to ``True``. - -With this example student submission, the SCT will fail and pythonwhat will automatically generate a meaningful feedback message. -However, because we are not merely string-matching the SCT allows for much more ways of (correctly solving) the problem that would be hard to cater for with ``has_code()``: - -.. code:: - - # example student submission (passing) - x = 4 - if x > 0: - print("x is strictly positive") - - # example student submission (passing) - x = 4 - if 0 < x: - print("x is strictly positive") - -.. note:: - - Notice that ``has_equal_value()`` is also used in the context of checking objects and function arguments. - When checking objects, ``has_equal_value()`` is executing the expression ```` and comparing the result. - When checking function arguments, ``has_equal_value()`` executes the expression that specifies an argument. - They are different applications of the same concept: zooming in on a part of the student/solution submission, - rerunning the expressions and comparing the results. - - -``check_for()`` -~~~~~~~~~~~~~~~ - -The following example checks whether the student properly iterates over a dictionary and does the appropriate printouts: - -.. code:: - - # solution - my_dict = {'a': 1, 'b': 2} - for key, value in my_dict.items(): - print(key + " - " + str(value)) - - # sct - Ex().check_object('my_dict').has_equal_value() - Ex().check_for_loop().multi( - check_iter().has_equal_value(), - check_body().multi( - set_context('a', 1).has_equal_output(), - set_context('b', 2).has_equal_output() - ) - ) - -Unlike the ``if`` statement, the ``for`` loop introduces two context values, ``key`` and ``value``. -pythonwhat treats them different from regular variables like ``x`` in the previous example to be robust to students using different names for these context variables, -Similar to ``set_env()``, you can now use ``set_context()`` to temporarily override the values of these context variables. -Next, you can use ``has_equal_x()`` like before to rerun the body of the for loop for different situations. - -- The ``check_object()`` chain verifies that ``my_dict`` is properly initialized. -- ``check_for_loop()`` zooms in on the ``for`` loop, and makes its parts available for further checking. -- ``check_iter()`` zooms in on the iterator part of the for loop, ``my_dict.items()`` in the solution. - - + ``has_equal_value()`` re-executes the expressions specified by student and solution and compares their results. - -- ``check_body()`` zooms in on the body part of the for loop, ``print(key + " - " + str(value))``: - - + Similar to ``set_env()``, we now use ``set_context()`` to temporarily override the values of the context variables ``key`` and ``value``, in this order. - Notice that the context values are not specified by name, this is on purpose. - + ``has_equal_output()`` re-executes the entire for loop body and captures the output this generates. It does this for both the student and solution body, and checks if the outputs are equal. - -Because pythonwhat treats context values differently from normal variables and we're not specifying the variables by name in ``set_context()``, -we can make the SCT robust against submissions that code the correct logic, but use different names for the context values. -Consider the following student submissions that would also pass the SCT: - -.. code:: - - # passing submission 1 - my_dict = {'a': 1, 'b': 2} - for k, v in my_dict.items(): - print(k + " - " + str(v)) - - # passing submission 2 - my_dict = {'a': 1, 'b': 2} - for first, second in my_dict.items(): - mess = first + " - " + str(second) - print(mess) - - -``check_function_def()`` -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following example checks whether students correctly defined their own function: - -.. code:: - - # solution - def shout_echo(word1, echo=1): - echo_word = word1 * echo - shout_words = echo_word + '!!!' - return shout_words - - # sct - Ex().check_function_def('shout_echo').check_correct( - multi( - check_call("f('hey', 3)".has_equal_value(), - check_call("f('hi', 2)".has_equal_value(), - check_call("f('hi')".has_equal_value() - ), - check_body().set_context('test', 1).multi( - has_equal_value(name = 'echo_word'), - has_equal_value(name = 'shout_words') - ) - ) - -Here: - -- ``check_function_def()`` zooms in on the function definition of ``shout_echo`` in both student and solution code (and process) -- ``check_correct()`` is used to - + First check whether the function gives the correct result when called in different ways (through ``check_call()``). - + Only if these 'function unit tests' don't pass, `check_correct()` will run the `check_body()` chain that dives deeper into the - function definition body. This chain sets the context variables - ``word1`` and ``echo``, the arguments of the function - to - the values ``'test'`` and ``1`` respectively, again while being agnostic to the actual name of these context variables. - -Notice how ``check_correct()`` is used to great effect here: why check the function definition internals if the I/O of the function works fine? -Because of this construct, all the following submissions will pass the SCT: - -.. code:: - - # passing submission 1 - def shout_echo(w, e=1): - ew = w * e - return ew + '!!!' - - # passing submission 2 - def shout_echo(a, b=1): - return a * b + '!!!' - -elif statements -~~~~~~~~~~~~~~~ - -In Python, when an if-else statement has an ``elif`` clause, it is held in the `orelse` part. -In this sense, an if-elif-else statement is represented by python as nested if-elses. More specifically, this if-else statement: - -.. code:: - - if x: - print(x) - elif y: - print(y) - else: - print('none') - -Is syntactically equivalent to: - -.. code:: - - if x: - print(x) - else: - if y: - print(y) - else: - print('none') - -The second representation has to be followed when writing the corresponding SCT: - -.. code:: - - Ex().check_if_else() \ - .check_orelse().check_if_else() \ - .check_orelse().has_equal_output() - - -Crazy combo -~~~~~~~~~~~ - -Suppose you want to check whether a function definition containing a for loop was coded correctly. Here's an example: - -.. code:: - - # solution def counter(lst, key): count = 0 for l in lst: count += l[key] return count - # sct that robustly checks this +The following SCT would robustly verify this: + +.. code:: + Ex().check_function_def('counter').check_correct( multi( check_call("f([{'a': 1}], 'a')").has_equal_value(), @@ -283,7 +59,7 @@ Suppose you want to check whether a function definition containing a for loop wa Some notes about this SCT: -- ``check_correct()`` is again used so the body is not further checked if calling the function in different ways produces the same value in both student and solution process. +- ``check_correct()`` is used so the body is not further checked if calling the function in different ways produces the same value in both student and solution process. - ``set_context()`` is used twice. Once to set the context variables introduced by the function definition, and once to set the context variable introducted by the for loop. - ``set_env()`` had to be used to initialize ``count`` to a variable that was scoped only to the function definition. @@ -375,3 +151,9 @@ The table below summarizes all checks that pythonwhat supports to test compound | | | | | | | | +------------------------+------------------------------------------------------+-------------------+ +|check_class_def('f') | .. code:: | | +| | | | +| | class KLS(BASES[0], BASES[1]): | | +| | BODY | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ diff --git a/docs/articles/checking_function_calls.rst b/docs/articles/checking_function_calls.rst index fc1fcb75..c7808a23 100644 --- a/docs/articles/checking_function_calls.rst +++ b/docs/articles/checking_function_calls.rst @@ -72,7 +72,7 @@ Now, the following submissions would fail: - ``val=2.718282; dig=3; round(number=val, dig)`` -- same - ``int_part = 2; dec_part = 0.718282; round(int_part + dec_part, 3)`` -- the string representation of ``int_part + dec_part`` in the student code is compered to ``2.718282`` in the solution code. -As you can see, doing exact string comparison of arguments is not a good idea, as it is very inflexible. +As you can see, doing exact string comparison of arguments is not a good idea here, as it is very inflexible. There are cases, however, where it makes sense to use this, e.g. when there are very big objects passed to functions, and you don't want to spend the processing power to fetch these objects from the student and solution processes. @@ -93,6 +93,27 @@ If the student did not properly call the function, ``check_function()`` will aut No matter how you import the function, you always have to refer to the function with its full name, e.g. ``package.subpackage1.subpackage2.function``. +has_equal_value? has_equal_ast? +=============================== + +In the customizations section above, you could already notice the difference between ``has_equal_value()`` and ``has_equal_ast()`` for checking +whether arguments are correct. The former **reruns** the expression used to specify the argument in both student and solution process +and compares their results, while the latter simply compares the expression's AST representations. Clearly, the former is more robust, but there +are some cases in which ``has_equal_ast()`` can be useful: + +- For better feedback. When using ``has_equal_ast()``, the 'expected x got y' message that is automatically generated when the arguments + don't match up will use the actual expressions used. ``has_equal_value()`` will use string representations of the evaluations of the expressions, + if they make sense, and this is typically less useful. +- To avoid very expensive object comparisons. If you are 100% sure that the object people have to pass as an argument is already correct (because + you checked it earlier in the SCT or because it was already specified in the pre exercise code) and doing an equality check on this object between + student and solution project is likely going to be expensive, then you can safely use ``has_equal_ast()`` to speed things up. +- If you want to save yourself the trouble of building exotic contexts. You'll often find yourself checking function calls in e.g. a for loop. + Typically, these function calls will use objects that were generated inside the loop. To easily unit test the body of a for loop, you'll typically + have to use ``set_context()`` and ``set_env()``. For exotic for loops, this can become tricky, and it might be a quick fix to be a little more + specific about the object names people should use, and just use ``has_equal_ast()`` for the argument comparison. That way, you're bypassing the need + to build up a context in the student/solution process and do object comparisions. + + Signatures ========== @@ -323,7 +344,7 @@ Here: - The second SCT is first checking whether ``df.groupby.mean()`` was called and whether calling it gives the right result. Notice several things: + We describe the entire chain of method calls, leaving out the parentheses and arguments used for method calls in between. - + We use ``sig_from_obj()`` to manually specify a Python expression that ``pythonwhat`` can use to derive the signature from. + + We use ``sig_from_obj()`` to manually specify a Python expression that pythonwhat can use to derive the signature from. If the string you use to describe the function to check evaluates to a method or function in the solution process, like for ``'df.groupby'``, pythonwhat can figure out the signature. However, for ``'df.groupby.mean'`` will `not` evaluate to a method object in the solution process, so we need to manually specify a valid expression that `will` evaluate to a valid signature with ``sig_from_obj()``. diff --git a/docs/articles/checking_imports.rst b/docs/articles/checking_imports.rst deleted file mode 100644 index aa7f1dbf..00000000 --- a/docs/articles/checking_imports.rst +++ /dev/null @@ -1,40 +0,0 @@ -Checking imports ----------------- - -Python features many ways to import packages. All of these different methods revolve around the ``import``, ``from`` and ``as`` keywords. -``has_import()`` provides a robust way to check whether a student correctly imported a certain package. - -Take this example to check whether a studnet imported ``matplotlib.pyplot`` as ``plt``: - -.. code:: - - # solution - import matplotlib.pyplot as plt - - # sct - Ex().has_import("matplotlib.pyplot") - - # passing submissions - import matplotlib.pyplot as plt - from matplotlib import pyplot as plt - import matplotlib.pyplot as pltttt - - # failing submissions - import matplotlib as mpl - -By default, ``has_import()`` allows for different ways of aliasing the imported package or function. If you want to make sure the correct alias was used to refer to the package or function that was imported, set ``same_as=True``. - -.. code:: - - # solution - import matplotlib.pyplot as plt - - # sct - Ex().has_import("matplotlib.pyplot", same_as=True) - - # passing submissions - import matplotlib.pyplot as plt - from matplotlib import pyplot as plt - - # failing submissions - import matplotlib.pyplot as pltttt diff --git a/docs/articles/checking_objects.rst b/docs/articles/checking_objects.rst deleted file mode 100644 index 1335e745..00000000 --- a/docs/articles/checking_objects.rst +++ /dev/null @@ -1,119 +0,0 @@ -Checking objects ----------------- - -In ``pythonbackend``, both the student's submission as well as the solution code are executed, in separate processes. -``check_object()`` looks at these processes and checks if the referenced object is available in the student process. -Next, you can use ``has_equal_value()`` to check whether the objects in the student and solution process correspond. - -.. note:: - - For more information how DataCamp's coding backends run code, check out `this article `_ - -Basic example -============= - -Consider the following solution, and corresponding SCT: - -.. code:: - - # solution - x = 15 - - # sct option 1 - Ex().check_object("x").has_equal_value() - - # submissions that will pass this sct - x = 10 - x = 12 + 3 - x = 3; x += 12 - - -- ``check_object()`` will check if the variable ``x`` is defined in the student process. -- ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process. - - -.. note:: - - If you only use ``Ex().check_object("x")`` without ``has_equal_value()``, you are only checking whether the object is defined. - -.. caution:: - - ``has_equal_value()`` only looks at **end result** of a variable in the student process. In the example, how the object ``x`` came about in the student's submission, does not matter. - -Checking the type of an object -============================== - -You can use ``is_instance()`` after ``check_object()`` to verify the class of an object: - -.. code:: - - # solution - x = 3 - - # sct - Ex().object("x").is_instance(int) - - # passing submissions - x = 3 - x = 4 - - # failing submissions - x = '3' - x = '4' - - -Checking objects with keys -========================== - -You can use the `check_keys()` function to 'zoom in' on a particular key in a dictionary or a pandas DataFrame: - -.. code:: - - # solution - import pandas as pd - my_df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - - # sct - Ex().check_df("my_df").check_key("a").has_equal_value() - - # passing submissions - my_df = pd.DataFrame({"a": [1, 1 + 1, 3], "b": [4, 5, 6]}) - my_df = pd.DataFrame({"b": [4, 5, 6], "a": [1, 2, 3]}) - -Here `check_df()` does two things: - -+ it checks whether the object ``my_df`` is defined in the student process (``check_object()`` behind the scenes). -+ it checks whether ``my_df`` is an object of type `pandas.DataFrame` (using ``is_instance()`` behind the scene). - -Exotic objects -============== - -pythonwhat compares the objects in the student and solution process with the ``==`` operator. -For basic objects, this ``==`` is operator is properly implemented, so that the objects can be effectively compared. -For more complex objects that are produced by third-party packages, however, it's possible that this equality operator is not implemented in a way you'd expect. -Often, for these object types the ``==`` will compare the actual object instances. - -.. code:: - - # pre exercise code - class Number(): - def __init__(self, n): - self.n = n - - # solution - x = Number(1) - - # sct that won't work - Ex().check_object().has_equal_value() - - # sct - Ex().check_object().has_equal_value(expr_code = 'x.n') - - # submissions that will pass this sct - x = Number(1) - x = Number(2 - 1) - -The basic SCT like in the previous example won't work here. -Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking. -Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n`` -and seeing if the result of running this expression in both student and solution process match. diff --git a/docs/articles/checking_output.rst b/docs/articles/checking_output.rst deleted file mode 100644 index c5fa09a8..00000000 --- a/docs/articles/checking_output.rst +++ /dev/null @@ -1,69 +0,0 @@ -Checking output ---------------- - -Checking any output -~~~~~~~~~~~~~~~~~~~ - -Among the student and solution process, the student submission and solution code as a string, -the ``Ex()`` state also contains the output that a student generated with his or her submission. - -With ``has_output()``, you can access this output and match it against a regular or fixed expression. - -As an example, suppose we want a student to print out a sentence: - -.. code:: - - # Print the "This is some ... stuff" - print("This is some weird stuff") - - -The following SCT tests whether the student prints out ``This is some weird stuff``: - -.. code:: - - # Using exact string matching - Ex().has_output("This is some weird stuff", pattern = False) - - # Using a regular expression (more robust) - pattern = True is the default - Ex().has_output(r"This is some \w* stuff", - no_output_msg = "Print out ``This is some ... stuff`` to the output, fill in ``...`` with a word you like.") - - -Checking ``print()`` calls -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Checking whether the right printouts were done is easy: - -.. code:: - - # solution - x = 4 - print(x) - - # sct - Ex().has_printout(0) - -``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in -the solution process, capture its output, and verify whether the output is present in the output of the student. - -Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed. -If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work: - -.. code:: - - # solution - x = 4 - print(x) - x = 6 - - # sct that won't work - Ex().has_printout(0) - -In this example, when the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated. -In cases like these, default to using the classical pattern to check function calls: - -.. code:: - - # sct that will work - Ex().check_function('print').check_args(0).has_equal_value()`` - diff --git a/docs/articles/electives.rst b/docs/articles/electives.rst index ea14c36d..9e577aed 100644 --- a/docs/articles/electives.rst +++ b/docs/articles/electives.rst @@ -12,7 +12,7 @@ When all tests in an SCT pass, pythonwhat will automatically generate a congratu success_msg("You are a hero when it comes to variable assignment!") -`This article `_ on the authoring docs describes how to write good success messages. +`This article `_ on the authoring docs describes how to write good success messages. Multiple choice exercises @@ -175,19 +175,10 @@ by name in the SCT above, they may also be given by position.. Ex().check_for_loop().check_body().set_context(0, 'a').has_equal_output() - -Instructor Errors -~~~~~~~~~~~~~~~~~ - -If you are unsure what variables can be set, it's often easiest to take a guess. -When you try to set context values that don't match any target variables in the solution code, -``set_context()`` raises an exception that lists the ones available. - - with_context ============ -.. autofunction:: pythonwhat.check_funcs.with_context +.. autofunction:: pythonwhat.checks.check_funcs.with_context :noindex: Runs subtests after setting the context for a ``with`` statement. @@ -216,21 +207,3 @@ This code runs by and replacing step (3) with any sub-tests given as arguments. -fail -==== - -.. autofunction:: pythonwhat.check_logic.fail - -Fails. This function takes a single argument, ``msg``, that is the feedback given to the student. -Note that this would be a terrible idea for grading submissions, but may be useful while writing SCTs. -For example, failing a test will highlight the code as if the previous test/check had failed. - -As a trivial SCT example, - -.. code:: - - Ex().check_for_loop().check_body().fail() # fails boo - -This can also be helpful for debugging SCTs, as it can be used to stop testing as a given point. - - diff --git a/docs/articles/make_your_sct_robust.rst b/docs/articles/make_your_sct_robust.rst index 88f6abc7..f3d94d50 100644 --- a/docs/articles/make_your_sct_robust.rst +++ b/docs/articles/make_your_sct_robust.rst @@ -111,7 +111,7 @@ or you can use ``check_or()`` with three separate ``has_code()`` functions: .. code:: - Ex().check_or(has_code('4'), + Ex().check_or(has_code('4'), has_code('5'), has_code('6')) diff --git a/docs/articles/processes.rst b/docs/articles/processes.rst index 877818ee..2823b5a4 100644 --- a/docs/articles/processes.rst +++ b/docs/articles/processes.rst @@ -1,7 +1,7 @@ Processes --------- -As explained on the `SCT authoring homepage `_, DataCamp's Python coding backends use uses two separate processes: one process to run the solution code, and one process to run the student's submission. +As explained on the `SCT authoring homepage `_, DataCamp's Python coding backends use uses two separate processes: one process to run the solution code, and one process to run the student's submission. As such, pythonwhat has access to the 'ideal ending scenario' of an exercises, which in turn makes it easier to write SCTs. Instead of having to specify which value an object should be, we can have pythonwhat look into the solution process and compare the object in that process with the object in the student process. diff --git a/docs/articles/single_process_exercise.rst b/docs/articles/single_process_exercise.rst new file mode 100644 index 00000000..e9a65a43 --- /dev/null +++ b/docs/articles/single_process_exercise.rst @@ -0,0 +1,106 @@ +SingleProcessExercise +--------------------- + +Introduction +============ + +Typical interactive exercises on DataCamp will be of the type ``NormalExercise`` or something similar. + +For these normal exercises, the pythonbackend (the Python package responsible +for running Python code that the student submitted) will execute: + +- the solution code in a solution process + (once, at exercise initialization), +- the student's submission in a student process + (every time the student hits submit, after which the process is restarted from scratch) +- the student's experimentation commands in the console in a console process + (every time the user executes a command, without restarting afterwards) + +These completely separate processes make sure that: + +- the different commands do not interfere with one another; + if you import a package in one process, + the package will not become available in the other process. +- pythonwhat has access to a 'target solution process' to easily do comparisons; + to compare an object ``x``, you simply have to use ``Ex().check_object('x').has_equal_value()`` and + pythonwhat will figure out the value ``x`` should have from the solution process. + +To learn more about how the backend works, you can visit +`this wiki article `_. + +Why does this exercise type exist? +================================== + +There are Python courses that make extensive use of programs running outside of Python. +Sometimes, these programs cannot handle it well when different +Python process are trying to interface with it. +An example of this is PySpark, where on container startup, +a Spark cluster is started up, that you can then interface with. +Things go horribly wrong if you try to access this PySpark cluster from different Python processes. + +To solve for this, a new exercise type was built, +that does not create three separate Python processes (solution, student, console). +Instead, only one process is created: + +- the solution code is not executed in this process. +- the student's sumission is executed in this process, but the process is not restarted afterwards +- the student's experimentation commands in the console are executed in the same process. + +From a user perspective, this shouldn't pose too much difficulties, +with the exception that the code execution is now stateful. + +So, what's the problem then? +============================ + +As mentioned earlier, pythonwhat depends heavily on the existence of +two separate process: a 'target' solution process, and a student process. +Functions such as ```has_equal_value()`` compare values and the results of expression in these processes. +In the ``SingleProcessExercise``, the student process and +the solution process are identical, it's one and the same process, +so these comparisons don't make any sense. + +Therefore, the 'process-based checks' in pythonwhat have to be used with care when +writing SCTs for a ``SingleProcessExercise``. More specifically: + +- ``check_object()`` should work okay, as there is some magic happening behind the scenes. +- ``has_equal_value()`` and ``has_equal_output()`` should be used with the ``override`` argument. + When this argument is specified, the expression that is 'zoomed in on' in the solution code will not be executed in the solution proces. + Instead, it will just take the value you pass to ``override`` to compare the result/output of the expression that is zoomed in on in the student code to. + +Example +======= + +As an example, suppose we want to check whether a student correctly created a list ``x``: + +.. code:: + + # solution + x = [1, 2, 3, 4, 5] + +If this solution were part of a traditional ``NormalExercise``, your SCT would be simple: + +.. code:: + + # SCT + Ex().check_object('x').has_equal_value() + +However, if this solution were part of a ``SingleProcessExercise``, the above SCT would not work. +Instead, you'll want to do the following: + +.. code:: + + # SCT + Ex().check_object('x').has_equal_value(override = [1, 2, 3, 4, 5]) + +Here, we use ``override`` to tell pythonwhat not to go look for the value of ``x`` in the solution process. +Instead, it uses the manually specified value in ``override`` to compare to. + +You can use ``override`` in combination with other arguments in ``has_equal_x()``, such as ``expr_code``. +Suppose you're only interested in the element at index 2 of the list ``x``: + +.. code:: + + # SCT + Ex().check_object('x').has_equal_value(expr_code = 'x[2]', override = 3) + +Tricky stuff, but it works! \ No newline at end of file diff --git a/docs/articles/tutorial.rst b/docs/articles/tutorial.rst index f0f6f8ce..ba97d5b4 100644 --- a/docs/articles/tutorial.rst +++ b/docs/articles/tutorial.rst @@ -227,6 +227,5 @@ However, when they do make a mistake, you want to be specific about the mistake These seemingly conflicting requirements can be satisfied with ``check_correct()``. It is an **extremely powerful function** that should be used whenever it makes sense. The `Make your SCT robust `_ article is highly recommended reading. -For other guidelines on writing good SCTs, check out the 'How to write good SCTs' section on DataCamp's `general SCT documentation page `_. +For other guidelines on writing good SCTs, check out the 'How to write good SCTs' section on DataCamp's `general SCT documentation page `_. - \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index c03b42cc..bace3095 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,12 +16,29 @@ import os import sys +import json +import pythonwhat + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) -import pythonwhat +# enables finding custom extensions in docs directory +sys.path.insert(0, os.path.abspath(".")) + +sys.path.insert(0, os.path.abspath("..")) + +TEST_DATA_FILE = "test_data.json" + +if not os.path.isfile(TEST_DATA_FILE): + import subprocess + + subprocess.call("pytest", cwd=os.path.abspath(os.path.join(os.getcwd(), os.pardir))) + +with open(TEST_DATA_FILE, "r") as read_file: + test_data = json.load(read_file) + +jinja_contexts = {"test_ctx": {"test_data": test_data}} # -- General configuration ------------------------------------------------ @@ -32,21 +49,25 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinxcontrib.jinja", +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'pythonwhat' -copyright = '2018, DataCamp' -author = 'DataCamp' +project = "pythonwhat" +copyright = "2019, DataCamp" +author = "DataCamp" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -67,14 +88,14 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -84,7 +105,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -101,19 +122,18 @@ # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'pythonwhatdoc' +htmlhelp_basename = "pythonwhatdoc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { } +latex_elements = {} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pythonwhat.tex', 'pythonwhat Documentation', - 'DataCamp', 'manual'), + (master_doc, "pythonwhat.tex", "pythonwhat Documentation", "DataCamp", "manual") ] @@ -121,10 +141,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pythonwhat', 'pythonwhat Documentation', - [author], 1) -] +man_pages = [(master_doc, "pythonwhat", "pythonwhat Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -132,8 +149,13 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pythonwhat', 'pythonwhat Documentation', - author, 'pythonwhat', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pythonwhat", + "pythonwhat Documentation", + author, + "pythonwhat", + "One line description of project.", + "Miscellaneous", + ) ] - diff --git a/docs/data_to_doc.py b/docs/data_to_doc.py new file mode 100644 index 00000000..d6d94f99 --- /dev/null +++ b/docs/data_to_doc.py @@ -0,0 +1,13 @@ +def render_data(app, docname, source): + """ + Render our pages as a jinja template for fancy templating goodness. + """ + if app.builder.format != "html": + return + src = source[0] + rendered = app.builder.templates.render_string(src, app.config.html_context) + source[0] = rendered + + +def setup(app): + app.connect("source-read", render_data) diff --git a/docs/glossary.rst b/docs/glossary.rst index 867386b0..3633ae83 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -211,96 +211,36 @@ Check output print("this is weird stuff") print("Thisis weird stuff") -Check import -~~~~~~~~~~~~ +Check Multiple Choice +~~~~~~~~~~~~~~~~~~~~~ .. code:: - # solution - import matplotlib.pyplot as plt + # solution (implicit) + # 3 is the correct answer # sct - Ex().has_import("matplotlib.pyplot") - - # passing submissions - import matplotlib.pyplot as plt - from matplotlib import pyplot as plt - import matplotlib.pyplot as pltttt + Ex().has_chosen(correct = 3, # 1-base indexed + msgs = ["That's someone who makes soups.", + "That's a clown who likes burgers.", + "Correct! Head over to the next exercise!"]) - # failing submissions - import matplotlib as mpl +Check import +~~~~~~~~~~~~ +`See has_import doc `_ Check if statement ~~~~~~~~~~~~~~~~~~ -.. code:: - - # solution - x = 4 - if x > 0: - print("x is strictly positive") - - # sct - Ex().check_if_else().multi( - check_test().multi([ has_equal_value(extra_env = {'x': i}) for i in [4, -1, 0, 1] ]), - check_body().check_function('print', 0).check_args('value').has_equal_value() - ) - - # passing submission - x = 4 - if 0 < x: - print("x is strictly positive") +`See check_if_else doc `_ Check function definition ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code:: - - # solution - def shout_echo(word1, echo=1): - echo_word = word1 * echo - shout_words = echo_word + '!!!' - return shout_words - - # sct - Ex().check_function_def('shout_echo').check_correct( - multi( - check_call("f('hey', 3)").has_equal_value(), - check_call("f('hi', 2)").has_equal_value(), - check_call("f('hi')").has_equal_value() - ), - check_body().set_context('test', 1).multi( - has_equal_value(name = 'echo_word'), - has_equal_value(name = 'shout_words') - ) - ) +`See check_function_def doc `_ Check list comprehensions ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code:: - - # solution - L2 = [ i*2 for i in range(0,10) if i>2 ] - - # sct - Ex().check_list_comp().multi( - check_body().has_code('i\*2'), - check_iter().has_equal_value(), - check_ifs(0).multi([has_equal_value(context_vals=[i]) for i in range(0,10)]) - ) - -Check Multiple Choice -~~~~~~~~~~~~~~~~~~~~~ - -.. code:: - - # solution (implicit) - # 3 is the correct answer - - # sct - Ex().has_chosen(correct = 3, # 1-base indexed - msgs = ["That's someone who makes soups.", - "That's a clown who likes burgers.", - "Correct! Head over to the next exercise!"]) \ No newline at end of file +`See check_list_comp doc `_ diff --git a/docs/index.rst b/docs/index.rst index fc7297b2..2a04317b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,6 @@ The reference docs become useful when you grasp all concepts and want to look up .. toctree:: :maxdepth: 2 - :caption: Reference reference @@ -30,11 +29,8 @@ The reference docs become useful when you grasp all concepts and want to look up :caption: Basic articles articles/tutorial.rst - articles/checking_objects.rst articles/checking_function_calls.rst articles/make_your_sct_robust.rst - articles/checking_output.rst - articles/checking_imports.rst articles/checking_through_string_matching.rst .. toctree:: @@ -44,7 +40,14 @@ The reference docs become useful when you grasp all concepts and want to look up articles/checking_compound_statements.rst articles/expression_tests.rst articles/processes.rst + articles/single_process_exercise.rst articles/electives.rst articles/test_to_check.rst +.. toctree:: + :maxdepth: 1 + :caption: Tests + + tests + For details, questions and suggestions, `contact us `_. diff --git a/docs/reference.rst b/docs/reference.rst index 4c9244fa..dc13556e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,56 +1,105 @@ -Simple tests ------------- +Reference +========= .. note:: - - ``check_`` functions produce a child state that 'dives' deeper into a part of the state it was passed. They are typically chained off of for further checking. - - ``has_`` functions always **return the state that they were intially passed** and are used at the 'end' of a chain. + - ``check_`` functions typically 'dive' deeper into a part of the state it was passed. They are typically chained for further checking. + - ``has_`` functions always return the state that they were intially passed and are used at the 'end' of a chain. -Basic building blocks -===================== +Objects +------- -.. autofunction:: pythonwhat.has_funcs.has_code -.. autofunction:: pythonwhat.has_funcs.has_output -.. autofunction:: pythonwhat.has_funcs.has_printout -.. autofunction:: pythonwhat.has_funcs.has_import -.. autofunction:: pythonwhat.has_funcs.has_equal_value -.. autofunction:: pythonwhat.has_funcs.has_equal_error -.. autofunction:: pythonwhat.has_funcs.has_equal_output -.. autofunction:: pythonwhat.has_funcs.has_equal_ast +.. autofunction:: pythonwhat.checks.check_object.check_object +.. autofunction:: pythonwhat.checks.check_object.is_instance +.. autofunction:: pythonwhat.checks.check_object.check_df +.. autofunction:: pythonwhat.checks.check_object.check_keys -Checking objects -================ +Function calls +-------------- -.. autofunction:: pythonwhat.check_object.check_object -.. autofunction:: pythonwhat.check_object.is_instance -.. autofunction:: pythonwhat.check_object.check_df -.. autofunction:: pythonwhat.check_object.check_keys +.. autofunction:: pythonwhat.checks.check_function.check_function +.. autofunction:: pythonwhat.checks.check_funcs.check_args -Checking function calls -======================= +Output +------ -.. autofunction:: pythonwhat.check_function.check_function -.. autofunction:: pythonwhat.check_funcs.check_args +.. autofunction:: pythonwhat.checks.has_funcs.has_output +.. autofunction:: pythonwhat.checks.has_funcs.has_printout +.. autofunction:: pythonwhat.checks.has_funcs.has_no_error -Checking function definitions -============================= +Code +---- -.. autofunction:: pythonwhat.has_funcs.has_equal_part_len -.. autofunction:: pythonwhat.check_funcs.check_args -.. autofunction:: pythonwhat.check_funcs.check_call +.. autofunction:: pythonwhat.checks.has_funcs.has_code +.. autofunction:: pythonwhat.checks.has_funcs.has_import -Logic tests +has_equal_x ----------- -.. autofunction:: pythonwhat.check_logic.multi -.. autofunction:: pythonwhat.check_logic.check_correct -.. autofunction:: pythonwhat.check_logic.check_or -.. autofunction:: pythonwhat.check_logic.check_not +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_value +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_output +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_error +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_ast + +Combining SCTs +-------------- + +.. autofunction:: protowhat.checks.check_logic.multi +.. autofunction:: protowhat.checks.check_logic.check_correct +.. autofunction:: protowhat.checks.check_logic.check_or +.. autofunction:: protowhat.checks.check_logic.check_not + +Function/Class/Lambda definitions +--------------------------------- + +.. autofunction:: pythonwhat.checks.check_wrappers.check_function_def +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_part_len +.. autofunction:: pythonwhat.checks.check_funcs.check_call +.. autofunction:: pythonwhat.checks.check_wrappers.check_class_def +.. autofunction:: pythonwhat.checks.check_wrappers.check_lambda_function + +Control flow +------------ + +.. autofunction:: pythonwhat.checks.check_wrappers.check_if_else +.. autofunction:: pythonwhat.checks.check_wrappers.check_try_except +.. autofunction:: pythonwhat.checks.check_wrappers.check_if_exp +.. autofunction:: pythonwhat.checks.check_wrappers.check_with -State-management +Loops +----- + +.. autofunction:: pythonwhat.checks.check_wrappers.check_for_loop +.. autofunction:: pythonwhat.checks.check_wrappers.check_while +.. autofunction:: pythonwhat.checks.check_wrappers.check_list_comp +.. autofunction:: pythonwhat.checks.check_wrappers.check_dict_comp +.. autofunction:: pythonwhat.checks.check_wrappers.check_generator_exp + +State management ---------------- -.. autofunction:: pythonwhat.check_logic.override -.. autofunction:: pythonwhat.check_logic.disable_highlighting -.. autofunction:: pythonwhat.check_logic.set_context -.. autofunction:: pythonwhat.check_logic.set_env +.. autofunction:: pythonwhat.checks.check_logic.override +.. autofunction:: pythonwhat.checks.check_logic.disable_highlighting +.. autofunction:: pythonwhat.checks.check_logic.set_context +.. autofunction:: pythonwhat.checks.check_logic.set_env + +Checking files +-------------- + +.. autofunction:: pythonwhat.checks.check_wrappers.check_file +.. autofunction:: pythonwhat.checks.check_wrappers.has_dir +.. autofunction:: pythonwhat.local.run + +Bash history checks +------------------- + +.. automodule:: protowhat.checks.check_bash_history + :members: + +Electives +--------- + +.. autofunction:: pythonwhat.checks.has_funcs.has_chosen +.. autofunction:: pythonwhat.test_exercise.success_msg +.. autofunction:: protowhat.checks.check_simple.allow_errors +.. autofunction:: protowhat.checks.check_logic.fail diff --git a/docs/tests.rst b/docs/tests.rst new file mode 100644 index 00000000..c6d9c5d6 --- /dev/null +++ b/docs/tests.rst @@ -0,0 +1,79 @@ +Tests +===== + +.. note:: + + The examples are numbered and linkable, + but numbers (and links) can change between builds of the documentation. + +.. jinja:: test_ctx + + {% for file, tests in test_data.items() %} + + {{ file }} + {{ "-" * 100 }} + + {% for test in tests %} + + Example {{loop.index}} + ~~~~~~~~~~~~~~~~~~~~~~ + {% if test.pre_exercise_code %} + PEC :: + + {{ test.pre_exercise_code | indent(4) }} + + {% else %} + No PEC + {% endif %} + {% if test.solution_code %} + Solution code :: + + {{ test.solution_code | indent(4) }} + + {% else %} + No solution code + {% endif %} + {% if test.student_code %} + Student code :: + + {{ test.student_code | indent(4) }} + + {% else %} + No student code + {% endif %} + {% if test.raw_student_output %} + Student output :: + + {{ test.raw_student_output | indent(4) }} + + {% else %} + No output + {% endif %} + {% if test.sct %} + SCT :: + + {{ test.sct | indent(4) }} + + {% else %} + No SCT + {% endif %} + {% if test.result %} + Result :: + + {{ test.result.message | indent(4) }} + + {% else %} + No result + {% endif %} + {% if test.error %} + Error :: + + {{ test.error | indent(4) }} + + {% else %} + No error + {% endif %} + + {% endfor %} + + {% endfor %} diff --git a/pytest.ini b/pytest.ini index 38b2dde6..724b2ae5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,6 @@ [pytest] testpaths = tests/ -addopts =-s -m "not compiled" +addopts =-m "not compiled" +markers = + slow + compiled diff --git a/pythonwhat/Feedback.py b/pythonwhat/Feedback.py deleted file mode 100644 index 7ea4302a..00000000 --- a/pythonwhat/Feedback.py +++ /dev/null @@ -1,18 +0,0 @@ -import re - -class Feedback(object): - - def __init__(self, message, state = None): - self.message = message - self.line_info = {} - try: - if state is not None and hasattr(state.highlight, "first_token") and \ - hasattr(state.highlight, "last_token") and not state.highlighting_disabled: - self.line_info["line_start"] = state.highlight.first_token.start[0] - self.line_info["column_start"] = state.highlight.first_token.start[1] - self.line_info["line_end"] = state.highlight.last_token.end[0] - self.line_info["column_end"] = state.highlight.last_token.end[1] - - except: - pass - diff --git a/pythonwhat/Reporter.py b/pythonwhat/Reporter.py deleted file mode 100644 index 6138471b..00000000 --- a/pythonwhat/Reporter.py +++ /dev/null @@ -1,74 +0,0 @@ -from pythonwhat.Feedback import Feedback -import re -import markdown2 -from pythonwhat.Test import TestFail, Test - -""" -This file holds the reporter class. -""" - -class Reporter(object): - """Do reporting. - - This class holds the feedback- or success message and tracks whether there are failed tests - or not. All tests are executed trough do_test() in the Reporter. - """ - active_reporter = None - - def __init__(self, error=None): - self.success_msg = "Great work!" - self.error = error - self.errors_allowed = False - - def do_test(self, testobj): - """Do test. - - Execute a given test, unless some previous test has failed. If the test has failed, - the state of the reporter changes and the feedback is kept. - """ - - if isinstance(testobj, Test): - testobj.test() - result = testobj.result - if (not result): - feedback = testobj.get_feedback() - raise TestFail(feedback, self.build_failed_payload(feedback)) - - else: - result = None - testobj() # run function for side effects - - return result - - def build_failed_payload(self, feedback): - if not feedback.line_info: - return { - "correct": False, - "message": Reporter.to_html(feedback.message) - } - else: - return { - "correct": False, - "message": Reporter.to_html(feedback.message), - "line_start": feedback.line_info["line_start"], - "column_start": feedback.line_info["column_start"] + 1, - "line_end": feedback.line_info["line_end"], - "column_end": feedback.line_info["column_end"] - } - - def build_final_payload(self): - if (self.error and not self.errors_allowed): - feedback_msg = "Have a look at the console: your code contains an error. Fix it and try again!" - return { - "correct": False, - "message": Reporter.to_html(feedback_msg) - } - else: - return({ - "correct": True, - "message": Reporter.to_html(self.success_msg) - }) - - @staticmethod - def to_html(msg): - return(re.sub("

(.*)

", "\\1", markdown2.markdown(msg)).strip()) diff --git a/pythonwhat/State.py b/pythonwhat/State.py index 9cf4510d..86a0f426 100644 --- a/pythonwhat/State.py +++ b/pythonwhat/State.py @@ -1,20 +1,25 @@ -import ast -import inspect -import string -from copy import copy -from functools import partial -from pythonwhat.parsing import TargetVars, FunctionParser, ObjectAccessParser, parser_dict -from pythonwhat.Reporter import Reporter -from pythonwhat.Feedback import Feedback -from pythonwhat.Test import Test +import asttokens + +from functools import partialmethod +from collections.abc import Mapping + +from protowhat.failure import debugger +from protowhat.Feedback import FeedbackComponent +from protowhat.selectors import DispatcherInterface +from protowhat.State import State as ProtoState +from protowhat.utils import parameters_attr from pythonwhat import signatures from pythonwhat.converters import get_manual_converters -from collections.abc import Mapping -from itertools import chain -from jinja2 import Template -import asttokens +from pythonwhat.feedback import Feedback +from pythonwhat.parsing import ( + TargetVars, + FunctionParser, + ObjectAccessParser, + parser_dict, +) from pythonwhat.utils_ast import wrap_in_module + class Context(Mapping): def __init__(self, context=None, prev=None): self.context = context if context else TargetVars() @@ -23,8 +28,7 @@ def __init__(self, context=None, prev=None): self._items = {**self.prev, **self.context.defined_items()} def update_ctx(self, new_ctx): - upd_prev = {**self.prev, **self.context.defined_items()} - return self.__class__(new_ctx, upd_prev) + return self.__class__(new_ctx, self._items) def __getitem__(self, x): return self._items[x] @@ -35,267 +39,300 @@ def __iter__(self): def __len__(self): return len(self._items) -class State(object): + +@parameters_attr +class State(ProtoState): """State of the SCT environment. This class holds all information relevevant to test the correctness of an exercise. It is coded suboptimally and it will be refactored soon, and documented thouroughly after that. - """ - def __init__(self, - student_context=None, solution_context=None, - student_env=None, solution_env=None, - student_parts=None, solution_parts=None, - highlight = None, - highlighting_disabled = None, messages=None, - **kwargs): - - # Set basic fields from kwargs - self.__dict__.update(kwargs) + kwargs: + ... + - reporter - self.student_parts = student_parts - self.solution_parts = solution_parts - self.messages = messages if messages else [] - - # parse code if didn't happen yet - if not hasattr(self, 'student_tree'): - self.student_tree_tokens, self.student_tree = State.parse_external(self.student_code) - - if not hasattr(self, 'solution_tree'): - self.solution_tree_tokens, self.solution_tree = State.parse_internal(self.solution_code) - - if not hasattr(self, 'pre_exercise_tree'): - _, self.pre_exercise_tree = State.parse_internal(self.pre_exercise_code) - - if not hasattr(self, 'parent_state'): - self.parent_state = None - - self.student_context = Context(student_context) if student_context is None else student_context - self.solution_context = Context(solution_context) if solution_context is None else solution_context - self.student_env = Context(student_env) if student_env is None else student_env - self.solution_env = Context(solution_env) if solution_env is None else solution_env - - self.highlight = self.student_tree if (not highlight) and self.parent_state else highlight - self.highlighting_disabled = highlighting_disabled + """ - self.converters = get_manual_converters() # accessed only from root state + feedback_cls = Feedback + + def __init__( + self, + student_code, + solution_code, + pre_exercise_code, + student_process, + solution_process, + raw_student_output, + # solution output + reporter, + force_diagnose=False, + highlight=None, + highlight_offset=None, + highlighting_disabled=None, + feedback_context=None, + creator=None, + student_ast=None, + solution_ast=None, + student_ast_tokens=None, + solution_ast_tokens=None, + student_parts=None, + solution_parts=None, + student_context=Context(), + solution_context=Context(), + student_env=Context(), + solution_env=Context(), + ): + args = locals().copy() + self.debug = False + + for k, v in args.items(): + if k != "self": + setattr(self, k, v) + + self.ast_dispatcher = self.get_dispatcher() + + # Parse solution and student code + # if possible, not done yet and wanted (ast arguments not False) + if isinstance(self.student_code, str) and student_ast is None: + self.student_ast = self.parse(student_code) + if isinstance(self.solution_code, str) and solution_ast is None: + with debugger(self): + self.solution_ast = self.parse(solution_code) + + if highlight is None: # todo: check parent_state? (move check to reporting?) + self.highlight = self.student_ast + + self.converters = get_manual_converters() # accessed only from root state self.manual_sigs = None - self._parser_cache = {} def get_manual_sigs(self): if self.manual_sigs is None: self.manual_sigs = signatures.get_manual_sigs() - return(self.manual_sigs) - - def build_message(self, tail="", fmt_kwargs=None, append=True): - - if not fmt_kwargs: fmt_kwargs = {} - out_list = [] - # add trailing message to msg list - msgs = self.messages[:] + [{'msg': tail or "", 'kwargs':fmt_kwargs}] - # format messages in list, by iterating over previous, current, and next message - for prev_d, d, next_d in zip([{}, *msgs[:-1]], msgs, [*msgs[1:], {}]): - tmp_kwargs = {'parent': prev_d.get('kwargs'), - 'child': next_d.get('kwargs'), - 'this': d['kwargs'], - **d['kwargs']} - # don't bother appending if there is no message - if not d['msg']: continue - if d['msg'].startswith('FMT:'): - out = d['msg'].replace('FMT:', "").format(**tmp_kwargs) - elif d['msg'].startswith('__JINJA__:'): - out = Template(d['msg'].replace('__JINJA__:', "")).render(**tmp_kwargs) - else: - out = d['msg'] - - out_list.append(out) - - # if highlighting info is available, don't put all expand messages - if self.highlight and not self.highlighting_disabled: - out_list = out_list[-3:] - - if append: - return "".join(out_list) - else: - return out_list[-1] - - def to_child_state(self, student_subtree=None, solution_subtree=None, - student_context=None, solution_context=None, - student_env=None, solution_env=None, - student_parts=None, solution_parts=None, - highlight = None, - highlighting_disabled = None, - append_message="", node_name=""): + return self.manual_sigs + + def to_child(self, append_message=None, node_name="", **kwargs): """Dive into nested tree. Set the current state as a state with a subtree of this syntax tree as student tree and solution tree. This is necessary when testing if statements or for loops for example. """ + bad_parameters = set(kwargs) - set(self.parameters) + if bad_parameters: + raise ValueError( + "Invalid init parameters for State: %s" % ", ".join(bad_parameters) + ) + + base_kwargs = { + attr: getattr(self, attr) + for attr in self.parameters + if hasattr(self, attr) and attr not in ["ast_dispatcher", "highlight"] + } + + if append_message and not isinstance(append_message, FeedbackComponent): + append_message = FeedbackComponent(append_message) + kwargs["feedback_context"] = append_message + kwargs["creator"] = {"type": "to_child", "args": {"state": self}} + + def update_kwarg(name, func): + kwargs[name] = func(kwargs[name]) + + def update_context(name): + update_kwarg(name, getattr(self, name).update_ctx) + + for ast_arg in ["student_ast", "solution_ast"]: + if isinstance(kwargs.get(ast_arg), list): + update_kwarg(ast_arg, wrap_in_module) + + if kwargs.get("student_ast") and kwargs.get("student_code") is None: + kwargs["student_code"] = self.student_ast_tokens.get_text( + kwargs["student_ast"] + ) + if kwargs.get("solution_ast") and kwargs.get("solution_code") is None: + kwargs["solution_code"] = self.solution_ast_tokens.get_text( + kwargs["solution_ast"] + ) + + for context in [ + "student_context", + "solution_context", + "student_env", + "solution_env", + ]: + if context in kwargs: + if kwargs[context] is not None: + update_context(context) + else: + kwargs.pop(context) + + klass = self.SUBCLASSES[node_name] if node_name else State + init_kwargs = {**base_kwargs, **kwargs} + child = klass(**init_kwargs) + + extra_attrs = set(vars(self)) - set(self.parameters) + for attr in extra_attrs: + # don't copy attrs set on new instances in init + # the cached manual_sigs is passed + if attr not in {"ast_dispatcher", "converters"}: + setattr(child, attr, getattr(self, attr)) - if isinstance(student_subtree, list): - student_subtree = wrap_in_module(student_subtree) - if isinstance(solution_subtree, list): - solution_subtree = wrap_in_module(solution_subtree) - - # get new contexts - if solution_context is not None: - solution_context = self.solution_context.update_ctx(solution_context) - else: - solution_context = self.solution_context - - if student_context is not None: - student_context = self.student_context.update_ctx(student_context) - else: - student_context = self.student_context - - # get new envs - if solution_env is not None: - solution_env = self.solution_env.update_ctx(solution_env) - else: - solution_env = self.solution_env - - if student_env is not None: - student_env = self.student_env.update_ctx(student_env) - else: - student_env = self.student_env - - if highlighting_disabled is None: - highlighting_disabled = self.highlighting_disabled - - if not isinstance(append_message, dict): - append_message = {'msg': append_message, 'kwargs': {}} - - messages = [*self.messages, append_message] - - if not (solution_subtree and student_subtree): - if student_parts and solution_parts: - self.update(student_parts = student_parts, solution_parts = solution_parts) - return self.update(student_context = student_context, solution_context = solution_context, - student_env = student_env, solution_env = solution_env, - highlight = highlight, - highlighting_disabled = highlighting_disabled, - messages = messages) - - klass = State if not node_name else self.SUBCLASSES[node_name] - child = klass(student_code = self.student_tree_tokens.get_text(student_subtree), - solution_code = self.solution_tree_tokens.get_text(solution_subtree), - student_tree_tokens = self.student_tree_tokens, - solution_tree_tokens = self.solution_tree_tokens, - pre_exercise_code = self.pre_exercise_code, - student_context = student_context, - solution_context = solution_context, - student_env = student_env, - solution_env = solution_env, - student_process = self.student_process, - solution_process = self.solution_process, - raw_student_output = self.raw_student_output, - pre_exercise_tree = self.pre_exercise_tree, - student_tree = student_subtree, - solution_tree = solution_subtree, - student_parts = student_parts, - solution_parts = solution_parts, - highlight = highlight, - highlighting_disabled = highlighting_disabled, - messages = messages, - parent_state = self) - return(child) - - def update(self, **kwargs): - """Return a copy of set, setting kwargs as attributes""" - child = copy(self) - for k, v in kwargs.items(): - setattr(child, k, v) return child - @staticmethod - def parse_external(x): - rep = Reporter.active_reporter - + def has_different_processes(self): + # process classes have an _identity field that is a tuple + try: + return ( + self.student_process._identity[0] != self.solution_process._identity[0] + ) + except: + # play it safe (most common) + return True + + def assert_execution_root(self, fun, extra_msg=""): + if not (self.is_root or self.is_creator_type("run")): + with debugger(self): + self.report( + "`%s()` should only be called focusing on a full script, following `Ex()` or `run()`. %s" + % (fun, extra_msg) + ) + + def is_creator_type(self, type): + return self.creator and self.creator.get("type") == type + + def assert_is(self, klasses, fun, prev_fun): + if self.__class__.__name__ not in klasses: + with debugger(self): + self.report( + "`%s()` can only be called on %s." + % (fun, " or ".join(["`%s()`" % pf for pf in prev_fun])) + ) + + def assert_is_not(self, klasses, fun, prev_fun): + if self.__class__.__name__ in klasses: + with debugger(self): + self.report( + "`%s()` should not be called on %s." + % (fun, " or ".join(["`%s()`" % pf for pf in prev_fun])) + ) + + def parse_external(self, code): res = (None, None) try: - res = asttokens.ASTTokens(x, parse = True) - return(res, res._tree) - + return self.ast_dispatcher.parse(code) except IndentationError as e: e.filename = "script.py" # no line info for now - rep.do_test(Test(Feedback("Your code could not be parsed due to an error in the indentation:
`%s.`" % str(e)))) + self.report( + "Your code could not be parsed due to an error in the indentation:
`%s.`" + % str(e) + ) except SyntaxError as e: e.filename = "script.py" # no line info for now - rep.do_test(Test(Feedback("Your code can not be executed due to a syntax error:
`%s.`" % str(e)))) + self.report( + "Your code can not be executed due to a syntax error:
`%s.`" % str(e) + ) # Can happen, can't catch this earlier because we can't differentiate between # TypeError in parsing or TypeError within code (at runtime). except: - rep.do_test(Test(Feedback("Something went wrong while parsing your code."))) + self.report("Something went wrong while parsing your code.") - return(res) + return res - @staticmethod - def parse_internal(x): - res = (None, None) + def parse_internal(self, code): try: - res = asttokens.ASTTokens(x, parse = True) - return(res, res._tree) + return self.ast_dispatcher.parse(code) + except Exception as e: + self.report( + "Something went wrong when parsing the solution code: %s" % str(e) + ) + + def parse(self, text): + if self.debug: + parse_method = self.parse_internal + token_attr = "solution_ast_tokens" + else: + parse_method = self.parse_external + token_attr = "student_ast_tokens" + + tokens, ast = parse_method(text) + setattr(self, token_attr, tokens) + + return ast + + def get_dispatcher(self): + try: + return Dispatcher(self.pre_exercise_code) + except Exception as e: + with debugger(self): + self.report("Something went wrong when parsing the PEC: %s" % str(e)) + + +class Dispatcher(DispatcherInterface): + _context_cache = dict() + + def __init__(self, context_code=""): + self._parser_cache = dict() + context_ast = getattr(self._context_cache, context_code, None) + if context_ast is None: + context_ast = self._context_cache[context_code] = self.parse(context_code)[ + 1 + ] + self.context_mappings = self._getx(FunctionParser, "mappings", context_ast) + + def find(self, name, node, *args, **kwargs): + return getattr(self, name)(node) + + def parse(self, code): + res = asttokens.ASTTokens(code, parse=True) + return res, res.tree + + # add methods for retrieving parser outputs -------------------------- + def _getx(self, Parser, ext_attr, tree): + """getter for Parser outputs""" + # return cached output if possible + cache_key = Parser.__name__ + str(hash(tree)) + if self._parser_cache.get(cache_key): + p = self._parser_cache[cache_key] + else: + # otherwise, run parser over tree + p = Parser() + # set mappings for parsers that inspect attribute access + if ext_attr != "mappings" and Parser in [ + FunctionParser, + ObjectAccessParser, + ]: + p.mappings = self.context_mappings.copy() + # run parser + p.visit(tree) + # cache + self._parser_cache[cache_key] = p + return getattr(p, ext_attr) + + +# put a function on the dispatcher +for k, Parser in parser_dict.items(): + setattr(Dispatcher, k, partialmethod(Dispatcher._getx, Parser, "out")) + +# mappings from ObjectAccessParser +prop_oa_map = partialmethod(Dispatcher._getx, ObjectAccessParser, "mappings") +setattr(Dispatcher, "oa_mappings", prop_oa_map) + +# mappings from FunctionParser +prop_map = partialmethod(Dispatcher._getx, FunctionParser, "mappings") +setattr(Dispatcher, "mappings", prop_map) - except SyntaxError as e: - raise SyntaxError(str(e)) - except TypeError as e: - raise TypeError(str(e)) - - return(res) - -# add property methods for retrieving parser outputs -------------------------- -# note that this code is an alternative means of using something like.. -# @property -# def student_withs(self): ... -# when defining the State class. -def getx(tree_name, Parser, ext_attr, self): - """getter for Parser outputs""" - # return cached output if possible - cache_key = tree_name + Parser.__name__ - if self._parser_cache.get(cache_key): - p = self._parser_cache[cache_key] - else: - # otherwise, run parser over tree - p = Parser() - # set mappings for parsers that inspect attribute access - if ext_attr != 'mappings' and Parser in [FunctionParser, ObjectAccessParser]: - p.mappings = self.pre_exercise_mappings.copy() - # run parser - p.visit(getattr(self, tree_name)) - # cache - self._parser_cache[cache_key] = p - return getattr(p, ext_attr) - -# put a property getter on state for each parsed ast tree output. -# since the getter takes only one argument, self, partial functions -# are used to set all other arguments on getx -for s in ['student', 'solution']: - tree_name = s+'_tree' - for k, Parser in parser_dict.items(): - setattr(State, s+'_'+k, property(partial(getx, tree_name, Parser, 'out'))) - - # mappings from ObjectAccessParser - prop_oa_map = property(partial(getx, tree_name, ObjectAccessParser, 'mappings')) - setattr(State, s+'_oa_mappings', prop_oa_map) - - # mappings from FunctionParser - prop_map = property(partial(getx, tree_name, FunctionParser, 'mappings')) - setattr(State, s+'_mappings', prop_map) - -# mappings for pre exercise code from FunctionParser -pec_prop_map = property(partial(getx, 'pre_exercise_tree', FunctionParser, 'mappings')) -setattr(State, 'pre_exercise_mappings', pec_prop_map) # State subclasses based on parsed output ------------------------------------- -State.SUBCLASSES = {node_name: type(node_name, (State,), {}) for node_name in parser_dict} +State.SUBCLASSES = { + node_name: type(node_name, (State,), {}) for node_name in parser_dict +} + # global setters on State ----------------------------------------------------- def set_converter(key, fundef): diff --git a/pythonwhat/Test.py b/pythonwhat/Test.py index ec5ff42b..47c0703c 100644 --- a/pythonwhat/Test.py +++ b/pythonwhat/Test.py @@ -1,74 +1,18 @@ import re -from pythonwhat.Feedback import Feedback -import numpy as np -import pandas as pd from pythonwhat.tasks import * +from protowhat.Test import Test """ -This file contains all tests that can be done on specific objects. All tests are represented +This file contains all tests that can be done on specific objects. All tests are represented as an object. Tests that are alike can inherit from the same superclass. A test is first initialized and can then be performed by calling the 'test()' function. The result will be stored inside the result boolean. A test contains a failure message, which can be used by the reporter to show when the test failed. """ -class TestFail(Exception): - def __init__(self, feedback, payload): - super().__init__(feedback.message) - self.feedback = feedback - self.payload = payload -class Test(object): - """ - The basic Test. It should only contain a failure message, as all tests should result in - a failure message when they fail. - - Note: - This test should not be used by itself, subclasses should be used. - - Attributes: - feedback (str): A string containing the failure message in case the test fails. - result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. - """ - - def __init__(self, feedback): - """ - Initialize the standard test. - - Args: - feedback: string or Feedback object - """ - if (issubclass(type(feedback), Feedback)): - self.feedback = feedback - elif (issubclass(type(feedback), str)): - self.feedback = Feedback(feedback) - else: - raise TypeError("When creating a test, specify either a string or a Feedback object") - - self.result = None - - def test(self): - """ - Wrapper around specific tests. Tests only get one chance. - """ - if self.result is None: - try: - self.specific_test() - self.result = np.array(self.result).all() - except: - self.result = False - - def specific_test(self): - """ - Perform the actual test. For the standard test, result will be set to False. - """ - self.result = False +# Testing definition - def get_feedback(self): - return(self.feedback) - - -## Testing definition class DefinedProcessTest(Test): def __init__(self, name, process, feedback): @@ -76,7 +20,7 @@ def __init__(self, name, process, feedback): self.name = name self.process = process - def specific_test(self): + def test(self): self.result = isDefinedInProcess(self.name, self.process) @@ -90,14 +34,16 @@ class DefinedCollTest(Test): collection (list/dict/set): Contains any object on which the 'in' operator can be performed. result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. """ + def __init__(self, name, collection, feedback): super().__init__(feedback) self.name = name self.collection = collection - def specific_test(self): + def test(self): self.result = self.name in self.collection + class DefinedCollProcessTest(Test): def __init__(self, name, key, process, feedback): super().__init__(feedback) @@ -105,20 +51,12 @@ def __init__(self, name, key, process, feedback): self.key = key self.process = process - def specific_test(self): + def test(self): self.result = isDefinedCollInProcess(self.name, self.key, self.process) -## Testing class - -class InstanceTest(Test): - def __init__(self, obj, cls, feedback): - super().__init__(feedback) - self.obj = obj - self.cls = cls +# Testing class - def specific_test(self): - self.result = isinstance(self.obj, self.cls) class InstanceProcessTest(Test): def __init__(self, name, klass, process, feedback): @@ -127,14 +65,13 @@ def __init__(self, name, klass, process, feedback): self.klass = klass self.process = process - def specific_test(self): + def test(self): self.result = isInstanceInProcess(self.name, self.klass, self.process) +# Testing equality -## Testing equality - class EqualTest(Test): """ Check if two objects are equal. Equal means the objects are exactly the same. @@ -147,87 +84,118 @@ class EqualTest(Test): result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. """ - def __init__(self, obj1, obj2, feedback, func = None): + def __init__(self, obj1, obj2, feedback, func=None): super().__init__(feedback) self.obj1 = obj1 self.obj2 = obj2 self.func = func if func is not None else is_equal - def specific_test(self): + def test(self): """ Perform the actual test. result is set to False if the objects differ, True otherwise. """ - self.result = self.func(self.obj1, self.obj2) + result = self.func(self.obj1, self.obj2) -## Helpers for testing equality + try: + import numpy as np + + self.result = np.array(result).all() + except ImportError: + self.result = result + + +# Helpers for testing equality -def objs_are(x, y, list_of_classes): - return ( - any([isinstance(x, klass) for klass in list_of_classes]) & - any([isinstance(y, klass) for klass in list_of_classes]) - ) + +def areinstance(x, y, tuple_of_classes): + return isinstance(x, tuple_of_classes) and isinstance(y, tuple_of_classes) + +def is_primitive(x): + return isinstance(x, (str, int, float, bool, type(None))) + +def is_collection_of_primitives(x): + return isinstance(x, (list, tuple, set)) and all(is_primitive(element) for element in x) # For equality of ndarrays, list, dicts, pd Series and pd DataFrames: # First try to the faster equality functions. If these don't pass, # Run the assertions that are typically slower. def is_equal(x, y): + try: + if areinstance(x, y, (str, int, float, bool, type(None))): + return x == y + + if is_collection_of_primitives(x): + return x == y + + if areinstance(x, y, (Exception,)): + # Types of errors don't matter (this is debatable) + return str(x) == str(y) + + if areinstance(x, y, (list, tuple,)): + if len(x) != len(y): + return False + return all(is_equal(x_element, y_element) for x_element, y_element in zip(x, y)) + + if areinstance(x, y, (map, filter,)): + x_list, y_list = list(x), list(y) + if len(x_list) != len(y_list): + return False + return all(is_equal(x_element, y_element) for x_element, y_element in zip(x_list, y_list)) + + if areinstance(x, y, (set,)): + return is_equal(sorted(x), sorted(y)) + + if areinstance(x, y, (dict,)): + if x.keys() != y.keys(): + return False + return all(is_equal(x[key], y[key]) for key in x) + + # Delay importing pandas / numpy until absolutely necessary. This is important for performance in Pyodide. + # Also, assume they may not be available, as Pyodide won't install them unless they are needed. try: - if objs_are(x, y, [Exception]): - # Types of errors don't matter - return str(x) == str(y) - if objs_are(x, y, [np.ndarray, dict, list]): - if np.array_equal(x, y): return True + import numpy as np + + if areinstance(x, y, (np.ndarray,)): np.testing.assert_equal(x, y) return True - elif objs_are(x, y, [map, filter]): - return np.array_equal(list(x), list(y)) - elif objs_are(x, y, [pd.DataFrame]): - if x.equals(y): return True - pd.util.testing.assert_frame_equal(x, y) + except ImportError: + if areinstance(x, y, (np.ndarray,)): + raise RuntimeError("NumPy is required for comparing NumPy objects.") + + try: + import pandas as pd + from pandas.testing import assert_frame_equal, assert_series_equal + + if areinstance(x, y, (pd.DataFrame,)): + if x.equals(y): + return True + assert_frame_equal(x, y) return True - elif objs_are(x, y, [pd.Series]): - if x.equals(y): return True - pd.util.testing.assert_series_equal(x, y) + elif areinstance(x, y, (pd.Series,)): + if x.equals(y): + return True + assert_series_equal(x, y) return True - elif objs_are(x, y, [Exception]): - return type(x) == type(y) and str(x) == str(y) - else: - return x == y + except ImportError: + if areinstance(x, y, (pd.DataFrame, pd.Series)): + raise RuntimeError("pandas is required for comparing pandas objects.") - except Exception: - return False + return x == y -## Others + except Exception: + return False -class BiggerTest(Test): - """ - Check if one object is greater than another. This test should only be used with numeric variables (for now). - Attributes: - feedback (str): A string containing the failure message in case the test fails. - obj1 (str): The first object, that should be the greatest - obj2 (str): The second object, that should be smaller - result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. - """ +# Others - def __init__(self, obj1, obj2, feedback): - """ - Initialize with two objects. - Args: - obj1 (str): The first object, obj1 will be set to this. - obj2 (str): The second object, obj2 will be set to this. - feedback (str): The failure message will be set to this. - """ - super().__init__(feedback) - self.obj1 = obj1 - self.obj2 = obj2 +class BiggerTest(EqualTest): + """ + Check if the first object is greater than another. + """ - def specific_test(self): - """ - Perform the actual test. result is set to False if the objects differ, True otherwise. - """ - self.result = (self.obj1 > self.obj2) + def __init__(self, *args): + super().__init__(*args, func=lambda obj1, obj2: obj1 > obj2) class StringContainsTest(Test): @@ -257,18 +225,12 @@ def __init__(self, string, search_string, pattern, feedback): self.search_string = search_string self.pattern = pattern - def specific_test(self): + def test(self): """ Perform the actual test. result will be True if string is found (whether or not with a pattern), False otherwise. """ if self.pattern: - self.result = ( - re.search( - self.search_string, - self.string) is not None) + self.result = re.search(self.search_string, self.string) is not None else: - self.result = (self.string.find(self.search_string) is not -1) - - - + self.result = self.string.find(self.search_string) != -1 diff --git a/pythonwhat/__init__.py b/pythonwhat/__init__.py index d328c25b..47b6d8fd 100644 --- a/pythonwhat/__init__.py +++ b/pythonwhat/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.15.2' +__version__ = "2.30.1" from .test_exercise import test_exercise, allow_errors diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py deleted file mode 100644 index 1de08039..00000000 --- a/pythonwhat/check_funcs.py +++ /dev/null @@ -1,409 +0,0 @@ -from pythonwhat.tasks import getResultInProcess, getOutputInProcess, getErrorInProcess, ReprFail, setUpNewEnvInProcess, breakDownNewEnvInProcess -from pythonwhat.has_funcs import has_part -from pythonwhat.check_logic import multi -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest, TestFail -from pythonwhat.Feedback import Feedback -from pythonwhat.utils import get_ord -from functools import partial -import ast - -class StubState(): - def __init__(self, highlight, highlighting_disabled): - self.highlight = highlight - self.highlighting_disabled = highlighting_disabled - -def part_to_child(stu_part, sol_part, append_message, state, node_name=None): - # stu_part and sol_part will be accessible on all templates - append_message['kwargs'].update({'stu_part': stu_part, 'sol_part': sol_part}) - - # if the parts are dictionaries, use to deck out child state - if all(isinstance(p, dict) for p in [stu_part, sol_part]): - return state.to_child_state(student_subtree=stu_part['node'], - solution_subtree=sol_part['node'], - student_context=stu_part.get('target_vars'), - solution_context=sol_part.get('target_vars'), - student_parts=stu_part, - solution_parts=sol_part, - highlight = stu_part.get('highlight'), - append_message = append_message, - node_name=node_name) - - # otherwise, assume they are just nodes - return state.to_child_state(student_subtree=stu_part, - solution_subtree=sol_part, - append_message=append_message) - - -def check_part(name, part_msg, - missing_msg=None, - expand_msg=None, - state=None): - """Return child state with name part as its ast tree""" - - if missing_msg is None: missing_msg = "__JINJA__:Are you sure you defined the {{part}}? " - if expand_msg is None: expand_msg = "__JINJA__:Did you correctly specify the {{part}}? " - - if not part_msg: part_msg = name - append_message = {'msg': expand_msg, 'kwargs': { 'part': part_msg }} - - has_part(name, missing_msg, state, append_message['kwargs']) - stu_part = state.student_parts[name] - sol_part = state.solution_parts[name] - - return part_to_child(stu_part, sol_part, append_message, state) - -def check_part_index(name, index, part_msg, - missing_msg=None, - expand_msg=None, - state=None): - """Return child state with indexed name part as its ast tree. - - ``index`` can be: - - - an integer, in which case the student/solution_parts are indexed by position. - - a string, in which case the student/solution_parts are expected to be a dictionary. - - a list of indices (which can be integer or string), in which case the student parts are indexed step by step. - """ - - if missing_msg is None: missing_msg = "__JINJA__:Are you sure you defined the {{part}}? " - if expand_msg is None: expand_msg = "__JINJA__:Did you correctly specify the {{part}}? " - - # create message - ordinal = get_ord(index+1) if isinstance(index, int) else "" - fmt_kwargs = {'index': index, 'ordinal': ordinal} - fmt_kwargs['part'] = part_msg.format(**fmt_kwargs) - - append_message = { - 'msg': expand_msg, - 'kwargs': fmt_kwargs - } - - # check there are enough parts for index - has_part(name, missing_msg, state, append_message['kwargs'], index) - - # get part at index - stu_part = state.student_parts[name] - sol_part = state.solution_parts[name] - - if isinstance(index, list): - for ind in index: - stu_part = stu_part[ind] - sol_part = sol_part[ind] - else: - stu_part = stu_part[index] - sol_part = sol_part[index] - - # return child state from part - return part_to_child(stu_part, sol_part, append_message, state) - -def check_node(name, index=0, typestr='{ordinal} node', - missing_msg=None, - expand_msg=None, - state=None): - - if missing_msg is None: missing_msg = "__JINJA__:The system wants to check the {{typestr}} but hasn't found it." - if expand_msg is None: expand_msg = "__JINJA__:Check the {{typestr}}. " - - rep = Reporter.active_reporter - stu_out = getattr(state, 'student_'+name) - sol_out = getattr(state, 'solution_'+name) - - # check if there are enough nodes for index - fmt_kwargs = {'ordinal': get_ord(index+1) if isinstance(index, int) else "", - 'index': index, - 'name': name} - fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs) - - # test if node can be indexed succesfully - try: stu_out[index] - except (KeyError, IndexError): # TODO comment errors - _msg = state.build_message(missing_msg, fmt_kwargs) - rep.do_test(Test(Feedback(_msg, state))) - - # get node at index - stu_part = stu_out[index] - sol_part = sol_out[index] - - append_message = { - 'msg': expand_msg, - 'kwargs': fmt_kwargs - } - - return part_to_child(stu_part, sol_part, append_message, state, node_name=name) - -# context functions ----------------------------------------------------------- - -def with_context(*args, state=None): - - rep = Reporter.active_reporter - - # set up context in processes - solution_res = setUpNewEnvInProcess(process = state.solution_process, - context = state.solution_parts['with_items']) - if isinstance(solution_res, Exception): - raise Exception("error in the solution, running test_with(): %s" % str(solution_res)) - - student_res = setUpNewEnvInProcess(process = state.student_process, - context = state.student_parts['with_items']) - if isinstance(student_res, AttributeError): - rep.do_test(Test(Feedback("In your `with` statement, you're not using a correct context manager.", child.highlight))) - - if isinstance(student_res, (AssertionError, ValueError, TypeError)): - rep.do_test(Test(Feedback("In your `with` statement, the number of values in your context manager " - "doesn't correspond to the number of variables you're trying to assign it to.", child.highlight))) - - # run subtests - try: - multi(*args, state=state) - finally: - # exit context - if breakDownNewEnvInProcess(process = state.solution_process): - raise Exception("error in the solution, closing the `with` fails with: %s" % (close_solution_context)) - - if breakDownNewEnvInProcess(process = state.student_process): - - rep.do_test(Test(Feedback("Your `with` statement can not be closed off correctly, you're " + \ - "not using the context manager correctly.", state))) - return state - -def check_args(name, missing_msg=None, state=None): - """Check whether a function argument is specified. - - This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified. - If you want to go on and check whether the argument was correctly specified, you can can continue chaining with - ``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check) - - This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been - specified. - - Args: - name (str): the name of the argument for which you want to check it is specified. This can also be - a number, in which case it refers to the positional arguments. Named argumetns take precedence. - missing_msg (str): If specified, this overrides an automatically generated feedback message in case - the student did specify the argument. - state (State): State object that is passed from the SCT Chain (don't specify this). - - :Examples: - - Student and solution code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - np.mean(arr) - - SCT:: - - # Verify whether arr was correctly set in np.mean - # has_equal_value() checks the value of arr, used to set argument a - Ex().check_function('numpy.mean').check_args('a').has_equal_value() - - # Verify whether arr was correctly set in np.mean - # has_equal_ast() checks the expression used to set argument a - Ex().check_function('numpy.mean').check_args('a').has_equal_ast() - - Student and solution code:: - - def my_power(x): - print("calculating sqrt...") - return(x * x) - - SCT:: - - Ex().check_function_def('my_power').multi( - check_args('x') # will fail if student used y as arg - check_args(0) # will still pass if student used y as arg - ) - - """ - if missing_msg is None: - missing_msg = '__JINJA__:Did you specify the {{part}}?' - - if name in ['*args', '**kwargs']: # for check_function_def - return check_part(name, name, state=state, missing_msg = missing_msg) - else: - if isinstance(name, list): # dealing with args or kwargs - if name[0] == 'args': - arg_str = "%s argument passed as a variable length argument"%get_ord(name[1]+1) - else: - arg_str = "argument `%s`"%name[1] - else: - arg_str = "%s argument" % get_ord(name+1) if isinstance(name, int) else "argument `%s`" % name - return check_part_index('args', name, arg_str, missing_msg = missing_msg, state=state) - - -# CALL CHECK ================================================================== - -evalCalls = {'value': getResultInProcess, - 'output': getOutputInProcess, - 'error': getErrorInProcess} - -call_warnings = { - 'value': 'in the solution process resulted in an error', - 'error': 'did not generate an error in the solution environment', - 'output': 'in the solution process resulted in an error' - } - -def fix_format(arguments): - if isinstance(arguments, str): - arguments = (arguments, ) - if isinstance(arguments, tuple): - arguments = list(arguments) - - if isinstance(arguments, list): - arguments = {'args': arguments, 'kwargs': {}} - - if not isinstance(arguments, dict) or 'args' not in arguments or 'kwargs' not in arguments: - raise ValueError("Wrong format of arguments in 'results', 'outputs' or 'errors'; either a list, or a dictionary with names args (a list) and kwargs (a dict)") - - return(arguments) - -def stringify(arguments): - vararg = str(arguments['args'])[1:-1] - kwarg = ', '.join(['%s = %s' % (key, value) for key, value in arguments['kwargs'].items()]) - if len(vararg) == 0: - if len(kwarg) == 0: - return "()" - else: - return "(" + kwarg + ")" - else : - if len(kwarg) == 0: - return "(" + vararg + ")" - else : - return "(" + ", ".join([vararg, kwarg]) + ")" - -# TODO: test string syntax with check_function_def -# test argument syntax with check_lambda_function -def run_call(args, node, process, get_func, **kwargs): - # Get function expression - if isinstance(node, ast.FunctionDef): # function name - func_expr = ast.Name(id=node.name, ctx=ast.Load()) - elif isinstance(node, ast.Lambda): # lambda body expr - func_expr = node - else: raise TypeError("Only function definition or lambda may be called") - - # args is a call string or argument list/dict - if isinstance(args, str): - parsed = ast.parse(args).body[0].value - parsed.func = func_expr - ast.fix_missing_locations(parsed) - return get_func(process = process, tree = parsed, **kwargs) - else: - # e.g. list -> {args: [...], kwargs: {}} - fmt_args = fix_format(args) - ast.fix_missing_locations(func_expr) - return get_func(process = process, tree=func_expr, call = fmt_args, **kwargs) - -MSG_CALL_INCORRECT = "__JINJA__:Calling {{argstr}} should {{action}} `{{str_sol}}`, instead got {{str_stu if str_stu == 'no printouts' else '`' + str_stu + '`'}}." -MSG_CALL_ERROR = "__JINJA__:Calling {{argstr}} should {{action}} `{{str_sol}}`, instead it errored out: `{{str_stu}}`." -MSG_CALL_ERROR_INV = "__JINJA__:Calling {{argstr}} should {{action}} `{{str_sol}}`, instead got `{{str_stu}}`." -def call(args, - test='value', - incorrect_msg=None, - error_msg=None, - argstr=None, - func=None, - state=None, **kwargs): - """Use ``check_call()`` in combination with ``has_equal_x()`` instead. - """ - - if incorrect_msg is None: - incorrect_msg = MSG_CALL_INCORRECT - if error_msg is None: - error_msg = MSG_CALL_ERROR_INV if test == 'error' else MSG_CALL_ERROR - - if argstr is None: - bracks = stringify(fix_format(args)) - if hasattr(state.student_parts['node'], 'name'): # Lambda function doesn't have name - argstr = '`{}{}`'.format(state.student_parts['node'].name, bracks) - else: - argstr = 'it with the arguments `{}`'.format(bracks) - - rep = Reporter.active_reporter - - assert test in ('value', 'output', 'error') - - get_func = evalCalls[test] - - # Run for Solution -------------------------------------------------------- - eval_sol, str_sol = run_call(args, state.solution_parts['node'], state.solution_process, get_func, **kwargs) - - if (test == 'error') ^ isinstance(eval_sol, Exception): - _msg = state.build_message("FMT:Calling {argstr} resulted in an error (or not an error if testing for one). Error message: {type_err} {str_sol}", - dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr)), - raise ValueError(_msg) - - if isinstance(eval_sol, ReprFail): - _msg = state.build_message("FMT:Can't get the result of calling {argstr}: {eval_sol.info}", - dict(argstr = argstr, eval_sol=eval_sol)) - raise ValueError(_msg) - - # Run for Submission ------------------------------------------------------ - eval_stu, str_stu = run_call(args, state.student_parts['node'], state.student_process, get_func, **kwargs) - action_strs = {'value': 'return', 'output': 'print out', 'error': 'error out with the message'} - fmt_kwargs = {'part': argstr, 'argstr': argstr, 'str_sol': str_sol, 'str_stu': str_stu, 'action': action_strs[test]} - - # either error test and no error, or vice-versa - stu_node = state.student_parts['node'] - stu_state = StubState(stu_node, state.highlighting_disabled) - if (test == 'error') ^ isinstance(eval_stu, Exception): - _msg = state.build_message(error_msg, fmt_kwargs) - rep.do_test(Test(Feedback(_msg, stu_state))) - - # incorrect result - _msg = state.build_message(incorrect_msg, fmt_kwargs) - rep.do_test(EqualTest(eval_sol, eval_stu, Feedback(_msg, stu_state), func)) - - return state - -def build_call(callstr, node): - if isinstance(node, ast.FunctionDef): # function name - func_expr = ast.Name(id=node.name, ctx=ast.Load()) - argstr = "`{}`".format(callstr.replace('f', node.name)) - elif isinstance(node, ast.Lambda): # lambda body expr - func_expr = node - argstr = 'it with the arguments `{}`'.format(callstr.replace('f', '')) - else: raise TypeError("You can use check_call() only on check_function_def() or check_lambda()") - - parsed = ast.parse(callstr).body[0].value - parsed.func = func_expr - ast.fix_missing_locations(parsed) - return parsed, argstr - -def check_call(callstr, argstr = None, expand_msg=None, state=None): - """When checking a function definition of lambda function, - prepare has_equal_x for checking the call of a user-defined function. - - Args: - callstr (str): call string that specifies how the function should be called, e.g. `f(1, a = 2)`. - ``check_call()`` will replace ``f`` with the function/lambda you're targeting. - argstr (str): If specified, this overrides the way the function call is refered to in the expand message. - expand_msg (str): If specified, this overrides the expand message. - state (State): state object that is chained from. - - :Example: - - Student and solution code:: - - def my_power(x): - print("calculating sqrt...") - return(x * x) - - SCT:: - - Ex().check_function_def('my_power').multi( - check_call("f(3)").has_equal_value() - check_call("f(3)").has_equal_output() - ) - """ - - if expand_msg is None: - expand_msg = "__JINJA__:To verify it, we reran {{argstr}}. " - - stu_part, _argstr = build_call(callstr, state.student_parts['node']) - sol_part, _ = build_call(callstr, state.solution_parts['node']) - - append_message = { 'msg': expand_msg, 'kwargs': {'argstr': argstr or _argstr }} - child = part_to_child(stu_part, sol_part, append_message, state) - - return child \ No newline at end of file diff --git a/pythonwhat/check_function.py b/pythonwhat/check_function.py deleted file mode 100644 index 5f3fe623..00000000 --- a/pythonwhat/check_function.py +++ /dev/null @@ -1,125 +0,0 @@ -from pythonwhat.Reporter import Reporter -from pythonwhat.check_funcs import part_to_child, StubState -from pythonwhat.tasks import getSignatureInProcess -from pythonwhat.utils import get_ord, get_times -from pythonwhat.Test import Test -from pythonwhat.Feedback import Feedback -from pythonwhat.parsing import IndexedDict -from functools import partial - -def bind_args(signature, args_part): - pos_args = []; kw_args = {} - for k, arg in args_part.items(): - if isinstance(k, int): pos_args.append(arg) - else: kw_args[k] = arg - - bound_args = signature.bind(*pos_args, **kw_args) - return IndexedDict(bound_args.arguments) - -def get_mapped_name(name, mappings): - # get name by splitting on periods - if "." in name: - for orig, full_name in mappings.items(): - if name.startswith(full_name): return name.replace(full_name, orig) - return name - -MISSING_MSG = "__JINJA__:Did you call `{{mapped_name}}()`{{' ' + times if index>0}}?" -SIG_ISSUE_MSG = "__JINJA__:Have you specified the arguments for `{{mapped_name}}()` using the right syntax?" -PREPEND_MSG = "__JINJA__:Check your {{ord + ' ' if index>0}}call of `{{mapped_name}}()`. " -def check_function(name, index=0, - missing_msg=None, - params_not_matched_msg=None, - expand_msg=None, - signature=True, - state=None): - """Check whether a particular function is called. - - This function is typically followed by ``check_args()`` to check whether the arguments were - specified correctly. - - Args: - name (str): the name of the function to be tested. When checking functions in packages, always - use the 'full path' of the function. - index (int): index of the function call to be checked. Defaults to 0. - missing_msg (str): If specified, this overrides an automatically generated feedback message in case - the student did not call the function correctly. - params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message - in case the function parameters were not successfully matched. - expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. - signature (Signature): Normally, check_function() can figure out what the function signature is, - but it might be necessary to use build_sig to manually build a signature and pass this along. - state (State): State object that is passed from the SCT Chain (don't specify this). - - :Examples: - - Student code and solution code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - np.mean(arr) - - SCT:: - - # Verify whether arr was correctly set in np.mean - Ex().check_function('numpy.mean').check_args('a').has_equal_value() - - # Verify whether np.mean(arr) produced the same result - Ex().check_function('numpy.mean').has_equal_value() - """ - - append_missing = missing_msg is None - append_params_not_matched = params_not_matched_msg is None - if missing_msg is None: - missing_msg = MISSING_MSG - if expand_msg is None: - expand_msg = PREPEND_MSG - if params_not_matched_msg is None: - params_not_matched_msg = SIG_ISSUE_MSG - - rep = Reporter.active_reporter - stu_out = state.student_function_calls - sol_out = state.solution_function_calls - - student_mappings = state.student_mappings - - fmt_kwargs = {'times': get_times(index+1), - 'ord': get_ord(index+1), - 'index': index, - 'mapped_name': get_mapped_name(name, student_mappings)} - - # Get Parts ---- - # Copy, otherwise signature binding overwrites sol_out[name][index]['args'] - sol_parts = {**sol_out[name][index]} - - try: - # Copy, otherwise signature binding overwrites stu_out[name][index]['args'] - stu_parts = {**stu_out[name][index]} - except (KeyError, IndexError): - _msg = state.build_message(missing_msg, fmt_kwargs, append=append_missing) - rep.do_test(Test(Feedback(_msg, state))) - - # Signatures ----- - if signature: - signature = None if isinstance(signature, bool) else signature - get_sig = partial(getSignatureInProcess, name=name, signature=signature, - manual_sigs = state.get_manual_sigs()) - - try: - sol_sig = get_sig(mapped_name=sol_parts['name'], process=state.solution_process) - sol_parts['args'] = bind_args(sol_sig, sol_parts['args']) - except: - raise ValueError("Something went wrong in matching call index {index} of {name} to its signature. " - "You might have to manually specify or correct the signature." - .format(index=index, name=name)) - - try: - stu_sig = get_sig(mapped_name=stu_parts['name'], process=state.student_process) - stu_parts['args'] = bind_args(stu_sig, stu_parts['args']) - except Exception: - _msg = state.build_message(params_not_matched_msg, fmt_kwargs, append=append_params_not_matched) - rep.do_test(Test(Feedback(_msg, StubState(stu_parts['node'], state.highlighting_disabled)))) - - # three types of parts: pos_args, keywords, args (e.g. these are bound to sig) - append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs} - child = part_to_child(stu_parts, sol_parts, append_message, state, node_name='function_calls') - return child diff --git a/pythonwhat/check_has_context.py b/pythonwhat/check_has_context.py deleted file mode 100644 index f6d6218b..00000000 --- a/pythonwhat/check_has_context.py +++ /dev/null @@ -1,80 +0,0 @@ -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest -from pythonwhat.Feedback import Feedback -from pythonwhat.State import State -from functools import singledispatch -from pythonwhat.check_funcs import check_part_index - -MSG_INCORRECT_LOOP = "FMT:Have you used the correct iterator variable names? Was expecting `{sol_vars}` but got `{stu_vars}`." -MSG_INCORRECT_WITH = "FMT:Make sure to use the correct context variable names. Was expecting `{sol_vars}` but got `{stu_vars}`." - -def has_context(incorrect_msg=None, exact_names=False, state=None): - # call _has_context, since the built-in singledispatch can only use 1st pos arg - return _has_context(state, incorrect_msg, exact_names) - -def _test(state, incorrect_msg, exact_names, tv_name, highlight_name): - rep = Reporter.active_reporter - # get parts for testing from state - # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. - stu_vars = state.student_parts[tv_name] - sol_vars = state.solution_parts[tv_name] - - child_state = state.to_child_state(student_subtree = state.student_parts.get(highlight_name), - solution_subtree = state.solution_parts.get(highlight_name)) - - # variables exposed to messages - d = { 'stu_vars': stu_vars, - 'sol_vars': sol_vars, - 'num_vars': len(sol_vars)} - - if exact_names: - # message for wrong iter var names - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, child_state))) - else: - # message for wrong number of iter vars - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, child_state))) - - return state - - -@singledispatch -def _has_context(state, incorrect_msg, exact_names): - raise BaseException("first argument to _has_context must be a State instance or subclass") - -@_has_context.register(State) -def has_context_state(*args, **kwargs): - return _test(*args, tv_name='target_vars', highlight_name='highlight', **kwargs) - -@_has_context.register(State.SUBCLASSES['for_loops']) -@_has_context.register(State.SUBCLASSES['whiles']) -@_has_context.register(State.SUBCLASSES['dict_comps']) -@_has_context.register(State.SUBCLASSES['generator_exps']) -@_has_context.register(State.SUBCLASSES['list_comps']) -def has_context_loop(state, incorrect_msg, exact_names): - """When dispatched on loops, has_context the target vars are the attribute _target_vars. - - Note: This is to allow people to call has_context on a node (e.g. for_loop) rather than - one of its attributes (e.g. body). Purely for convenience. - """ - return _test(state, incorrect_msg or MSG_INCORRECT_LOOP, exact_names, - tv_name='_target_vars', highlight_name='target') - -@_has_context.register(State.SUBCLASSES['withs']) -def has_context_with(state, incorrect_msg, exact_names): - """When dispatched on with statements, has_context loops over each context manager. - - Note: This is to allow people to call has_context on the with statement, rather than - having to manually loop over each context manager. - - e.g. Ex().check_with(0).has_context() vs Ex().check_with(0).check_context(0).has_context() - """ - - for i in range(len(state.solution_parts['context'])): - ctxt_state = check_part_index('context', i, '{ordinal} context', state=state) - _has_context(ctxt_state, incorrect_msg or MSG_INCORRECT_WITH, exact_names) - - return state diff --git a/pythonwhat/check_object.py b/pythonwhat/check_object.py deleted file mode 100644 index ecb6a6d0..00000000 --- a/pythonwhat/check_object.py +++ /dev/null @@ -1,181 +0,0 @@ -from pythonwhat.parsing import ObjectAssignmentParser -from pythonwhat.Test import DefinedProcessTest, InstanceProcessTest, DefinedCollProcessTest -from pythonwhat.Reporter import Reporter -from pythonwhat.Feedback import Feedback -from pythonwhat.tasks import isDefinedInProcess, isInstanceInProcess, isDefinedCollInProcess -from pythonwhat.check_funcs import part_to_child -from pythonwhat.has_funcs import has_equal_value -import pandas as pd -import ast - -def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr="variable"): - """Check object existence (and equality) - - Check whether an object is defined in the student's environment, and zoom in on its value in both - student and solution environment to inspect quality (with has_equal_value(). - - Args: - index (str): the name of the object which value has to be checked. - missing_msg (str): feedback message when the object is not defined in the student's environment. - expect_msg (str): prepending message to put in front. - - :Example: - - Student code:: - - b = 1 - c = 3 - - Solution code:: - - a = 1 - b = 2 - c = 3 - - SCT:: - - Ex().check_object("a") # fail - Ex().check_object("b") # pass - Ex().check_object("b").has_equal_value() # fail - Ex().check_object("c").has_equal_value() # pass - - """ - - if missing_msg is None: - missing_msg = "__JINJA__:Did you define the {{typestr}} `{{index}}` without errors?" - - if expand_msg is None: - expand_msg = "__JINJA__:Did you correctly define the {{typestr}} `{{index}}`? " - - rep = Reporter.active_reporter - - if not isDefinedInProcess(index, state.solution_process): - raise NameError("%r not in solution environment " % index) - - append_message = {'msg': expand_msg, 'kwargs': {'index': index, 'typestr': typestr}} - - # create child state, using either parser output, or create part from name - fallback = lambda: ObjectAssignmentParser.get_part(index) - stu_part = state.student_object_assignments.get(index, fallback()) - sol_part = state.solution_object_assignments.get(index, fallback()) - - # test object exists - _msg = state.build_message(missing_msg, append_message['kwargs']) - rep.do_test(DefinedProcessTest(index, state.student_process, Feedback(_msg))) - - child = part_to_child(stu_part, sol_part, append_message, state) - - return child - -def is_instance(inst, not_instance_msg=None, state=None): - """Check whether an object is an instance of a certain class. - - ``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is - used to 'zoom in' on the object of interest. - - Args: - inst (class): The class that the object should have. - not_instance_msg (str): When specified, this overrides the automatically generated message in case - the object does not have the expected class. - state (State): The state that is passed in through the SCT chain (don't specify this). - - :Example: - - Student code and solution code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - - SCT:: - - # Verify the class of arr - import numpy - Ex().check_object('arr').is_instance(numpy.ndarray) - """ - rep = Reporter.active_reporter - - sol_name = state.solution_parts.get('name') - stu_name = state.student_parts.get('name') - - if not_instance_msg is None: not_instance_msg = "__JINJA__:Is it a {{inst.__name__}}?" - - if not isInstanceInProcess(sol_name, inst, state.solution_process): - raise ValueError("%r is not a %s in the solution environment" % (sol_name, type(inst))) - - _msg = state.build_message(not_instance_msg, {'inst': inst}) - feedback = Feedback(_msg, state) - rep.do_test(InstanceProcessTest(stu_name, inst, state.student_process, feedback)) - - return state - -def check_df(index, missing_msg=None, not_instance_msg=None, expand_msg=None, state=None): - """Check whether a DataFrame was defined and it is the right type""" - child = check_object(index, missing_msg=missing_msg, expand_msg=expand_msg, state=state, typestr="pandas DataFrame") - is_instance(pd.DataFrame, not_instance_msg=not_instance_msg, state=child) - return child - -def check_keys(key, missing_msg=None, expand_msg=None, state=None): - """Check whether an object (dict, DataFrame, etc) has a key. - - ``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is - used to 'zoom in' on the object of interest. - - Args: - key (str): Name of the key that the object should have. - missing_msg (str): When specified, this overrides the automatically generated - message in case the key does not exist. - state (State): The state that is passed in through the SCT chain (don't specify this). - - :Example: - - Student code and solution code:: - - x = {'a': 2} - - SCT:: - - # Verify that x contains a key a - Ex().check_object('x').check_keys('a') - - # Verify that x contains a key a and a is correct. - Ex().check_object('x').check_keys('a').has_equal_value() - - """ - - if missing_msg is None: - missing_msg = "__JINJA__:There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`." - if expand_msg is None: - expand_msg = "__JINJA__:Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? " - - rep = Reporter.active_reporter - - sol_name = state.solution_parts.get('name') - stu_name = state.student_parts.get('name') - - if not isDefinedCollInProcess(sol_name, key, state.solution_process): - raise NameError("Not all keys you specified are actually keys in %s in the solution process" % sol_name) - - # check if key available - _msg = state.build_message(missing_msg, {'key': key}) - rep.do_test(DefinedCollProcessTest(stu_name, key, state.student_process, - Feedback(_msg, state))) - - def get_part(name, key, highlight): - if isinstance(key, str): - slice_val = ast.Str(s=key) - else: - slice_val = ast.parse('{}'.format(key)).body[0].value - expr = ast.Subscript(value=ast.Name(id=name, ctx=ast.Load()), - slice=ast.Index(value=slice_val), - ctx=ast.Load()) - ast.fix_missing_locations(expr) - return { - 'node': expr, - 'highlight': highlight - } - - stu_part = get_part(stu_name, key, state.student_parts.get('highlight')) - sol_part = get_part(sol_name, key, state.solution_parts.get('highlight')) - append_message = {'msg': expand_msg, 'kwargs': {'key': key }} - child = part_to_child(stu_part, sol_part, append_message, state) - return child diff --git a/pythonwhat/check_syntax.py b/pythonwhat/check_syntax.py deleted file mode 100644 index 3434f513..00000000 --- a/pythonwhat/check_syntax.py +++ /dev/null @@ -1,127 +0,0 @@ -from pythonwhat.check_wrappers import scts -from pythonwhat.State import State -from pythonwhat.probe import Node, Probe, TEST_NAMES -from pythonwhat.utils import include_v1 -from pythonwhat import test_funcs -from functools import partial, reduce, wraps -import inspect -import copy -import os - -# TODO: could define scts for check_wrappers at the module level -ATTR_SCTS = scts.copy() - -def multi_dec(f): - """Decorator for multi to remove nodes for original test functions from root node""" - - @wraps(f) - def wrapper(*args, **kwargs): - args = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args - for arg in args: - if isinstance(arg, Node) and arg.parent.name is 'root': - arg.parent.remove_child(arg) - arg.update_child_calls() - return f(*args, **kwargs) - return wrapper - - -def state_dec(f): - """Decorate check_* functions to return F chain if no state passed""" - - @wraps(f) - def wrapper(*args, **kwargs): - ba = inspect.signature(f).bind(*args, **kwargs) - ba.apply_defaults() - - state_arg = ba.arguments.get('state') - if isinstance(state_arg, State): # proper state, run function - return f(*args, **kwargs) - elif state_arg is None: # default state arg, make partial - return F._from_func(partial(f, *args, **kwargs)) - else: # passed improper state arg - raise BaseException("Did you use the right number of arguments in your SCT?") - - return wrapper - -class Chain: - def __init__(self, state): - self._state = state - self._crnt_sct = None - self._waiting_on_call = False - - def _double_attr_error(self): - raise AttributeError("Did you forget to call a statement? " - "e.g. Ex().check_list_comp.check_body()") - - def __getattr__(self, attr): - if attr not in ATTR_SCTS: raise AttributeError("No SCT named %s"%attr) - elif self._waiting_on_call: self._double_attr_error() - else: - # make a copy to return, - # in case someone does: a = chain.a; b = chain.b - return self._sct_copy(ATTR_SCTS[attr]) - - def __call__(self, *args, **kwargs): - self._state = self._crnt_sct(state=self._state, *args, **kwargs) - self._waiting_on_call = False - return self - - def __rshift__(self, f): - if self._waiting_on_call: - self._double_attr_error() - elif type(f) == Chain: - raise BaseException("did you use a result of the Ex() function on the right hand side of the >> operator?") - elif not callable(f): - raise BaseException("right hand side of >> operator should be an SCT, so must be callable!") - else: - chain = self._sct_copy(f) - return chain() - - def _sct_copy(self, f): - chain = copy.copy(self) - chain._crnt_sct = f - chain._waiting_on_call = True - return chain - -class F(Chain): - def __init__(self, stack = None): - self._crnt_sct = None - self._stack = [] if stack is None else stack - self._waiting_on_call = False - - def __call__(self, *args, **kwargs): - if not self._crnt_sct: - state = kwargs.get('state') or args[0] - return reduce(lambda s, f: f(state=s), self._stack, state) - else: - pf = partial(self._crnt_sct, *args, **kwargs) - return self.__class__(self._stack + [pf]) - - @classmethod - def _from_func(cls, f): - func_chain = cls() - func_chain._stack.append(f) - return func_chain - -def Ex(state = None): - return Chain(state or State.root_state) - -if include_v1(): - # Prepare SCTs that may be chained attributes ---------------------- - # decorate functions that may try to run test_* function nodes as subtests - # so they remove those nodes from the tree - for k in ['multi', 'with_context']: - ATTR_SCTS[k] = multi_dec(ATTR_SCTS[k]) - - # allow test_* functions as chained attributes - for k in TEST_NAMES: - ATTR_SCTS[k] = Probe(tree = None, f = getattr(test_funcs, k), eval_on_call=True) - - # original logical test_* functions behave like multi - # this is necessary to allow them to take check_* funcs as args - # since probe behavior will try to call all SCTs passed (assuming they're also probes) - for k in ['test_or', 'test_correct']: - ATTR_SCTS[k] = multi_dec(getattr(test_funcs, k)) - -# Prepare check_funcs to be used alone (e.g. test = check_with().check_body()) -v2_check_functions = {k : state_dec(v) for k, v in scts.items()} diff --git a/pythonwhat/check_wrappers.py b/pythonwhat/check_wrappers.py deleted file mode 100644 index 6b6b3c7b..00000000 --- a/pythonwhat/check_wrappers.py +++ /dev/null @@ -1,72 +0,0 @@ -from pythonwhat.check_funcs import check_part, check_part_index, check_node -from pythonwhat.has_funcs import has_equal_part -from pythonwhat import check_funcs, has_funcs, check_logic, check_object -from pythonwhat.check_function import check_function -from pythonwhat.check_has_context import has_context - -from functools import partial -import inspect - -__PART_WRAPPERS__ = { - 'iter': 'iterable part', - 'body': 'body', - 'key' : 'key part', - 'value': 'value part', - 'orelse': 'else part', - 'finalbody': 'finally part', - 'test': 'condition' - } - -__PART_INDEX_WRAPPERS__ = { - 'ifs': '{ordinal} if', - 'handlers': '`{index}` `except` block', - 'context': '{ordinal} context' - } - -__NODE_WRAPPERS__ = { - 'list_comp': '{ordinal} list comprehension', - 'generator_exp': '{ordinal} generator expression', - 'dict_comp': '{ordinal} dictionary comprehension', - 'for_loop': '{ordinal} for statement', - 'function_def': 'definition of `{index}()`', - 'if_exp': '{ordinal} if expression', - 'if_else': '{ordinal} if statement', - 'lambda_function': '{ordinal} lambda function', - 'try_except': '{ordinal} try statement', - 'while': '{ordinal} `while` loop', - 'with': '{ordinal} `with` statement' - } - -scts = {} - -# make has_equal_part wrappers - -scts['has_equal_name'] = partial(has_equal_part, 'name', msg='Make sure to use the correct {name}, was expecting {sol_part[name]}, instead got {stu_part[name]}.') -scts['is_default'] = partial(has_equal_part, 'is_default', msg="__JINJA__:Make sure it {{ 'has' if sol_part.is_default else 'does not have'}} a default argument.") - -# include rest of wrappers -for k, v in __PART_WRAPPERS__.items(): - scts['check_'+k] = partial(check_part, k, v) - -for k, v in __PART_INDEX_WRAPPERS__.items(): - scts['check_'+k] = partial(check_part_index, k, part_msg=v) - -for k, v in __NODE_WRAPPERS__.items(): - scts['check_'+k] = partial(check_node, k+'s', typestr=v) - -for k in ['set_context', 'set_env', 'disable_highlighting', 'check_not', 'check_or', 'check_correct', 'fail', 'quiet', 'override', 'multi']: - scts[k] = getattr(check_logic, k) - -for k in ['with_context', 'check_args', 'check_call']: - scts[k] = getattr(check_funcs, k) - -for k in ['has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'has_equal_part_len', - 'has_equal_part', 'has_import', 'has_output', 'has_printout', 'has_code', 'has_chosen']: - scts[k] = getattr(has_funcs, k) - -# include check_object and friends ------ -for k in ['check_object', 'is_instance', 'check_df', 'check_keys']: - scts[k] = getattr(check_object, k) - -scts['has_context'] = has_context -scts['check_function'] = check_function \ No newline at end of file diff --git a/pythonwhat/checks/__init__.py b/pythonwhat/checks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pythonwhat/checks/check_funcs.py b/pythonwhat/checks/check_funcs.py new file mode 100644 index 00000000..be618aa4 --- /dev/null +++ b/pythonwhat/checks/check_funcs.py @@ -0,0 +1,329 @@ +from protowhat.Feedback import FeedbackComponent +from pythonwhat.checks.check_logic import multi +from pythonwhat.checks.has_funcs import has_part +from protowhat.failure import debugger +from pythonwhat.tasks import setUpNewEnvInProcess, breakDownNewEnvInProcess +from protowhat.utils_messaging import get_ord +from pythonwhat.utils_ast import assert_ast +import ast +from jinja2 import Template + + +def render(template, kwargs): + return Template(template).render(**kwargs) + + +def part_to_child(stu_part, sol_part, append_message, state, node_name=None): + # stu_part and sol_part will be accessible on all templates + append_message.kwargs.update({"stu_part": stu_part, "sol_part": sol_part}) + + # if the parts are dictionaries, use to deck out child state + if all(isinstance(p, dict) for p in [stu_part, sol_part]): + child_state = state.to_child( + student_ast=stu_part["node"], + solution_ast=sol_part["node"], + student_context=stu_part.get("target_vars"), + solution_context=sol_part.get("target_vars"), + student_parts=stu_part, + solution_parts=sol_part, + highlight=stu_part.get("highlight"), + append_message=append_message, + node_name=node_name, + ) + else: + # otherwise, assume they are just nodes + child_state = state.to_child( + student_ast=stu_part, + solution_ast=sol_part, + append_message=append_message, + node_name=node_name, + ) + + return child_state + + +def check_part(state, name, part_msg, missing_msg=None, expand_msg=None): + """Return child state with name part as its ast tree""" + + if missing_msg is None: + missing_msg = "Are you sure you defined the {{part}}? " + if expand_msg is None: + expand_msg = "Did you correctly specify the {{part}}? " + + if not part_msg: + part_msg = name + append_message = FeedbackComponent(expand_msg, {"part": part_msg}) + + has_part(state, name, missing_msg, append_message.kwargs) + + stu_part = state.student_parts[name] + sol_part = state.solution_parts[name] + + assert_ast(state, sol_part, append_message.kwargs) + + return part_to_child(stu_part, sol_part, append_message, state) + + +def check_part_index(state, name, index, part_msg, missing_msg=None, expand_msg=None): + """Return child state with indexed name part as its ast tree. + + ``index`` can be: + + - an integer, in which case the student/solution_parts are indexed by position. + - a string, in which case the student/solution_parts are expected to be a dictionary. + - a list of indices (which can be integer or string), in which case the student parts are indexed step by step. + """ + + if missing_msg is None: + missing_msg = "Are you sure you defined the {{part}}? " + if expand_msg is None: + expand_msg = "Did you correctly specify the {{part}}? " + + # create message + ordinal = get_ord(index + 1) if isinstance(index, int) else "" + fmt_kwargs = {"index": index, "ordinal": ordinal} + fmt_kwargs.update(part=render(part_msg, fmt_kwargs)) + + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + + # check there are enough parts for index + has_part(state, name, missing_msg, fmt_kwargs, index) + + # get part at index + stu_part = state.student_parts[name] + sol_part = state.solution_parts[name] + + if isinstance(index, list): + for ind in index: + stu_part = stu_part[ind] + sol_part = sol_part[ind] + else: + stu_part = stu_part[index] + sol_part = sol_part[index] + + assert_ast(state, sol_part, fmt_kwargs) + + # return child state from part + return part_to_child(stu_part, sol_part, append_message, state) + + +def check_node( + state, name, index=0, typestr="{{ordinal}} node", missing_msg=None, expand_msg=None +): + + if missing_msg is None: + missing_msg = "The system wants to check the {{typestr}} but hasn't found it." + if expand_msg is None: + expand_msg = "Check the {{typestr}}. " + + stu_out = state.ast_dispatcher.find(name, state.student_ast) + sol_out = state.ast_dispatcher.find(name, state.solution_ast) + + # check if there are enough nodes for index + fmt_kwargs = { + "ordinal": get_ord(index + 1) if isinstance(index, int) else "", + "index": index, + "name": name, + } + fmt_kwargs["typestr"] = render(typestr, fmt_kwargs) + + # test if node can be indexed succesfully + try: + stu_out[index] + except (KeyError, IndexError): # TODO comment errors + state.report(missing_msg, fmt_kwargs) + + # get node at index + stu_part = stu_out[index] + sol_part = sol_out[index] + + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + + return part_to_child(stu_part, sol_part, append_message, state, node_name=name) + + +# context functions ----------------------------------------------------------- +# TODO: check if still useful +def with_context(state, *args, child=None): + + # set up context in processes + solution_res = setUpNewEnvInProcess( + process=state.solution_process, context=state.solution_parts["with_items"] + ) + if isinstance(solution_res, Exception): + with debugger(state): + state.report( + "error in the solution, running test_with(): %s" % str(solution_res) + ) + + student_res = setUpNewEnvInProcess( + process=state.student_process, context=state.student_parts["with_items"] + ) + if isinstance(student_res, AttributeError): + child.report( + "In your `with` statement, you're not using a correct context manager." + ) + + if isinstance(student_res, (AssertionError, ValueError, TypeError)): + child.report( + "In your `with` statement, the number of values in your context manager " + "doesn't correspond to the number of variables you're trying to assign it to." + ) + + # run subtests + try: + multi(state, *args) + finally: + # exit context + close_solution_context = breakDownNewEnvInProcess( + process=state.solution_process + ) + if isinstance(close_solution_context, Exception): + with debugger(state): + state.report( + "error in the solution, closing the `with` fails with: %s" + % close_solution_context + ) + + close_student_context = breakDownNewEnvInProcess(process=state.student_process) + if isinstance(close_student_context, Exception): + state.report( + "Your `with` statement can not be closed off correctly, you're " + "not using the context manager correctly." + ) + return state + + +def check_args(state, name, missing_msg=None): + """Check whether a function argument is specified. + + This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified. + If you want to go on and check whether the argument was correctly specified, you can can continue chaining with + ``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check) + + This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been + specified. + + Args: + name (str): the name of the argument for which you want to check if it is specified. This can also be + a number, in which case it refers to the positional arguments. Named arguments take precedence. + missing_msg (str): If specified, this overrides the automatically generated feedback message in case + the student did specify the argument. + state (State): State object that is passed from the SCT Chain (don't specify this). + + :Examples: + + Student and solution code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + np.mean(arr) + + SCT:: + + # Verify whether arr was correctly set in np.mean + # has_equal_value() checks the value of arr, used to set argument a + Ex().check_function('numpy.mean').check_args('a').has_equal_value() + + # Verify whether arr was correctly set in np.mean + # has_equal_ast() checks the expression used to set argument a + Ex().check_function('numpy.mean').check_args('a').has_equal_ast() + + Student and solution code:: + + def my_power(x): + print("calculating sqrt...") + return(x * x) + + SCT:: + + Ex().check_function_def('my_power').multi( + check_args('x') # will fail if student used y as arg + check_args(0) # will still pass if student used y as arg + ) + + """ + if missing_msg is None: + missing_msg = "Did you specify the {{part}}?" + + if name in ["*args", "**kwargs"]: # for check_function_def + return check_part(state, name, name, missing_msg=missing_msg) + else: + if isinstance(name, list): # dealing with args or kwargs + if name[0] == "args": + arg_str = "{} argument passed as a variable length argument".format( + get_ord(name[1] + 1) + ) + else: + arg_str = "argument `{}`".format(name[1]) + else: + arg_str = ( + "{} argument".format(get_ord(name + 1)) + if isinstance(name, int) + else "argument `{}`".format(name) + ) + return check_part_index(state, "args", name, arg_str, missing_msg=missing_msg) + + +# CALL CHECK ================================================================== + + +def build_call(callstr, node): + if isinstance(node, ast.FunctionDef): # function name + func_expr = ast.Name(id=node.name, ctx=ast.Load()) + argstr = "`%s`" % callstr.replace("f", node.name) + elif isinstance(node, ast.Lambda): # lambda body expr + func_expr = node + argstr = "it with the arguments `%s`" % callstr.replace("f", "") + else: + raise TypeError("Can't handle AST that is passed.") + + parsed = ast.parse(callstr).body[0].value + parsed.func = func_expr + ast.fix_missing_locations(parsed) + return parsed, argstr + + +def check_call(state, callstr, argstr=None, expand_msg=None): + """When checking a function definition of lambda function, + prepare has_equal_x for checking the call of a user-defined function. + + Args: + callstr (str): call string that specifies how the function should be called, e.g. `f(1, a = 2)`. + ``check_call()`` will replace ``f`` with the function/lambda you're targeting. + argstr (str): If specified, this overrides the way the function call is refered to in the expand message. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + state (State): state object that is chained from. + + :Example: + + Student and solution code:: + + def my_power(x): + print("calculating sqrt...") + return(x * x) + + SCT:: + + Ex().check_function_def('my_power').multi( + check_call("f(3)").has_equal_value() + check_call("f(3)").has_equal_output() + ) + """ + + state.assert_is( + ["function_defs", "lambda_functions"], + "check_call", + ["check_function_def", "check_lambda_function"], + ) + + if expand_msg is None: + expand_msg = "To verify it, we reran {{argstr}}. " + + stu_part, _argstr = build_call(callstr, state.student_parts["node"]) + sol_part, _ = build_call(callstr, state.solution_parts["node"]) + + append_message = FeedbackComponent(expand_msg, {"argstr": argstr or _argstr}) + child = part_to_child(stu_part, sol_part, append_message, state) + + return child diff --git a/pythonwhat/checks/check_function.py b/pythonwhat/checks/check_function.py new file mode 100644 index 00000000..ca093c24 --- /dev/null +++ b/pythonwhat/checks/check_function.py @@ -0,0 +1,172 @@ +from protowhat.Feedback import FeedbackComponent +from pythonwhat.checks.check_funcs import part_to_child +from pythonwhat.tasks import getSignatureInProcess +from protowhat.utils_messaging import get_ord, get_times +from protowhat.failure import debugger +from pythonwhat.parsing import IndexedDict +from functools import partial + + +def bind_args(signature, args_part): + pos_args = [] + kw_args = {} + for k, arg in args_part.items(): + if isinstance(k, int): + pos_args.append(arg) + else: + kw_args[k] = arg + + bound_args = signature.bind(*pos_args, **kw_args) + return IndexedDict(bound_args.arguments) + + +def get_mapped_name(name, mappings): + # get name by splitting on periods + if "." in name: + for orig, full_name in mappings.items(): + if name.startswith(full_name): + return name.replace(full_name, orig) + return name + + +MISSING_MSG = "Did you call `{{mapped_name}}()`{{' ' + times if index>0}}?" +SIG_ISSUE_MSG = ( + "Have you specified the arguments for `{{mapped_name}}()` using the right syntax?" +) +PREPEND_MSG = "Check your {{ord + ' ' if index>0}}call of `{{mapped_name}}()`. " + + +def check_function( + state, + name, + index=0, + missing_msg=None, + params_not_matched_msg=None, + expand_msg=None, + signature=True, +): + """Check whether a particular function is called. + + ``check_function()`` is typically followed by: + + - ``check_args()`` to check whether the arguments were specified. + In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()`` + to assert that the arguments were correctly specified. + - ``has_equal_value()`` to check whether rerunning the function call coded by the student + gives the same result as calling the function call as in the solution. + + Checking function calls is a tricky topic. Please visit the + `dedicated article `_ for more explanation, + edge cases and best practices. + + Args: + name (str): the name of the function to be tested. When checking functions in packages, always + use the 'full path' of the function. + index (int): index of the function call to be checked. Defaults to 0. + missing_msg (str): If specified, this overrides an automatically generated feedback message in case + the student did not call the function correctly. + params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message + in case the function parameters were not successfully matched. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + signature (Signature): Normally, check_function() can figure out what the function signature is, + but it might be necessary to use ``sig_from_params()`` to manually build a signature and pass this along. + state (State): State object that is passed from the SCT Chain (don't specify this). + + :Examples: + + Student code and solution code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + np.mean(arr) + + SCT:: + + # Verify whether arr was correctly set in np.mean + Ex().check_function('numpy.mean').check_args('a').has_equal_value() + + # Verify whether np.mean(arr) produced the same result + Ex().check_function('numpy.mean').has_equal_value() + """ + + append_missing = missing_msg is None + append_params_not_matched = params_not_matched_msg is None + if missing_msg is None: + missing_msg = MISSING_MSG + if expand_msg is None: + expand_msg = PREPEND_MSG + if params_not_matched_msg is None: + params_not_matched_msg = SIG_ISSUE_MSG + + stu_out = state.ast_dispatcher.find("function_calls", state.student_ast) + sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast) + + student_mappings = state.ast_dispatcher.find("mappings", state.student_ast) + + fmt_kwargs = { + "times": get_times(index + 1), + "ord": get_ord(index + 1), + "index": index, + "mapped_name": get_mapped_name(name, student_mappings), + } + + # Get Parts ---- + # Copy, otherwise signature binding overwrites sol_out[name][index]['args'] + with debugger(state): + try: + sol_parts = {**sol_out[name][index]} + except KeyError: + state.report( + "`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!" + % name + ) + except IndexError: + state.report( + "`check_function()` couldn't find %s calls of `%s()` in your solution code." + % (index + 1, name) + ) + + try: + # Copy, otherwise signature binding overwrites stu_out[name][index]['args'] + stu_parts = {**stu_out[name][index]} + except (KeyError, IndexError): + state.report(missing_msg, fmt_kwargs, append=append_missing) + + # Signatures ----- + if signature: + signature = None if isinstance(signature, bool) else signature + get_sig = partial( + getSignatureInProcess, + name=name, + signature=signature, + manual_sigs=state.get_manual_sigs(), + ) + + try: + sol_sig = get_sig( + mapped_name=sol_parts["name"], process=state.solution_process + ) + sol_parts["args"] = bind_args(sol_sig, sol_parts["args"]) + except Exception as e: + with debugger(state): + state.report( + "`check_function()` couldn't match the %s call of `%s` to its signature:\n%s " + % (get_ord(index + 1), name, e) + ) + + try: + stu_sig = get_sig( + mapped_name=stu_parts["name"], process=state.student_process + ) + stu_parts["args"] = bind_args(stu_sig, stu_parts["args"]) + except Exception: + state.to_child(highlight=stu_parts["node"]).report( + params_not_matched_msg, fmt_kwargs, append=append_params_not_matched + ) + + # three types of parts: pos_args, keywords, args (e.g. these are bound to sig) + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + child = part_to_child( + stu_parts, sol_parts, append_message, state, node_name="function_calls" + ) + return child diff --git a/pythonwhat/checks/check_has_context.py b/pythonwhat/checks/check_has_context.py new file mode 100644 index 00000000..2a2fef24 --- /dev/null +++ b/pythonwhat/checks/check_has_context.py @@ -0,0 +1,92 @@ +from pythonwhat.Test import EqualTest +from protowhat.Feedback import FeedbackComponent +from protowhat.failure import debugger +from pythonwhat.State import State +from functools import singledispatch +from pythonwhat.checks.check_funcs import check_part_index + +MSG_INCORRECT_LOOP = "Have you used the correct iterator variable names? Was expecting `{{sol_vars}}` but got `{{stu_vars}}`." +MSG_INCORRECT_WITH = "Make sure to use the correct context variable names. Was expecting `{{sol_vars}}` but got `{{stu_vars}}`." + + +def has_context(state, incorrect_msg=None, exact_names=False): + # call _has_context, since the built-in singledispatch can only use 1st pos arg + return _has_context(state, incorrect_msg, exact_names) + + +def _test(state, incorrect_msg, exact_names, tv_name, highlight_name): + # get parts for testing from state + # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. + stu_vars = state.student_parts[tv_name] + sol_vars = state.solution_parts[tv_name] + + child_state = state.to_child( + student_ast=state.student_parts.get(highlight_name), + solution_ast=state.solution_parts.get(highlight_name), + ) + + # variables exposed to messages + d = {"stu_vars": stu_vars, "sol_vars": sol_vars, "num_vars": len(sol_vars)} + + if exact_names: + # feedback for wrong iter var names + child_state.do_test( + EqualTest(stu_vars, sol_vars, FeedbackComponent(incorrect_msg, d)) + ) + else: + # feedback for wrong number of iter vars + child_state.do_test( + EqualTest(len(stu_vars), len(sol_vars), FeedbackComponent(incorrect_msg, d)) + ) + + return state + + +@singledispatch +def _has_context(state, incorrect_msg, exact_names): + with debugger(state): + state.report( + "first argument to _has_context must be a State instance or subclass" + ) + + +@_has_context.register(State) +def has_context_state(*args, **kwargs): + return _test(*args, tv_name="target_vars", highlight_name="highlight", **kwargs) + + +@_has_context.register(State.SUBCLASSES["for_loops"]) +@_has_context.register(State.SUBCLASSES["whiles"]) +@_has_context.register(State.SUBCLASSES["dict_comps"]) +@_has_context.register(State.SUBCLASSES["generator_exps"]) +@_has_context.register(State.SUBCLASSES["list_comps"]) +def has_context_loop(state, incorrect_msg, exact_names): + """When dispatched on loops, has_context the target vars are the attribute _target_vars. + + Note: This is to allow people to call has_context on a node (e.g. for_loop) rather than + one of its attributes (e.g. body). Purely for convenience. + """ + return _test( + state, + incorrect_msg or MSG_INCORRECT_LOOP, + exact_names, + tv_name="_target_vars", + highlight_name="target", + ) + + +@_has_context.register(State.SUBCLASSES["withs"]) +def has_context_with(state, incorrect_msg, exact_names): + """When dispatched on with statements, has_context loops over each context manager. + + Note: This is to allow people to call has_context on the with statement, rather than + having to manually loop over each context manager. + + e.g. Ex().check_with(0).has_context() vs Ex().check_with(0).check_context(0).has_context() + """ + + for i in range(len(state.solution_parts["context"])): + ctxt_state = check_part_index(state, "context", i, "{{ordinal}} context") + _has_context(ctxt_state, incorrect_msg or MSG_INCORRECT_WITH, exact_names) + + return state diff --git a/pythonwhat/check_logic.py b/pythonwhat/checks/check_logic.py similarity index 53% rename from pythonwhat/check_logic.py rename to pythonwhat/checks/check_logic.py index 8cdf0f95..e9b86ab5 100644 --- a/pythonwhat/check_logic.py +++ b/pythonwhat/checks/check_logic.py @@ -1,105 +1,127 @@ -from types import GeneratorType -from functools import partial -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, TestFail -from pythonwhat.Feedback import Feedback -import copy +from protowhat.Feedback import FeedbackComponent +from protowhat.checks.check_logic import ( + multi, + check_not, + check_or, + check_correct, + disable_highlighting, + fail, +) +from protowhat.failure import InstructorError import ast -def multi(*args, state=None): - """Run multiple subtests. Return original state (for chaining).""" - if any(args): - rep = Reporter.active_reporter - # when input is a single list of subtests - if len(args) == 1 and isinstance(args[0], (list, tuple, GeneratorType)): - args = args[0] +multi.__doc__ = ( + str(multi.__doc__) + + """ + :Example: - for test in args: - # assume test is function needing a state argument - # partial state so reporter can test - rep.do_test(partial(test, state=state)) + Suppose we want to verify the following function call: :: - # return original state, so can be chained - return state + round(1.2345, ndigits=2) -def check_not(*tests, msg, state=None): - """Run multiple subtests that should fail. If all subtests fail, returns original state (for chaining) + The following SCT would verify this, using ``multi`` to + 'branch out' the state to two sub-SCTs: :: - Args: - state: State instance describing student and solution code. Can be omitted if used with Ex(). - args: one or more sub-SCTs to run. + Ex().check_function('round').multi( + check_args(0).has_equal_value(), + check_args('ndigits').has_equal_value() + ) + """ +) + + +check_not.__doc__ = ( + str(check_not.__doc__) + + """ :Example: - Thh SCT below runs two test_student_typed cases.. :: + The SCT fails with feedback for a specific incorrect value, defined using an override: :: + + Ex().check_object('result').multi( + check_not( + has_equal_value(override=100), + msg='100 is incorrect for reason xyz.' + ), + has_equal_value() + ) - Ex().multi(test_student_typed('INNER'), test_student_typed('OUTER')) + Notice that ``check_not`` comes before the ``has_equal_value`` test + that checks if the student value is equal to the solution value. - If students use INNER (JOIN) or OUTER (JOIN) in their code, this test will fail. + :Example: + The SCT below runs two ``has_code`` cases: :: + + Ex().check_not( + has_code('mean'), + has_code('median'), + msg='Check your code' + ) + + If students use ``mean`` or ``median`` anywhere in their code, this SCT will fail. Note: - - This function is currently only tested in working with test_student_typed in the subtests. + - This function is not yet tested with all checks, please report unexpected behaviour. - This function can be thought as a NOT(x OR y OR ...) statement, since all tests it runs must fail - This function can be considered a direct counterpart of multi. """ - rep = Reporter.active_reporter - - for test in tests: - try: - multi(test, state=state) - except TestFail: - # it fails, as expected, off to next one - continue - return rep.do_test(Test(msg)) - - # return original state, so can be chained - return state - -def check_or(*tests, state=None): - """Test whether at least one SCT passes.""" - - rep = Reporter.active_reporter - - success = False - first_feedback = None - for test in tests: - try: - multi(test, state=state) - success = True - except TestFail as e: - if not first_feedback: first_feedback = e.feedback - if success: - return - - rep.do_test(Test(first_feedback)) - -def check_correct(check, diagnose, state=None): - """Allows feedback from a diagnostic SCT, only if a check SCT fails. """ - def diagnose_and_check(state=None): - # use multi twice, since diagnose and check may be lists of tests - multi(diagnose, state=state) - multi(check, state=state) - - check_or(diagnose_and_check, check, state=state) +) + + +check_or.__doc__ = ( + str(check_or.__doc__) + + """ + :Example: + + The SCT below tests that the student typed either 'mean' or 'median': :: + + Ex().check_or( + has_code('mean'), + has_code('median') + ) + + If the student didn't type either, the feedback message generated by ``has_code(mean)``, + the first SCT, will be presented to the student. + + """ +) + + +check_correct.__doc__ = ( + str(check_correct.__doc__) + + """ + :Example: + + The SCT below tests whether an object is correct. Only if the object is not correct, will + the function calling checks be executed :: + + Ex().check_correct( + check_object('x').has_equal_value(), + check_function('round').check_args(0).has_equal_value() + ) + + """ +) + # utility functions ----------------------------------------------------------- -def quiet(n = 0, state=None): - """Turn off prepended messages. Defaults to turning all off.""" - cpy = copy.copy(state) - hushed = [{**m, 'msg': ""} for m in cpy.messages] - cpy.messages = hushed - return cpy -def fail(msg="", state=None): - """Fail test with message""" - rep = Reporter.active_reporter - _msg = state.build_message(msg) - rep.do_test(Test(Feedback(_msg, state))) +fail.__doc__ = ( + str(fail.__doc__) + + """ + :Example: + + As a trivial SCT example, :: + + Ex().check_for_loop().check_body().fail() - return state + This can also be helpful for debugging SCTs, as it can be used to stop testing as a given point. + """ +) -def override(solution, state=None): + +def override(state, solution): """Override the solution code with something arbitrary. There might be cases in which you want to temporarily override the solution code @@ -118,37 +140,39 @@ def override(solution, state=None): # (1) ast.Module, or for single expressions... # (2) whatever was grabbed using module.body[0] # (3) module.body[0].value, when module.body[0] is an Expr node - old_ast = state.solution_tree + old_ast = state.solution_ast new_ast = ast.parse(solution) if not isinstance(old_ast, ast.Module) and len(new_ast.body) == 1: expr = new_ast.body[0] candidates = [expr, expr.value] if isinstance(expr, ast.Expr) else [expr] for node in candidates: - if isinstance(node, old_ast.__class__): + if isinstance(node, old_ast.__class__): new_ast = node break - kwargs = state.messages[-1] if state.messages else {} - child = state.to_child_state( - solution_subtree = new_ast, - student_subtree = state.student_tree, - highlight = state.highlight, - append_message = {'msg': "", 'kwargs': kwargs} - ) + kwargs = state.feedback_context.kwargs if state.feedback_context else {} + child = state.to_child( + solution_ast=new_ast, + student_ast=state.student_ast, + highlight=state.highlight, + append_message=FeedbackComponent("", kwargs), + ) return child -def set_context(*args, state=None, **kwargs): +def set_context(state, *args, **kwargs): """Update context values for student and solution environments. - + When ``has_equal_x()`` is used after this, the context values (in ``for`` loops and function definitions, for example) - will have the values specified throught his function. It is the function equivalent of the ``context_vals`` argument of + will have the values specified through his function. It is the function equivalent of the ``context_vals`` argument of the ``has_equal_x()`` functions. - Note 1: excess args and unmatched kwargs will be unused in the student environment. - - Note 2: positional arguments are more robust to the student using different names for context values. - - Note 3: You have to specify arguments either by position, either by name. A combination is not possible. + - Note 2: When you try to set context values that don't match any target variables in the solution code, + ``set_context()`` raises an exception that lists the ones available. + - Note 3: positional arguments are more robust to the student using different names for context values. + - Note 4: You have to specify arguments either by position, either by name. A combination is not possible. :Example: @@ -184,14 +208,19 @@ def set_context(*args, state=None, **kwargs): # for now, you can't specify both if len(args) > 0 and len(kwargs) > 0: - raise ValueError("In set_context() make sure to specify arguments either by position, either by name") + raise InstructorError.from_message( + "In `set_context()`, specify arguments either by position, either by name." + ) # set args specified by pos ----------------------------------------------- if args: # stop if too many pos args for solution - if len(args) > len(sol_crnt): - raise IndexError("Too many positional args. There are {} context vals, but tried to set {}" - .format(len(sol_crnt), len(args))) + if len(args) > len(sol_crnt): + raise InstructorError.from_message( + "Too many positional args. There are {} context vals, but tried to set {}".format( + len(sol_crnt), len(args) + ) + ) # set pos args upd_sol = sol_crnt.update(dict(zip(sol_crnt.keys(), args))) upd_stu = stu_crnt.update(dict(zip(stu_crnt.keys(), args))) @@ -203,22 +232,28 @@ def set_context(*args, state=None, **kwargs): if kwargs: # stop if keywords don't match with solution if set(kwargs) - set(upd_sol): - raise KeyError("Context val names are {}, but tried to set {}" - .format(upd_sol or "none", kwargs.keys())) + raise InstructorError.from_message( + "`set_context()` failed: context val names are {}, but you tried to set {}.".format( + upd_sol or "missing", sorted(list(kwargs.keys())) + ) + ) out_sol = upd_sol.update(kwargs) # need to match keys in kwargs with corresponding keys in stu context # in case they used, e.g., different loop variable names match_keys = dict(zip(sol_crnt.keys(), stu_crnt.keys())) - out_stu = upd_stu.update({match_keys[k]: v for k,v in kwargs.items() if k in match_keys}) + out_stu = upd_stu.update( + {match_keys[k]: v for k, v in kwargs.items() if k in match_keys} + ) else: out_sol = upd_sol out_stu = upd_stu - return state.to_child_state(student_context = out_stu, - solution_context = out_sol, - highlight = state.highlight) + return state.to_child( + student_context=out_stu, solution_context=out_sol, highlight=state.highlight + ) -def set_env(state = None, **kwargs): + +def set_env(state, **kwargs): """Update/set environemnt variables for student and solution environments. When ``has_equal_x()`` is used after this, the variables specified through this function will @@ -238,8 +273,8 @@ def set_env(state = None, **kwargs): # check if condition works with different values of a Ex().check_if_else().check_test().multi( - set_env(a = 3).has_equal_value() - set_env(a = 4).has_equal_value() + set_env(a = 3).has_equal_value(), + set_env(a = 4).has_equal_value(), set_env(a = 5).has_equal_value() ) @@ -254,15 +289,14 @@ def set_env(state = None, **kwargs): stu_new = stu_crnt.update(kwargs) sol_new = sol_crnt.update(kwargs) - return state.to_child_state(student_env = stu_new, - solution_env = sol_new, - highlight = state.highlight) - -def disable_highlighting(state = None): - """Disable highlighting in the remainder of the SCT chain. + return state.to_child( + student_env=stu_new, solution_env=sol_new, highlight=state.highlight + ) - Include this function if you want to avoid that pythonwhat marks which part of the student submission is incorrect. +disable_highlighting.__doc__ = ( + str(disable_highlighting.__doc__) + + """ :Examples: SCT that will mark the 'number' portion if it is incorrect:: @@ -275,4 +309,4 @@ def disable_highlighting(state = None): Ex().check_function('round').disable_highlighting().check_args(0).has_equal_ast() Ex().check_function('round').check_args(0).disable_highlighting().has_equal_ast() """ - return state.to_child_state(highlighting_disabled = True) +) diff --git a/pythonwhat/checks/check_object.py b/pythonwhat/checks/check_object.py new file mode 100644 index 00000000..2b0b8856 --- /dev/null +++ b/pythonwhat/checks/check_object.py @@ -0,0 +1,372 @@ +from pythonwhat.parsing import ObjectAssignmentParser +from pythonwhat.Test import ( + DefinedProcessTest, + InstanceProcessTest, + DefinedCollProcessTest, +) +from protowhat.Feedback import FeedbackComponent +from protowhat.failure import InstructorError +from pythonwhat.tasks import ( + isDefinedInProcess, + isInstanceInProcess, + isDefinedCollInProcess, +) +from pythonwhat.checks.check_funcs import part_to_child +from pythonwhat.utils import v2_only +import ast + + +def check_object(state, index, missing_msg=None, expand_msg=None, typestr="variable"): + """Check object existence (and equality) + + Check whether an object is defined in the student's process, and zoom in on its value in both + student and solution process to inspect quality (with has_equal_value(). + + In ``pythonbackend``, both the student's submission as well as the solution code are executed, in separate processes. + ``check_object()`` looks at these processes and checks if the referenced object is available in the student process. + Next, you can use ``has_equal_value()`` to check whether the objects in the student and solution process correspond. + + Args: + index (str): the name of the object which value has to be checked. + missing_msg (str): feedback message when the object is not defined in the student process. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + + :Example: + + Suppose you want the student to create a variable ``x``, equal to 15: :: + + x = 15 + + The following SCT will verify this: :: + + Ex().check_object("x").has_equal_value() + + - ``check_object()`` will check if the variable ``x`` is defined in the student process. + - ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process. + + Note that ``has_equal_value()`` only looks at **end result** of a variable in the student process. + In the example, how the object ``x`` came about in the student's submission, does not matter. + This means that all of the following submission will also pass the above SCT: :: + + x = 15 + x = 12 + 3 + x = 3; x += 12 + + :Example: + + As the previous example mentioned, ``has_equal_value()`` only looks at the **end result**. If your exercise is + first initializing and object and further down the script is updating the object, you can only look at the final value! + + Suppose you want the student to initialize and populate a list `my_list` as follows: :: + + my_list = [] + for i in range(20): + if i % 3 == 0: + my_list.append(i) + + There is no robust way to verify whether `my_list = [0]` was coded correctly in a separate way. + The best SCT would look something like this: :: + + msg = "Have you correctly initialized `my_list`?" + Ex().check_correct( + check_object('my_list').has_equal_value(), + multi( + # check initialization: [] or list() + check_or( + has_equal_ast(code = "[]", incorrect_msg = msg), + check_function('list') + ), + check_for_loop().multi( + check_iter().has_equal_value(), + check_body().check_if_else().multi( + check_test().multi( + set_context(2).has_equal_value(), + set_context(3).has_equal_value() + ), + check_body().set_context(3).\\ + set_env(my_list = [0]).\\ + has_equal_value(name = 'my_list') + ) + ) + ) + ) + + - ``check_correct()`` is used to robustly check whether ``my_list`` was built correctly. + - If ``my_list`` is not correct, **both** the initialization and the population code are checked. + + :Example: + + Because checking object correctness incorrectly is such a common misconception, we're adding another example: :: + + import pandas as pd + df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}) + df['c'] = [7, 8, 9] + + The following SCT would be **wrong**, as it does not factor in the possibility that the 'add column ``c``' step could've been wrong: :: + + Ex().check_correct( + check_object('df').has_equal_value(), + check_function('pandas.DataFrame').check_args(0).has_equal_value() + ) + + The following SCT would be better, as it is specific to the steps: :: + + # verify the df = pd.DataFrame(...) step + Ex().check_correct( + check_df('df').multi( + check_keys('a').has_equal_value(), + check_keys('b').has_equal_value() + ), + check_function('pandas.DataFrame').check_args(0).has_equal_value() + ) + + # verify the df['c'] = [...] step + Ex().check_df('df').check_keys('c').has_equal_value() + + :Example: + + pythonwhat compares the objects in the student and solution process with the ``==`` operator. + For basic objects, this ``==`` is operator is properly implemented, so that the objects can be effectively compared. + For more complex objects that are produced by third-party packages, however, it's possible that this equality operator is not implemented in a way you'd expect. + Often, for these object types the ``==`` will compare the actual object instances: :: + + # pre exercise code + class Number(): + def __init__(self, n): + self.n = n + + # solution + x = Number(1) + + # sct that won't work + Ex().check_object().has_equal_value() + + # sct + Ex().check_object().has_equal_value(expr_code = 'x.n') + + # submissions that will pass this sct + x = Number(1) + x = Number(2 - 1) + + The basic SCT like in the previous example will notwork here. + Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking. + Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n`` + and seeing if the result of running this expression in both student and solution process match. + + """ + + # Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1' + if v2_only(): + extra_msg = "If you want to check the value of an object in e.g. a for loop, use `has_equal_value(name = 'my_obj')` instead." + state.assert_execution_root("check_object", extra_msg=extra_msg) + + if missing_msg is None: + missing_msg = "Did you define the {{typestr}} `{{index}}` without errors?" + + if expand_msg is None: + expand_msg = "Did you correctly define the {{typestr}} `{{index}}`? " + + if ( + not isDefinedInProcess(index, state.solution_process) + and state.has_different_processes() + ): + raise InstructorError.from_message( + "`check_object()` couldn't find object `%s` in the solution process." + % index + ) + + append_message = FeedbackComponent(expand_msg, {"index": index, "typestr": typestr}) + + # create child state, using either parser output, or create part from name + fallback = lambda: ObjectAssignmentParser.get_part(index) + stu_part = state.ast_dispatcher.find("object_assignments", state.student_ast).get( + index, fallback() + ) + sol_part = state.ast_dispatcher.find("object_assignments", state.solution_ast).get( + index, fallback() + ) + + # test object exists + state.do_test( + DefinedProcessTest( + index, + state.student_process, + FeedbackComponent(missing_msg, append_message.kwargs), + ) + ) + + child = part_to_child( + stu_part, sol_part, append_message, state, node_name="object_assignments" + ) + + return child + + +def is_instance(state, inst, not_instance_msg=None): + """Check whether an object is an instance of a certain class. + + ``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is + used to 'zoom in' on the object of interest. + + Args: + inst (class): The class that the object should have. + not_instance_msg (str): When specified, this overrides the automatically generated message in case + the object does not have the expected class. + state (State): The state that is passed in through the SCT chain (don't specify this). + + :Example: + + Student code and solution code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + + SCT:: + + # Verify the class of arr + import numpy + Ex().check_object('arr').is_instance(numpy.ndarray) + """ + + state.assert_is(["object_assignments"], "is_instance", ["check_object"]) + + sol_name = state.solution_parts.get("name") + stu_name = state.student_parts.get("name") + + if not_instance_msg is None: + not_instance_msg = "Is it a {{inst.__name__}}?" + + if not isInstanceInProcess(sol_name, inst, state.solution_process): + raise InstructorError.from_message( + "`is_instance()` noticed that `%s` is not a `%s` in the solution process." + % (sol_name, inst.__name__) + ) + + feedback = FeedbackComponent(not_instance_msg, {"inst": inst}) + state.do_test(InstanceProcessTest(stu_name, inst, state.student_process, feedback)) + + return state + + +def check_df(state, index, missing_msg=None, not_instance_msg=None, expand_msg=None): + """Check whether a DataFrame was defined and it is the right type + + ``check_df()`` is a combo of ``check_object()`` and ``is_instance()`` that checks whether the specified object exists + and whether the specified object is pandas DataFrame. + + You can continue checking the data frame with ``check_keys()`` function to 'zoom in' on a particular column in the pandas DataFrame: + + Args: + index (str): Name of the data frame to zoom in on. + missing_msg (str): See ``check_object()``. + not_instance_msg (str): See ``is_instance()``. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + + :Example: + + Suppose you want the student to create a DataFrame ``my_df`` with two columns. + The column ``a`` should contain the numbers 1 to 3, + while the contents of column ``b`` can be anything: :: + + import pandas as pd + my_df = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "n", "y"]}) + + The following SCT would robustly check that: :: + + Ex().check_df("my_df").multi( + check_keys("a").has_equal_value(), + check_keys("b") + ) + + - ``check_df()`` checks if ``my_df`` exists (``check_object()`` behind the scenes) and is a DataFrame (``is_instance()``) + - ``check_keys("a")`` zooms in on the column ``a`` of the data frame, and ``has_equal_value()`` checks if the columns correspond between student and solution process. + - ``check_keys("b")`` zooms in on hte column ``b`` of the data frame, but there's no 'equality checking' happening + + The following submissions would pass the SCT above: :: + + my_df = pd.DataFrame({"a": [1, 1 + 1, 3], "b": ["a", "l", "l"]}) + my_df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + + """ + import pandas as pd + child = check_object( + state, + index, + missing_msg=missing_msg, + expand_msg=expand_msg, + typestr="pandas DataFrame", + ) + is_instance(child, pd.DataFrame, not_instance_msg=not_instance_msg) + return child + + +def check_keys(state, key, missing_msg=None, expand_msg=None): + """Check whether an object (dict, DataFrame, etc) has a key. + + ``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is + used to 'zoom in' on the object of interest. + + Args: + key (str): Name of the key that the object should have. + missing_msg (str): When specified, this overrides the automatically generated + message in case the key does not exist. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + state (State): The state that is passed in through the SCT chain (don't specify this). + + :Example: + + Student code and solution code:: + + x = {'a': 2} + + SCT:: + + # Verify that x contains a key a + Ex().check_object('x').check_keys('a') + + # Verify that x contains a key a and a is correct. + Ex().check_object('x').check_keys('a').has_equal_value() + + """ + + state.assert_is(["object_assignments"], "is_instance", ["check_object", "check_df"]) + + if missing_msg is None: + missing_msg = "There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`." + if expand_msg is None: + expand_msg = "Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? " + + sol_name = state.solution_parts.get("name") + stu_name = state.student_parts.get("name") + + if not isDefinedCollInProcess(sol_name, key, state.solution_process): + raise InstructorError.from_message( + "`check_keys()` couldn't find key `%s` in object `%s` in the solution process." + % (key, sol_name) + ) + + # check if key available + state.do_test( + DefinedCollProcessTest( + stu_name, key, state.student_process, FeedbackComponent(missing_msg, {"key": key}) + ) + ) + + def get_part(name, key, highlight): + if isinstance(key, str): + slice_val = ast.Constant(value=key) + else: + slice_val = ast.parse(str(key)).body[0].value + expr = ast.Subscript( + value=ast.Name(id=name, ctx=ast.Load()), + slice=ast.Index(value=slice_val), + ctx=ast.Load(), + ) + ast.fix_missing_locations(expr) + return {"node": expr, "highlight": highlight} + + stu_part = get_part(stu_name, key, state.student_parts.get("highlight")) + sol_part = get_part(sol_name, key, state.solution_parts.get("highlight")) + append_message = FeedbackComponent(expand_msg, {"key": key}) + child = part_to_child(stu_part, sol_part, append_message, state) + return child diff --git a/pythonwhat/checks/check_wrappers.py b/pythonwhat/checks/check_wrappers.py new file mode 100644 index 00000000..f10067a9 --- /dev/null +++ b/pythonwhat/checks/check_wrappers.py @@ -0,0 +1,799 @@ +from protowhat.failure import _debug +from protowhat.checks.check_simple import allow_errors +from protowhat.checks.check_bash_history import has_command +from protowhat.checks.check_files import check_file, has_dir +from pythonwhat.checks.check_funcs import check_part, check_part_index, check_node +from pythonwhat.checks.has_funcs import has_equal_part +from pythonwhat.checks.check_function import check_function +from pythonwhat.checks.check_has_context import has_context +from pythonwhat.checks import check_object, check_logic, check_funcs, has_funcs +from pythonwhat.local import run + +from inspect import signature, Parameter +from functools import partial, wraps +from jinja2 import Template + +__PART_WRAPPERS__ = { + "iter": "iterable part", + "body": "body", + "key": "key part", + "value": "value part", + "orelse": "else part", + "finalbody": "finally part", + "test": "condition", +} + +__PART_INDEX_WRAPPERS__ = { + "ifs": "{{ordinal}} if", + "bases": "{{ordinal}} base class", + "handlers": "`{{index}}` `except` block", + "context": "{{ordinal}} context", +} + +__NODE_WRAPPERS__ = { + "list_comp": { + "typestr": "{{ordinal}} list comprehension", + "docstr": """Check whether a list comprehension was coded and zoom in on it. + + Can be chained with ``check_iter()``, ``check_body()``, and ``check_ifs()``. + + Args: + index: Index of the list comprehension (0-based) + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you expect students to create a list ``my_list`` as follows: :: + + my_list = [ i*2 for i in range(0,10) if i>2 ] + + The following SCT would robustly verify this: :: + + Ex().check_correct( + check_object('my_list').has_equal_value(), + check_list_comp().multi( + check_iter().has_equal_value(), + check_body().set_context(4).has_equal_value(), + check_ifs(0).multi( + set_context(0).has_equal_value(), + set_context(3).has_equal_value(), + set_context(5).has_equal_value() + ) + ) + ) + + - With ``check_correct()``, we're making sure that the list comprehension + checking is not executed if ``my_list`` was calculated properly. + - If ``my_list`` is not correct, the 'diagnose' chain will run: ``check_list_comp()`` looks + for the first list comprehension in the student's submission. + - Next, ``check_iter()`` zooms in on the iterator, ``range(0, 10)`` in the case of the solution. + ``has_equal_value()`` verifies whether the expression that the student used evaluates to the + same value as the expression that the solution used. + - ``check_body()`` zooms in on the body, ``i*2`` in the case of the solution. + ``set_context()`` sets the iterator to 4, allowing for the fact that the student used another name instead of ``i`` for this iterator. + ``has_equal_value()`` reruns the body in the student and solution code with the iterator set to 4, and checks if the results are the same. + - ``check_ifs(0)`` zooms in on the first ``if`` of the list comprehension, ``i>2`` in case of the solution. + With a series of ``set_context()`` and ``has_equal_value()``, it is verifies whether this condition evaluates to the same value in student + and solution code for different values of the iterator (`i` in the case of the solution, whatever in the case of the student). + + """, + }, + "generator_exp": { + "typestr": "{{ordinal}} generator expression", + "docstr": """Check whether a generator expression was coded and zoom in on it. + + Can be chained with ``check_iter()``, ``check_body()``, and ``check_ifs()``. + + Args: + index: Index of the generator expression (0-based) + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you expect students to create a generator ``my_gen`` as follows: :: + + my_gen = ( i*2 for i in range(0,10) ) + + The following SCT would robustly verify this: :: + + Ex().check_correct( + check_object('my_gen').has_equal_value(), + check_generator_exp().multi( + check_iter().has_equal_value(), + check_body().set_context(4).has_equal_value() + ) + ) + + Have a look at ``check_list_comp`` to understand what's going on; it is very similar. + + """, + }, + "dict_comp": { + "typestr": "{{ordinal}} dictionary comprehension", + "docstr": """Check whether a dictionary comprehension was coded and zoom in on it. + + Can be chained with ``check_key()``, ``check_value()``, and ``check_ifs()``. + + Args: + index: Index of the dictionary comprehension (0-based) + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you expect students to create a dictionary ``my_dict`` as follows: :: + + my_dict = { m:len(m) for m in ['a', 'ab', 'abc'] } + + The following SCT would robustly verify this: :: + + Ex().check_correct( + check_object('my_dict').has_equal_value(), + check_dict_comp().multi( + check_iter().has_equal_value(), + check_key().set_context('ab').has_equal_value(), + check_value().set_context('ab').has_equal_value() + ) + ) + + - With ``check_correct()``, we're making sure that the dictionary comprehension + checking is not executed if ``my_dict`` was created properly. + - If ``my_dict`` is not correct, the 'diagnose' chain will run: ``check_dict_comp()`` looks + for the first dictionary comprehension in the student's submission. + - Next, ``check_iter()`` zooms in on the iterator, ``['a', 'ab', 'abc']`` in the case of the solution. + ``has_equal_value()`` verifies whether the expression that the student used evaluates to the + same value as the expression that the solution used. + - ``check_key()`` zooms in on the key of the comprehension, ``m`` in the case of the solution. + ``set_context()`` temporaritly sets the iterator to ``'ab'``, allowing for the fact that the student used another name instead of ``m`` for this iterator. + ``has_equal_value()`` reruns the key expression in the student and solution code with the iterator set to ``'ab'``, and checks if the results are the same. + - ``check_value()`` zooms in on the value of the comprehension, ``len(m)`` in the case of the solution. + ``has_equal_value()`` reruns the value expression in the student and solution code with the iterator set to ``'ab'``, and checks if the results are the same. + + """, + }, + "for_loop": { + "typestr": "{{ordinal}} for loop", + "docstr": """Check whether a for loop was coded and zoom in on it. + + Can be chained with ``check_iter()`` and ``check_body()``. + + Args: + index: Index of the for loop (0-based). + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want a student to iterate over a predefined dictionary ``my_dict`` and do the appropriate printouts: :: + + for key, value in my_dict.items(): + print(key + " - " + str(value)) + + The following SCT would verify this: :: + + Ex().check_for_loop().multi( + check_iter().has_equal_value(), + check_body().multi( + set_context('a', 1).has_equal_output(), + set_context('b', 2).has_equal_output() + ) + ) + + - ``check_for_loop()`` zooms in on the ``for`` loop, and makes its parts available for further checking. + - ``check_iter()`` zooms in on the iterator part of the for loop, ``my_dict.items()`` in the solution. + ``has_equal_value()`` re-executes the expressions specified by student and solution and compares their results. + - ``check_body()`` zooms in on the body part of the for loop, ``print(key + " - " + str(value))``. + For different values of ``key`` and ``value``, the student's body and solution's body are executed again and the printouts are captured and compared to see if they are equal. + + Notice how you do not need to specify the variables by name in ``set_context()``. pythonwhat can figure out the variable names used in both student and solution code, and + can do the verification independent of that. That way, we can make the SCT robust against submissions that code the correct logic, but use different names for the context values. + In other words, the following student submissions that would also pass the SCT: :: + + # passing submission 1 + my_dict = {'a': 1, 'b': 2} + for k, v in my_dict.items(): + print(k + " - " + str(v)) + + # passing submission 2 + my_dict = {'a': 1, 'b': 2} + for first, second in my_dict.items(): + mess = first + " - " + str(second) + print(mess) + + :Example: + + As another example, suppose you want the student to build a list of doubles as follows: :: + + even = [] + for i in range(10): + even.append(2*i) + + The following SCT would robustly verify this: :: + + Ex().check_correct( + check_object('even').has_equal_value(), + check_for_loop().multi( + check_iter().has_equal_value(), + check_body().set_context(2).set_env(even = []).\\ + has_equal_value(name = 'even') + ) + ) + + - ``check_correct()`` makes sure that we do not dive into the ``for`` loop if the array ``even`` is correctly populated in the end. + - If ``even`` was not correctly populated, ``check_for_loop()`` will zoom in on the for loop. + - The ``check_iter()`` chain verifies whether `range(10)` (or something equivalent) was used to iterate over. + - ``check_body()`` zooms in on the body, and reruns the body (``even.append(2*i)`` in the solution) for ``i`` equal to 2, and even temporarily set to an empty array. + Notice how we use ``set_context()`` to robustly set the context value (the student can use a different variable name), while we have to explicitly set ``even`` with ``set_env()``. + Also notice how we use ``has_equal_value(name = 'even')`` instead of the usual ``check_object()``; ``check_object()`` can only be called from the root state ``Ex()``. + + :Example: + + As a follow-up example, suppose you want the student to build a list of doubles of the even numbers only: :: + + even = [] + for i in range(10): + if i % 2 == 0: + even.append(2*i) + + The following SCT would robustly verify this: :: + + Ex().check_correct( + check_object('even').has_equal_value(), + check_for_loop().multi( + check_iter().has_equal_value(), + check_body().check_if_else().multi( + check_test().multi( + set_context(1).has_equal_value(), + set_context(2).has_equal_value() + ), + check_body().set_context(2).\\ + set_env(even = []).has_equal_value(name = 'even') + ) + ) + ) + + """, + }, + "function_def": { + "typestr": "definition of `{{index}}()`", + "docstr": """Check whether a function was defined and zoom in on it. + + Can be chained with ``check_call()``, ``check_args()`` and ``check_body()``. + + Args: + index: the name of the function definition. + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want a student to create a function ``shout_echo()``: :: + + def shout_echo(word1, echo=1): + echo_word = word1 * echo + shout_words = echo_word + '!!!' + return shout_words + + The following SCT robustly checks this: :: + + Ex().check_function_def('shout_echo').check_correct( + multi( + check_call("f('hey', 3)").has_equal_value(), + check_call("f('hi', 2)").has_equal_value(), + check_call("f('hi')").has_equal_value() + ), + check_body().set_context('test', 1).multi( + has_equal_value(name = 'echo_word'), + has_equal_value(name = 'shout_words') + ) + ) + + Here: + + - ``check_function_def()`` zooms in on the function definition of ``shout_echo`` in both student and solution code (and process). + - ``check_correct()`` is used to + + + First check whether the function gives the correct result when called in different ways (through ``check_call()``). + + Only if these 'function unit tests' don't pass, ``check_correct()`` will run the `check_body()` chain that dives deeper into the + function definition body. This chain sets the context variables - ``word1`` and ``echo``, the arguments of the function - to + the values ``'test'`` and ``1`` respectively, again while being agnostic to the actual name of these context variables. + + Notice how ``check_correct()`` is used to great effect here: why check the function definition internals if the I/O of the function works fine? + Because of this construct, all the following submissions will pass the SCT: :: + + # passing submission 1 + def shout_echo(w, e=1): + ew = w * e + return ew + '!!!' + + # passing submission 2 + def shout_echo(a, b=1): + return a * b + '!!!' + + :Example: + + ``check_args()`` is most commonly used in combination with ``check_function()`` + to verify the arguments of function **calls**, but it can also be used + to verify the arguments specified in the signature of a function definition. + + We can extend the SCT for the previous example to explicitly verify the signature: :: + + + msg1 = "Make sure to specify 2 arguments!" + msg2 = "don't specify default arg!" + msg3 = "specify a default arg!" + Ex().check_function_def('shout_echo').check_correct( + multi( + check_call("f('hey', 3)").has_equal_value(), + check_call("f('hi', 2)").has_equal_value(), + check_call("f('hi')").has_equal_value() + ), + multi( + has_equal_part_len("args", unequal_msg=1), + check_args(0).has_equal_part('is_default', msg=msg2), + check_args('word1').has_equal_part('is_default', msg=msg2), + check_args(1).\\ + has_equal_part('is_default', msg=msg3).has_equal_value(), + check_args('echo').\\ + has_equal_part('is_default', msg=msg3).has_equal_value(), + check_body().set_context('test', 1).multi( + has_equal_value(name = 'echo_word'), + has_equal_value(name = 'shout_words') + ) + ) + ) + + - ``has_equal_part_len("args")`` verifies whether student and solution function + definition have the same number of arguments. + - ``check_args(0)`` refers to the first argument in the signature by position, + and the chain checks whether the student did not specify a default as in the solution. + - An alternative for the ``check_args(0)`` chain is to use ``check_args('word1')`` + to refer to the first argument. This is more restrictive, as the requires the + student to use the exact same name. + - ``check_args(1)`` refers to the second argument in the signature by position, + and the chain checks whether the student specified a default, as in the solution, and + whether the value of this default corresponds to the one in the solution. + - The ``check_args('echo')`` chain is a more restrictive alternative for the ``check_args(1)`` + chain. + + Notice that support for verifying arguments is not great yet: + + - A lot of work is needed to verify the number of arguments and whether or not defaults are set. + - You have to specify custom messages because pythonwhat doesn't automatically generate messages. + + We are working on it! + + """, + }, + "class_def": { + "typestr": "class definition of `{{index}}`", + "docstr": """Check whether a class was defined and zoom in on its definition + + Can be chained with ``check_bases()`` and ``check_body()``. + + Args: + index: the name of the function definition. + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want to check whether a class was defined correctly: :: + + class MyInt(int): + def __init__(self, i): + super().__init__(i + 1) + + The following SCT would verify this: :: + + Ex().check_class_def('MyInt').multi( + check_bases(0).has_equal_ast(), + check_body().check_function_def('__init__').multi( + check_args('self'), + check_args('i'), + check_body().set_context(i = 2).multi( + check_function('super', signature=False), + check_function('super.__init__').check_args(0).has_equal_value() + ) + ) + ) + + - ``check_class_def()`` looks for the class definition itself. + - With ``check_bases()``, you can zoom in on the different basse classes that the class definition inherits from. + - With ``check_body()``, you zoom in on the class body, after which you can use other functions such + as ``check_function_def()`` to look for class methods. + - Of course, just like for other examples, you can use ``check_correct()`` where necessary, + e.g. to verify whether class methods give the right behavior with ``check_call()`` + before diving into the body of the method itself. + + """, + }, + "if_exp": { + "typestr": "{{ordinal}} if expression", + "docstr": """Check whether an if expression was coded zoom in on it. + + This function works the exact same way as ``check_if_else()``. + """, + }, + "if_else": { + "typestr": "{{ordinal}} if statement", + "docstr": """Check whether an if statement was coded zoom in on it. + + Args: + index: the index of the if statement to look for (0 based) + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want students to print out a message if ``x`` is larger than 0: :: + + x = 4 + if x > 0: + print("x is strictly positive") + + The following SCT would verify that: :: + + Ex().check_if_else().multi( + check_test().multi( + set_env(x = -1).has_equal_value(), + set_env(x = 1).has_equal_value(), + set_env(x = 0).has_equal_value() + ), + check_body().check_function('print', 0).\\ + check_args('value').has_equal_value() + ) + + - ``check_if_else()`` zooms in on the first if statement in the student and solution submission. + - ``check_test()`` zooms in on the 'test' portion of the if statement, ``x > 0`` in case of the solution. + ``has_equal_value()`` reruns this expression and the corresponding expression in the student code for + different values of ``x`` (set with ``set_env()``) and compare there results. + This way, you can robustly verify whether the if test was coded up correctly. If the student + codes up the condition as ``0 < x``, this would also be accepted. + - ``check_body()`` zooms in on the 'body' portion of the if statement, ``print("...")`` in case of the solution. + With a classical ``check_function()`` chain, it is verified whether the if statement contains a + function ``print()`` and whether its argument is set correctly. + + :Example: + + In Python, when an if-else statement has an ``elif`` clause, it is held in the `orelse` part. + In this sense, an if-elif-else statement is represented by python as nested if-elses. + More specifically, this if-else statement: :: + + if x > 0: + print(x) + elif y > 0: + print(y) + else: + print('none') + + Is syntactically equivalent to: :: + + if x > 0: + print(x) + else: + if y > 0: + print(y) + else: + print('none') + + The second representation has to be followed when writing the corresponding SCT: :: + + Ex().check_if_else().multi( + check_test(), # zoom in on x > 0 + check_body(), # zoom in on print(x) + check_orelse().check_if_else().multi( + check_test(), # zoom in on y > 0 + check_body(), # zoom in on print(y) + check_orelse() # zoom in on print('none') + ) + ) + + """, + }, + "lambda_function": { + "typestr": "{{ordinal}} lambda function", + "docstr": """Check whether a lambda function was coded zoom in on it. + + Can be chained with ``check_call()``, ``check_args()`` and ``check_body()``. + + Args: + index: the index of the lambda function (0-based). + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want a student to create a lambda function + that returns the length of an array times two: :: + + lambda x: len(x)*2 + + The following SCT robustly checks this: :: + + Ex().check_lambda_function().check_correct( + multi( + check_call("f([1])").has_equal_value(), + check_call("f([1, 2])").has_equal_value() + ), + check_body().set_context([1, 2, 3]).has_equal_value() + ) + + Here: + + - ``check_lambda_function()`` zooms in on the first lambda function in both student and solution code. + - ``check_correct()`` is used to + + + First check whether the lambda function gives the correct result when called in different ways (through ``check_call()``). + + Only if these 'function unit tests' don't pass, ``check_correct()`` will run the `check_body()` chain that dives deeper into the + lambda function's body. This chain sets the context variable `x`, the argument of the function, to + the values ``[1, 2, 3]``, while being agnostic to the actual name the student used for this context variable. + + Notice how ``check_correct()`` is used to great effect here: why check the function definition internals if the I/O of the function works fine? + Because of this construct, all the following submissions will pass the SCT: :: + + # passing submission 1 + lambda x: len(x) + len(x) + + # passing submission 2 + lambda y, times=2: len(y) * times + """, + }, + "try_except": { + "typestr": "{{ordinal}} try statement", + "docstr": """Check whether a try except statement was coded zoom in on it. + + Can be chained with ``check_body()``, ``check_handlers()``, ``check_orelse()`` and ``check_finalbody()``. + + Args: + index: the index of the try except statement (0-based). + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want to verify whether the student did a `try-except` statement properly: :: + + do_dangerous_thing = lambda n: n + + try: + x = do_dangerous_thing(n = 4) + except ValueError as e: + x = 'something wrong with inputs' + except: + x = 'something went wrong' + finally: + print('ciao!') + + The following SCT can be used to verify this: :: + + Ex().check_try_except().multi( + check_body().\\ + check_function('do_dangerous_thing').\\ + check_args('n').has_equal_value(), + check_handlers('ValueError').\\ + has_equal_value(name = 'x'), + check_handlers('all').\\ + has_equal_value(name = 'x'), + check_finalbody().\\ + check_function('print').check_args(0).has_equal_value() + ) + + """, + }, + "while": { + "typestr": "{{ordinal}} `while` loop", + "docstr": """Check whether a while loop was coded and zoom in on it. + + Can be chained with ``check_test()``, ``check_body()`` and ``check_orelse()``. + + Args: + index: the index of the while loop to verify (0-based). + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + :Example: + + Suppose you want a student to code a while loop that counts down a counter from 50 until + a multilpe of 11 is found. If it is found, the value should be printed out. :: + + i = 50 + while i % 11 != 0: + i -= 1 + + The following SCT robustly verifies this: :: + + Ex().check_correct( + check_object('i').has_equal_value(), + check_while().multi( + check_test().multi( + set_env(i = 45).has_equal_value(), + set_env(i = 44).has_equal_value() + ), + check_body().set_env(i = 3).has_equal_value(name = 'i') + ) + ) + + - ``check_correct()`` first checks whether the end result of ``i`` is correct. If it is, the entire chain that checks the ``while`` loop is skipped. + - If ``i`` is not correctly calculated, ``check_while_loop()`` zooms in on the while loop. + - ``check_test()`` zooms in on the condition of the ``while`` loop, ``i % 11 != 0`` in the solution, and verifies whether + the expression gives the same results for different values of ``i``, set through ``set_env()``, when comparing student and solution. + - ``check_body()`` zooms in on the body of the ``while`` loop, and ``has_equal_value()`` checks whether rerunning this body + updates ``i`` as expected when ``i`` is temporarily set to 3 with ``set_env()``. + + """, + }, + "with": { + "typestr": "{{ordinal}} `with` statement", + "docstr": """Check whether a with statement was coded zoom in on it. + + Args: + index: the index of the``with`` statement to verify (0-based) + {{typestr}} + {{missing_msg}} + {{expand_msg}} + + """, + }, +} + +scts = dict() + +# make has_equal_part wrappers + + +def partial_with_offset(offset=1): + def bound_partial_with_offset(func, *partial_args, **partial_kwargs): + kwargs_partial = partial(func, **partial_kwargs) + + @wraps(func) + def full_partial(*args, **kwargs): + full_args = args[:offset] + partial_args + args[offset:] + return kwargs_partial(*full_args, **kwargs) + + # set correct signature of returned partial + # todo: pass arguments as keywords to partial, instead of this decorator? + # (where args are always the same) + func_sig = signature(full_partial) + parameter_names = tuple(func_sig.parameters) + + partialed_positional_indices = [] + for kwarg in partial_kwargs: + param = func_sig.parameters[kwarg] + if param.default is param.empty: + partialed_positional_indices.append(parameter_names.index(kwarg)) + + partial_params = list(func_sig.parameters.values()) + for index in sorted(partialed_positional_indices, reverse=True): + # appending isn't needed for functionality, but more similar to partial + # and it shows that these arguments can still be updated as kwargs + partial_params.append( + partial_params[index].replace( + kind=Parameter.KEYWORD_ONLY, + default=partial_kwargs[partial_params[index].name], + ) + ) + del partial_params[index] + del partial_params[offset : offset + len(partial_args)] + + full_partial.__signature__ = func_sig.replace(parameters=partial_params) + + return full_partial + + return bound_partial_with_offset + + +state_partial = partial_with_offset() + + +def rename_function(func, name): + # see functools.wraps + func.__name__ = func.__qualname__ = name + + +def add_partial_sct(func, name): + rename_function(func, name) + scts[name] = func + + +add_partial_sct( + state_partial( + has_equal_part, + "name", + msg="Make sure to use the correct {{name}}, was expecting {{sol_part[name]}}, instead got {{stu_part[name]}}.", + ), + "has_equal_name", +) +add_partial_sct( + state_partial( + has_equal_part, + "is_default", + msg="Make sure it {{ 'has' if sol_part.is_default else 'does not have'}} a default argument.", + ), + "is_default", +) + +# include rest of wrappers +for k, v in __PART_WRAPPERS__.items(): + check_fun = state_partial(check_part, k, v) + add_partial_sct(check_fun, "check_" + k) + +for k, v in __PART_INDEX_WRAPPERS__.items(): + check_fun = state_partial(check_part_index, k, part_msg=v) + add_partial_sct(check_fun, "check_" + k) + +for k, v in __NODE_WRAPPERS__.items(): + check_fun = state_partial(check_node, k + "s", typestr=v["typestr"]) + check_fun.__doc__ = Template(v["docstr"]).render( + typestr="typestr: If specified, this overrides the standard way of referring to the construct you're zooming in on.", + missing_msg="missing_msg: If specified, this overrides the automatically generated feedback message in case the construct could not be found.", + expand_msg="expand_msg: If specified, this overrides the automatically generated feedback message that is prepended to feedback messages that are thrown further in the SCT chain.", + ) + add_partial_sct(check_fun, "check_" + k) + +for k in [ + "set_context", + "set_env", + "disable_highlighting", + "check_not", + "check_or", + "check_correct", + "fail", + "override", + "multi", +]: + scts[k] = getattr(check_logic, k) + +for k in ["with_context", "check_args", "check_call"]: + scts[k] = getattr(check_funcs, k) + +for k in [ + "has_equal_value", + "has_equal_output", + "has_equal_error", + "has_equal_ast", + "has_equal_part_len", + "has_equal_part", + "has_import", + "has_output", + "has_printout", + "has_code", + "has_no_error", + "has_chosen", +]: + sct = getattr(has_funcs, k) + if not hasattr(sct, "__name__"): + rename_function(sct, k) + scts[k] = sct + +# include check_object and friends ------ +for k in ["check_object", "is_instance", "check_df", "check_keys"]: + scts[k] = getattr(check_object, k) + +scts["has_context"] = has_context +scts["check_function"] = check_function + +scts["run"] = run + +scts["has_command"] = has_command + +scts["check_file"] = check_file +scts["has_dir"] = has_dir + +scts["allow_errors"] = allow_errors + +scts["_debug"] = _debug + +locals().update(scts) diff --git a/pythonwhat/checks/has_funcs.py b/pythonwhat/checks/has_funcs.py new file mode 100644 index 00000000..870e1069 --- /dev/null +++ b/pythonwhat/checks/has_funcs.py @@ -0,0 +1,852 @@ +from protowhat.utils_messaging import get_ord +from pythonwhat.tasks import ( + getResultInProcess, + getOutputInProcess, + getErrorInProcess, + ReprFail, + isDefinedInProcess, + getOptionFromProcess, + UndefinedValue, +) +from pythonwhat.Test import EqualTest, DefinedCollTest +from protowhat.Feedback import Feedback, FeedbackComponent +from protowhat.failure import InstructorError, debugger +from pythonwhat import utils +from functools import partial +import re +import copy +import ast + +evalCalls = { + "value": getResultInProcess, + "output": getOutputInProcess, + "error": getErrorInProcess, +} + + +def has_part(state, name, msg, fmt_kwargs=None, index=None): + d = { + "sol_part": state.solution_parts, + "stu_part": state.student_parts, + **fmt_kwargs, + } + + def verify(part, index): + if index is not None: + if isinstance(index, list): + for ind in index: + part = part[ind] + else: + part = part[index] + if part is None: + raise KeyError + + # TODO: instructor error if msg is not str + # Check if it's there in the solution + try: + verify(state.solution_parts[name], index) + except (KeyError, IndexError): + with debugger(state): + err_msg = "SCT fails on solution: {}".format(msg) + state.report(err_msg, d) + + try: + verify(state.student_parts[name], index) + except (KeyError, IndexError): + state.report(msg, d) + + return state + + +def has_equal_part(state, name, msg): + d = { + "stu_part": state.student_parts, + "sol_part": state.solution_parts, + "name": name, + } + + state.do_test( + EqualTest(d["stu_part"][name], d["sol_part"][name], FeedbackComponent(msg, d)) + ) + + return state + + +# TODO: shouldn't have to hardcode message +def has_equal_part_len(state, name, unequal_msg): + """Verify that a part that is zoomed in on has equal length. + + Typically used in the context of ``check_function_def()`` + + Arguments: + name (str): name of the part for which to check the length to the corresponding part in the solution. + unequal_msg (str): Message in case the lengths do not match. + state (State): state as passed by the SCT chain. Don't specify this explicitly. + + :Examples: + + Student and solution code:: + + def shout(word): + return word + '!!!' + + SCT that checks number of arguments:: + + Ex().check_function_def('shout').has_equal_part_len('args', 'not enough args!') + """ + d = dict( + stu_len=len(state.student_parts[name]), sol_len=len(state.solution_parts[name]) + ) + + if d["stu_len"] != d["sol_len"]: + state.report(unequal_msg, d) + + return state + + +# Expression tests ----------------------------------------------------------- + + +def has_equal_ast(state, incorrect_msg=None, code=None, exact=True, append=None): + """Test whether abstract syntax trees match between the student and solution code. + + ``has_equal_ast()`` can be used in two ways: + + * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission. + But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST. + Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``. + * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments. + + Args: + incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this. + code: optional code to use instead of the solution AST. + exact: whether the representations must match exactly. If false, the solution AST + only needs to be contained within the student AST (similar to using test student typed). + Defaults to ``True``, unless the ``code`` argument has been specified. + + :Example: + + Student and Solution Code:: + + dict(a = 'value').keys() + + SCT:: + + # all pass + Ex().has_equal_ast() + Ex().has_equal_ast(code = "dict(a = 'value').keys()") + Ex().has_equal_ast(code = "dict(a = 'value')", exact = False) + + Student and Solution Code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + np.mean(arr) + + SCT:: + + # Check underlying value of arugment a of np.mean: + Ex().check_function('numpy.mean').check_args('a').has_equal_ast() + + # Only check AST equality of expression used to specify argument a: + Ex().check_function('numpy.mean').check_args('a').has_equal_ast() + + """ + if utils.v2_only(): + state.assert_is_not(["object_assignments"], "has_equal_ast", ["check_object"]) + state.assert_is_not(["function_calls"], "has_equal_ast", ["check_function"]) + + if code and incorrect_msg is None: + raise InstructorError.from_message( + "If you manually specify the code to match inside has_equal_ast(), " + "you have to explicitly set the `incorrect_msg` argument." + ) + + if ( + append is None + ): # if not specified, set to False if incorrect_msg was manually specified + append = incorrect_msg is None + if incorrect_msg is None: + incorrect_msg = ( + "Expected \n```\n{{sol_str}}\n```\n, but got \n```\n{{stu_str}}\n```\n" + if utils.is_multiline_code(state.student_code, state.solution_code) + else "Expected `{{sol_str}}`, but got `{{stu_str}}`." + ) + + def parse_tree(tree): + # get contents of module.body if only 1 element + crnt = ( + tree.body[0] + if isinstance(tree, ast.Module) and len(tree.body) == 1 + else tree + ) + + # remove Expr if it exists + return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt) + + stu_rep = parse_tree(state.student_ast) + sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code)) + + if utils.is_multiline_code(state.student_code, state.solution_code): + fmt_kwargs = { + "sol_str": utils.format_code(state.solution_code) + if not code + else utils.format_code(code), + "stu_str": utils.format_code(state.student_code), + } + else: + fmt_kwargs = { + "sol_str": state.solution_code if not code else code, + "stu_str": state.student_code, + } + + if exact and not code: + state.do_test( + EqualTest( + stu_rep, + sol_rep, + FeedbackComponent(incorrect_msg, fmt_kwargs, append=append), + ) + ) + elif sol_rep not in stu_rep: + state.report(incorrect_msg, fmt_kwargs, append=append) + + return state + + +DEFAULT_INCORRECT_MSG = "Expected {{test_desc}}`{{sol_eval}}`, but got `{{stu_eval}}`." +DEFAULT_ERROR_MSG = "Running {{'it' if parent['part'] else 'the highlighted expression'}} generated an error: `{{stu_str}}`." +DEFAULT_ERROR_MSG_INV = "Running {{'it' if parent['part'] else 'the highlighted expression'}} didn't generate an error, but it should!" +DEFAULT_UNDEFINED_NAME_MSG = "Running {{'it' if parent['part'] else 'the highlighted expression'}} should define a variable `{{name}}` without errors, but it doesn't." +DEFAULT_INCORRECT_NAME_MSG = ( + "Are you sure you assigned the correct value to `{{name}}`?" +) +DEFAULT_INCORRECT_EXPR_CODE_MSG = ( + "Running the expression `{{expr_code}}` didn't generate the expected result." +) + +args_string = """ + + Args: + incorrect_msg (str): feedback message if the {0} of the expression in the solution + doesn't match the one of the student. This feedback message will be expanded if it is used + in the context of another check function, like ``check_if_else``. + error_msg (str): feedback message if there was an error when running the targeted student code. + Note that when testing for an error, this message is displayed when none is raised. + undefined_msg (str): feedback message if the ``name`` argument is defined, but a variable + with that name doesn't exist after running the targeted student code. + extra_env (dict): set variables to the extra environment. They will update the student and solution environment in + the active state before the student/solution code in the active state is ran. This argument should contain a + dictionary with the keys the names of the variables you want to set, and the values are the values of these variables. + You can also use ``set_env()`` for this. + context_vals (list): set variables which are bound in a ``for`` loop to certain values. + This argument is only useful when checking a for loop (or list comprehensions). + It contains a list with the values of the bound variables. + You can also use ``set_context()`` for this. + pre_code (str): the code in string form that should be executed before the expression is executed. + This is the ideal place to set a random seed, for example. + expr_code (str): If this argument is set, the expression in the student/solution code will not + be ran. Instead, the given piece of code will be ran in the student as well as the solution environment + and the result will be compared. However if the string contains one or more placeholders ``__focus__``, + they will be substituted by the currently focused code. + name (str): If this is specified, the {0} of running this expression after running the focused expression + is returned, instead of the {0} of the focused expression in itself. This is typically used to inspect the + {0} of an object after executing the body of e.g. a ``for`` loop. + copy (bool): whether to try to deep copy objects in the environment, such as lists, that could + accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues. + func (function): custom binary function of form f(stu_result, sol_result), for equality testing. + override: If specified, this avoids the execution of the targeted code in the solution process. Instead, it + will compare the {0} of the expression in the student process with the value specified in ``override``. + Typically used in a ``SingleProcessExercise`` or if you want to allow for different solutions other than + the one coded up in the solution. + """ + + +def has_expr( + state, + incorrect_msg=None, + error_msg=None, + undefined_msg=None, + append=None, + extra_env=None, + context_vals=None, + pre_code=None, + expr_code=None, + name=None, + copy=True, + func=None, + override=None, + test=None, # todo: default or arg before state +): + + if ( + append is None + ): # if not specified, set to False if incorrect_msg was manually specified + append = incorrect_msg is None + if incorrect_msg is None: + if name: + incorrect_msg = DEFAULT_INCORRECT_NAME_MSG + elif expr_code: + incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG + else: + incorrect_msg = DEFAULT_INCORRECT_MSG + if undefined_msg is None: + undefined_msg = DEFAULT_UNDEFINED_NAME_MSG + if error_msg is None: + if test == "error": + error_msg = DEFAULT_ERROR_MSG_INV + else: + error_msg = DEFAULT_ERROR_MSG + + if state.solution_code is not None and isinstance(expr_code, str): + expr_code = expr_code.replace("__focus__", state.solution_code) + + get_func = partial( + evalCalls[test], + extra_env=extra_env, + context_vals=context_vals, + pre_code=pre_code, + expr_code=expr_code, + name=name, + copy=copy, + ) + + if override is not None: + # don't bother with running expression and fetching output/value + # eval_sol, str_sol = eval + eval_sol, str_sol = override, str(override) + else: + eval_sol, str_sol = get_func( + tree=state.solution_ast, + process=state.solution_process, + context=state.solution_context, + env=state.solution_env, + ) + + if (test == "error") ^ isinstance(eval_sol, Exception): + raise InstructorError.from_message( + "Evaluating expression raised error in solution process (or didn't raise if testing for one). " + "Error: {} - {}".format(type(eval_sol), str_sol) + ) + if isinstance(eval_sol, ReprFail): + raise InstructorError.from_message( + "Couldn't extract the value for the highlighted expression from the solution process: " + + eval_sol.info + ) + + eval_stu, str_stu = get_func( + tree=state.student_ast, + process=state.student_process, + context=state.student_context, + env=state.student_env, + ) + + # kwargs --- + fmt_kwargs = { + "stu_part": state.student_parts, + "sol_part": state.solution_parts, + "name": name, + "test": test, + "test_desc": "" if test == "value" else "the %s " % test, + "expr_code": expr_code, + } + + fmt_kwargs["stu_eval"] = str(eval_stu) + fmt_kwargs["sol_eval"] = str(eval_sol) + + # wrap in quotes if eval_sol or eval_stu are strings + if test == "value": + if isinstance(eval_stu, str): + fmt_kwargs["stu_eval"] = "'{}'".format(fmt_kwargs["stu_eval"]) + if isinstance(eval_sol, str): + fmt_kwargs["sol_eval"] = "'{}'".format(fmt_kwargs["sol_eval"]) + + # reformat student evaluation string if it is too long + fmt_kwargs["stu_eval"] = utils.shorten_string(fmt_kwargs["stu_eval"]) + + # check if student or solution evaluations are too long or contain newlines + if incorrect_msg == DEFAULT_INCORRECT_MSG and ( + len(fmt_kwargs["sol_eval"]) > 50 + or utils.has_newline(fmt_kwargs["stu_eval"]) + or utils.has_newline(fmt_kwargs["sol_eval"]) + or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"] + ): + fmt_kwargs["stu_eval"] = None + fmt_kwargs["sol_eval"] = None + incorrect_msg = "Expected something different." + + # tests --- + # error in process + if (test == "error") ^ isinstance(eval_stu, Exception): + fmt_kwargs["stu_str"] = str_stu + state.report(error_msg, fmt_kwargs, append=append) + + # name is undefined after running expression + if isinstance(eval_stu, UndefinedValue): + state.report(undefined_msg, fmt_kwargs, append=append) + + # test equality of results + state.do_test( + EqualTest( + eval_stu, + eval_sol, + FeedbackComponent(incorrect_msg, fmt_kwargs, append=append), + func, + ) + ) + + return state + + +has_equal_value = partial(has_expr, test="value") +has_equal_value.__name__ = "has_equal_value" +has_equal_value.__doc__ = ( + """Run targeted student and solution code, and compare returned value. + + When called on an SCT chain, ``has_equal_value()`` will execute the student and solution + code that is 'zoomed in on' and compare the returned values. + """ + + args_string.format("returned value", "value") + + """ + :Example: + + Student code and solution code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + np.mean(arr) + + SCT:: + + # Verify equality of arr: + Ex().check_object('arr').has_equal_value() + + # Verify whether arr was correctly set in np.mean + Ex().check_function('numpy.mean').check_args('a').has_equal_value() + + # Verify whether np.mean(arr) produced the same result + Ex().check_function('numpy.mean').has_equal_value() + + """ +) + + +has_equal_output = partial(has_expr, test="output") +has_equal_output.__name__ = "has_equal_output" +has_equal_output.__doc__ = """Run targeted student and solution code, and compare output. + + When called on an SCT chain, ``has_equal_output()`` will execute the student and solution + code that is 'zoomed in on' and compare the output. + """ + args_string.format( + "output" +) + +has_equal_error = partial(has_expr, test="error") +has_equal_error.__name__ = "has_equal_error" +has_equal_error.__doc__ = """Run targeted student and solution code, and compare generated errors. + + When called on an SCT chain, ``has_equal_error()`` will execute the student and solution + code that is 'zoomed in on' and compare the errors that they generate. + """ + args_string.format( + "error" +) + +## Various has tests ---------------------------------------------------------- + +from pythonwhat.Test import StringContainsTest + + +def has_code(state, text, pattern=True, not_typed_msg=None): + """Test the student code. + + Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``, + as it is more robust to small syntactical differences that don't change the code's behavior. + + Args: + text (str): the text that is searched for + pattern (bool): if True (the default), the text is treated as a pattern. If False, it is treated as plain text. + not_typed_msg (str): feedback message to be displayed if the student did not type the text. + + :Example: + + Student code and solution code:: + + y = 1 + 2 + 3 + + SCT:: + + # Verify that student code contains pattern (not robust!!): + Ex().has_code(r"1\\s*\\+2\\s*\\+3") + + """ + if not not_typed_msg: + if pattern: + not_typed_msg = "Could not find the correct pattern in your code." + else: + not_typed_msg = "Could not find the following text in your code: %r" % text + + student_code = state.student_code + + state.do_test(StringContainsTest(student_code, text, pattern, not_typed_msg)) + + return state + + +def has_import( + state, + name, + same_as=False, + not_imported_msg="Did you import `{{pkg}}`?", + incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?", +): + """Checks whether student imported a package or function correctly. + + Python features many ways to import packages. + All of these different methods revolve around the ``import``, ``from`` and ``as`` keywords. + ``has_import()`` provides a robust way to check whether a student correctly imported a certain package. + + By default, ``has_import()`` allows for different ways of aliasing the imported package or function. + If you want to make sure the correct alias was used to refer to the package or function that was imported, + set ``same_as=True``. + + Args: + name (str): the name of the package that has to be checked. + same_as (bool): if True, the alias of the package or function has to be the same. Defaults to False. + not_imported_msg (str): feedback message when the package is not imported. + incorrect_as_msg (str): feedback message if the alias is wrong. + + :Example: + + Example 1, where aliases don't matter (defaut): :: + + # solution + import matplotlib.pyplot as plt + + # sct + Ex().has_import("matplotlib.pyplot") + + # passing submissions + import matplotlib.pyplot as plt + from matplotlib import pyplot as plt + import matplotlib.pyplot as pltttt + + # failing submissions + import matplotlib as mpl + + Example 2, where the SCT is coded so aliases do matter: :: + + # solution + import matplotlib.pyplot as plt + + # sct + Ex().has_import("matplotlib.pyplot", same_as=True) + + # passing submissions + import matplotlib.pyplot as plt + from matplotlib import pyplot as plt + + # failing submissions + import matplotlib.pyplot as pltttt + + """ + student_imports = state.ast_dispatcher.find("imports", state.student_ast) + solution_imports = state.ast_dispatcher.find("imports", state.solution_ast) + + if name not in solution_imports: + raise InstructorError.from_message( + "`has_import()` couldn't find an import of the package %s in your solution code." + % name + ) + + fmt_kwargs = {"pkg": name, "alias": solution_imports[name]} + + state.do_test( + DefinedCollTest( + name, student_imports, FeedbackComponent(not_imported_msg, fmt_kwargs) + ) + ) + + if same_as: + state.do_test( + EqualTest( + solution_imports[name], + student_imports[name], + FeedbackComponent(incorrect_as_msg, fmt_kwargs), + ) + ) + + return state + + +def has_output(state, text, pattern=True, no_output_msg=None): + r"""Search student output for a pattern. + + Among the student and solution process, the student submission and solution code as a string, + the ``Ex()`` state also contains the output that a student generated with his or her submission. + + With ``has_output()``, you can access this output and match it against a regular or fixed expression. + + Args: + text (str): the text that is searched for + pattern (bool): if True (default), the text is treated as a pattern. If False, it is treated as plain text. + no_output_msg (str): feedback message to be displayed if the output is not found. + + :Example: + + As an example, suppose we want a student to print out a sentence: :: + + # Print the "This is some ... stuff" + print("This is some weird stuff") + + The following SCT tests whether the student prints out ``This is some weird stuff``: :: + + # Using exact string matching + Ex().has_output("This is some weird stuff", pattern = False) + + # Using a regular expression (more robust) + # pattern = True is the default + msg = "Print out ``This is some ... stuff`` to the output, " + \\ + "fill in ``...`` with a word you like." + Ex().has_output(r"This is some \w* stuff", no_output_msg = msg) + + """ + if not no_output_msg: + no_output_msg = "You did not output the correct things." + + state.do_test( + StringContainsTest(state.raw_student_output, text, pattern, no_output_msg) + ) + + return state + + +def has_printout( + state, index, not_printed_msg=None, pre_code=None, name=None, copy=False +): + """Check if the right printouts happened. + + ``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in + the solution process, capture its output, and verify whether the output is present in the output of the student. + + This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many + printouts as they want, as long as they do the correct one somewhere. + + Args: + index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output. + not_printed_msg (str): if specified, this overrides the default message that is generated when the output + is not found in the student output. + pre_code (str): Python code as a string that is executed before running the targeted student call. + This is the ideal place to set a random seed, for example. + copy (bool): whether to try to deep copy objects in the environment, such as lists, that could + accidentally be mutated. Disabled by default, which speeds up SCTs. + state (State): state as passed by the SCT chain. Don't specify this explicitly. + + :Example: + + Suppose you want somebody to print out 4: :: + + print(1, 2, 3, 4) + + The following SCT would check that: :: + + Ex().has_printout(0) + + All of the following SCTs would pass: :: + + print(1, 2, 3, 4) + print('1 2 3 4') + print(1, 2, '3 4') + print("random"); print(1, 2, 3, 4) + + :Example: + + Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed. + If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work. + + Suppose you have the following solution: :: + + x = 4 + print(x) + x = 6 + + The following SCT will not work: :: + + Ex().has_printout(0) + + Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated. + In cases like these, ``has_printout()`` cannot be used. + + :Example: + + Inside a for loop ``has_printout()`` + + Suppose you have the following solution: :: + + for i in range(5): + print(i) + + The following SCT will not work: :: + + Ex().check_for_loop().check_body().has_printout(0) + + The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``. + If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: :: + + Ex().check_for_loop().check_body().\\ + set_context(0).check_function('print').\\ + check_args(0).has_equal_value() + + """ + + extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead." + state.assert_execution_root("has_printout", extra_msg=extra_msg) + + if not_printed_msg is None: + not_printed_msg = ( + "Have you used `{{sol_call}}` to do the appropriate printouts?" + ) + + try: + sol_call_ast = state.ast_dispatcher.find("function_calls", state.solution_ast)[ + "print" + ][index]["node"] + except (KeyError, IndexError): + raise InstructorError.from_message( + "`has_printout({})` couldn't find the {} print call in your solution.".format( + index, get_ord(index + 1) + ) + ) + + out_sol, str_sol = getOutputInProcess( + tree=sol_call_ast, + process=state.solution_process, + context=state.solution_context, + env=state.solution_env, + pre_code=pre_code, + copy=copy, + ) + + sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast) + + if isinstance(str_sol, Exception): + with debugger(state): + state.report( + "Evaluating the solution expression {} raised error in solution process." + "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol) + ) + + has_output( + state, + out_sol.strip(), + pattern=False, + no_output_msg=FeedbackComponent(not_printed_msg, {"sol_call": sol_call_str}), + ) + + return state + + +def has_no_error( + state, + incorrect_msg="Have a look at the console: your code contains an error. Fix it and try again!", +): + """Check whether the submission did not generate a runtime error. + + If all SCTs for an exercise pass, before marking the submission as correct pythonwhat will automatically check whether + the student submission generated an error. This means it is not needed to use ``has_no_error()`` explicitly. + + However, in some cases, using ``has_no_error()`` explicitly somewhere throughout your SCT execution can be helpful: + + - If you want to make sure people didn't write typos when writing a long function name. + - If you want to first verify whether a function actually runs, before checking whether the arguments were specified correctly. + - More generally, if, because of the content, it's instrumental that the script runs without + errors before doing any other verifications. + + Args: + incorrect_msg: if specified, this overrides the default message if the student code generated an error. + + :Example: + + Suppose you're verifying an exercise about model training and validation: :: + + # pre exercise code + import numpy as np + from sklearn.model_selection import train_test_split + from sklearn import datasets + from sklearn import svm + + iris = datasets.load_iris() + iris.data.shape, iris.target.shape + + # solution + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, test_size=0.4, random_state=0) + + If you want to make sure that ``train_test_split()`` ran without errors, + which would check if the student typed the function without typos and used + sensical arguments, you could use the following SCT: :: + + Ex().has_no_error() + Ex().check_function('sklearn.model_selection.train_test_split').multi( + check_args(['arrays', 0]).has_equal_value(), + check_args(['arrays', 0]).has_equal_value(), + check_args(['options', 'test_size']).has_equal_value(), + check_args(['options', 'random_state']).has_equal_value() + ) + + If, on the other hand, you want to fall back onto pythonwhat's built in behavior, + that checks for an error before marking the exercise as correct, you can simply + leave of the ``has_no_error()`` step. + + """ + state.assert_execution_root("has_no_error") + + if state.reporter.errors: + state.report(incorrect_msg, {"error": str(state.reporter.errors[0])}) + + return state + + +MC_VAR_NAME = "selected_option" + + +def has_chosen(state, correct, msgs): + """Test multiple choice exercise. + + Test for a MultipleChoiceExercise. The correct answer (as an integer) and feedback messages + are passed to this function. + + Args: + correct (int): the index of the correct answer (should be an instruction). Starts at 1. + msgs (list(str)): a list containing all feedback messages belonging to each choice of the + student. The list should have the same length as the number of options. + """ + if not issubclass(type(correct), int): + raise InstructorError.from_message( + "Inside `has_chosen()`, the argument `correct` should be an integer." + ) + + student_process = state.student_process + if not isDefinedInProcess(MC_VAR_NAME, student_process): + raise InstructorError.from_message( + "Option not available in the student process" + ) + else: + selected_option = getOptionFromProcess(student_process, MC_VAR_NAME) + if not issubclass(type(selected_option), int): + raise InstructorError.from_message("selected_option should be an integer") + + if selected_option < 1 or correct < 1: + raise InstructorError.from_message( + "selected_option and correct should be greater than zero" + ) + + if selected_option > len(msgs) or correct > len(msgs): + raise InstructorError.from_message( + "there are not enough feedback messages defined" + ) + + feedback_msg = msgs[selected_option - 1] + + state.reporter.success_msg = msgs[correct - 1] + + state.do_test(EqualTest(selected_option, correct, feedback_msg)) diff --git a/pythonwhat/converters.py b/pythonwhat/converters.py index 65c784e9..16d48ba2 100644 --- a/pythonwhat/converters.py +++ b/pythonwhat/converters.py @@ -1,19 +1,23 @@ import inspect import pythonwhat + def get_manual_converters(): converters = { - 'pandas.io.excel.ExcelFile': lambda x: x.io, - 'builtins.dict_keys': lambda x: sorted(x), - 'builtins.dict_items': lambda x: sorted(x), - 'bs4.BeautifulSoup': lambda x: str(x), - 'bs4.element.Tag': lambda x: str(x), - 'bs4.element.NavigableString': lambda x: str(x), - 'bs4.element.ResultSet': lambda x: [str(res) for res in x], - 'h5py._hl.files.File': lambda x: x.file.filename, - 'h5py._hl.group.Group': lambda x: x.file.filename + '_' + str([x for x in x.keys()]), - 'sqlalchemy.engine.base.Engine': lambda x: x.url.database + "pandas.io.excel.ExcelFile": lambda x: x.io, + "pandas.io.excel._base.ExcelFile": lambda x: x.io, + "builtins.dict_keys": lambda x: sorted(x), + "builtins.dict_items": lambda x: sorted(x), + "bs4.BeautifulSoup": lambda x: str(x), + "bs4.element.Tag": lambda x: str(x), + "bs4.element.NavigableString": lambda x: str(x), + "bs4.element.ResultSet": lambda x: [str(res) for res in x], + "h5py._hl.files.File": lambda x: x.file.filename, + "h5py._hl.group.Group": lambda x: x.file.filename + + "_" + + str([x for x in x.keys()]), + "sqlalchemy.engine.base.Engine": lambda x: x.url.database, } - return(converters) + return converters diff --git a/pythonwhat/feedback.py b/pythonwhat/feedback.py new file mode 100644 index 00000000..dae753f0 --- /dev/null +++ b/pythonwhat/feedback.py @@ -0,0 +1,19 @@ +from typing import Dict + +from protowhat.Feedback import Feedback as ProtoFeedback + + +class Feedback(ProtoFeedback): + ast_highlight_offset = {"column_start": 1} + + @classmethod + def get_highlight_position(cls, highlight) -> Dict[str, int]: + if getattr(highlight, "first_token", None) and getattr( + highlight, "last_token", None + ): + return { + "line_start": highlight.first_token.start[0], + "column_start": highlight.first_token.start[1], + "line_end": highlight.last_token.end[0], + "column_end": highlight.last_token.end[1], + } diff --git a/pythonwhat/has_funcs.py b/pythonwhat/has_funcs.py deleted file mode 100644 index 10239e29..00000000 --- a/pythonwhat/has_funcs.py +++ /dev/null @@ -1,583 +0,0 @@ -from pythonwhat.tasks import getResultInProcess, getOutputInProcess, getErrorInProcess, ReprFail, isDefinedInProcess, getOptionFromProcess, ReprFail, UndefinedValue -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest -from pythonwhat.Feedback import Feedback -from pythonwhat import utils -from functools import partial -import re -import copy -import ast - -evalCalls = {'value': getResultInProcess, - 'output': getOutputInProcess, - 'error': getErrorInProcess} - -def has_part(name, msg, state=None, fmt_kwargs=None, index=None): - rep = Reporter.active_reporter - d = {'sol_part': state.solution_parts, - 'stu_part': state.student_parts, - **fmt_kwargs - } - - try: - part = state.student_parts[name] - if index is not None: - if isinstance(index, list): - for ind in index: - part = part[ind] - else: - part = part[index] - if part is None: raise KeyError - except (KeyError, IndexError): - _msg = state.build_message(msg, d) - rep.do_test(Test(Feedback(_msg, state))) - - return state - - -def has_equal_part(name, msg, state): - rep = Reporter.active_reporter - d = {'stu_part': state.student_parts, - 'sol_part': state.solution_parts, - 'name': name} - - _msg = state.build_message(msg, d) - rep.do_test(EqualTest(d['stu_part'][name], d['sol_part'][name], Feedback(_msg, state))) - - return state - -# TODO: shouldn't have to hardcode message -def has_equal_part_len(name, unequal_msg, state=None): - """Verify that a part that is zoomed in on has equal length. - - Typically used in the context of ``check_function_def()`` - - Arguments: - name (str): name of the part for which to check the length to the corresponding part in the solution. - unequal_msg (str): Message in case the lengths do not match. - state (State): state as passed by the SCT chain. Don't specify this explicitly. - - :Examples: - - Student and solution code:: - - def shout(word): - return word + '!!!' - - SCT that checks number of arguments:: - - Ex().check_function_def('shout').has_equal_part_len('args', 'not enough args!') - - """ - rep = Reporter.active_reporter - d = dict(stu_len = len(state.student_parts[name]), - sol_len = len(state.solution_parts[name])) - - if d['stu_len'] != d['sol_len']: - _msg = state.build_message(unequal_msg, d) - rep.do_test(Test(Feedback(_msg, state))) - - return state - -## Expression tests ----------------------------------------------------------- - -def has_equal_ast(incorrect_msg=None, - code=None, - exact=True, - append=None, - state=None): - """Test whether abstract syntax trees match between the student and solution code. - - ``has_equal_ast()`` can be used in two ways: - - * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission. - * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments. - - Args: - incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this. - code: optional code to use instead of the solution AST. - exact: whether the representations must match exactly. If false, the solution AST - only needs to be contained within the student AST (similar to using test student typed). - Defaults to ``True``, unless the ``code`` argument has been specified. - - :Example: - - Student and Solution Code:: - - dict(a = 'value').keys() - - SCT:: - - # all pass - Ex().has_equal_ast() - Ex().has_equal_ast(code = "dict(a = 'value').keys()") - Ex().has_equal_ast(code = "dict(a = 'value')", exact = False) - - Student and Solution Code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - np.mean(arr) - - SCT:: - - # Check underlying value of arugment a of np.mean: - Ex().check_function('numpy.mean').check_args('a').has_equal_ast() - - # Only check AST equality of expression used to specify argument a: - Ex().check_function('numpy.mean').check_args('a').has_equal_ast() - - """ - rep = Reporter.active_reporter - - if code and incorrect_msg is None: - raise ValueError("If you manually specify the code to match inside has_equal_ast(), you have to explicitly set the `incorrect_msg` arugment.") - - if append is None: # if not specified, set to False if incorrect_msg was manually specified - append = incorrect_msg is None - if incorrect_msg is None: - incorrect_msg = "__JINJA__:Expected `{{sol_str}}`, but got `{{stu_str}}`." - - def parse_tree(tree): - # get contents of module.body if only 1 element - crnt = tree.body[0] if isinstance(tree, ast.Module) and len(tree.body) == 1 else tree - - # remove Expr if it exists - return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt) - - stu_rep = parse_tree(state.student_tree) - sol_rep = parse_tree(state.solution_tree if not code else ast.parse(code)) - - fmt_kwargs = { - 'sol_str': state.solution_code if not code else code, - 'stu_str': state.student_code - } - - _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) - - if exact and not code: - rep.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state))) - elif not sol_rep in stu_rep: - rep.do_test(Test(Feedback(_msg, state))) - - return state - -DEFAULT_INCORRECT_MSG="__JINJA__:Expected {{test_desc}}`{{sol_eval}}`, but got `{{stu_eval}}`." -DEFAULT_ERROR_MSG="__JINJA__:Running {{'it' if parent['part'] else 'the higlighted expression'}} generated an error: `{{stu_str}}`." -DEFAULT_ERROR_MSG_INV="__JINJA__:Running {{'it' if parent['part'] else 'the higlighted expression'}} didn't generate an error, but it should!" -DEFAULT_UNDEFINED_NAME_MSG="__JINJA__:Running {{'it' if parent['part'] else 'the higlighted expression'}} should define a variable `{{name}}` without errors, but it doesn't." -DEFAULT_INCORRECT_NAME_MSG="__JINJA__:Are you sure you assigned the correct value to `{{name}}`?" -def has_expr(incorrect_msg=None, - error_msg=None, - undefined_msg=None, - append=None, - extra_env=None, - context_vals=None, - pre_code=None, - expr_code=None, - name=None, - copy=True, - func=None, - state=None, - test=None): - - if append is None: # if not specified, set to False if incorrect_msg was manually specified - append = incorrect_msg is None - if incorrect_msg is None: - incorrect_msg = DEFAULT_INCORRECT_MSG if name is None else DEFAULT_INCORRECT_NAME_MSG - if undefined_msg is None: - undefined_msg = DEFAULT_UNDEFINED_NAME_MSG - if error_msg is None: - error_msg = DEFAULT_ERROR_MSG_INV if test == 'error' else DEFAULT_ERROR_MSG - - rep = Reporter.active_reporter - - get_func = partial(evalCalls[test], - extra_env=extra_env, - context_vals=context_vals, - pre_code=pre_code, - expr_code=expr_code, - name=name, - copy=copy) - - eval_sol, str_sol = get_func(tree=state.solution_tree, - process=state.solution_process, - context=state.solution_context, - env=state.solution_env) - - if (test == 'error') ^ isinstance(eval_sol, Exception): - raise ValueError("Evaluating expression raised error in solution process (or not an error if testing for one). " - "Error: {} - {}".format(type(eval_sol), str_sol)) - if isinstance(eval_sol, ReprFail): - raise ValueError("Couldn't figure out the value of a default argument: " + eval_sol.info) - - eval_stu, str_stu = get_func(tree=state.student_tree, - process=state.student_process, - context=state.student_context, - env=state.student_env) - - # kwargs --- - fmt_kwargs = { - 'stu_part': state.student_parts, - 'sol_part': state.solution_parts, - 'name': name, 'test': test, - 'test_desc': '' if test == 'value' else 'the %s ' % test - } - - fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) - fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) - if incorrect_msg == DEFAULT_INCORRECT_MSG and \ - ( fmt_kwargs['stu_eval'] is None or - fmt_kwargs['sol_eval'] is None or - fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ): - incorrect_msg = "Expected something different." - - # tests --- - # error in process - if (test == 'error') ^ isinstance(eval_stu, Exception): - fmt_kwargs['stu_str'] = str_stu - _msg = state.build_message(error_msg, fmt_kwargs, append=append) - feedback = Feedback(_msg, state) - rep.do_test(Test(feedback)) - - # name is undefined after running expression - if isinstance(eval_stu, UndefinedValue): - _msg = state.build_message(undefined_msg, fmt_kwargs, append=append) - rep.do_test(Test(Feedback(_msg, state))) - - # test equality of results - _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) - rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func)) - - return state - - - -args_string = """ - - Args: - incorrect_msg (str): feedback message if the {0} of the expression in the solution - doesn't match the one of the student. This feedback message will be expanded if it is used - in the context of another check function, like ``check_if_else``. - error_msg (str): feedback message if there was an error when running the targeted student code. - Note that when testing for an error, this message is displayed when none is raised. - undefined_msg (str): feedback message if the ``name`` argument is defined, but a variable - with that name doesn't exist after running the targeted student code. - extra_env (dict): set variables to the extra environment. They will update the student and solution environment in - the active state before the student/solution code in the active state is ran. This argument should contain a - dictionary with the keys the names of the variables you want to set, and the values are the values of these variables. - You can also use ``set_env()`` for this. - context_vals (list): set variables which are bound in a ``for`` loop to certain values. - This argument is only useful when checking a for loop (or list comprehensions). - It contains a list with the values of the bound variables. - You can also use ``set_context()`` for this. - pre_code (str): the code in string form that should be executed before the expression is executed. - This is the ideal place to set a random seed, for example. - expr_code (str): if this argument is set, the expression in the student/solution code will not - be ran. Instead, the given piece of code will be ran in the student as well as the solution environment - and the result will be compared. - name (str): If this is specified, the {1} of running this expression after running the focused expression - is returned, instead of the {1} of the focussed expression in itself. This is typically used to inspect the - {1} of an object after executing the body of e.g. a ``for`` loop. - copy (bool): whether to try to deep copy objects in the environment, such as lists, that could - accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues. - func: custom binary function of form f(stu_result, sol_result), for equality testing. - """ - -has_equal_value = partial(has_expr, test = 'value') -has_equal_value.__doc__ = """Run targeted student and solution code, and compare returned value. - - When called on an SCT chain, ``has_equal_value()`` will execute the student and solution - code that is 'zoomed in on' and compare the returned values. - """ + args_string.format("returned value", "value") + """ - :Example: - - Student code and solution code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - np.mean(arr) - - SCT:: - - # Verify equality of arr: - Ex().check_object('arr').has_equal_value() - - # Verify whether arr was correctly set in np.mean - Ex().check_function('numpy.mean').check_args('a').has_equal_value() - - # Verify whether np.mean(arr) produced the same result - Ex().check_function('numpy.mean').has_equal_value() - - """ - - -has_equal_output = partial(has_expr, test = 'output') -has_equal_output.__doc__ = """Run targeted student and solution code, and compare output. - - When called on an SCT chain, ``has_equal_output()`` will execute the student and solution - code that is 'zoomed in on' and compare the output. - """ + args_string.format("output", "output") - -has_equal_error = partial(has_expr, test = 'error') -has_equal_error.__doc__ = """Run targeted student and solution code, and compare generated errors. - - When called on an SCT chain, ``has_equal_error()`` will execute the student and solution - code that is 'zoomed in on' and compare the errors that they generate. - """ + args_string.format("error", "error") - -## Various has tests ---------------------------------------------------------- - -from pythonwhat.Test import StringContainsTest - -def has_code(text, - pattern=True, - not_typed_msg=None, - state=None): - """Test the student code. - - Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``, - as it is more robust to small syntactical differences that don't change the code's behavior. - - Args: - text (str): the text that is searched for - pattern (bool): if True (the default), the text is treated as a pattern. If False, it is treated as plain text. - not_typed_msg (str): feedback message to be displayed if the student did not type the text. - - :Example: - - Student code and solution code:: - - y = 1 + 2 + 3 - - SCT:: - - # Verify that student code contains pattern (not robust!!): - Ex().has_code(r"1\s*\+2\s*\+3") - - """ - rep = Reporter.active_reporter - - if not not_typed_msg: - if pattern: - not_typed_msg = "Could not find the correct pattern in your code." - else: - not_typed_msg = "Could not find the following text in your code: %r" % text - - student_code = state.student_code - - _msg = state.build_message(not_typed_msg) - rep.do_test(StringContainsTest(student_code, text, pattern, Feedback(_msg, state))) - - return state - -from pythonwhat.Test import Test, DefinedCollTest, EqualTest - -def has_import(name, - same_as=False, - not_imported_msg="__JINJA__:Did you import `{{pkg}}`?", - incorrect_as_msg="__JINJA__:Did you import `{{pkg}}` as `{{alias}}`?", - state=None): - """Checks whether student imported a package or function correctly. - - Args: - name (str): the name of the package that has to be checked. - same_as (bool): if True, the alias of the package or function has to be the same. Defaults to False. - not_imported_msg (str): feedback message when the package is not imported. - incorrect_as_msg (str): feedback message if the alias is wrong. - - - :Example: - - Student code:: - - import numpy as np - import pandas as pa - - Solution code:: - - import numpy as np - import pandas as pd - - SCT:: - - Ex().has_import("numpy") # pass - Ex().has_import("pandas") # pass - Ex().has_import("pandas", same_as=True) # fail - - """ - - rep = Reporter.active_reporter - - student_imports = state.student_imports - solution_imports = state.solution_imports - - if name not in solution_imports: - raise NameError("The package you specified is not in the solution imports itself. %r not in solution imports" % name) - - fmt_kwargs = { 'pkg': name, 'alias': solution_imports[name] } - - _msg = state.build_message(not_imported_msg, fmt_kwargs) - rep.do_test(DefinedCollTest(name, student_imports, _msg)) - - if (same_as): - _msg = state.build_message(incorrect_as_msg, fmt_kwargs) - rep.do_test(EqualTest(solution_imports[name], student_imports[name], _msg)) - - return state - -def has_output(text, - pattern=True, - no_output_msg=None, - state=None): - """Search student output. - - Checks if the output contains a (pattern of) text. - - Args: - text (str): the text that is searched for - pattern (bool): if True (default), the text is treated as a pattern. If False, it is treated as plain text. - no_output_msg (str): feedback message to be displayed if the output is not found. - - :Example: - - SCT:: - - Ex().has_output(r'[H|h]i,*\\s+there!') - - Submissions:: - - print("Hi, there!") # pass - print("hi there!") # pass - print("Hello there") # fail - """ - rep = Reporter.active_reporter - - if not no_output_msg: - no_output_msg = "You did not output the correct things." - # raise ValueError("Inside has_output(), specify the `no_output_msg` manually.") - - student_output = state.raw_student_output - - _msg = state.build_message(no_output_msg) - rep.do_test( - StringContainsTest( - student_output, - text, - pattern, - _msg)) - - return state - -def has_printout(index, - not_printed_msg=None, - pre_code=None, - name=None, - copy=False, - state=None): - """Check if the output of print() statement in the solution is in the output the student generated. - - This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many - printouts as they want, as long as they do the correct one somewhere. - - .. note:: - - When zooming in on parts of the student submission (with e.g. ``check_for_loop()``), we are not - zooming in on the piece of the student output that is related to that piece of the student code. - In other words, ``has_printout()`` always considers the entire student output. - - Args: - index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output. - not_printed_msg (str): if specified, this overrides the default message that is generated when the output - is not found in the student output. - pre_code (str): Python code as a string that is executed before running the targeted student call. - This is the ideal place to set a random seed, for example. - copy (bool): whether to try to deep copy objects in the environment, such as lists, that could - accidentally be mutated. Disabled by default, which speeds up SCTs. - state (State): state as passed by the SCT chain. Don't specify this explicitly. - - :Example: - - Solution:: - - print(1, 2, 3, 4) - - SCT:: - - Ex().has_printout(0) - - Each of these submissions will pass:: - - print(1, 2, 3, 4) - print('1 2 3 4') - print(1, 2, '3 4') - print("random"); print(1, 2, 3, 4) - """ - - if not_printed_msg is None: - not_printed_msg = "__JINJA__:Have you used `{{sol_call}}` to do the appropriate printouts?" - - try: - sol_call_ast = state.solution_function_calls['print'][index]['node'] - except (KeyError, IndexError): - raise ValueError("Using has_printout() with index {} expects that there is/are at least {} print() call(s) in your solution." - "Is that the case?".format(index, index+1)) - - out_sol, str_sol = getOutputInProcess( - tree = sol_call_ast, - process = state.solution_process, - context = state.solution_context, - env = state.solution_env, - pre_code = pre_code, - copy = copy - ) - - sol_call_str = state.solution_tree_tokens.get_text(sol_call_ast) - - if isinstance(str_sol, Exception): - raise ValueError("Evaluating the solution expression {} raised error in solution process." - "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol)) - - _msg = state.build_message(not_printed_msg, { 'sol_call': sol_call_str }) - - has_output(out_sol.strip(), pattern = False, no_output_msg=_msg, state=state) - - return state - -MC_VAR_NAME = "selected_option" - -def has_chosen(correct, msgs, state=None): - """Test multiple choice exercise. - - Test for a MultipleChoiceExercise. The correct answer (as an integer) and feedback messages - are passed to this function. - - Args: - correct (int): the index of the correct answer (should be an instruction). Starts at 1. - msgs (list(str)): a list containing all feedback messages belonging to each choice of the - student. The list should have the same length as the number of instructions. - """ - if not issubclass(type(correct), int): - raise ValueError("correct should be an integer") - - rep = Reporter.active_reporter - student_process = state.student_process - if not isDefinedInProcess(MC_VAR_NAME, student_process): - raise NameError("Option not available in the student process") - else: - selected_option = getOptionFromProcess(student_process, MC_VAR_NAME) - if not issubclass(type(selected_option), int): - raise ValueError("selected_option should be an integer") - - if selected_option < 1 or correct < 1: - raise ValueError( - "selected_option and correct should be greater than zero") - - if selected_option > len(msgs) or correct > len(msgs): - raise ValueError("there are not enough feedback messages defined") - - feedback_msg = msgs[selected_option - 1] - - rep.success_msg = msgs[correct - 1] - - rep.do_test(EqualTest(selected_option, correct, feedback_msg)) diff --git a/pythonwhat/local.py b/pythonwhat/local.py index 85814e94..7dee518e 100644 --- a/pythonwhat/local.py +++ b/pythonwhat/local.py @@ -1,47 +1,293 @@ import io -from pythonwhat.check_syntax import Ex -from pythonwhat.State import State -from pythonwhat.Reporter import Reporter +import os +import random +from pathlib import Path from contextlib import redirect_stdout -class StubShell(object): - - def __init__(self, init_code = None): +from multiprocessing import Process, Queue +from protowhat.Reporter import Reporter + +try: + from pythonbackend.shell_utils import create + from pythonbackend.tasks import TaskCaptureFullOutput + + BACKEND_AVAILABLE = True +except: + BACKEND_AVAILABLE = False + + +class StubShell: + def __init__(self, init_code=None): self.user_ns = {} if init_code: self.run_code(init_code) - + def run_code(self, code): - exec(code, None, self.user_ns) + exec(code, self.user_ns) -class StubProcess(object): - def __init__(self, init_code = None): +class StubProcess: + def __init__(self, init_code=None, pid=None): self.shell = StubShell(init_code) + self._identity = (pid,) if pid else (random.randint(0, int(1e12)),) def executeTask(self, task): return task(self.shell) -def setup_state(stu_code = "", sol_code = "", pec = ""): - stu_output = io.StringIO() - with redirect_stdout(stu_output): - stu_process = StubProcess(init_code = "%s\n%s" % (pec, stu_code)) +class TaskCaptureOutput: + def __init__(self, code): + self.code = code + + def __call__(self, shell): + return run_code(shell.run_code, self.code) + + +class TaskKillProcess: + def __call__(self, shell): + return None + + +class CaptureErrors: + def __init__(self, output): + self.output = output + + def __enter__(self): + pass + + def __exit__(self, exc_type, exception, traceback): + if exc_type is not None: + self.output.append({"type": "backend-error", "payload": str(exception)}) + return True + + +class WorkerProcess(Process): + instances = [] + + def __init__(self, pid=None): + Process.__init__(self) + self.task_queue = Queue() + self.result_queue = Queue() + self.daemon = ( + True + ) # when parent process is killed, sub/childprocess get also killed + self.instances.append(self) + # used to detect single process exercise + self._identity = (pid,) if pid else (random.randint(0, int(1e12)),) + + def get_shell(self): + return create({}) + + def run(self): + shell = self.get_shell() + while True: + output = [] + with CaptureErrors(output): + next_task = self.task_queue.get() + answer = next_task(shell) + if len(output) > 0: # means backend error happened + answer = output + output = [] + with CaptureErrors(output): + self.result_queue.put_nowait(answer) + if len(output) > 0: # means backend error happened + self.result_queue.put_nowait(output) + if isinstance(next_task, TaskKillProcess): + break # break while loop -> we do not wait upon new task + return + + def executeTask(self, task): + self.task_queue.put_nowait(task) + return self.result_queue.get() # wait and fetches next item in queue + + def kill(self): + try: + if self.is_alive(): + self.executeTask(TaskKillProcess()) + self.join(timeout=3.0) + if self.is_alive(): + self.terminate() + self.join(timeout=3.0) + if self in self.instances: + self.instances.remove(self) + finally: + pass + # python 3.7: + # self.close() + + @classmethod + def kill_all(cls): + for instance in list(cls.instances): + instance.kill() + + +class SimpleProcess(WorkerProcess): + def get_shell(self): + return StubShell() + + +class ChDir(object): + """ + Step into a directory temporarily. + """ + + def __init__(self, path): + self.old_dir = os.getcwd() + self.new_dir = str(path) + + def __enter__(self): + os.chdir(self.new_dir) + + def __exit__(self, *args): + os.chdir(self.old_dir) + + +def run_code(executor, code): + with io.StringIO() as output: + try: + with redirect_stdout(output): + executor(code) + raw_output = output.getvalue() + error = None + except BaseException as e: + raw_output = "" + error = str(e) + return raw_output, error + + +def run_single_process(pec, code, pid=None, mode="simple"): + if mode == "stub": + # no isolation + process = StubProcess(init_code=pec, pid=pid) + raw_output, error = run_code(process.shell.run_code, code) + + elif mode == "simple": + # no advanced functionality + process = SimpleProcess(pid) + process.start() + _ = process.executeTask(TaskCaptureOutput(pec)) + raw_output, error = process.executeTask(TaskCaptureOutput(code)) + + elif mode == "full" and BACKEND_AVAILABLE: + # slow + process = WorkerProcess(pid) + process.start() + _ = process.executeTask( + TaskCaptureFullOutput((pec,), "", None, silent=True) + ) + output, raw_output = process.executeTask( + TaskCaptureFullOutput((code,), "script.py", None, silent=True) + ) + raw_output = raw_output["output_stream"] + error = raw_output["error"] + + else: + raise ValueError("Invalid mode") + + return process, raw_output, error + + +def run_exercise(pec, sol_code, stu_code, sol_wd=None, stu_wd=None, **kwargs): + with ChDir(sol_wd or os.getcwd()): + sol_process, _, _ = run_single_process(pec, sol_code, **kwargs) + + with ChDir(stu_wd or os.getcwd()): + stu_process, raw_stu_output, error = run_single_process(pec, stu_code, **kwargs) + + return sol_process, stu_process, raw_stu_output, error + + +# todo: +# imports from local modules (solution needs to be materialised somewhere) +# converge with xbackend (pythonbackend + look at scalabackend) +# move towards xwhat controlling all execution and xbackend providing the execution interface? +# running with arbitrary wd + path + flags (now only wd) needed? +# e.g. `python -m project.run +# allow setting env vars? e.g. PYTHONPATH, could help running more complex setup +# allow prepending code? set_env? e.g. (automatically) setting __file__? +def run(state, relative_working_dir=None, solution_dir="../solution", run_solution=True): + """Run the focused student and solution code in the specified location + + This function can be used after ``check_file`` to execute student and solution code. + The arguments allow configuring the correct context for execution. + + SCT functions chained after this one that execute pieces of code (custom expressions or the focused part of a file) + execute in the same student and solution locations as the file. + + .. note:: + + This function does not execute the file itself, but code in memory. + This can have an impact when: + + - the solution code imports from a different file in the expected solution (code that is not installed) + - using functionality depending on e.g. ``__file__`` and ``inspect`` + + When the expected code has imports from a different file that is part of the exercise, + it can only work if the solution code provided earlier does not have these imports but instead + has all that functionality inlined. + + Args: + relative_working_dir (str): if specified, this relative path is the subdirectory + inside the student and solution context in which the code is executed + solution_dir (str): a relative path, ``solution`` by default, + that sets the root of the solution context, relative to that of the student execution context + state (State): state as passed by the SCT chain. Don't specify this explicitly. + + If ``relative_working_dir`` is not set, it will be the directory the file was loaded from by ``check_file`` + and fall back to the root of the student execution context (the working directory pythonwhat runs in). + + The ``solution_dir`` helps to prevent solution side effects from conflicting with those of the student. + If the set or derived value of ``relative_working_dir`` is an absolute path, + ``relative_working_dir`` will not be used to form the solution execution working directory: + the solution code will be executed in the root of the solution execution context. + + :Example: + + Suppose the student and solution have a file ``script.py`` in ``/home/repl/``:: + + if True: + a = 1 + + print("Hi!") + + We can check it with this SCT (with ``file_content`` containing the expected file content):: - sol_output = io.StringIO() - with redirect_stdout(sol_output): - sol_process = StubProcess(init_code = "%s\n%s" % (pec, sol_code)) + Ex().check_file( + "script.py", + solution_code=file_content + ).run().multi( + check_object("a").has_equal_value(), + has_printout(0) + ) + """ + # todo: + # look into executing the file itself + # and keeping the process alive to extract values + if relative_working_dir is None: + if getattr(state, "path", False): + relative_working_dir = state.path.parent + else: + relative_working_dir = "" - rep = Reporter() - Reporter.active_reporter = rep + if not os.path.isabs(str(relative_working_dir)): + sol_wd = Path(os.getcwd(), solution_dir, relative_working_dir) + else: + sol_wd = Path(os.getcwd(), solution_dir) - state = State( - student_code = stu_code, - solution_code = sol_code, - pre_exercise_code = pec, - student_process = stu_process, - solution_process = sol_process, - raw_student_output = stu_output.getvalue()) + os.makedirs(str(sol_wd), exist_ok=True) + stu_wd = Path(os.getcwd(), relative_working_dir) + sol_code = state.solution_code or "" if run_solution else "" - State.root_state = state - return(Ex(state)) + sol_process, stu_process, raw_stu_output, error = run_exercise( + pec="", + sol_code=sol_code, + stu_code=state.student_code, + sol_wd=sol_wd, + stu_wd=stu_wd, + ) + return state.to_child( + student_process=stu_process, + solution_process=sol_process, + raw_student_output=raw_stu_output, + reporter=Reporter(state.reporter, errors=[error] if error else []), + ) diff --git a/pythonwhat/parsing.py b/pythonwhat/parsing.py index f404ccef..cd9102d4 100644 --- a/pythonwhat/parsing.py +++ b/pythonwhat/parsing.py @@ -16,7 +16,10 @@ https://greentreesnakes.readthedocs.org/en/latest/ """ -class EmptyTargetVar: pass + +class EmptyTargetVar: + pass + class TargetVars(Mapping): """Immutable ordered mapping from target variables to their values.""" @@ -24,15 +27,20 @@ class TargetVars(Mapping): EMPTY = EmptyTargetVar() def __init__(self, target_vars=tuple(), is_empty=True): - if is_empty: + if is_empty: target_vars = [(v, self.EMPTY) for v in target_vars] self._od = OrderedDict(target_vars) # getitem, len, iter wrap OrderedDict behavior - def __getitem__(self, k): return self._od.__getitem__(k) - def __len__(self): return self._od.__len__() - def __iter__(self): return self._od.__iter__() + def __getitem__(self, k): + return self._od.__getitem__(k) + + def __len__(self): + return self._od.__len__() + + def __iter__(self): + return self._od.__iter__() def update(self, *args, **kwargs): cpy = self.copy() @@ -44,12 +52,17 @@ def copy(self): def __str__(self): """Format target vars for printing""" - if len(self) > 1: return "({})".format(", ".join(self._od.keys())) - else: return "".join(self._od.keys()) + if len(self) > 1: + return "({})".format(", ".join(self._od.keys())) + else: + return "".join(self._od.keys()) def defined_items(self): """Return copy of instance, omitting entries that are EMPTY""" - return self.__class__([(k, v) for k,v in self.items() if v is not self.EMPTY], is_empty=False) + return self.__class__( + [(k, v) for k, v in self.items() if v is not self.EMPTY], is_empty=False + ) + class IndexedDict(Mapping): """Wrapper around OrderedDict that allows access via item position or key""" @@ -57,15 +70,19 @@ class IndexedDict(Mapping): def __init__(self, *args, **kwargs): self._od = OrderedDict(*args, **kwargs) - def __getitem__(self, k): - try: return list(self._od.values())[k] - except TypeError: return self._od[k] - - def __len__(self): return self._od.__len__() - def __iter__(self): return self._od.__iter__() + def __getitem__(self, k): + try: + return list(self._od.values())[k] + except TypeError: + return self._od[k] + + def __len__(self): + return self._od.__len__() + + def __iter__(self): + return self._od.__iter__() - class Parser(ast.NodeVisitor): """Basic parser. @@ -106,7 +123,7 @@ def generic_visit(self, node): node (ast.Node): The node which is visited. """ pass # This ignore is necessary to keep the parser at base level, also look comment above in - # the visit_Module function body. + # the visit_Module function body. def visit_each(self, lst): for el in lst: @@ -115,11 +132,12 @@ def visit_each(self, lst): @staticmethod def get_target_vars(target): get_id = lambda n: n.id if not isinstance(n, ast.Starred) else n.value.id - if isinstance(target, (ast.Name, ast.Starred)): + if isinstance(target, (ast.Name, ast.Starred)): tv = [get_id(target)] - elif isinstance(target, ast.Tuple): + elif isinstance(target, ast.Tuple): tv = [get_id(node) for node in target.elts] - else: tv = [] + else: + tv = [] return TargetVars(tv) @@ -127,7 +145,7 @@ def get_target_vars(target): def get_arg(el): if el is None: return None - else : + else: return el.arg @staticmethod @@ -138,7 +156,7 @@ def get_arg_tuples(arguments, defaults): @staticmethod def get_arg_parts(arguments, defaults, type): - # only difference is that it doesn't pull out arg.arg, so we can + # only difference is that it doesn't pull out arg.arg, so we can # use all the information on the arg node down the road match_def = [None] * (len(arguments) - len(defaults)) + defaults part_list = [] @@ -149,18 +167,19 @@ def get_arg_parts(arguments, defaults, type): @staticmethod def get_arg_part(_arg, _def, type=None): # type is arg, kwonly, kwarg, vararg - if not _arg: return None + if not _arg: + return None # part uses default highlighting, so will highlight "node" entry return { - 'node': _def or _arg, - 'arg': _arg, - # TODO: need to fill out - 'type': type, - 'is_default': True if _def else False, - 'name': _arg.arg, - 'annotation': _arg.annotation - } + "node": _def or _arg, + "arg": _arg, + # TODO: need to fill out + "type": type, + "is_default": True if _def else False, + "name": _arg.arg, + "annotation": _arg.annotation, + } # class OperatorParser(Parser): @@ -261,8 +280,8 @@ class FunctionParser(Parser): """ def __init__(self): - self.gen_name = '' - self.raw_name = '' + self.gen_name = "" + self.raw_name = "" self.mappings = {} self.out = {} self.call_lookup_active = False @@ -278,6 +297,7 @@ def visit_AugAssign(self, node): self.visit(node.value) def visit_Compare(self, node): + self.visit(node.left) self.visit_each(node.comparators) def visit_UnaryOp(self, node): @@ -288,7 +308,7 @@ def visit_Import(self, node): if imp.asname is not None: self.mappings[imp.asname] = imp.name else: - pass # e.g. numpy import as numpy, so no action needed. + pass # e.g. numpy import as numpy, so no action needed. def visit_ImportFrom(self, node): for imp in node.names: @@ -298,24 +318,26 @@ def visit_Expr(self, node): self.visit(node.value) def visit_List(self, node): - [ self.visit(el) for el in node.elts ] + [self.visit(el) for el in node.elts] def visit_Dict(self, node): - [ self.visit(el) for el in node.values ] + [self.visit(el) for el in node.values] def visit_Call(self, node): if self.call_lookup_active: self.visit(node.func) - else : + else: self.call_lookup_active = True - self.visit(node.func) # Need to visit func to start recording the current function name. + self.visit( + node.func + ) # Need to visit func to start recording the current function name. if self.gen_name: - if (self.gen_name not in self.out): + if self.gen_name not in self.out: self.out[self.gen_name] = [] self.out[self.gen_name].append(self.get_call_part(node)) - #self.out[self.current].append((node, node.args, node.keywords)) + # self.out[self.current].append((node, node.args, node.keywords)) self.gen_name = self.raw_name = "" self.call_lookup_active = False @@ -341,44 +363,46 @@ def visit_Subscript(self, node): def visit_Name(self, node): self.gen_name = self.mappings.get(node.id) or node.id self.raw_name = node.id - + def get_call_part(self, node): args = [self.get_pos_arg_part(n, ii) for ii, n in enumerate(node.args)] keywords = [self.get_kw_arg_part(n) for n in node.keywords] - return {'node': node, - # TODO: right now, args and keywords can be indexed by pos or name. - # Note that a pos args name is its position. - # Problems will arise if SCT tests a position, but the submission - # has too few positional arguments, since it will then grab a kw arg :( - # This is not necessarily a bad thing, but instructors would need to be - # Careful deciding when to test a pos arg, and when to test using kw. - # Could use check_pos_args with pos_args entry below to solve. - 'args': IndexedDict((n['name'], n) for n in [*args, *keywords]), - #'pos_args': args, - #'keywords': keywords, - 'name': self.raw_name - } - - def get_pos_arg_part(self, arg, indx_pos): + return { + "node": node, + # TODO: right now, args and keywords can be indexed by pos or name. + # Note that a pos args name is its position. + # Problems will arise if SCT tests a position, but the submission + # has too few positional arguments, since it will then grab a kw arg :( + # This is not necessarily a bad thing, but instructors would need to be + # Careful deciding when to test a pos arg, and when to test using kw. + # Could use check_pos_args with pos_args entry below to solve. + "args": IndexedDict((n["name"], n) for n in [*args, *keywords]), + #'pos_args': args, + #'keywords': keywords, + "name": self.raw_name, + } + + @staticmethod + def get_pos_arg_part(arg, indx_pos): is_star = isinstance(arg, ast.Starred) return { - 'node': arg if not is_star else arg.value, - 'highlight': arg, - 'type': 'argument', - 'is_starred': is_star, - 'name': indx_pos - } - - def get_kw_arg_part(self, arg): + "node": arg if not is_star else arg.value, + "highlight": arg, + "type": "argument", + "is_starred": is_star, + "name": indx_pos, + } + + @staticmethod + def get_kw_arg_part(arg): is_kwarg = arg.arg is None return { - 'node': arg.value, - 'highlight': arg, - 'type': 'keyword', - 'is_kwarg': is_kwarg, - 'name': arg.arg - } - + "node": arg.value, + "highlight": arg, + "type": "keyword", + "is_kwarg": is_kwarg, + "name": arg.arg, + } class ObjectAccessParser(FunctionParser): @@ -416,12 +440,13 @@ def visit_Name(self, node): # if name refers to an import, replace prefix = self.mappings.get(node.id) or node.id - self.gen_name = prefix + "." + self.gen_name if self.gen_name else prefix + self.gen_name = prefix + "." + self.gen_name if self.gen_name else prefix self.raw_name = node.id + "." + self.raw_name if self.raw_name else node.id self.out.append(self.gen_name) self.gen_name = self.raw_name = "" + class ObjectAssignmentParser(Parser): """Find object assignmnts @@ -438,8 +463,7 @@ def visit_Name(self, node): if node.id not in self.out: self.out[node.id] = self.get_part(node, self.active_assignment) else: - self.out[node.id]['highlight'] = None - self.out[node.id]['assignments'].append(self.active_assignment) + self.out[node.id]["highlight"] = None self.active_assignment = None def visit_Attribute(self, node): @@ -472,21 +496,13 @@ def visit_Try(self, node): self.visit_each(node.body) self.visit_each(node.finalbody) - def visit_TryFinally(self, node): - self.visit_each(node.body) - self.visit_each(node.finalbody) - @staticmethod def get_part(name_node, ass_node=None): # either name node or simply str or name itself - name = getattr(name_node, 'id', name_node) + name = getattr(name_node, "id", name_node) load_name = ast.Name(id=name, ctx=ast.Load()) ast.fix_missing_locations(load_name) - return {'name': name, - 'node': load_name, - 'highlight': ass_node or name_node, - 'assignments': [] if not ass_node else [ass_node] - } + return {"name": name, "node": load_name, "highlight": ass_node or name_node} class IfParser(Parser): @@ -500,12 +516,9 @@ def __init__(self): self.out = [] def visit_If(self, node): - self.out.append({ - 'node': node, - 'test': node.test, - 'body': node.body, - 'orelse': node.orelse, - }) + self.out.append( + {"node": node, "test": node.test, "body": node.body, "orelse": node.orelse} + ) class IfExpParser(IfParser): @@ -515,7 +528,8 @@ class IfExpParser(IfParser): Only 'top-level' if structures will be found! """ - def visit_If(self, node): return + def visit_If(self, node): + return def visit_IfExp(self, node): super().visit_If(node) @@ -536,15 +550,6 @@ def visit_Compare(self, node): def visit_UnaryOp(self, node): self.visit(node.operand) - def visit_Expr(self, node): - self.visit(node.value) - - def visit_Call(self, node): - self.visit(node.func) - - def visit_Return(self, node): - self.visit(node.value) - class WhileParser(Parser): """Find while structures. @@ -557,12 +562,9 @@ def __init__(self): self.out = [] def visit_While(self, node): - self.out.append({ - 'node': node, - 'test': node.test, - 'body': node.body, - 'orelse': node.orelse - }) + self.out.append( + {"node": node, "test": node.test, "body": node.body, "orelse": node.orelse} + ) class ForParser(Parser): @@ -577,14 +579,31 @@ def __init__(self): def visit_For(self, node): tv = Parser.get_target_vars(node.target) - self.out.append({ - 'node': node, - 'iter': node.iter, - 'body': {'node': node.body, 'target_vars': tv}, - 'orelse': {'node': node.orelse, 'target_vars': tv}, - 'target': node.target, - '_target_vars': tv - }) + self.out.append( + { + "node": node, + "iter": node.iter, + "body": {"node": node.body, "target_vars": tv}, + "orelse": {"node": node.orelse, "target_vars": tv}, + "target": node.target, + "_target_vars": tv, + } + ) + + +class ClassDefParser(Parser): + """Find class definitions + """ + + def __init__(self): + self.out = {} + + def visit_ClassDef(self, node): + self.out[node.name] = { + "node": node, + "bases": [{"node": node} for node in node.bases], + "body": node.body, + } class FunctionDefParser(Parser): @@ -593,46 +612,50 @@ class FunctionDefParser(Parser): A parser which inherits from the basic parser to find function definitions. Only 'top-level' for structures will be found! """ + def __init__(self): self.out = {} def visit_FunctionDef(self, node): self.out[node.name] = self.parse_node(node) - @classmethod def parse_node(cls, node): normal_args = cls.get_arg_tuples(node.args.args, node.args.defaults) kwonlyargs = cls.get_arg_tuples(node.args.kwonlyargs, node.args.kw_defaults) # TODO: all single args should be tuples like this vararg = cls.get_arg(node.args.vararg) - kwarg = cls.get_arg(node.args.kwarg) + kwarg = cls.get_arg(node.args.kwarg) # create context variables target_vars = [arg[0] for arg in normal_args] - if vararg: target_vars.append(vararg) - if kwarg: target_vars.append(kwarg) - - args = cls.get_arg_parts(node.args.args, node.args.defaults, 'arg') - kw_args = cls.get_arg_parts(node.args.kwonlyargs, node.args.kw_defaults, 'kwonly') - varargs = cls.get_arg_part(node.args.vararg, None, 'vararg') - kwargs = cls.get_arg_part(node.args.kwarg, None, 'kwarg') + if vararg: + target_vars.append(vararg) + if kwarg: + target_vars.append(kwarg) + + args = cls.get_arg_parts(node.args.args, node.args.defaults, "arg") + kw_args = cls.get_arg_parts( + node.args.kwonlyargs, node.args.kw_defaults, "kwonly" + ) + varargs = cls.get_arg_part(node.args.vararg, None, "vararg") + kwargs = cls.get_arg_part(node.args.kwarg, None, "kwarg") all_args = [*args, varargs, *kw_args, kwargs] - - if isinstance(node, ast.Lambda): body_node = node.body + + if isinstance(node, ast.Lambda): + body_node = node.body else: bodyMod = wrap_in_module(node.body) body_node = FunctionBodyTransformer().visit(bodyMod) return { "node": node, - "name": getattr(node, 'name', None), - "args": IndexedDict([ (p['name'], p) for p in all_args if p is not None]), + "name": getattr(node, "name", None), + "args": IndexedDict([(p["name"], p) for p in all_args if p is not None]), # TODO: arg is the node counterpart to target_vars "_spec1_args": args, "*args": varargs, - "**kwargs": kwargs, - "body": {'node': body_node, - 'target_vars': TargetVars(target_vars)} + "**kwargs": kwargs, + "body": {"node": body_node, "target_vars": TargetVars(target_vars)}, } @@ -658,29 +681,6 @@ def visit_Call(self, node): for key in node.keywords: self.visit(key.value) - def visit_If(self, node): - self.visit_each(node.body) - self.visit_each(node.orelse) - - def visit_While(self, node): - self.visit_each(node.body) - self.visit_each(node.orelse) - - def visit_For(self, node): - self.visit_each(node.body) - self.visit_each(node.orelse) - - def visit_With(self, node): - self.visit_each(node.body) - - def visit_Try(self, node): - self.visit_each(node.body) - self.visit_each(node.finalbody) - - def visit_TryFinally(self, node): - self.visit_each(node.body) - self.visit_each(node.finalbody) - def visit_Lambda(self, node): self.out.append(FunctionDefParser.parse_node(node)) @@ -695,34 +695,30 @@ def visit_Assign(self, node): def visit_AugAssign(self, node): self.visit(node.value) - def visit_Try(self, node): - self.visit_each(node.body) - self.visit_each(node.finalbody) - - def visit_TryFinally(self, node): - self.visit_each(node.body) - self.visit_each(node.finalbody) - def build_comp(self, node): target = node.generators[0].target tv = Parser.get_target_vars(target) ifs = node.generators[0].ifs - self.out.append({ + self.out.append( + { "node": node, - "body": {'node': node.elt, 'target_vars': tv}, + "body": {"node": node.elt, "target_vars": tv}, "target": target, "iter": node.generators[0].iter, - "ifs": [{'node': ifnode, 'target_vars': tv} for ifnode in ifs], + "ifs": [{"node": ifnode, "target_vars": tv} for ifnode in ifs], # TODO: 'private' _target_vars, since it shouldn't be set when selecting node, # see remarks in test_list_comp on rewriting - "_target_vars": tv - }) + "_target_vars": tv, + } + ) + class ListCompParser(CompParser): """Find list comprehensions A parser which inherits from the CompParser to find list comprehensions. """ + def visit_ListComp(self, node): self.build_comp(node) @@ -736,36 +732,40 @@ class GeneratorExpParser(CompParser): def visit_GeneratorExp(self, node): self.build_comp(node) + class DictCompParser(CompParser): """Find dictionary comprehensions A parser which inherits from the CompParser to find dict comprehensions. """ + def visit_DictComp(self, node): target = node.generators[0].target tv = Parser.get_target_vars(target) ifs = node.generators[0].ifs - self.out.append({ + self.out.append( + { "node": node, - "key": {'node': node.key, 'target_vars': tv}, - "value": {'node': node.value, 'target_vars': tv}, + "key": {"node": node.key, "target_vars": tv}, + "value": {"node": node.value, "target_vars": tv}, "target": target, "iter": node.generators[0].iter, - "ifs": [{'node': ifnode, 'target_vars': tv} for ifnode in ifs], + "ifs": [{"node": ifnode, "target_vars": tv} for ifnode in ifs], # TODO: 'private' _target_vars, since it shouldn't be set when selecting node, # see remarks in test_list_comp on rewriting - "_target_vars": tv - }) + "_target_vars": tv, + } + ) class FunctionBodyTransformer(ast.NodeTransformer): # TODO this does not automatically contain line_end information! def visit_Nonlocal(self, node): - new_node = ast.copy_location(ast.Global(names = node.names), node) + new_node = ast.copy_location(ast.Global(names=node.names), node) return FunctionBodyTransformer.decorate(new_node, node) def visit_Return(self, node): - new_node = ast.copy_location(ast.Expr(value = node.value), node) + new_node = ast.copy_location(ast.Expr(value=node.value), node) return FunctionBodyTransformer.decorate(new_node, node) @staticmethod @@ -774,6 +774,7 @@ def decorate(new_node, node): new_node.last_token = node.last_token return new_node + class WithParser(Parser): def __init__(self): self.out = [] @@ -781,21 +782,26 @@ def __init__(self): def visit_With(self, node): items = node.items context = [ - {"node" : item.context_expr, - "target_vars": self.get_target_vars(item.optional_vars), - "with_items": item, - "highlight": node - } - for item in items] - - tv_all = TargetVars(sum([list(c['target_vars'].items()) for c in context], [])) - - self.out.append({ - "context": context, - "body": {'node': node.body, 'with_items': items}, - "node": node, - "n_vars": len(items) - }) + { + "node": item.context_expr, + "target_vars": self.get_target_vars(item.optional_vars), + "with_items": item, + "highlight": node, + } + for item in items + ] + + tv_all = TargetVars(sum([list(c["target_vars"].items()) for c in context], [])) + + self.out.append( + { + "context": context, + "body": {"node": node.body, "with_items": items}, + "node": node, + "n_vars": len(items), + } + ) + class TryExceptParser(Parser): def __init__(self): @@ -811,22 +817,35 @@ def visit_Try(self, node): handlers[el.id] = self.parse_handler(handler) else: # either general handler, or single error handler - k = 'all' if not handler.type else handler.type.id + k = "all" if not handler.type else self.get_identifier(handler.type) handlers[k] = self.parse_handler(handler) - self.out.append({ - "node": node, - "body": node.body, - "orelse": node.orelse or None, - "finalbody": node.finalbody or None, - "handlers": handlers, - }) + self.out.append( + { + "node": node, + "body": node.body, + "orelse": node.orelse or None, + "finalbody": node.finalbody or None, + "handlers": handlers, + } + ) @staticmethod - def parse_handler(handler): return { - 'node': handler.body, - 'target_vars': TargetVars([handler.name]) - } + def get_identifier(obj): + # if only name needs to be correct + return getattr(obj, "id", None) or getattr(obj, "attr") + # if full attr chain must match + # name = "" + # while hasattr(obj, "attr"): + # name = "." + obj.attr + name + # obj = obj.value + # name = obj.id + name + # return name + + @staticmethod + def parse_handler(handler): + return {"node": handler.body, "target_vars": TargetVars([handler.name])} + parser_dict = { "object_accesses": ObjectAccessParser, @@ -837,6 +856,7 @@ def parse_handler(handler): return { "if_exps": IfExpParser, "whiles": WhileParser, "for_loops": ForParser, + "class_defs": ClassDefParser, "function_defs": FunctionDefParser, "lambda_functions": LambdaFunctionParser, "list_comps": ListCompParser, @@ -844,5 +864,5 @@ def parse_handler(handler): return { "generator_exps": GeneratorExpParser, "withs": WithParser, "try_excepts": TryExceptParser, - "function_calls": FunctionParser + "function_calls": FunctionParser, } diff --git a/pythonwhat/probe.py b/pythonwhat/probe.py index dcb8a1af..9a041e4d 100644 --- a/pythonwhat/probe.py +++ b/pythonwhat/probe.py @@ -4,6 +4,8 @@ from functools import partial from collections import OrderedDict from pythonwhat import test_funcs +from pythonwhat.State import State +from pythonwhat.checks.check_wrappers import state_partial TEST_NAMES = [ "test_mc", @@ -13,7 +15,6 @@ "test_object", "test_correct", "test_if_else", - "test_if_exp", "test_for_loop", "test_function", "test_list_comp", @@ -26,21 +27,21 @@ "test_expression_result", "test_expression_output", "test_function_definition", - "test_object_after_expression" + "test_object_after_expression", ] SUB_TESTS = { - "test_if_else": ['test', 'body', 'orelse'], - "test_if_exp": ['test', 'body', 'orelse'], - "test_list_comp": ['comp_iter', 'body', 'ifs'], - "check_correct": ['check', 'diagnose'], - "test_for_loop": ['for_iter', 'body', 'orelse'], - "test_while_loop": ['test', 'body', 'orelse'], - "test_with": ['context_tests', 'body'], - "test_function_definition": ['body'], - "check_or": ['tests'] + "test_if_else": ["test", "body", "orelse"], + "test_list_comp": ["comp_iter", "body", "ifs"], + "check_correct": ["check", "diagnose"], + "test_for_loop": ["for_iter", "body", "orelse"], + "test_while_loop": ["test", "body", "orelse"], + "test_with": ["context_tests", "body"], + "test_function_definition": ["body"], + "check_or": ["tests"], } + class Tree(object): def __init__(self): """ @@ -56,10 +57,19 @@ def __init__(self): self.crnt_node = self.root @classmethod - def str_branch(cls, node, str_func=lambda s: ""): - f = node.data.get('func') - this_node = " "*node.depth + getattr(f, '__name__', node.name) + str_func(node) + "\n" - return this_node + "".join(map(lambda x: cls.str_branch(x, str_func), node.child_list)) + def str_branch(cls, node, str_func=lambda s: str(s)): + # dict(getattr(s.data.get("bound_args", {}), 'arguments', {})) + f = node.data.get("func") + this_node = ( + " " * node.depth + + "(" + + getattr(f, "__name__", node.name) + + str_func(node) + + ")\n" + ) + return this_node + "".join( + map(lambda x: cls.str_branch(x, str_func), node.child_list) + ) def __str__(self): return self.str_branch(self.crnt_node) @@ -71,10 +81,12 @@ def descend(self, node=None): return sum(children, base) def __iter__(self): - for ii in self.descend(self.crnt_node): yield ii + for ii in self.descend(self.crnt_node): + yield ii + class Node(object): - def __init__(self, child_list = None, data = None, name="unnamed", arg_name=""): + def __init__(self, child_list=None, data=None, name="unnamed", arg_name=""): """ Hold a function call with its bound arguments, along with child nodes. @@ -89,49 +101,55 @@ def __init__(self, child_list = None, data = None, name="unnamed", arg_name=""): def __call__(self, state=None): """Call original function with its arguments, and optional state""" - ba = self.data['bound_args'] + ba = self.data["bound_args"] if state: - self.data['func'](state=state, *ba.args, **ba.kwargs) + func = self.data["func"] + ba = inspect.signature(func).bind(state, *ba.args[1:], **ba.kwargs) + func(*ba.args, **ba.kwargs) return state else: - self.data['func'](*ba.args, **ba.kwargs) + self.data["func"](*ba.args, **ba.kwargs) ba.apply_defaults() - return ba.arguments['state'] + return ba.arguments["state"] def __str__(self): - # TODO print function signature without defaults (or with) - return pp.pformat(self.data) + return pp.pformat( + dict(getattr(self.data.get("bound_args", {}), "arguments", {})) + ) def __iter__(self): - for c in self.child_list: yield c + for c in self.child_list: + yield c def partial(self): """Return partial of original function call""" - ba = self.data['bound_args'] - return partial(self.data['func'], *ba.args, **ba.kwargs) + ba = self.data["bound_args"] + return state_partial(self.data["func"], *ba.args[1:], **ba.kwargs) def update_child_calls(self): """Replace child nodes on original function call with their partials""" for node in filter(lambda n: len(n.arg_name), self.child_list): - self.data['bound_args'].arguments[node.arg_name] = node.partial() + self.data["bound_args"].arguments[node.arg_name] = node.partial() self.updated = True def remove_child(self, node): - indx = self.child_list.index(node) - del self.child_list[indx] - return indx + index = self.child_list.index(node) + del self.child_list[index] + return index def add_child(self, child): - # since it is a tree, there is only one parent + # since it is a tree, there is only one parent # note this means we do not allow edges between same layer units - if child.parent: child.parent.remove_child(child) + if child.parent: + child.parent.remove_child(child) child.parent = self self.child_list.append(child) def descend(self, include_me=True): """Descend depth first into all child nodes""" - if include_me: yield self + if include_me: + yield self for child in self.child_list: yield child @@ -139,8 +157,11 @@ def descend(self, include_me=True): @property def depth(self): - if self.parent: return self.parent.depth + 1 - else: return 0 + if self.parent: + return self.parent.depth + 1 + else: + return 0 + class NodeList(Node): def partial(self): @@ -149,12 +170,6 @@ def partial(self): def update_child_calls(self): pass -class NodeDict(Node): - def partial(self): - return OrderedDict((node.arg_name, node.partial()) for node in self.child_list) - - def update_child_calls(self): - pass class Probe(object): def __init__(self, tree, f, eval_on_call=False): @@ -164,7 +179,7 @@ def __init__(self, tree, f, eval_on_call=False): self.eval_on_call = eval_on_call # TODO: auto sub_test detection self.sub_tests = SUB_TESTS.get(self.test_name) or [] - + def __call__(self, *args, **kwargs): """Bind arguments to original function signature, and store in a Node @@ -173,44 +188,46 @@ def __call__(self, *args, **kwargs): instances are assembled into a tree. """ + if (len(args) > 0 and not isinstance(args[0], State)) or len(args) == 0: + # no state placeholder if a state is passed + args = ["state_placeholder"] + list(args) bound_args = inspect.signature(self.f).bind(*args, **kwargs) - data = dict( - bound_args = bound_args, - func = self.f) + data = dict(bound_args=bound_args, func=self.f) this_node = Node(data=data, name=self.test_name) if self.tree is not None: self.tree.crnt_node.add_child(this_node) # First pass to set up branches off node - da = bound_args.arguments - for st in self.sub_tests: # TODO: auto sub test detection - if st in da and da[st]: - self.build_sub_test_nodes(da[st], self.tree, this_node, st) + arguments = bound_args.arguments + for subtest in self.sub_tests: # TODO: auto sub test detection + if subtest in arguments and arguments[subtest]: + self.build_sub_test_nodes( + arguments[subtest], self.tree, this_node, subtest + ) # Second pass to build node and all its children into a subtest for node in this_node.descend(include_me=True): - if node.updated: # already built, e.g. node used multiple times + if node.updated: # already built, e.g. node used multiple times continue else: node.update_child_calls() - - if self.eval_on_call: return this_node() - else: return this_node + + if self.eval_on_call: + return this_node() + else: + return this_node @staticmethod def build_sub_test_nodes(test, tree, node, arg_name): # note that I've made the strong assumption that # if not a function, then test is a dict, list or tuple of them - if isinstance(test, dict): - nd = NodeDict(name = "Dict", arg_name = arg_name) - node.add_child(nd) - for k, f in test.items(): Probe.build_sub_test_nodes(f, tree, nd, k) - elif isinstance(test, (list, tuple)): - nl = NodeList(name = "List", arg_name = arg_name) + if isinstance(test, (list, tuple)): + nl = NodeList(name="List", arg_name=arg_name) node.add_child(nl) - for ii, f in enumerate(test): Probe.build_sub_test_nodes(f, tree, nl, str(ii)) + for ii, f in enumerate(test): + Probe.build_sub_test_nodes(f, tree, nl, str(ii)) elif isinstance(test, Node): # test was a lambdaless subtest call, which produced a node # so need to tell it what its arg_name was on parent test @@ -219,7 +236,7 @@ def build_sub_test_nodes(test, tree, node, arg_name): elif callable(test): # test was inside a lambda, function containing subtests or v2 F() chain object with subtests # since either may contain multiple subtests, we put them in a node list - nl = NodeList(name = "ListDeferred", arg_name = arg_name) + nl = NodeList(name="ListDeferred", arg_name=arg_name) node.add_child(nl) if tree is not None: prev_node, tree.crnt_node = tree.crnt_node, nl diff --git a/pythonwhat/sct_syntax.py b/pythonwhat/sct_syntax.py new file mode 100644 index 00000000..2ff2af6c --- /dev/null +++ b/pythonwhat/sct_syntax.py @@ -0,0 +1,70 @@ +from protowhat.sct_syntax import EagerChain, ExGen, LazyChainStart, state_dec_gen, LazyChain +from pythonwhat.checks.check_wrappers import scts +from pythonwhat.State import State +from pythonwhat.probe import Node, Probe, TEST_NAMES +from pythonwhat.utils import include_v1 +from pythonwhat import test_funcs +from functools import wraps + +# TODO: could define scts for check_wrappers at the module level +sct_dict = scts.copy() + + +def multi_dec(f): + """Decorator for multi to remove nodes for original test functions from root node""" + + @wraps(f) + def wrapper(*args, **kwargs): + args = ( + args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args + ) + for arg in args: + if isinstance(arg, Node) and arg.parent.name == "root": + arg.parent.remove_child(arg) + arg.update_child_calls() + return f(*args, **kwargs) + + return wrapper + + +state_dec = state_dec_gen(sct_dict) + +# todo: __all__? +assert ExGen +assert LazyChainStart + + +def Ex(state=None): + return EagerChain(state=state or State.root_state, chainable_functions=sct_dict) + + +def F(): + return LazyChain(chainable_functions=sct_dict) + + +def get_chains(): + return { + "Ex": ExGen(sct_dict, State.root_state), + "F": LazyChainStart(sct_dict), + } + + +if include_v1(): + # Prepare SCTs that may be chained attributes ---------------------- + # decorate functions that may try to run test_* function nodes as subtests + # so they remove those nodes from the tree + for k in ["multi", "with_context"]: + sct_dict[k] = multi_dec(sct_dict[k]) + + # allow test_* functions as chained attributes + for k in TEST_NAMES: + sct_dict[k] = Probe(tree=None, f=getattr(test_funcs, k), eval_on_call=True) + + # original logical test_* functions behave like multi + # this is necessary to allow them to take check_* funcs as args + # since probe behavior will try to call all SCTs passed (assuming they're also probes) + for k in ["test_or", "test_correct"]: + sct_dict[k] = multi_dec(getattr(test_funcs, k)) + +# Prepare check_funcs to be used alone (e.g. test = check_with().check_body()) +v2_check_functions = {k: state_dec(v) for k, v in scts.items()} diff --git a/pythonwhat/signatures.py b/pythonwhat/signatures.py index 8004eb34..94ffaa72 100644 --- a/pythonwhat/signatures.py +++ b/pythonwhat/signatures.py @@ -3,132 +3,175 @@ import pythonwhat from pythonwhat.tasks import getSignatureFromObjInProcess + def sig_from_params(*args): - return(inspect.Signature(list(args))) + return inspect.Signature(list(args)) + def sig_from_obj(obj_char): - return getSignatureFromObjInProcess(obj_char, - pythonwhat.State.State.root_state.solution_process) + return getSignatureFromObjInProcess( + obj_char, pythonwhat.State.State.root_state.solution_process + ) + def get_manual_sigs(): manual_sigs = { # builtins - 'abs': [param('x', param.POSITIONAL_ONLY)], - 'all': [param('iterable', param.POSITIONAL_ONLY)], - 'any': [param('iterable', param.POSITIONAL_ONLY)], - 'ascii': [param('obj', param.POSITIONAL_ONLY)], - 'bin': [param('number', param.POSITIONAL_ONLY)], - 'bool': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'chr': [param('i', param.POSITIONAL_ONLY)], - 'callable': [param('obj', param.POSITIONAL_ONLY)], - 'classmethod': [param('function', param.POSITIONAL_ONLY)], - 'complex': [param('imag', param.POSITIONAL_OR_KEYWORD, default=0), - param('real', param.POSITIONAL_OR_KEYWORD, default=0)], - 'delattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY)], - 'dir': [param('object', param.POSITIONAL_OR_KEYWORD, default=None)], - 'divmod': [param('x', param.POSITIONAL_ONLY), - param('y', param.POSITIONAL_ONLY)], - 'enumerate': [param('iterable', param.POSITIONAL_ONLY), - param('start', param.POSITIONAL_OR_KEYWORD, default=0)], - 'float': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'getattr': [param('object', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY), - param('default', param.POSITIONAL_ONLY, default=None)], - 'hasattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY)], - 'hash': [param('obj', param.POSITIONAL_ONLY)], - 'hex': [param('number', param.POSITIONAL_ONLY)], - 'id': [param('obj', param.POSITIONAL_ONLY)], - 'int': [param('x', param.POSITIONAL_OR_KEYWORD), - param('base', param.POSITIONAL_OR_KEYWORD, default=10)], - 'isinstance': [param('obj', param.POSITIONAL_ONLY), - param('class_or_tuple', param.POSITIONAL_ONLY)], - 'issubclass': [param('cls', param.POSITIONAL_ONLY), - param('class_or_tuple', param.POSITIONAL_ONLY)], - 'list': [param('iterable', param.POSITIONAL_ONLY, default=None)], - 'len': [param('obj', param.POSITIONAL_ONLY)], - 'oct': [param('number', param.POSITIONAL_ONLY)], - 'open': [param('file', param.POSITIONAL_OR_KEYWORD), - param('mode', param.POSITIONAL_OR_KEYWORD, default='r'), - param('buffering', param.POSITIONAL_OR_KEYWORD, default=1), - param('encoding', param.POSITIONAL_OR_KEYWORD, default=None), - param('errors', param.POSITIONAL_OR_KEYWORD, default=None), - param('newline', param.POSITIONAL_OR_KEYWORD, default=None), - param('closefd', param.POSITIONAL_OR_KEYWORD, default=None), - param('opener', param.POSITIONAL_OR_KEYWORD, default=None)], - 'ord': [param('c', param.POSITIONAL_ONLY)], - 'pow': [param('x', param.POSITIONAL_ONLY), - param('y', param.POSITIONAL_ONLY), - param('z', param.POSITIONAL_ONLY, default=None)], - 'print': [param('value', param.POSITIONAL_ONLY)], - 'repr': [param('obj', param.POSITIONAL_ONLY)], - 'reversed': [param('sequence', param.POSITIONAL_ONLY)], - 'round': [param('number', param.POSITIONAL_OR_KEYWORD), - param('ndigits', param.POSITIONAL_OR_KEYWORD, default=0)], - 'set': [param('iterable', param.POSITIONAL_ONLY, default=None)], - + "abs": [param("x", param.POSITIONAL_ONLY)], + "all": [param("iterable", param.POSITIONAL_ONLY)], + "any": [param("iterable", param.POSITIONAL_ONLY)], + "ascii": [param("obj", param.POSITIONAL_ONLY)], + "bin": [param("number", param.POSITIONAL_ONLY)], + "bool": [param("x", param.POSITIONAL_OR_KEYWORD)], + "chr": [param("i", param.POSITIONAL_ONLY)], + "callable": [param("obj", param.POSITIONAL_ONLY)], + "classmethod": [param("function", param.POSITIONAL_ONLY)], + "complex": [ + param("imag", param.POSITIONAL_OR_KEYWORD, default=0), + param("real", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "delattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + ], + "dir": [param("object", param.POSITIONAL_OR_KEYWORD, default=None)], + "divmod": [ + param("x", param.POSITIONAL_ONLY), + param("y", param.POSITIONAL_ONLY), + ], + "enumerate": [ + param("iterable", param.POSITIONAL_ONLY), + param("start", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "float": [param("x", param.POSITIONAL_OR_KEYWORD)], + "getattr": [ + param("object", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + param("default", param.POSITIONAL_ONLY, default=None), + ], + "hasattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + ], + "hash": [param("obj", param.POSITIONAL_ONLY)], + "hex": [param("number", param.POSITIONAL_ONLY)], + "id": [param("obj", param.POSITIONAL_ONLY)], + "int": [ + param("x", param.POSITIONAL_OR_KEYWORD), + param("base", param.POSITIONAL_OR_KEYWORD, default=10), + ], + "isinstance": [ + param("obj", param.POSITIONAL_ONLY), + param("class_or_tuple", param.POSITIONAL_ONLY), + ], + "issubclass": [ + param("cls", param.POSITIONAL_ONLY), + param("class_or_tuple", param.POSITIONAL_ONLY), + ], + "list": [param("iterable", param.POSITIONAL_ONLY, default=None)], + "len": [param("obj", param.POSITIONAL_ONLY)], + "oct": [param("number", param.POSITIONAL_ONLY)], + "open": [ + param("file", param.POSITIONAL_OR_KEYWORD), + param("mode", param.POSITIONAL_OR_KEYWORD, default="r"), + param("buffering", param.POSITIONAL_OR_KEYWORD, default=1), + param("encoding", param.POSITIONAL_OR_KEYWORD, default=None), + param("errors", param.POSITIONAL_OR_KEYWORD, default=None), + param("newline", param.POSITIONAL_OR_KEYWORD, default=None), + param("closefd", param.POSITIONAL_OR_KEYWORD, default=None), + param("opener", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "ord": [param("c", param.POSITIONAL_ONLY)], + "pow": [ + param("x", param.POSITIONAL_ONLY), + param("y", param.POSITIONAL_ONLY), + param("z", param.POSITIONAL_ONLY, default=None), + ], + "print": [param("value", param.POSITIONAL_ONLY)], + "repr": [param("obj", param.POSITIONAL_ONLY)], + "reversed": [param("sequence", param.POSITIONAL_ONLY)], + "round": [ + param("number", param.POSITIONAL_OR_KEYWORD), + param("ndigits", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "set": [param("iterable", param.POSITIONAL_ONLY, default=None)], # Difference v3.4 vs v3.5!!! - 'setattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY), - param('value', param.POSITIONAL_ONLY)], - 'sorted': [param('iterable', param.POSITIONAL_ONLY), - param('key', param.POSITIONAL_OR_KEYWORD, default=None), - param('reverse', param.POSITIONAL_OR_KEYWORD, default=False)], - 'str': [param('object', param.POSITIONAL_OR_KEYWORD)], - 'sum': [param('iterable', param.POSITIONAL_ONLY), - param('start', param.POSITIONAL_ONLY, default=0)], - 'tuple': [param('iterable', param.POSITIONAL_ONLY, default=None)], - 'type': [param('object', param.POSITIONAL_ONLY)], - 'vars': [param('object', param.POSITIONAL_ONLY)], - + "setattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + param("value", param.POSITIONAL_ONLY), + ], + "sorted": [ + param("iterable", param.POSITIONAL_ONLY), + param("key", param.POSITIONAL_OR_KEYWORD, default=None), + param("reverse", param.POSITIONAL_OR_KEYWORD, default=False), + ], + "str": [param("object", param.POSITIONAL_OR_KEYWORD)], + "sum": [ + param("iterable", param.POSITIONAL_ONLY), + param("start", param.POSITIONAL_ONLY, default=0), + ], + "tuple": [param("iterable", param.POSITIONAL_ONLY, default=None)], + "type": [param("object", param.POSITIONAL_ONLY)], + "vars": [param("object", param.POSITIONAL_ONLY)], # int - # str - 'str.center': [param('width', param.POSITIONAL_ONLY), - param('fillchar', param.POSITIONAL_ONLY, default=" ")], - + "str.center": [ + param("width", param.POSITIONAL_ONLY), + param("fillchar", param.POSITIONAL_ONLY, default=" "), + ], # list - 'list.append': [param('object', param.POSITIONAL_ONLY)], - 'list.count': [param('value', param.POSITIONAL_ONLY)], - + "list.append": [param("object", param.POSITIONAL_ONLY)], + "list.count": [param("value", param.POSITIONAL_ONLY)], # dict - # numpy - 'numpy.array': [param('object', param.POSITIONAL_OR_KEYWORD), - param('dtype', param.POSITIONAL_OR_KEYWORD, default=None), - param('copy', param.POSITIONAL_OR_KEYWORD, default=True), - param('order', param.POSITIONAL_OR_KEYWORD, default=None), - param('subok', param.POSITIONAL_OR_KEYWORD, default=False), - param('ndmin', param.POSITIONAL_OR_KEYWORD, default=0)], - 'numpy.random.seed': [param('seed', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.rand': [param('d0', param.POSITIONAL_ONLY, default=None), - param('d1', param.POSITIONAL_ONLY, default=None), - param('d2', param.POSITIONAL_ONLY, default=None), - param('d3', param.POSITIONAL_ONLY, default=None), - param('d4', param.POSITIONAL_ONLY, default=None), - param('d5', param.POSITIONAL_ONLY, default=None), - param('d6', param.POSITIONAL_ONLY, default=None)], - 'numpy.random.randint': [param('low', param.POSITIONAL_OR_KEYWORD), - param('high', param.POSITIONAL_OR_KEYWORD, default=None), - param('size', param.POSITIONAL_OR_KEYWORD, default=None), - param('dtype', param.POSITIONAL_OR_KEYWORD, default='l')], - 'numpy.random.choice': [param('a', param.POSITIONAL_OR_KEYWORD), - param('size', param.POSITIONAL_OR_KEYWORD, default=None), - param('replace', param.POSITIONAL_OR_KEYWORD, default=True), - param('p', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.normal': [param('loc', param.POSITIONAL_OR_KEYWORD, default=0.0), - param('scale', param.POSITIONAL_OR_KEYWORD, default=1.0), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.poisson': [param('lam', param.POSITIONAL_OR_KEYWORD, default=1.0), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.binomial': [param('n', param.POSITIONAL_OR_KEYWORD), - param('p', param.POSITIONAL_OR_KEYWORD), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.shuffle': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'numpy.random.permutation': [param('x', param.POSITIONAL_OR_KEYWORD)], - + "numpy.array": [ + param("object", param.POSITIONAL_OR_KEYWORD), + param("dtype", param.POSITIONAL_OR_KEYWORD, default=None), + param("copy", param.POSITIONAL_OR_KEYWORD, default=True), + param("order", param.POSITIONAL_OR_KEYWORD, default=None), + param("subok", param.POSITIONAL_OR_KEYWORD, default=False), + param("ndmin", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "numpy.random.seed": [param("seed", param.POSITIONAL_OR_KEYWORD, default=None)], + "numpy.random.rand": [ + param("d0", param.POSITIONAL_ONLY, default=None), + param("d1", param.POSITIONAL_ONLY, default=None), + param("d2", param.POSITIONAL_ONLY, default=None), + param("d3", param.POSITIONAL_ONLY, default=None), + param("d4", param.POSITIONAL_ONLY, default=None), + param("d5", param.POSITIONAL_ONLY, default=None), + param("d6", param.POSITIONAL_ONLY, default=None), + ], + "numpy.random.randint": [ + param("low", param.POSITIONAL_OR_KEYWORD), + param("high", param.POSITIONAL_OR_KEYWORD, default=None), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + param("dtype", param.POSITIONAL_OR_KEYWORD, default="l"), + ], + "numpy.random.choice": [ + param("a", param.POSITIONAL_OR_KEYWORD), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + param("replace", param.POSITIONAL_OR_KEYWORD, default=True), + param("p", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.normal": [ + param("loc", param.POSITIONAL_OR_KEYWORD, default=0.0), + param("scale", param.POSITIONAL_OR_KEYWORD, default=1.0), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.poisson": [ + param("lam", param.POSITIONAL_OR_KEYWORD, default=1.0), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.binomial": [ + param("n", param.POSITIONAL_OR_KEYWORD), + param("p", param.POSITIONAL_OR_KEYWORD), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.shuffle": [param("x", param.POSITIONAL_OR_KEYWORD)], + "numpy.random.permutation": [param("x", param.POSITIONAL_OR_KEYWORD)], # others - 'math.radians': [param('x', param.POSITIONAL_ONLY)] + "math.radians": [param("x", param.POSITIONAL_ONLY)], } - return(manual_sigs) + return manual_sigs diff --git a/pythonwhat/tasks.py b/pythonwhat/tasks.py index cfff1d57..3516ff7b 100644 --- a/pythonwhat/tasks.py +++ b/pythonwhat/tasks.py @@ -1,5 +1,4 @@ from pythonwhat import utils -import os import dill import pickle import pythonwhat @@ -9,37 +8,48 @@ from pickle import PicklingError from pythonwhat.utils_env import set_context_vals, assign_from_ast from contextlib import contextmanager -from functools import partial, wraps, update_wrapper +from functools import partial, wraps +from protowhat.failure import InstructorError + + +# Shell is passed as a parameter to partially applied functions in executeTask +# Process is passed as a parameter in SCT function + def process_task(f): """Decorator to (optionally) run function in a process.""" sig = inspect.signature(f) + @wraps(f) def wrapper(*args, **kwargs): # get bound arguments for call ba = sig.bind_partial(*args, **kwargs) # when process is specified, remove from args and use to execute - process = ba.arguments.get('process') + process = ba.arguments.get("process") if process: - ba.arguments['process'] = None + ba.arguments["process"] = None # partial function since shell argument may have been left # unspecified, as it will be passed when the process executes pf = partial(wrapper, *ba.args, **ba.kwargs) return process.executeTask(pf) # otherwise, run original function return f(*ba.args, **ba.kwargs) + return wrapper + def get_env(ns): - if '__env__' in ns: - return ns['__env__'] + if "__env__" in ns: + return ns["__env__"] else: return ns + @contextmanager def capture_output(): import sys from io import StringIO + oldout, olderr = sys.stdout, sys.stderr out = [StringIO(), StringIO()] sys.stdout, sys.stderr = out @@ -54,44 +64,47 @@ def capture_output(): def getOptionFromProcess(process, name, shell): return shell.user_ns[name] + # Is a variable is defined in the process? @process_task def isDefinedInProcess(name, process, shell): return name in get_env(shell.user_ns) + # Is a variable is of a certain class in the process? @process_task def isInstanceInProcess(name, klass, process, shell): return isinstance(get_env(shell.user_ns)[name], klass) + # Get the columns of a Pandas data frame in the process @process_task def getColumnsInProcess(name, process, shell): - try: - return list(get_env(shell.user_ns)[name].columns) - except: - return None + return list(get_env(shell.user_ns)[name].columns) + # Is a key defined in a collection in the process? @process_task def isDefinedCollInProcess(name, key, process, shell): return key in get_env(shell.user_ns)[name] + # Get the signature of a function inside the process + def get_signature(name, mapped_name, signature, manual_sigs, env): if isinstance(signature, str): if signature in manual_sigs: signature = inspect.Signature(manual_sigs[signature]) else: - raise ValueError('signature error - specified signature not found') + raise InstructorError.from_message("signature error - specified signature not found") if signature is None: # establish function try: fun = eval(mapped_name, env) except: - raise ValueError("%s() was not found." % mapped_name) + raise InstructorError.from_message("%s() was not found." % mapped_name) # first go through manual sigs # try to get signature @@ -106,33 +119,36 @@ def get_signature(name, mapped_name, signature, manual_sigs, env): els[0] = type(eval(els[0], env)).__name__ generic_name = ".".join(els[:]) except: - raise ValueError('signature error - cannot convert call') + raise InstructorError.from_message("signature error - cannot convert call") if generic_name in manual_sigs: signature = inspect.Signature(manual_sigs[generic_name]) else: - raise ValueError('signature error - %s not in builtins' % generic_name) + raise InstructorError.from_message( + "signature error - %s not in builtins" % generic_name + ) else: - raise ValueError('manual signature not found') - except: + raise InstructorError.from_message("manual signature not found") + except Exception as e: try: signature = inspect.signature(fun) except: - raise ValueError('signature error - cannot determine signature') + raise InstructorError.from_message(e.args[0] + " and cannot determine signature") - return signature + params = [param.replace(annotation=inspect._empty) for param in signature.parameters.values()] + return signature.replace(parameters=params) # Get the signature of a function based on an object inside the process @process_task def getSignatureInProcess(name, mapped_name, signature, manual_sigs, process, shell): - try: - return get_signature(name = name, - mapped_name = mapped_name, - signature = signature, - manual_sigs = manual_sigs, - env = get_env(shell.user_ns)) - except: - return None + return get_signature( + name=name, + mapped_name=mapped_name, + signature=signature, + manual_sigs=manual_sigs, + env=get_env(shell.user_ns), + ) + @process_task def getSignatureFromObjInProcess(obj_char, process, shell): @@ -142,17 +158,17 @@ def getSignatureFromObjInProcess(obj_char, process, shell): return None - - # Stuff for test_with from contextlib import ExitStack + + def context_env_update(context_list, env): es = ExitStack() for item in context_list: # create context manager and enter - tmp_name = '__pw_cm' - cm_code = compile(ast.Expression(item.context_expr), '', 'eval') + tmp_name = "__pw_cm" + cm_code = compile(ast.Expression(item.context_expr), "", "eval") env[tmp_name] = es.enter_context(eval(cm_code, env)) # assign to its optional_vars in separte dict @@ -165,33 +181,35 @@ def context_env_update(context_list, env): @process_task def setUpNewEnvInProcess(context, process, shell): - shell.user_ns['__env__'] = utils.copy_env(shell.user_ns) + shell.user_ns["__env__"] = utils.copy_env(shell.user_ns) try: - es = context_env_update(context, shell.user_ns['__env__']) - shell.user_ns['__exit_stack__'] = es + es = context_env_update(context, shell.user_ns["__env__"]) + shell.user_ns["__exit_stack__"] = es return True except Exception as e: return e + # break down environment def context_objs_exit(es): try: es.close() - return False + return True except Exception as e: - raise e return e + @process_task def breakDownNewEnvInProcess(process, shell): try: - res = context_objs_exit(shell.user_ns['__exit_stack__']) - del shell.user_ns['__exit_stack__'] - del shell.user_ns['__env__'] + res = context_objs_exit(shell.user_ns["__exit_stack__"]) + del shell.user_ns["__exit_stack__"] + del shell.user_ns["__env__"] return res except: return False + # Tasks that may need to serialize across processes =========================== # Get a bytes or string representation of an object in the process @@ -204,10 +222,12 @@ def getClass(name, process, shell): except: return None + @process_task def convert(name, converter, process, shell): return dill.loads(converter)(get_env(shell.user_ns)[name]) + @process_task def getStreamPickle(name, process, shell): try: @@ -215,6 +235,7 @@ def getStreamPickle(name, process, shell): except: return None + @process_task def getStreamDill(name, process, shell): try: @@ -223,70 +244,87 @@ def getStreamDill(name, process, shell): return None -class ReprFail(object): +class ReprFail: def __init__(self, info): self.info = info + def getRepresentation(name, process): obj_class = getClass(name, process) converters = pythonwhat.State.State.root_state.converters if obj_class in converters: repres = convert(name, dill.dumps(converters[obj_class]), process) - if (errored(repres)): - return ReprFail("manual conversion failed") - else: + if errored(repres): + return ReprFail("manual conversion failed: {}".format(repres)) + else: return repres else: # first try to pickle try: stream = getStreamPickle(name, process) - if not errored(stream): return pickle.loads(stream) - except: + if not errored(stream): + return pickle.loads(stream) + except: pass # if it failed, try to dill try: stream = getStreamDill(name, process) - if not errored(stream): return dill.loads(stream) - return ReprFail("dilling inside process failed for %s - write manual converter" % obj_class) + if not errored(stream): + return dill.loads(stream) + return ReprFail( + "dilling inside process failed for %s - write manual converter" + % obj_class + ) except PicklingError: - return ReprFail("undilling of bytestream failed with PicklingError - write manual converter") + return ReprFail( + "undilling of bytestream failed with PicklingError - write manual converter" + ) except Exception as e: - return ReprFail("undilling of bytestream failed for class %s - write manual converter." - "Error: %s - %s" % (obj_class, type(e), e)) + return ReprFail( + "undilling of bytestream failed for class %s - write manual converter." + "Error: %s - %s" % (obj_class, type(e), e) + ) + def errored(el): - return el is None or (isinstance(el, list) and 'backend-error' in str(el)) + return el is None or (isinstance(el, list) and "backend-error" in str(el)) # Make wrapper for getting an object representation from process -------------- -class UndefinedValue: pass +class UndefinedValue: + pass + def getResultFromProcess(res, tempname, process): """Get a value from process, return tuple of value, res if succesful""" if not isinstance(res, (UndefinedValue, Exception)): value = getRepresentation(tempname, process) - return (value, res) - else: - return (res, str(res)) + return value, res + else: + return res, str(res) + # decorator to automatically get value after running process task function def get_rep(f): sig = inspect.signature(f) + @wraps(f) - def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): # get bound arguments for call ba = sig.bind_partial(*args, **kwargs) ba.apply_defaults() # get tempname, process arg values - tempname = ba.arguments['tempname'] - process = ba.arguments['process'] + tempname = ba.arguments["tempname"] + process = ba.arguments["process"] # run process task - res = f(*args,**kwargs) + res = f(*args, **kwargs) # get result from task return getResultFromProcess(res, tempname, process) + return wrapper + ## Get the output of a tree (with setting envs, pre_code and/er expr_code) @process_task def get_output(f, process, shell, *args, **kwargs): @@ -296,62 +334,122 @@ def get_output(f, process, shell, *args, **kwargs): out_str = out[0].strip() if not isinstance(res, Exception): toret = out_str or "no printouts" - return (toret, toret) + return toret, toret else: - return (res, str(res)) + return res, str(res) + @process_task def get_error(f, *args, **kwargs): res = f(*args, **kwargs) return (res, str(res)) if isinstance(res, Exception) else (None, res) + # General tasks to eval or exec code, with decorated counterparts ------------- -# Eval an expression tree or node (with setting envs, pre_code and/or expr_code) + @process_task -def taskRunEval(tree, - process, shell, - env=None, extra_env = None, context=None, context_vals=None, - pre_code = "", expr_code = "", name="", copy=True, tempname='_evaluation_object_', - call=None): - try: - # Prepare code -------------------------------------------------------- - # If no name given, the object of interest is the output of eval - # otherwise, we'll use name to get the object from the environment - if not isinstance(tree, ast.Module): - mode = 'eval' - tree = ast.Expression(tree) +def taskRunEval( + tree, + process, + shell, + env=None, + extra_env=None, + context=None, + context_vals=None, + pre_code="", + expr_code="", + name="", + copy=True, + tempname="_evaluation_object_", + call=None, +): + """ + Eval an expression tree (with setting envs, pre_code and/or expr_code) + Utility function later wrapped to extract either result (end state of a variable), output (stdout) or error + + Args: + tree (ast): current focused ast, used to get code to execute + process: manages shell (see local.py) + shell: link to get process namespace from execution up until now + env: update value in focused code by name + extra_env: variables to be replaced in focused code by name from extra_env in has_expr + context: sum of set_context in sct chain + context_vals: extra context argument in has_expr + pre_code: argument in has_expr to execute code before evaluating, for example to set a seed + expr_code: code to execute instead of focused code + name: extract value after executing focused expr_code (~post_code) + copy: copy entire env because our expr_code could have side effects + tempname: key for the result when it is added to context, only for v1 sct's + call: only used in v1 sct's + + Returns: + str: output of the executed code + """ + try: + # Prepare code and mode ----------------------------------------------- + # Verify if expr_code is expression code (returning a value) or just runnable code. + if ( # expr_code returns nothing and then we will extract a value + expr_code and name + ) or ( # No expr_code and the tree is of a node type that does not evaluate to have output + not expr_code and isinstance(tree, ast.Module) + ): + # We are not focused on an expression (no output) + mode = "exec" else: - mode = 'exec' + mode = "eval" + # Wrap the focused node in the tree so it can be run with eval() + if not isinstance(tree, (ast.Module, ast.Expression, ast.Expr)): + tree = ast.Expression(tree) # Expression code takes precedence over tree code - if expr_code: code = expr_code - else: code = compile(tree, "