diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..acb1e62a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,46 @@ +version: 2.1 + +orbs: + python: circleci/python@1.2 + +jobs: + build-and-test: + docker: + - image: cimg/python:3.12 + steps: + - checkout + - python/install-packages: + pip-dependency-file: requirements-test.txt + pkg-manager: pip + - run: + name: Run tests + command: pytest -m "not compiled" --cov=pythonwhat + publish: + docker: + - image: cimg/python:3.9 + steps: + - checkout + - run: + command: | + python setup.py sdist bdist_wheel + pip install pipenv + pipenv install twine + pipenv run twine upload --verbose --repository pypi dist/* + +workflows: + build: + jobs: + - build-and-test: + context: org-global + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ + - publish: + context: org-global + requires: + - build-and-test + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ + branches: + ignore: /.*/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..7f0089f5 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,20 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.5 + install: + - requirements: requirements.txt + - method: pip + path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a9840af6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: false -language: python -python: -- '3.5' -before_install: -- pip install -r requirements.txt -- pip install -e . -script: pytest -m "not compiled" --cov=pythonwhat -after_success: codecov --token=$CODECOV_TOKEN -deploy: - provider: pypi - user: datacamp - password: - secure: g0lu0u/tDLNxPa0VsmPQhRfPUzA+3EavKljX9goinCbp0o6RWY7LSBEwCqmMcqIlVJ7gL2yMkBC21HsEGkIvkJhoTpWFXVWtbZEUcmyrfviu1HCrOUqsLvy9WOd+h8ZVelsPT4PEuEKtXD8K9qMjRjtiWH9xuoG+LyMPWvO2WGuIxk+z7DNIrPNm48kuC97yKKr2NWXoj9MjhfqPNAizqAiolYdLBwQX/biXQqsGNErSWp1rvojeA27PrtROLFHZO3cni7b3QjmpmaDkAYERfSbfUfom1gcAKQNeM3OfecBVxNsFc8kjhriktQyp9pWGZ44Hn9dmeYVeGfmVOEmIlCppLlQYcRm07QBtOpgN/QqOftAgZBKbd0AXftrxu64G//l9fLXGaW7i9vsqUMc7ttD4FKF5c7Du4+v3i5ouRXnnaMf2bER/FwezPn5tBrAJ2jW96CDYCnUzQzve/NRmjMUSJbNM4YAB5apXq1PyeacLUEPWphZBcurZe0/Gy3ERTDFTMpHC1fS6ciIDnzMLl69HDuCwOfT7Spvj0hjUgDPR650l66rbEG5UHDmymK2r+qsNrmSicahmKOUT8NJ478qP68G1eNemqe2iT6m8dTvMGD5vSqSgPLwrbfX/ECLk4Gx39Xn+Guj1o/TYF8MTMy/CxXIdXXx9P2Z9/4X5Xso= - on: - tags: true - distributions: sdist bdist_wheel - repo: datacamp/pythonwhat - skip_upload_docs: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed71871..cd1ab6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,55 @@ All notable changes to the pythonwhat project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 2.24.0 + +- Support Python version 3.9 +- Update dependencies to up-to-date versions + +## 2.23.2 + +- Update behaviour of has_expr() default feedback message. If the student's evaluation is too long, it is now shortened and an ellipsis is added. + +## 2.23.1 + +- Fix string formatting in has_expr(). Strings will now have quotes in error messages. Leading and trailing whitespace is no longer removed. + +## 2.23.0 + +- Update to protowhat v2 (embedding xwhats, `prepare_validation` helper, for checking bash history, autodebug) + +## 2.22.0 + +- Add support for replacing the placeholder `__focus__` with the focused code in the `expr_code` argument +- Expose bash history functionality from protowhat + +## 2.21.0 + +- Use path info in `run` + +## 2.20.2 + +- Improve checking exceptions +- Improve checking file content + +## 2.20.1 + +- Speed improvements + +## 2.20.0 + +- Expose `_debug` function +- Update protowhat + +## 2.19.0 + +- `state` is now the first argument to SCT functions (instead of a keyword argument) +- functionality is shared with protowhat: + - `Reporter`, `Chain`, `F` and logic SCTs are reused + - `State` and `Feedback` inherit + - `Dispatcher` handles parsing (decoupled from `State`) + - `Test` moved + ## 2.18.1 ### Fixed @@ -206,9 +255,9 @@ _Small changes to follow up on `2.13.0`_ ```python # Check whether the function was called: - Ex().check_function('df.a.sum', signature = False) + Ex().check_function('df.a.sum', signature=False) # Check whether the function was called and generated the correct result: - Ex().check_function('df.a.sum', signature = False).has_equal_value() + Ex().check_function('df.a.sum', signature=False).has_equal_value() ``` This update means that you should no longer need to use two `has_equal_ast()`'s inside a `test_correct()` to allow for two @@ -247,7 +296,7 @@ _Small changes to follow up on `2.13.0`_ - You can now robustly test printouts with `Ex().has_printout(index = x)`. This function will look for the `x`'th `print()` call in the solution code, rerun that call while capturing the output, and then look for that output in the output that was generated by the student's code submission. ```python - Ex().has_printout(index = 0) + Ex().has_printout(index=0) ``` This approach is far easier and more robust than using `Ex().check_function().check_args().has_equal_value()`. @@ -329,7 +378,7 @@ _Small changes to follow up on `2.13.0`_ ```Python from pythonwhat.local import setup_state -s = setup_state(sol_code = "x = 4", stu_code = "x = 5") +s = setup_state(sol_code="x = 4", stu_code="x = 5") s.check_object('x').has_equal_value() # # pythonwhat.Test.TestFail: Check the variable `x`. Unexpected expression value: expected `4`, got `5`. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..096dd503 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include requirements.txt diff --git a/README.md b/README.md index 2ccad917..f50b0e0b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # pythonwhat [![Build Status](https://travis-ci.org/datacamp/pythonwhat.svg?branch=master)](https://travis-ci.org/datacamp/pythonwhat) -[![codecov](https://codecov.io/gh/datacamp/pythonwhat/branch/master/graph/badge.svg)](https://codecov.io/gh/datacamp/pythonwhat) [![PyPI version](https://badge.fury.io/py/pythonwhat.svg)](https://badge.fury.io/py/pythonwhat) [![Documentation Status](https://readthedocs.org/projects/pythonwhat/badge/?version=stable)](http://pythonwhat.readthedocs.io/en/stable/?badge=stable) +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat?ref=badge_shield) Verify Python code submissions and auto-generate meaningful feedback messages. Originally developed for Python exercises on DataCamp for so-called Submission Correctness Tests, but can also be used independently. @@ -33,7 +33,7 @@ _, ctxt = prep_context() globals().update(ctxt) # initialize state with student and solution submission -from pythonwhat.local import setup_state +from pythonwhat.test_exercise import setup_state setup_state(stu_code = "x = 5", sol_code = "x = 4") Ex().check_object('x') @@ -52,11 +52,17 @@ To learn how to include an SCT in a DataCamp course, visit https://instructor-su ## Run tests -``` -pyenv local 3.5.2 -pip install -r requirements.txt -pip install -e . +```bash +pyenv local 3.12.7 +pip3.12 install -r requirements-test.txt +pip3.12 install -e . pytest ``` +## Contributing + Bugs? Questions? Suggestions? [Create an issue](https://github.com/datacamp/pythonwhat/issues/new), or [contact us](mailto:content-engineering@datacamp.com)! + +## License + +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fdatacamp%2Fpythonwhat?ref=badge_large) diff --git a/docs/articles/electives.rst b/docs/articles/electives.rst index 23acaf33..9e577aed 100644 --- a/docs/articles/electives.rst +++ b/docs/articles/electives.rst @@ -178,7 +178,7 @@ by name in the SCT above, they may also be given by position.. with_context ============ -.. autofunction:: pythonwhat.check_funcs.with_context +.. autofunction:: pythonwhat.checks.check_funcs.with_context :noindex: Runs subtests after setting the context for a ``with`` statement. diff --git a/docs/articles/make_your_sct_robust.rst b/docs/articles/make_your_sct_robust.rst index 88f6abc7..f3d94d50 100644 --- a/docs/articles/make_your_sct_robust.rst +++ b/docs/articles/make_your_sct_robust.rst @@ -111,7 +111,7 @@ or you can use ``check_or()`` with three separate ``has_code()`` functions: .. code:: - Ex().check_or(has_code('4'), + Ex().check_or(has_code('4'), has_code('5'), has_code('6')) diff --git a/docs/conf.py b/docs/conf.py index c03b42cc..bace3095 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,12 +16,29 @@ import os import sys +import json +import pythonwhat + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) -import pythonwhat +# enables finding custom extensions in docs directory +sys.path.insert(0, os.path.abspath(".")) + +sys.path.insert(0, os.path.abspath("..")) + +TEST_DATA_FILE = "test_data.json" + +if not os.path.isfile(TEST_DATA_FILE): + import subprocess + + subprocess.call("pytest", cwd=os.path.abspath(os.path.join(os.getcwd(), os.pardir))) + +with open(TEST_DATA_FILE, "r") as read_file: + test_data = json.load(read_file) + +jinja_contexts = {"test_ctx": {"test_data": test_data}} # -- General configuration ------------------------------------------------ @@ -32,21 +49,25 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinxcontrib.jinja", +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'pythonwhat' -copyright = '2018, DataCamp' -author = 'DataCamp' +project = "pythonwhat" +copyright = "2019, DataCamp" +author = "DataCamp" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -67,14 +88,14 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -84,7 +105,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -101,19 +122,18 @@ # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'pythonwhatdoc' +htmlhelp_basename = "pythonwhatdoc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { } +latex_elements = {} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pythonwhat.tex', 'pythonwhat Documentation', - 'DataCamp', 'manual'), + (master_doc, "pythonwhat.tex", "pythonwhat Documentation", "DataCamp", "manual") ] @@ -121,10 +141,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pythonwhat', 'pythonwhat Documentation', - [author], 1) -] +man_pages = [(master_doc, "pythonwhat", "pythonwhat Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -132,8 +149,13 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pythonwhat', 'pythonwhat Documentation', - author, 'pythonwhat', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pythonwhat", + "pythonwhat Documentation", + author, + "pythonwhat", + "One line description of project.", + "Miscellaneous", + ) ] - diff --git a/docs/data_to_doc.py b/docs/data_to_doc.py new file mode 100644 index 00000000..d6d94f99 --- /dev/null +++ b/docs/data_to_doc.py @@ -0,0 +1,13 @@ +def render_data(app, docname, source): + """ + Render our pages as a jinja template for fancy templating goodness. + """ + if app.builder.format != "html": + return + src = source[0] + rendered = app.builder.templates.render_string(src, app.config.html_context) + source[0] = rendered + + +def setup(app): + app.connect("source-read", render_data) diff --git a/docs/index.rst b/docs/index.rst index 325b8a87..2a04317b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,4 +44,10 @@ The reference docs become useful when you grasp all concepts and want to look up articles/electives.rst articles/test_to_check.rst +.. toctree:: + :maxdepth: 1 + :caption: Tests + + tests + For details, questions and suggestions, `contact us `_. diff --git a/docs/reference.rst b/docs/reference.rst index 25aabae4..dc13556e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -9,83 +9,97 @@ Reference Objects ------- -.. autofunction:: pythonwhat.check_object.check_object -.. autofunction:: pythonwhat.check_object.is_instance -.. autofunction:: pythonwhat.check_object.check_df -.. autofunction:: pythonwhat.check_object.check_keys +.. autofunction:: pythonwhat.checks.check_object.check_object +.. autofunction:: pythonwhat.checks.check_object.is_instance +.. autofunction:: pythonwhat.checks.check_object.check_df +.. autofunction:: pythonwhat.checks.check_object.check_keys Function calls -------------- -.. autofunction:: pythonwhat.check_function.check_function -.. autofunction:: pythonwhat.check_funcs.check_args +.. autofunction:: pythonwhat.checks.check_function.check_function +.. autofunction:: pythonwhat.checks.check_funcs.check_args Output ------ -.. autofunction:: pythonwhat.has_funcs.has_output -.. autofunction:: pythonwhat.has_funcs.has_printout -.. autofunction:: pythonwhat.has_funcs.has_no_error +.. autofunction:: pythonwhat.checks.has_funcs.has_output +.. autofunction:: pythonwhat.checks.has_funcs.has_printout +.. autofunction:: pythonwhat.checks.has_funcs.has_no_error Code ---- -.. autofunction:: pythonwhat.has_funcs.has_code -.. autofunction:: pythonwhat.has_funcs.has_import +.. autofunction:: pythonwhat.checks.has_funcs.has_code +.. autofunction:: pythonwhat.checks.has_funcs.has_import has_equal_x ----------- -.. autofunction:: pythonwhat.has_funcs.has_equal_value -.. autofunction:: pythonwhat.has_funcs.has_equal_output -.. autofunction:: pythonwhat.has_funcs.has_equal_error -.. autofunction:: pythonwhat.has_funcs.has_equal_ast +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_value +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_output +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_error +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_ast Combining SCTs -------------- -.. autofunction:: pythonwhat.check_logic.multi -.. autofunction:: pythonwhat.check_logic.check_correct -.. autofunction:: pythonwhat.check_logic.check_or -.. autofunction:: pythonwhat.check_logic.check_not +.. autofunction:: protowhat.checks.check_logic.multi +.. autofunction:: protowhat.checks.check_logic.check_correct +.. autofunction:: protowhat.checks.check_logic.check_or +.. autofunction:: protowhat.checks.check_logic.check_not Function/Class/Lambda definitions --------------------------------- -.. autofunction:: pythonwhat.check_wrappers.check_function_def -.. autofunction:: pythonwhat.has_funcs.has_equal_part_len -.. autofunction:: pythonwhat.check_funcs.check_call -.. autofunction:: pythonwhat.check_wrappers.check_class_def -.. autofunction:: pythonwhat.check_wrappers.check_lambda_function +.. autofunction:: pythonwhat.checks.check_wrappers.check_function_def +.. autofunction:: pythonwhat.checks.has_funcs.has_equal_part_len +.. autofunction:: pythonwhat.checks.check_funcs.check_call +.. autofunction:: pythonwhat.checks.check_wrappers.check_class_def +.. autofunction:: pythonwhat.checks.check_wrappers.check_lambda_function Control flow ------------ -.. autofunction:: pythonwhat.check_wrappers.check_if_else -.. autofunction:: pythonwhat.check_wrappers.check_try_except -.. autofunction:: pythonwhat.check_wrappers.check_if_exp -.. autofunction:: pythonwhat.check_wrappers.check_with +.. autofunction:: pythonwhat.checks.check_wrappers.check_if_else +.. autofunction:: pythonwhat.checks.check_wrappers.check_try_except +.. autofunction:: pythonwhat.checks.check_wrappers.check_if_exp +.. autofunction:: pythonwhat.checks.check_wrappers.check_with Loops ----- -.. autofunction:: pythonwhat.check_wrappers.check_for_loop -.. autofunction:: pythonwhat.check_wrappers.check_while -.. autofunction:: pythonwhat.check_wrappers.check_list_comp -.. autofunction:: pythonwhat.check_wrappers.check_dict_comp -.. autofunction:: pythonwhat.check_wrappers.check_generator_exp +.. autofunction:: pythonwhat.checks.check_wrappers.check_for_loop +.. autofunction:: pythonwhat.checks.check_wrappers.check_while +.. autofunction:: pythonwhat.checks.check_wrappers.check_list_comp +.. autofunction:: pythonwhat.checks.check_wrappers.check_dict_comp +.. autofunction:: pythonwhat.checks.check_wrappers.check_generator_exp -State Management +State management ---------------- -.. autofunction:: pythonwhat.check_logic.override -.. autofunction:: pythonwhat.check_logic.disable_highlighting -.. autofunction:: pythonwhat.check_logic.set_context -.. autofunction:: pythonwhat.check_logic.set_env +.. autofunction:: pythonwhat.checks.check_logic.override +.. autofunction:: pythonwhat.checks.check_logic.disable_highlighting +.. autofunction:: pythonwhat.checks.check_logic.set_context +.. autofunction:: pythonwhat.checks.check_logic.set_env + +Checking files +-------------- + +.. autofunction:: pythonwhat.checks.check_wrappers.check_file +.. autofunction:: pythonwhat.checks.check_wrappers.has_dir +.. autofunction:: pythonwhat.local.run + +Bash history checks +------------------- + +.. automodule:: protowhat.checks.check_bash_history + :members: Electives --------- -.. autofunction:: pythonwhat.has_funcs.has_chosen +.. autofunction:: pythonwhat.checks.has_funcs.has_chosen .. autofunction:: pythonwhat.test_exercise.success_msg -.. autofunction:: pythonwhat.check_logic.fail \ No newline at end of file +.. autofunction:: protowhat.checks.check_simple.allow_errors +.. autofunction:: protowhat.checks.check_logic.fail diff --git a/docs/tests.rst b/docs/tests.rst new file mode 100644 index 00000000..c6d9c5d6 --- /dev/null +++ b/docs/tests.rst @@ -0,0 +1,79 @@ +Tests +===== + +.. note:: + + The examples are numbered and linkable, + but numbers (and links) can change between builds of the documentation. + +.. jinja:: test_ctx + + {% for file, tests in test_data.items() %} + + {{ file }} + {{ "-" * 100 }} + + {% for test in tests %} + + Example {{loop.index}} + ~~~~~~~~~~~~~~~~~~~~~~ + {% if test.pre_exercise_code %} + PEC :: + + {{ test.pre_exercise_code | indent(4) }} + + {% else %} + No PEC + {% endif %} + {% if test.solution_code %} + Solution code :: + + {{ test.solution_code | indent(4) }} + + {% else %} + No solution code + {% endif %} + {% if test.student_code %} + Student code :: + + {{ test.student_code | indent(4) }} + + {% else %} + No student code + {% endif %} + {% if test.raw_student_output %} + Student output :: + + {{ test.raw_student_output | indent(4) }} + + {% else %} + No output + {% endif %} + {% if test.sct %} + SCT :: + + {{ test.sct | indent(4) }} + + {% else %} + No SCT + {% endif %} + {% if test.result %} + Result :: + + {{ test.result.message | indent(4) }} + + {% else %} + No result + {% endif %} + {% if test.error %} + Error :: + + {{ test.error | indent(4) }} + + {% else %} + No error + {% endif %} + + {% endfor %} + + {% endfor %} diff --git a/pytest.ini b/pytest.ini index 015f8591..724b2ae5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,6 @@ [pytest] testpaths = tests/ addopts =-m "not compiled" +markers = + slow + compiled diff --git a/pythonwhat/Feedback.py b/pythonwhat/Feedback.py deleted file mode 100644 index 2325ad6d..00000000 --- a/pythonwhat/Feedback.py +++ /dev/null @@ -1,19 +0,0 @@ -import re - -class Feedback(object): - - def __init__(self, message, state = None): - self.message = message - self.line_info = {} - try: - if state is not None and hasattr(state.highlight, "first_token") and \ - hasattr(state.highlight, "last_token") and not state.highlighting_disabled: - self.line_info["line_start"] = state.highlight.first_token.start[0] - self.line_info["column_start"] = state.highlight.first_token.start[1] - self.line_info["line_end"] = state.highlight.last_token.end[0] - self.line_info["column_end"] = state.highlight.last_token.end[1] - except: - pass - -class InstructorError(Exception): - pass diff --git a/pythonwhat/Reporter.py b/pythonwhat/Reporter.py deleted file mode 100644 index 6138471b..00000000 --- a/pythonwhat/Reporter.py +++ /dev/null @@ -1,74 +0,0 @@ -from pythonwhat.Feedback import Feedback -import re -import markdown2 -from pythonwhat.Test import TestFail, Test - -""" -This file holds the reporter class. -""" - -class Reporter(object): - """Do reporting. - - This class holds the feedback- or success message and tracks whether there are failed tests - or not. All tests are executed trough do_test() in the Reporter. - """ - active_reporter = None - - def __init__(self, error=None): - self.success_msg = "Great work!" - self.error = error - self.errors_allowed = False - - def do_test(self, testobj): - """Do test. - - Execute a given test, unless some previous test has failed. If the test has failed, - the state of the reporter changes and the feedback is kept. - """ - - if isinstance(testobj, Test): - testobj.test() - result = testobj.result - if (not result): - feedback = testobj.get_feedback() - raise TestFail(feedback, self.build_failed_payload(feedback)) - - else: - result = None - testobj() # run function for side effects - - return result - - def build_failed_payload(self, feedback): - if not feedback.line_info: - return { - "correct": False, - "message": Reporter.to_html(feedback.message) - } - else: - return { - "correct": False, - "message": Reporter.to_html(feedback.message), - "line_start": feedback.line_info["line_start"], - "column_start": feedback.line_info["column_start"] + 1, - "line_end": feedback.line_info["line_end"], - "column_end": feedback.line_info["column_end"] - } - - def build_final_payload(self): - if (self.error and not self.errors_allowed): - feedback_msg = "Have a look at the console: your code contains an error. Fix it and try again!" - return { - "correct": False, - "message": Reporter.to_html(feedback_msg) - } - else: - return({ - "correct": True, - "message": Reporter.to_html(self.success_msg) - }) - - @staticmethod - def to_html(msg): - return(re.sub("

(.*)

", "\\1", markdown2.markdown(msg)).strip()) diff --git a/pythonwhat/State.py b/pythonwhat/State.py index a5ffe593..86a0f426 100644 --- a/pythonwhat/State.py +++ b/pythonwhat/State.py @@ -1,20 +1,25 @@ -import ast -import inspect -import string -from copy import copy -from functools import partial -from pythonwhat.parsing import TargetVars, FunctionParser, ObjectAccessParser, parser_dict -from pythonwhat.Reporter import Reporter -from pythonwhat.Feedback import Feedback, InstructorError -from pythonwhat.Test import Test +import asttokens + +from functools import partialmethod +from collections.abc import Mapping + +from protowhat.failure import debugger +from protowhat.Feedback import FeedbackComponent +from protowhat.selectors import DispatcherInterface +from protowhat.State import State as ProtoState +from protowhat.utils import parameters_attr from pythonwhat import signatures from pythonwhat.converters import get_manual_converters -from collections.abc import Mapping -from itertools import chain -from jinja2 import Template -import asttokens +from pythonwhat.feedback import Feedback +from pythonwhat.parsing import ( + TargetVars, + FunctionParser, + ObjectAccessParser, + parser_dict, +) from pythonwhat.utils_ast import wrap_in_module + class Context(Mapping): def __init__(self, context=None, prev=None): self.context = context if context else TargetVars() @@ -23,8 +28,7 @@ def __init__(self, context=None, prev=None): self._items = {**self.prev, **self.context.defined_items()} def update_ctx(self, new_ctx): - upd_prev = {**self.prev, **self.context.defined_items()} - return self.__class__(new_ctx, upd_prev) + return self.__class__(new_ctx, self._items) def __getitem__(self, x): return self._items[x] @@ -35,282 +39,300 @@ def __iter__(self): def __len__(self): return len(self._items) -class State(object): + +@parameters_attr +class State(ProtoState): """State of the SCT environment. This class holds all information relevevant to test the correctness of an exercise. It is coded suboptimally and it will be refactored soon, and documented thouroughly after that. - """ - def __init__(self, - student_context=None, solution_context=None, - student_env=None, solution_env=None, - student_parts=None, solution_parts=None, - highlight = None, - highlighting_disabled = None, messages=None, - force_diagnose = False, - **kwargs): - - # Set basic fields from kwargs - self.__dict__.update(kwargs) - - self.student_parts = student_parts - self.solution_parts = solution_parts - self.messages = messages if messages else [] - self.force_diagnose = force_diagnose - - # parse code if didn't happen yet - if not hasattr(self, 'student_tree'): - self.student_tree_tokens, self.student_tree = State.parse_external(self.student_code) - - if not hasattr(self, 'solution_tree'): - self.solution_tree_tokens, self.solution_tree = State.parse_internal(self.solution_code) + kwargs: + ... + - reporter - if not hasattr(self, 'pre_exercise_tree'): - _, self.pre_exercise_tree = State.parse_internal(self.pre_exercise_code) - - if not hasattr(self, 'parent_state'): - self.parent_state = None - - self.student_context = Context(student_context) if student_context is None else student_context - self.solution_context = Context(solution_context) if solution_context is None else solution_context - self.student_env = Context(student_env) if student_env is None else student_env - self.solution_env = Context(solution_env) if solution_env is None else solution_env - - self.highlight = self.student_tree if (not highlight) and self.parent_state else highlight - self.highlighting_disabled = highlighting_disabled + """ - self.converters = get_manual_converters() # accessed only from root state + feedback_cls = Feedback + + def __init__( + self, + student_code, + solution_code, + pre_exercise_code, + student_process, + solution_process, + raw_student_output, + # solution output + reporter, + force_diagnose=False, + highlight=None, + highlight_offset=None, + highlighting_disabled=None, + feedback_context=None, + creator=None, + student_ast=None, + solution_ast=None, + student_ast_tokens=None, + solution_ast_tokens=None, + student_parts=None, + solution_parts=None, + student_context=Context(), + solution_context=Context(), + student_env=Context(), + solution_env=Context(), + ): + args = locals().copy() + self.debug = False + + for k, v in args.items(): + if k != "self": + setattr(self, k, v) + + self.ast_dispatcher = self.get_dispatcher() + + # Parse solution and student code + # if possible, not done yet and wanted (ast arguments not False) + if isinstance(self.student_code, str) and student_ast is None: + self.student_ast = self.parse(student_code) + if isinstance(self.solution_code, str) and solution_ast is None: + with debugger(self): + self.solution_ast = self.parse(solution_code) + + if highlight is None: # todo: check parent_state? (move check to reporting?) + self.highlight = self.student_ast + + self.converters = get_manual_converters() # accessed only from root state self.manual_sigs = None - self._parser_cache = {} def get_manual_sigs(self): if self.manual_sigs is None: self.manual_sigs = signatures.get_manual_sigs() - return(self.manual_sigs) - - def build_message(self, tail="", fmt_kwargs=None, append=True): - - if not fmt_kwargs: fmt_kwargs = {} - out_list = [] - # add trailing message to msg list - msgs = self.messages[:] + [{'msg': tail or "", 'kwargs':fmt_kwargs}] - # format messages in list, by iterating over previous, current, and next message - for prev_d, d, next_d in zip([{}, *msgs[:-1]], msgs, [*msgs[1:], {}]): - tmp_kwargs = {'parent': prev_d.get('kwargs'), - 'child': next_d.get('kwargs'), - 'this': d['kwargs'], - **d['kwargs']} - # don't bother appending if there is no message - if not d['msg']: - continue - out = Template(d['msg'].replace('__JINJA__:', "")).render(**tmp_kwargs) - out_list.append(out) - - # if highlighting info is available, don't put all expand messages - if self.highlight and not self.highlighting_disabled: - out_list = out_list[-3:] - - if append: - return "".join(out_list) - else: - return out_list[-1] - - def to_child_state(self, student_subtree=None, solution_subtree=None, - student_context=None, solution_context=None, - student_env=None, solution_env=None, - student_parts=None, solution_parts=None, - highlight = None, - highlighting_disabled = None, - append_message="", node_name=""): + return self.manual_sigs + + def to_child(self, append_message=None, node_name="", **kwargs): """Dive into nested tree. Set the current state as a state with a subtree of this syntax tree as student tree and solution tree. This is necessary when testing if statements or for loops for example. """ + bad_parameters = set(kwargs) - set(self.parameters) + if bad_parameters: + raise ValueError( + "Invalid init parameters for State: %s" % ", ".join(bad_parameters) + ) - if isinstance(student_subtree, list): - student_subtree = wrap_in_module(student_subtree) - if isinstance(solution_subtree, list): - solution_subtree = wrap_in_module(solution_subtree) + base_kwargs = { + attr: getattr(self, attr) + for attr in self.parameters + if hasattr(self, attr) and attr not in ["ast_dispatcher", "highlight"] + } - # get new contexts - if solution_context is not None: - solution_context = self.solution_context.update_ctx(solution_context) - else: - solution_context = self.solution_context + if append_message and not isinstance(append_message, FeedbackComponent): + append_message = FeedbackComponent(append_message) + kwargs["feedback_context"] = append_message + kwargs["creator"] = {"type": "to_child", "args": {"state": self}} - if student_context is not None: - student_context = self.student_context.update_ctx(student_context) - else: - student_context = self.student_context + def update_kwarg(name, func): + kwargs[name] = func(kwargs[name]) - # get new envs - if solution_env is not None: - solution_env = self.solution_env.update_ctx(solution_env) - else: - solution_env = self.solution_env + def update_context(name): + update_kwarg(name, getattr(self, name).update_ctx) + + for ast_arg in ["student_ast", "solution_ast"]: + if isinstance(kwargs.get(ast_arg), list): + update_kwarg(ast_arg, wrap_in_module) + + if kwargs.get("student_ast") and kwargs.get("student_code") is None: + kwargs["student_code"] = self.student_ast_tokens.get_text( + kwargs["student_ast"] + ) + if kwargs.get("solution_ast") and kwargs.get("solution_code") is None: + kwargs["solution_code"] = self.solution_ast_tokens.get_text( + kwargs["solution_ast"] + ) + + for context in [ + "student_context", + "solution_context", + "student_env", + "solution_env", + ]: + if context in kwargs: + if kwargs[context] is not None: + update_context(context) + else: + kwargs.pop(context) + + klass = self.SUBCLASSES[node_name] if node_name else State + init_kwargs = {**base_kwargs, **kwargs} + child = klass(**init_kwargs) + + extra_attrs = set(vars(self)) - set(self.parameters) + for attr in extra_attrs: + # don't copy attrs set on new instances in init + # the cached manual_sigs is passed + if attr not in {"ast_dispatcher", "converters"}: + setattr(child, attr, getattr(self, attr)) - if student_env is not None: - student_env = self.student_env.update_ctx(student_env) - else: - student_env = self.student_env - - if highlighting_disabled is None: - highlighting_disabled = self.highlighting_disabled - - if not isinstance(append_message, dict): - append_message = {'msg': append_message, 'kwargs': {}} - - messages = [*self.messages, append_message] - - if not (solution_subtree and student_subtree): - return self.update(student_context = student_context, solution_context = solution_context, - student_env = student_env, solution_env = solution_env, - highlight = highlight, - highlighting_disabled = highlighting_disabled, - messages = messages) - - klass = State if not node_name else self.SUBCLASSES[node_name] - child = klass(student_code = self.student_tree_tokens.get_text(student_subtree), - solution_code = self.solution_tree_tokens.get_text(solution_subtree), - student_tree_tokens = self.student_tree_tokens, - solution_tree_tokens = self.solution_tree_tokens, - pre_exercise_code = self.pre_exercise_code, - student_context = student_context, - solution_context = solution_context, - student_env = student_env, - solution_env = solution_env, - student_process = self.student_process, - solution_process = self.solution_process, - raw_student_output = self.raw_student_output, - pre_exercise_tree = self.pre_exercise_tree, - student_tree = student_subtree, - solution_tree = solution_subtree, - student_parts = student_parts, - solution_parts = solution_parts, - highlight = highlight, - highlighting_disabled = highlighting_disabled, - messages = messages, - parent_state = self, - force_diagnose=self.force_diagnose) - return(child) - - def update(self, **kwargs): - """Return a copy of set, setting kwargs as attributes""" - child = copy(self) - for k, v in kwargs.items(): - setattr(child, k, v) return child def has_different_processes(self): # process classes have an _identity field that is a tuple try: - return self.student_process._identity[0] != self.solution_process._identity[0] + return ( + self.student_process._identity[0] != self.solution_process._identity[0] + ) except: # play it safe (most common) return True - def assert_root(self, fun, extra_msg=""): - if self.parent_state is not None: - raise InstructorError("`%s()` should only be called from the root state, `Ex()`. %s" % (fun, extra_msg)) + def assert_execution_root(self, fun, extra_msg=""): + if not (self.is_root or self.is_creator_type("run")): + with debugger(self): + self.report( + "`%s()` should only be called focusing on a full script, following `Ex()` or `run()`. %s" + % (fun, extra_msg) + ) + + def is_creator_type(self, type): + return self.creator and self.creator.get("type") == type def assert_is(self, klasses, fun, prev_fun): if self.__class__.__name__ not in klasses: - raise InstructorError("`%s()` can only be called on %s." % - (fun, " or ".join([ '`%s()`' % pf for pf in prev_fun ])) - ) + with debugger(self): + self.report( + "`%s()` can only be called on %s." + % (fun, " or ".join(["`%s()`" % pf for pf in prev_fun])) + ) def assert_is_not(self, klasses, fun, prev_fun): if self.__class__.__name__ in klasses: - raise InstructorError("`%s()` should not be called on %s." % - (fun, " or ".join([ '`%s()`' % pf for pf in prev_fun ])) - ) - - @staticmethod - def parse_external(x): - rep = Reporter.active_reporter + with debugger(self): + self.report( + "`%s()` should not be called on %s." + % (fun, " or ".join(["`%s()`" % pf for pf in prev_fun])) + ) + def parse_external(self, code): res = (None, None) try: - res = asttokens.ASTTokens(x, parse = True) - return(res, res._tree) - + return self.ast_dispatcher.parse(code) except IndentationError as e: e.filename = "script.py" # no line info for now - rep.do_test(Test(Feedback("Your code could not be parsed due to an error in the indentation:
`%s.`" % str(e)))) + self.report( + "Your code could not be parsed due to an error in the indentation:
`%s.`" + % str(e) + ) except SyntaxError as e: e.filename = "script.py" # no line info for now - rep.do_test(Test(Feedback("Your code can not be executed due to a syntax error:
`%s.`" % str(e)))) + self.report( + "Your code can not be executed due to a syntax error:
`%s.`" % str(e) + ) # Can happen, can't catch this earlier because we can't differentiate between # TypeError in parsing or TypeError within code (at runtime). except: - rep.do_test(Test(Feedback("Something went wrong while parsing your code."))) + self.report("Something went wrong while parsing your code.") - return(res) + return res - @staticmethod - def parse_internal(x): - res = (None, None) + def parse_internal(self, code): + try: + return self.ast_dispatcher.parse(code) + except Exception as e: + self.report( + "Something went wrong when parsing the solution code: %s" % str(e) + ) + + def parse(self, text): + if self.debug: + parse_method = self.parse_internal + token_attr = "solution_ast_tokens" + else: + parse_method = self.parse_external + token_attr = "student_ast_tokens" + + tokens, ast = parse_method(text) + setattr(self, token_attr, tokens) + + return ast + + def get_dispatcher(self): try: - res = asttokens.ASTTokens(x, parse = True) - return(res, res._tree) + return Dispatcher(self.pre_exercise_code) except Exception as e: - raise InstructorError("Something went wrong when parsing PEC or solution code: %s" % str(e)) - -# add property methods for retrieving parser outputs -------------------------- -# note that this code is an alternative means of using something like.. -# @property -# def student_withs(self): ... -# when defining the State class. -def getx(tree_name, Parser, ext_attr, self): - """getter for Parser outputs""" - # return cached output if possible - cache_key = tree_name + Parser.__name__ - if self._parser_cache.get(cache_key): - p = self._parser_cache[cache_key] - else: - # otherwise, run parser over tree - p = Parser() - # set mappings for parsers that inspect attribute access - if ext_attr != 'mappings' and Parser in [FunctionParser, ObjectAccessParser]: - p.mappings = self.pre_exercise_mappings.copy() - # run parser - p.visit(getattr(self, tree_name)) - # cache - self._parser_cache[cache_key] = p - return getattr(p, ext_attr) - -# put a property getter on state for each parsed ast tree output. -# since the getter takes only one argument, self, partial functions -# are used to set all other arguments on getx -for s in ['student', 'solution']: - tree_name = s+'_tree' - for k, Parser in parser_dict.items(): - setattr(State, s+'_'+k, property(partial(getx, tree_name, Parser, 'out'))) - - # mappings from ObjectAccessParser - prop_oa_map = property(partial(getx, tree_name, ObjectAccessParser, 'mappings')) - setattr(State, s+'_oa_mappings', prop_oa_map) - - # mappings from FunctionParser - prop_map = property(partial(getx, tree_name, FunctionParser, 'mappings')) - setattr(State, s+'_mappings', prop_map) - -# mappings for pre exercise code from FunctionParser -pec_prop_map = property(partial(getx, 'pre_exercise_tree', FunctionParser, 'mappings')) -setattr(State, 'pre_exercise_mappings', pec_prop_map) + with debugger(self): + self.report("Something went wrong when parsing the PEC: %s" % str(e)) + + +class Dispatcher(DispatcherInterface): + _context_cache = dict() + + def __init__(self, context_code=""): + self._parser_cache = dict() + context_ast = getattr(self._context_cache, context_code, None) + if context_ast is None: + context_ast = self._context_cache[context_code] = self.parse(context_code)[ + 1 + ] + self.context_mappings = self._getx(FunctionParser, "mappings", context_ast) + + def find(self, name, node, *args, **kwargs): + return getattr(self, name)(node) + + def parse(self, code): + res = asttokens.ASTTokens(code, parse=True) + return res, res.tree + + # add methods for retrieving parser outputs -------------------------- + def _getx(self, Parser, ext_attr, tree): + """getter for Parser outputs""" + # return cached output if possible + cache_key = Parser.__name__ + str(hash(tree)) + if self._parser_cache.get(cache_key): + p = self._parser_cache[cache_key] + else: + # otherwise, run parser over tree + p = Parser() + # set mappings for parsers that inspect attribute access + if ext_attr != "mappings" and Parser in [ + FunctionParser, + ObjectAccessParser, + ]: + p.mappings = self.context_mappings.copy() + # run parser + p.visit(tree) + # cache + self._parser_cache[cache_key] = p + return getattr(p, ext_attr) + + +# put a function on the dispatcher +for k, Parser in parser_dict.items(): + setattr(Dispatcher, k, partialmethod(Dispatcher._getx, Parser, "out")) + +# mappings from ObjectAccessParser +prop_oa_map = partialmethod(Dispatcher._getx, ObjectAccessParser, "mappings") +setattr(Dispatcher, "oa_mappings", prop_oa_map) + +# mappings from FunctionParser +prop_map = partialmethod(Dispatcher._getx, FunctionParser, "mappings") +setattr(Dispatcher, "mappings", prop_map) + # State subclasses based on parsed output ------------------------------------- -State.SUBCLASSES = {node_name: type(node_name, (State,), {}) for node_name in parser_dict} +State.SUBCLASSES = { + node_name: type(node_name, (State,), {}) for node_name in parser_dict +} + # global setters on State ----------------------------------------------------- def set_converter(key, fundef): diff --git a/pythonwhat/Test.py b/pythonwhat/Test.py index 079f45a8..47c0703c 100644 --- a/pythonwhat/Test.py +++ b/pythonwhat/Test.py @@ -1,74 +1,18 @@ import re -from pythonwhat.Feedback import Feedback -import numpy as np -import pandas as pd from pythonwhat.tasks import * +from protowhat.Test import Test """ -This file contains all tests that can be done on specific objects. All tests are represented +This file contains all tests that can be done on specific objects. All tests are represented as an object. Tests that are alike can inherit from the same superclass. A test is first initialized and can then be performed by calling the 'test()' function. The result will be stored inside the result boolean. A test contains a failure message, which can be used by the reporter to show when the test failed. """ -class TestFail(Exception): - def __init__(self, feedback, payload): - super().__init__(feedback.message) - self.feedback = feedback - self.payload = payload -class Test(object): - """ - The basic Test. It should only contain a failure message, as all tests should result in - a failure message when they fail. - - Note: - This test should not be used by itself, subclasses should be used. - - Attributes: - feedback (str): A string containing the failure message in case the test fails. - result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. - """ - - def __init__(self, feedback): - """ - Initialize the standard test. - - Args: - feedback: string or Feedback object - """ - if (issubclass(type(feedback), Feedback)): - self.feedback = feedback - elif (issubclass(type(feedback), str)): - self.feedback = Feedback(feedback) - else: - raise TypeError("When creating a test, specify either a string or a Feedback object") - - self.result = None - - def test(self): - """ - Wrapper around specific tests. Tests only get one chance. - """ - if self.result is None: - try: - self.specific_test() - self.result = np.array(self.result).all() - except: - self.result = False - - def specific_test(self): - """ - Perform the actual test. For the standard test, result will be set to False. - """ - self.result = False +# Testing definition - def get_feedback(self): - return(self.feedback) - - -## Testing definition class DefinedProcessTest(Test): def __init__(self, name, process, feedback): @@ -76,7 +20,7 @@ def __init__(self, name, process, feedback): self.name = name self.process = process - def specific_test(self): + def test(self): self.result = isDefinedInProcess(self.name, self.process) @@ -90,14 +34,16 @@ class DefinedCollTest(Test): collection (list/dict/set): Contains any object on which the 'in' operator can be performed. result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. """ + def __init__(self, name, collection, feedback): super().__init__(feedback) self.name = name self.collection = collection - def specific_test(self): + def test(self): self.result = self.name in self.collection + class DefinedCollProcessTest(Test): def __init__(self, name, key, process, feedback): super().__init__(feedback) @@ -105,11 +51,12 @@ def __init__(self, name, key, process, feedback): self.key = key self.process = process - def specific_test(self): + def test(self): self.result = isDefinedCollInProcess(self.name, self.key, self.process) -## Testing class +# Testing class + class InstanceProcessTest(Test): def __init__(self, name, klass, process, feedback): @@ -118,10 +65,12 @@ def __init__(self, name, klass, process, feedback): self.klass = klass self.process = process - def specific_test(self): + def test(self): self.result = isInstanceInProcess(self.name, self.klass, self.process) -## Testing equality + +# Testing equality + class EqualTest(Test): """ @@ -135,85 +84,118 @@ class EqualTest(Test): result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. """ - def __init__(self, obj1, obj2, feedback, func = None): + def __init__(self, obj1, obj2, feedback, func=None): super().__init__(feedback) self.obj1 = obj1 self.obj2 = obj2 self.func = func if func is not None else is_equal - def specific_test(self): + def test(self): """ Perform the actual test. result is set to False if the objects differ, True otherwise. """ - self.result = self.func(self.obj1, self.obj2) + result = self.func(self.obj1, self.obj2) -## Helpers for testing equality + try: + import numpy as np + + self.result = np.array(result).all() + except ImportError: + self.result = result + + +# Helpers for testing equality -def objs_are(x, y, list_of_classes): - return ( - any([isinstance(x, klass) for klass in list_of_classes]) & - any([isinstance(y, klass) for klass in list_of_classes]) - ) + +def areinstance(x, y, tuple_of_classes): + return isinstance(x, tuple_of_classes) and isinstance(y, tuple_of_classes) + +def is_primitive(x): + return isinstance(x, (str, int, float, bool, type(None))) + +def is_collection_of_primitives(x): + return isinstance(x, (list, tuple, set)) and all(is_primitive(element) for element in x) # For equality of ndarrays, list, dicts, pd Series and pd DataFrames: # First try to the faster equality functions. If these don't pass, # Run the assertions that are typically slower. def is_equal(x, y): + try: + if areinstance(x, y, (str, int, float, bool, type(None))): + return x == y + + if is_collection_of_primitives(x): + return x == y + + if areinstance(x, y, (Exception,)): + # Types of errors don't matter (this is debatable) + return str(x) == str(y) + + if areinstance(x, y, (list, tuple,)): + if len(x) != len(y): + return False + return all(is_equal(x_element, y_element) for x_element, y_element in zip(x, y)) + + if areinstance(x, y, (map, filter,)): + x_list, y_list = list(x), list(y) + if len(x_list) != len(y_list): + return False + return all(is_equal(x_element, y_element) for x_element, y_element in zip(x_list, y_list)) + + if areinstance(x, y, (set,)): + return is_equal(sorted(x), sorted(y)) + + if areinstance(x, y, (dict,)): + if x.keys() != y.keys(): + return False + return all(is_equal(x[key], y[key]) for key in x) + + # Delay importing pandas / numpy until absolutely necessary. This is important for performance in Pyodide. + # Also, assume they may not be available, as Pyodide won't install them unless they are needed. try: - if objs_are(x, y, [Exception]): - # Types of errors don't matter (this is debatable) - return str(x) == str(y) - if objs_are(x, y, [np.ndarray, dict, list, tuple]): - if np.array_equal(x, y): return True + import numpy as np + + if areinstance(x, y, (np.ndarray,)): np.testing.assert_equal(x, y) return True - elif objs_are(x, y, [map, filter]): - return np.array_equal(list(x), list(y)) - elif objs_are(x, y, [pd.DataFrame]): - if x.equals(y): return True - pd.util.testing.assert_frame_equal(x, y) + except ImportError: + if areinstance(x, y, (np.ndarray,)): + raise RuntimeError("NumPy is required for comparing NumPy objects.") + + try: + import pandas as pd + from pandas.testing import assert_frame_equal, assert_series_equal + + if areinstance(x, y, (pd.DataFrame,)): + if x.equals(y): + return True + assert_frame_equal(x, y) return True - elif objs_are(x, y, [pd.Series]): - if x.equals(y): return True - pd.util.testing.assert_series_equal(x, y) + elif areinstance(x, y, (pd.Series,)): + if x.equals(y): + return True + assert_series_equal(x, y) return True - else: - return x == y + except ImportError: + if areinstance(x, y, (pd.DataFrame, pd.Series)): + raise RuntimeError("pandas is required for comparing pandas objects.") - except Exception: - return False + return x == y -## Others + except Exception: + return False -class BiggerTest(Test): - """ - Check if one object is greater than another. This test should only be used with numeric variables (for now). - Attributes: - feedback (str): A string containing the failure message in case the test fails. - obj1 (str): The first object, that should be the greatest - obj2 (str): The second object, that should be smaller - result (bool): True if the test succeed, False if it failed. None if it hasn't been tested yet. - """ +# Others - def __init__(self, obj1, obj2, feedback): - """ - Initialize with two objects. - Args: - obj1 (str): The first object, obj1 will be set to this. - obj2 (str): The second object, obj2 will be set to this. - feedback (str): The failure message will be set to this. - """ - super().__init__(feedback) - self.obj1 = obj1 - self.obj2 = obj2 +class BiggerTest(EqualTest): + """ + Check if the first object is greater than another. + """ - def specific_test(self): - """ - Perform the actual test. result is set to False if the objects differ, True otherwise. - """ - self.result = (self.obj1 > self.obj2) + def __init__(self, *args): + super().__init__(*args, func=lambda obj1, obj2: obj1 > obj2) class StringContainsTest(Test): @@ -243,18 +225,12 @@ def __init__(self, string, search_string, pattern, feedback): self.search_string = search_string self.pattern = pattern - def specific_test(self): + def test(self): """ Perform the actual test. result will be True if string is found (whether or not with a pattern), False otherwise. """ if self.pattern: - self.result = ( - re.search( - self.search_string, - self.string) is not None) + self.result = re.search(self.search_string, self.string) is not None else: - self.result = (self.string.find(self.search_string) is not -1) - - - + self.result = self.string.find(self.search_string) != -1 diff --git a/pythonwhat/__init__.py b/pythonwhat/__init__.py index e5c1c4e3..47b6d8fd 100644 --- a/pythonwhat/__init__.py +++ b/pythonwhat/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.18.1' +__version__ = "2.30.1" from .test_exercise import test_exercise, allow_errors diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py deleted file mode 100644 index 6280c258..00000000 --- a/pythonwhat/check_funcs.py +++ /dev/null @@ -1,417 +0,0 @@ -from pythonwhat.tasks import getResultInProcess, getOutputInProcess, getErrorInProcess, ReprFail, setUpNewEnvInProcess, breakDownNewEnvInProcess -from pythonwhat.has_funcs import has_part -from pythonwhat.check_logic import multi -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest, TestFail -from pythonwhat.Feedback import Feedback, InstructorError -from pythonwhat.utils import get_ord -from pythonwhat.utils_ast import assert_ast -from functools import partial -import ast -from jinja2 import Template - -def render(template, kwargs): - return Template(template).render(**kwargs) - -class StubState(): - def __init__(self, highlight, highlighting_disabled): - self.highlight = highlight - self.highlighting_disabled = highlighting_disabled - -def part_to_child(stu_part, sol_part, append_message, state, node_name=None): - # stu_part and sol_part will be accessible on all templates - append_message['kwargs'].update({'stu_part': stu_part, 'sol_part': sol_part}) - - # if the parts are dictionaries, use to deck out child state - if all(isinstance(p, dict) for p in [stu_part, sol_part]): - return state.to_child_state(student_subtree=stu_part['node'], - solution_subtree=sol_part['node'], - student_context=stu_part.get('target_vars'), - solution_context=sol_part.get('target_vars'), - student_parts=stu_part, - solution_parts=sol_part, - highlight = stu_part.get('highlight'), - append_message = append_message, - node_name=node_name) - - # otherwise, assume they are just nodes - return state.to_child_state(student_subtree=stu_part, - solution_subtree=sol_part, - append_message=append_message, - node_name=node_name) - -def check_part(name, part_msg, - missing_msg=None, - expand_msg=None, - state=None): - """Return child state with name part as its ast tree""" - - if missing_msg is None: missing_msg = "Are you sure you defined the {{part}}? " - if expand_msg is None: expand_msg = "Did you correctly specify the {{part}}? " - - if not part_msg: part_msg = name - append_message = {'msg': expand_msg, 'kwargs': { 'part': part_msg }} - - has_part(name, missing_msg, state, append_message['kwargs']) - - stu_part = state.student_parts[name] - sol_part = state.solution_parts[name] - - assert_ast(state, sol_part, append_message['kwargs']) - - return part_to_child(stu_part, sol_part, append_message, state) - -def check_part_index(name, index, part_msg, - missing_msg=None, - expand_msg=None, - state=None): - """Return child state with indexed name part as its ast tree. - - ``index`` can be: - - - an integer, in which case the student/solution_parts are indexed by position. - - a string, in which case the student/solution_parts are expected to be a dictionary. - - a list of indices (which can be integer or string), in which case the student parts are indexed step by step. - """ - - if missing_msg is None: missing_msg = "Are you sure you defined the {{part}}? " - if expand_msg is None: expand_msg = "Did you correctly specify the {{part}}? " - - # create message - ordinal = get_ord(index+1) if isinstance(index, int) else "" - fmt_kwargs = { - 'index': index, - 'ordinal': ordinal - } - fmt_kwargs.update(part = render(part_msg, fmt_kwargs)) - - append_message = { - 'msg': expand_msg, - 'kwargs': fmt_kwargs - } - - # check there are enough parts for index - has_part(name, missing_msg, state, fmt_kwargs, index) - - # get part at index - stu_part = state.student_parts[name] - sol_part = state.solution_parts[name] - - if isinstance(index, list): - for ind in index: - stu_part = stu_part[ind] - sol_part = sol_part[ind] - else: - stu_part = stu_part[index] - sol_part = sol_part[index] - - assert_ast(state, sol_part, fmt_kwargs) - - # return child state from part - return part_to_child(stu_part, sol_part, append_message, state) - -def check_node(name, - index=0, - typestr='{{ordinal}} node', - missing_msg=None, - expand_msg=None, - state=None): - - if missing_msg is None: missing_msg = "The system wants to check the {{typestr}} but hasn't found it." - if expand_msg is None: expand_msg = "Check the {{typestr}}. " - - rep = Reporter.active_reporter - stu_out = getattr(state, 'student_'+name) - sol_out = getattr(state, 'solution_'+name) - - # check if there are enough nodes for index - fmt_kwargs = { - 'ordinal': get_ord(index+1) if isinstance(index, int) else "", - 'index': index, - 'name': name - } - fmt_kwargs['typestr'] = render(typestr, fmt_kwargs) - - # test if node can be indexed succesfully - try: stu_out[index] - except (KeyError, IndexError): # TODO comment errors - _msg = state.build_message(missing_msg, fmt_kwargs) - rep.do_test(Test(Feedback(_msg, state))) - - # get node at index - stu_part = stu_out[index] - sol_part = sol_out[index] - - append_message = { - 'msg': expand_msg, - 'kwargs': fmt_kwargs - } - - return part_to_child(stu_part, sol_part, append_message, state, node_name=name) - -# context functions ----------------------------------------------------------- - -def with_context(*args, state=None): - - rep = Reporter.active_reporter - - # set up context in processes - solution_res = setUpNewEnvInProcess(process = state.solution_process, - context = state.solution_parts['with_items']) - if isinstance(solution_res, Exception): - raise InstructorError("error in the solution, running test_with(): %s" % str(solution_res)) - - student_res = setUpNewEnvInProcess(process = state.student_process, - context = state.student_parts['with_items']) - if isinstance(student_res, AttributeError): - rep.do_test(Test(Feedback("In your `with` statement, you're not using a correct context manager.", child.highlight))) - - if isinstance(student_res, (AssertionError, ValueError, TypeError)): - rep.do_test(Test(Feedback("In your `with` statement, the number of values in your context manager " - "doesn't correspond to the number of variables you're trying to assign it to.", child.highlight))) - - # run subtests - try: - multi(*args, state=state) - finally: - # exit context - if breakDownNewEnvInProcess(process = state.solution_process): - raise InstructorError("error in the solution, closing the `with` fails with: %s" % (close_solution_context)) - - if breakDownNewEnvInProcess(process = state.student_process): - - rep.do_test(Test(Feedback("Your `with` statement can not be closed off correctly, you're " + \ - "not using the context manager correctly.", state))) - return state - -def check_args(name, missing_msg=None, state=None): - """Check whether a function argument is specified. - - This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified. - If you want to go on and check whether the argument was correctly specified, you can can continue chaining with - ``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check) - - This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been - specified. - - Args: - name (str): the name of the argument for which you want to check it is specified. This can also be - a number, in which case it refers to the positional arguments. Named argumetns take precedence. - missing_msg (str): If specified, this overrides an automatically generated feedback message in case - the student did specify the argument. - state (State): State object that is passed from the SCT Chain (don't specify this). - - :Examples: - - Student and solution code:: - - import numpy as np - arr = np.array([1, 2, 3, 4, 5]) - np.mean(arr) - - SCT:: - - # Verify whether arr was correctly set in np.mean - # has_equal_value() checks the value of arr, used to set argument a - Ex().check_function('numpy.mean').check_args('a').has_equal_value() - - # Verify whether arr was correctly set in np.mean - # has_equal_ast() checks the expression used to set argument a - Ex().check_function('numpy.mean').check_args('a').has_equal_ast() - - Student and solution code:: - - def my_power(x): - print("calculating sqrt...") - return(x * x) - - SCT:: - - Ex().check_function_def('my_power').multi( - check_args('x') # will fail if student used y as arg - check_args(0) # will still pass if student used y as arg - ) - - """ - if missing_msg is None: - missing_msg = 'Did you specify the {{part}}?' - - if name in ['*args', '**kwargs']: # for check_function_def - return check_part(name, name, state=state, missing_msg = missing_msg) - else: - if isinstance(name, list): # dealing with args or kwargs - if name[0] == 'args': - arg_str = "%s argument passed as a variable length argument"%get_ord(name[1]+1) - else: - arg_str = "argument `%s`"%name[1] - else: - arg_str = "%s argument" % get_ord(name+1) if isinstance(name, int) else "argument `%s`" % name - return check_part_index('args', name, arg_str, missing_msg = missing_msg, state=state) - - -# CALL CHECK ================================================================== - -evalCalls = {'value': getResultInProcess, - 'output': getOutputInProcess, - 'error': getErrorInProcess} - -call_warnings = { - 'value': 'in the solution process resulted in an error', - 'error': 'did not generate an error in the solution environment', - 'output': 'in the solution process resulted in an error' - } - -def fix_format(arguments): - if isinstance(arguments, str): - arguments = (arguments, ) - if isinstance(arguments, tuple): - arguments = list(arguments) - - if isinstance(arguments, list): - arguments = {'args': arguments, 'kwargs': {}} - - if not isinstance(arguments, dict) or 'args' not in arguments or 'kwargs' not in arguments: - raise ValueError("Wrong format of arguments in 'results', 'outputs' or 'errors'; either a list, or a dictionary with names args (a list) and kwargs (a dict)") - - return(arguments) - -def stringify(arguments): - vararg = str(arguments['args'])[1:-1] - kwarg = ', '.join(['%s = %s' % (key, value) for key, value in arguments['kwargs'].items()]) - if len(vararg) == 0: - if len(kwarg) == 0: - return "()" - else: - return "(" + kwarg + ")" - else : - if len(kwarg) == 0: - return "(" + vararg + ")" - else : - return "(" + ", ".join([vararg, kwarg]) + ")" - -# TODO: test string syntax with check_function_def -# test argument syntax with check_lambda_function -def run_call(args, node, process, get_func, **kwargs): - # Get function expression - if isinstance(node, ast.FunctionDef): # function name - func_expr = ast.Name(id=node.name, ctx=ast.Load()) - elif isinstance(node, ast.Lambda): # lambda body expr - func_expr = node - else: raise InstructorError("Only function definition or lambda may be called") - - ast.fix_missing_locations(func_expr) - return get_func(process = process, tree=func_expr, call = args, **kwargs) - -MSG_CALL_INCORRECT = "Calling {{argstr}} should {{action}} `{{str_sol}}`, instead got {{str_stu if str_stu == 'no printouts' else '`' + str_stu + '`'}}." -MSG_CALL_ERROR = "Calling {{argstr}} should {{action}} `{{str_sol}}`, instead it errored out: `{{str_stu}}`." -MSG_CALL_ERROR_INV = "Calling {{argstr}} should {{action}} `{{str_sol}}`, instead got `{{str_stu}}`." -def call(args, - test='value', - incorrect_msg=None, - error_msg=None, - argstr=None, - func=None, - state=None, **kwargs): - """Use ``check_call()`` in combination with ``has_equal_x()`` instead. - """ - - if incorrect_msg is None: - incorrect_msg = MSG_CALL_INCORRECT - if error_msg is None: - error_msg = MSG_CALL_ERROR_INV if test == 'error' else MSG_CALL_ERROR - - rep = Reporter.active_reporter - - assert test in ('value', 'output', 'error') - - get_func = evalCalls[test] - - # Run for Solution -------------------------------------------------------- - eval_sol, str_sol = run_call(args, state.solution_parts['node'], state.solution_process, get_func, **kwargs) - - if (test == 'error') ^ isinstance(eval_sol, Exception): - _msg = state.build_message("Calling {{argstr}} resulted in an error (or not an error if testing for one). Error message: {{type_err}} {{str_sol}}", - dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr)), - raise InstructorError(_msg) - - if isinstance(eval_sol, ReprFail): - _msg = state.build_message("Can't get the result of calling {{argstr}}: {{eval_sol.info}}", - dict(argstr = argstr, eval_sol=eval_sol)) - raise InstructorError(_msg) - - # Run for Submission ------------------------------------------------------ - eval_stu, str_stu = run_call(args, state.student_parts['node'], state.student_process, get_func, **kwargs) - action_strs = {'value': 'return', 'output': 'print out', 'error': 'error out with the message'} - fmt_kwargs = {'part': argstr, 'argstr': argstr, 'str_sol': str_sol, 'str_stu': str_stu, 'action': action_strs[test]} - - # either error test and no error, or vice-versa - stu_node = state.student_parts['node'] - stu_state = StubState(stu_node, state.highlighting_disabled) - if (test == 'error') ^ isinstance(eval_stu, Exception): - _msg = state.build_message(error_msg, fmt_kwargs) - rep.do_test(Test(Feedback(_msg, stu_state))) - - # incorrect result - _msg = state.build_message(incorrect_msg, fmt_kwargs) - rep.do_test(EqualTest(eval_sol, eval_stu, Feedback(_msg, stu_state), func)) - - return state - -def build_call(callstr, node): - if isinstance(node, ast.FunctionDef): # function name - func_expr = ast.Name(id=node.name, ctx=ast.Load()) - argstr = "`%s`" % callstr.replace('f', node.name) - elif isinstance(node, ast.Lambda): # lambda body expr - func_expr = node - argstr = 'it with the arguments `%s`' % callstr.replace('f', '') - else: - raise TypeError("Can't handle AST that is passed.") - - parsed = ast.parse(callstr).body[0].value - parsed.func = func_expr - ast.fix_missing_locations(parsed) - return parsed, argstr - -def check_call(callstr, argstr = None, expand_msg=None, state=None): - """When checking a function definition of lambda function, - prepare has_equal_x for checking the call of a user-defined function. - - Args: - callstr (str): call string that specifies how the function should be called, e.g. `f(1, a = 2)`. - ``check_call()`` will replace ``f`` with the function/lambda you're targeting. - argstr (str): If specified, this overrides the way the function call is refered to in the expand message. - expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. - state (State): state object that is chained from. - - :Example: - - Student and solution code:: - - def my_power(x): - print("calculating sqrt...") - return(x * x) - - SCT:: - - Ex().check_function_def('my_power').multi( - check_call("f(3)").has_equal_value() - check_call("f(3)").has_equal_output() - ) - """ - - state.assert_is( - ['function_defs', 'lambda_functions'], - 'check_call', - ['check_function_def', 'check_lambda_function'] - ) - - if expand_msg is None: - expand_msg = "To verify it, we reran {{argstr}}. " - - stu_part, _argstr = build_call(callstr, state.student_parts['node']) - sol_part, _ = build_call(callstr, state.solution_parts['node']) - - append_message = { 'msg': expand_msg, 'kwargs': {'argstr': argstr or _argstr }} - child = part_to_child(stu_part, sol_part, append_message, state) - - return child \ No newline at end of file diff --git a/pythonwhat/check_syntax.py b/pythonwhat/check_syntax.py deleted file mode 100644 index 0e2f5807..00000000 --- a/pythonwhat/check_syntax.py +++ /dev/null @@ -1,130 +0,0 @@ -from pythonwhat.check_wrappers import scts -from pythonwhat.State import State -from pythonwhat.probe import Node, Probe, TEST_NAMES -from pythonwhat.utils import include_v1 -from pythonwhat import test_funcs -from functools import partial, reduce, wraps -import inspect -import copy -import os - -# TODO: could define scts for check_wrappers at the module level -ATTR_SCTS = scts.copy() - -def multi_dec(f): - """Decorator for multi to remove nodes for original test functions from root node""" - - @wraps(f) - def wrapper(*args, **kwargs): - args = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args - for arg in args: - if isinstance(arg, Node) and arg.parent.name is 'root': - arg.parent.remove_child(arg) - arg.update_child_calls() - return f(*args, **kwargs) - return wrapper - - -def state_dec(f): - """Decorate check_* functions to return F chain if no state passed""" - - @wraps(f) - def wrapper(*args, **kwargs): - ba = inspect.signature(f).bind(*args, **kwargs) - ba.apply_defaults() - - state_arg = ba.arguments.get('state') - if isinstance(state_arg, State): # proper state, run function - return f(*args, **kwargs) - elif state_arg is None: # default state arg, make partial - return F._from_func(partial(f, *args, **kwargs)) - else: # passed improper state arg - raise BaseException("Did you use the right number of arguments in your SCT?") - - return wrapper - -class Chain: - def __init__(self, state): - self._state = state - self._crnt_sct = None # last called SCT - self._waiting_on_call = False - - def _double_attr_error(self): - raise AttributeError("Did you forget to call a statement? " - "e.g. Ex().check_list_comp.check_body()") - - def __getattr__(self, attr): - if attr not in ATTR_SCTS: raise AttributeError("No SCT named %s"%attr) - elif self._waiting_on_call: self._double_attr_error() - else: - # make a copy to return, - # in case someone does: a = chain.a; b = chain.b - return self._sct_copy(ATTR_SCTS[attr]) - - def __call__(self, *args, **kwargs): - self._state = self._crnt_sct(state=self._state, *args, **kwargs) - self._waiting_on_call = False - return self - - def __rshift__(self, f): - if self._waiting_on_call: - self._double_attr_error() - elif type(f) == Chain: - raise BaseException("did you use a result of the Ex() function on the right hand side of the >> operator?") - elif not callable(f): - raise BaseException("right hand side of >> operator should be an SCT, so must be callable!") - else: - chain = self._sct_copy(f) - return chain() - - def _sct_copy(self, f): - chain = copy.copy(self) - chain._crnt_sct = f - chain._waiting_on_call = True - return chain - -class F(Chain): - """ - Chain with deferred State passing - """ - def __init__(self, stack = None): - self._crnt_sct = None - self._stack = [] if stack is None else stack - self._waiting_on_call = False - - def __call__(self, *args, **kwargs): - if not self._crnt_sct: - # first function in chain - state = kwargs.get('state') or args[0] - return reduce(lambda s, f: f(state=s), self._stack, state) - else: - pf = partial(self._crnt_sct, *args, **kwargs) - return self.__class__(self._stack + [pf]) - - @classmethod - def _from_func(cls, f): - func_chain = cls(stack = [f]) - return func_chain - -def Ex(state = None): - return Chain(state or State.root_state) - -if include_v1(): - # Prepare SCTs that may be chained attributes ---------------------- - # decorate functions that may try to run test_* function nodes as subtests - # so they remove those nodes from the tree - for k in ['multi', 'with_context']: - ATTR_SCTS[k] = multi_dec(ATTR_SCTS[k]) - - # allow test_* functions as chained attributes - for k in TEST_NAMES: - ATTR_SCTS[k] = Probe(tree = None, f = getattr(test_funcs, k), eval_on_call=True) - - # original logical test_* functions behave like multi - # this is necessary to allow them to take check_* funcs as args - # since probe behavior will try to call all SCTs passed (assuming they're also probes) - for k in ['test_or', 'test_correct']: - ATTR_SCTS[k] = multi_dec(getattr(test_funcs, k)) - -# Prepare check_funcs to be used alone (e.g. test = check_with().check_body()) -v2_check_functions = {k : state_dec(v) for k, v in scts.items()} diff --git a/pythonwhat/checks/__init__.py b/pythonwhat/checks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pythonwhat/checks/check_funcs.py b/pythonwhat/checks/check_funcs.py new file mode 100644 index 00000000..be618aa4 --- /dev/null +++ b/pythonwhat/checks/check_funcs.py @@ -0,0 +1,329 @@ +from protowhat.Feedback import FeedbackComponent +from pythonwhat.checks.check_logic import multi +from pythonwhat.checks.has_funcs import has_part +from protowhat.failure import debugger +from pythonwhat.tasks import setUpNewEnvInProcess, breakDownNewEnvInProcess +from protowhat.utils_messaging import get_ord +from pythonwhat.utils_ast import assert_ast +import ast +from jinja2 import Template + + +def render(template, kwargs): + return Template(template).render(**kwargs) + + +def part_to_child(stu_part, sol_part, append_message, state, node_name=None): + # stu_part and sol_part will be accessible on all templates + append_message.kwargs.update({"stu_part": stu_part, "sol_part": sol_part}) + + # if the parts are dictionaries, use to deck out child state + if all(isinstance(p, dict) for p in [stu_part, sol_part]): + child_state = state.to_child( + student_ast=stu_part["node"], + solution_ast=sol_part["node"], + student_context=stu_part.get("target_vars"), + solution_context=sol_part.get("target_vars"), + student_parts=stu_part, + solution_parts=sol_part, + highlight=stu_part.get("highlight"), + append_message=append_message, + node_name=node_name, + ) + else: + # otherwise, assume they are just nodes + child_state = state.to_child( + student_ast=stu_part, + solution_ast=sol_part, + append_message=append_message, + node_name=node_name, + ) + + return child_state + + +def check_part(state, name, part_msg, missing_msg=None, expand_msg=None): + """Return child state with name part as its ast tree""" + + if missing_msg is None: + missing_msg = "Are you sure you defined the {{part}}? " + if expand_msg is None: + expand_msg = "Did you correctly specify the {{part}}? " + + if not part_msg: + part_msg = name + append_message = FeedbackComponent(expand_msg, {"part": part_msg}) + + has_part(state, name, missing_msg, append_message.kwargs) + + stu_part = state.student_parts[name] + sol_part = state.solution_parts[name] + + assert_ast(state, sol_part, append_message.kwargs) + + return part_to_child(stu_part, sol_part, append_message, state) + + +def check_part_index(state, name, index, part_msg, missing_msg=None, expand_msg=None): + """Return child state with indexed name part as its ast tree. + + ``index`` can be: + + - an integer, in which case the student/solution_parts are indexed by position. + - a string, in which case the student/solution_parts are expected to be a dictionary. + - a list of indices (which can be integer or string), in which case the student parts are indexed step by step. + """ + + if missing_msg is None: + missing_msg = "Are you sure you defined the {{part}}? " + if expand_msg is None: + expand_msg = "Did you correctly specify the {{part}}? " + + # create message + ordinal = get_ord(index + 1) if isinstance(index, int) else "" + fmt_kwargs = {"index": index, "ordinal": ordinal} + fmt_kwargs.update(part=render(part_msg, fmt_kwargs)) + + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + + # check there are enough parts for index + has_part(state, name, missing_msg, fmt_kwargs, index) + + # get part at index + stu_part = state.student_parts[name] + sol_part = state.solution_parts[name] + + if isinstance(index, list): + for ind in index: + stu_part = stu_part[ind] + sol_part = sol_part[ind] + else: + stu_part = stu_part[index] + sol_part = sol_part[index] + + assert_ast(state, sol_part, fmt_kwargs) + + # return child state from part + return part_to_child(stu_part, sol_part, append_message, state) + + +def check_node( + state, name, index=0, typestr="{{ordinal}} node", missing_msg=None, expand_msg=None +): + + if missing_msg is None: + missing_msg = "The system wants to check the {{typestr}} but hasn't found it." + if expand_msg is None: + expand_msg = "Check the {{typestr}}. " + + stu_out = state.ast_dispatcher.find(name, state.student_ast) + sol_out = state.ast_dispatcher.find(name, state.solution_ast) + + # check if there are enough nodes for index + fmt_kwargs = { + "ordinal": get_ord(index + 1) if isinstance(index, int) else "", + "index": index, + "name": name, + } + fmt_kwargs["typestr"] = render(typestr, fmt_kwargs) + + # test if node can be indexed succesfully + try: + stu_out[index] + except (KeyError, IndexError): # TODO comment errors + state.report(missing_msg, fmt_kwargs) + + # get node at index + stu_part = stu_out[index] + sol_part = sol_out[index] + + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + + return part_to_child(stu_part, sol_part, append_message, state, node_name=name) + + +# context functions ----------------------------------------------------------- +# TODO: check if still useful +def with_context(state, *args, child=None): + + # set up context in processes + solution_res = setUpNewEnvInProcess( + process=state.solution_process, context=state.solution_parts["with_items"] + ) + if isinstance(solution_res, Exception): + with debugger(state): + state.report( + "error in the solution, running test_with(): %s" % str(solution_res) + ) + + student_res = setUpNewEnvInProcess( + process=state.student_process, context=state.student_parts["with_items"] + ) + if isinstance(student_res, AttributeError): + child.report( + "In your `with` statement, you're not using a correct context manager." + ) + + if isinstance(student_res, (AssertionError, ValueError, TypeError)): + child.report( + "In your `with` statement, the number of values in your context manager " + "doesn't correspond to the number of variables you're trying to assign it to." + ) + + # run subtests + try: + multi(state, *args) + finally: + # exit context + close_solution_context = breakDownNewEnvInProcess( + process=state.solution_process + ) + if isinstance(close_solution_context, Exception): + with debugger(state): + state.report( + "error in the solution, closing the `with` fails with: %s" + % close_solution_context + ) + + close_student_context = breakDownNewEnvInProcess(process=state.student_process) + if isinstance(close_student_context, Exception): + state.report( + "Your `with` statement can not be closed off correctly, you're " + "not using the context manager correctly." + ) + return state + + +def check_args(state, name, missing_msg=None): + """Check whether a function argument is specified. + + This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified. + If you want to go on and check whether the argument was correctly specified, you can can continue chaining with + ``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check) + + This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been + specified. + + Args: + name (str): the name of the argument for which you want to check if it is specified. This can also be + a number, in which case it refers to the positional arguments. Named arguments take precedence. + missing_msg (str): If specified, this overrides the automatically generated feedback message in case + the student did specify the argument. + state (State): State object that is passed from the SCT Chain (don't specify this). + + :Examples: + + Student and solution code:: + + import numpy as np + arr = np.array([1, 2, 3, 4, 5]) + np.mean(arr) + + SCT:: + + # Verify whether arr was correctly set in np.mean + # has_equal_value() checks the value of arr, used to set argument a + Ex().check_function('numpy.mean').check_args('a').has_equal_value() + + # Verify whether arr was correctly set in np.mean + # has_equal_ast() checks the expression used to set argument a + Ex().check_function('numpy.mean').check_args('a').has_equal_ast() + + Student and solution code:: + + def my_power(x): + print("calculating sqrt...") + return(x * x) + + SCT:: + + Ex().check_function_def('my_power').multi( + check_args('x') # will fail if student used y as arg + check_args(0) # will still pass if student used y as arg + ) + + """ + if missing_msg is None: + missing_msg = "Did you specify the {{part}}?" + + if name in ["*args", "**kwargs"]: # for check_function_def + return check_part(state, name, name, missing_msg=missing_msg) + else: + if isinstance(name, list): # dealing with args or kwargs + if name[0] == "args": + arg_str = "{} argument passed as a variable length argument".format( + get_ord(name[1] + 1) + ) + else: + arg_str = "argument `{}`".format(name[1]) + else: + arg_str = ( + "{} argument".format(get_ord(name + 1)) + if isinstance(name, int) + else "argument `{}`".format(name) + ) + return check_part_index(state, "args", name, arg_str, missing_msg=missing_msg) + + +# CALL CHECK ================================================================== + + +def build_call(callstr, node): + if isinstance(node, ast.FunctionDef): # function name + func_expr = ast.Name(id=node.name, ctx=ast.Load()) + argstr = "`%s`" % callstr.replace("f", node.name) + elif isinstance(node, ast.Lambda): # lambda body expr + func_expr = node + argstr = "it with the arguments `%s`" % callstr.replace("f", "") + else: + raise TypeError("Can't handle AST that is passed.") + + parsed = ast.parse(callstr).body[0].value + parsed.func = func_expr + ast.fix_missing_locations(parsed) + return parsed, argstr + + +def check_call(state, callstr, argstr=None, expand_msg=None): + """When checking a function definition of lambda function, + prepare has_equal_x for checking the call of a user-defined function. + + Args: + callstr (str): call string that specifies how the function should be called, e.g. `f(1, a = 2)`. + ``check_call()`` will replace ``f`` with the function/lambda you're targeting. + argstr (str): If specified, this overrides the way the function call is refered to in the expand message. + expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. + state (State): state object that is chained from. + + :Example: + + Student and solution code:: + + def my_power(x): + print("calculating sqrt...") + return(x * x) + + SCT:: + + Ex().check_function_def('my_power').multi( + check_call("f(3)").has_equal_value() + check_call("f(3)").has_equal_output() + ) + """ + + state.assert_is( + ["function_defs", "lambda_functions"], + "check_call", + ["check_function_def", "check_lambda_function"], + ) + + if expand_msg is None: + expand_msg = "To verify it, we reran {{argstr}}. " + + stu_part, _argstr = build_call(callstr, state.student_parts["node"]) + sol_part, _ = build_call(callstr, state.solution_parts["node"]) + + append_message = FeedbackComponent(expand_msg, {"argstr": argstr or _argstr}) + child = part_to_child(stu_part, sol_part, append_message, state) + + return child diff --git a/pythonwhat/check_function.py b/pythonwhat/checks/check_function.py similarity index 56% rename from pythonwhat/check_function.py rename to pythonwhat/checks/check_function.py index 4cf90f9e..ca093c24 100644 --- a/pythonwhat/check_function.py +++ b/pythonwhat/checks/check_function.py @@ -1,41 +1,54 @@ -from pythonwhat.Reporter import Reporter -from pythonwhat.check_funcs import part_to_child, StubState +from protowhat.Feedback import FeedbackComponent +from pythonwhat.checks.check_funcs import part_to_child from pythonwhat.tasks import getSignatureInProcess -from pythonwhat.utils import get_ord, get_times -from pythonwhat.Test import Test -from pythonwhat.Feedback import Feedback, InstructorError +from protowhat.utils_messaging import get_ord, get_times +from protowhat.failure import debugger from pythonwhat.parsing import IndexedDict from functools import partial + def bind_args(signature, args_part): - pos_args = []; kw_args = {} + pos_args = [] + kw_args = {} for k, arg in args_part.items(): - if isinstance(k, int): pos_args.append(arg) - else: kw_args[k] = arg - + if isinstance(k, int): + pos_args.append(arg) + else: + kw_args[k] = arg + bound_args = signature.bind(*pos_args, **kw_args) return IndexedDict(bound_args.arguments) + def get_mapped_name(name, mappings): # get name by splitting on periods if "." in name: for orig, full_name in mappings.items(): - if name.startswith(full_name): return name.replace(full_name, orig) + if name.startswith(full_name): + return name.replace(full_name, orig) return name + MISSING_MSG = "Did you call `{{mapped_name}}()`{{' ' + times if index>0}}?" -SIG_ISSUE_MSG = "Have you specified the arguments for `{{mapped_name}}()` using the right syntax?" +SIG_ISSUE_MSG = ( + "Have you specified the arguments for `{{mapped_name}}()` using the right syntax?" +) PREPEND_MSG = "Check your {{ord + ' ' if index>0}}call of `{{mapped_name}}()`. " -def check_function(name, index=0, - missing_msg=None, - params_not_matched_msg=None, - expand_msg=None, - signature=True, - state=None): + + +def check_function( + state, + name, + index=0, + missing_msg=None, + params_not_matched_msg=None, + expand_msg=None, + signature=True, +): """Check whether a particular function is called. ``check_function()`` is typically followed by: - + - ``check_args()`` to check whether the arguments were specified. In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()`` to assert that the arguments were correctly specified. @@ -85,53 +98,75 @@ def check_function(name, index=0, if params_not_matched_msg is None: params_not_matched_msg = SIG_ISSUE_MSG - rep = Reporter.active_reporter - stu_out = state.student_function_calls - sol_out = state.solution_function_calls + stu_out = state.ast_dispatcher.find("function_calls", state.student_ast) + sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast) - student_mappings = state.student_mappings + student_mappings = state.ast_dispatcher.find("mappings", state.student_ast) - fmt_kwargs = {'times': get_times(index+1), - 'ord': get_ord(index+1), - 'index': index, - 'mapped_name': get_mapped_name(name, student_mappings)} + fmt_kwargs = { + "times": get_times(index + 1), + "ord": get_ord(index + 1), + "index": index, + "mapped_name": get_mapped_name(name, student_mappings), + } # Get Parts ---- # Copy, otherwise signature binding overwrites sol_out[name][index]['args'] - try: - sol_parts = {**sol_out[name][index]} - except KeyError: - raise InstructorError("`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!" % name) - except IndexError: - raise InstructorError("`check_function()` couldn't find %s calls of `%s()` in your solution code." % (index+1, name)) + with debugger(state): + try: + sol_parts = {**sol_out[name][index]} + except KeyError: + state.report( + "`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!" + % name + ) + except IndexError: + state.report( + "`check_function()` couldn't find %s calls of `%s()` in your solution code." + % (index + 1, name) + ) try: # Copy, otherwise signature binding overwrites stu_out[name][index]['args'] stu_parts = {**stu_out[name][index]} except (KeyError, IndexError): - _msg = state.build_message(missing_msg, fmt_kwargs, append=append_missing) - rep.do_test(Test(Feedback(_msg, state))) + state.report(missing_msg, fmt_kwargs, append=append_missing) # Signatures ----- if signature: signature = None if isinstance(signature, bool) else signature - get_sig = partial(getSignatureInProcess, name=name, signature=signature, - manual_sigs = state.get_manual_sigs()) + get_sig = partial( + getSignatureInProcess, + name=name, + signature=signature, + manual_sigs=state.get_manual_sigs(), + ) try: - sol_sig = get_sig(mapped_name=sol_parts['name'], process=state.solution_process) - sol_parts['args'] = bind_args(sol_sig, sol_parts['args']) - except: - raise InstructorError("`check_function()` couldn't match the %s call of `%s` to its signature. " % (get_ord(index + 1), name)) + sol_sig = get_sig( + mapped_name=sol_parts["name"], process=state.solution_process + ) + sol_parts["args"] = bind_args(sol_sig, sol_parts["args"]) + except Exception as e: + with debugger(state): + state.report( + "`check_function()` couldn't match the %s call of `%s` to its signature:\n%s " + % (get_ord(index + 1), name, e) + ) try: - stu_sig = get_sig(mapped_name=stu_parts['name'], process=state.student_process) - stu_parts['args'] = bind_args(stu_sig, stu_parts['args']) + stu_sig = get_sig( + mapped_name=stu_parts["name"], process=state.student_process + ) + stu_parts["args"] = bind_args(stu_sig, stu_parts["args"]) except Exception: - _msg = state.build_message(params_not_matched_msg, fmt_kwargs, append=append_params_not_matched) - rep.do_test(Test(Feedback(_msg, StubState(stu_parts['node'], state.highlighting_disabled)))) + state.to_child(highlight=stu_parts["node"]).report( + params_not_matched_msg, fmt_kwargs, append=append_params_not_matched + ) # three types of parts: pos_args, keywords, args (e.g. these are bound to sig) - append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs} - child = part_to_child(stu_parts, sol_parts, append_message, state, node_name='function_calls') + append_message = FeedbackComponent(expand_msg, fmt_kwargs) + child = part_to_child( + stu_parts, sol_parts, append_message, state, node_name="function_calls" + ) return child diff --git a/pythonwhat/check_has_context.py b/pythonwhat/checks/check_has_context.py similarity index 50% rename from pythonwhat/check_has_context.py rename to pythonwhat/checks/check_has_context.py index e2800daa..2a2fef24 100644 --- a/pythonwhat/check_has_context.py +++ b/pythonwhat/checks/check_has_context.py @@ -1,69 +1,81 @@ -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest -from pythonwhat.Feedback import Feedback, InstructorError +from pythonwhat.Test import EqualTest +from protowhat.Feedback import FeedbackComponent +from protowhat.failure import debugger from pythonwhat.State import State from functools import singledispatch -from pythonwhat.check_funcs import check_part_index +from pythonwhat.checks.check_funcs import check_part_index MSG_INCORRECT_LOOP = "Have you used the correct iterator variable names? Was expecting `{{sol_vars}}` but got `{{stu_vars}}`." MSG_INCORRECT_WITH = "Make sure to use the correct context variable names. Was expecting `{{sol_vars}}` but got `{{stu_vars}}`." -def has_context(incorrect_msg=None, exact_names=False, state=None): + +def has_context(state, incorrect_msg=None, exact_names=False): # call _has_context, since the built-in singledispatch can only use 1st pos arg return _has_context(state, incorrect_msg, exact_names) + def _test(state, incorrect_msg, exact_names, tv_name, highlight_name): - rep = Reporter.active_reporter # get parts for testing from state # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. stu_vars = state.student_parts[tv_name] sol_vars = state.solution_parts[tv_name] - child_state = state.to_child_state(student_subtree = state.student_parts.get(highlight_name), - solution_subtree = state.solution_parts.get(highlight_name)) + child_state = state.to_child( + student_ast=state.student_parts.get(highlight_name), + solution_ast=state.solution_parts.get(highlight_name), + ) # variables exposed to messages - d = { 'stu_vars': stu_vars, - 'sol_vars': sol_vars, - 'num_vars': len(sol_vars)} + d = {"stu_vars": stu_vars, "sol_vars": sol_vars, "num_vars": len(sol_vars)} if exact_names: - # message for wrong iter var names - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, child_state))) + # feedback for wrong iter var names + child_state.do_test( + EqualTest(stu_vars, sol_vars, FeedbackComponent(incorrect_msg, d)) + ) else: - # message for wrong number of iter vars - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, child_state))) + # feedback for wrong number of iter vars + child_state.do_test( + EqualTest(len(stu_vars), len(sol_vars), FeedbackComponent(incorrect_msg, d)) + ) return state @singledispatch def _has_context(state, incorrect_msg, exact_names): - raise InstructorError("first argument to _has_context must be a State instance or subclass") + with debugger(state): + state.report( + "first argument to _has_context must be a State instance or subclass" + ) + @_has_context.register(State) def has_context_state(*args, **kwargs): - return _test(*args, tv_name='target_vars', highlight_name='highlight', **kwargs) + return _test(*args, tv_name="target_vars", highlight_name="highlight", **kwargs) -@_has_context.register(State.SUBCLASSES['for_loops']) -@_has_context.register(State.SUBCLASSES['whiles']) -@_has_context.register(State.SUBCLASSES['dict_comps']) -@_has_context.register(State.SUBCLASSES['generator_exps']) -@_has_context.register(State.SUBCLASSES['list_comps']) + +@_has_context.register(State.SUBCLASSES["for_loops"]) +@_has_context.register(State.SUBCLASSES["whiles"]) +@_has_context.register(State.SUBCLASSES["dict_comps"]) +@_has_context.register(State.SUBCLASSES["generator_exps"]) +@_has_context.register(State.SUBCLASSES["list_comps"]) def has_context_loop(state, incorrect_msg, exact_names): """When dispatched on loops, has_context the target vars are the attribute _target_vars. Note: This is to allow people to call has_context on a node (e.g. for_loop) rather than one of its attributes (e.g. body). Purely for convenience. """ - return _test(state, incorrect_msg or MSG_INCORRECT_LOOP, exact_names, - tv_name='_target_vars', highlight_name='target') + return _test( + state, + incorrect_msg or MSG_INCORRECT_LOOP, + exact_names, + tv_name="_target_vars", + highlight_name="target", + ) + -@_has_context.register(State.SUBCLASSES['withs']) +@_has_context.register(State.SUBCLASSES["withs"]) def has_context_with(state, incorrect_msg, exact_names): """When dispatched on with statements, has_context loops over each context manager. @@ -73,8 +85,8 @@ def has_context_with(state, incorrect_msg, exact_names): e.g. Ex().check_with(0).has_context() vs Ex().check_with(0).check_context(0).has_context() """ - for i in range(len(state.solution_parts['context'])): - ctxt_state = check_part_index('context', i, '{{ordinal}} context', state=state) + for i in range(len(state.solution_parts["context"])): + ctxt_state = check_part_index(state, "context", i, "{{ordinal}} context") _has_context(ctxt_state, incorrect_msg or MSG_INCORRECT_WITH, exact_names) return state diff --git a/pythonwhat/check_logic.py b/pythonwhat/checks/check_logic.py similarity index 57% rename from pythonwhat/check_logic.py rename to pythonwhat/checks/check_logic.py index 4601fe78..e9b86ab5 100644 --- a/pythonwhat/check_logic.py +++ b/pythonwhat/checks/check_logic.py @@ -1,18 +1,19 @@ -from types import GeneratorType -from functools import partial -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, TestFail -from pythonwhat.Feedback import Feedback, InstructorError -import copy +from protowhat.Feedback import FeedbackComponent +from protowhat.checks.check_logic import ( + multi, + check_not, + check_or, + check_correct, + disable_highlighting, + fail, +) +from protowhat.failure import InstructorError import ast -def multi(*args, state=None): - """Run multiple subtests. Return original state (for chaining). - - Args: - state: State instance describing student and solution code. Can be omitted if used with Ex(). - tests: sub-SCTs that all should pass. +multi.__doc__ = ( + str(multi.__doc__) + + """ :Example: Suppose we want to verify the following function call: :: @@ -26,69 +27,50 @@ def multi(*args, state=None): check_args(0).has_equal_value(), check_args('ndigits').has_equal_value() ) - """ - if any(args): - rep = Reporter.active_reporter - - # when input is a single list of subtests - if len(args) == 1 and isinstance(args[0], (list, tuple, GeneratorType)): - args = args[0] +) - for test in args: - # assume test is function needing a state argument - # partial state so reporter can test - rep.do_test(partial(test, state=state)) - # return original state, so can be chained - return state +check_not.__doc__ = ( + str(check_not.__doc__) + + """ + :Example: + The SCT fails with feedback for a specific incorrect value, defined using an override: :: + + Ex().check_object('result').multi( + check_not( + has_equal_value(override=100), + msg='100 is incorrect for reason xyz.' + ), + has_equal_value() + ) -def check_not(*tests, msg, state=None): - """Run multiple subtests that should fail. If all subtests fail, returns original state (for chaining) - - Args: - state: State instance describing student and solution code. Can be omitted if used with Ex(). - tests: one or more sub-SCTs that all should not pass. - args: one or more sub-SCTs to run. + Notice that ``check_not`` comes before the ``has_equal_value`` test + that checks if the student value is equal to the solution value. :Example: The SCT below runs two ``has_code`` cases: :: Ex().check_not( has_code('mean'), - has_code('median') + has_code('median'), + msg='Check your code' ) If students use ``mean`` or ``median`` anywhere in their code, this SCT will fail. Note: - - This function is currently only tested in working with has_code in the subtests. + - This function is not yet tested with all checks, please report unexpected behaviour. - This function can be thought as a NOT(x OR y OR ...) statement, since all tests it runs must fail - This function can be considered a direct counterpart of multi. """ - rep = Reporter.active_reporter - - for test in tests: - try: - multi(test, state=state) - except TestFail: - # it fails, as expected, off to next one - continue - return rep.do_test(Test(msg)) - - # return original state, so can be chained - return state +) -def check_or(*tests, state=None): - """Test whether at least one SCT passes. - - If all of the tests fail, the feedback of the first test will be presented to the student. - - Args: - state: State instance describing student and solution code. Can be omitted if used with Ex(). - tests: one or more sub-SCTs to run. +check_or.__doc__ = ( + str(check_or.__doc__) + + """ :Example: The SCT below tests that the student typed either 'mean' or 'median': :: @@ -102,30 +84,12 @@ def check_or(*tests, state=None): the first SCT, will be presented to the student. """ +) - rep = Reporter.active_reporter - - success = False - first_feedback = None - for test in tests: - try: - multi(test, state=state) - success = True - except TestFail as e: - if not first_feedback: first_feedback = e.feedback - if success: - return - - rep.do_test(Test(first_feedback)) - -def check_correct(check, diagnose, state=None): - """Allows feedback from a diagnostic SCT, only if a check SCT fails. - - Args: - state: State instance describing student and solution code. Can be omitted if used with Ex(). - check: An sct chain that must succeed. - diagnose: An sct chain to run if the check fails. +check_correct.__doc__ = ( + str(check_correct.__doc__) + + """ :Example: The SCT below tests whether an object is correct. Only if the object is not correct, will @@ -137,47 +101,27 @@ def check_correct(check, diagnose, state=None): ) """ - feedback = None - try: - multi(check, state=state) - except TestFail as e: - feedback = e.feedback - - try: - multi(diagnose, state=state) - except TestFail as e: - if feedback is not None or state.force_diagnose: - feedback = e.feedback - - if feedback is not None: - rep = Reporter.active_reporter - rep.do_test(Test(feedback)) +) + # utility functions ----------------------------------------------------------- -def fail(msg="", state=None): - """Fail SCT - - This function takes a single argument, ``msg``, that is the feedback given to the student. - Note that this would be a terrible idea for grading submissions, but may be useful while writing SCTs. - For example, failing a test will highlight the code as if the previous test/check had failed. +fail.__doc__ = ( + str(fail.__doc__) + + """ :Example: - + As a trivial SCT example, :: Ex().check_for_loop().check_body().fail() This can also be helpful for debugging SCTs, as it can be used to stop testing as a given point. + """ +) - -""" - rep = Reporter.active_reporter - _msg = state.build_message(msg) - rep.do_test(Test(Feedback(_msg, state))) - -def override(solution, state=None): +def override(state, solution): """Override the solution code with something arbitrary. There might be cases in which you want to temporarily override the solution code @@ -196,32 +140,32 @@ def override(solution, state=None): # (1) ast.Module, or for single expressions... # (2) whatever was grabbed using module.body[0] # (3) module.body[0].value, when module.body[0] is an Expr node - old_ast = state.solution_tree + old_ast = state.solution_ast new_ast = ast.parse(solution) if not isinstance(old_ast, ast.Module) and len(new_ast.body) == 1: expr = new_ast.body[0] candidates = [expr, expr.value] if isinstance(expr, ast.Expr) else [expr] for node in candidates: - if isinstance(node, old_ast.__class__): + if isinstance(node, old_ast.__class__): new_ast = node break - kwargs = state.messages[-1] if state.messages else {} - child = state.to_child_state( - solution_subtree = new_ast, - student_subtree = state.student_tree, - highlight = state.highlight, - append_message = {'msg': "", 'kwargs': kwargs} - ) + kwargs = state.feedback_context.kwargs if state.feedback_context else {} + child = state.to_child( + solution_ast=new_ast, + student_ast=state.student_ast, + highlight=state.highlight, + append_message=FeedbackComponent("", kwargs), + ) return child -def set_context(*args, state=None, **kwargs): +def set_context(state, *args, **kwargs): """Update context values for student and solution environments. - + When ``has_equal_x()`` is used after this, the context values (in ``for`` loops and function definitions, for example) - will have the values specified throught his function. It is the function equivalent of the ``context_vals`` argument of + will have the values specified through his function. It is the function equivalent of the ``context_vals`` argument of the ``has_equal_x()`` functions. - Note 1: excess args and unmatched kwargs will be unused in the student environment. @@ -264,14 +208,19 @@ def set_context(*args, state=None, **kwargs): # for now, you can't specify both if len(args) > 0 and len(kwargs) > 0: - raise InstructorError("In `set_context()`, specify arguments either by position, either by name.") + raise InstructorError.from_message( + "In `set_context()`, specify arguments either by position, either by name." + ) # set args specified by pos ----------------------------------------------- if args: # stop if too many pos args for solution - if len(args) > len(sol_crnt): - raise InstructorError("Too many positional args. There are {} context vals, but tried to set {}" - .format(len(sol_crnt), len(args))) + if len(args) > len(sol_crnt): + raise InstructorError.from_message( + "Too many positional args. There are {} context vals, but tried to set {}".format( + len(sol_crnt), len(args) + ) + ) # set pos args upd_sol = sol_crnt.update(dict(zip(sol_crnt.keys(), args))) upd_stu = stu_crnt.update(dict(zip(stu_crnt.keys(), args))) @@ -283,22 +232,28 @@ def set_context(*args, state=None, **kwargs): if kwargs: # stop if keywords don't match with solution if set(kwargs) - set(upd_sol): - raise InstructorError("`set_context()` failed: context val names are {}, but you tried to set {}." - .format(upd_sol or "missing", sorted(list(kwargs.keys())))) + raise InstructorError.from_message( + "`set_context()` failed: context val names are {}, but you tried to set {}.".format( + upd_sol or "missing", sorted(list(kwargs.keys())) + ) + ) out_sol = upd_sol.update(kwargs) # need to match keys in kwargs with corresponding keys in stu context # in case they used, e.g., different loop variable names match_keys = dict(zip(sol_crnt.keys(), stu_crnt.keys())) - out_stu = upd_stu.update({match_keys[k]: v for k,v in kwargs.items() if k in match_keys}) + out_stu = upd_stu.update( + {match_keys[k]: v for k, v in kwargs.items() if k in match_keys} + ) else: out_sol = upd_sol out_stu = upd_stu - return state.to_child_state(student_context = out_stu, - solution_context = out_sol, - highlight = state.highlight) + return state.to_child( + student_context=out_stu, solution_context=out_sol, highlight=state.highlight + ) -def set_env(state = None, **kwargs): + +def set_env(state, **kwargs): """Update/set environemnt variables for student and solution environments. When ``has_equal_x()`` is used after this, the variables specified through this function will @@ -334,15 +289,14 @@ def set_env(state = None, **kwargs): stu_new = stu_crnt.update(kwargs) sol_new = sol_crnt.update(kwargs) - return state.to_child_state(student_env = stu_new, - solution_env = sol_new, - highlight = state.highlight) - -def disable_highlighting(state = None): - """Disable highlighting in the remainder of the SCT chain. + return state.to_child( + student_env=stu_new, solution_env=sol_new, highlight=state.highlight + ) - Include this function if you want to avoid that pythonwhat marks which part of the student submission is incorrect. +disable_highlighting.__doc__ = ( + str(disable_highlighting.__doc__) + + """ :Examples: SCT that will mark the 'number' portion if it is incorrect:: @@ -355,4 +309,4 @@ def disable_highlighting(state = None): Ex().check_function('round').disable_highlighting().check_args(0).has_equal_ast() Ex().check_function('round').check_args(0).disable_highlighting().has_equal_ast() """ - return state.to_child_state(highlighting_disabled = True) +) diff --git a/pythonwhat/check_object.py b/pythonwhat/checks/check_object.py similarity index 76% rename from pythonwhat/check_object.py rename to pythonwhat/checks/check_object.py index 6830e433..2b0b8856 100644 --- a/pythonwhat/check_object.py +++ b/pythonwhat/checks/check_object.py @@ -1,15 +1,22 @@ from pythonwhat.parsing import ObjectAssignmentParser -from pythonwhat.Test import DefinedProcessTest, InstanceProcessTest, DefinedCollProcessTest -from pythonwhat.Reporter import Reporter -from pythonwhat.Feedback import Feedback, InstructorError -from pythonwhat.tasks import isDefinedInProcess, isInstanceInProcess, isDefinedCollInProcess -from pythonwhat.check_funcs import part_to_child +from pythonwhat.Test import ( + DefinedProcessTest, + InstanceProcessTest, + DefinedCollProcessTest, +) +from protowhat.Feedback import FeedbackComponent +from protowhat.failure import InstructorError +from pythonwhat.tasks import ( + isDefinedInProcess, + isInstanceInProcess, + isDefinedCollInProcess, +) +from pythonwhat.checks.check_funcs import part_to_child from pythonwhat.utils import v2_only -from pythonwhat.has_funcs import has_equal_value -import pandas as pd import ast -def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr="variable"): + +def check_object(state, index, missing_msg=None, expand_msg=None, typestr="variable"): """Check object existence (and equality) Check whether an object is defined in the student's process, and zoom in on its value in both @@ -25,7 +32,7 @@ def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr=" expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. :Example: - + Suppose you want the student to create a variable ``x``, equal to 15: :: x = 15 @@ -36,9 +43,9 @@ def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr=" - ``check_object()`` will check if the variable ``x`` is defined in the student process. - ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process. - + Note that ``has_equal_value()`` only looks at **end result** of a variable in the student process. - In the example, how the object ``x`` came about in the student's submission, does not matter. + In the example, how the object ``x`` came about in the student's submission, does not matter. This means that all of the following submission will also pass the above SCT: :: x = 15 @@ -83,10 +90,10 @@ def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr=" ) ) ) - + - ``check_correct()`` is used to robustly check whether ``my_list`` was built correctly. - If ``my_list`` is not correct, **both** the initialization and the population code are checked. - + :Example: Because checking object correctness incorrectly is such a common misconception, we're adding another example: :: @@ -94,7 +101,7 @@ def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr=" import pandas as pd df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}) df['c'] = [7, 8, 9] - + The following SCT would be **wrong**, as it does not factor in the possibility that the 'add column ``c``' step could've been wrong: :: Ex().check_correct( @@ -140,7 +147,7 @@ def __init__(self, n): # submissions that will pass this sct x = Number(1) x = Number(2 - 1) - + The basic SCT like in the previous example will notwork here. Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking. Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n`` @@ -151,7 +158,7 @@ def __init__(self, n): # Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1' if v2_only(): extra_msg = "If you want to check the value of an object in e.g. a for loop, use `has_equal_value(name = 'my_obj')` instead." - state.assert_root('check_object', extra_msg=extra_msg) + state.assert_execution_root("check_object", extra_msg=extra_msg) if missing_msg is None: missing_msg = "Did you define the {{typestr}} `{{index}}` without errors?" @@ -159,28 +166,43 @@ def __init__(self, n): if expand_msg is None: expand_msg = "Did you correctly define the {{typestr}} `{{index}}`? " - rep = Reporter.active_reporter + if ( + not isDefinedInProcess(index, state.solution_process) + and state.has_different_processes() + ): + raise InstructorError.from_message( + "`check_object()` couldn't find object `%s` in the solution process." + % index + ) - if not isDefinedInProcess(index, state.solution_process) and state.has_different_processes(): - raise InstructorError("`check_object()` couldn't find object `%s` in the solution process." % index) - - append_message = {'msg': expand_msg, 'kwargs': {'index': index, 'typestr': typestr}} + append_message = FeedbackComponent(expand_msg, {"index": index, "typestr": typestr}) # create child state, using either parser output, or create part from name fallback = lambda: ObjectAssignmentParser.get_part(index) - stu_part = state.student_object_assignments.get(index, fallback()) - sol_part = state.solution_object_assignments.get(index, fallback()) + stu_part = state.ast_dispatcher.find("object_assignments", state.student_ast).get( + index, fallback() + ) + sol_part = state.ast_dispatcher.find("object_assignments", state.solution_ast).get( + index, fallback() + ) # test object exists - _msg = state.build_message(missing_msg, append_message['kwargs']) - rep.do_test(DefinedProcessTest(index, state.student_process, Feedback(_msg))) - - child = part_to_child(stu_part, sol_part, append_message, state, - node_name='object_assignments') + state.do_test( + DefinedProcessTest( + index, + state.student_process, + FeedbackComponent(missing_msg, append_message.kwargs), + ) + ) + + child = part_to_child( + stu_part, sol_part, append_message, state, node_name="object_assignments" + ) return child -def is_instance(inst, not_instance_msg=None, state=None): + +def is_instance(state, inst, not_instance_msg=None): """Check whether an object is an instance of a certain class. ``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is @@ -206,27 +228,29 @@ def is_instance(inst, not_instance_msg=None, state=None): Ex().check_object('arr').is_instance(numpy.ndarray) """ - state.assert_is(['object_assignments'], 'is_instance', ['check_object']) - - rep = Reporter.active_reporter + state.assert_is(["object_assignments"], "is_instance", ["check_object"]) - sol_name = state.solution_parts.get('name') - stu_name = state.student_parts.get('name') + sol_name = state.solution_parts.get("name") + stu_name = state.student_parts.get("name") - if not_instance_msg is None: not_instance_msg = "Is it a {{inst.__name__}}?" + if not_instance_msg is None: + not_instance_msg = "Is it a {{inst.__name__}}?" if not isInstanceInProcess(sol_name, inst, state.solution_process): - raise InstructorError("`is_instance()` noticed that `%s` is not a `%s` in the solution process." % (sol_name, inst.__name__)) + raise InstructorError.from_message( + "`is_instance()` noticed that `%s` is not a `%s` in the solution process." + % (sol_name, inst.__name__) + ) - _msg = state.build_message(not_instance_msg, {'inst': inst}) - feedback = Feedback(_msg, state) - rep.do_test(InstanceProcessTest(stu_name, inst, state.student_process, feedback)) + feedback = FeedbackComponent(not_instance_msg, {"inst": inst}) + state.do_test(InstanceProcessTest(stu_name, inst, state.student_process, feedback)) return state -def check_df(index, missing_msg=None, not_instance_msg=None, expand_msg=None, state=None): + +def check_df(state, index, missing_msg=None, not_instance_msg=None, expand_msg=None): """Check whether a DataFrame was defined and it is the right type - + ``check_df()`` is a combo of ``check_object()`` and ``is_instance()`` that checks whether the specified object exists and whether the specified object is pandas DataFrame. @@ -257,18 +281,26 @@ def check_df(index, missing_msg=None, not_instance_msg=None, expand_msg=None, st - ``check_df()`` checks if ``my_df`` exists (``check_object()`` behind the scenes) and is a DataFrame (``is_instance()``) - ``check_keys("a")`` zooms in on the column ``a`` of the data frame, and ``has_equal_value()`` checks if the columns correspond between student and solution process. - ``check_keys("b")`` zooms in on hte column ``b`` of the data frame, but there's no 'equality checking' happening - + The following submissions would pass the SCT above: :: - + my_df = pd.DataFrame({"a": [1, 1 + 1, 3], "b": ["a", "l", "l"]}) my_df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) """ - child = check_object(index, missing_msg=missing_msg, expand_msg=expand_msg, state=state, typestr="pandas DataFrame") - is_instance(pd.DataFrame, not_instance_msg=not_instance_msg, state=child) + import pandas as pd + child = check_object( + state, + index, + missing_msg=missing_msg, + expand_msg=expand_msg, + typestr="pandas DataFrame", + ) + is_instance(child, pd.DataFrame, not_instance_msg=not_instance_msg) return child -def check_keys(key, missing_msg=None, expand_msg=None, state=None): + +def check_keys(state, key, missing_msg=None, expand_msg=None): """Check whether an object (dict, DataFrame, etc) has a key. ``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is @@ -297,42 +329,44 @@ def check_keys(key, missing_msg=None, expand_msg=None, state=None): """ - state.assert_is(['object_assignments'], 'is_instance', ['check_object', 'check_df']) + state.assert_is(["object_assignments"], "is_instance", ["check_object", "check_df"]) if missing_msg is None: missing_msg = "There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`." if expand_msg is None: expand_msg = "Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? " - rep = Reporter.active_reporter - - sol_name = state.solution_parts.get('name') - stu_name = state.student_parts.get('name') + sol_name = state.solution_parts.get("name") + stu_name = state.student_parts.get("name") if not isDefinedCollInProcess(sol_name, key, state.solution_process): - raise InstructorError("`check_keys()` couldn't find key `%s` in object `%s` in the solution process." % (key, sol_name)) + raise InstructorError.from_message( + "`check_keys()` couldn't find key `%s` in object `%s` in the solution process." + % (key, sol_name) + ) # check if key available - _msg = state.build_message(missing_msg, {'key': key}) - rep.do_test(DefinedCollProcessTest(stu_name, key, state.student_process, - Feedback(_msg, state))) + state.do_test( + DefinedCollProcessTest( + stu_name, key, state.student_process, FeedbackComponent(missing_msg, {"key": key}) + ) + ) def get_part(name, key, highlight): if isinstance(key, str): - slice_val = ast.Str(s=key) + slice_val = ast.Constant(value=key) else: slice_val = ast.parse(str(key)).body[0].value - expr = ast.Subscript(value=ast.Name(id=name, ctx=ast.Load()), - slice=ast.Index(value=slice_val), - ctx=ast.Load()) + expr = ast.Subscript( + value=ast.Name(id=name, ctx=ast.Load()), + slice=ast.Index(value=slice_val), + ctx=ast.Load(), + ) ast.fix_missing_locations(expr) - return { - 'node': expr, - 'highlight': highlight - } - - stu_part = get_part(stu_name, key, state.student_parts.get('highlight')) - sol_part = get_part(sol_name, key, state.solution_parts.get('highlight')) - append_message = {'msg': expand_msg, 'kwargs': {'key': key }} + return {"node": expr, "highlight": highlight} + + stu_part = get_part(stu_name, key, state.student_parts.get("highlight")) + sol_part = get_part(sol_name, key, state.solution_parts.get("highlight")) + append_message = FeedbackComponent(expand_msg, {"key": key}) child = part_to_child(stu_part, sol_part, append_message, state) return child diff --git a/pythonwhat/check_wrappers.py b/pythonwhat/checks/check_wrappers.py similarity index 80% rename from pythonwhat/check_wrappers.py rename to pythonwhat/checks/check_wrappers.py index c334924d..f10067a9 100644 --- a/pythonwhat/check_wrappers.py +++ b/pythonwhat/checks/check_wrappers.py @@ -1,35 +1,40 @@ -from pythonwhat.check_funcs import check_part, check_part_index, check_node -from pythonwhat.has_funcs import has_equal_part -from pythonwhat import check_funcs, has_funcs, check_logic, check_object -from pythonwhat.check_function import check_function -from pythonwhat.check_has_context import has_context - -from functools import partial, update_wrapper -import inspect +from protowhat.failure import _debug +from protowhat.checks.check_simple import allow_errors +from protowhat.checks.check_bash_history import has_command +from protowhat.checks.check_files import check_file, has_dir +from pythonwhat.checks.check_funcs import check_part, check_part_index, check_node +from pythonwhat.checks.has_funcs import has_equal_part +from pythonwhat.checks.check_function import check_function +from pythonwhat.checks.check_has_context import has_context +from pythonwhat.checks import check_object, check_logic, check_funcs, has_funcs +from pythonwhat.local import run + +from inspect import signature, Parameter +from functools import partial, wraps from jinja2 import Template __PART_WRAPPERS__ = { - 'iter': 'iterable part', - 'body': 'body', - 'key' : 'key part', - 'value': 'value part', - 'orelse': 'else part', - 'finalbody': 'finally part', - 'test': 'condition', + "iter": "iterable part", + "body": "body", + "key": "key part", + "value": "value part", + "orelse": "else part", + "finalbody": "finally part", + "test": "condition", } __PART_INDEX_WRAPPERS__ = { - 'ifs': '{{ordinal}} if', - 'bases': '{{ordinal}} base class', - 'handlers': '`{{index}}` `except` block', - 'context': '{{ordinal}} context', + "ifs": "{{ordinal}} if", + "bases": "{{ordinal}} base class", + "handlers": "`{{index}}` `except` block", + "context": "{{ordinal}} context", } __NODE_WRAPPERS__ = { - 'list_comp': { - 'typestr': '{{ordinal}} list comprehension', - 'docstr': """Check whether a list comprehension was coded and zoom in on it. - + "list_comp": { + "typestr": "{{ordinal}} list comprehension", + "docstr": """Check whether a list comprehension was coded and zoom in on it. + Can be chained with ``check_iter()``, ``check_body()``, and ``check_ifs()``. Args: @@ -75,10 +80,10 @@ """, }, - 'generator_exp': { - 'typestr': '{{ordinal}} generator expression', - 'docstr': """Check whether a generator expression was coded and zoom in on it. - + "generator_exp": { + "typestr": "{{ordinal}} generator expression", + "docstr": """Check whether a generator expression was coded and zoom in on it. + Can be chained with ``check_iter()``, ``check_body()``, and ``check_ifs()``. Args: @@ -107,10 +112,10 @@ """, }, - 'dict_comp': { - 'typestr': '{{ordinal}} dictionary comprehension', - 'docstr': """Check whether a dictionary comprehension was coded and zoom in on it. - + "dict_comp": { + "typestr": "{{ordinal}} dictionary comprehension", + "docstr": """Check whether a dictionary comprehension was coded and zoom in on it. + Can be chained with ``check_key()``, ``check_value()``, and ``check_ifs()``. Args: @@ -151,12 +156,12 @@ """, }, - 'for_loop': { - 'typestr': '{{ordinal}} for loop', - 'docstr': """Check whether a for loop was coded and zoom in on it. + "for_loop": { + "typestr": "{{ordinal}} for loop", + "docstr": """Check whether a for loop was coded and zoom in on it. Can be chained with ``check_iter()`` and ``check_body()``. - + Args: index: Index of the for loop (0-based). {{typestr}} @@ -179,7 +184,7 @@ set_context('b', 2).has_equal_output() ) ) - + - ``check_for_loop()`` zooms in on the ``for`` loop, and makes its parts available for further checking. - ``check_iter()`` zooms in on the iterator part of the for loop, ``my_dict.items()`` in the solution. ``has_equal_value()`` re-executes the expressions specified by student and solution and compares their results. @@ -198,7 +203,7 @@ # passing submission 2 my_dict = {'a': 1, 'b': 2} for first, second in my_dict.items(): - mess = first + " - " + str(second) + mess = first + " - " + str(second) print(mess) :Example: @@ -252,12 +257,12 @@ ) ) ) - - """ + + """, }, - 'function_def': { - 'typestr': 'definition of `{{index}}()`', - 'docstr': """Check whether a function was defined and zoom in on it. + "function_def": { + "typestr": "definition of `{{index}}()`", + "docstr": """Check whether a function was defined and zoom in on it. Can be chained with ``check_call()``, ``check_args()`` and ``check_body()``. @@ -365,12 +370,12 @@ def shout_echo(a, b=1): We are working on it! - """ + """, }, - 'class_def': { - 'typestr': 'class definition of `{{index}}`', - 'docstr': """Check whether a class was defined and zoom in on its definition - + "class_def": { + "typestr": "class definition of `{{index}}`", + "docstr": """Check whether a class was defined and zoom in on its definition + Can be chained with ``check_bases()`` and ``check_body()``. Args: @@ -389,7 +394,7 @@ def __init__(self, i): The following SCT would verify this: :: - check_class_def('MyInt').multi( + Ex().check_class_def('MyInt').multi( check_bases(0).has_equal_ast(), check_body().check_function_def('__init__').multi( check_args('self'), @@ -409,18 +414,18 @@ def __init__(self, i): e.g. to verify whether class methods give the right behavior with ``check_call()`` before diving into the body of the method itself. - """ + """, }, - 'if_exp': { - 'typestr': '{{ordinal}} if expression', - 'docstr': """Check whether an if expression was coded zoom in on it. + "if_exp": { + "typestr": "{{ordinal}} if expression", + "docstr": """Check whether an if expression was coded zoom in on it. This function works the exact same way as ``check_if_else()``. - """ + """, }, - 'if_else': { - 'typestr': '{{ordinal}} if statement', - 'docstr': """Check whether an if statement was coded zoom in on it. + "if_else": { + "typestr": "{{ordinal}} if statement", + "docstr": """Check whether an if statement was coded zoom in on it. Args: index: the index of the if statement to look for (0 based) @@ -442,7 +447,7 @@ def __init__(self, i): check_test().multi( set_env(x = -1).has_equal_value(), set_env(x = 1).has_equal_value(), - set_env(x = 0).has_equal_value() + set_env(x = 0).has_equal_value() ), check_body().check_function('print', 0).\\ check_args('value').has_equal_value() @@ -459,11 +464,11 @@ def __init__(self, i): function ``print()`` and whether its argument is set correctly. :Example: - + In Python, when an if-else statement has an ``elif`` clause, it is held in the `orelse` part. In this sense, an if-elif-else statement is represented by python as nested if-elses. More specifically, this if-else statement: :: - + if x > 0: print(x) elif y > 0: @@ -493,11 +498,11 @@ def __init__(self, i): ) ) - """ + """, }, - 'lambda_function': { - 'typestr': '{{ordinal}} lambda function', - 'docstr': """Check whether a lambda function was coded zoom in on it. + "lambda_function": { + "typestr": "{{ordinal}} lambda function", + "docstr": """Check whether a lambda function was coded zoom in on it. Can be chained with ``check_call()``, ``check_args()`` and ``check_body()``. @@ -542,11 +547,11 @@ def __init__(self, i): # passing submission 2 lambda y, times=2: len(y) * times - """ + """, }, - 'try_except': { - 'typestr': '{{ordinal}} try statement', - 'docstr': """Check whether a try except statement was coded zoom in on it. + "try_except": { + "typestr": "{{ordinal}} try statement", + "docstr": """Check whether a try except statement was coded zoom in on it. Can be chained with ``check_body()``, ``check_handlers()``, ``check_orelse()`` and ``check_finalbody()``. @@ -585,11 +590,11 @@ def __init__(self, i): check_function('print').check_args(0).has_equal_value() ) - """ + """, }, - 'while': { - 'typestr': '{{ordinal}} `while` loop', - 'docstr': """Check whether a while loop was coded and zoom in on it. + "while": { + "typestr": "{{ordinal}} `while` loop", + "docstr": """Check whether a while loop was coded and zoom in on it. Can be chained with ``check_test()``, ``check_body()`` and ``check_orelse()``. @@ -628,11 +633,11 @@ def __init__(self, i): - ``check_body()`` zooms in on the body of the ``while`` loop, and ``has_equal_value()`` checks whether rerunning this body updates ``i`` as expected when ``i`` is temporarily set to 3 with ``set_env()``. - """ + """, }, - 'with': { - 'typestr': '{{ordinal}} `with` statement', - 'docstr': """Check whether a with statement was coded zoom in on it. + "with": { + "typestr": "{{ordinal}} `with` statement", + "docstr": """Check whether a with statement was coded zoom in on it. Args: index: the index of the``with`` statement to verify (0-based) @@ -640,50 +645,155 @@ def __init__(self, i): {{missing_msg}} {{expand_msg}} - """ + """, }, } -scts = {} +scts = dict() # make has_equal_part wrappers -scts['has_equal_name'] = partial(has_equal_part, 'name', msg='Make sure to use the correct {{name}}, was expecting {{sol_part[name]}}, instead got {{stu_part[name]}}.') -scts['is_default'] = partial(has_equal_part, 'is_default', msg="Make sure it {{ 'has' if sol_part.is_default else 'does not have'}} a default argument.") + +def partial_with_offset(offset=1): + def bound_partial_with_offset(func, *partial_args, **partial_kwargs): + kwargs_partial = partial(func, **partial_kwargs) + + @wraps(func) + def full_partial(*args, **kwargs): + full_args = args[:offset] + partial_args + args[offset:] + return kwargs_partial(*full_args, **kwargs) + + # set correct signature of returned partial + # todo: pass arguments as keywords to partial, instead of this decorator? + # (where args are always the same) + func_sig = signature(full_partial) + parameter_names = tuple(func_sig.parameters) + + partialed_positional_indices = [] + for kwarg in partial_kwargs: + param = func_sig.parameters[kwarg] + if param.default is param.empty: + partialed_positional_indices.append(parameter_names.index(kwarg)) + + partial_params = list(func_sig.parameters.values()) + for index in sorted(partialed_positional_indices, reverse=True): + # appending isn't needed for functionality, but more similar to partial + # and it shows that these arguments can still be updated as kwargs + partial_params.append( + partial_params[index].replace( + kind=Parameter.KEYWORD_ONLY, + default=partial_kwargs[partial_params[index].name], + ) + ) + del partial_params[index] + del partial_params[offset : offset + len(partial_args)] + + full_partial.__signature__ = func_sig.replace(parameters=partial_params) + + return full_partial + + return bound_partial_with_offset + + +state_partial = partial_with_offset() + + +def rename_function(func, name): + # see functools.wraps + func.__name__ = func.__qualname__ = name + + +def add_partial_sct(func, name): + rename_function(func, name) + scts[name] = func + + +add_partial_sct( + state_partial( + has_equal_part, + "name", + msg="Make sure to use the correct {{name}}, was expecting {{sol_part[name]}}, instead got {{stu_part[name]}}.", + ), + "has_equal_name", +) +add_partial_sct( + state_partial( + has_equal_part, + "is_default", + msg="Make sure it {{ 'has' if sol_part.is_default else 'does not have'}} a default argument.", + ), + "is_default", +) # include rest of wrappers for k, v in __PART_WRAPPERS__.items(): + check_fun = state_partial(check_part, k, v) + add_partial_sct(check_fun, "check_" + k) - scts['check_'+k] = partial(check_part, k, v) - -for k, v in __PART_INDEX_WRAPPERS__.items(): - scts['check_'+k] = partial(check_part_index, k, part_msg=v) +for k, v in __PART_INDEX_WRAPPERS__.items(): + check_fun = state_partial(check_part_index, k, part_msg=v) + add_partial_sct(check_fun, "check_" + k) for k, v in __NODE_WRAPPERS__.items(): - check_fun = partial(check_node, k+'s', typestr=v['typestr']) - check_fun.__doc__ = Template(v['docstr']).render( + check_fun = state_partial(check_node, k + "s", typestr=v["typestr"]) + check_fun.__doc__ = Template(v["docstr"]).render( typestr="typestr: If specified, this overrides the standard way of referring to the construct you're zooming in on.", missing_msg="missing_msg: If specified, this overrides the automatically generated feedback message in case the construct could not be found.", - expand_msg="expand_msg: If specified, this overrides the automatically generated feedback message that is prepended to feedback messages that are thrown further in the SCT chain." + expand_msg="expand_msg: If specified, this overrides the automatically generated feedback message that is prepended to feedback messages that are thrown further in the SCT chain.", ) - check_fun.__name__ = 'check_'+k - scts[check_fun.__name__] = check_fun - -for k in ['set_context', 'set_env', 'disable_highlighting', 'check_not', 'check_or', 'check_correct', 'fail', 'override', 'multi']: + add_partial_sct(check_fun, "check_" + k) + +for k in [ + "set_context", + "set_env", + "disable_highlighting", + "check_not", + "check_or", + "check_correct", + "fail", + "override", + "multi", +]: scts[k] = getattr(check_logic, k) -for k in ['with_context', 'check_args', 'check_call']: +for k in ["with_context", "check_args", "check_call"]: scts[k] = getattr(check_funcs, k) -for k in ['has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'has_equal_part_len', - 'has_equal_part', 'has_import', 'has_output', 'has_printout', 'has_code', 'has_no_error', 'has_chosen']: - scts[k] = getattr(has_funcs, k) +for k in [ + "has_equal_value", + "has_equal_output", + "has_equal_error", + "has_equal_ast", + "has_equal_part_len", + "has_equal_part", + "has_import", + "has_output", + "has_printout", + "has_code", + "has_no_error", + "has_chosen", +]: + sct = getattr(has_funcs, k) + if not hasattr(sct, "__name__"): + rename_function(sct, k) + scts[k] = sct # include check_object and friends ------ -for k in ['check_object', 'is_instance', 'check_df', 'check_keys']: +for k in ["check_object", "is_instance", "check_df", "check_keys"]: scts[k] = getattr(check_object, k) -scts['has_context'] = has_context -scts['check_function'] = check_function +scts["has_context"] = has_context +scts["check_function"] = check_function + +scts["run"] = run + +scts["has_command"] = has_command + +scts["check_file"] = check_file +scts["has_dir"] = has_dir + +scts["allow_errors"] = allow_errors + +scts["_debug"] = _debug -locals().update(scts) \ No newline at end of file +locals().update(scts) diff --git a/pythonwhat/has_funcs.py b/pythonwhat/checks/has_funcs.py similarity index 60% rename from pythonwhat/has_funcs.py rename to pythonwhat/checks/has_funcs.py index 7fe7c93c..870e1069 100644 --- a/pythonwhat/has_funcs.py +++ b/pythonwhat/checks/has_funcs.py @@ -1,23 +1,34 @@ -from pythonwhat.tasks import getResultInProcess, getOutputInProcess, getErrorInProcess, ReprFail, isDefinedInProcess, getOptionFromProcess, ReprFail, UndefinedValue -from pythonwhat.Reporter import Reporter -from pythonwhat.Test import Test, EqualTest -from pythonwhat.Feedback import Feedback, InstructorError +from protowhat.utils_messaging import get_ord +from pythonwhat.tasks import ( + getResultInProcess, + getOutputInProcess, + getErrorInProcess, + ReprFail, + isDefinedInProcess, + getOptionFromProcess, + UndefinedValue, +) +from pythonwhat.Test import EqualTest, DefinedCollTest +from protowhat.Feedback import Feedback, FeedbackComponent +from protowhat.failure import InstructorError, debugger from pythonwhat import utils from functools import partial import re import copy import ast -evalCalls = {'value': getResultInProcess, - 'output': getOutputInProcess, - 'error': getErrorInProcess} +evalCalls = { + "value": getResultInProcess, + "output": getOutputInProcess, + "error": getErrorInProcess, +} -def has_part(name, msg, state=None, fmt_kwargs=None, index=None): - rep = Reporter.active_reporter + +def has_part(state, name, msg, fmt_kwargs=None, index=None): d = { - 'sol_part': state.solution_parts, - 'stu_part': state.student_parts, - **fmt_kwargs + "sol_part": state.solution_parts, + "stu_part": state.student_parts, + **fmt_kwargs, } def verify(part, index): @@ -30,35 +41,39 @@ def verify(part, index): if part is None: raise KeyError - # Chceck if it's there in the solution - _msg = state.build_message(msg, d) - _err_msg = "SCT fails on solution: " + _msg + # TODO: instructor error if msg is not str + # Check if it's there in the solution try: verify(state.solution_parts[name], index) except (KeyError, IndexError): - raise InstructorError(_err_msg) + with debugger(state): + err_msg = "SCT fails on solution: {}".format(msg) + state.report(err_msg, d) - try: + try: verify(state.student_parts[name], index) except (KeyError, IndexError): - rep.do_test(Test(Feedback(_msg, state))) + state.report(msg, d) return state -def has_equal_part(name, msg, state): - rep = Reporter.active_reporter - d = {'stu_part': state.student_parts, - 'sol_part': state.solution_parts, - 'name': name} +def has_equal_part(state, name, msg): + d = { + "stu_part": state.student_parts, + "sol_part": state.solution_parts, + "name": name, + } - _msg = state.build_message(msg, d) - rep.do_test(EqualTest(d['stu_part'][name], d['sol_part'][name], Feedback(_msg, state))) + state.do_test( + EqualTest(d["stu_part"][name], d["sol_part"][name], FeedbackComponent(msg, d)) + ) return state + # TODO: shouldn't have to hardcode message -def has_equal_part_len(name, unequal_msg, state=None): +def has_equal_part_len(state, name, unequal_msg): """Verify that a part that is zoomed in on has equal length. Typically used in the context of ``check_function_def()`` @@ -79,28 +94,27 @@ def shout(word): Ex().check_function_def('shout').has_equal_part_len('args', 'not enough args!') """ - rep = Reporter.active_reporter - d = dict(stu_len = len(state.student_parts[name]), - sol_len = len(state.solution_parts[name])) + d = dict( + stu_len=len(state.student_parts[name]), sol_len=len(state.solution_parts[name]) + ) - if d['stu_len'] != d['sol_len']: - _msg = state.build_message(unequal_msg, d) - rep.do_test(Test(Feedback(_msg, state))) + if d["stu_len"] != d["sol_len"]: + state.report(unequal_msg, d) return state -## Expression tests ----------------------------------------------------------- -def has_equal_ast(incorrect_msg=None, - code=None, - exact=True, - append=None, - state=None): +# Expression tests ----------------------------------------------------------- + + +def has_equal_ast(state, incorrect_msg=None, code=None, exact=True, append=None): """Test whether abstract syntax trees match between the student and solution code. ``has_equal_ast()`` can be used in two ways: * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission. + But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST. + Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``. * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments. Args: @@ -138,51 +152,78 @@ def has_equal_ast(incorrect_msg=None, Ex().check_function('numpy.mean').check_args('a').has_equal_ast() """ - rep = Reporter.active_reporter - if utils.v2_only(): - state.assert_is_not(['object_assignments'], 'has_equal_ast', ['check_object']) - state.assert_is_not(['function_calls'], 'has_equal_ast', ['check_function']) + state.assert_is_not(["object_assignments"], "has_equal_ast", ["check_object"]) + state.assert_is_not(["function_calls"], "has_equal_ast", ["check_function"]) if code and incorrect_msg is None: - raise InstructorError("If you manually specify the code to match inside has_equal_ast(), " - "you have to explicitly set the `incorrect_msg` argument.") - - if append is None: # if not specified, set to False if incorrect_msg was manually specified + raise InstructorError.from_message( + "If you manually specify the code to match inside has_equal_ast(), " + "you have to explicitly set the `incorrect_msg` argument." + ) + + if ( + append is None + ): # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: - incorrect_msg = "Expected `{{sol_str}}`, but got `{{stu_str}}`." + incorrect_msg = ( + "Expected \n```\n{{sol_str}}\n```\n, but got \n```\n{{stu_str}}\n```\n" + if utils.is_multiline_code(state.student_code, state.solution_code) + else "Expected `{{sol_str}}`, but got `{{stu_str}}`." + ) def parse_tree(tree): # get contents of module.body if only 1 element - crnt = tree.body[0] if isinstance(tree, ast.Module) and len(tree.body) == 1 else tree + crnt = ( + tree.body[0] + if isinstance(tree, ast.Module) and len(tree.body) == 1 + else tree + ) # remove Expr if it exists return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt) - stu_rep = parse_tree(state.student_tree) - sol_rep = parse_tree(state.solution_tree if not code else ast.parse(code)) - - fmt_kwargs = { - 'sol_str': state.solution_code if not code else code, - 'stu_str': state.student_code - } + stu_rep = parse_tree(state.student_ast) + sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code)) - _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) + if utils.is_multiline_code(state.student_code, state.solution_code): + fmt_kwargs = { + "sol_str": utils.format_code(state.solution_code) + if not code + else utils.format_code(code), + "stu_str": utils.format_code(state.student_code), + } + else: + fmt_kwargs = { + "sol_str": state.solution_code if not code else code, + "stu_str": state.student_code, + } if exact and not code: - rep.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state))) - elif not sol_rep in stu_rep: - rep.do_test(Test(Feedback(_msg, state))) + state.do_test( + EqualTest( + stu_rep, + sol_rep, + FeedbackComponent(incorrect_msg, fmt_kwargs, append=append), + ) + ) + elif sol_rep not in stu_rep: + state.report(incorrect_msg, fmt_kwargs, append=append) return state -DEFAULT_INCORRECT_MSG="Expected {{test_desc}}`{{sol_eval}}`, but got `{{stu_eval}}`." -DEFAULT_ERROR_MSG="Running {{'it' if parent['part'] else 'the higlighted expression'}} generated an error: `{{stu_str}}`." -DEFAULT_ERROR_MSG_INV="Running {{'it' if parent['part'] else 'the higlighted expression'}} didn't generate an error, but it should!" -DEFAULT_UNDEFINED_NAME_MSG="Running {{'it' if parent['part'] else 'the higlighted expression'}} should define a variable `{{name}}` without errors, but it doesn't." -DEFAULT_INCORRECT_NAME_MSG="Are you sure you assigned the correct value to `{{name}}`?" -DEFAULT_INCORRECT_EXPR_CODE_MSG="Running the expression `{{expr_code}}` didn't generate the expected result." + +DEFAULT_INCORRECT_MSG = "Expected {{test_desc}}`{{sol_eval}}`, but got `{{stu_eval}}`." +DEFAULT_ERROR_MSG = "Running {{'it' if parent['part'] else 'the highlighted expression'}} generated an error: `{{stu_str}}`." +DEFAULT_ERROR_MSG_INV = "Running {{'it' if parent['part'] else 'the highlighted expression'}} didn't generate an error, but it should!" +DEFAULT_UNDEFINED_NAME_MSG = "Running {{'it' if parent['part'] else 'the highlighted expression'}} should define a variable `{{name}}` without errors, but it doesn't." +DEFAULT_INCORRECT_NAME_MSG = ( + "Are you sure you assigned the correct value to `{{name}}`?" +) +DEFAULT_INCORRECT_EXPR_CODE_MSG = ( + "Running the expression `{{expr_code}}` didn't generate the expected result." +) args_string = """ @@ -204,36 +245,43 @@ def parse_tree(tree): You can also use ``set_context()`` for this. pre_code (str): the code in string form that should be executed before the expression is executed. This is the ideal place to set a random seed, for example. - expr_code (str): if this argument is set, the expression in the student/solution code will not + expr_code (str): If this argument is set, the expression in the student/solution code will not be ran. Instead, the given piece of code will be ran in the student as well as the solution environment - and the result will be compared. + and the result will be compared. However if the string contains one or more placeholders ``__focus__``, + they will be substituted by the currently focused code. name (str): If this is specified, the {0} of running this expression after running the focused expression - is returned, instead of the {0} of the focussed expression in itself. This is typically used to inspect the + is returned, instead of the {0} of the focused expression in itself. This is typically used to inspect the {0} of an object after executing the body of e.g. a ``for`` loop. copy (bool): whether to try to deep copy objects in the environment, such as lists, that could accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues. - func: custom binary function of form f(stu_result, sol_result), for equality testing. + func (function): custom binary function of form f(stu_result, sol_result), for equality testing. override: If specified, this avoids the execution of the targeted code in the solution process. Instead, it will compare the {0} of the expression in the student process with the value specified in ``override``. Typically used in a ``SingleProcessExercise`` or if you want to allow for different solutions other than the one coded up in the solution. """ -def has_expr(incorrect_msg=None, - error_msg=None, - undefined_msg=None, - append=None, - extra_env=None, - context_vals=None, - pre_code=None, - expr_code=None, - name=None, - copy=True, - func=None, - override=None, - state=None, - test=None): - - if append is None: # if not specified, set to False if incorrect_msg was manually specified + + +def has_expr( + state, + incorrect_msg=None, + error_msg=None, + undefined_msg=None, + append=None, + extra_env=None, + context_vals=None, + pre_code=None, + expr_code=None, + name=None, + copy=True, + func=None, + override=None, + test=None, # todo: default or arg before state +): + + if ( + append is None + ): # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: if name: @@ -245,84 +293,121 @@ def has_expr(incorrect_msg=None, if undefined_msg is None: undefined_msg = DEFAULT_UNDEFINED_NAME_MSG if error_msg is None: - if test == 'error': + if test == "error": error_msg = DEFAULT_ERROR_MSG_INV else: error_msg = DEFAULT_ERROR_MSG - rep = Reporter.active_reporter - - get_func = partial(evalCalls[test], - extra_env=extra_env, - context_vals=context_vals, - pre_code=pre_code, - expr_code=expr_code, - name=name, - copy=copy) + if state.solution_code is not None and isinstance(expr_code, str): + expr_code = expr_code.replace("__focus__", state.solution_code) + + get_func = partial( + evalCalls[test], + extra_env=extra_env, + context_vals=context_vals, + pre_code=pre_code, + expr_code=expr_code, + name=name, + copy=copy, + ) if override is not None: # don't bother with running expression and fetching output/value # eval_sol, str_sol = eval eval_sol, str_sol = override, str(override) else: - eval_sol, str_sol = get_func(tree=state.solution_tree, - process=state.solution_process, - context=state.solution_context, - env=state.solution_env) - - if (test == 'error') ^ isinstance(eval_sol, Exception): - raise InstructorError("Evaluating expression raised error in solution process (or not an error if testing for one). " - "Error: {} - {}".format(type(eval_sol), str_sol)) + eval_sol, str_sol = get_func( + tree=state.solution_ast, + process=state.solution_process, + context=state.solution_context, + env=state.solution_env, + ) + + if (test == "error") ^ isinstance(eval_sol, Exception): + raise InstructorError.from_message( + "Evaluating expression raised error in solution process (or didn't raise if testing for one). " + "Error: {} - {}".format(type(eval_sol), str_sol) + ) if isinstance(eval_sol, ReprFail): - raise InstructorError("Couldn't extract the value for the highlighted expression from the solution process: " + eval_sol.info) + raise InstructorError.from_message( + "Couldn't extract the value for the highlighted expression from the solution process: " + + eval_sol.info + ) - eval_stu, str_stu = get_func(tree=state.student_tree, - process=state.student_process, - context=state.student_context, - env=state.student_env) + eval_stu, str_stu = get_func( + tree=state.student_ast, + process=state.student_process, + context=state.student_context, + env=state.student_env, + ) # kwargs --- fmt_kwargs = { - 'stu_part': state.student_parts, - 'sol_part': state.solution_parts, - 'name': name, 'test': test, - 'test_desc': '' if test == 'value' else 'the %s ' % test, - 'expr_code': expr_code + "stu_part": state.student_parts, + "sol_part": state.solution_parts, + "name": name, + "test": test, + "test_desc": "" if test == "value" else "the %s " % test, + "expr_code": expr_code, } - fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) - fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) - if incorrect_msg == DEFAULT_INCORRECT_MSG and \ - ( fmt_kwargs['stu_eval'] is None or - fmt_kwargs['sol_eval'] is None or - fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ): + fmt_kwargs["stu_eval"] = str(eval_stu) + fmt_kwargs["sol_eval"] = str(eval_sol) + + # wrap in quotes if eval_sol or eval_stu are strings + if test == "value": + if isinstance(eval_stu, str): + fmt_kwargs["stu_eval"] = "'{}'".format(fmt_kwargs["stu_eval"]) + if isinstance(eval_sol, str): + fmt_kwargs["sol_eval"] = "'{}'".format(fmt_kwargs["sol_eval"]) + + # reformat student evaluation string if it is too long + fmt_kwargs["stu_eval"] = utils.shorten_string(fmt_kwargs["stu_eval"]) + + # check if student or solution evaluations are too long or contain newlines + if incorrect_msg == DEFAULT_INCORRECT_MSG and ( + len(fmt_kwargs["sol_eval"]) > 50 + or utils.has_newline(fmt_kwargs["stu_eval"]) + or utils.has_newline(fmt_kwargs["sol_eval"]) + or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"] + ): + fmt_kwargs["stu_eval"] = None + fmt_kwargs["sol_eval"] = None incorrect_msg = "Expected something different." # tests --- # error in process - if (test == 'error') ^ isinstance(eval_stu, Exception): - fmt_kwargs['stu_str'] = str_stu - _msg = state.build_message(error_msg, fmt_kwargs, append=append) - feedback = Feedback(_msg, state) - rep.do_test(Test(feedback)) + if (test == "error") ^ isinstance(eval_stu, Exception): + fmt_kwargs["stu_str"] = str_stu + state.report(error_msg, fmt_kwargs, append=append) # name is undefined after running expression if isinstance(eval_stu, UndefinedValue): - _msg = state.build_message(undefined_msg, fmt_kwargs, append=append) - rep.do_test(Test(Feedback(_msg, state))) + state.report(undefined_msg, fmt_kwargs, append=append) # test equality of results - _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) - rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func)) + state.do_test( + EqualTest( + eval_stu, + eval_sol, + FeedbackComponent(incorrect_msg, fmt_kwargs, append=append), + func, + ) + ) return state -has_equal_value = partial(has_expr, test = 'value') -has_equal_value.__doc__ = """Run targeted student and solution code, and compare returned value. + +has_equal_value = partial(has_expr, test="value") +has_equal_value.__name__ = "has_equal_value" +has_equal_value.__doc__ = ( + """Run targeted student and solution code, and compare returned value. When called on an SCT chain, ``has_equal_value()`` will execute the student and solution code that is 'zoomed in on' and compare the returned values. - """ + args_string.format("returned value", "value") + """ + """ + + args_string.format("returned value", "value") + + """ :Example: Student code and solution code:: @@ -343,30 +428,35 @@ def has_expr(incorrect_msg=None, Ex().check_function('numpy.mean').has_equal_value() """ +) -has_equal_output = partial(has_expr, test = 'output') +has_equal_output = partial(has_expr, test="output") +has_equal_output.__name__ = "has_equal_output" has_equal_output.__doc__ = """Run targeted student and solution code, and compare output. When called on an SCT chain, ``has_equal_output()`` will execute the student and solution code that is 'zoomed in on' and compare the output. - """ + args_string.format("output") + """ + args_string.format( + "output" +) -has_equal_error = partial(has_expr, test = 'error') +has_equal_error = partial(has_expr, test="error") +has_equal_error.__name__ = "has_equal_error" has_equal_error.__doc__ = """Run targeted student and solution code, and compare generated errors. When called on an SCT chain, ``has_equal_error()`` will execute the student and solution code that is 'zoomed in on' and compare the errors that they generate. - """ + args_string.format("error") + """ + args_string.format( + "error" +) ## Various has tests ---------------------------------------------------------- from pythonwhat.Test import StringContainsTest -def has_code(text, - pattern=True, - not_typed_msg=None, - state=None): + +def has_code(state, text, pattern=True, not_typed_msg=None): """Test the student code. Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``, @@ -389,8 +479,6 @@ def has_code(text, Ex().has_code(r"1\\s*\\+2\\s*\\+3") """ - rep = Reporter.active_reporter - if not not_typed_msg: if pattern: not_typed_msg = "Could not find the correct pattern in your code." @@ -399,18 +487,18 @@ def has_code(text, student_code = state.student_code - _msg = state.build_message(not_typed_msg) - rep.do_test(StringContainsTest(student_code, text, pattern, Feedback(_msg, state))) + state.do_test(StringContainsTest(student_code, text, pattern, not_typed_msg)) return state -from pythonwhat.Test import Test, DefinedCollTest, EqualTest -def has_import(name, - same_as=False, - not_imported_msg="Did you import `{{pkg}}`?", - incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?", - state=None): +def has_import( + state, + name, + same_as=False, + not_imported_msg="Did you import `{{pkg}}`?", + incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?", +): """Checks whether student imported a package or function correctly. Python features many ways to import packages. @@ -461,31 +549,37 @@ def has_import(name, import matplotlib.pyplot as pltttt """ - - rep = Reporter.active_reporter - - student_imports = state.student_imports - solution_imports = state.solution_imports + student_imports = state.ast_dispatcher.find("imports", state.student_ast) + solution_imports = state.ast_dispatcher.find("imports", state.solution_ast) if name not in solution_imports: - raise InstructorError("`has_import()` couldn't find an import of the package %s in your solution code." % name) + raise InstructorError.from_message( + "`has_import()` couldn't find an import of the package %s in your solution code." + % name + ) - fmt_kwargs = { 'pkg': name, 'alias': solution_imports[name] } + fmt_kwargs = {"pkg": name, "alias": solution_imports[name]} - _msg = state.build_message(not_imported_msg, fmt_kwargs) - rep.do_test(DefinedCollTest(name, student_imports, _msg)) + state.do_test( + DefinedCollTest( + name, student_imports, FeedbackComponent(not_imported_msg, fmt_kwargs) + ) + ) - if (same_as): - _msg = state.build_message(incorrect_as_msg, fmt_kwargs) - rep.do_test(EqualTest(solution_imports[name], student_imports[name], _msg)) + if same_as: + state.do_test( + EqualTest( + solution_imports[name], + student_imports[name], + FeedbackComponent(incorrect_as_msg, fmt_kwargs), + ) + ) return state -def has_output(text, - pattern=True, - no_output_msg=None, - state=None): - """Search student output for a pattern. + +def has_output(state, text, pattern=True, no_output_msg=None): + r"""Search student output for a pattern. Among the student and solution process, the student submission and solution code as a string, the ``Ex()`` state also contains the output that a student generated with his or her submission. @@ -516,27 +610,19 @@ def has_output(text, Ex().has_output(r"This is some \w* stuff", no_output_msg = msg) """ - rep = Reporter.active_reporter - if not no_output_msg: no_output_msg = "You did not output the correct things." - _msg = state.build_message(no_output_msg) - rep.do_test( - StringContainsTest( - state.raw_student_output, - text, - pattern, - _msg)) + state.do_test( + StringContainsTest(state.raw_student_output, text, pattern, no_output_msg) + ) return state -def has_printout(index, - not_printed_msg=None, - pre_code=None, - name=None, - copy=False, - state=None): + +def has_printout( + state, index, not_printed_msg=None, pre_code=None, name=None, copy=False +): """Check if the right printouts happened. ``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in @@ -584,7 +670,7 @@ def has_printout(index, x = 6 The following SCT will not work: :: - + Ex().has_printout(0) Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated. @@ -592,7 +678,7 @@ def has_printout(index, :Example: - Inside a for loop ``has_printout()`` + Inside a for loop ``has_printout()`` Suppose you have the following solution: :: @@ -600,7 +686,7 @@ def has_printout(index, print(i) The following SCT will not work: :: - + Ex().check_for_loop().check_body().has_printout(0) The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``. @@ -613,45 +699,63 @@ def has_printout(index, """ extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead." - state.assert_root('has_printout', extra_msg=extra_msg) + state.assert_execution_root("has_printout", extra_msg=extra_msg) if not_printed_msg is None: - not_printed_msg = "Have you used `{{sol_call}}` to do the appropriate printouts?" + not_printed_msg = ( + "Have you used `{{sol_call}}` to do the appropriate printouts?" + ) try: - sol_call_ast = state.solution_function_calls['print'][index]['node'] + sol_call_ast = state.ast_dispatcher.find("function_calls", state.solution_ast)[ + "print" + ][index]["node"] except (KeyError, IndexError): - raise InstructorError("`has_printout({})` couldn't find the {} print call in your solution.".format(index, utils.get_ord(index + 1))) + raise InstructorError.from_message( + "`has_printout({})` couldn't find the {} print call in your solution.".format( + index, get_ord(index + 1) + ) + ) out_sol, str_sol = getOutputInProcess( - tree = sol_call_ast, - process = state.solution_process, - context = state.solution_context, - env = state.solution_env, - pre_code = pre_code, - copy = copy + tree=sol_call_ast, + process=state.solution_process, + context=state.solution_context, + env=state.solution_env, + pre_code=pre_code, + copy=copy, ) - sol_call_str = state.solution_tree_tokens.get_text(sol_call_ast) + sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast) if isinstance(str_sol, Exception): - raise InstructorError("Evaluating the solution expression {} raised error in solution process." - "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol)) - - _msg = state.build_message(not_printed_msg, { 'sol_call': sol_call_str }) + with debugger(state): + state.report( + "Evaluating the solution expression {} raised error in solution process." + "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol) + ) - has_output(out_sol.strip(), pattern = False, no_output_msg=_msg, state=state) + has_output( + state, + out_sol.strip(), + pattern=False, + no_output_msg=FeedbackComponent(not_printed_msg, {"sol_call": sol_call_str}), + ) return state -def has_no_error(incorrect_msg="Have a look at the console: your code contains an error. Fix it and try again!", state=None): + +def has_no_error( + state, + incorrect_msg="Have a look at the console: your code contains an error. Fix it and try again!", +): """Check whether the submission did not generate a runtime error. If all SCTs for an exercise pass, before marking the submission as correct pythonwhat will automatically check whether the student submission generated an error. This means it is not needed to use ``has_no_error()`` explicitly. However, in some cases, using ``has_no_error()`` explicitly somewhere throughout your SCT execution can be helpful: - + - If you want to make sure people didn't write typos when writing a long function name. - If you want to first verify whether a function actually runs, before checking whether the arguments were specified correctly. - More generally, if, because of the content, it's instrumental that the script runs without @@ -680,7 +784,7 @@ def has_no_error(incorrect_msg="Have a look at the console: your code contains a If you want to make sure that ``train_test_split()`` ran without errors, which would check if the student typed the function without typos and used sensical arguments, you could use the following SCT: :: - + Ex().has_no_error() Ex().check_function('sklearn.model_selection.train_test_split').multi( check_args(['arrays', 0]).has_equal_value(), @@ -688,24 +792,24 @@ def has_no_error(incorrect_msg="Have a look at the console: your code contains a check_args(['options', 'test_size']).has_equal_value(), check_args(['options', 'random_state']).has_equal_value() ) - + If, on the other hand, you want to fall back onto pythonwhat's built in behavior, that checks for an error before marking the exercise as correct, you can simply leave of the ``has_no_error()`` step. """ - state.assert_root('has_no_error') + state.assert_execution_root("has_no_error") - rep = Reporter.active_reporter - if rep.error: - _msg = state.build_message(incorrect_msg, { "error": str(rep.error) }) - rep.do_test(Test(Feedback(_msg, state))) + if state.reporter.errors: + state.report(incorrect_msg, {"error": str(state.reporter.errors[0])}) return state + MC_VAR_NAME = "selected_option" -def has_chosen(correct, msgs, state=None): + +def has_chosen(state, correct, msgs): """Test multiple choice exercise. Test for a MultipleChoiceExercise. The correct answer (as an integer) and feedback messages @@ -714,28 +818,35 @@ def has_chosen(correct, msgs, state=None): Args: correct (int): the index of the correct answer (should be an instruction). Starts at 1. msgs (list(str)): a list containing all feedback messages belonging to each choice of the - student. The list should have the same length as the number of instructions. + student. The list should have the same length as the number of options. """ if not issubclass(type(correct), int): - raise InstructorError("Inside `has_chosen()`, the argument `correct` should be an integer.") + raise InstructorError.from_message( + "Inside `has_chosen()`, the argument `correct` should be an integer." + ) - rep = Reporter.active_reporter student_process = state.student_process if not isDefinedInProcess(MC_VAR_NAME, student_process): - raise InstructorError("Option not available in the student process") + raise InstructorError.from_message( + "Option not available in the student process" + ) else: selected_option = getOptionFromProcess(student_process, MC_VAR_NAME) if not issubclass(type(selected_option), int): - raise InstructorError("selected_option should be an integer") + raise InstructorError.from_message("selected_option should be an integer") if selected_option < 1 or correct < 1: - raise InstructorError("selected_option and correct should be greater than zero") + raise InstructorError.from_message( + "selected_option and correct should be greater than zero" + ) if selected_option > len(msgs) or correct > len(msgs): - raise InstructorError("there are not enough feedback messages defined") + raise InstructorError.from_message( + "there are not enough feedback messages defined" + ) feedback_msg = msgs[selected_option - 1] - rep.success_msg = msgs[correct - 1] + state.reporter.success_msg = msgs[correct - 1] - rep.do_test(EqualTest(selected_option, correct, feedback_msg)) + state.do_test(EqualTest(selected_option, correct, feedback_msg)) diff --git a/pythonwhat/converters.py b/pythonwhat/converters.py index 65c784e9..16d48ba2 100644 --- a/pythonwhat/converters.py +++ b/pythonwhat/converters.py @@ -1,19 +1,23 @@ import inspect import pythonwhat + def get_manual_converters(): converters = { - 'pandas.io.excel.ExcelFile': lambda x: x.io, - 'builtins.dict_keys': lambda x: sorted(x), - 'builtins.dict_items': lambda x: sorted(x), - 'bs4.BeautifulSoup': lambda x: str(x), - 'bs4.element.Tag': lambda x: str(x), - 'bs4.element.NavigableString': lambda x: str(x), - 'bs4.element.ResultSet': lambda x: [str(res) for res in x], - 'h5py._hl.files.File': lambda x: x.file.filename, - 'h5py._hl.group.Group': lambda x: x.file.filename + '_' + str([x for x in x.keys()]), - 'sqlalchemy.engine.base.Engine': lambda x: x.url.database + "pandas.io.excel.ExcelFile": lambda x: x.io, + "pandas.io.excel._base.ExcelFile": lambda x: x.io, + "builtins.dict_keys": lambda x: sorted(x), + "builtins.dict_items": lambda x: sorted(x), + "bs4.BeautifulSoup": lambda x: str(x), + "bs4.element.Tag": lambda x: str(x), + "bs4.element.NavigableString": lambda x: str(x), + "bs4.element.ResultSet": lambda x: [str(res) for res in x], + "h5py._hl.files.File": lambda x: x.file.filename, + "h5py._hl.group.Group": lambda x: x.file.filename + + "_" + + str([x for x in x.keys()]), + "sqlalchemy.engine.base.Engine": lambda x: x.url.database, } - return(converters) + return converters diff --git a/pythonwhat/feedback.py b/pythonwhat/feedback.py new file mode 100644 index 00000000..dae753f0 --- /dev/null +++ b/pythonwhat/feedback.py @@ -0,0 +1,19 @@ +from typing import Dict + +from protowhat.Feedback import Feedback as ProtoFeedback + + +class Feedback(ProtoFeedback): + ast_highlight_offset = {"column_start": 1} + + @classmethod + def get_highlight_position(cls, highlight) -> Dict[str, int]: + if getattr(highlight, "first_token", None) and getattr( + highlight, "last_token", None + ): + return { + "line_start": highlight.first_token.start[0], + "column_start": highlight.first_token.start[1], + "line_end": highlight.last_token.end[0], + "column_end": highlight.last_token.end[1], + } diff --git a/pythonwhat/local.py b/pythonwhat/local.py index 3a1e9be7..7dee518e 100644 --- a/pythonwhat/local.py +++ b/pythonwhat/local.py @@ -1,14 +1,23 @@ import io +import os import random - -from pythonwhat.check_syntax import Ex -from pythonwhat.State import State -from pythonwhat.Reporter import Reporter +from pathlib import Path from contextlib import redirect_stdout -class StubShell(object): +from multiprocessing import Process, Queue +from protowhat.Reporter import Reporter + +try: + from pythonbackend.shell_utils import create + from pythonbackend.tasks import TaskCaptureFullOutput + + BACKEND_AVAILABLE = True +except: + BACKEND_AVAILABLE = False + - def __init__(self, init_code = None): +class StubShell: + def __init__(self, init_code=None): self.user_ns = {} if init_code: self.run_code(init_code) @@ -16,35 +25,269 @@ def __init__(self, init_code = None): def run_code(self, code): exec(code, self.user_ns) -class StubProcess(object): - def __init__(self, init_code = None, pid = None): +class StubProcess: + def __init__(self, init_code=None, pid=None): self.shell = StubShell(init_code) - self._identity = (pid,) if pid else (random.randint(0, 1e12),) + self._identity = (pid,) if pid else (random.randint(0, int(1e12)),) def executeTask(self, task): return task(self.shell) -def setup_state(stu_code = "", sol_code = "", pec = "", pid = None): - stu_output = io.StringIO() - with redirect_stdout(stu_output): - stu_process = StubProcess("%s\n%s" % (pec, stu_code), pid) +class TaskCaptureOutput: + def __init__(self, code): + self.code = code + + def __call__(self, shell): + return run_code(shell.run_code, self.code) + + +class TaskKillProcess: + def __call__(self, shell): + return None + + +class CaptureErrors: + def __init__(self, output): + self.output = output + + def __enter__(self): + pass + + def __exit__(self, exc_type, exception, traceback): + if exc_type is not None: + self.output.append({"type": "backend-error", "payload": str(exception)}) + return True + + +class WorkerProcess(Process): + instances = [] + + def __init__(self, pid=None): + Process.__init__(self) + self.task_queue = Queue() + self.result_queue = Queue() + self.daemon = ( + True + ) # when parent process is killed, sub/childprocess get also killed + self.instances.append(self) + # used to detect single process exercise + self._identity = (pid,) if pid else (random.randint(0, int(1e12)),) + + def get_shell(self): + return create({}) + + def run(self): + shell = self.get_shell() + while True: + output = [] + with CaptureErrors(output): + next_task = self.task_queue.get() + answer = next_task(shell) + if len(output) > 0: # means backend error happened + answer = output + output = [] + with CaptureErrors(output): + self.result_queue.put_nowait(answer) + if len(output) > 0: # means backend error happened + self.result_queue.put_nowait(output) + if isinstance(next_task, TaskKillProcess): + break # break while loop -> we do not wait upon new task + return + + def executeTask(self, task): + self.task_queue.put_nowait(task) + return self.result_queue.get() # wait and fetches next item in queue + + def kill(self): + try: + if self.is_alive(): + self.executeTask(TaskKillProcess()) + self.join(timeout=3.0) + if self.is_alive(): + self.terminate() + self.join(timeout=3.0) + if self in self.instances: + self.instances.remove(self) + finally: + pass + # python 3.7: + # self.close() + + @classmethod + def kill_all(cls): + for instance in list(cls.instances): + instance.kill() + + +class SimpleProcess(WorkerProcess): + def get_shell(self): + return StubShell() + + +class ChDir(object): + """ + Step into a directory temporarily. + """ + + def __init__(self, path): + self.old_dir = os.getcwd() + self.new_dir = str(path) + + def __enter__(self): + os.chdir(self.new_dir) + + def __exit__(self, *args): + os.chdir(self.old_dir) + + +def run_code(executor, code): + with io.StringIO() as output: + try: + with redirect_stdout(output): + executor(code) + raw_output = output.getvalue() + error = None + except BaseException as e: + raw_output = "" + error = str(e) + return raw_output, error + + +def run_single_process(pec, code, pid=None, mode="simple"): + if mode == "stub": + # no isolation + process = StubProcess(init_code=pec, pid=pid) + raw_output, error = run_code(process.shell.run_code, code) + + elif mode == "simple": + # no advanced functionality + process = SimpleProcess(pid) + process.start() + _ = process.executeTask(TaskCaptureOutput(pec)) + raw_output, error = process.executeTask(TaskCaptureOutput(code)) + + elif mode == "full" and BACKEND_AVAILABLE: + # slow + process = WorkerProcess(pid) + process.start() + _ = process.executeTask( + TaskCaptureFullOutput((pec,), "", None, silent=True) + ) + output, raw_output = process.executeTask( + TaskCaptureFullOutput((code,), "script.py", None, silent=True) + ) + raw_output = raw_output["output_stream"] + error = raw_output["error"] + + else: + raise ValueError("Invalid mode") + + return process, raw_output, error + + +def run_exercise(pec, sol_code, stu_code, sol_wd=None, stu_wd=None, **kwargs): + with ChDir(sol_wd or os.getcwd()): + sol_process, _, _ = run_single_process(pec, sol_code, **kwargs) + + with ChDir(stu_wd or os.getcwd()): + stu_process, raw_stu_output, error = run_single_process(pec, stu_code, **kwargs) + + return sol_process, stu_process, raw_stu_output, error + + +# todo: +# imports from local modules (solution needs to be materialised somewhere) +# converge with xbackend (pythonbackend + look at scalabackend) +# move towards xwhat controlling all execution and xbackend providing the execution interface? +# running with arbitrary wd + path + flags (now only wd) needed? +# e.g. `python -m project.run +# allow setting env vars? e.g. PYTHONPATH, could help running more complex setup +# allow prepending code? set_env? e.g. (automatically) setting __file__? +def run(state, relative_working_dir=None, solution_dir="../solution", run_solution=True): + """Run the focused student and solution code in the specified location + + This function can be used after ``check_file`` to execute student and solution code. + The arguments allow configuring the correct context for execution. + + SCT functions chained after this one that execute pieces of code (custom expressions or the focused part of a file) + execute in the same student and solution locations as the file. + + .. note:: + + This function does not execute the file itself, but code in memory. + This can have an impact when: + + - the solution code imports from a different file in the expected solution (code that is not installed) + - using functionality depending on e.g. ``__file__`` and ``inspect`` + + When the expected code has imports from a different file that is part of the exercise, + it can only work if the solution code provided earlier does not have these imports but instead + has all that functionality inlined. + + Args: + relative_working_dir (str): if specified, this relative path is the subdirectory + inside the student and solution context in which the code is executed + solution_dir (str): a relative path, ``solution`` by default, + that sets the root of the solution context, relative to that of the student execution context + state (State): state as passed by the SCT chain. Don't specify this explicitly. + + If ``relative_working_dir`` is not set, it will be the directory the file was loaded from by ``check_file`` + and fall back to the root of the student execution context (the working directory pythonwhat runs in). + + The ``solution_dir`` helps to prevent solution side effects from conflicting with those of the student. + If the set or derived value of ``relative_working_dir`` is an absolute path, + ``relative_working_dir`` will not be used to form the solution execution working directory: + the solution code will be executed in the root of the solution execution context. + + :Example: + + Suppose the student and solution have a file ``script.py`` in ``/home/repl/``:: + + if True: + a = 1 + + print("Hi!") + + We can check it with this SCT (with ``file_content`` containing the expected file content):: - sol_output = io.StringIO() - with redirect_stdout(sol_output): - sol_process = StubProcess("%s\n%s" % (pec, sol_code), pid) + Ex().check_file( + "script.py", + solution_code=file_content + ).run().multi( + check_object("a").has_equal_value(), + has_printout(0) + ) + """ + # todo: + # look into executing the file itself + # and keeping the process alive to extract values + if relative_working_dir is None: + if getattr(state, "path", False): + relative_working_dir = state.path.parent + else: + relative_working_dir = "" - rep = Reporter() - Reporter.active_reporter = rep + if not os.path.isabs(str(relative_working_dir)): + sol_wd = Path(os.getcwd(), solution_dir, relative_working_dir) + else: + sol_wd = Path(os.getcwd(), solution_dir) - state = State( - student_code = stu_code, - solution_code = sol_code, - pre_exercise_code = pec, - student_process = stu_process, - solution_process = sol_process, - raw_student_output = stu_output.getvalue()) + os.makedirs(str(sol_wd), exist_ok=True) + stu_wd = Path(os.getcwd(), relative_working_dir) + sol_code = state.solution_code or "" if run_solution else "" - State.root_state = state - return(Ex(state)) + sol_process, stu_process, raw_stu_output, error = run_exercise( + pec="", + sol_code=sol_code, + stu_code=state.student_code, + sol_wd=sol_wd, + stu_wd=stu_wd, + ) + return state.to_child( + student_process=stu_process, + solution_process=sol_process, + raw_student_output=raw_stu_output, + reporter=Reporter(state.reporter, errors=[error] if error else []), + ) diff --git a/pythonwhat/parsing.py b/pythonwhat/parsing.py index 430a00fd..cd9102d4 100644 --- a/pythonwhat/parsing.py +++ b/pythonwhat/parsing.py @@ -16,7 +16,10 @@ https://greentreesnakes.readthedocs.org/en/latest/ """ -class EmptyTargetVar: pass + +class EmptyTargetVar: + pass + class TargetVars(Mapping): """Immutable ordered mapping from target variables to their values.""" @@ -24,15 +27,20 @@ class TargetVars(Mapping): EMPTY = EmptyTargetVar() def __init__(self, target_vars=tuple(), is_empty=True): - if is_empty: + if is_empty: target_vars = [(v, self.EMPTY) for v in target_vars] self._od = OrderedDict(target_vars) # getitem, len, iter wrap OrderedDict behavior - def __getitem__(self, k): return self._od.__getitem__(k) - def __len__(self): return self._od.__len__() - def __iter__(self): return self._od.__iter__() + def __getitem__(self, k): + return self._od.__getitem__(k) + + def __len__(self): + return self._od.__len__() + + def __iter__(self): + return self._od.__iter__() def update(self, *args, **kwargs): cpy = self.copy() @@ -44,12 +52,17 @@ def copy(self): def __str__(self): """Format target vars for printing""" - if len(self) > 1: return "({})".format(", ".join(self._od.keys())) - else: return "".join(self._od.keys()) + if len(self) > 1: + return "({})".format(", ".join(self._od.keys())) + else: + return "".join(self._od.keys()) def defined_items(self): """Return copy of instance, omitting entries that are EMPTY""" - return self.__class__([(k, v) for k,v in self.items() if v is not self.EMPTY], is_empty=False) + return self.__class__( + [(k, v) for k, v in self.items() if v is not self.EMPTY], is_empty=False + ) + class IndexedDict(Mapping): """Wrapper around OrderedDict that allows access via item position or key""" @@ -57,15 +70,19 @@ class IndexedDict(Mapping): def __init__(self, *args, **kwargs): self._od = OrderedDict(*args, **kwargs) - def __getitem__(self, k): - try: return list(self._od.values())[k] - except TypeError: return self._od[k] - - def __len__(self): return self._od.__len__() - def __iter__(self): return self._od.__iter__() + def __getitem__(self, k): + try: + return list(self._od.values())[k] + except TypeError: + return self._od[k] + + def __len__(self): + return self._od.__len__() + + def __iter__(self): + return self._od.__iter__() - class Parser(ast.NodeVisitor): """Basic parser. @@ -106,7 +123,7 @@ def generic_visit(self, node): node (ast.Node): The node which is visited. """ pass # This ignore is necessary to keep the parser at base level, also look comment above in - # the visit_Module function body. + # the visit_Module function body. def visit_each(self, lst): for el in lst: @@ -115,11 +132,12 @@ def visit_each(self, lst): @staticmethod def get_target_vars(target): get_id = lambda n: n.id if not isinstance(n, ast.Starred) else n.value.id - if isinstance(target, (ast.Name, ast.Starred)): + if isinstance(target, (ast.Name, ast.Starred)): tv = [get_id(target)] - elif isinstance(target, ast.Tuple): + elif isinstance(target, ast.Tuple): tv = [get_id(node) for node in target.elts] - else: tv = [] + else: + tv = [] return TargetVars(tv) @@ -127,7 +145,7 @@ def get_target_vars(target): def get_arg(el): if el is None: return None - else : + else: return el.arg @staticmethod @@ -138,7 +156,7 @@ def get_arg_tuples(arguments, defaults): @staticmethod def get_arg_parts(arguments, defaults, type): - # only difference is that it doesn't pull out arg.arg, so we can + # only difference is that it doesn't pull out arg.arg, so we can # use all the information on the arg node down the road match_def = [None] * (len(arguments) - len(defaults)) + defaults part_list = [] @@ -149,18 +167,19 @@ def get_arg_parts(arguments, defaults, type): @staticmethod def get_arg_part(_arg, _def, type=None): # type is arg, kwonly, kwarg, vararg - if not _arg: return None + if not _arg: + return None # part uses default highlighting, so will highlight "node" entry return { - 'node': _def or _arg, - 'arg': _arg, - # TODO: need to fill out - 'type': type, - 'is_default': True if _def else False, - 'name': _arg.arg, - 'annotation': _arg.annotation - } + "node": _def or _arg, + "arg": _arg, + # TODO: need to fill out + "type": type, + "is_default": True if _def else False, + "name": _arg.arg, + "annotation": _arg.annotation, + } # class OperatorParser(Parser): @@ -261,8 +280,8 @@ class FunctionParser(Parser): """ def __init__(self): - self.gen_name = '' - self.raw_name = '' + self.gen_name = "" + self.raw_name = "" self.mappings = {} self.out = {} self.call_lookup_active = False @@ -278,6 +297,7 @@ def visit_AugAssign(self, node): self.visit(node.value) def visit_Compare(self, node): + self.visit(node.left) self.visit_each(node.comparators) def visit_UnaryOp(self, node): @@ -288,7 +308,7 @@ def visit_Import(self, node): if imp.asname is not None: self.mappings[imp.asname] = imp.name else: - pass # e.g. numpy import as numpy, so no action needed. + pass # e.g. numpy import as numpy, so no action needed. def visit_ImportFrom(self, node): for imp in node.names: @@ -298,24 +318,26 @@ def visit_Expr(self, node): self.visit(node.value) def visit_List(self, node): - [ self.visit(el) for el in node.elts ] + [self.visit(el) for el in node.elts] def visit_Dict(self, node): - [ self.visit(el) for el in node.values ] + [self.visit(el) for el in node.values] def visit_Call(self, node): if self.call_lookup_active: self.visit(node.func) - else : + else: self.call_lookup_active = True - self.visit(node.func) # Need to visit func to start recording the current function name. + self.visit( + node.func + ) # Need to visit func to start recording the current function name. if self.gen_name: - if (self.gen_name not in self.out): + if self.gen_name not in self.out: self.out[self.gen_name] = [] self.out[self.gen_name].append(self.get_call_part(node)) - #self.out[self.current].append((node, node.args, node.keywords)) + # self.out[self.current].append((node, node.args, node.keywords)) self.gen_name = self.raw_name = "" self.call_lookup_active = False @@ -341,44 +363,46 @@ def visit_Subscript(self, node): def visit_Name(self, node): self.gen_name = self.mappings.get(node.id) or node.id self.raw_name = node.id - + def get_call_part(self, node): args = [self.get_pos_arg_part(n, ii) for ii, n in enumerate(node.args)] keywords = [self.get_kw_arg_part(n) for n in node.keywords] - return {'node': node, - # TODO: right now, args and keywords can be indexed by pos or name. - # Note that a pos args name is its position. - # Problems will arise if SCT tests a position, but the submission - # has too few positional arguments, since it will then grab a kw arg :( - # This is not necessarily a bad thing, but instructors would need to be - # Careful deciding when to test a pos arg, and when to test using kw. - # Could use check_pos_args with pos_args entry below to solve. - 'args': IndexedDict((n['name'], n) for n in [*args, *keywords]), - #'pos_args': args, - #'keywords': keywords, - 'name': self.raw_name - } - - def get_pos_arg_part(self, arg, indx_pos): + return { + "node": node, + # TODO: right now, args and keywords can be indexed by pos or name. + # Note that a pos args name is its position. + # Problems will arise if SCT tests a position, but the submission + # has too few positional arguments, since it will then grab a kw arg :( + # This is not necessarily a bad thing, but instructors would need to be + # Careful deciding when to test a pos arg, and when to test using kw. + # Could use check_pos_args with pos_args entry below to solve. + "args": IndexedDict((n["name"], n) for n in [*args, *keywords]), + #'pos_args': args, + #'keywords': keywords, + "name": self.raw_name, + } + + @staticmethod + def get_pos_arg_part(arg, indx_pos): is_star = isinstance(arg, ast.Starred) return { - 'node': arg if not is_star else arg.value, - 'highlight': arg, - 'type': 'argument', - 'is_starred': is_star, - 'name': indx_pos - } - - def get_kw_arg_part(self, arg): + "node": arg if not is_star else arg.value, + "highlight": arg, + "type": "argument", + "is_starred": is_star, + "name": indx_pos, + } + + @staticmethod + def get_kw_arg_part(arg): is_kwarg = arg.arg is None return { - 'node': arg.value, - 'highlight': arg, - 'type': 'keyword', - 'is_kwarg': is_kwarg, - 'name': arg.arg - } - + "node": arg.value, + "highlight": arg, + "type": "keyword", + "is_kwarg": is_kwarg, + "name": arg.arg, + } class ObjectAccessParser(FunctionParser): @@ -416,12 +440,13 @@ def visit_Name(self, node): # if name refers to an import, replace prefix = self.mappings.get(node.id) or node.id - self.gen_name = prefix + "." + self.gen_name if self.gen_name else prefix + self.gen_name = prefix + "." + self.gen_name if self.gen_name else prefix self.raw_name = node.id + "." + self.raw_name if self.raw_name else node.id self.out.append(self.gen_name) self.gen_name = self.raw_name = "" + class ObjectAssignmentParser(Parser): """Find object assignmnts @@ -438,7 +463,7 @@ def visit_Name(self, node): if node.id not in self.out: self.out[node.id] = self.get_part(node, self.active_assignment) else: - self.out[node.id]['highlight'] = None + self.out[node.id]["highlight"] = None self.active_assignment = None def visit_Attribute(self, node): @@ -474,14 +499,10 @@ def visit_Try(self, node): @staticmethod def get_part(name_node, ass_node=None): # either name node or simply str or name itself - name = getattr(name_node, 'id', name_node) + name = getattr(name_node, "id", name_node) load_name = ast.Name(id=name, ctx=ast.Load()) ast.fix_missing_locations(load_name) - return { - 'name': name, - 'node': load_name, - 'highlight': ass_node or name_node, - } + return {"name": name, "node": load_name, "highlight": ass_node or name_node} class IfParser(Parser): @@ -495,12 +516,9 @@ def __init__(self): self.out = [] def visit_If(self, node): - self.out.append({ - 'node': node, - 'test': node.test, - 'body': node.body, - 'orelse': node.orelse, - }) + self.out.append( + {"node": node, "test": node.test, "body": node.body, "orelse": node.orelse} + ) class IfExpParser(IfParser): @@ -510,7 +528,8 @@ class IfExpParser(IfParser): Only 'top-level' if structures will be found! """ - def visit_If(self, node): return + def visit_If(self, node): + return def visit_IfExp(self, node): super().visit_If(node) @@ -543,12 +562,9 @@ def __init__(self): self.out = [] def visit_While(self, node): - self.out.append({ - 'node': node, - 'test': node.test, - 'body': node.body, - 'orelse': node.orelse - }) + self.out.append( + {"node": node, "test": node.test, "body": node.body, "orelse": node.orelse} + ) class ForParser(Parser): @@ -563,14 +579,17 @@ def __init__(self): def visit_For(self, node): tv = Parser.get_target_vars(node.target) - self.out.append({ - 'node': node, - 'iter': node.iter, - 'body': {'node': node.body, 'target_vars': tv}, - 'orelse': {'node': node.orelse, 'target_vars': tv}, - 'target': node.target, - '_target_vars': tv - }) + self.out.append( + { + "node": node, + "iter": node.iter, + "body": {"node": node.body, "target_vars": tv}, + "orelse": {"node": node.orelse, "target_vars": tv}, + "target": node.target, + "_target_vars": tv, + } + ) + class ClassDefParser(Parser): """Find class definitions @@ -581,17 +600,19 @@ def __init__(self): def visit_ClassDef(self, node): self.out[node.name] = { - 'node': node, - 'bases': [ {'node': node } for node in node.bases ], - 'body': node.body, + "node": node, + "bases": [{"node": node} for node in node.bases], + "body": node.body, } + class FunctionDefParser(Parser): """Find function definitions A parser which inherits from the basic parser to find function definitions. Only 'top-level' for structures will be found! """ + def __init__(self): self.out = {} @@ -604,33 +625,37 @@ def parse_node(cls, node): kwonlyargs = cls.get_arg_tuples(node.args.kwonlyargs, node.args.kw_defaults) # TODO: all single args should be tuples like this vararg = cls.get_arg(node.args.vararg) - kwarg = cls.get_arg(node.args.kwarg) + kwarg = cls.get_arg(node.args.kwarg) # create context variables target_vars = [arg[0] for arg in normal_args] - if vararg: target_vars.append(vararg) - if kwarg: target_vars.append(kwarg) - - args = cls.get_arg_parts(node.args.args, node.args.defaults, 'arg') - kw_args = cls.get_arg_parts(node.args.kwonlyargs, node.args.kw_defaults, 'kwonly') - varargs = cls.get_arg_part(node.args.vararg, None, 'vararg') - kwargs = cls.get_arg_part(node.args.kwarg, None, 'kwarg') + if vararg: + target_vars.append(vararg) + if kwarg: + target_vars.append(kwarg) + + args = cls.get_arg_parts(node.args.args, node.args.defaults, "arg") + kw_args = cls.get_arg_parts( + node.args.kwonlyargs, node.args.kw_defaults, "kwonly" + ) + varargs = cls.get_arg_part(node.args.vararg, None, "vararg") + kwargs = cls.get_arg_part(node.args.kwarg, None, "kwarg") all_args = [*args, varargs, *kw_args, kwargs] - - if isinstance(node, ast.Lambda): body_node = node.body + + if isinstance(node, ast.Lambda): + body_node = node.body else: bodyMod = wrap_in_module(node.body) body_node = FunctionBodyTransformer().visit(bodyMod) return { "node": node, - "name": getattr(node, 'name', None), - "args": IndexedDict([ (p['name'], p) for p in all_args if p is not None]), + "name": getattr(node, "name", None), + "args": IndexedDict([(p["name"], p) for p in all_args if p is not None]), # TODO: arg is the node counterpart to target_vars "_spec1_args": args, "*args": varargs, - "**kwargs": kwargs, - "body": {'node': body_node, - 'target_vars': TargetVars(target_vars)} + "**kwargs": kwargs, + "body": {"node": body_node, "target_vars": TargetVars(target_vars)}, } @@ -674,22 +699,26 @@ def build_comp(self, node): target = node.generators[0].target tv = Parser.get_target_vars(target) ifs = node.generators[0].ifs - self.out.append({ + self.out.append( + { "node": node, - "body": {'node': node.elt, 'target_vars': tv}, + "body": {"node": node.elt, "target_vars": tv}, "target": target, "iter": node.generators[0].iter, - "ifs": [{'node': ifnode, 'target_vars': tv} for ifnode in ifs], + "ifs": [{"node": ifnode, "target_vars": tv} for ifnode in ifs], # TODO: 'private' _target_vars, since it shouldn't be set when selecting node, # see remarks in test_list_comp on rewriting - "_target_vars": tv - }) + "_target_vars": tv, + } + ) + class ListCompParser(CompParser): """Find list comprehensions A parser which inherits from the CompParser to find list comprehensions. """ + def visit_ListComp(self, node): self.build_comp(node) @@ -703,36 +732,40 @@ class GeneratorExpParser(CompParser): def visit_GeneratorExp(self, node): self.build_comp(node) + class DictCompParser(CompParser): """Find dictionary comprehensions A parser which inherits from the CompParser to find dict comprehensions. """ + def visit_DictComp(self, node): target = node.generators[0].target tv = Parser.get_target_vars(target) ifs = node.generators[0].ifs - self.out.append({ + self.out.append( + { "node": node, - "key": {'node': node.key, 'target_vars': tv}, - "value": {'node': node.value, 'target_vars': tv}, + "key": {"node": node.key, "target_vars": tv}, + "value": {"node": node.value, "target_vars": tv}, "target": target, "iter": node.generators[0].iter, - "ifs": [{'node': ifnode, 'target_vars': tv} for ifnode in ifs], + "ifs": [{"node": ifnode, "target_vars": tv} for ifnode in ifs], # TODO: 'private' _target_vars, since it shouldn't be set when selecting node, # see remarks in test_list_comp on rewriting - "_target_vars": tv - }) + "_target_vars": tv, + } + ) class FunctionBodyTransformer(ast.NodeTransformer): # TODO this does not automatically contain line_end information! def visit_Nonlocal(self, node): - new_node = ast.copy_location(ast.Global(names = node.names), node) + new_node = ast.copy_location(ast.Global(names=node.names), node) return FunctionBodyTransformer.decorate(new_node, node) def visit_Return(self, node): - new_node = ast.copy_location(ast.Expr(value = node.value), node) + new_node = ast.copy_location(ast.Expr(value=node.value), node) return FunctionBodyTransformer.decorate(new_node, node) @staticmethod @@ -741,6 +774,7 @@ def decorate(new_node, node): new_node.last_token = node.last_token return new_node + class WithParser(Parser): def __init__(self): self.out = [] @@ -748,21 +782,26 @@ def __init__(self): def visit_With(self, node): items = node.items context = [ - {"node" : item.context_expr, - "target_vars": self.get_target_vars(item.optional_vars), - "with_items": item, - "highlight": node - } - for item in items] - - tv_all = TargetVars(sum([list(c['target_vars'].items()) for c in context], [])) - - self.out.append({ - "context": context, - "body": {'node': node.body, 'with_items': items}, - "node": node, - "n_vars": len(items) - }) + { + "node": item.context_expr, + "target_vars": self.get_target_vars(item.optional_vars), + "with_items": item, + "highlight": node, + } + for item in items + ] + + tv_all = TargetVars(sum([list(c["target_vars"].items()) for c in context], [])) + + self.out.append( + { + "context": context, + "body": {"node": node.body, "with_items": items}, + "node": node, + "n_vars": len(items), + } + ) + class TryExceptParser(Parser): def __init__(self): @@ -778,22 +817,35 @@ def visit_Try(self, node): handlers[el.id] = self.parse_handler(handler) else: # either general handler, or single error handler - k = 'all' if not handler.type else handler.type.id + k = "all" if not handler.type else self.get_identifier(handler.type) handlers[k] = self.parse_handler(handler) - self.out.append({ - "node": node, - "body": node.body, - "orelse": node.orelse or None, - "finalbody": node.finalbody or None, - "handlers": handlers, - }) + self.out.append( + { + "node": node, + "body": node.body, + "orelse": node.orelse or None, + "finalbody": node.finalbody or None, + "handlers": handlers, + } + ) @staticmethod - def parse_handler(handler): return { - 'node': handler.body, - 'target_vars': TargetVars([handler.name]) - } + def get_identifier(obj): + # if only name needs to be correct + return getattr(obj, "id", None) or getattr(obj, "attr") + # if full attr chain must match + # name = "" + # while hasattr(obj, "attr"): + # name = "." + obj.attr + name + # obj = obj.value + # name = obj.id + name + # return name + + @staticmethod + def parse_handler(handler): + return {"node": handler.body, "target_vars": TargetVars([handler.name])} + parser_dict = { "object_accesses": ObjectAccessParser, @@ -812,5 +864,5 @@ def parse_handler(handler): return { "generator_exps": GeneratorExpParser, "withs": WithParser, "try_excepts": TryExceptParser, - "function_calls": FunctionParser + "function_calls": FunctionParser, } diff --git a/pythonwhat/probe.py b/pythonwhat/probe.py index ce3aa21d..9a041e4d 100644 --- a/pythonwhat/probe.py +++ b/pythonwhat/probe.py @@ -4,6 +4,8 @@ from functools import partial from collections import OrderedDict from pythonwhat import test_funcs +from pythonwhat.State import State +from pythonwhat.checks.check_wrappers import state_partial TEST_NAMES = [ "test_mc", @@ -25,20 +27,21 @@ "test_expression_result", "test_expression_output", "test_function_definition", - "test_object_after_expression" + "test_object_after_expression", ] SUB_TESTS = { - "test_if_else": ['test', 'body', 'orelse'], - "test_list_comp": ['comp_iter', 'body', 'ifs'], - "check_correct": ['check', 'diagnose'], - "test_for_loop": ['for_iter', 'body', 'orelse'], - "test_while_loop": ['test', 'body', 'orelse'], - "test_with": ['context_tests', 'body'], - "test_function_definition": ['body'], - "check_or": ['tests'] + "test_if_else": ["test", "body", "orelse"], + "test_list_comp": ["comp_iter", "body", "ifs"], + "check_correct": ["check", "diagnose"], + "test_for_loop": ["for_iter", "body", "orelse"], + "test_while_loop": ["test", "body", "orelse"], + "test_with": ["context_tests", "body"], + "test_function_definition": ["body"], + "check_or": ["tests"], } + class Tree(object): def __init__(self): """ @@ -54,10 +57,19 @@ def __init__(self): self.crnt_node = self.root @classmethod - def str_branch(cls, node, str_func=lambda s: ""): - f = node.data.get('func') - this_node = " "*node.depth + getattr(f, '__name__', node.name) + str_func(node) + "\n" - return this_node + "".join(map(lambda x: cls.str_branch(x, str_func), node.child_list)) + def str_branch(cls, node, str_func=lambda s: str(s)): + # dict(getattr(s.data.get("bound_args", {}), 'arguments', {})) + f = node.data.get("func") + this_node = ( + " " * node.depth + + "(" + + getattr(f, "__name__", node.name) + + str_func(node) + + ")\n" + ) + return this_node + "".join( + map(lambda x: cls.str_branch(x, str_func), node.child_list) + ) def __str__(self): return self.str_branch(self.crnt_node) @@ -69,10 +81,12 @@ def descend(self, node=None): return sum(children, base) def __iter__(self): - for ii in self.descend(self.crnt_node): yield ii + for ii in self.descend(self.crnt_node): + yield ii + class Node(object): - def __init__(self, child_list = None, data = None, name="unnamed", arg_name=""): + def __init__(self, child_list=None, data=None, name="unnamed", arg_name=""): """ Hold a function call with its bound arguments, along with child nodes. @@ -87,49 +101,55 @@ def __init__(self, child_list = None, data = None, name="unnamed", arg_name=""): def __call__(self, state=None): """Call original function with its arguments, and optional state""" - ba = self.data['bound_args'] + ba = self.data["bound_args"] if state: - self.data['func'](state=state, *ba.args, **ba.kwargs) + func = self.data["func"] + ba = inspect.signature(func).bind(state, *ba.args[1:], **ba.kwargs) + func(*ba.args, **ba.kwargs) return state else: - self.data['func'](*ba.args, **ba.kwargs) + self.data["func"](*ba.args, **ba.kwargs) ba.apply_defaults() - return ba.arguments['state'] + return ba.arguments["state"] def __str__(self): - # TODO print function signature without defaults (or with) - return pp.pformat(self.data) + return pp.pformat( + dict(getattr(self.data.get("bound_args", {}), "arguments", {})) + ) def __iter__(self): - for c in self.child_list: yield c + for c in self.child_list: + yield c def partial(self): """Return partial of original function call""" - ba = self.data['bound_args'] - return partial(self.data['func'], *ba.args, **ba.kwargs) + ba = self.data["bound_args"] + return state_partial(self.data["func"], *ba.args[1:], **ba.kwargs) def update_child_calls(self): """Replace child nodes on original function call with their partials""" for node in filter(lambda n: len(n.arg_name), self.child_list): - self.data['bound_args'].arguments[node.arg_name] = node.partial() + self.data["bound_args"].arguments[node.arg_name] = node.partial() self.updated = True def remove_child(self, node): - indx = self.child_list.index(node) - del self.child_list[indx] - return indx + index = self.child_list.index(node) + del self.child_list[index] + return index def add_child(self, child): - # since it is a tree, there is only one parent + # since it is a tree, there is only one parent # note this means we do not allow edges between same layer units - if child.parent: child.parent.remove_child(child) + if child.parent: + child.parent.remove_child(child) child.parent = self self.child_list.append(child) def descend(self, include_me=True): """Descend depth first into all child nodes""" - if include_me: yield self + if include_me: + yield self for child in self.child_list: yield child @@ -137,8 +157,11 @@ def descend(self, include_me=True): @property def depth(self): - if self.parent: return self.parent.depth + 1 - else: return 0 + if self.parent: + return self.parent.depth + 1 + else: + return 0 + class NodeList(Node): def partial(self): @@ -147,6 +170,7 @@ def partial(self): def update_child_calls(self): pass + class Probe(object): def __init__(self, tree, f, eval_on_call=False): self.tree = tree @@ -155,7 +179,7 @@ def __init__(self, tree, f, eval_on_call=False): self.eval_on_call = eval_on_call # TODO: auto sub_test detection self.sub_tests = SUB_TESTS.get(self.test_name) or [] - + def __call__(self, *args, **kwargs): """Bind arguments to original function signature, and store in a Node @@ -164,40 +188,46 @@ def __call__(self, *args, **kwargs): instances are assembled into a tree. """ + if (len(args) > 0 and not isinstance(args[0], State)) or len(args) == 0: + # no state placeholder if a state is passed + args = ["state_placeholder"] + list(args) bound_args = inspect.signature(self.f).bind(*args, **kwargs) - data = dict( - bound_args = bound_args, - func = self.f) + data = dict(bound_args=bound_args, func=self.f) this_node = Node(data=data, name=self.test_name) if self.tree is not None: self.tree.crnt_node.add_child(this_node) # First pass to set up branches off node - da = bound_args.arguments - for st in self.sub_tests: # TODO: auto sub test detection - if st in da and da[st]: - self.build_sub_test_nodes(da[st], self.tree, this_node, st) + arguments = bound_args.arguments + for subtest in self.sub_tests: # TODO: auto sub test detection + if subtest in arguments and arguments[subtest]: + self.build_sub_test_nodes( + arguments[subtest], self.tree, this_node, subtest + ) # Second pass to build node and all its children into a subtest for node in this_node.descend(include_me=True): - if node.updated: # already built, e.g. node used multiple times + if node.updated: # already built, e.g. node used multiple times continue else: node.update_child_calls() - - if self.eval_on_call: return this_node() - else: return this_node + + if self.eval_on_call: + return this_node() + else: + return this_node @staticmethod def build_sub_test_nodes(test, tree, node, arg_name): # note that I've made the strong assumption that # if not a function, then test is a dict, list or tuple of them - if isinstance(test, (list, tuple)): - nl = NodeList(name = "List", arg_name = arg_name) + if isinstance(test, (list, tuple)): + nl = NodeList(name="List", arg_name=arg_name) node.add_child(nl) - for ii, f in enumerate(test): Probe.build_sub_test_nodes(f, tree, nl, str(ii)) + for ii, f in enumerate(test): + Probe.build_sub_test_nodes(f, tree, nl, str(ii)) elif isinstance(test, Node): # test was a lambdaless subtest call, which produced a node # so need to tell it what its arg_name was on parent test @@ -206,7 +236,7 @@ def build_sub_test_nodes(test, tree, node, arg_name): elif callable(test): # test was inside a lambda, function containing subtests or v2 F() chain object with subtests # since either may contain multiple subtests, we put them in a node list - nl = NodeList(name = "ListDeferred", arg_name = arg_name) + nl = NodeList(name="ListDeferred", arg_name=arg_name) node.add_child(nl) if tree is not None: prev_node, tree.crnt_node = tree.crnt_node, nl diff --git a/pythonwhat/sct_syntax.py b/pythonwhat/sct_syntax.py new file mode 100644 index 00000000..2ff2af6c --- /dev/null +++ b/pythonwhat/sct_syntax.py @@ -0,0 +1,70 @@ +from protowhat.sct_syntax import EagerChain, ExGen, LazyChainStart, state_dec_gen, LazyChain +from pythonwhat.checks.check_wrappers import scts +from pythonwhat.State import State +from pythonwhat.probe import Node, Probe, TEST_NAMES +from pythonwhat.utils import include_v1 +from pythonwhat import test_funcs +from functools import wraps + +# TODO: could define scts for check_wrappers at the module level +sct_dict = scts.copy() + + +def multi_dec(f): + """Decorator for multi to remove nodes for original test functions from root node""" + + @wraps(f) + def wrapper(*args, **kwargs): + args = ( + args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args + ) + for arg in args: + if isinstance(arg, Node) and arg.parent.name == "root": + arg.parent.remove_child(arg) + arg.update_child_calls() + return f(*args, **kwargs) + + return wrapper + + +state_dec = state_dec_gen(sct_dict) + +# todo: __all__? +assert ExGen +assert LazyChainStart + + +def Ex(state=None): + return EagerChain(state=state or State.root_state, chainable_functions=sct_dict) + + +def F(): + return LazyChain(chainable_functions=sct_dict) + + +def get_chains(): + return { + "Ex": ExGen(sct_dict, State.root_state), + "F": LazyChainStart(sct_dict), + } + + +if include_v1(): + # Prepare SCTs that may be chained attributes ---------------------- + # decorate functions that may try to run test_* function nodes as subtests + # so they remove those nodes from the tree + for k in ["multi", "with_context"]: + sct_dict[k] = multi_dec(sct_dict[k]) + + # allow test_* functions as chained attributes + for k in TEST_NAMES: + sct_dict[k] = Probe(tree=None, f=getattr(test_funcs, k), eval_on_call=True) + + # original logical test_* functions behave like multi + # this is necessary to allow them to take check_* funcs as args + # since probe behavior will try to call all SCTs passed (assuming they're also probes) + for k in ["test_or", "test_correct"]: + sct_dict[k] = multi_dec(getattr(test_funcs, k)) + +# Prepare check_funcs to be used alone (e.g. test = check_with().check_body()) +v2_check_functions = {k: state_dec(v) for k, v in scts.items()} diff --git a/pythonwhat/signatures.py b/pythonwhat/signatures.py index 8004eb34..94ffaa72 100644 --- a/pythonwhat/signatures.py +++ b/pythonwhat/signatures.py @@ -3,132 +3,175 @@ import pythonwhat from pythonwhat.tasks import getSignatureFromObjInProcess + def sig_from_params(*args): - return(inspect.Signature(list(args))) + return inspect.Signature(list(args)) + def sig_from_obj(obj_char): - return getSignatureFromObjInProcess(obj_char, - pythonwhat.State.State.root_state.solution_process) + return getSignatureFromObjInProcess( + obj_char, pythonwhat.State.State.root_state.solution_process + ) + def get_manual_sigs(): manual_sigs = { # builtins - 'abs': [param('x', param.POSITIONAL_ONLY)], - 'all': [param('iterable', param.POSITIONAL_ONLY)], - 'any': [param('iterable', param.POSITIONAL_ONLY)], - 'ascii': [param('obj', param.POSITIONAL_ONLY)], - 'bin': [param('number', param.POSITIONAL_ONLY)], - 'bool': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'chr': [param('i', param.POSITIONAL_ONLY)], - 'callable': [param('obj', param.POSITIONAL_ONLY)], - 'classmethod': [param('function', param.POSITIONAL_ONLY)], - 'complex': [param('imag', param.POSITIONAL_OR_KEYWORD, default=0), - param('real', param.POSITIONAL_OR_KEYWORD, default=0)], - 'delattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY)], - 'dir': [param('object', param.POSITIONAL_OR_KEYWORD, default=None)], - 'divmod': [param('x', param.POSITIONAL_ONLY), - param('y', param.POSITIONAL_ONLY)], - 'enumerate': [param('iterable', param.POSITIONAL_ONLY), - param('start', param.POSITIONAL_OR_KEYWORD, default=0)], - 'float': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'getattr': [param('object', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY), - param('default', param.POSITIONAL_ONLY, default=None)], - 'hasattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY)], - 'hash': [param('obj', param.POSITIONAL_ONLY)], - 'hex': [param('number', param.POSITIONAL_ONLY)], - 'id': [param('obj', param.POSITIONAL_ONLY)], - 'int': [param('x', param.POSITIONAL_OR_KEYWORD), - param('base', param.POSITIONAL_OR_KEYWORD, default=10)], - 'isinstance': [param('obj', param.POSITIONAL_ONLY), - param('class_or_tuple', param.POSITIONAL_ONLY)], - 'issubclass': [param('cls', param.POSITIONAL_ONLY), - param('class_or_tuple', param.POSITIONAL_ONLY)], - 'list': [param('iterable', param.POSITIONAL_ONLY, default=None)], - 'len': [param('obj', param.POSITIONAL_ONLY)], - 'oct': [param('number', param.POSITIONAL_ONLY)], - 'open': [param('file', param.POSITIONAL_OR_KEYWORD), - param('mode', param.POSITIONAL_OR_KEYWORD, default='r'), - param('buffering', param.POSITIONAL_OR_KEYWORD, default=1), - param('encoding', param.POSITIONAL_OR_KEYWORD, default=None), - param('errors', param.POSITIONAL_OR_KEYWORD, default=None), - param('newline', param.POSITIONAL_OR_KEYWORD, default=None), - param('closefd', param.POSITIONAL_OR_KEYWORD, default=None), - param('opener', param.POSITIONAL_OR_KEYWORD, default=None)], - 'ord': [param('c', param.POSITIONAL_ONLY)], - 'pow': [param('x', param.POSITIONAL_ONLY), - param('y', param.POSITIONAL_ONLY), - param('z', param.POSITIONAL_ONLY, default=None)], - 'print': [param('value', param.POSITIONAL_ONLY)], - 'repr': [param('obj', param.POSITIONAL_ONLY)], - 'reversed': [param('sequence', param.POSITIONAL_ONLY)], - 'round': [param('number', param.POSITIONAL_OR_KEYWORD), - param('ndigits', param.POSITIONAL_OR_KEYWORD, default=0)], - 'set': [param('iterable', param.POSITIONAL_ONLY, default=None)], - + "abs": [param("x", param.POSITIONAL_ONLY)], + "all": [param("iterable", param.POSITIONAL_ONLY)], + "any": [param("iterable", param.POSITIONAL_ONLY)], + "ascii": [param("obj", param.POSITIONAL_ONLY)], + "bin": [param("number", param.POSITIONAL_ONLY)], + "bool": [param("x", param.POSITIONAL_OR_KEYWORD)], + "chr": [param("i", param.POSITIONAL_ONLY)], + "callable": [param("obj", param.POSITIONAL_ONLY)], + "classmethod": [param("function", param.POSITIONAL_ONLY)], + "complex": [ + param("imag", param.POSITIONAL_OR_KEYWORD, default=0), + param("real", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "delattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + ], + "dir": [param("object", param.POSITIONAL_OR_KEYWORD, default=None)], + "divmod": [ + param("x", param.POSITIONAL_ONLY), + param("y", param.POSITIONAL_ONLY), + ], + "enumerate": [ + param("iterable", param.POSITIONAL_ONLY), + param("start", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "float": [param("x", param.POSITIONAL_OR_KEYWORD)], + "getattr": [ + param("object", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + param("default", param.POSITIONAL_ONLY, default=None), + ], + "hasattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + ], + "hash": [param("obj", param.POSITIONAL_ONLY)], + "hex": [param("number", param.POSITIONAL_ONLY)], + "id": [param("obj", param.POSITIONAL_ONLY)], + "int": [ + param("x", param.POSITIONAL_OR_KEYWORD), + param("base", param.POSITIONAL_OR_KEYWORD, default=10), + ], + "isinstance": [ + param("obj", param.POSITIONAL_ONLY), + param("class_or_tuple", param.POSITIONAL_ONLY), + ], + "issubclass": [ + param("cls", param.POSITIONAL_ONLY), + param("class_or_tuple", param.POSITIONAL_ONLY), + ], + "list": [param("iterable", param.POSITIONAL_ONLY, default=None)], + "len": [param("obj", param.POSITIONAL_ONLY)], + "oct": [param("number", param.POSITIONAL_ONLY)], + "open": [ + param("file", param.POSITIONAL_OR_KEYWORD), + param("mode", param.POSITIONAL_OR_KEYWORD, default="r"), + param("buffering", param.POSITIONAL_OR_KEYWORD, default=1), + param("encoding", param.POSITIONAL_OR_KEYWORD, default=None), + param("errors", param.POSITIONAL_OR_KEYWORD, default=None), + param("newline", param.POSITIONAL_OR_KEYWORD, default=None), + param("closefd", param.POSITIONAL_OR_KEYWORD, default=None), + param("opener", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "ord": [param("c", param.POSITIONAL_ONLY)], + "pow": [ + param("x", param.POSITIONAL_ONLY), + param("y", param.POSITIONAL_ONLY), + param("z", param.POSITIONAL_ONLY, default=None), + ], + "print": [param("value", param.POSITIONAL_ONLY)], + "repr": [param("obj", param.POSITIONAL_ONLY)], + "reversed": [param("sequence", param.POSITIONAL_ONLY)], + "round": [ + param("number", param.POSITIONAL_OR_KEYWORD), + param("ndigits", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "set": [param("iterable", param.POSITIONAL_ONLY, default=None)], # Difference v3.4 vs v3.5!!! - 'setattr': [param('obj', param.POSITIONAL_ONLY), - param('name', param.POSITIONAL_ONLY), - param('value', param.POSITIONAL_ONLY)], - 'sorted': [param('iterable', param.POSITIONAL_ONLY), - param('key', param.POSITIONAL_OR_KEYWORD, default=None), - param('reverse', param.POSITIONAL_OR_KEYWORD, default=False)], - 'str': [param('object', param.POSITIONAL_OR_KEYWORD)], - 'sum': [param('iterable', param.POSITIONAL_ONLY), - param('start', param.POSITIONAL_ONLY, default=0)], - 'tuple': [param('iterable', param.POSITIONAL_ONLY, default=None)], - 'type': [param('object', param.POSITIONAL_ONLY)], - 'vars': [param('object', param.POSITIONAL_ONLY)], - + "setattr": [ + param("obj", param.POSITIONAL_ONLY), + param("name", param.POSITIONAL_ONLY), + param("value", param.POSITIONAL_ONLY), + ], + "sorted": [ + param("iterable", param.POSITIONAL_ONLY), + param("key", param.POSITIONAL_OR_KEYWORD, default=None), + param("reverse", param.POSITIONAL_OR_KEYWORD, default=False), + ], + "str": [param("object", param.POSITIONAL_OR_KEYWORD)], + "sum": [ + param("iterable", param.POSITIONAL_ONLY), + param("start", param.POSITIONAL_ONLY, default=0), + ], + "tuple": [param("iterable", param.POSITIONAL_ONLY, default=None)], + "type": [param("object", param.POSITIONAL_ONLY)], + "vars": [param("object", param.POSITIONAL_ONLY)], # int - # str - 'str.center': [param('width', param.POSITIONAL_ONLY), - param('fillchar', param.POSITIONAL_ONLY, default=" ")], - + "str.center": [ + param("width", param.POSITIONAL_ONLY), + param("fillchar", param.POSITIONAL_ONLY, default=" "), + ], # list - 'list.append': [param('object', param.POSITIONAL_ONLY)], - 'list.count': [param('value', param.POSITIONAL_ONLY)], - + "list.append": [param("object", param.POSITIONAL_ONLY)], + "list.count": [param("value", param.POSITIONAL_ONLY)], # dict - # numpy - 'numpy.array': [param('object', param.POSITIONAL_OR_KEYWORD), - param('dtype', param.POSITIONAL_OR_KEYWORD, default=None), - param('copy', param.POSITIONAL_OR_KEYWORD, default=True), - param('order', param.POSITIONAL_OR_KEYWORD, default=None), - param('subok', param.POSITIONAL_OR_KEYWORD, default=False), - param('ndmin', param.POSITIONAL_OR_KEYWORD, default=0)], - 'numpy.random.seed': [param('seed', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.rand': [param('d0', param.POSITIONAL_ONLY, default=None), - param('d1', param.POSITIONAL_ONLY, default=None), - param('d2', param.POSITIONAL_ONLY, default=None), - param('d3', param.POSITIONAL_ONLY, default=None), - param('d4', param.POSITIONAL_ONLY, default=None), - param('d5', param.POSITIONAL_ONLY, default=None), - param('d6', param.POSITIONAL_ONLY, default=None)], - 'numpy.random.randint': [param('low', param.POSITIONAL_OR_KEYWORD), - param('high', param.POSITIONAL_OR_KEYWORD, default=None), - param('size', param.POSITIONAL_OR_KEYWORD, default=None), - param('dtype', param.POSITIONAL_OR_KEYWORD, default='l')], - 'numpy.random.choice': [param('a', param.POSITIONAL_OR_KEYWORD), - param('size', param.POSITIONAL_OR_KEYWORD, default=None), - param('replace', param.POSITIONAL_OR_KEYWORD, default=True), - param('p', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.normal': [param('loc', param.POSITIONAL_OR_KEYWORD, default=0.0), - param('scale', param.POSITIONAL_OR_KEYWORD, default=1.0), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.poisson': [param('lam', param.POSITIONAL_OR_KEYWORD, default=1.0), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.binomial': [param('n', param.POSITIONAL_OR_KEYWORD), - param('p', param.POSITIONAL_OR_KEYWORD), - param('size', param.POSITIONAL_OR_KEYWORD, default=None)], - 'numpy.random.shuffle': [param('x', param.POSITIONAL_OR_KEYWORD)], - 'numpy.random.permutation': [param('x', param.POSITIONAL_OR_KEYWORD)], - + "numpy.array": [ + param("object", param.POSITIONAL_OR_KEYWORD), + param("dtype", param.POSITIONAL_OR_KEYWORD, default=None), + param("copy", param.POSITIONAL_OR_KEYWORD, default=True), + param("order", param.POSITIONAL_OR_KEYWORD, default=None), + param("subok", param.POSITIONAL_OR_KEYWORD, default=False), + param("ndmin", param.POSITIONAL_OR_KEYWORD, default=0), + ], + "numpy.random.seed": [param("seed", param.POSITIONAL_OR_KEYWORD, default=None)], + "numpy.random.rand": [ + param("d0", param.POSITIONAL_ONLY, default=None), + param("d1", param.POSITIONAL_ONLY, default=None), + param("d2", param.POSITIONAL_ONLY, default=None), + param("d3", param.POSITIONAL_ONLY, default=None), + param("d4", param.POSITIONAL_ONLY, default=None), + param("d5", param.POSITIONAL_ONLY, default=None), + param("d6", param.POSITIONAL_ONLY, default=None), + ], + "numpy.random.randint": [ + param("low", param.POSITIONAL_OR_KEYWORD), + param("high", param.POSITIONAL_OR_KEYWORD, default=None), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + param("dtype", param.POSITIONAL_OR_KEYWORD, default="l"), + ], + "numpy.random.choice": [ + param("a", param.POSITIONAL_OR_KEYWORD), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + param("replace", param.POSITIONAL_OR_KEYWORD, default=True), + param("p", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.normal": [ + param("loc", param.POSITIONAL_OR_KEYWORD, default=0.0), + param("scale", param.POSITIONAL_OR_KEYWORD, default=1.0), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.poisson": [ + param("lam", param.POSITIONAL_OR_KEYWORD, default=1.0), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.binomial": [ + param("n", param.POSITIONAL_OR_KEYWORD), + param("p", param.POSITIONAL_OR_KEYWORD), + param("size", param.POSITIONAL_OR_KEYWORD, default=None), + ], + "numpy.random.shuffle": [param("x", param.POSITIONAL_OR_KEYWORD)], + "numpy.random.permutation": [param("x", param.POSITIONAL_OR_KEYWORD)], # others - 'math.radians': [param('x', param.POSITIONAL_ONLY)] + "math.radians": [param("x", param.POSITIONAL_ONLY)], } - return(manual_sigs) + return manual_sigs diff --git a/pythonwhat/tasks.py b/pythonwhat/tasks.py index af320c2e..3516ff7b 100644 --- a/pythonwhat/tasks.py +++ b/pythonwhat/tasks.py @@ -1,5 +1,4 @@ from pythonwhat import utils -import os import dill import pickle import pythonwhat @@ -9,38 +8,48 @@ from pickle import PicklingError from pythonwhat.utils_env import set_context_vals, assign_from_ast from contextlib import contextmanager -from functools import partial, wraps, update_wrapper -from pythonwhat.Feedback import InstructorError +from functools import partial, wraps +from protowhat.failure import InstructorError + + +# Shell is passed as a parameter to partially applied functions in executeTask +# Process is passed as a parameter in SCT function + def process_task(f): """Decorator to (optionally) run function in a process.""" sig = inspect.signature(f) + @wraps(f) def wrapper(*args, **kwargs): # get bound arguments for call ba = sig.bind_partial(*args, **kwargs) # when process is specified, remove from args and use to execute - process = ba.arguments.get('process') + process = ba.arguments.get("process") if process: - ba.arguments['process'] = None + ba.arguments["process"] = None # partial function since shell argument may have been left # unspecified, as it will be passed when the process executes pf = partial(wrapper, *ba.args, **ba.kwargs) return process.executeTask(pf) # otherwise, run original function return f(*ba.args, **ba.kwargs) + return wrapper + def get_env(ns): - if '__env__' in ns: - return ns['__env__'] + if "__env__" in ns: + return ns["__env__"] else: return ns + @contextmanager def capture_output(): import sys from io import StringIO + oldout, olderr = sys.stdout, sys.stderr out = [StringIO(), StringIO()] sys.stdout, sys.stderr = out @@ -55,41 +64,47 @@ def capture_output(): def getOptionFromProcess(process, name, shell): return shell.user_ns[name] + # Is a variable is defined in the process? @process_task def isDefinedInProcess(name, process, shell): return name in get_env(shell.user_ns) + # Is a variable is of a certain class in the process? @process_task def isInstanceInProcess(name, klass, process, shell): return isinstance(get_env(shell.user_ns)[name], klass) + # Get the columns of a Pandas data frame in the process @process_task def getColumnsInProcess(name, process, shell): return list(get_env(shell.user_ns)[name].columns) + # Is a key defined in a collection in the process? @process_task def isDefinedCollInProcess(name, key, process, shell): return key in get_env(shell.user_ns)[name] + # Get the signature of a function inside the process + def get_signature(name, mapped_name, signature, manual_sigs, env): if isinstance(signature, str): if signature in manual_sigs: signature = inspect.Signature(manual_sigs[signature]) else: - raise InstructorError('signature error - specified signature not found') + raise InstructorError.from_message("signature error - specified signature not found") if signature is None: # establish function try: fun = eval(mapped_name, env) except: - raise InstructorError("%s() was not found." % mapped_name) + raise InstructorError.from_message("%s() was not found." % mapped_name) # first go through manual sigs # try to get signature @@ -104,33 +119,36 @@ def get_signature(name, mapped_name, signature, manual_sigs, env): els[0] = type(eval(els[0], env)).__name__ generic_name = ".".join(els[:]) except: - raise InstructorError('signature error - cannot convert call') + raise InstructorError.from_message("signature error - cannot convert call") if generic_name in manual_sigs: signature = inspect.Signature(manual_sigs[generic_name]) else: - raise InstructorError('signature error - %s not in builtins' % generic_name) + raise InstructorError.from_message( + "signature error - %s not in builtins" % generic_name + ) else: - raise InstructorError('manual signature not found') - except: + raise InstructorError.from_message("manual signature not found") + except Exception as e: try: signature = inspect.signature(fun) except: - raise InstructorError('signature error - cannot determine signature') + raise InstructorError.from_message(e.args[0] + " and cannot determine signature") - return signature + params = [param.replace(annotation=inspect._empty) for param in signature.parameters.values()] + return signature.replace(parameters=params) # Get the signature of a function based on an object inside the process @process_task def getSignatureInProcess(name, mapped_name, signature, manual_sigs, process, shell): - try: - return get_signature(name = name, - mapped_name = mapped_name, - signature = signature, - manual_sigs = manual_sigs, - env = get_env(shell.user_ns)) - except: - return None + return get_signature( + name=name, + mapped_name=mapped_name, + signature=signature, + manual_sigs=manual_sigs, + env=get_env(shell.user_ns), + ) + @process_task def getSignatureFromObjInProcess(obj_char, process, shell): @@ -140,17 +158,17 @@ def getSignatureFromObjInProcess(obj_char, process, shell): return None - - # Stuff for test_with from contextlib import ExitStack + + def context_env_update(context_list, env): es = ExitStack() for item in context_list: # create context manager and enter - tmp_name = '__pw_cm' - cm_code = compile(ast.Expression(item.context_expr), '', 'eval') + tmp_name = "__pw_cm" + cm_code = compile(ast.Expression(item.context_expr), "", "eval") env[tmp_name] = es.enter_context(eval(cm_code, env)) # assign to its optional_vars in separte dict @@ -163,33 +181,35 @@ def context_env_update(context_list, env): @process_task def setUpNewEnvInProcess(context, process, shell): - shell.user_ns['__env__'] = utils.copy_env(shell.user_ns) + shell.user_ns["__env__"] = utils.copy_env(shell.user_ns) try: - es = context_env_update(context, shell.user_ns['__env__']) - shell.user_ns['__exit_stack__'] = es + es = context_env_update(context, shell.user_ns["__env__"]) + shell.user_ns["__exit_stack__"] = es return True except Exception as e: return e + # break down environment def context_objs_exit(es): try: es.close() - return False + return True except Exception as e: - raise e return e + @process_task def breakDownNewEnvInProcess(process, shell): try: - res = context_objs_exit(shell.user_ns['__exit_stack__']) - del shell.user_ns['__exit_stack__'] - del shell.user_ns['__env__'] + res = context_objs_exit(shell.user_ns["__exit_stack__"]) + del shell.user_ns["__exit_stack__"] + del shell.user_ns["__env__"] return res except: return False + # Tasks that may need to serialize across processes =========================== # Get a bytes or string representation of an object in the process @@ -202,10 +222,12 @@ def getClass(name, process, shell): except: return None + @process_task def convert(name, converter, process, shell): return dill.loads(converter)(get_env(shell.user_ns)[name]) + @process_task def getStreamPickle(name, process, shell): try: @@ -213,6 +235,7 @@ def getStreamPickle(name, process, shell): except: return None + @process_task def getStreamDill(name, process, shell): try: @@ -221,70 +244,87 @@ def getStreamDill(name, process, shell): return None -class ReprFail(object): +class ReprFail: def __init__(self, info): self.info = info + def getRepresentation(name, process): obj_class = getClass(name, process) converters = pythonwhat.State.State.root_state.converters if obj_class in converters: repres = convert(name, dill.dumps(converters[obj_class]), process) - if (errored(repres)): - return ReprFail("manual conversion failed") - else: + if errored(repres): + return ReprFail("manual conversion failed: {}".format(repres)) + else: return repres else: # first try to pickle try: stream = getStreamPickle(name, process) - if not errored(stream): return pickle.loads(stream) - except: + if not errored(stream): + return pickle.loads(stream) + except: pass # if it failed, try to dill try: stream = getStreamDill(name, process) - if not errored(stream): return dill.loads(stream) - return ReprFail("dilling inside process failed for %s - write manual converter" % obj_class) + if not errored(stream): + return dill.loads(stream) + return ReprFail( + "dilling inside process failed for %s - write manual converter" + % obj_class + ) except PicklingError: - return ReprFail("undilling of bytestream failed with PicklingError - write manual converter") + return ReprFail( + "undilling of bytestream failed with PicklingError - write manual converter" + ) except Exception as e: - return ReprFail("undilling of bytestream failed for class %s - write manual converter." - "Error: %s - %s" % (obj_class, type(e), e)) + return ReprFail( + "undilling of bytestream failed for class %s - write manual converter." + "Error: %s - %s" % (obj_class, type(e), e) + ) + def errored(el): - return el is None or (isinstance(el, list) and 'backend-error' in str(el)) + return el is None or (isinstance(el, list) and "backend-error" in str(el)) # Make wrapper for getting an object representation from process -------------- -class UndefinedValue: pass +class UndefinedValue: + pass + def getResultFromProcess(res, tempname, process): """Get a value from process, return tuple of value, res if succesful""" if not isinstance(res, (UndefinedValue, Exception)): value = getRepresentation(tempname, process) - return (value, res) - else: - return (res, str(res)) + return value, res + else: + return res, str(res) + # decorator to automatically get value after running process task function def get_rep(f): sig = inspect.signature(f) + @wraps(f) - def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): # get bound arguments for call ba = sig.bind_partial(*args, **kwargs) ba.apply_defaults() # get tempname, process arg values - tempname = ba.arguments['tempname'] - process = ba.arguments['process'] + tempname = ba.arguments["tempname"] + process = ba.arguments["process"] # run process task - res = f(*args,**kwargs) + res = f(*args, **kwargs) # get result from task return getResultFromProcess(res, tempname, process) + return wrapper + ## Get the output of a tree (with setting envs, pre_code and/er expr_code) @process_task def get_output(f, process, shell, *args, **kwargs): @@ -294,60 +334,122 @@ def get_output(f, process, shell, *args, **kwargs): out_str = out[0].strip() if not isinstance(res, Exception): toret = out_str or "no printouts" - return (toret, toret) + return toret, toret else: - return (res, str(res)) + return res, str(res) + @process_task def get_error(f, *args, **kwargs): res = f(*args, **kwargs) return (res, str(res)) if isinstance(res, Exception) else (None, res) + # General tasks to eval or exec code, with decorated counterparts ------------- -# Eval an expression tree or node (with setting envs, pre_code and/or expr_code) + @process_task -def taskRunEval(tree, - process, shell, - env=None, extra_env = None, context=None, context_vals=None, - pre_code = "", expr_code = "", name="", copy=True, tempname='_evaluation_object_', - call=None): - try: +def taskRunEval( + tree, + process, + shell, + env=None, + extra_env=None, + context=None, + context_vals=None, + pre_code="", + expr_code="", + name="", + copy=True, + tempname="_evaluation_object_", + call=None, +): + """ + Eval an expression tree (with setting envs, pre_code and/or expr_code) + Utility function later wrapped to extract either result (end state of a variable), output (stdout) or error + + Args: + tree (ast): current focused ast, used to get code to execute + process: manages shell (see local.py) + shell: link to get process namespace from execution up until now + env: update value in focused code by name + extra_env: variables to be replaced in focused code by name from extra_env in has_expr + context: sum of set_context in sct chain + context_vals: extra context argument in has_expr + pre_code: argument in has_expr to execute code before evaluating, for example to set a seed + expr_code: code to execute instead of focused code + name: extract value after executing focused expr_code (~post_code) + copy: copy entire env because our expr_code could have side effects + tempname: key for the result when it is added to context, only for v1 sct's + call: only used in v1 sct's + + Returns: + str: output of the executed code + """ + try: # Prepare code and mode ----------------------------------------------- - if (expr_code and name) or (not expr_code and isinstance(tree, ast.Module)): - mode = 'exec' + # Verify if expr_code is expression code (returning a value) or just runnable code. + if ( # expr_code returns nothing and then we will extract a value + expr_code and name + ) or ( # No expr_code and the tree is of a node type that does not evaluate to have output + not expr_code and isinstance(tree, ast.Module) + ): + # We are not focused on an expression (no output) + mode = "exec" else: - mode = 'eval' - tree = ast.Expression(tree) + mode = "eval" + # Wrap the focused node in the tree so it can be run with eval() + if not isinstance(tree, (ast.Module, ast.Expression, ast.Expr)): + tree = ast.Expression(tree) # Expression code takes precedence over tree code - if expr_code: code = expr_code - else: code = compile(tree, "