From 69b67a33b26def59a4aaf6c2f5d560c6eff0fc91 Mon Sep 17 00:00:00 2001 From: Filip Schouwenaars Date: Mon, 16 Jan 2017 11:11:58 -0500 Subject: [PATCH 001/418] Delete CONTRIBUTING.md --- CONTRIBUTING.md | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index d663f34e..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,8 +0,0 @@ -Some exercises are very challenging to write appropriate SCTs for. In your issue, be sure to include, - -1. Where you looked in [the wiki](https://www.github.com/datacamp/pythonwhat/wiki). -2. A link to the __source code__ (i.e. an RMarkdown file), or a copy of the exact pre-exercise and solution code. -3. What you want to test with the SCT, and which message you want it to return for a given submission. (Be exact). -4. What you have already tried (include the code) and why the documentation is not helping you (that way, the documentation can be updated afterwards). - -Thanks! From 0be2da14f5416359501efeb96d6c5805b02495de Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Mon, 27 Feb 2017 15:29:37 -0500 Subject: [PATCH 002/418] add travis yaml --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..2857f5b6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +sudo: false +language: python +python: + - "3.5" + +install: pip install -r requirements.txt + +script: python tests/run_all.py From bc6228df3f4410f894cec17410429b21b24591d0 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 28 Feb 2017 09:39:34 -0500 Subject: [PATCH 003/418] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 09daaba3..80b40bc1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ For a detailed guide on how to use `pythonwhat`, head over to [the online docume Visit [DataCamp Teach](https://www.datacamp.com/teach) to create your own DataCamp Python course, powered by `pythonwhat`. +## Documentation + +* [pythonwhat documentation](http://pythonwhat.readthedocs.io) +* [full tutorial](https://github.com/datacamp/courses-pythonwhat-tutorial) + ## Installation ``` From 1dadce419b72645b4ac8eb1a69615173f696fb33 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 28 Feb 2017 09:35:42 -0500 Subject: [PATCH 004/418] mark matplotlib tests, add requirements.txt --- .travis.yml | 5 ++++- requirements.txt | 15 +++++++++++++++ tests/helper.py | 6 +++--- tests/test_content.py | 2 ++ tests/test_test_function_v2.py | 2 ++ tests/test_test_importing.py | 2 ++ tests/test_test_with.py | 2 ++ 7 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 2857f5b6..3692920f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ language: python python: - "3.5" +before_install: + - pip install -e . + - "git clone https://$GH_TOKEN@github.com/datacamp/pythonbackend.git && pip install ./pythonbackend && rm -rf pythonbackend" install: pip install -r requirements.txt -script: python tests/run_all.py +script: pytest tests -m "not dep_matplotlib" -s diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..06e48ab9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +dill +markdown2 +numpy +pandas +matplotlib +boto3 +bs4 +h5py + +sqlalchemy +requests +xlrd +tweepy +scipy +sas7bdat diff --git a/tests/helper.py b/tests/helper.py index c5828694..4c3e6d7b 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -3,9 +3,9 @@ import re def get_sct_payload(output): - output = [out for out in output if out['type'] == 'sct'] - if (len(output) > 0): - return(output[0]['payload']) + sct_output = [out for out in output if out['type'] == 'sct'] + if (len(sct_output) > 0): + return(sct_output[0]['payload']) else: print(output) return(None) diff --git a/tests/test_content.py b/tests/test_content.py index 00d00cdc..97e21939 100644 --- a/tests/test_content.py +++ b/tests/test_content.py @@ -1,6 +1,8 @@ import unittest import helper +import pytest +@pytest.mark.dep_matplotlib class TestTemplate(unittest.TestCase): def setUp(self): self.data = { diff --git a/tests/test_test_function_v2.py b/tests/test_test_function_v2.py index 4b56a298..f4ef3370 100644 --- a/tests/test_test_function_v2.py +++ b/tests/test_test_function_v2.py @@ -1,5 +1,6 @@ import unittest import helper +import pytest class TestFunctionBase(unittest.TestCase): @@ -955,6 +956,7 @@ def test_pandas(self): self.assertFalse(sct_payload['correct']) self.assertIn("Did you call pad.DataFrame() with the correct arguments?", sct_payload['message']) + @pytest.mark.dep_matplotlib def test_mpl(self): self.data = { "DC_PEC": "", diff --git a/tests/test_test_importing.py b/tests/test_test_importing.py index c4d7e9c1..3e116084 100644 --- a/tests/test_test_importing.py +++ b/tests/test_test_importing.py @@ -1,5 +1,6 @@ import unittest import helper +import pytest class TestExercise1(unittest.TestCase): @@ -536,6 +537,7 @@ def test_for_body(): self.assertTrue(sct_payload['correct']) +@pytest.mark.dep_matplotlib class TestImportWhenTestFunction(unittest.TestCase): def test_pass(self): diff --git a/tests/test_test_with.py b/tests/test_test_with.py index 1b759376..d91b28cd 100644 --- a/tests/test_test_with.py +++ b/tests/test_test_with.py @@ -1,5 +1,6 @@ import unittest import helper +import pytest class TestExercise1(unittest.TestCase): @@ -307,6 +308,7 @@ def test_Pass1_no_lam(self): sct_payload = helper.run(self.data) self.assertTrue(sct_payload['correct']) +@pytest.mark.dep_matplotlib class TestExercise5(unittest.TestCase): def setUp(self): From 236dd0048c2fc9ce7e8f79ba07ca3a6fd63b41b1 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 28 Feb 2017 13:43:22 -0500 Subject: [PATCH 005/418] move iter var check into check funcs --- pythonwhat/check_funcs.py | 27 ++++++++++++++++++++++++ pythonwhat/test_funcs/test_comp.py | 34 +++++------------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index a8596c56..427e940e 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -138,6 +138,33 @@ def has_equal_part_len(name, insufficient_msg, state=None): return state +def has_iter_vars(incorrect_msg, exact_names=False, state=None): + rep = Reporter.active_reporter + # get parts for testing from state + # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. + stu_vars = state.student_parts['_target_vars'] + sol_vars = state.solution_parts['_target_vars'] + stu_target = state.student_parts['target'] + + # variables exposed to messages + d = { 'stu_vars': stu_vars, + 'sol_vars': sol_vars, + 'num_vars': len(sol_vars)} + + if exact_names: + # message for wrong iter var names + _msg = state.build_message(incorrect_msg, d) + # test + rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, stu_target))) + else: + # message for wrong number of iter vars + _msg = state.build_message(incorrect_msg, d) + # test + rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, stu_target))) + + return state + + # functions for running multiple sub-tests ------------------------------------ def extend(*args, state=None): diff --git a/pythonwhat/test_funcs/test_comp.py b/pythonwhat/test_funcs/test_comp.py index e7345325..386b7e39 100644 --- a/pythonwhat/test_funcs/test_comp.py +++ b/pythonwhat/test_funcs/test_comp.py @@ -2,7 +2,7 @@ from pythonwhat.Feedback import Feedback from pythonwhat.Test import EqualTest from pythonwhat.utils import get_ord -from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len +from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len, has_iter_vars MSG_NOT_CALLED = "FMT:The system wants to check the {ordinal} {typestr} you defined but hasn't found it." MSG_PREPEND = "FMT:Check your code in the {child[part]} of the {ordinal} {typestr}. " @@ -86,7 +86,10 @@ def test_comp(typestr, comptype, index, iter_vars_names, # test comprehension iter and its variable names (or number of variables) if comp_iter: multi(comp_iter, state=check_part("iter", "iterable part", state)) - has_iter_vars(incorrect_iter_vars_msg, iter_vars_names, state=quiet_state) + + # test iterator variables + default_msg = MSG_INCORRECT_ITER_VARS if iter_vars_names else MSG_INCORRECT_NUM_ITER_VARS + has_iter_vars(incorrect_iter_vars_msg or default_msg, iter_vars_names, state=quiet_state) # test the main expressions. if body: multi(body, state=check_part("body", "body", state)) # list and gen comp @@ -100,30 +103,3 @@ def test_comp(typestr, comptype, index, iter_vars_names, # test individual ifs multi(if_test, state=check_part_index("ifs", i, get_ord(i+1) + " if", state=state)) - -def has_iter_vars(incorrect_iter_vars_msg, exact_names=False, state=None): - rep = Reporter.active_reporter - # get parts for testing from state - # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. - stu_vars = state.student_parts['_target_vars'] - sol_vars = state.solution_parts['_target_vars'] - stu_target = state.student_parts['target'] - - # variables exposed to messages - d = { 'stu_vars': stu_vars, - 'sol_vars': sol_vars, - 'num_vars': len(sol_vars)} - - if exact_names: - # message for wrong iter var names - _msg = state.build_message(incorrect_iter_vars_msg or MSG_INCORRECT_ITER_VARS, d) - # test - rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, stu_target))) - else: - # message for wrong number of iter vars - _msg = state.build_message(incorrect_iter_vars_msg or MSG_INCORRECT_NUM_ITER_VARS, d) - # test - rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, stu_target))) - - return state - From 7af83a88f8acd1c4abb3f33930e4c3eb9ef590c7 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 28 Feb 2017 15:41:48 -0500 Subject: [PATCH 006/418] update and add context tests for with, for, comps --- pythonwhat/check_funcs.py | 8 ++++---- pythonwhat/check_wrappers.py | 2 +- pythonwhat/parsing.py | 5 ++++- pythonwhat/test_funcs/test_comp.py | 4 ++-- tests/test_test_comp.py | 16 +++++++++++++++ tests/test_test_for_loop.py | 25 ++++++++++++++++++++++ tests/test_test_with.py | 33 ++++++++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 8 deletions(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index 427e940e..4813c2aa 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -138,13 +138,13 @@ def has_equal_part_len(name, insufficient_msg, state=None): return state -def has_iter_vars(incorrect_msg, exact_names=False, state=None): +def has_context(incorrect_msg="Incorrect iterator variables", exact_names=False, state=None): rep = Reporter.active_reporter # get parts for testing from state # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. - stu_vars = state.student_parts['_target_vars'] - sol_vars = state.solution_parts['_target_vars'] - stu_target = state.student_parts['target'] + stu_vars = state.student_parts.get('_target_vars') or state.student_parts['target_vars'] + sol_vars = state.solution_parts.get('_target_vars') or state.solution_parts['target_vars'] + stu_target = state.student_parts.get('target') # variables exposed to messages d = { 'stu_vars': stu_vars, diff --git a/pythonwhat/check_wrappers.py b/pythonwhat/check_wrappers.py index deae9ace..0a63971d 100644 --- a/pythonwhat/check_wrappers.py +++ b/pythonwhat/check_wrappers.py @@ -57,7 +57,7 @@ scts['check_function'] = check_function for k in ['set_context', - 'has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'call', + 'has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'has_context', 'call', 'extend', 'multi', 'test_not', 'fail', 'quiet', 'override', 'with_context', 'check_args', diff --git a/pythonwhat/parsing.py b/pythonwhat/parsing.py index 001f9264..fb6c5991 100644 --- a/pythonwhat/parsing.py +++ b/pythonwhat/parsing.py @@ -574,6 +574,7 @@ def visit_For(self, node): 'body': {'node': node.body, 'target_vars': tv}, 'orelse': {'node': node.orelse, 'target_vars': tv}, 'target': node.target, + '_target_vars': tv }) @@ -779,6 +780,7 @@ def visit_With(self, node): } for item in items] + tv_all = TargetVars(sum([list(c['target_vars'].items()) for c in context], [])) #body_tv = [] #for c in context: body_tv.extend(c['target_vars']) @@ -786,7 +788,8 @@ def visit_With(self, node): "context": context, "body": {'node': node.body, 'with_items': items}, "node": node, - "n_vars": len(items) + "n_vars": len(items), + "_target_vars": tv_all }) class TryExceptParser(Parser): diff --git a/pythonwhat/test_funcs/test_comp.py b/pythonwhat/test_funcs/test_comp.py index 386b7e39..9d9760de 100644 --- a/pythonwhat/test_funcs/test_comp.py +++ b/pythonwhat/test_funcs/test_comp.py @@ -2,7 +2,7 @@ from pythonwhat.Feedback import Feedback from pythonwhat.Test import EqualTest from pythonwhat.utils import get_ord -from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len, has_iter_vars +from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len, has_context MSG_NOT_CALLED = "FMT:The system wants to check the {ordinal} {typestr} you defined but hasn't found it." MSG_PREPEND = "FMT:Check your code in the {child[part]} of the {ordinal} {typestr}. " @@ -89,7 +89,7 @@ def test_comp(typestr, comptype, index, iter_vars_names, # test iterator variables default_msg = MSG_INCORRECT_ITER_VARS if iter_vars_names else MSG_INCORRECT_NUM_ITER_VARS - has_iter_vars(incorrect_iter_vars_msg or default_msg, iter_vars_names, state=quiet_state) + has_context(incorrect_iter_vars_msg or default_msg, iter_vars_names, state=quiet_state) # test the main expressions. if body: multi(body, state=check_part("body", "body", state)) # list and gen comp diff --git a/tests/test_test_comp.py b/tests/test_test_comp.py index 2be615d2..1dfc91f9 100644 --- a/tests/test_test_comp.py +++ b/tests/test_test_comp.py @@ -259,6 +259,22 @@ def test_pass(self): sct_payload = helper.run(self.data) self.assertTrue(sct_payload['correct']) + def test_fail_spec2(self): + self.data["DC_SCT"] = "Ex().check_list_comp(0).has_context()" + self.data["DC_CODE"] = "[a for a in x.items()]" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + + def test_pass_spec2(self): + self.data["DC_SCT"] = "Ex().check_list_comp(0).has_context()" + self.test_pass() + + def test_fail_spec2_exact_names(self): + self.data["DC_CODE"] = "[a for a,b in x.items()]" + self.data["DC_SCT"] = "Ex().check_list_comp(0).has_context(exact_names=True)" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + class TestListDestructuring(unittest.TestCase): def setUp(self): diff --git a/tests/test_test_for_loop.py b/tests/test_test_for_loop.py index ee60798c..4ffbb102 100644 --- a/tests/test_test_for_loop.py +++ b/tests/test_test_for_loop.py @@ -75,6 +75,31 @@ def test_Pass_exchain(self): self.data["DC_SCT"] = "Ex().\\" + helper.remove_lambdas(self.data["DC_SCT"]) self.test_Pass() + def test_has_context_pass(self): + self.data["DC_CODE"] = "for i in range(10): pass" + self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context()" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_has_context_mult_pass(self): + self.data["DC_SOLUTION"] = "for x,y in zip(range(10), range(10)): pass" + self.data["DC_CODE"] = "for i,j in zip(range(10), range(10)): pass" + self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context()" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_has_context_fail(self): + self.data["DC_CODE"] = "for i in range(10): pass" + self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context(exact_names=True)" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + + def test_has_context_pass(self): + self.data["DC_SOLUTION"] = "for x,y in zip(range(10), range(10)): pass" + self.data["DC_CODE"] = "for i,j in zip(range(10), range(10)): pass" + self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context(exact_names=True)" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) class TestForLoop2(unittest.TestCase): diff --git a/tests/test_test_with.py b/tests/test_test_with.py index 1b759376..1c0e6763 100644 --- a/tests/test_test_with.py +++ b/tests/test_test_with.py @@ -382,6 +382,39 @@ def test_pass(self): sct_payload = helper.run(self.data) self.assertTrue(sct_payload['correct']) +class TestHasContext(unittest.TestCase): + def setUp(self): + self.data = { + "DC_PEC": "from io import StringIO", + "DC_SOLUTION": "with StringIO() as f1, StringIO() as f2: pass", + "DC_CODE": "with StringIO() as f1, StringIO() as f2: pass", + "DC_SCT": "Ex().check_with(0).has_context()" + } + + def test_pass(self): + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_fail(self): + self.data["DC_CODE"] = "with StringIO() as f1: pass" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + + def test_fail_exact_names(self): + self.data["DC_CODE"] = "with StringIO() as f3, StringIO() as f4: pass" + self.data["DC_SCT"] = "Ex().check_with(0).has_context(exact_names=True)" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + + def test_context_pass(self): + self.data["DC_SCT"] = "Ex().check_with(0).check_context(0).has_context()" + self.test_pass() + + def test_context_fail(self): + self.data["DC_CODE"] = "with StringIO() as f3: pass" + self.data["DC_SCT"] = "Ex().check_with(0).check_context(0).has_context(exact_names=True)" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) if __name__ == "__main__": unittest.main() From ead450d3d07ae1a1d5f615b47cf5e4aaaca69e9d Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 28 Feb 2017 16:45:14 -0500 Subject: [PATCH 007/418] doc update --- docs/source/Home.md | 5 +++- docs/source/expression_tests.md | 4 +++ docs/source/part_checks.rst | 15 +++++++---- docs/source/simple_tests/has_equal_ast.md | 3 +-- pythonwhat/check_funcs.py | 33 +++++++++++++++++++++++ 5 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/source/Home.md b/docs/source/Home.md index c4bb782a..b48a9818 100644 --- a/docs/source/Home.md +++ b/docs/source/Home.md @@ -34,7 +34,10 @@ Overview To get started, make sure to check out the [Quickstart Guide](quickstart_guide.md). -To robustly test the equality of objects, and results of evaluations, it has to fetch the information from the respective processes, i.e. the student and solution processes. By default, this is done through a process of 'dilling' and 'undilling', but it's also possible to define your own converters to customize the way objects and results are compared. For more background on this, check out the [Processes article](expression_tests.md). For some more background on the principle of 'sub-SCTs', i.e. sets of tests to be called on a particular part or a particular state of a student's submission, have a look at the [Part Checks article](part_checks.rst). +```eval_rst + +To robustly test the equality of objects, and results of evaluations, it has to fetch the information from the respective processes, i.e. the student and solution processes. By default, this is done through a process of 'dilling' and 'undilling', but it's also possible to define your own converters to customize the way objects and results are compared. For more background on this, check out :ref:`managing-processes`. For some more background on the principle of 'sub-SCTs', i.e. sets of tests to be called on a particular part or a particular state of a student's submission, have a look at the :doc:`Part Checks article `. +``` The remainder of the wiki goes over every test function that `pythonwhat` features, explaining all arguments and covering different use cases. They will give you an idea of how, why and when to use them. diff --git a/docs/source/expression_tests.md b/docs/source/expression_tests.md index 0ded5b4f..4d92afc7 100644 --- a/docs/source/expression_tests.md +++ b/docs/source/expression_tests.md @@ -320,6 +320,10 @@ For example, the SCT below shows how to run some `pre_code`, and then evaluate t Ex().check_function_def('my_fun').call("f(1, 2)", test="output", pre_code="x = 1") ``` +```eval_rst +.. _managing-processes: +``` + Managing Processes ----------------- diff --git a/docs/source/part_checks.rst b/docs/source/part_checks.rst index 1d8d0896..e9026125 100644 --- a/docs/source/part_checks.rst +++ b/docs/source/part_checks.rst @@ -161,8 +161,7 @@ Helper Functions multi ~~~~~~~ -Runs multiple subtests. - +.. autofunction:: pythonwhat.check_funcs.multi Comma separated arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -214,6 +213,8 @@ followed by its iterator. set_context ~~~~~~~~~~~~~ +.. autofunction:: pythonwhat.check_funcs.set_context + Sets the value of a temporary variable, such as ``ii`` in the list comprehension below. .. code:: @@ -261,6 +262,8 @@ When you try to set context values that don't match any target variables in the with_context ~~~~~~~~~~~~~~ +.. autofunction:: pythonwhat.check_funcs.with_context + Runs subtests after setting the context for a ``with`` statement. This function takes arguments in the same form as ``multi``. @@ -291,6 +294,8 @@ and replacing step (3) with any sub-tests given as arguments. fail ~~~~~~ +.. autofunction:: pythonwhat.check_funcs.fail + Fails. This function takes a single argument, ``msg``, that is the feedback given to the student. Note that this would be a terrible idea for grading submissions, but may be useful while writing SCTs. For example, failing a test will highlight the code as if the previous test/check had failed. @@ -315,7 +320,6 @@ Check Functions (e.g. ``Ex().check_list_comp(0).check_ifs(0)``) * **missing_msg**: optional feedback message if node or part doesn't exist. - Note that code in all caps indicates the name of a piece of code that may be inspected using, ``check_{part}``, where ``{part}`` is replaced by the name in caps (e.g. ``check_if_else(0).check_test()``). Target variables are those that may be set using ``set_context``. @@ -323,6 +327,7 @@ These variables may only be set in places where python would set them. For example, this means that a list comprehension's ITER part has no target variables, but its BODY does. + +------------------------+------------------------------------------------------+-------------------+ | check | parts | target variables | +========================+======================================================+===================+ @@ -380,9 +385,9 @@ but its BODY does. | | FINALBODY | | | | | | +------------------------+------------------------------------------------------+-------------------+ -|check_with(0) | .. code:: python | `f`` | +|check_with(0) | .. code:: python | ``f`` | | | | | -| | with CONTEXT_TEST as f: | | +| | with CONTEXT[0] as f1, CONTEXT[1] as f2: | | | | BODY | | | | | | +------------------------+------------------------------------------------------+-------------------+ diff --git a/docs/source/simple_tests/has_equal_ast.md b/docs/source/simple_tests/has_equal_ast.md index 25cfda6b..6294eaf5 100644 --- a/docs/source/simple_tests/has_equal_ast.md +++ b/docs/source/simple_tests/has_equal_ast.md @@ -2,8 +2,7 @@ has_equal_ast -------------- ```eval_rst -.. automodule:: pythonwhat.check_funcs.has_equal_ast - :members: +.. autofunction:: pythonwhat.check_funcs.has_equal_ast ``` An abstract syntax tree (AST) is a way of representing the high-level structure of python code. diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index a8596c56..ed2977a1 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -402,6 +402,11 @@ def call(args, from pythonwhat import utils def has_equal_ast(incorrect_msg="FMT: Your code does not seem to match the solution.", state=None): + """Test whether abstract syntax trees match between the student and solution code. + + Args: + incorrect_msg: message displayed when ASTs mismatch. + """ rep = Reporter.active_reporter stu_rep = ast.dump(state.student_tree) @@ -424,6 +429,34 @@ def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eva highlight=None, state=None, test=None): + """Run student and solution code, compare returned value, printed output, or errors. + + Args: + incorrect_msg (str): feedback message if the output of the expression in the solution doesn't match + the one of the student. This feedback message will be expanded if it is used in the context of + another test function, like test_if_else. + error_msg (str): feedback message if there was an error when running the student code. + Note that when testing for an error, this message is displayed when none is raised. + undefined_msg (str): feedback message if the name argument is defined, but a variable + with that name doesn't exist after running the student code. + extra_env (dict): set variables to the extra environment. They will update the student + and solution environment in the active state before the student/solution code in the active + state is ran. This argument should contain a dictionary with the keys the names of + the variables you want to set, and the values are the values of these variables. + context_vals (list): set variables which are bound in a for loop to certain values. This argument is + only useful if you use the function in a test_for_loop. It contains a list with the values + of the bound variables. + expr_code (str): if this variable is not None, the expression in the student/solution code will not + be ran. Instead, the given piece of code will be ran in the student as well as the solution environment + and the result will be compared. + pre_code (str): the code in string form that should be executed before the expression is executed. + This is the ideal place to set a random seed, for example. + keep_obj_in_env (list()): a list of variable names that should be hold in the copied environment where + the expression is evaluated. All primitive types are copied automatically, other objects have to + be passed explicitely. + name (str): the name of a variable, or expression, whose value will be tested after running the + student and solution code. This could be thought of as post code. + """ rep = Reporter.active_reporter # run function to highlight a block of code From 289eb35a2c50a5cbeb100566da2d7befbebafde4 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Thu, 2 Mar 2017 11:40:40 -0500 Subject: [PATCH 008/418] make has_context use single dispatch --- pythonwhat/check_funcs.py | 28 ----------- pythonwhat/check_has_context.py | 74 ++++++++++++++++++++++++++++++ pythonwhat/check_wrappers.py | 6 +-- pythonwhat/parsing.py | 1 - pythonwhat/test_funcs/test_comp.py | 4 +- pythonwhat/test_funcs/test_with.py | 10 ++-- 6 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 pythonwhat/check_has_context.py diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index 7194ef2e..afde34b2 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -137,34 +137,6 @@ def has_equal_part_len(name, insufficient_msg, state=None): return state - -def has_context(incorrect_msg="Incorrect iterator variables", exact_names=False, state=None): - rep = Reporter.active_reporter - # get parts for testing from state - # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. - stu_vars = state.student_parts.get('_target_vars') or state.student_parts['target_vars'] - sol_vars = state.solution_parts.get('_target_vars') or state.solution_parts['target_vars'] - stu_target = state.student_parts.get('target') - - # variables exposed to messages - d = { 'stu_vars': stu_vars, - 'sol_vars': sol_vars, - 'num_vars': len(sol_vars)} - - if exact_names: - # message for wrong iter var names - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, stu_target))) - else: - # message for wrong number of iter vars - _msg = state.build_message(incorrect_msg, d) - # test - rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, stu_target))) - - return state - - # functions for running multiple sub-tests ------------------------------------ def extend(*args, state=None): diff --git a/pythonwhat/check_has_context.py b/pythonwhat/check_has_context.py new file mode 100644 index 00000000..6dad8136 --- /dev/null +++ b/pythonwhat/check_has_context.py @@ -0,0 +1,74 @@ +from pythonwhat.Reporter import Reporter +from pythonwhat.Test import Test, EqualTest +from pythonwhat.Feedback import Feedback +from pythonwhat.State import State +from functools import singledispatch +from pythonwhat.check_funcs import check_part_index + +def has_context(incorrect_msg="Incorrect iterator variables", exact_names=False, state=None): + # call _has_context, since the built-in singledispatch can only use 1st pos arg + return _has_context(state, incorrect_msg, exact_names) + +def _test(state, incorrect_msg, exact_names, tv_name, highlight_name): + rep = Reporter.active_reporter + # get parts for testing from state + # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. + stu_vars = state.student_parts[tv_name] + sol_vars = state.solution_parts[tv_name] + stu_target = state.student_parts.get(highlight_name) # TODO should be node? + + # variables exposed to messages + d = { 'stu_vars': stu_vars, + 'sol_vars': sol_vars, + 'num_vars': len(sol_vars)} + + if exact_names: + # message for wrong iter var names + _msg = state.build_message(incorrect_msg, d) + # test + rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, stu_target))) + else: + # message for wrong number of iter vars + _msg = state.build_message(incorrect_msg, d) + # test + rep.do_test(EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, stu_target))) + + return state + + +@singledispatch +def _has_context(state, incorrect_msg, exact_names): + raise BaseException("first argument to _has_context must be a State instance or subclass") + +@_has_context.register(State) +def has_context_state(*args, **kwargs): + return _test(*args, tv_name='target_vars', highlight_name='highlight', **kwargs) + +@_has_context.register(State.SUBCLASSES['for_loops']) +@_has_context.register(State.SUBCLASSES['whiles']) +@_has_context.register(State.SUBCLASSES['dict_comps']) +@_has_context.register(State.SUBCLASSES['generator_exps']) +@_has_context.register(State.SUBCLASSES['list_comps']) +def has_context_loop(*args, **kwargs): + """When dispatched on loops, has_context the target vars are the attribute _target_vars. + + Note: This is to allow people to call has_context on a node (e.g. for_loop) rather than + one of its attributes (e.g. body). Purely for convenience. + """ + return _test(*args, tv_name='_target_vars', highlight_name='target', **kwargs) + +@_has_context.register(State.SUBCLASSES['withs']) +def has_context_with(state, incorrect_msg, exact_names): + """When dispatched on with statements, has_context loops over each context manager. + + Note: This is to allow people to call has_context on the with statement, rather than + having to manually loop over each context manager. + + e.g. Ex().check_with(0).has_context() vs Ex().check_with(0).check_context(0).has_context() + """ + + for i in range(len(state.solution_parts['context'])): + ctxt_state = check_part_index('context', i, "", state=state) + _has_context(ctxt_state, incorrect_msg, exact_names) + + return state diff --git a/pythonwhat/check_wrappers.py b/pythonwhat/check_wrappers.py index 0a63971d..33c59cf3 100644 --- a/pythonwhat/check_wrappers.py +++ b/pythonwhat/check_wrappers.py @@ -1,6 +1,7 @@ from pythonwhat.check_funcs import check_part, check_part_index, check_node, has_equal_part from pythonwhat import check_funcs, check_object from pythonwhat.check_function import check_function +from pythonwhat.check_has_context import has_context from pythonwhat.test_funcs.test_data_frame import check_df from pythonwhat.test_funcs.test_dictionary import check_dict from pythonwhat import test_funcs @@ -57,7 +58,7 @@ scts['check_function'] = check_function for k in ['set_context', - 'has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'has_context', 'call', + 'has_equal_value', 'has_equal_output', 'has_equal_error', 'has_equal_ast', 'call', 'extend', 'multi', 'test_not', 'fail', 'quiet', 'override', 'with_context', 'check_args', @@ -70,5 +71,4 @@ scts['check_df'] = check_df scts['check_dict'] = check_dict - - +scts['has_context'] = has_context diff --git a/pythonwhat/parsing.py b/pythonwhat/parsing.py index fb6c5991..92859f5d 100644 --- a/pythonwhat/parsing.py +++ b/pythonwhat/parsing.py @@ -789,7 +789,6 @@ def visit_With(self, node): "body": {'node': node.body, 'with_items': items}, "node": node, "n_vars": len(items), - "_target_vars": tv_all }) class TryExceptParser(Parser): diff --git a/pythonwhat/test_funcs/test_comp.py b/pythonwhat/test_funcs/test_comp.py index 9d9760de..59d43a09 100644 --- a/pythonwhat/test_funcs/test_comp.py +++ b/pythonwhat/test_funcs/test_comp.py @@ -2,7 +2,9 @@ from pythonwhat.Feedback import Feedback from pythonwhat.Test import EqualTest from pythonwhat.utils import get_ord -from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len, has_context +from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, has_equal_part_len +from pythonwhat.check_has_context import has_context + MSG_NOT_CALLED = "FMT:The system wants to check the {ordinal} {typestr} you defined but hasn't found it." MSG_PREPEND = "FMT:Check your code in the {child[part]} of the {ordinal} {typestr}. " diff --git a/pythonwhat/test_funcs/test_with.py b/pythonwhat/test_funcs/test_with.py index 00f380e3..f0654698 100644 --- a/pythonwhat/test_funcs/test_with.py +++ b/pythonwhat/test_funcs/test_with.py @@ -4,6 +4,7 @@ from pythonwhat import utils from pythonwhat.tasks import setUpNewEnvInProcess, breakDownNewEnvInProcess from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, quiet, has_equal_part, with_context +from pythonwhat.check_has_context import has_context from functools import partial @@ -12,7 +13,7 @@ MSG_PREPEND2 = "FMT:Check the {child[part]} of the {ordinal} `with` statement. " MSG_NUM_CTXT = "make sure to use the correct number of context variables. It seems you defined too many." MSG_NUM_CTXT2 = "make sure to use the correct number of context variables. It seems you defined too little." -MSG_CTXT_NAMES = "FMT:make sure to use the correct context variable names. Was expecting `{sol_part[target_vars]}` but got `{stu_part[target_vars]}`." +MSG_CTXT_NAMES = "FMT:make sure to use the correct context variable names. Was expecting `{sol_vars}` but got `{stu_vars}`." def test_with(index, @@ -51,9 +52,10 @@ def test_with(index, rep.do_test(Test(Feedback(_msg, child.student_tree))) # test context var names ---- - for i in range(len(child.solution_parts['context'])): - ctxt_state = check_part_index('context', i, "", state=child) - has_equal_part('target_vars', MSG_CTXT_NAMES, state=ctxt_state) + has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES, exact_names = True, state=child) + #for i in range(len(child.solution_parts['context'])): + # ctxt_state = check_part_index('context', i, "", state=child) + # has_equal_part('target_vars', MSG_CTXT_NAMES, state=ctxt_state) # Context sub tests ---- From 2306bc53f7401545ffe5a11e84b364e884bfd2b8 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Thu, 2 Mar 2017 14:56:30 -0500 Subject: [PATCH 009/418] add proper messages to has_context --- pythonwhat/check_has_context.py | 12 ++++++++---- pythonwhat/test_funcs/test_with.py | 4 ---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pythonwhat/check_has_context.py b/pythonwhat/check_has_context.py index 6dad8136..1a6b2a08 100644 --- a/pythonwhat/check_has_context.py +++ b/pythonwhat/check_has_context.py @@ -5,7 +5,10 @@ from functools import singledispatch from pythonwhat.check_funcs import check_part_index -def has_context(incorrect_msg="Incorrect iterator variables", exact_names=False, state=None): +MSG_INCORRECT_LOOP = "FMT:Have you used the correct iterator variable names? Was expecting `{sol_vars}` but got `{stu_vars}`." +MSG_INCORRECT_WITH = "FMT:Make sure to use the correct context variable names. Was expecting `{sol_vars}` but got `{stu_vars}`." + +def has_context(incorrect_msg=None, exact_names=False, state=None): # call _has_context, since the built-in singledispatch can only use 1st pos arg return _has_context(state, incorrect_msg, exact_names) @@ -49,13 +52,14 @@ def has_context_state(*args, **kwargs): @_has_context.register(State.SUBCLASSES['dict_comps']) @_has_context.register(State.SUBCLASSES['generator_exps']) @_has_context.register(State.SUBCLASSES['list_comps']) -def has_context_loop(*args, **kwargs): +def has_context_loop(state, incorrect_msg, exact_names): """When dispatched on loops, has_context the target vars are the attribute _target_vars. Note: This is to allow people to call has_context on a node (e.g. for_loop) rather than one of its attributes (e.g. body). Purely for convenience. """ - return _test(*args, tv_name='_target_vars', highlight_name='target', **kwargs) + return _test(state, incorrect_msg or MSG_INCORRECT_LOOP, exact_names, + tv_name='_target_vars', highlight_name='target') @_has_context.register(State.SUBCLASSES['withs']) def has_context_with(state, incorrect_msg, exact_names): @@ -69,6 +73,6 @@ def has_context_with(state, incorrect_msg, exact_names): for i in range(len(state.solution_parts['context'])): ctxt_state = check_part_index('context', i, "", state=state) - _has_context(ctxt_state, incorrect_msg, exact_names) + _has_context(ctxt_state, incorrect_msg or MSG_INCORRECT_WITH, exact_names) return state diff --git a/pythonwhat/test_funcs/test_with.py b/pythonwhat/test_funcs/test_with.py index f0654698..dfcc4130 100644 --- a/pythonwhat/test_funcs/test_with.py +++ b/pythonwhat/test_funcs/test_with.py @@ -53,10 +53,6 @@ def test_with(index, # test context var names ---- has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES, exact_names = True, state=child) - #for i in range(len(child.solution_parts['context'])): - # ctxt_state = check_part_index('context', i, "", state=child) - # has_equal_part('target_vars', MSG_CTXT_NAMES, state=ctxt_state) - # Context sub tests ---- if context_tests and not isinstance(context_tests, list): context_tests = [context_tests] From 4d017cf5c942769cdc9418d5d5dcfa892ac898e4 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Thu, 2 Mar 2017 14:59:21 -0500 Subject: [PATCH 010/418] add travis badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 09daaba3..65e7be00 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ![banner](https://s3.amazonaws.com/assets.datacamp.com/img/github/content-engineering-repos/pythonwhat_banner_v2.png) +[![Build Status](https://travis-ci.org/datacamp/pythonwhat.svg?branch=master)](https://travis-ci.org/datacamp/pythonwhat) + The `pythonwhat` package provides rich functionality to write Submission Correctness Tests for interactive Python exercises on the DataCamp platform. DataCamp operates with **Python 3**. For a detailed guide on how to use `pythonwhat`, head over to [the online documentation](http://pythonwhat.readthedocs.io). Before, all documentation was on the [wiki](https://github.com/datacamp/pythonwhat/wiki), but things are steadily being moved to the _readthedocs_ format. From 9433d83221d5cb8acefcac83fbdb1c4032231a23 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Thu, 2 Mar 2017 16:32:16 -0500 Subject: [PATCH 011/418] remove exclamations from negative feedback --- pythonwhat/test_funcs/test_comp.py | 2 +- pythonwhat/test_funcs/test_function_definition.py | 2 +- pythonwhat/test_funcs/test_lambda_function.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonwhat/test_funcs/test_comp.py b/pythonwhat/test_funcs/test_comp.py index e7345325..4de8f528 100644 --- a/pythonwhat/test_funcs/test_comp.py +++ b/pythonwhat/test_funcs/test_comp.py @@ -7,7 +7,7 @@ MSG_NOT_CALLED = "FMT:The system wants to check the {ordinal} {typestr} you defined but hasn't found it." MSG_PREPEND = "FMT:Check your code in the {child[part]} of the {ordinal} {typestr}. " -MSG_INCORRECT_ITER_VARS = "FMT:Have you used the correct iterator variables in the {parent[ordinal]} {parent[typestr]}? Make sure you use the correct names!" +MSG_INCORRECT_ITER_VARS = "FMT:Have you used the correct iterator variables in the {parent[ordinal]} {parent[typestr]}? Be sure to use the correct names." MSG_INCORRECT_NUM_ITER_VARS = "FMT:Have you used {num_vars} iterator variables in the {parent[ordinal]} {parent[typestr]}?" MSG_INSUFFICIENT_IFS = "FMT:Have you used {sol_len} ifs inside the {parent[ordinal]} {parent[typestr]}?" diff --git a/pythonwhat/test_funcs/test_function_definition.py b/pythonwhat/test_funcs/test_function_definition.py index 15aa6fd5..0feab9c7 100644 --- a/pythonwhat/test_funcs/test_function_definition.py +++ b/pythonwhat/test_funcs/test_function_definition.py @@ -21,7 +21,7 @@ # TODO some need to reference the eval rather than str MSG_RES_ERROR = "FMT:Calling `{argstr}` should result in `{str_sol}`, instead got an error." MSG_RES_INCORRECT = "FMT:Calling `{argstr}` should result in `{str_sol}`, instead got `{str_stu}`." -MSG_ERR_NONE = "FMT:Calling `{argstr}` doesn't result in an error, but it should!" +MSG_ERR_NONE = "FMT:Calling `{argstr}` doesn't result in an error, but it should." MSG_ERR_INCORRECT = "FMT:Calling `{argstr}` should result in a `{str_sol.__class__.__name__}`, instead got a `{str_stu.__class__.__name__}`." MSG_OUT_ERROR = "FMT:Calling `{argstr}` should output {str_sol}, instead got an error." MSG_OUT_INCORRECT = "FMT:Calling `{argstr}` should output `{str_sol}`, instead got {str_stu}." diff --git a/pythonwhat/test_funcs/test_lambda_function.py b/pythonwhat/test_funcs/test_lambda_function.py index 96189d78..fff59fe3 100644 --- a/pythonwhat/test_funcs/test_lambda_function.py +++ b/pythonwhat/test_funcs/test_lambda_function.py @@ -10,7 +10,7 @@ MSG_RES_ERROR = "FMT:Calling it with arguments `{argstr}` should result in `{str_sol}`, instead got an error." MSG_RES_INCORRECT = "FMT:Calling it with arguments `{argstr}` should result in `{str_sol}`, instead got `{str_stu}`." -MSG_ERR_WRONG = "FMT:Calling it with arguments `{argstr}` doesn't result in an error, but it should!" +MSG_ERR_WRONG = "FMT:Calling it with arguments `{argstr}` doesn't result in an error, but it should." def test_lambda_function(index, arg_names=True, arg_defaults=True, From aa3a91150c3934af6f393d2c9107a42d443386af Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Mon, 6 Mar 2017 11:05:15 -0500 Subject: [PATCH 012/418] add marker for printing feedback, fix has_context messaging --- .travis.yml | 3 ++- pythonwhat/check_funcs.py | 16 +++++++++------- pythonwhat/check_has_context.py | 2 +- tests/helper.py | 6 +++++- tests/test_test_for_loop.py | 5 ++++- tests/test_test_with.py | 2 ++ 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3692920f..41b165ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,5 @@ before_install: - "git clone https://$GH_TOKEN@github.com/datacamp/pythonbackend.git && pip install ./pythonbackend && rm -rf pythonbackend" install: pip install -r requirements.txt -script: pytest tests -m "not dep_matplotlib" -s +script: pytest tests -s -m "not dep_matplotlib" +after_script: "export PYTHONWHAT_DEBUG_FEEDBACK='True' && pytest tests -s -m 'feedback'" diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index afde34b2..c4c844b3 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -52,11 +52,12 @@ def check_part_index(name, index, part_msg, 'kwargs': fmt_kwargs} # check there are enough parts for index - stu_parts = state.student_parts[name] - try: stu_parts[index] - except (KeyError, IndexError): - _msg = state.build_message(missing_msg, append_message['kwargs']) - rep.do_test(Test(Feedback(_msg, state.highlight))) + has_part(name, missing_msg, state, append_message['kwargs'], index) + #stu_parts = state.student_parts[name] + #try: stu_parts[index] + #except (KeyError, IndexError): + # _msg = state.build_message(missing_msg, append_message['kwargs']) + # rep.do_test(Test(Feedback(_msg, state.highlight))) # get part at index stu_part = state.student_parts[name][index] @@ -66,7 +67,7 @@ def check_part_index(name, index, part_msg, return part_to_child(stu_part, sol_part, append_message, state) MSG_MISSING = "FMT:The system wants to check the {typestr} you defined but hasn't found it." -MSG_PREPEND = "__JINJA__:Check your code in the {{child['part']+ ' of the' if child['part']}} {{typestr}}. " +MSG_PREPEND = "__JINJA__:Check your code in the{{' ' + child['part']+ ' of the' if child['part']}} {{typestr}}. " def check_node(name, index, typestr, missing_msg=MSG_MISSING, expand_msg=MSG_PREPEND, state=None): rep = Reporter.active_reporter stu_out = getattr(state, 'student_'+name) @@ -97,7 +98,7 @@ def check_node(name, index, typestr, missing_msg=MSG_MISSING, expand_msg=MSG_PRE # Part tests ------------------------------------------------------------------ -def has_part(name, msg, state=None, fmt_kwargs=None): +def has_part(name, msg, state=None, fmt_kwargs=None, index=None): rep = Reporter.active_reporter d = {'sol_part': state.solution_parts, 'stu_part': state.student_parts, @@ -106,6 +107,7 @@ def has_part(name, msg, state=None, fmt_kwargs=None): try: part = state.student_parts[name] + if index is not None: part = part[index] if part is None: raise KeyError except (KeyError, IndexError): _msg = state.build_message(msg, d) diff --git a/pythonwhat/check_has_context.py b/pythonwhat/check_has_context.py index 1a6b2a08..0eb79863 100644 --- a/pythonwhat/check_has_context.py +++ b/pythonwhat/check_has_context.py @@ -72,7 +72,7 @@ def has_context_with(state, incorrect_msg, exact_names): """ for i in range(len(state.solution_parts['context'])): - ctxt_state = check_part_index('context', i, "", state=state) + ctxt_state = check_part_index('context', i, '{ordinal} context', state=state) _has_context(ctxt_state, incorrect_msg or MSG_INCORRECT_WITH, exact_names) return state diff --git a/tests/helper.py b/tests/helper.py index 4c3e6d7b..7637ff2b 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -1,6 +1,7 @@ from pythonbackend.Exercise import Exercise from pythonbackend import utils import re +import os def get_sct_payload(output): sct_output = [out for out in output if out['type'] == 'sct'] @@ -17,7 +18,10 @@ def run(data): print(output) raise(ValueError("Backend error")) output = exercise.runSubmit(data) - return(get_sct_payload(output)) + sct_payload = get_sct_payload(output) + if os.environ.get('PYTHONWHAT_DEBUG_FEEDBACK'): + print('message: %s'%sct_payload.get('message')) + return(sct_payload) def test_lines(test, sct_payload, ls, le, cs, ce): test.assertEqual(sct_payload['line_start'], ls) diff --git a/tests/test_test_for_loop.py b/tests/test_test_for_loop.py index 4ffbb102..4b3d95ef 100644 --- a/tests/test_test_for_loop.py +++ b/tests/test_test_for_loop.py @@ -1,5 +1,6 @@ import unittest import helper +import pytest class TestForLoop(unittest.TestCase): @@ -88,13 +89,15 @@ def test_has_context_mult_pass(self): sct_payload = helper.run(self.data) self.assertTrue(sct_payload['correct']) + @pytest.mark.feedback def test_has_context_fail(self): self.data["DC_CODE"] = "for i in range(10): pass" self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context(exact_names=True)" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - def test_has_context_pass(self): + @pytest.mark.feedback + def test_has_context_mult_fail(self): self.data["DC_SOLUTION"] = "for x,y in zip(range(10), range(10)): pass" self.data["DC_CODE"] = "for i,j in zip(range(10), range(10)): pass" self.data["DC_SCT"] = "Ex().check_for_loop(0).has_context(exact_names=True)" diff --git a/tests/test_test_with.py b/tests/test_test_with.py index ac11e271..b519ada0 100644 --- a/tests/test_test_with.py +++ b/tests/test_test_with.py @@ -397,11 +397,13 @@ def test_pass(self): sct_payload = helper.run(self.data) self.assertTrue(sct_payload['correct']) + @pytest.mark.feedback def test_fail(self): self.data["DC_CODE"] = "with StringIO() as f1: pass" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) + @pytest.mark.feedback def test_fail_exact_names(self): self.data["DC_CODE"] = "with StringIO() as f3, StringIO() as f4: pass" self.data["DC_SCT"] = "Ex().check_with(0).has_context(exact_names=True)" From 37081d2d1e2f1d5504913000ac681527998444a2 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Mon, 6 Mar 2017 11:15:29 -0500 Subject: [PATCH 013/418] minor cleanup --- pythonwhat/check_funcs.py | 5 ----- pythonwhat/parsing.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index c4c844b3..6823f2c2 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -53,11 +53,6 @@ def check_part_index(name, index, part_msg, # check there are enough parts for index has_part(name, missing_msg, state, append_message['kwargs'], index) - #stu_parts = state.student_parts[name] - #try: stu_parts[index] - #except (KeyError, IndexError): - # _msg = state.build_message(missing_msg, append_message['kwargs']) - # rep.do_test(Test(Feedback(_msg, state.highlight))) # get part at index stu_part = state.student_parts[name][index] diff --git a/pythonwhat/parsing.py b/pythonwhat/parsing.py index 92859f5d..f46c8c25 100644 --- a/pythonwhat/parsing.py +++ b/pythonwhat/parsing.py @@ -788,7 +788,7 @@ def visit_With(self, node): "context": context, "body": {'node': node.body, 'with_items': items}, "node": node, - "n_vars": len(items), + "n_vars": len(items) }) class TryExceptParser(Parser): From c2aa35c914523307cf1b6ec83ea9da6f6d159b5c Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 7 Mar 2017 09:18:23 -0500 Subject: [PATCH 014/418] update unit tests with exclamations --- tests/test_test_comp.py | 6 +++--- tests/test_test_function_definition.py | 2 +- tests/test_test_lambda_function.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_test_comp.py b/tests/test_test_comp.py index 2be615d2..c997d177 100644 --- a/tests/test_test_comp.py +++ b/tests/test_test_comp.py @@ -39,7 +39,7 @@ def test_fail_3(self): self.data["DC_CODE"] = "[a + str(b) for a,b in x.items()]" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - self.assertEqual(sct_payload['message'], "Have you used the correct iterator variables in the first list comprehension? Make sure you use the correct names!") + self.assertEqual(sct_payload['message'], "Have you used the correct iterator variables in the first list comprehension? Be sure to use the correct names.") helper.test_lines(self, sct_payload, 1, 1, 17, 19) def test_fail_4(self): @@ -334,7 +334,7 @@ def test_fail_3(self): self.data["DC_CODE"] = "{ a:a for a in lst }" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - self.assertEqual("Have you used the correct iterator variables in the first dictionary comprehension? Make sure you use the correct names!", sct_payload['message']) + self.assertEqual("Have you used the correct iterator variables in the first dictionary comprehension? Be sure to use the correct names.", sct_payload['message']) helper.test_lines(self, sct_payload, 1, 1, 11, 11) def test_fail_4(self): @@ -419,7 +419,7 @@ def test_fail_3(self): self.data["DC_CODE"] = "(a + str(b) for a,b in x.items())" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - self.assertEqual(sct_payload['message'], "Have you used the correct iterator variables in the first generator expression? Make sure you use the correct names!") + self.assertEqual(sct_payload['message'], "Have you used the correct iterator variables in the first generator expression? Be sure to use the correct names.") helper.test_lines(self, sct_payload, 1, 1, 17, 19) def test_fail_4(self): diff --git a/tests/test_test_function_definition.py b/tests/test_test_function_definition.py index 4dd46378..132a29b3 100644 --- a/tests/test_test_function_definition.py +++ b/tests/test_test_function_definition.py @@ -464,7 +464,7 @@ def inc(num): ''' sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - self.assertEqual("Calling inc(-1) doesn't result in an error, but it should!", sct_payload['message']) + self.assertEqual("Calling inc(-1) doesn't result in an error, but it should.", sct_payload['message']) def test_fail_2(self): self.data["DC_CODE"] = ''' diff --git a/tests/test_test_lambda_function.py b/tests/test_test_lambda_function.py index 5fcbdfd7..05238364 100644 --- a/tests/test_test_lambda_function.py +++ b/tests/test_test_lambda_function.py @@ -68,7 +68,7 @@ def test_fail_8(self): self.data["DC_CODE"] = "echo_word = lambda word, echo = 1: word * int(echo)" sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) - self.assertEqual(sct_payload['message'], "Check your definition of the first lambda function. Calling it with arguments ('a', '2') doesn't result in an error, but it should!") + self.assertEqual(sct_payload['message'], "Check your definition of the first lambda function. Calling it with arguments ('a', '2') doesn't result in an error, but it should.") helper.test_lines(self, sct_payload, 1, 1, 13, 51) def test_pass(self): From 25ebe564cf65f464da1b80e55ca71e4329a77425 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 7 Mar 2017 09:42:02 -0500 Subject: [PATCH 015/418] test_with cleaned up to use has_equal_part_len --- pythonwhat/test_funcs/test_with.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pythonwhat/test_funcs/test_with.py b/pythonwhat/test_funcs/test_with.py index dfcc4130..4dcbb1d5 100644 --- a/pythonwhat/test_funcs/test_with.py +++ b/pythonwhat/test_funcs/test_with.py @@ -3,7 +3,7 @@ from pythonwhat.Test import EqualTest, Test from pythonwhat import utils from pythonwhat.tasks import setUpNewEnvInProcess, breakDownNewEnvInProcess -from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, quiet, has_equal_part, with_context +from pythonwhat.check_funcs import check_node, check_part, check_part_index, multi, quiet, has_equal_part, with_context, has_equal_part_len from pythonwhat.check_has_context import has_context from functools import partial @@ -45,14 +45,12 @@ def test_with(index, quiet_child = quiet(1, child) if context_vals: - # test num context vars ---- - too_many = len(child.student_parts['context']) > len(child.solution_parts['context']) - if too_many: - _msg = child.build_message(MSG_NUM_CTXT) - rep.do_test(Test(Feedback(_msg, child.student_tree))) - # test context var names ---- has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES, exact_names = True, state=child) + + # test num context vars ---- + has_equal_part_len('context', MSG_NUM_CTXT, state=child) + # Context sub tests ---- if context_tests and not isinstance(context_tests, list): context_tests = [context_tests] From 79061eef08812848c551918fc61a4b0d9258ab2f Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 7 Mar 2017 16:15:21 -0500 Subject: [PATCH 016/418] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0b808aad..74371ee6 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='pythonwhat', - version='2.3.0', + version='2.4.0', packages=['pythonwhat', 'pythonwhat.test_funcs'], install_requires=["dill", "IPython", "numpy", "pandas", "markdown2", "jinja2"] ) From 504b717d93989b0f931313ba0c55435bfc09301b Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 7 Mar 2017 16:22:16 -0500 Subject: [PATCH 017/418] cleanup reporter --- .travis.yml | 2 +- pythonwhat/Reporter.py | 78 ++++---------------------------- pythonwhat/test_exercise.py | 5 +- pythonwhat/test_funcs/test_mc.py | 2 +- 4 files changed, 15 insertions(+), 72 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41b165ad..e1c57d9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,5 +8,5 @@ before_install: - "git clone https://$GH_TOKEN@github.com/datacamp/pythonbackend.git && pip install ./pythonbackend && rm -rf pythonbackend" install: pip install -r requirements.txt -script: pytest tests -s -m "not dep_matplotlib" +script: pytest after_script: "export PYTHONWHAT_DEBUG_FEEDBACK='True' && pytest tests -s -m 'feedback'" diff --git a/pythonwhat/Reporter.py b/pythonwhat/Reporter.py index 9c075200..9f96bf17 100644 --- a/pythonwhat/Reporter.py +++ b/pythonwhat/Reporter.py @@ -20,24 +20,10 @@ def __init__(self): self.feedback = Feedback("Oh no, your solution is incorrect! Please, try again.") self.success_msg = "Great work!" self.errors_allowed = False - self.tags = {} self.failure_msg = "" self.fallback_ast = None - self.test_stack = [] - self.test_mode = None - def set_success_msg(self, success_msg): - self.success_msg = success_msg - - def allow_errors(self): - self.errors_allowed = True - - def reject_errors(self): - self.errors_allowed = False - - def fail(self, failure_msg): - self.failed_test = True - self.feedback = Feedback(failure_msg) + def set_tag(self, *args, **kwargs): pass def do_test(self, testobj, prepend_on_fail="", fallback_ast=None): """Do test. @@ -46,16 +32,9 @@ def do_test(self, testobj, prepend_on_fail="", fallback_ast=None): the state of the reporter changes and the feedback is kept. """ - if self.test_mode is 'or': - return self.test_stack.append([testobj, prepend_on_fail, fallback_ast]) - if prepend_on_fail: self.failure_msg = prepend_on_fail if fallback_ast: self.fallback_ast = fallback_ast - if self.failed_test: - self.feedback.message = self.failure_msg + self.feedback.message - raise TestFail - return if isinstance(testobj, Test): testobj.test() result = testobj.result @@ -74,57 +53,20 @@ def do_test(self, testobj, prepend_on_fail="", fallback_ast=None): #self.failure_msg_stack.pop() return result - def do_tests(self, testobjs): - """Do multiple tests. - - Execute an array of tests. - """ - for testobj in testobjs: - if self.failed_test: - break - - self.do_test(testobj) - - def start_or_test(self): - self.test_mode = 'or' - self.test_stack = [] - - def end_or_test(self): - self.test_mode = None - first_message = None - success = False - for sct_args in self.test_stack: - try: - self.do_test(*sct_args) - success = True - except TestFail as e: - if not first_message: first_message = self.feedback.message - self.failed_test = False - - if success: return - - self.failed_test = True - self.feedback.message = first_message - raise TestFail - - def set_tag(self, key, value): - self.tags[key] = value - - def build_payload(self, error): if (error and not self.failed_test and not self.errors_allowed): feedback_msg = "Your code contains an error: `%s`" % str(error[1]) - return({ + return { "correct": False, - "message": Reporter.to_html(feedback_msg), - "tags": {"fun": "runtime_error"}}) + "message": Reporter.to_html(feedback_msg) + } if self.failed_test: if not self.feedback.line_info: - return({ + return { "correct": False, - "message": Reporter.to_html(self.feedback.message), - "tags": self.tags}) + "message": Reporter.to_html(self.feedback.message) + } else: # Hack to make it work with campus app implementation if self.feedback.line_info["column_start"] is None: @@ -132,14 +74,14 @@ def build_payload(self, error): else: col_start = self.feedback.line_info["column_start"] + 1 - return({ + return { "correct": False, "message": Reporter.to_html(self.feedback.message), "line_start": self.feedback.line_info["line_start"], "column_start": col_start, "line_end": self.feedback.line_info["line_end"], - "column_end": self.feedback.line_info["column_end"], - "tags": self.tags}) + "column_end": self.feedback.line_info["column_end"] + } else: diff --git a/pythonwhat/test_exercise.py b/pythonwhat/test_exercise.py index 88750d7e..a5411d7e 100644 --- a/pythonwhat/test_exercise.py +++ b/pythonwhat/test_exercise.py @@ -65,6 +65,7 @@ def test_exercise(sct, sct_cntxt.update(spec_2_context) try: + # if statement detects, for example, if State hit a parsing error w/student code if not rep.failed_test: exec(sct, sct_cntxt) # Spec v2 tests run immediately for test in tree.crnt_node: # Spec v1 tests run after @@ -81,11 +82,11 @@ def success_msg(message): message (str): A string containing the feedback message. """ rep = Reporter.active_reporter - rep.set_success_msg(message) + rep.success_msg = message def allow_errors(): rep = Reporter.active_reporter - rep.allow_errors() + rep.errors_allowed = True cntxt['success_msg'] = success_msg diff --git a/pythonwhat/test_funcs/test_mc.py b/pythonwhat/test_funcs/test_mc.py index 45f29e63..32da03f8 100644 --- a/pythonwhat/test_funcs/test_mc.py +++ b/pythonwhat/test_funcs/test_mc.py @@ -40,6 +40,6 @@ def test_mc(correct, msgs, state=None): feedback_msg = msgs[selected_option - 1] - rep.set_success_msg(msgs[correct - 1]) + rep.success_msg = msgs[correct - 1] rep.do_test(EqualTest(selected_option, correct, feedback_msg)) From d5ab1970b039ec152de194e1b03b22239fa136c5 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 7 Mar 2017 16:34:24 -0500 Subject: [PATCH 018/418] add pytest.ini --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..e31e731d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests/ +addopts =-s -m "not dep_matplotlib" From ac483247ec4d064f3109da681d6c93c2d84252bb Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Fri, 17 Mar 2017 10:02:42 -0400 Subject: [PATCH 019/418] remove tag check from failing tests --- tests/test_test_exercise.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_test_exercise.py b/tests/test_test_exercise.py index b411b7d3..5f68aa45 100644 --- a/tests/test_test_exercise.py +++ b/tests/test_test_exercise.py @@ -44,7 +44,6 @@ def test_syntax_error(self): self.assertEqual(output[0]['type'], "error") self.assertFalse(output[1]['payload']['correct']) self.assertEqual(output[1]['payload']['message'], "Your code can not be executed due to a syntax error:
Missing parentheses in call to 'print' (script.py, line 1).") - self.assertEqual(output[1]['payload']['tags']['fun'], "syntax_error") def test_indentation_error(self): self.data = { @@ -59,7 +58,6 @@ def test_indentation_error(self): self.assertEqual(output[0]['type'], "script-output") self.assertFalse(output[1]['payload']['correct']) self.assertEqual(output[1]['payload']['message'], "Your code could not be parsed due to an error in the indentation:
unexpected indent (script.py, line 1).") - self.assertEqual(output[1]['payload']['tags']['fun'], "indentation_error") def test_enrichment_error(self): self.data = { From 8b40dd77213abf1aaf7092d7162cb612280883bb Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Fri, 17 Mar 2017 10:05:58 -0400 Subject: [PATCH 020/418] remove all uses of set_tag --- pythonwhat/Reporter.py | 2 -- pythonwhat/State.py | 3 --- pythonwhat/test_funcs/test_comp.py | 3 --- pythonwhat/test_funcs/test_correct.py | 1 - pythonwhat/test_funcs/test_data_frame.py | 1 - pythonwhat/test_funcs/test_dictionary.py | 1 - pythonwhat/test_funcs/test_for_loop.py | 1 - pythonwhat/test_funcs/test_function.py | 2 -- pythonwhat/test_funcs/test_function_definition.py | 1 - pythonwhat/test_funcs/test_if_else.py | 1 - pythonwhat/test_funcs/test_import.py | 1 - pythonwhat/test_funcs/test_lambda_function.py | 1 - pythonwhat/test_funcs/test_mc.py | 1 - pythonwhat/test_funcs/test_object.py | 1 - pythonwhat/test_funcs/test_object_accessed.py | 1 - pythonwhat/test_funcs/test_operator.py | 1 - pythonwhat/test_funcs/test_or.py | 1 - pythonwhat/test_funcs/test_output_contains.py | 1 - pythonwhat/test_funcs/test_student_typed.py | 1 - pythonwhat/test_funcs/test_try_except.py | 1 - pythonwhat/test_funcs/test_while_loop.py | 1 - pythonwhat/test_funcs/test_with.py | 1 - 22 files changed, 28 deletions(-) diff --git a/pythonwhat/Reporter.py b/pythonwhat/Reporter.py index 9f96bf17..51bf381c 100644 --- a/pythonwhat/Reporter.py +++ b/pythonwhat/Reporter.py @@ -23,8 +23,6 @@ def __init__(self): self.failure_msg = "" self.fallback_ast = None - def set_tag(self, *args, **kwargs): pass - def do_test(self, testobj, prepend_on_fail="", fallback_ast=None): """Do test. diff --git a/pythonwhat/State.py b/pythonwhat/State.py index 037d0605..3d2d3b8f 100644 --- a/pythonwhat/State.py +++ b/pythonwhat/State.py @@ -232,14 +232,12 @@ def parse_ext(x): utils_ast.mark_text_ranges(res, x + '\n') except IndentationError as e: - rep.set_tag("fun", "indentation_error") e.filename = "script.py" # no line info for now rep.feedback = Feedback("Your code could not be parsed due to an error in the indentation:
`%s.`" % str(e)) rep.failed_test = True except SyntaxError as e: - rep.set_tag("fun", "syntax_error") e.filename = "script.py" # no line info for now rep.feedback = Feedback("Your code can not be executed due to a syntax error:
`%s.`" % str(e)) @@ -248,7 +246,6 @@ def parse_ext(x): # Can happen, can't catch this earlier because we can't differentiate between # TypeError in parsing or TypeError within code (at runtime). except: - rep.set_tag("fun", "other_error") rep.feedback.message = "Something went wrong while parsing your code." rep.failed_test = True diff --git a/pythonwhat/test_funcs/test_comp.py b/pythonwhat/test_funcs/test_comp.py index 79ebe1a3..53276416 100644 --- a/pythonwhat/test_funcs/test_comp.py +++ b/pythonwhat/test_funcs/test_comp.py @@ -27,7 +27,6 @@ def test_list_comp(index=1, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_list_comp") test_comp("list comprehension", 'list_comps', **(locals())) @@ -44,7 +43,6 @@ def test_generator_exp(index=1, """Test generator expressions """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_generator_exp") test_comp("generator expression", 'generator_exps', **(locals())) @@ -63,7 +61,6 @@ def test_dict_comp(index=1, """Test dict comprehension. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_dict_comp") test_comp("dictionary comprehension", 'dict_comps', **(locals())) diff --git a/pythonwhat/test_funcs/test_correct.py b/pythonwhat/test_funcs/test_correct.py index 91c110be..e3abc36f 100644 --- a/pythonwhat/test_funcs/test_correct.py +++ b/pythonwhat/test_funcs/test_correct.py @@ -9,7 +9,6 @@ def test_correct(check, diagnose, state=None): """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_correct") def diagnose_and_check(state=None): # use multi twice, since diagnose and check may be lists of tests diff --git a/pythonwhat/test_funcs/test_data_frame.py b/pythonwhat/test_funcs/test_data_frame.py index 2374e86c..f00467d2 100644 --- a/pythonwhat/test_funcs/test_data_frame.py +++ b/pythonwhat/test_funcs/test_data_frame.py @@ -20,7 +20,6 @@ def test_data_frame(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_data_frame") child = check_object(name, undefined_msg or MSG_UNDEFINED, expand_msg="", state=state, typestr="pandas DataFrame") is_instance(pd.DataFrame, not_data_frame_msg or MSG_NOT_INSTANCE, state=child) # test instance diff --git a/pythonwhat/test_funcs/test_dictionary.py b/pythonwhat/test_funcs/test_dictionary.py index 45dc4472..b7ba861b 100644 --- a/pythonwhat/test_funcs/test_dictionary.py +++ b/pythonwhat/test_funcs/test_dictionary.py @@ -18,7 +18,6 @@ def test_dictionary(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_dictionary") child = check_object(name, undefined_msg or MSG_UNDEFINED, expand_msg = "", state=state, typestr="dictionary") is_instance(dict, not_dictionary_msg or MSG_NOT_INSTANCE, state=child) # test instance diff --git a/pythonwhat/test_funcs/test_for_loop.py b/pythonwhat/test_funcs/test_for_loop.py index a8fc1173..bd4089ae 100644 --- a/pythonwhat/test_funcs/test_for_loop.py +++ b/pythonwhat/test_funcs/test_for_loop.py @@ -62,7 +62,6 @@ def test_for_loop(index=1, :code:`test_exression_output()` will pass on the body code. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_for_loop") state = check_node('for_loops', index-1, "`for` loops", MSG_MISSING, MSG_PREPEND, state=state) diff --git a/pythonwhat/test_funcs/test_function.py b/pythonwhat/test_funcs/test_function.py index db46d2d0..622d27f6 100644 --- a/pythonwhat/test_funcs/test_function.py +++ b/pythonwhat/test_funcs/test_function.py @@ -65,7 +65,6 @@ def test_function(name, test_function(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_function") index = index - 1 @@ -259,7 +258,6 @@ def test_function_v2(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_function") index = index - 1 eq_map = {"equal": EqualTest} diff --git a/pythonwhat/test_funcs/test_function_definition.py b/pythonwhat/test_funcs/test_function_definition.py index 0feab9c7..8d9a636e 100644 --- a/pythonwhat/test_funcs/test_function_definition.py +++ b/pythonwhat/test_funcs/test_function_definition.py @@ -117,7 +117,6 @@ def shout( word = 'help', times = 3 ): body = test_function('print', args = []])) """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_function_definition") # what the function will be referred to as typestr = "`{}()`".format(name) diff --git a/pythonwhat/test_funcs/test_if_else.py b/pythonwhat/test_funcs/test_if_else.py index 109bbf12..11ea90ae 100644 --- a/pythonwhat/test_funcs/test_if_else.py +++ b/pythonwhat/test_funcs/test_if_else.py @@ -69,7 +69,6 @@ def test_if_else(index=1, the same thing in the solution as in the student code. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_if_else") # get state with specific if block diff --git a/pythonwhat/test_funcs/test_import.py b/pythonwhat/test_funcs/test_import.py index 34e79181..4f71c63e 100644 --- a/pythonwhat/test_funcs/test_import.py +++ b/pythonwhat/test_funcs/test_import.py @@ -40,7 +40,6 @@ def test_import(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_import") student_imports = state.student_imports solution_imports = state.solution_imports diff --git a/pythonwhat/test_funcs/test_lambda_function.py b/pythonwhat/test_funcs/test_lambda_function.py index fff59fe3..f6e7d78d 100644 --- a/pythonwhat/test_funcs/test_lambda_function.py +++ b/pythonwhat/test_funcs/test_lambda_function.py @@ -64,7 +64,6 @@ def test_lambda_function(index, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_lambda_function") # what the lambda will be referred to as typestr = "the {} lambda function".format(get_ord(index)) diff --git a/pythonwhat/test_funcs/test_mc.py b/pythonwhat/test_funcs/test_mc.py index 32da03f8..357981ba 100644 --- a/pythonwhat/test_funcs/test_mc.py +++ b/pythonwhat/test_funcs/test_mc.py @@ -22,7 +22,6 @@ def test_mc(correct, msgs, state=None): raise ValueError("correct should be an integer") rep = Reporter.active_reporter - rep.set_tag("fun", "test_mc") student_process = state.student_process if not isDefinedInProcess(MC_VAR_NAME, student_process): raise NameError("Option not available in the student process") diff --git a/pythonwhat/test_funcs/test_object.py b/pythonwhat/test_funcs/test_object.py index 0b4e6eca..276d9ea5 100644 --- a/pythonwhat/test_funcs/test_object.py +++ b/pythonwhat/test_funcs/test_object.py @@ -51,7 +51,6 @@ def test_object(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_object") child = check_object(name, undefined_msg or MSG_UNDEFINED, expand_msg = "", state=state) diff --git a/pythonwhat/test_funcs/test_object_accessed.py b/pythonwhat/test_funcs/test_object_accessed.py index f56bc439..8d98ad74 100644 --- a/pythonwhat/test_funcs/test_object_accessed.py +++ b/pythonwhat/test_funcs/test_object_accessed.py @@ -40,7 +40,6 @@ def test_object_accessed(name, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_object_accessed") student_object_accesses = state.student_object_accesses student_mappings = state.student_oa_mappings diff --git a/pythonwhat/test_funcs/test_operator.py b/pythonwhat/test_funcs/test_operator.py index 86f391ed..74e2787f 100644 --- a/pythonwhat/test_funcs/test_operator.py +++ b/pythonwhat/test_funcs/test_operator.py @@ -58,7 +58,6 @@ def test_operator(index=1, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_operator") # Indexing starts at 1 for the pythonwhat user. index = index - 1 diff --git a/pythonwhat/test_funcs/test_or.py b/pythonwhat/test_funcs/test_or.py index 83b26032..93b6f37b 100644 --- a/pythonwhat/test_funcs/test_or.py +++ b/pythonwhat/test_funcs/test_or.py @@ -6,7 +6,6 @@ def test_or(*tests, state=None): """Test whether at least one SCT passes.""" rep = Reporter.active_reporter - rep.set_tag("fun", "test_or") success = False first_feedback = None diff --git a/pythonwhat/test_funcs/test_output_contains.py b/pythonwhat/test_funcs/test_output_contains.py index b46b864d..377a01df 100644 --- a/pythonwhat/test_funcs/test_output_contains.py +++ b/pythonwhat/test_funcs/test_output_contains.py @@ -18,7 +18,6 @@ def test_output_contains(text, no_output_msg (str): feedback message to be displayed if the output is not found. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_output_contains") if not no_output_msg: no_output_msg = "You did not output the correct things." diff --git a/pythonwhat/test_funcs/test_student_typed.py b/pythonwhat/test_funcs/test_student_typed.py index 8038c630..d7cf37ea 100644 --- a/pythonwhat/test_funcs/test_student_typed.py +++ b/pythonwhat/test_funcs/test_student_typed.py @@ -18,7 +18,6 @@ def test_student_typed(text, not_typed_msg (str): feedback message to be displayed if the student did not type the text. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_student_typed") if not not_typed_msg: if pattern: diff --git a/pythonwhat/test_funcs/test_try_except.py b/pythonwhat/test_funcs/test_try_except.py index e7ccff61..93d830b6 100644 --- a/pythonwhat/test_funcs/test_try_except.py +++ b/pythonwhat/test_funcs/test_try_except.py @@ -41,7 +41,6 @@ def test_try_except(index=1, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_try_except") # TODO: alternatively, could have missing_msg not use prepended messages # then we wouldn't have to run check_part twice for everything diff --git a/pythonwhat/test_funcs/test_while_loop.py b/pythonwhat/test_funcs/test_while_loop.py index a5c5ac00..170691d4 100644 --- a/pythonwhat/test_funcs/test_while_loop.py +++ b/pythonwhat/test_funcs/test_while_loop.py @@ -67,7 +67,6 @@ def test_while_loop(index=1, and solution code and `test_exression_output()` will pass on the body code. """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_while_loop") state = check_node('whiles', index-1, "`while` loops", MSG_MISSING, MSG_PREPEND if expand_message else "", state=state) diff --git a/pythonwhat/test_funcs/test_with.py b/pythonwhat/test_funcs/test_with.py index 4dcbb1d5..0fca69f1 100644 --- a/pythonwhat/test_funcs/test_with.py +++ b/pythonwhat/test_funcs/test_with.py @@ -36,7 +36,6 @@ def test_with(index, """ rep = Reporter.active_reporter - rep.set_tag("fun", "test_with") check_with = partial(check_node, 'withs', index-1, "`with` statement", MSG_MISSING, state=state) From e8e13eb19c00873331861c749d3683936b5f27cf Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 11 Apr 2017 14:44:17 -0400 Subject: [PATCH 021/418] expand has_equal_ast to see if student code contains solution, manual code --- pythonwhat/check_funcs.py | 17 ++++++++++---- tests/test_spec.py | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index 6823f2c2..54850088 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -397,19 +397,28 @@ def call(args, from pythonwhat.tasks import ReprFail, UndefinedValue from pythonwhat import utils -def has_equal_ast(incorrect_msg="FMT: Your code does not seem to match the solution.", state=None): +def has_equal_ast(incorrect_msg="FMT: Your code does not seem to match the solution.", code=None, exact=True, state=None): """Test whether abstract syntax trees match between the student and solution code. Args: incorrect_msg: message displayed when ASTs mismatch. + code: optional code to use instead of the solution AST + exact: whether the representations must match exactly. If false, the solution AST + only needs to be contained within the student AST (similar to using test student typed). """ rep = Reporter.active_reporter - stu_rep = ast.dump(state.student_tree) - sol_rep = ast.dump(state.solution_tree) + parse_tree = lambda n: ast.dump(n.body[0] if isinstance(n, ast.Module) and len(n.body) == 1 else n) + + stu_rep = parse_tree(state.student_tree) + sol_rep = parse_tree(state.solution_tree if not code else ast.parse(code)) _msg = state.build_message(incorrect_msg) - rep.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state.highlight))) + + if exact: + rep.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state.highlight))) + elif not sol_rep in stu_rep: + rep.do_test(Test(Feedback(_msg, state.highlight))) return state diff --git a/tests/test_spec.py b/tests/test_spec.py index 9dcd57fd..cb3aa5f8 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -172,6 +172,53 @@ def test_fail(self): sct_payload = helper.run(self.data) self.assertFalse(sct_payload['correct']) +class TestHasEqualAst(unittest.TestCase): + def setUp(self): + self.data = { + "DC_SOLUTION": """dict(a = "a").keys()""", + "DC_CODE": """dict(a = 'a') .keys()"""} + + def failing_submission(self): + self.data["DC_CODE"] = "dict(A = 'a').keys(somearg = 2)""" + sct_payload = helper.run(self.data) + self.assertFalse(sct_payload['correct']) + + def test_simple_pass(self): + self.data["DC_SCT"] = "Ex().has_equal_ast()" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_simple_fail(self): + self.data["DC_SCT"] = "Ex().has_equal_ast()" + self.failing_submission() + + def test_function_pass(self): + self.data["DC_SCT"] = "Ex().check_function('dict', 0, signature=False).has_equal_ast()" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_function_fail(self): + self.data["DC_SCT"] = "Ex().check_function('dict', 0, signature=False).has_equal_ast()" + self.failing_submission() + + def test_function_code_pass(self): + self.data["DC_SCT"] = """Ex().has_equal_ast(code = 'dict(a = "a").keys()')""" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_function_code_fail(self): + self.data["DC_SCT"] = """Ex().has_equal_ast(code = 'dict(a = "a").keys()')""" + self.failing_submission() + + def test_exact_false_pass(self): + self.data["DC_CODE"] = """dict(a = 'a').keys()\nprint('extra')""" + self.data["DC_SCT"] = "Ex().has_equal_ast(exact=False)" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_exact_false_fail(self): + self.data["DC_SCT"] = "Ex().has_equal_ast(exact=False)" + self.failing_submission() class TestOverride(unittest.TestCase): """ From 686a96e2b2b8ff6a37291456d0cbcda1fa123f88 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 12 Apr 2017 09:33:46 -0400 Subject: [PATCH 022/418] add doc example to has_equal_ast --- pythonwhat/check_funcs.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index 54850088..709607f7 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -405,6 +405,20 @@ def has_equal_ast(incorrect_msg="FMT: Your code does not seem to match the solut code: optional code to use instead of the solution AST exact: whether the representations must match exactly. If false, the solution AST only needs to be contained within the student AST (similar to using test student typed). + + :Example: + + Student and Solution Code:: + + dict(a = 'value').keys() + + SCT:: + + # all pass + Ex().has_equal_ast() + Ex().has_equal_ast(code = "dict(a = 'value').keys()") + Ex().has_equal_ast(code = "dict(a = 'value')", exact = False) + """ rep = Reporter.active_reporter From 799aa799d5c9db35229850d1d61b113ea6c15989 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 12 Apr 2017 14:26:46 -0400 Subject: [PATCH 023/418] bump minor version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 74371ee6..df85b510 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='pythonwhat', - version='2.4.0', + version='2.5.0', packages=['pythonwhat', 'pythonwhat.test_funcs'], install_requires=["dill", "IPython", "numpy", "pandas", "markdown2", "jinja2"] ) From 35abd937fee856a0087d80378b0ee78733e33269 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Fri, 14 Apr 2017 09:29:11 -0400 Subject: [PATCH 024/418] Squashed commit of the following: commit d4e55c128b8fd68c24aaef514fb89f7d4773c5c7 Author: Michael Chow Date: Thu Mar 2 17:09:19 2017 -0500 Update part_checks.rst commit b1342c9fb80d2984508399085523805725853a96 Author: Michael Chow Date: Thu Mar 2 16:57:34 2017 -0500 stub check_object tests commit 03028516ff28087263f7250c7703c85f25a26309 Author: Michael Chow Date: Thu Mar 2 16:52:39 2017 -0500 Update quickstart_guide.md commit f7b983467236230accc427853ede34b6824b03c4 Author: Michael Chow Date: Thu Mar 2 16:46:27 2017 -0500 Update Home.md --- docs/source/Home.md | 4 +-- docs/source/part_checks.rst | 46 +++++++++++++++++++++++++ docs/source/quickstart_guide.md | 4 +-- docs/source/simple_tests/test_object.md | 28 +++++++++++++++ 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/docs/source/Home.md b/docs/source/Home.md index b48a9818..ae1c01c4 100644 --- a/docs/source/Home.md +++ b/docs/source/Home.md @@ -41,8 +41,6 @@ To robustly test the equality of objects, and results of evaluations, it has to The remainder of the wiki goes over every test function that `pythonwhat` features, explaining all arguments and covering different use cases. They will give you an idea of how, why and when to use them. -For more full examples of SCTs for Python exercises on DataCamp, check out the [source files of the introduction to Python course](http://www.github.com/datacamp/courses-intro-to-python). In the chapter files there, you can can see the SCTs that have been written for several exercises. - -To test your understanding of writing SCTs for Python exercises on the DataCamp platform, you can take the course [Writing SCTs with pythonwhat](https://www.datacamp.com/courses/writing-scts-with-pythonwhat) course. +For more full examples of SCTs for Python exercises on DataCamp, check out the [source files of the pythonwhat tutorial course](http://www.github.com/datacamp/courses-pythonwhat-tutorial). In the chapter files there, you can can see the SCTs that have been written for several exercises. Another useful course is [introduction to Python](http://www.github.com/datacamp/courses-intro-to-python). After reading through this documentation, we hope writing SCTs for Python exercises on DataCamp becomes a painless experience. If this is not the case and you think improvements to `pythonwhat` and this documentation are possible, [please let us know](mailto:content-engineering@datacamp.com)! diff --git a/docs/source/part_checks.rst b/docs/source/part_checks.rst index e9026125..dc19e0c1 100644 --- a/docs/source/part_checks.rst +++ b/docs/source/part_checks.rst @@ -209,6 +209,52 @@ followed by its iterator. .multi(check_body().has_equal_value()) \ .check_iter().has_equal_value() +has_context +~~~~~~~~~~~~~ + +.. autofunction:: pythonwhat.check_has_context.has_context + +Tests whether context variables defined by the student match the solution, for a selected block of code. +A context variable is one that is defined in a looping or block statement. +For example, ``ii`` in the code below. + +.. code:: + + [ii + 1 for ii in range(3)] + +By default, the test fails if the submission code does not have the same number of context variables. +This is illustrated below. + +**Solution Code** + +.. code:: + + # ii and ltr are context variables + for ii, ltr in enumerate(['a']): pass + +**SCT** + +.. code:: + + Ex().check_for_loop(0).check_body().has_context() + +**Passing Submission** + +.. code:: + + # still 2 variables, just different names + for jj, Ltr in enumerate(['a']): pass + +**Failing Submission** + +.. code:: + + # only 1 variable + for ii in enumerate(['a']): pass + +.. note:: + + that if you use ``has_context(exact_names = True)``, then the submission must use the same names for the context variables, which would cause the passing submission above to fail. set_context ~~~~~~~~~~~~~ diff --git a/docs/source/quickstart_guide.md b/docs/source/quickstart_guide.md index 8fdcb08c..a10def9c 100644 --- a/docs/source/quickstart_guide.md +++ b/docs/source/quickstart_guide.md @@ -47,13 +47,13 @@ The SCT uses three `pythonwhat` chains to test the correctness of the student's In all the test statements above, feedback messages will be automatically generated when something goes wrong. However, it is possible to manually set these feedback messages. For example, in the code below, ```python -Ex().check_object(undefined_msg="`x` is undefined!") \ +Ex().check_object(missing_msg="`x` is undefined!") \ .has_equal_value(incorrect_msg="wrong value for `x`") ``` the automatic messages for when `x` is undefined or incorrect are replaced with manual feedback. Now, if students submit `x = 4` instead of `x = 5`, they will see the message, "wrong value for `x`". Finally, notice that you can use Markdown syntax inside the strings here. -The same holds for `test_output_contains()`: you can use the `no_output_msg` argument to specify a custom message. For more information on all the different arguments you can set in the different `pythonwhat` functions, have a look at the articles in this wiki, describing them in detail. +The same holds for `has_equal_value()`: you can use the `incorrect_msg` argument to specify a custom message. For more information on all the different arguments you can set in the different `pythonwhat` functions, have a look at the articles in this wiki, describing them in detail. Next Steps ---------- diff --git a/docs/source/simple_tests/test_object.md b/docs/source/simple_tests/test_object.md index 4e0120fd..fc1a52a6 100644 --- a/docs/source/simple_tests/test_object.md +++ b/docs/source/simple_tests/test_object.md @@ -97,3 +97,31 @@ Of course primitive classes like `str`, `int`, `list`, `dict`, ... can be tested As explained in the [Processes article](../expression_tests.md), objects are extracted from their respected processes by 'dilling' and 'undilling' them. However, you can manually set a 'converter' with the `set_converter()` function. This will override the default dilling and undilling behavior, and enables you to make simplified representations of custom objects, testing only exactly what you want to test. **NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](../expression_tests.md). + +check_object +------------ + +```eval_rst +.. autofunction:: pythonwhat.check_object.check_object +``` + +is_instance +----------- + +```eval_rst +.. autofunction:: pythonwhat.check_object.is_instance +``` + +has_key +----------- + +```eval_rst +.. autofunction:: pythonwhat.check_object.has_key +``` + +has_equal_key +--------------- + +```eval_rst +.. autofunction:: pythonwhat.check_object.has_equal_key +``` From 4fccc96ed3485b51e55536838c98be44f04a744f Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 19 Apr 2017 09:06:28 -0400 Subject: [PATCH 025/418] remove extra Expr root node in has_equal_ast --- pythonwhat/check_funcs.py | 7 ++++++- tests/test_spec.py | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index 709607f7..b3d0cda2 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -422,8 +422,13 @@ def has_equal_ast(incorrect_msg="FMT: Your code does not seem to match the solut """ rep = Reporter.active_reporter - parse_tree = lambda n: ast.dump(n.body[0] if isinstance(n, ast.Module) and len(n.body) == 1 else n) + def parse_tree(n): + # get contents of module.body if only 1 element + crnt = n.body[0] if isinstance(n, ast.Module) and len(n.body) == 1 else n + # remove Expr if it exists + return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt) + stu_rep = parse_tree(state.student_tree) sol_rep = parse_tree(state.solution_tree if not code else ast.parse(code)) diff --git a/tests/test_spec.py b/tests/test_spec.py index cb3aa5f8..4938e9dc 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -220,6 +220,15 @@ def test_exact_false_fail(self): self.data["DC_SCT"] = "Ex().has_equal_ast(exact=False)" self.failing_submission() + def test_part_of_method_pass(self): + self.data["DC_SCT"] = """Ex().has_equal_ast(code = 'dict(a = "a")', exact=False)""" + sct_payload = helper.run(self.data) + self.assertTrue(sct_payload['correct']) + + def test_part_of_method_fail(self): + self.data["DC_SCT"] = """Ex().has_equal_ast(code = 'dict(a = "a")', exact=False)""" + self.failing_submission() + class TestOverride(unittest.TestCase): """ This class is used to test overriding w/ correct and incorrect code. Tests are From 5522baed72c8158275e001b24c4db4a3c0a553be Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 19 Apr 2017 09:44:20 -0400 Subject: [PATCH 026/418] bump patch version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df85b510..8e48373c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='pythonwhat', - version='2.5.0', + version='2.5.1', packages=['pythonwhat', 'pythonwhat.test_funcs'], install_requires=["dill", "IPython", "numpy", "pandas", "markdown2", "jinja2"] ) From ac40f02c80e854d03d5dbedf43d543744aa40c04 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Thu, 4 May 2017 18:10:31 -0400 Subject: [PATCH 027/418] doc updates --- docs/source/Home.md | 10 ++++------ docs/source/index.rst | 24 ++++++++++++++++++------ docs/source/interactive_examples.rst | 8 ++++++++ docs/source/logic_tests/index.rst | 6 +++++- docs/source/simple_tests/index.rst | 3 ++- docs/source/simple_tests/test_object.md | 12 ++++-------- 6 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 docs/source/interactive_examples.rst diff --git a/docs/source/Home.md b/docs/source/Home.md index ae1c01c4..2e1abd76 100644 --- a/docs/source/Home.md +++ b/docs/source/Home.md @@ -1,10 +1,8 @@ -Home -==== +Background +========== -At DataCamp we build tools to [learn data science](https://www.datacamp.com) interactively. See e.g. our online [R tutorial](https://www.datacamp.com/courses/free-introduction-to-r) to learn R Programming and our Python For Data Science tutorial to [learn Python](https://www.datacamp.com/courses/intro-to-python-for-data-science). - -pythonwhat? ------------ +What is pythonwhat? +------------------- A major part of DataCamp's interactive learning is centered around automated and meaningful feedback. When a student submits an incorrect answer, the system tells the student what he or she is doing wrong. This happens through so-called submission correctness tests, or SCTs. An SCT is a test script that compares the different steps in a student's submission to the ideal solution, and generates meaningful feedback along the way. diff --git a/docs/source/index.rst b/docs/source/index.rst index d30a9c36..19ea37ca 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,26 +1,38 @@ pythonwhat ================================================ -Contents: ---------- +At DataCamp we build tools to `learn data science `_ interactively. +See e.g. our online `R tutorial `_ to learn R Programming and our Python For Data Science tutorial to `learn Python `_. + +To jump in to pythonwhat, check out the `Interactive Examples `_ and the :doc:`Quickstart Guide `. + .. toctree:: + :hidden: :maxdepth: 2 + :caption: General Home quickstart_guide + interactive_examples + + +.. toctree:: + :hidden: + :maxdepth: 2 + :caption: API + simple_tests/index.rst part_checks expression_tests logic_tests/index.rst - spec2_summary -Pythonwhat V1 -------------- - .. toctree:: + :hidden: + :caption: Upgrading from v1 + spec2_summary pythonwhat.wiki/index.rst diff --git a/docs/source/interactive_examples.rst b/docs/source/interactive_examples.rst new file mode 100644 index 00000000..6094abf1 --- /dev/null +++ b/docs/source/interactive_examples.rst @@ -0,0 +1,8 @@ +Interactive Examples +==================== + +Click `here `_ for walkthrough +made up of interactive exercises. + +In order to raise issues or make changes to the walkthrough, see the +`courses-pythonwhat-tutorial `_ github repo. diff --git a/docs/source/logic_tests/index.rst b/docs/source/logic_tests/index.rst index 8227204b..5bca93ac 100644 --- a/docs/source/logic_tests/index.rst +++ b/docs/source/logic_tests/index.rst @@ -1,9 +1,13 @@ Logic Tests =========== +These tests are used to combine different SCTs. For example, ``test_or`` takes +one or more SCTs, and fails only if all of those SCTs fail. + .. toctree:: :maxdepth: 2 + :hidden: test_correct test_or - test_no + test_not diff --git a/docs/source/simple_tests/index.rst b/docs/source/simple_tests/index.rst index cf037998..fddcc2f4 100644 --- a/docs/source/simple_tests/index.rst +++ b/docs/source/simple_tests/index.rst @@ -2,11 +2,12 @@ Simple Tests ============ Simple tests are the most basic tests available in pythonwhat. -They usually don't focus on specific pieces of a submission (:doc:`like part checks `), or re-run any code (:doc:`like expression tests ). +They usually don't focus on specific pieces of a submission (:doc:`like part checks `). Instead, they simply look at things like imports, printed output, or raw code text. A final, common use is to test the value of a variable in the final environment (that is, after the submission of solution code have been run). .. toctree:: + :hidden: :maxdepth: 2 test_import diff --git a/docs/source/simple_tests/test_object.md b/docs/source/simple_tests/test_object.md index fc1a52a6..38bec3ff 100644 --- a/docs/source/simple_tests/test_object.md +++ b/docs/source/simple_tests/test_object.md @@ -98,29 +98,25 @@ As explained in the [Processes article](../expression_tests.md), objects are ext **NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](../expression_tests.md). -check_object ------------- +### check_object syntax ```eval_rst .. autofunction:: pythonwhat.check_object.check_object ``` -is_instance ------------ +#### is_instance ```eval_rst .. autofunction:: pythonwhat.check_object.is_instance ``` -has_key ------------ +#### has_key ```eval_rst .. autofunction:: pythonwhat.check_object.has_key ``` -has_equal_key ---------------- +#### has_equal_key ```eval_rst .. autofunction:: pythonwhat.check_object.has_equal_key From fb1ce223c1aa19506f1e76ee07e784eb3b1bd00d Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 24 May 2017 16:26:34 -0400 Subject: [PATCH 028/418] give has_expr a copy arg, to speed up scts --- pythonwhat/check_funcs.py | 2 ++ pythonwhat/tasks.py | 31 ++++++++++++++++++---------- tests/test_test_expression_result.py | 14 +++++++++++++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pythonwhat/check_funcs.py b/pythonwhat/check_funcs.py index b3d0cda2..8b89423f 100644 --- a/pythonwhat/check_funcs.py +++ b/pythonwhat/check_funcs.py @@ -451,6 +451,7 @@ def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eva keep_objs_in_env=None, name=None, highlight=None, + copy=True, state=None, test=None): """Run student and solution code, compare returned value, printed output, or errors. @@ -496,6 +497,7 @@ def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eva expr_code = expr_code, keep_objs_in_env = keep_objs_in_env, name=name, + copy=copy, do_exec = True if test == 'output' else False) eval_sol, str_sol = get_func(tree = state.solution_tree, diff --git a/pythonwhat/tasks.py b/pythonwhat/tasks.py index 53c13dc1..034c5cf1 100644 --- a/pythonwhat/tasks.py +++ b/pythonwhat/tasks.py @@ -5,7 +5,7 @@ import pythonwhat import ast import inspect -import copy +from copy import deepcopy from pickle import PicklingError from pythonwhat.utils_env import set_context_vals, assign_from_ast from contextlib import contextmanager @@ -330,17 +330,10 @@ def get_error(f, *args, **kwargs): def taskRunEval(tree, process, shell, keep_objs_in_env = None, extra_env = None, context=None, context_vals=None, - pre_code = "", expr_code = "", name="", tempname='_evaluation_object_', do_exec=False, - call=None): - new_env = utils.copy_env(get_env(shell.user_ns), keep_objs_in_env) - if extra_env is not None: - new_env.update(copy.deepcopy(extra_env)) - if context is not None: - set_context_vals(new_env, context, context_vals) + pre_code = "", expr_code = "", name="", copy=True, tempname='_evaluation_object_', + do_exec=False, call=None): try: - # Execute pre_code if specified - if pre_code: exec(pre_code, new_env) - + # Prepare code -------------------------------------------------------- # If no name given, the object of interest is the output of eval # otherwise, we'll use name to get the object from the environment if not (name or do_exec): @@ -353,6 +346,22 @@ def taskRunEval(tree, if expr_code: code = expr_code else: code = compile(tree, "