diff --git a/Instruction_set_Project_2.pdf b/Instruction_set_Project_2.pdf new file mode 100644 index 0000000..99175c7 Binary files /dev/null and b/Instruction_set_Project_2.pdf differ diff --git a/Python150kExtractor/Untitled.ipynb b/Python150kExtractor/Untitled.ipynb new file mode 100644 index 0000000..9cf2347 --- /dev/null +++ b/Python150kExtractor/Untitled.ipynb @@ -0,0 +1,101 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "!PYTHON150K_DIR=./PYTHON150_data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Usage: mkdir [OPTION]... DIRECTORY...\r\n", + "Create the DIRECTORY(ies), if they do not already exist.\r\n", + "\r\n", + "Mandatory arguments to long options are mandatory for short options too.\r\n", + " -m, --mode=MODE set file mode (as in chmod), not a=rwx - umask\r\n", + " -p, --parents no error if existing, make parent directories as needed\r\n", + " -v, --verbose print a message for each created directory\r\n", + " -Z set SELinux security context of each created directory\r\n", + " to the default type\r\n", + " --context[=CTX] like -Z, or if CTX is specified then set the SELinux\r\n", + " or SMACK security context to CTX\r\n", + " --help display this help and exit\r\n", + " --version output version information and exit\r\n", + "\r\n", + "GNU coreutils online help: \r\n", + "Full documentation at: \r\n", + "or available locally via: info '(coreutils) mkdir invocation'\r\n" + ] + } + ], + "source": [ + "!mkdir --help" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: missing operand\r\n", + "Try 'mkdir --help' for more information.\r\n" + ] + } + ], + "source": [ + "!mkdir $PYTHON150K_DIR" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!echo $PYTHON150K_DIR" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Python150kExtractor/data/dir/PYTHON150_K_data/example.py b/Python150kExtractor/data/dir/PYTHON150_K_data/example.py new file mode 100644 index 0000000..7a8bc6e --- /dev/null +++ b/Python150kExtractor/data/dir/PYTHON150_K_data/example.py @@ -0,0 +1,3 @@ +import pprint + +print("hello world") diff --git a/Python150kExtractor/data/dir/PYTHON150_K_data/parse_python.py b/Python150kExtractor/data/dir/PYTHON150_K_data/parse_python.py new file mode 100755 index 0000000..c714de2 --- /dev/null +++ b/Python150kExtractor/data/dir/PYTHON150_K_data/parse_python.py @@ -0,0 +1,147 @@ +#!/usr/bin/python + +import sys +import json as json +import ast + +def PrintUsage(): + sys.stderr.write(""" +Usage: + parse_python.py + +""") + exit(1) + +def read_file_to_string(filename): + f = open(filename, 'rt') + s = f.read() + f.close() + return s + +def parse_file(filename): + global c, d + tree = ast.parse(read_file_to_string(filename), filename) + + json_tree = [] + def gen_identifier(identifier, node_type = 'identifier'): + pos = len(json_tree) + json_node = {} + json_tree.append(json_node) + json_node['type'] = node_type + json_node['value'] = identifier + return pos + + def traverse_list(l, node_type = 'list'): + pos = len(json_tree) + json_node = {} + json_tree.append(json_node) + json_node['type'] = node_type + children = [] + for item in l: + children.append(traverse(item)) + if (len(children) != 0): + json_node['children'] = children + return pos + + def traverse(node): + pos = len(json_tree) + json_node = {} + json_tree.append(json_node) + json_node['type'] = type(node).__name__ + children = [] + if isinstance(node, ast.Name): + json_node['value'] = node.id + elif isinstance(node, ast.Num): + json_node['value'] = str(node.n) + elif isinstance(node, ast.Str): + json_node['value'] = node.s + elif isinstance(node, ast.alias): + json_node['value'] = str(node.name) + if node.asname: + children.append(gen_identifier(node.asname)) + elif isinstance(node, ast.FunctionDef): + json_node['value'] = str(node.name) + elif isinstance(node, ast.ClassDef): + json_node['value'] = str(node.name) + elif isinstance(node, ast.ImportFrom): + if node.module: + json_node['value'] = str(node.module) + elif isinstance(node, ast.Global): + for n in node.names: + children.append(gen_identifier(n)) + elif isinstance(node, ast.keyword): + json_node['value'] = str(node.arg) + + + # Process children. + if isinstance(node, ast.For): + children.append(traverse(node.target)) + children.append(traverse(node.iter)) + children.append(traverse_list(node.body, 'body')) + if node.orelse: + children.append(traverse_list(node.orelse, 'orelse')) + elif isinstance(node, ast.If) or isinstance(node, ast.While): + children.append(traverse(node.test)) + children.append(traverse_list(node.body, 'body')) + if node.orelse: + children.append(traverse_list(node.orelse, 'orelse')) + elif isinstance(node, ast.With): + children.append(traverse(node.context_expr)) + if node.optional_vars: + children.append(traverse(node.optional_vars)) + children.append(traverse_list(node.body, 'body')) + elif isinstance(node, ast.Try): + children.append(traverse_list(node.body, 'body')) + children.append(traverse_list(node.handlers, 'handlers')) + if node.orelse: + children.append(traverse_list(node.orelse, 'orelse')) + elif isinstance(node, ast.Try): + children.append(traverse_list(node.body, 'body')) + children.append(traverse_list(node.finalbody, 'finalbody')) + elif isinstance(node, ast.arguments): + children.append(traverse_list(node.args, 'args')) + children.append(traverse_list(node.defaults, 'defaults')) + if node.vararg: + children.append(gen_identifier(node.vararg, 'vararg')) + if node.kwarg: + children.append(gen_identifier(node.kwarg, 'kwarg')) + elif isinstance(node, ast.ExceptHandler): + if node.type: + children.append(traverse_list([node.type], 'type')) + if node.name: + children.append(traverse_list([node.name], 'name')) + children.append(traverse_list(node.body, 'body')) + elif isinstance(node, ast.ClassDef): + children.append(traverse_list(node.bases, 'bases')) + children.append(traverse_list(node.body, 'body')) + children.append(traverse_list(node.decorator_list, 'decorator_list')) + elif isinstance(node, ast.FunctionDef): + children.append(traverse(node.args)) + children.append(traverse_list(node.body, 'body')) + children.append(traverse_list(node.decorator_list, 'decorator_list')) + else: + # Default handling: iterate over children. + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.expr_context) or isinstance(child, ast.operator) or isinstance(child, ast.boolop) or isinstance(child, ast.unaryop) or isinstance(child, ast.cmpop): + # Directly include expr_context, and operators into the type instead of creating a child. + json_node['type'] = json_node['type'] + type(child).__name__ + else: + children.append(traverse(child)) + + if isinstance(node, ast.Attribute): + children.append(gen_identifier(node.attr, 'attr')) + + if (len(children) != 0): + json_node['children'] = children + return pos + + traverse(tree) + return json.dumps(json_tree, separators=(',', ':'), ensure_ascii=False) + +if __name__ == "__main__": + if len(sys.argv) != 2: + PrintUsage() + try: + print(parse_file(sys.argv[1])) + except(strEncodeError, strDecodeError): + pass diff --git a/Python150kExtractor/preprocess.sh b/Python150kExtractor/preprocess.sh old mode 100644 new mode 100755 diff --git a/Requirements.txt b/Requirements.txt new file mode 100644 index 0000000..e17c8b8 --- /dev/null +++ b/Requirements.txt @@ -0,0 +1,180 @@ +absl-py==0.9.0 +annoy==1.16.3 +astor==0.8.1 +attrs==19.3.0 +awscli==1.18.36 +backcall==0.1.0 +bcolz==1.2.1 +beautifulsoup4==4.6.3 +bleach==3.1.4 +blis==0.4.1 +botocore==1.15.36 +cachetools==4.0.0 +catalogue==1.0.0 +certifi==2019.11.28 +chardet==3.0.4 +cliff==2.8.3 +cmd2==1.0.1 +colorama==0.4.3 +configparser==5.0.0 +cssselect==1.0.3 +cycler==0.10.0 +cymem==2.0.3 +cytoolz==0.10.1 +dask==2.14.0 +decorator==4.4.2 +defusedxml==0.6.0 +descartes==1.1.0 +dill==0.3.1.1 +docutils==0.15.2 +en-core-web-sm==2.2.5 +entrypoints==0.3 +fastai==0.6 +feather-format==0.4.0 +ftfy==4.4.3 +gast==0.2.2 +generalutils==0.1.6 +glob3==0.0.1 +google-auth==1.13.1 +google-auth-oauthlib==0.4.1 +google-pasta==0.2.0 +graphviz==0.13.2 +grpcio==1.28.1 +h5py==2.10.0 +html5lib==1.0.1 +idna==2.9 +ijson==3.0 +importlib-metadata==1.6.0 +ipykernel==5.2.0 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +isoweek==1.3.3 +jedi==0.16.0 +Jinja2==2.11.1 +jmespath==0.9.5 +joblib==0.14.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==6.1.2 +jupyter-console==6.1.0 +jupyter-contrib-core==0.3.3 +jupyter-contrib-nbextensions==0.5.1 +jupyter-core==4.6.3 +jupyter-highlight-selected-word==0.2.0 +jupyter-latex-envs==1.4.6 +jupyter-nbextensions-configurator==0.4.1 +kaggle-cli==0.12.13 +Keras==2.3.1 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.0 +kiwisolver==1.2.0 +ktext==0.40 +lxml==4.5.0 +Markdown==3.2.1 +MarkupSafe==1.1.1 +matplotlib==3.2.1 +MechanicalSoup==0.8.0 +mistune==0.8.4 +mizani==0.6.0 +more-itertools==8.2.0 +msgpack==1.0.0 +msgpack-numpy==0.4.4.3 +multiprocess==0.70.9 +murmurhash==1.0.2 +nbconvert==5.6.1 +nbformat==5.0.5 +networkx==2.4 +nltk==3.4.5 +nmslib==2.0.5 +notebook==6.0.3 +numpy==1.18.2 +oauthlib==3.1.0 +olefile==0.46 +opencv-python==4.2.0.34 +opt-einsum==3.2.0 +palettable==3.3.0 +pandas==1.0.3 +pandas-summary==0.0.7 +pandocfilters==1.4.2 +parso==0.6.2 +pathos==0.2.5 +patsy==0.5.1 +pbr==5.4.4 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==7.1.1 +plac==1.1.3 +plotnine==0.6.0 +pox==0.2.7 +ppft==1.6.6.1 +preshed==3.0.2 +prettytable==0.7.2 +progressbar2==3.34.3 +prometheus-client==0.7.1 +prompt-toolkit==3.0.5 +protobuf==3.11.3 +psutil==5.7.0 +ptyprocess==0.6.0 +pyarrow==0.16.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pybind11==2.5.0 +pyemd==0.5.1 +Pygments==2.6.1 +pyparsing==2.4.6 +pyperclip==1.8.0 +Pyphen==0.9.5 +pyrsistent==0.16.0 +python-dateutil==2.8.1 +python-Levenshtein==0.12.0 +python-utils==2.4.0 +pytz==2019.3 +PyYAML==5.3.1 +pyzmq==19.0.0 +qtconsole==4.7.2 +QtPy==1.9.0 +requests==2.23.0 +requests-oauthlib==1.3.0 +rogue==0.0.2 +rouge==1.0.0 +rsa==3.4.2 +s3transfer==0.3.3 +scikit-learn==0.22.2.post1 +scipy==1.4.1 +seaborn==0.10.0 +Send2Trash==1.5.0 +sentencepiece==0.1.85 +simplegeneric==0.8.1 +six==1.14.0 +sklearn==0.0 +sklearn-pandas==1.8.0 +spacy==2.2.4 +srsly==1.0.2 +statsmodels==0.11.1 +stevedore==1.32.0 +tensorboard==1.15.0 +tensorflow-estimator==1.15.1 +tensorflow-gpu==1.15.2 +termcolor==1.1.0 +terminado==0.8.3 +testpath==0.4.4 +textacy==0.6.2 +thinc==7.4.0 +toolz==0.10.0 +torch==1.3.1 +torchtext==0.5.0 +torchvision==0.2.0 +tornado==6.0.4 +tqdm==4.45.0 +traitlets==4.3.3 +Unidecode==1.1.1 +urllib3==1.25.8 +wasabi==0.6.0 +wcwidth==0.1.9 +webencodings==0.5.1 +Werkzeug==1.0.1 +wget==3.2 +widgetsnbextension==3.5.1 +wrapt==1.12.1 +zipp==3.1.0 diff --git a/model.py b/model.py index df1fed3..0da9ccc 100644 --- a/model.py +++ b/model.py @@ -225,8 +225,8 @@ def evaluate(self, release=False): elapsed = int(time.time() - eval_start_time) precision, recall, f1 = self.calculate_results(true_positive, false_positive, false_negative) - files_rouge = FilesRouge(predicted_file_name, ref_file_name) - rouge = files_rouge.get_scores(avg=True, ignore_empty=True) + files_rouge = FilesRouge() + rouge = files_rouge.get_scores(predicted_file_name, ref_file_name,avg=True, ignore_empty=True) print("Evaluation time: %sh%sm%ss" % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60)) return num_correct_predictions / total_predictions, \ precision, recall, f1, rouge diff --git a/train_python150k.sh b/train_python150k.sh old mode 100644 new mode 100755