"));
+ }
+}
diff --git a/JavaExtractor/JPredict/src/test/java/JavaExtractor/StopWordFilterTest.java b/JavaExtractor/JPredict/src/test/java/JavaExtractor/StopWordFilterTest.java
new file mode 100644
index 0000000..3c5555d
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/java/JavaExtractor/StopWordFilterTest.java
@@ -0,0 +1,15 @@
+package JavaExtractor;
+
+import JavaExtractor.Common.StopWordsFilter;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class StopWordFilterTest {
+
+ @Test
+ public void removeStopWordsTest() {
+ String filtered = StopWordsFilter.removeStopWords("a all be as if in hello world");
+ assertEquals(filtered, "hello world");
+ }
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/TestCSN.java b/JavaExtractor/JPredict/src/test/resources/TestCSN.java
new file mode 100644
index 0000000..66898ed
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/TestCSN.java
@@ -0,0 +1,4 @@
+{
+ docstring: "test string",
+ original_string: "int fooBar() { \n // return sum of the two numbers \n return 2 + 3;\n}"
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/TestDefault.java b/JavaExtractor/JPredict/src/test/resources/TestDefault.java
new file mode 100644
index 0000000..d7101b5
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/TestDefault.java
@@ -0,0 +1,7 @@
+/**
+* TEST JAVADOC
+*/
+public int addSum(int a, int b) {
+ // INLINE COMMENT
+ return a+b;
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/TestFuncom.java b/JavaExtractor/JPredict/src/test/resources/TestFuncom.java
new file mode 100644
index 0000000..cce39f1
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/TestFuncom.java
@@ -0,0 +1 @@
+{'\n/**\ngets the sort name of the env entry editor object\n*/\n\tpublic String getSortName() {\n return "_BeanEnvEnvEntry";\n }\n'}
diff --git a/JavaExtractor/JPredict/src/test/resources/bad_examples/spelling.java b/JavaExtractor/JPredict/src/test/resources/bad_examples/spelling.java
new file mode 100644
index 0000000..a8ab518
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/bad_examples/spelling.java
@@ -0,0 +1,7 @@
+pubic class spellingMistake{
+
+ private vod doSomething(){
+
+ }
+
+}
\ No newline at end of file
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/bracket.java b/JavaExtractor/JPredict/src/test/resources/examples/bracket.java
new file mode 100644
index 0000000..d504dda
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/bracket.java
@@ -0,0 +1,5 @@
+public class missingBracket{
+
+ private void doSomething(){
+
+ }
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/comments.java b/JavaExtractor/JPredict/src/test/resources/examples/comments.java
new file mode 100644
index 0000000..2441271
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/comments.java
@@ -0,0 +1,9 @@
+public class exampleOne{
+ /**
+ * TEST JAVADOC
+ */
+ public int add(int a, int b) {
+ // INLINE COMMENT
+ return a+b;
+ }
+}
\ No newline at end of file
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/comments2.java b/JavaExtractor/JPredict/src/test/resources/examples/comments2.java
new file mode 100644
index 0000000..c912fbe
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/comments2.java
@@ -0,0 +1,9 @@
+public class exampleThree{
+ /**
+ *TEST JAVADOC
+ */
+ public int add(int a, int b) {
+ //INLINE COMMENT
+ return a+b;
+ }
+}
\ No newline at end of file
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/longNames.java b/JavaExtractor/JPredict/src/test/resources/examples/longNames.java
new file mode 100644
index 0000000..fe39608
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/longNames.java
@@ -0,0 +1,16 @@
+public class LongNames{
+
+ private String megaSizedClassVariableName = "Some very cool String describing the megaSizedClassVariableName";
+
+ /**
+ * This method holds very long Names and is very necessary.
+ * Bananaboattourguide.
+ * @param veryLongParameterName
+ */
+ public void veryLongMethodNameThatDoesALot(int veryLongParameterName){
+ double extraLargeVariableName = 15200000000.0d;
+ // Seagulltransportationbox
+ return;
+ }
+
+}
\ No newline at end of file
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/nocomments.java b/JavaExtractor/JPredict/src/test/resources/examples/nocomments.java
new file mode 100644
index 0000000..e5411de
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/nocomments.java
@@ -0,0 +1,5 @@
+public class exampleTwo{
+ public int add(int a, int b) {
+ return a+b;
+ }
+}
\ No newline at end of file
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/onlyMethod.java b/JavaExtractor/JPredict/src/test/resources/examples/onlyMethod.java
new file mode 100644
index 0000000..ef6e8d5
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/onlyMethod.java
@@ -0,0 +1,7 @@
+/**
+ * TEST JAVADOC
+ */
+public int add(int a, int b) {
+ // INLINE COMMENT
+ return a+b;
+ }
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/orphanComment.java b/JavaExtractor/JPredict/src/test/resources/examples/orphanComment.java
new file mode 100644
index 0000000..f420493
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/orphanComment.java
@@ -0,0 +1,17 @@
+public class orphanComment {
+ /** TEST JAVADOC */
+ public int add(int a, int b) {
+ // first
+ // second
+ // third
+ b = 0;
+
+ // hello
+ // world
+ a = 0;
+
+ // re
+ // turn
+ return a + b;
+ }
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/orphanCommentNewLine.java b/JavaExtractor/JPredict/src/test/resources/examples/orphanCommentNewLine.java
new file mode 100644
index 0000000..56831dd
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/orphanCommentNewLine.java
@@ -0,0 +1,11 @@
+public class orphanComment {
+ /** TEST JAVADOC */
+ public int add(int a, int b) {
+ // first
+
+ // second
+
+ // third
+ return a + b;
+ }
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/examples/stopWordsComment.java b/JavaExtractor/JPredict/src/test/resources/examples/stopWordsComment.java
new file mode 100644
index 0000000..8e6daee
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/examples/stopWordsComment.java
@@ -0,0 +1,7 @@
+public class exampleOne {
+ /** TEST JAVADOC */
+ public int add(int x, int y) {
+ // and or at be
+ return x + y;
+ }
+}
diff --git a/JavaExtractor/JPredict/src/test/resources/jsonls/jsonTest.jsonl b/JavaExtractor/JPredict/src/test/resources/jsonls/jsonTest.jsonl
new file mode 100644
index 0000000..c84b938
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/jsonls/jsonTest.jsonl
@@ -0,0 +1 @@
+{"repo": "ReactiveX/RxJava", "path": "src/main/java/io/reactivex/internal/observers/QueueDrainObserver.java", "func_name": "QueueDrainObserver.fastPathOrderedEmit", "original_string": "public int add() {\n // Addition \n return 1+1; \n}", "language": "java", "code": " ", "code_tokens": ["protected", "final", "void", "fastPathOrderedEmit", "(", "U", "value", ",", "boolean", "delayError", ",", "Disposable", "disposable", ")", "{", "final", "Observer", "<", "?", "super", "V", ">", "observer", "=", "downstream", ";", "final", "SimplePlainQueue", "<", "U", ">", "q", "=", "queue", ";", "if", "(", "wip", ".", "get", "(", ")", "==", "0", "&&", "wip", ".", "compareAndSet", "(", "0", ",", "1", ")", ")", "{", "if", "(", "q", ".", "isEmpty", "(", ")", ")", "{", "accept", "(", "observer", ",", "value", ")", ";", "if", "(", "leave", "(", "-", "1", ")", "==", "0", ")", "{", "return", ";", "}", "}", "else", "{", "q", ".", "offer", "(", "value", ")", ";", "}", "}", "else", "{", "q", ".", "offer", "(", "value", ")", ";", "if", "(", "!", "enter", "(", ")", ")", "{", "return", ";", "}", "}", "QueueDrainHelper", ".", "drainLoop", "(", "q", ",", "observer", ",", "delayError", ",", "disposable", ",", "this", ")", ";", "}"], "docstring": "return sum of two numbers", "docstring_tokens": ["Makes", "sure", "the", "fast", "-", "path", "emits", "in", "order", "."], "sha": "ac84182aa2bd866b53e01c8e3fe99683b882c60e", "url": "https://github.com/ReactiveX/RxJava/blob/ac84182aa2bd866b53e01c8e3fe99683b882c60e/src/main/java/io/reactivex/internal/observers/QueueDrainObserver.java#L88-L108", "partition": "test"}
diff --git a/JavaExtractor/JPredict/src/test/resources/jsonls/jsonWithHtml.jsonl b/JavaExtractor/JPredict/src/test/resources/jsonls/jsonWithHtml.jsonl
new file mode 100644
index 0000000..17ed779
--- /dev/null
+++ b/JavaExtractor/JPredict/src/test/resources/jsonls/jsonWithHtml.jsonl
@@ -0,0 +1 @@
+{"repo": "ReactiveX/RxJava", "path": "src/main/java/io/reactivex/internal/observers/QueueDrainObserver.java", "func_name": "QueueDrainObserver.fastPathOrderedEmit", "original_string": "public int add() {\n // Addition \n return 1+1; \n}", "language": "java", "code": " ", "code_tokens": ["protected", "final", "void", "fastPathOrderedEmit", "(", "U", "value", ",", "boolean", "delayError", ",", "Disposable", "disposable", ")", "{", "final", "Observer", "<", "?", "super", "V", ">", "observer", "=", "downstream", ";", "final", "SimplePlainQueue", "<", "U", ">", "q", "=", "queue", ";", "if", "(", "wip", ".", "get", "(", ")", "==", "0", "&&", "wip", ".", "compareAndSet", "(", "0", ",", "1", ")", ")", "{", "if", "(", "q", ".", "isEmpty", "(", ")", ")", "{", "accept", "(", "observer", ",", "value", ")", ";", "if", "(", "leave", "(", "-", "1", ")", "==", "0", ")", "{", "return", ";", "}", "}", "else", "{", "q", ".", "offer", "(", "value", ")", ";", "}", "}", "else", "{", "q", ".", "offer", "(", "value", ")", ";", "if", "(", "!", "enter", "(", ")", ")", "{", "return", ";", "}", "}", "QueueDrainHelper", ".", "drainLoop", "(", "q", ",", "observer", ",", "delayError", ",", "disposable", ",", "this", ")", ";", "}"], "docstring": "return sum of two numbers of SOME_OBJECT
", "docstring_tokens": ["Makes", "sure", "the", "fast", "-", "path", "emits", "in", "order", "."], "sha": "ac84182aa2bd866b53e01c8e3fe99683b882c60e", "url": "https://github.com/ReactiveX/RxJava/blob/ac84182aa2bd866b53e01c8e3fe99683b882c60e/src/main/java/io/reactivex/internal/observers/QueueDrainObserver.java#L88-L108", "partition": "test"}
diff --git a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar b/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar
deleted file mode 100644
index 8972be5..0000000
Binary files a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar and /dev/null differ
diff --git a/JavaExtractor/extract.py b/JavaExtractor/extract.py
index 29ac6e3..b49a025 100644
--- a/JavaExtractor/extract.py
+++ b/JavaExtractor/extract.py
@@ -11,8 +11,11 @@
def get_immediate_subdirectories(a_dir):
- return [(os.path.join(a_dir, name)) for name in os.listdir(a_dir)
- if os.path.isdir(os.path.join(a_dir, name))]
+ return [
+ (os.path.join(a_dir, name))
+ for name in os.listdir(a_dir)
+ if os.path.isdir(os.path.join(a_dir, name))
+ ]
TMP_DIR = ""
@@ -23,18 +26,44 @@ def ParallelExtractDir(args, dir):
def ExtractFeaturesForDir(args, dir, prefix):
- command = ['java', '-Xmx100g', '-XX:MaxNewSize=60g', '-cp', args.jar, 'JavaExtractor.App',
- '--max_path_length', str(args.max_path_length), '--max_path_width', str(args.max_path_width),
- '--dir', dir, '--num_threads', str(args.num_threads)]
+
+ command = [
+ "java",
+ "-Xmx100g",
+ "-XX:MaxNewSize=60g",
+ "-cp",
+ args.jar,
+ "JavaExtractor.App",
+ "--max_path_length",
+ str(args.max_path_length),
+ "--max_path_width",
+ str(args.max_path_width),
+ "--dir",
+ dir,
+ "--num_threads",
+ str(args.num_threads),
+ "--include_comments",
+ str(args.include_comments),
+ "--exclude_stopwords",
+ str(args.exclude_stopwords),
+ "--include_tfidf",
+ str(args.include_tfidf),
+ "--number_keywords",
+ str(args.number_keywords),
+ ]
+
+ if args.dataset != "default":
+ command.append("--dataset")
+ command.append(str(args.dataset))
# print command
# os.system(command)
kill = lambda process: process.kill()
- outputFileName = TMP_DIR + prefix + dir.split('/')[-1]
+ outputFileName = TMP_DIR + prefix + dir.split("/")[-1]
failed = False
- with open(outputFileName, 'a') as outputFile:
+ with open(outputFileName, "a") as outputFile:
sleeper = subprocess.Popen(command, stdout=outputFile, stderr=subprocess.PIPE)
- timer = Timer(60 * 60, kill, [sleeper])
+ timer = Timer(60 * 60 * 60 * 60, kill, [sleeper])
try:
timer.start()
@@ -46,14 +75,15 @@ def ExtractFeaturesForDir(args, dir, prefix):
if len(stderr) > 0:
print(stderr, file=sys.stderr)
else:
- print('dir: ' + str(dir) + ' was not completed in time', file=sys.stderr)
+ print("dir: " + str(dir) + " was not completed in time", file=sys.stderr)
failed = True
subdirs = get_immediate_subdirectories(dir)
for subdir in subdirs:
- ExtractFeaturesForDir(args, subdir, prefix + dir.split('/')[-1] + '_')
+ ExtractFeaturesForDir(args, subdir, prefix + dir.split("/")[-1] + "_")
if failed:
if os.path.exists(outputFileName):
os.remove(outputFileName)
+ sys.exit(1)
def ExtractFeaturesForDirsList(args, dirs):
@@ -74,20 +104,74 @@ def ExtractFeaturesForDirsList(args, dirs):
shutil.rmtree(TMP_DIR, ignore_errors=True)
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
- parser.add_argument("-maxlen", "--max_path_length", dest="max_path_length", required=False, default=8)
- parser.add_argument("-maxwidth", "--max_path_width", dest="max_path_width", required=False, default=2)
- parser.add_argument("-threads", "--num_threads", dest="num_threads", required=False, default=64)
+ parser.add_argument(
+ "-maxlen",
+ "--max_path_length",
+ dest="max_path_length",
+ required=False,
+ default=8,
+ )
+ parser.add_argument(
+ "-maxwidth",
+ "--max_path_width",
+ dest="max_path_width",
+ required=False,
+ default=2,
+ )
+ parser.add_argument(
+ "-threads", "--num_threads", dest="num_threads", required=False, default=64
+ )
parser.add_argument("-j", "--jar", dest="jar", required=True)
parser.add_argument("-dir", "--dir", dest="dir", required=False)
parser.add_argument("-file", "--file", dest="file", required=False)
+ parser.add_argument(
+ "-inclcomm",
+ "--include_comments",
+ dest="include_comments",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "-exclstop",
+ "--exclude_stopwords",
+ dest="exclude_stopwords",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "-incltfidf",
+ "--include_tfidf",
+ dest="include_tfidf",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "-numkeywords",
+ "--number_keywords",
+ dest="number_keywords",
+ required=False,
+ default=4,
+ )
+ parser.add_argument(
+ "-d", "--dataset", dest="dataset", required=False, default="default"
+ )
args = parser.parse_args()
if args.file is not None:
- command = 'java -cp ' + args.jar + ' JavaExtractor.App --max_path_length ' + \
- str(args.max_path_length) + ' --max_path_width ' + str(args.max_path_width) + ' --file ' + args.file
- os.system(command)
+ command = (
+ "java -cp "
+ + args.jar
+ + " JavaExtractor.App --max_path_length "
+ + str(args.max_path_length)
+ + " --max_path_width "
+ + str(args.max_path_width)
+ + " --file "
+ + args.file
+ )
+ exit_code = os.system(command)
+ sys.exit(exit_code)
elif args.dir is not None:
subdirs = get_immediate_subdirectories(args.dir)
if len(subdirs) == 0:
diff --git a/Python150kExtractor/extract.py b/Python150kExtractor/extract.py
index ce38bce..7701674 100644
--- a/Python150kExtractor/extract.py
+++ b/Python150kExtractor/extract.py
@@ -10,23 +10,23 @@
from pathlib import Path
from sklearn import model_selection as sklearn_model_selection
-METHOD_NAME, NUM = 'METHODNAME', 'NUM'
+METHOD_NAME, NUM = "METHODNAME", "NUM"
parser = argparse.ArgumentParser()
-parser.add_argument('--data_dir', required=True, type=str)
-parser.add_argument('--valid_p', type=float, default=0.2)
-parser.add_argument('--max_path_length', type=int, default=8)
-parser.add_argument('--max_path_width', type=int, default=2)
-parser.add_argument('--use_method_name', type=bool, default=True)
-parser.add_argument('--use_nums', type=bool, default=True)
-parser.add_argument('--output_dir', required=True, type=str)
-parser.add_argument('--n_jobs', type=int, default=multiprocessing.cpu_count())
-parser.add_argument('--seed', type=int, default=239)
+parser.add_argument("--data_dir", required=True, type=str)
+parser.add_argument("--valid_p", type=float, default=0.2)
+parser.add_argument("--max_path_length", type=int, default=8)
+parser.add_argument("--max_path_width", type=int, default=2)
+parser.add_argument("--use_method_name", type=bool, default=True)
+parser.add_argument("--use_nums", type=bool, default=True)
+parser.add_argument("--output_dir", required=True, type=str)
+parser.add_argument("--n_jobs", type=int, default=multiprocessing.cpu_count())
+parser.add_argument("--seed", type=int, default=239)
def __collect_asts(json_file):
asts = []
- with open(json_file, 'r', encoding='utf-8') as f:
+ with open(json_file, "r", encoding="utf-8") as f:
for line in f:
ast = json.loads(line.strip())
asts.append(ast)
@@ -42,22 +42,22 @@ def dfs(v):
v_node = ast[v]
- if 'value' in v_node:
+ if "value" in v_node:
if v == node_index: # Top-level func def node.
if args.use_method_name:
paths.append((stack.copy(), METHOD_NAME))
else:
- v_type = v_node['type']
+ v_type = v_node["type"]
- if v_type.startswith('Name'):
- paths.append((stack.copy(), v_node['value']))
- elif args.use_nums and v_type == 'Num':
+ if v_type.startswith("Name"):
+ paths.append((stack.copy(), v_node["value"]))
+ elif args.use_nums and v_type == "Num":
paths.append((stack.copy(), NUM))
else:
pass
- if 'children' in v_node:
- for child in v_node['children']:
+ if "children" in v_node:
+ for child in v_node["children"]:
dfs(child)
stack.pop()
@@ -84,12 +84,13 @@ def __raw_tree_paths(ast, node_index, args):
tree_paths = []
for (v_path, v_value), (u_path, u_value) in itertools.combinations(
- iterable=tnodes,
- r=2,
+ iterable=tnodes,
+ r=2,
):
prefix, lca, suffix = __merge_terminals2_paths(v_path, u_path)
- if (len(prefix) + 1 + len(suffix) <= args.max_path_length) \
- and (abs(len(prefix) - len(suffix)) <= args.max_path_width):
+ if (len(prefix) + 1 + len(suffix) <= args.max_path_length) and (
+ abs(len(prefix) - len(suffix)) <= args.max_path_width
+ ):
path = prefix + [lca] + suffix
tree_path = v_value, path, u_value
tree_paths.append(tree_path)
@@ -103,24 +104,24 @@ def __delim_name(name):
def camel_case_split(identifier):
matches = re.finditer(
- '.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)',
+ ".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)",
identifier,
)
return [m.group(0) for m in matches]
blocks = []
- for underscore_block in name.split('_'):
+ for underscore_block in name.split("_"):
blocks.extend(camel_case_split(underscore_block))
- return '|'.join(block.lower() for block in blocks)
+ return "|".join(block.lower() for block in blocks)
def __collect_sample(ast, fd_index, args):
root = ast[fd_index]
- if root['type'] != 'FunctionDef':
- raise ValueError('Wrong node type.')
+ if root["type"] != "FunctionDef":
+ raise ValueError("Wrong node type.")
- target = root['value']
+ target = root["value"]
tree_paths = __raw_tree_paths(ast, fd_index, args)
contexts = []
@@ -128,24 +129,24 @@ def __collect_sample(ast, fd_index, args):
start, connector, finish = tree_path
start, finish = __delim_name(start), __delim_name(finish)
- connector = '|'.join(ast[v]['type'] for v in connector)
+ connector = "|".join(ast[v]["type"] for v in connector)
- context = f'{start},{connector},{finish}'
+ context = f"{start},{connector},{finish}"
contexts.append(context)
if len(contexts) == 0:
return None
target = __delim_name(target)
- context = ' '.join(contexts)
+ context = " ".join(contexts)
- return f'{target} {context}'
+ return f"{target} {context}"
def __collect_samples(ast, args):
samples = []
for node_index, node in enumerate(ast):
- if node['type'] == 'FunctionDef':
+ if node["type"] == "FunctionDef":
sample = __collect_sample(ast, node_index, args)
if sample is not None:
samples.append(sample)
@@ -160,9 +161,9 @@ def __collect_all_and_save(asts, args, output_file):
samples = parallel(func(ast, args) for ast in tqdm.tqdm(asts))
samples = list(itertools.chain.from_iterable(samples))
- with open(output_file, 'w') as f:
+ with open(output_file, "w") as f:
for line_index, line in enumerate(samples):
- f.write(line + ('' if line_index == len(samples) - 1 else '\n'))
+ f.write(line + ("" if line_index == len(samples) - 1 else "\n"))
def main():
@@ -170,8 +171,8 @@ def main():
np.random.seed(args.seed)
data_dir = Path(args.data_dir)
- trains = __collect_asts(data_dir / 'python100k_train.json')
- evals = __collect_asts(data_dir / 'python50k_eval.json')
+ trains = __collect_asts(data_dir / "python100k_train.json")
+ evals = __collect_asts(data_dir / "python50k_eval.json")
train, valid = sklearn_model_selection.train_test_split(
trains,
@@ -182,12 +183,12 @@ def main():
output_dir = Path(args.output_dir)
output_dir.mkdir(exist_ok=True)
for split_name, split in zip(
- ('train', 'valid', 'test'),
- (train, valid, test),
+ ("train", "valid", "test"),
+ (train, valid, test),
):
- output_file = output_dir / f'{split_name}_output_file.txt'
+ output_file = output_dir / f"{split_name}_output_file.txt"
__collect_all_and_save(split, args, output_file)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/args.py b/args.py
index 1f38ac2..31d56f1 100644
--- a/args.py
+++ b/args.py
@@ -5,21 +5,59 @@ def read_args():
parser = ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
- group.add_argument("-d", "--data", dest="data_path",
- help="path to preprocessed dataset")
- group.add_argument("-l", "--load_path", dest="load_path",
- help="path to load model files", metavar="FILE")
+ group.add_argument(
+ "-d", "--data", dest="data_path", help="path to preprocessed dataset"
+ )
+ group.add_argument(
+ "-l",
+ "--load_path",
+ dest="load_path",
+ help="path to load model files",
+ metavar="FILE",
+ )
- parser.add_argument("-m", "--model_path", dest="model_path",
- help="path to save and load checkpoints", metavar="FILE", required=False)
- parser.add_argument("-s", "--save_path", dest="save_path",
- help="path to save model files", metavar="FILE", required=False)
+ parser.add_argument(
+ "-m",
+ "--model_path",
+ dest="model_path",
+ help="path to save and load checkpoints",
+ metavar="FILE",
+ required=False,
+ )
+ parser.add_argument(
+ "-s",
+ "--save_path",
+ dest="save_path",
+ help="path to save model files",
+ metavar="FILE",
+ required=False,
+ )
- parser.add_argument("-t", "--test", dest="test_path",
- help="path to test file", metavar="FILE", required=False)
+ parser.add_argument(
+ "-t",
+ "--test",
+ dest="test_path",
+ help="path to test file",
+ metavar="FILE",
+ required=False,
+ )
- parser.add_argument('-p', '--predict', dest='predict', type=str, default='java',
- help='starts prediction mode, argument is "cpp" or "java" dependin on language model')
- parser.add_argument('--debug', action='store_true')
- parser.add_argument('--seed', type=int, default=239)
+ parser.add_argument(
+ "-p",
+ "--predict",
+ dest="predict",
+ type=str,
+ default="",
+ help='starts prediction mode, argument is "cpp" or "java" dependin on language model',
+ )
+ parser.add_argument(
+ "-c",
+ "--continue_training_from_checkpoint",
+ dest="continue_from_checkpoint",
+ type=str,
+ help='Continue training model from a previous checkpoint, if it exists.',
+ )
+
+ parser.add_argument("--debug", action="store_true")
+ parser.add_argument("--seed", type=int, default=239)
return parser.parse_args()
diff --git a/baseline_tokenization/javalang/__init__.py b/baseline_tokenization/javalang/__init__.py
index 8ee0b30..23adac1 100644
--- a/baseline_tokenization/javalang/__init__.py
+++ b/baseline_tokenization/javalang/__init__.py
@@ -1,4 +1,3 @@
-
from . import parser
from . import parse
from . import tokenizer
diff --git a/baseline_tokenization/javalang/ast.py b/baseline_tokenization/javalang/ast.py
index 66f9312..072f752 100644
--- a/baseline_tokenization/javalang/ast.py
+++ b/baseline_tokenization/javalang/ast.py
@@ -5,14 +5,14 @@
class MetaNode(type):
def __new__(mcs, name, bases, dict):
- attrs = list(dict['attrs'])
- dict['attrs'] = list()
+ attrs = list(dict["attrs"])
+ dict["attrs"] = list()
for base in bases:
- if hasattr(base, 'attrs'):
- dict['attrs'].extend(base.attrs)
+ if hasattr(base, "attrs"):
+ dict["attrs"].extend(base.attrs)
- dict['attrs'].extend(attrs)
+ dict["attrs"].extend(attrs)
return type.__new__(mcs, name, bases, dict)
@@ -29,7 +29,7 @@ def __init__(self, **kwargs):
setattr(self, attr_name, value)
if values:
- raise ValueError('Extraneous arguments')
+ raise ValueError("Extraneous arguments")
def __equals__(self, other):
if type(other) is not type(self):
@@ -49,14 +49,16 @@ def __iter__(self):
def filter(self, pattern):
for path, node in self:
- if ((isinstance(pattern, type) and isinstance(node, pattern)) or
- (node == pattern)):
+ if (isinstance(pattern, type) and isinstance(node, pattern)) or (
+ node == pattern
+ ):
yield path, node
@property
def children(self):
return [getattr(self, attr_name) for attr_name in self.attrs]
+
def walk_tree(root):
children = None
@@ -71,8 +73,10 @@ def walk_tree(root):
for path, node in walk_tree(child):
yield (root,) + path, node
+
def dump(ast, file):
pickle.dump(ast, file)
+
def load(file):
return pickle.load(file)
diff --git a/baseline_tokenization/javalang/javadoc.py b/baseline_tokenization/javalang/javadoc.py
index ee3635d..624986f 100644
--- a/baseline_tokenization/javalang/javadoc.py
+++ b/baseline_tokenization/javalang/javadoc.py
@@ -1,12 +1,13 @@
-
import re
+
def join(s):
- return ' '.join(l.strip() for l in s.split('\n'))
+ return " ".join(l.strip() for l in s.split("\n"))
+
class DocBlock(object):
def __init__(self):
- self.description = ''
+ self.description = ""
self.return_doc = None
self.params = []
@@ -22,59 +23,64 @@ def __init__(self):
def add_block(self, name, value):
value = value.strip()
- if name == 'param':
+ if name == "param":
try:
param, description = value.split(None, 1)
except ValueError:
- param, description = value, ''
+ param, description = value, ""
self.params.append((param, join(description)))
- elif name in ('throws', 'exception'):
+ elif name in ("throws", "exception"):
try:
ex, description = value.split(None, 1)
except ValueError:
- ex, description = value, ''
+ ex, description = value, ""
self.throws[ex] = join(description)
- elif name == 'return':
+ elif name == "return":
self.return_doc = value
- elif name == 'author':
+ elif name == "author":
self.authors.append(value)
- elif name == 'deprecated':
+ elif name == "deprecated":
self.deprecated = True
self.tags.setdefault(name, []).append(value)
-blocks_re = re.compile('(^@)', re.MULTILINE)
-leading_space_re = re.compile(r'^\s*\*', re.MULTILINE)
-blocks_justify_re = re.compile(r'^\s*@', re.MULTILINE)
+
+blocks_re = re.compile("(^@)", re.MULTILINE)
+leading_space_re = re.compile(r"^\s*\*", re.MULTILINE)
+blocks_justify_re = re.compile(r"^\s*@", re.MULTILINE)
+
def _sanitize(s):
s = s.strip()
- if not (s[:3] == '/**' and s[-2:] == '*/'):
- raise ValueError('not a valid Javadoc comment')
+ if not (s[:3] == "/**" and s[-2:] == "*/"):
+ raise ValueError("not a valid Javadoc comment")
- s = s.replace('\t', ' ')
+ s = s.replace("\t", " ")
return s
+
def _uncomment(s):
# Remove /** and */
s = s[3:-2].strip()
- return leading_space_re.sub('', s)
+ return leading_space_re.sub("", s)
+
def _get_indent_level(s):
return len(s) - len(s.lstrip())
+
def _left_justify(s):
lines = s.rstrip().splitlines()
if not lines:
- return ''
+ return ""
indent_levels = []
for line in lines:
@@ -87,10 +93,12 @@ def _left_justify(s):
return s
else:
lines = [line[common_indent:] for line in lines]
- return '\n'.join(lines)
+ return "\n".join(lines)
+
def _force_blocks_left(s):
- return blocks_justify_re.sub('@', s)
+ return blocks_justify_re.sub("@", s)
+
def parse(raw):
sanitized = _sanitize(raw)
@@ -103,7 +111,7 @@ def parse(raw):
doc = DocBlock()
- if blocks[0] != '@':
+ if blocks[0] != "@":
doc.description = blocks[0].strip()
blocks = blocks[2::2]
else:
@@ -113,7 +121,7 @@ def parse(raw):
try:
tag, value = block.split(None, 1)
except ValueError:
- tag, value = block, ''
+ tag, value = block, ""
doc.add_block(tag, value)
diff --git a/baseline_tokenization/javalang/parse.py b/baseline_tokenization/javalang/parse.py
index 0451fed..0732ee8 100644
--- a/baseline_tokenization/javalang/parse.py
+++ b/baseline_tokenization/javalang/parse.py
@@ -1,52 +1,57 @@
-
from .parser import Parser
from .tokenizer import tokenize
+
def parse_expression(exp):
- if not exp.endswith(';'):
- exp = exp + ';'
+ if not exp.endswith(";"):
+ exp = exp + ";"
tokens = tokenize(exp)
parser = Parser(tokens)
return parser.parse_expression()
+
def parse_member_signature(sig):
- if not sig.endswith(';'):
- sig = sig + ';'
+ if not sig.endswith(";"):
+ sig = sig + ";"
tokens = tokenize(sig)
parser = Parser(tokens)
return parser.parse_member_declaration()
+
def parse_constructor_signature(sig):
# Add an empty body to the signature, replacing a ; if necessary
- if sig.endswith(';'):
+ if sig.endswith(";"):
sig = sig[:-1]
- sig = sig + '{ }'
+ sig = sig + "{ }"
tokens = tokenize(sig)
parser = Parser(tokens)
return parser.parse_member_declaration()
+
def parse_type(s):
tokens = tokenize(s)
parser = Parser(tokens)
return parser.parse_type()
+
def parse_type_signature(sig):
- if sig.endswith(';'):
+ if sig.endswith(";"):
sig = sig[:-1]
- sig = sig + '{ }'
+ sig = sig + "{ }"
tokens = tokenize(sig)
parser = Parser(tokens)
return parser.parse_class_or_interface_declaration()
+
def parse(s):
tokens = tokenize(s)
parser = Parser(tokens)
diff --git a/baseline_tokenization/javalang/parser.py b/baseline_tokenization/javalang/parser.py
index c78a9f4..f47b768 100644
--- a/baseline_tokenization/javalang/parser.py
+++ b/baseline_tokenization/javalang/parser.py
@@ -3,18 +3,27 @@
from . import util
from . import tree
from .tokenizer import (
- EndOfInput, Keyword, Modifier, BasicType, Identifier,
- Annotation, Literal, Operator, JavaToken,
- )
+ EndOfInput,
+ Keyword,
+ Modifier,
+ BasicType,
+ Identifier,
+ Annotation,
+ Literal,
+ Operator,
+ JavaToken,
+)
ENABLE_DEBUG_SUPPORT = False
+
def parse_debug(method):
global ENABLE_DEBUG_SUPPORT
if ENABLE_DEBUG_SUPPORT:
+
def _method(self):
- if not hasattr(self, 'recursion_depth'):
+ if not hasattr(self, "recursion_depth"):
self.recursion_depth = 0
if self.debug:
@@ -22,7 +31,7 @@ def _method(self):
token = six.text_type(self.tokens.look())
start_value = self.tokens.look().value
name = method.__name__
- sep = ("-" * self.recursion_depth)
+ sep = "-" * self.recursion_depth
e_message = ""
print("%s %s> %s(%s)" % (depth, sep, name, token))
@@ -42,8 +51,10 @@ def _method(self):
finally:
token = six.text_type(self.tokens.last())
- print("%s <%s %s(%s, %s) %s" %
- (depth, sep, name, start_value, token, e_message))
+ print(
+ "%s <%s %s(%s, %s) %s"
+ % (depth, sep, name, start_value, token, e_message)
+ )
self.recursion_depth -= 1
else:
self.recursion_depth += 1
@@ -59,13 +70,16 @@ def _method(self):
else:
return method
+
# ------------------------------------------------------------------------------
# ---- Parsing exception ----
+
class JavaParserBaseException(Exception):
- def __init__(self, message=''):
+ def __init__(self, message=""):
super(JavaParserBaseException, self).__init__(message)
+
class JavaSyntaxError(JavaParserBaseException):
def __init__(self, description, at=None):
super(JavaSyntaxError, self).__init__()
@@ -73,23 +87,28 @@ def __init__(self, description, at=None):
self.description = description
self.at = at
+
class JavaParserError(JavaParserBaseException):
pass
+
# ------------------------------------------------------------------------------
# ---- Parser class ----
+
class Parser(object):
- operator_precedence = [ set(('||',)),
- set(('&&',)),
- set(('|',)),
- set(('^',)),
- set(('&',)),
- set(('==', '!=')),
- set(('<', '>', '>=', '<=', 'instanceof')),
- set(('<<', '>>', '>>>')),
- set(('+', '-')),
- set(('*', '/', '%')) ]
+ operator_precedence = [
+ set(("||",)),
+ set(("&&",)),
+ set(("|",)),
+ set(("^",)),
+ set(("&",)),
+ set(("==", "!=")),
+ set(("<", ">", ">=", "<=", "instanceof")),
+ set(("<<", ">>", ">>>")),
+ set(("+", "-")),
+ set(("*", "/", "%")),
+ ]
def __init__(self, tokens):
self.tokens = util.LookAheadListIterator(tokens)
@@ -97,20 +116,20 @@ def __init__(self, tokens):
self.debug = False
-# ------------------------------------------------------------------------------
-# ---- Debug control ----
+ # ------------------------------------------------------------------------------
+ # ---- Debug control ----
def set_debug(self, debug=True):
self.debug = debug
-# ------------------------------------------------------------------------------
-# ---- Parsing entry point ----
+ # ------------------------------------------------------------------------------
+ # ---- Parsing entry point ----
def parse(self):
return self.parse_compilation_unit()
-# ------------------------------------------------------------------------------
-# ---- Helper methods ----
+ # ------------------------------------------------------------------------------
+ # ---- Helper methods ----
def illegal(self, description, at=None):
if not at:
@@ -126,8 +145,7 @@ def accept(self, *accepts):
for accept in accepts:
token = next(self.tokens)
- if isinstance(accept, six.string_types) and (
- not token.value == accept):
+ if isinstance(accept, six.string_types) and (not token.value == accept):
self.illegal("Expected '%s'" % (accept,))
elif isinstance(accept, type) and not isinstance(token, accept):
self.illegal("Expected %s" % (accept.__name__,))
@@ -143,8 +161,7 @@ def would_accept(self, *accepts):
for i, accept in enumerate(accepts):
token = self.tokens.look(i)
- if isinstance(accept, six.string_types) and (
- not token.value == accept):
+ if isinstance(accept, six.string_types) and (not token.value == accept):
return False
elif isinstance(accept, type) and not isinstance(token, accept):
return False
@@ -158,8 +175,7 @@ def try_accept(self, *accepts):
for i, accept in enumerate(accepts):
token = self.tokens.look(i)
- if isinstance(accept, six.string_types) and (
- not token.value == accept):
+ if isinstance(accept, six.string_types) and (not token.value == accept):
return False
elif isinstance(accept, type) and not isinstance(token, accept):
return False
@@ -204,28 +220,32 @@ def build_binary_operation(self, parts, start_level=0):
return operation
def is_annotation(self, i=0):
- """ Returns true if the position is the start of an annotation application
+ """Returns true if the position is the start of an annotation application
(as opposed to an annotation declaration)
"""
- return (isinstance(self.tokens.look(i), Annotation)
- and not self.tokens.look(i + 1).value == 'interface')
+ return (
+ isinstance(self.tokens.look(i), Annotation)
+ and not self.tokens.look(i + 1).value == "interface"
+ )
def is_annotation_declaration(self, i=0):
- """ Returns true if the position is the start of an annotation application
+ """Returns true if the position is the start of an annotation application
(as opposed to an annotation declaration)
"""
- return (isinstance(self.tokens.look(i), Annotation)
- and self.tokens.look(i + 1).value == 'interface')
+ return (
+ isinstance(self.tokens.look(i), Annotation)
+ and self.tokens.look(i + 1).value == "interface"
+ )
-# ------------------------------------------------------------------------------
-# ---- Parsing methods ----
+ # ------------------------------------------------------------------------------
+ # ---- Parsing methods ----
-# ------------------------------------------------------------------------------
-# -- Identifiers --
+ # ------------------------------------------------------------------------------
+ # -- Identifiers --
@parse_debug
def parse_identifier(self):
@@ -239,10 +259,10 @@ def parse_qualified_identifier(self):
identifier = self.parse_identifier()
qualified_identifier.append(identifier)
- if not self.try_accept('.'):
+ if not self.try_accept("."):
break
- return '.'.join(qualified_identifier)
+ return ".".join(qualified_identifier)
@parse_debug
def parse_qualified_identifier_list(self):
@@ -252,13 +272,13 @@ def parse_qualified_identifier_list(self):
qualified_identifier = self.parse_qualified_identifier()
qualified_identifiers.append(qualified_identifier)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return qualified_identifiers
-# ------------------------------------------------------------------------------
-# -- Top level units --
+ # ------------------------------------------------------------------------------
+ # -- Top level units --
@parse_debug
def parse_compilation_unit(self):
@@ -276,18 +296,20 @@ def parse_compilation_unit(self):
if self.is_annotation():
package_annotations = self.parse_annotations()
- if self.try_accept('package'):
+ if self.try_accept("package"):
self.tokens.pop_marker(False)
package_name = self.parse_qualified_identifier()
- package = tree.PackageDeclaration(annotations=package_annotations,
- name=package_name,
- documentation=javadoc)
- self.accept(';')
+ package = tree.PackageDeclaration(
+ annotations=package_annotations,
+ name=package_name,
+ documentation=javadoc,
+ )
+ self.accept(";")
else:
self.tokens.pop_marker(True)
package_annotations = None
- while self.would_accept('import'):
+ while self.would_accept("import"):
import_declaration = self.parse_import_declaration()
import_declarations.append(import_declaration)
@@ -300,9 +322,9 @@ def parse_compilation_unit(self):
if type_declaration:
type_declarations.append(type_declaration)
- return tree.CompilationUnit(package=package,
- imports=import_declarations,
- types=type_declarations)
+ return tree.CompilationUnit(
+ package=package, imports=import_declarations, types=type_declarations
+ )
@parse_debug
def parse_import_declaration(self):
@@ -310,32 +332,32 @@ def parse_import_declaration(self):
static = False
import_all = False
- self.accept('import')
+ self.accept("import")
- if self.try_accept('static'):
+ if self.try_accept("static"):
static = True
while True:
identifier = self.parse_identifier()
qualified_identifier.append(identifier)
- if self.try_accept('.'):
- if self.try_accept('*'):
- self.accept(';')
+ if self.try_accept("."):
+ if self.try_accept("*"):
+ self.accept(";")
import_all = True
break
else:
- self.accept(';')
+ self.accept(";")
break
- return tree.Import(path='.'.join(qualified_identifier),
- static=static,
- wildcard=import_all)
+ return tree.Import(
+ path=".".join(qualified_identifier), static=static, wildcard=import_all
+ )
@parse_debug
def parse_type_declaration(self):
- if self.try_accept(';'):
+ if self.try_accept(";"):
return None
else:
return self.parse_class_or_interface_declaration()
@@ -346,11 +368,11 @@ def parse_class_or_interface_declaration(self):
type_declaration = None
token = self.tokens.look()
- if token.value == 'class':
+ if token.value == "class":
type_declaration = self.parse_normal_class_declaration()
- elif token.value == 'enum':
+ elif token.value == "enum":
type_declaration = self.parse_enum_declaration()
- elif token.value == 'interface':
+ elif token.value == "interface":
type_declaration = self.parse_normal_interface_declaration()
elif self.is_annotation_declaration():
type_declaration = self.parse_annotation_type_declaration()
@@ -371,26 +393,28 @@ def parse_normal_class_declaration(self):
implements = None
body = None
- self.accept('class')
+ self.accept("class")
name = self.parse_identifier()
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_params = self.parse_type_parameters()
- if self.try_accept('extends'):
+ if self.try_accept("extends"):
extends = self.parse_type()
- if self.try_accept('implements'):
+ if self.try_accept("implements"):
implements = self.parse_type_list()
body = self.parse_class_body()
- return tree.ClassDeclaration(name=name,
- type_parameters=type_params,
- extends=extends,
- implements=implements,
- body=body)
+ return tree.ClassDeclaration(
+ name=name,
+ type_parameters=type_params,
+ extends=extends,
+ implements=implements,
+ body=body,
+ )
@parse_debug
def parse_enum_declaration(self):
@@ -398,17 +422,15 @@ def parse_enum_declaration(self):
implements = None
body = None
- self.accept('enum')
+ self.accept("enum")
name = self.parse_identifier()
- if self.try_accept('implements'):
+ if self.try_accept("implements"):
implements = self.parse_type_list()
body = self.parse_enum_body()
- return tree.EnumDeclaration(name=name,
- implements=implements,
- body=body)
+ return tree.EnumDeclaration(name=name, implements=implements, body=body)
@parse_debug
def parse_normal_interface_declaration(self):
@@ -417,37 +439,35 @@ def parse_normal_interface_declaration(self):
extends = None
body = None
- self.accept('interface')
+ self.accept("interface")
name = self.parse_identifier()
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_parameters = self.parse_type_parameters()
- if self.try_accept('extends'):
+ if self.try_accept("extends"):
extends = self.parse_type_list()
body = self.parse_interface_body()
- return tree.InterfaceDeclaration(name=name,
- type_parameters=type_parameters,
- extends=extends,
- body=body)
+ return tree.InterfaceDeclaration(
+ name=name, type_parameters=type_parameters, extends=extends, body=body
+ )
@parse_debug
def parse_annotation_type_declaration(self):
name = None
body = None
- self.accept('@', 'interface')
+ self.accept("@", "interface")
name = self.parse_identifier()
body = self.parse_annotation_type_body()
- return tree.AnnotationDeclaration(name=name,
- body=body)
+ return tree.AnnotationDeclaration(name=name, body=body)
-# ------------------------------------------------------------------------------
-# -- Types --
+ # ------------------------------------------------------------------------------
+ # -- Types --
@parse_debug
def parse_type(self):
@@ -476,10 +496,10 @@ def parse_reference_type(self):
while True:
tail.name = self.parse_identifier()
- if self.would_accept('<'):
+ if self.would_accept("<"):
tail.arguments = self.parse_type_arguments()
- if self.try_accept('.'):
+ if self.try_accept("."):
tail.sub_type = tree.ReferenceType()
tail = tail.sub_type
else:
@@ -491,16 +511,16 @@ def parse_reference_type(self):
def parse_type_arguments(self):
type_arguments = list()
- self.accept('<')
+ self.accept("<")
while True:
type_argument = self.parse_type_argument()
type_arguments.append(type_argument)
- if self.try_accept('>'):
+ if self.try_accept(">"):
break
- self.accept(',')
+ self.accept(",")
return type_arguments
@@ -509,15 +529,15 @@ def parse_type_argument(self):
pattern_type = None
base_type = None
- if self.try_accept('?'):
- if self.tokens.look().value in ('extends', 'super'):
+ if self.try_accept("?"):
+ if self.tokens.look().value in ("extends", "super"):
pattern_type = self.tokens.next().value
else:
- return tree.TypeArgument(pattern_type='?')
+ return tree.TypeArgument(pattern_type="?")
if self.would_accept(BasicType):
base_type = self.parse_basic_type()
- self.accept('[', ']')
+ self.accept("[", "]")
base_type.dimensions = [None]
else:
base_type = self.parse_reference_type()
@@ -525,14 +545,13 @@ def parse_type_argument(self):
base_type.dimensions += self.parse_array_dimension()
- return tree.TypeArgument(type=base_type,
- pattern_type=pattern_type)
+ return tree.TypeArgument(type=base_type, pattern_type=pattern_type)
@parse_debug
def parse_nonwildcard_type_arguments(self):
- self.accept('<')
+ self.accept("<")
type_arguments = self.parse_type_list()
- self.accept('>')
+ self.accept(">")
return [tree.TypeArgument(type=t) for t in type_arguments]
@@ -543,7 +562,7 @@ def parse_type_list(self):
while True:
if self.would_accept(BasicType):
base_type = self.parse_basic_type()
- self.accept('[', ']')
+ self.accept("[", "]")
base_type.dimensions = [None]
else:
base_type = self.parse_reference_type()
@@ -552,21 +571,21 @@ def parse_type_list(self):
base_type.dimensions += self.parse_array_dimension()
types.append(base_type)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return types
@parse_debug
def parse_type_arguments_or_diamond(self):
- if self.try_accept('<', '>'):
+ if self.try_accept("<", ">"):
return list()
else:
return self.parse_type_arguments()
@parse_debug
def parse_nonwildcard_type_arguments_or_diamond(self):
- if self.try_accept('<', '>'):
+ if self.try_accept("<", ">"):
return list()
else:
return self.parse_nonwildcard_type_arguments()
@@ -575,16 +594,16 @@ def parse_nonwildcard_type_arguments_or_diamond(self):
def parse_type_parameters(self):
type_parameters = list()
- self.accept('<')
+ self.accept("<")
while True:
type_parameter = self.parse_type_parameter()
type_parameters.append(type_parameter)
- if self.try_accept('>'):
+ if self.try_accept(">"):
break
else:
- self.accept(',')
+ self.accept(",")
return type_parameters
@@ -593,30 +612,29 @@ def parse_type_parameter(self):
identifier = self.parse_identifier()
extends = None
- if self.try_accept('extends'):
+ if self.try_accept("extends"):
extends = list()
while True:
reference_type = self.parse_reference_type()
extends.append(reference_type)
- if not self.try_accept('&'):
+ if not self.try_accept("&"):
break
- return tree.TypeParameter(name=identifier,
- extends=extends)
+ return tree.TypeParameter(name=identifier, extends=extends)
@parse_debug
def parse_array_dimension(self):
array_dimension = 0
- while self.try_accept('[', ']'):
+ while self.try_accept("[", "]"):
array_dimension += 1
return [None] * array_dimension
-# ------------------------------------------------------------------------------
-# -- Annotations and modifiers --
+ # ------------------------------------------------------------------------------
+ # -- Annotations and modifiers --
@parse_debug
def parse_modifiers(self):
@@ -659,20 +677,19 @@ def parse_annotation(self):
qualified_identifier = None
annotation_element = None
- self.accept('@')
+ self.accept("@")
qualified_identifier = self.parse_qualified_identifier()
- if self.try_accept('('):
- if not self.would_accept(')'):
+ if self.try_accept("("):
+ if not self.would_accept(")"):
annotation_element = self.parse_annotation_element()
- self.accept(')')
+ self.accept(")")
- return tree.Annotation(name=qualified_identifier,
- element=annotation_element)
+ return tree.Annotation(name=qualified_identifier, element=annotation_element)
@parse_debug
def parse_annotation_element(self):
- if self.would_accept(Identifier, '='):
+ if self.would_accept(Identifier, "="):
return self.parse_element_value_pairs()
else:
return self.parse_element_value()
@@ -685,7 +702,7 @@ def parse_element_value_pairs(self):
pair = self.parse_element_value_pair()
pairs.append(pair)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return pairs
@@ -693,18 +710,17 @@ def parse_element_value_pairs(self):
@parse_debug
def parse_element_value_pair(self):
identifier = self.parse_identifier()
- self.accept('=')
+ self.accept("=")
value = self.parse_element_value()
- return tree.ElementValuePair(name=identifier,
- value=value)
+ return tree.ElementValuePair(name=identifier, value=value)
@parse_debug
def parse_element_value(self):
if self.is_annotation():
return self.parse_annotation()
- elif self.would_accept('{'):
+ elif self.would_accept("{"):
return self.parse_element_value_array_initializer()
else:
@@ -712,14 +728,14 @@ def parse_element_value(self):
@parse_debug
def parse_element_value_array_initializer(self):
- self.accept('{')
+ self.accept("{")
- if self.try_accept('}'):
+ if self.try_accept("}"):
return list()
element_values = self.parse_element_values()
- self.try_accept(',')
- self.accept('}')
+ self.try_accept(",")
+ self.accept("}")
return tree.ElementArrayValue(values=element_values)
@@ -731,28 +747,28 @@ def parse_element_values(self):
element_value = self.parse_element_value()
element_values.append(element_value)
- if self.would_accept('}') or self.would_accept(',', '}'):
+ if self.would_accept("}") or self.would_accept(",", "}"):
break
- self.accept(',')
+ self.accept(",")
return element_values
-# ------------------------------------------------------------------------------
-# -- Class body --
+ # ------------------------------------------------------------------------------
+ # -- Class body --
@parse_debug
def parse_class_body(self):
declarations = list()
- self.accept('{')
+ self.accept("{")
- while not self.would_accept('}'):
+ while not self.would_accept("}"):
declaration = self.parse_class_body_declaration()
if declaration:
declarations.append(declaration)
- self.accept('}')
+ self.accept("}")
return declarations
@@ -760,14 +776,14 @@ def parse_class_body(self):
def parse_class_body_declaration(self):
token = self.tokens.look()
- if self.try_accept(';'):
+ if self.try_accept(";"):
return None
- elif self.would_accept('static', '{'):
- self.accept('static')
+ elif self.would_accept("static", "{"):
+ self.accept("static")
return self.parse_block()
- elif self.would_accept('{'):
+ elif self.would_accept("{"):
return self.parse_block()
else:
@@ -779,27 +795,27 @@ def parse_member_declaration(self):
member = None
token = self.tokens.look()
- if self.try_accept('void'):
+ if self.try_accept("void"):
method_name = self.parse_identifier()
member = self.parse_void_method_declarator_rest()
member.name = method_name
- elif token.value == '<':
+ elif token.value == "<":
member = self.parse_generic_method_or_constructor_declaration()
- elif token.value == 'class':
+ elif token.value == "class":
member = self.parse_normal_class_declaration()
- elif token.value == 'enum':
+ elif token.value == "enum":
member = self.parse_enum_declaration()
- elif token.value == 'interface':
+ elif token.value == "interface":
member = self.parse_normal_interface_declaration()
elif self.is_annotation_declaration():
member = self.parse_annotation_type_declaration()
- elif self.would_accept(Identifier, '('):
+ elif self.would_accept(Identifier, "("):
constructor_name = self.parse_identifier()
member = self.parse_constructor_declarator_rest()
member.name = constructor_name
@@ -834,20 +850,21 @@ def parse_method_or_field_declaraction(self):
@parse_debug
def parse_method_or_field_rest(self):
- if self.would_accept('('):
+ if self.would_accept("("):
return self.parse_method_declarator_rest()
else:
rest = self.parse_field_declarators_rest()
- self.accept(';')
+ self.accept(";")
return rest
@parse_debug
def parse_field_declarators_rest(self):
array_dimension, initializer = self.parse_variable_declarator_rest()
- declarators = [tree.VariableDeclarator(dimensions=array_dimension,
- initializer=initializer)]
+ declarators = [
+ tree.VariableDeclarator(dimensions=array_dimension, initializer=initializer)
+ ]
- while self.try_accept(','):
+ while self.try_accept(","):
declarator = self.parse_variable_declarator()
declarators.append(declarator)
@@ -860,18 +877,20 @@ def parse_method_declarator_rest(self):
throws = None
body = None
- if self.try_accept('throws'):
+ if self.try_accept("throws"):
throws = self.parse_qualified_identifier_list()
- if self.would_accept('{'):
+ if self.would_accept("{"):
body = self.parse_block()
else:
- self.accept(';')
+ self.accept(";")
- return tree.MethodDeclaration(parameters=formal_parameters,
- throws=throws,
- body=body,
- return_type=tree.Type(dimensions=additional_dimensions))
+ return tree.MethodDeclaration(
+ parameters=formal_parameters,
+ throws=throws,
+ body=body,
+ return_type=tree.Type(dimensions=additional_dimensions),
+ )
@parse_debug
def parse_void_method_declarator_rest(self):
@@ -879,17 +898,17 @@ def parse_void_method_declarator_rest(self):
throws = None
body = None
- if self.try_accept('throws'):
+ if self.try_accept("throws"):
throws = self.parse_qualified_identifier_list()
- if self.would_accept('{'):
+ if self.would_accept("{"):
body = self.parse_block()
else:
- self.accept(';')
+ self.accept(";")
- return tree.MethodDeclaration(parameters=formal_parameters,
- throws=throws,
- body=body)
+ return tree.MethodDeclaration(
+ parameters=formal_parameters, throws=throws, body=body
+ )
@parse_debug
def parse_constructor_declarator_rest(self):
@@ -897,25 +916,25 @@ def parse_constructor_declarator_rest(self):
throws = None
body = None
- if self.try_accept('throws'):
+ if self.try_accept("throws"):
throws = self.parse_qualified_identifier_list()
body = self.parse_block()
- return tree.ConstructorDeclaration(parameters=formal_parameters,
- throws=throws,
- body=body)
+ return tree.ConstructorDeclaration(
+ parameters=formal_parameters, throws=throws, body=body
+ )
@parse_debug
def parse_generic_method_or_constructor_declaration(self):
type_parameters = self.parse_type_parameters()
method = None
- if self.would_accept(Identifier, '('):
+ if self.would_accept(Identifier, "("):
constructor_name = self.parse_identifier()
method = self.parse_constructor_declarator_rest()
method.name = constructor_name
- elif self.try_accept('void'):
+ elif self.try_accept("void"):
method_name = self.parse_identifier()
method = self.parse_void_method_declarator_rest()
method.name = method_name
@@ -933,26 +952,26 @@ def parse_generic_method_or_constructor_declaration(self):
method.type_parameters = type_parameters
return method
-# ------------------------------------------------------------------------------
-# -- Interface body --
+ # ------------------------------------------------------------------------------
+ # -- Interface body --
@parse_debug
def parse_interface_body(self):
declarations = list()
- self.accept('{')
- while not self.would_accept('}'):
+ self.accept("{")
+ while not self.would_accept("}"):
declaration = self.parse_interface_body_declaration()
if declaration:
declarations.append(declaration)
- self.accept('}')
+ self.accept("}")
return declarations
@parse_debug
def parse_interface_body_declaration(self):
- if self.try_accept(';'):
+ if self.try_accept(";"):
return None
modifiers, annotations, javadoc = self.parse_modifiers()
@@ -968,17 +987,17 @@ def parse_interface_body_declaration(self):
def parse_interface_member_declaration(self):
declaration = None
- if self.would_accept('class'):
+ if self.would_accept("class"):
declaration = self.parse_normal_class_declaration()
- elif self.would_accept('interface'):
+ elif self.would_accept("interface"):
declaration = self.parse_normal_interface_declaration()
- elif self.would_accept('enum'):
+ elif self.would_accept("enum"):
declaration = self.parse_enum_declaration()
elif self.is_annotation_declaration():
declaration = self.parse_annotation_type_declaration()
- elif self.would_accept('<'):
+ elif self.would_accept("<"):
declaration = self.parse_interface_generic_method_declarator()
- elif self.try_accept('void'):
+ elif self.try_accept("void"):
method_name = self.parse_identifier()
declaration = self.parse_void_interface_method_declarator_rest()
declaration.name = method_name
@@ -1007,21 +1026,22 @@ def parse_interface_method_or_field_declaration(self):
def parse_interface_method_or_field_rest(self):
rest = None
- if self.would_accept('('):
+ if self.would_accept("("):
rest = self.parse_interface_method_declarator_rest()
else:
rest = self.parse_constant_declarators_rest()
- self.accept(';')
+ self.accept(";")
return rest
@parse_debug
def parse_constant_declarators_rest(self):
array_dimension, initializer = self.parse_constant_declarator_rest()
- declarators = [tree.VariableDeclarator(dimensions=array_dimension,
- initializer=initializer)]
+ declarators = [
+ tree.VariableDeclarator(dimensions=array_dimension, initializer=initializer)
+ ]
- while self.try_accept(','):
+ while self.try_accept(","):
declarator = self.parse_constant_declarator()
declarators.append(declarator)
@@ -1030,7 +1050,7 @@ def parse_constant_declarators_rest(self):
@parse_debug
def parse_constant_declarator_rest(self):
array_dimension = self.parse_array_dimension()
- self.accept('=')
+ self.accept("=")
initializer = self.parse_variable_initializer()
return (array_dimension, initializer)
@@ -1040,9 +1060,9 @@ def parse_constant_declarator(self):
name = self.parse_identifier()
additional_dimension, initializer = self.parse_constant_declarator_rest()
- return tree.VariableDeclarator(name=name,
- dimensions=additional_dimension,
- initializer=initializer)
+ return tree.VariableDeclarator(
+ name=name, dimensions=additional_dimension, initializer=initializer
+ )
@parse_debug
def parse_interface_method_declarator_rest(self):
@@ -1051,18 +1071,20 @@ def parse_interface_method_declarator_rest(self):
throws = None
body = None
- if self.try_accept('throws'):
+ if self.try_accept("throws"):
throws = self.parse_qualified_identifier_list()
- if self.would_accept('{'):
+ if self.would_accept("{"):
body = self.parse_block()
else:
- self.accept(';')
+ self.accept(";")
- return tree.MethodDeclaration(parameters=parameters,
- throws=throws,
- body=body,
- return_type=tree.Type(dimensions=array_dimension))
+ return tree.MethodDeclaration(
+ parameters=parameters,
+ throws=throws,
+ body=body,
+ return_type=tree.Type(dimensions=array_dimension),
+ )
@parse_debug
def parse_void_interface_method_declarator_rest(self):
@@ -1070,17 +1092,15 @@ def parse_void_interface_method_declarator_rest(self):
throws = None
body = None
- if self.try_accept('throws'):
+ if self.try_accept("throws"):
throws = self.parse_qualified_identifier_list()
- if self.would_accept('{'):
+ if self.would_accept("{"):
body = self.parse_block()
else:
- self.accept(';')
+ self.accept(";")
- return tree.MethodDeclaration(parameters=parameters,
- throws=throws,
- body=body)
+ return tree.MethodDeclaration(parameters=parameters, throws=throws, body=body)
@parse_debug
def parse_interface_generic_method_declarator(self):
@@ -1088,7 +1108,7 @@ def parse_interface_generic_method_declarator(self):
return_type = None
method_name = None
- if not self.try_accept('void'):
+ if not self.try_accept("void"):
return_type = self.parse_type()
method_name = self.parse_identifier()
@@ -1099,16 +1119,16 @@ def parse_interface_generic_method_declarator(self):
return method
-# ------------------------------------------------------------------------------
-# -- Parameters and variables --
+ # ------------------------------------------------------------------------------
+ # -- Parameters and variables --
@parse_debug
def parse_formal_parameters(self):
formal_parameters = list()
- self.accept('(')
+ self.accept("(")
- if self.try_accept(')'):
+ if self.try_accept(")"):
return formal_parameters
while True:
@@ -1116,17 +1136,19 @@ def parse_formal_parameters(self):
parameter_type = self.parse_type()
varargs = False
- if self.try_accept('...'):
+ if self.try_accept("..."):
varargs = True
parameter_name = self.parse_identifier()
parameter_type.dimensions += self.parse_array_dimension()
- parameter = tree.FormalParameter(modifiers=modifiers,
- annotations=annotations,
- type=parameter_type,
- name=parameter_name,
- varargs=varargs)
+ parameter = tree.FormalParameter(
+ modifiers=modifiers,
+ annotations=annotations,
+ type=parameter_type,
+ name=parameter_name,
+ varargs=varargs,
+ )
formal_parameters.append(parameter)
@@ -1134,10 +1156,10 @@ def parse_formal_parameters(self):
# varargs parameter must be the last
break
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
- self.accept(')')
+ self.accept(")")
return formal_parameters
@@ -1147,8 +1169,8 @@ def parse_variable_modifiers(self):
annotations = list()
while True:
- if self.try_accept('final'):
- modifiers.add('final')
+ if self.try_accept("final"):
+ modifiers.add("final")
elif self.is_annotation():
annotation = self.parse_annotation()
annotations.append(annotation)
@@ -1165,7 +1187,7 @@ def parse_variable_declators(self):
declarator = self.parse_variable_declator()
declarators.append(declarator)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return declarators
@@ -1178,7 +1200,7 @@ def parse_variable_declarators(self):
declarator = self.parse_variable_declarator()
declarators.append(declarator)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return declarators
@@ -1188,23 +1210,23 @@ def parse_variable_declarator(self):
identifier = self.parse_identifier()
array_dimension, initializer = self.parse_variable_declarator_rest()
- return tree.VariableDeclarator(name=identifier,
- dimensions=array_dimension,
- initializer=initializer)
+ return tree.VariableDeclarator(
+ name=identifier, dimensions=array_dimension, initializer=initializer
+ )
@parse_debug
def parse_variable_declarator_rest(self):
array_dimension = self.parse_array_dimension()
initializer = None
- if self.try_accept('='):
+ if self.try_accept("="):
initializer = self.parse_variable_initializer()
return (array_dimension, initializer)
@parse_debug
def parse_variable_initializer(self):
- if self.would_accept('{'):
+ if self.would_accept("{"):
return self.parse_array_initializer()
else:
return self.parse_expression()
@@ -1213,48 +1235,48 @@ def parse_variable_initializer(self):
def parse_array_initializer(self):
array_initializer = tree.ArrayInitializer(initializers=list())
- self.accept('{')
+ self.accept("{")
- if self.try_accept(','):
- self.accept('}')
+ if self.try_accept(","):
+ self.accept("}")
return array_initializer
- if self.try_accept('}'):
+ if self.try_accept("}"):
return array_initializer
while True:
initializer = self.parse_variable_initializer()
array_initializer.initializers.append(initializer)
- if not self.would_accept('}'):
- self.accept(',')
+ if not self.would_accept("}"):
+ self.accept(",")
- if self.try_accept('}'):
+ if self.try_accept("}"):
return array_initializer
-# ------------------------------------------------------------------------------
-# -- Blocks and statements --
+ # ------------------------------------------------------------------------------
+ # -- Blocks and statements --
@parse_debug
def parse_block(self):
statements = list()
- self.accept('{')
+ self.accept("{")
- while not self.would_accept('}'):
+ while not self.would_accept("}"):
statement = self.parse_block_statement()
statements.append(statement)
- self.accept('}')
+ self.accept("}")
return statements
@parse_debug
def parse_block_statement(self):
- if self.would_accept(Identifier, ':'):
+ if self.would_accept(Identifier, ":"):
# Labeled statement
return self.parse_statement()
- if self.would_accept('synchronized'):
+ if self.would_accept("synchronized"):
return self.parse_statement()
token = None
@@ -1267,25 +1289,25 @@ def parse_block_statement(self):
token = self.tokens.look(i)
if isinstance(token, Modifier):
- if not token.value == 'final':
+ if not token.value == "final":
return self.parse_class_or_interface_declaration()
elif self.is_annotation(i):
found_annotations = True
i += 2
- while self.tokens.look(i).value == '.':
+ while self.tokens.look(i).value == ".":
i += 2
- if self.tokens.look(i).value == '(':
+ if self.tokens.look(i).value == "(":
parens = 1
i += 1
while parens > 0:
token = self.tokens.look(i)
- if token.value == '(':
+ if token.value == "(":
parens += 1
- elif token.value == ')':
+ elif token.value == ")":
parens -= 1
i += 1
continue
@@ -1295,7 +1317,7 @@ def parse_block_statement(self):
i += 1
- if token.value in ('class', 'enum', 'interface', '@'):
+ if token.value in ("class", "enum", "interface", "@"):
return self.parse_class_or_interface_declaration()
if found_annotations or isinstance(token, BasicType):
@@ -1320,147 +1342,145 @@ def parse_local_variable_declaration_statement(self):
modifiers, annotations = self.parse_variable_modifiers()
java_type = self.parse_type()
declarators = self.parse_variable_declarators()
- self.accept(';')
-
- var = tree.LocalVariableDeclaration(modifiers=modifiers,
- annotations=annotations,
- type=java_type,
- declarators=declarators)
+ self.accept(";")
+
+ var = tree.LocalVariableDeclaration(
+ modifiers=modifiers,
+ annotations=annotations,
+ type=java_type,
+ declarators=declarators,
+ )
return var
@parse_debug
def parse_statement(self):
token = self.tokens.look()
- if self.would_accept('{'):
+ if self.would_accept("{"):
block = self.parse_block()
return tree.BlockStatement(statements=block)
- elif self.try_accept(';'):
+ elif self.try_accept(";"):
return tree.Statement()
- elif self.would_accept(Identifier, ':'):
+ elif self.would_accept(Identifier, ":"):
identifer = self.parse_identifier()
- self.accept(':')
+ self.accept(":")
statement = self.parse_statement()
statement.label = identifer
return statement
- elif self.try_accept('if'):
+ elif self.try_accept("if"):
condition = self.parse_par_expression()
then = self.parse_statement()
else_statement = None
- if self.try_accept('else'):
+ if self.try_accept("else"):
else_statement = self.parse_statement()
- return tree.IfStatement(condition=condition,
- then_statement=then,
- else_statement=else_statement)
+ return tree.IfStatement(
+ condition=condition, then_statement=then, else_statement=else_statement
+ )
- elif self.try_accept('assert'):
+ elif self.try_accept("assert"):
condition = self.parse_expression()
value = None
- if self.try_accept(':'):
+ if self.try_accept(":"):
value = self.parse_expression()
- self.accept(';')
+ self.accept(";")
- return tree.AssertStatement(condition=condition,
- value=value)
+ return tree.AssertStatement(condition=condition, value=value)
- elif self.try_accept('switch'):
+ elif self.try_accept("switch"):
switch_expression = self.parse_par_expression()
- self.accept('{')
+ self.accept("{")
switch_block = self.parse_switch_block_statement_groups()
- self.accept('}')
+ self.accept("}")
- return tree.SwitchStatement(expression=switch_expression,
- cases=switch_block)
+ return tree.SwitchStatement(
+ expression=switch_expression, cases=switch_block
+ )
- elif self.try_accept('while'):
+ elif self.try_accept("while"):
condition = self.parse_par_expression()
action = self.parse_statement()
- return tree.WhileStatement(condition=condition,
- body=action)
+ return tree.WhileStatement(condition=condition, body=action)
- elif self.try_accept('do'):
+ elif self.try_accept("do"):
action = self.parse_statement()
- self.accept('while')
+ self.accept("while")
condition = self.parse_par_expression()
- self.accept(';')
+ self.accept(";")
- return tree.DoStatement(condition=condition,
- body=action)
+ return tree.DoStatement(condition=condition, body=action)
- elif self.try_accept('for'):
- self.accept('(')
+ elif self.try_accept("for"):
+ self.accept("(")
for_control = self.parse_for_control()
- self.accept(')')
+ self.accept(")")
for_statement = self.parse_statement()
- return tree.ForStatement(control=for_control,
- body=for_statement)
+ return tree.ForStatement(control=for_control, body=for_statement)
- elif self.try_accept('break'):
+ elif self.try_accept("break"):
label = None
if self.would_accept(Identifier):
label = self.parse_identifier()
- self.accept(';')
+ self.accept(";")
return tree.BreakStatement(goto=label)
- elif self.try_accept('continue'):
+ elif self.try_accept("continue"):
label = None
if self.would_accept(Identifier):
label = self.parse_identifier()
- self.accept(';')
+ self.accept(";")
return tree.ContinueStatement(goto=label)
- elif self.try_accept('return'):
+ elif self.try_accept("return"):
value = None
- if not self.would_accept(';'):
+ if not self.would_accept(";"):
value = self.parse_expression()
- self.accept(';')
+ self.accept(";")
return tree.ReturnStatement(expression=value)
- elif self.try_accept('throw'):
+ elif self.try_accept("throw"):
value = self.parse_expression()
- self.accept(';')
+ self.accept(";")
return tree.ThrowStatement(expression=value)
- elif self.try_accept('synchronized'):
+ elif self.try_accept("synchronized"):
lock = self.parse_par_expression()
block = self.parse_block()
- return tree.SynchronizedStatement(lock=lock,
- block=block)
+ return tree.SynchronizedStatement(lock=lock, block=block)
- elif self.try_accept('try'):
+ elif self.try_accept("try"):
resource_specification = None
block = None
catches = None
finally_block = None
- if self.would_accept('{'):
+ if self.would_accept("{"):
block = self.parse_block()
- if self.would_accept('catch'):
+ if self.would_accept("catch"):
catches = self.parse_catches()
- if self.try_accept('finally'):
+ if self.try_accept("finally"):
finally_block = self.parse_block()
if catches == None and finally_block == None:
@@ -1470,25 +1490,27 @@ def parse_statement(self):
resource_specification = self.parse_resource_specification()
block = self.parse_block()
- if self.would_accept('catch'):
+ if self.would_accept("catch"):
catches = self.parse_catches()
- if self.try_accept('finally'):
+ if self.try_accept("finally"):
finally_block = self.parse_block()
- return tree.TryStatement(resources=resource_specification,
- block=block,
- catches=catches,
- finally_block=finally_block)
+ return tree.TryStatement(
+ resources=resource_specification,
+ block=block,
+ catches=catches,
+ finally_block=finally_block,
+ )
else:
expression = self.parse_expression()
- self.accept(';')
+ self.accept(";")
return tree.StatementExpression(expression=expression)
-# ------------------------------------------------------------------------------
-# -- Try / catch --
+ # ------------------------------------------------------------------------------
+ # -- Try / catch --
@parse_debug
def parse_catches(self):
@@ -1498,14 +1520,14 @@ def parse_catches(self):
catch = self.parse_catch_clause()
catches.append(catch)
- if not self.would_accept('catch'):
+ if not self.would_accept("catch"):
break
return catches
@parse_debug
def parse_catch_clause(self):
- self.accept('catch', '(')
+ self.accept("catch", "(")
modifiers, annotations = self.parse_variable_modifiers()
catch_parameter = tree.CatchClauseParameter(types=list())
@@ -1514,30 +1536,29 @@ def parse_catch_clause(self):
catch_type = self.parse_qualified_identifier()
catch_parameter.types.append(catch_type)
- if not self.try_accept('|'):
+ if not self.try_accept("|"):
break
catch_parameter.name = self.parse_identifier()
- self.accept(')')
+ self.accept(")")
block = self.parse_block()
- return tree.CatchClause(parameter=catch_parameter,
- block=block)
+ return tree.CatchClause(parameter=catch_parameter, block=block)
@parse_debug
def parse_resource_specification(self):
resources = list()
- self.accept('(')
+ self.accept("(")
while True:
resource = self.parse_resource()
resources.append(resource)
- if not self.would_accept(')'):
- self.accept(';')
+ if not self.would_accept(")"):
+ self.accept(";")
- if self.try_accept(')'):
+ if self.try_accept(")"):
break
return resources
@@ -1549,23 +1570,25 @@ def parse_resource(self):
reference_type.dimensions = self.parse_array_dimension()
name = self.parse_identifier()
reference_type.dimensions += self.parse_array_dimension()
- self.accept('=')
+ self.accept("=")
value = self.parse_expression()
- return tree.TryResource(modifiers=modifiers,
- annotations=annotations,
- type=reference_type,
- name=name,
- value=value)
+ return tree.TryResource(
+ modifiers=modifiers,
+ annotations=annotations,
+ type=reference_type,
+ name=name,
+ value=value,
+ )
-# ------------------------------------------------------------------------------
-# -- Switch and for statements ---
+ # ------------------------------------------------------------------------------
+ # -- Switch and for statements ---
@parse_debug
def parse_switch_block_statement_groups(self):
statement_groups = list()
- while self.tokens.look().value in ('case', 'default'):
+ while self.tokens.look().value in ("case", "default"):
statement_group = self.parse_switch_block_statement_group()
statement_groups.append(statement_group)
@@ -1580,27 +1603,26 @@ def parse_switch_block_statement_group(self):
case_type = self.tokens.next().value
case_value = None
- if case_type == 'case':
- if self.would_accept(Identifier, ':'):
+ if case_type == "case":
+ if self.would_accept(Identifier, ":"):
case_value = self.parse_identifier()
else:
case_value = self.parse_expression()
labels.append(case_value)
- elif not case_type == 'default':
+ elif not case_type == "default":
self.illegal("Expected switch case")
- self.accept(':')
+ self.accept(":")
- if self.tokens.look().value not in ('case', 'default'):
+ if self.tokens.look().value not in ("case", "default"):
break
- while self.tokens.look().value not in ('case', 'default', '}'):
+ while self.tokens.look().value not in ("case", "default", "}"):
statement = self.parse_block_statement()
statements.append(statement)
- return tree.SwitchStatementCase(case=labels,
- statements=statements)
+ return tree.SwitchStatementCase(case=labels, statements=statements)
@parse_debug
def parse_for_control(self):
@@ -1613,24 +1635,22 @@ def parse_for_control(self):
pass
init = None
- if not self.would_accept(';'):
+ if not self.would_accept(";"):
init = self.parse_for_init_or_update()
- self.accept(';')
+ self.accept(";")
condition = None
- if not self.would_accept(';'):
+ if not self.would_accept(";"):
condition = self.parse_expression()
- self.accept(';')
+ self.accept(";")
update = None
- if not self.would_accept(')'):
+ if not self.would_accept(")"):
update = self.parse_for_init_or_update()
- return tree.ForControl(init=init,
- condition=condition,
- update=update)
+ return tree.ForControl(init=init, condition=condition, update=update)
@parse_debug
def parse_for_var_control(self):
@@ -1639,44 +1659,41 @@ def parse_for_var_control(self):
var_name = self.parse_identifier()
var_type.dimensions += self.parse_array_dimension()
- var = tree.VariableDeclaration(modifiers=modifiers,
- annotations=annotations,
- type=var_type)
+ var = tree.VariableDeclaration(
+ modifiers=modifiers, annotations=annotations, type=var_type
+ )
rest = self.parse_for_var_control_rest()
if isinstance(rest, tree.Expression):
var.declarators = [tree.VariableDeclarator(name=var_name)]
- return tree.EnhancedForControl(var=var,
- iterable=rest)
+ return tree.EnhancedForControl(var=var, iterable=rest)
else:
declarators, condition, update = rest
declarators[0].name = var_name
var.declarators = declarators
- return tree.ForControl(init=var,
- condition=condition,
- update=update)
+ return tree.ForControl(init=var, condition=condition, update=update)
@parse_debug
def parse_for_var_control_rest(self):
- if self.try_accept(':'):
+ if self.try_accept(":"):
expression = self.parse_expression()
return expression
declarators = None
- if not self.would_accept(';'):
+ if not self.would_accept(";"):
declarators = self.parse_for_variable_declarator_rest()
else:
declarators = [tree.VariableDeclarator()]
- self.accept(';')
+ self.accept(";")
condition = None
- if not self.would_accept(';'):
+ if not self.would_accept(";"):
condition = self.parse_expression()
- self.accept(';')
+ self.accept(";")
update = None
- if not self.would_accept(')'):
+ if not self.would_accept(")"):
update = self.parse_for_init_or_update()
return (declarators, condition, update)
@@ -1685,12 +1702,12 @@ def parse_for_var_control_rest(self):
def parse_for_variable_declarator_rest(self):
initializer = None
- if self.try_accept('='):
+ if self.try_accept("="):
initializer = self.parse_variable_initializer()
declarators = [tree.VariableDeclarator(initializer=initializer)]
- while self.try_accept(','):
+ while self.try_accept(","):
declarator = self.parse_variable_declarator()
declarators.append(declarator)
@@ -1704,13 +1721,13 @@ def parse_for_init_or_update(self):
expression = self.parse_expression()
expressions.append(expression)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
return expressions
-# ------------------------------------------------------------------------------
-# -- Expressions --
+ # ------------------------------------------------------------------------------
+ # -- Expressions --
@parse_debug
def parse_expression(self):
@@ -1721,9 +1738,11 @@ def parse_expression(self):
if self.tokens.look().value in Operator.ASSIGNMENT:
assignment_type = self.tokens.next().value
assignment_expression = self.parse_expression()
- return tree.Assignment(expressionl=expressionl,
- type=assignment_type,
- value=assignment_expression)
+ return tree.Assignment(
+ expressionl=expressionl,
+ type=assignment_type,
+ value=assignment_expression,
+ )
else:
return expressionl
@@ -1733,31 +1752,33 @@ def parse_expressionl(self):
true_expression = None
false_expression = None
- if self.try_accept('?'):
+ if self.try_accept("?"):
true_expression = self.parse_expression()
- self.accept(':')
+ self.accept(":")
false_expression = self.parse_expressionl()
- return tree.TernaryExpression(condition=expression_2,
- if_true=true_expression,
- if_false=false_expression)
- if self.would_accept('->'):
+ return tree.TernaryExpression(
+ condition=expression_2,
+ if_true=true_expression,
+ if_false=false_expression,
+ )
+ if self.would_accept("->"):
body = self.parse_lambda_method_body()
- return tree.LambdaExpression(parameters=[expression_2],
- body=body)
- if self.try_accept('::'):
+ return tree.LambdaExpression(parameters=[expression_2], body=body)
+ if self.try_accept("::"):
method_reference, type_arguments = self.parse_method_reference()
return tree.MethodReference(
expression=expression_2,
method=method_reference,
- type_arguments=type_arguments)
+ type_arguments=type_arguments,
+ )
return expression_2
@parse_debug
def parse_expression_2(self):
expression_3 = self.parse_expression_3()
token = self.tokens.look()
- if token.value in Operator.INFIX or token.value == 'instanceof':
+ if token.value in Operator.INFIX or token.value == "instanceof":
parts = self.parse_expression_2_rest()
parts.insert(0, expression_3)
return self.build_binary_operation(parts)
@@ -1769,10 +1790,10 @@ def parse_expression_2_rest(self):
parts = list()
token = self.tokens.look()
- while token.value in Operator.INFIX or token.value == 'instanceof':
- if self.try_accept('instanceof'):
+ while token.value in Operator.INFIX or token.value == "instanceof":
+ if self.try_accept("instanceof"):
comparison_type = self.parse_type()
- parts.extend(('instanceof', comparison_type))
+ parts.extend(("instanceof", comparison_type))
else:
operator = self.parse_infix_operator()
expression = self.parse_expression_3()
@@ -1782,8 +1803,8 @@ def parse_expression_2_rest(self):
return parts
-# ------------------------------------------------------------------------------
-# -- Expression operators --
+ # ------------------------------------------------------------------------------
+ # -- Expression operators --
@parse_debug
def parse_expression_3(self):
@@ -1791,23 +1812,22 @@ def parse_expression_3(self):
while self.tokens.look().value in Operator.PREFIX:
prefix_operators.append(self.tokens.next().value)
- if self.would_accept('('):
+ if self.would_accept("("):
try:
with self.tokens:
- lambda_exp = self.parse_lambda_expression()
- if lambda_exp:
- return lambda_exp
+ lambda_exp = self.parse_lambda_expression()
+ if lambda_exp:
+ return lambda_exp
except JavaSyntaxError:
pass
try:
with self.tokens:
- self.accept('(')
+ self.accept("(")
cast_target = self.parse_type()
- self.accept(')')
+ self.accept(")")
expression = self.parse_expression_3()
- return tree.Cast(type=cast_target,
- expression=expression)
+ return tree.Cast(type=cast_target, expression=expression)
except JavaSyntaxError:
pass
@@ -1817,7 +1837,7 @@ def parse_expression_3(self):
primary.postfix_operators = list()
token = self.tokens.look()
- while token.value in '[.':
+ while token.value in "[.":
selector = self.parse_selector()
primary.selectors.append(selector)
@@ -1832,10 +1852,10 @@ def parse_expression_3(self):
@parse_debug
def parse_method_reference(self):
type_arguments = list()
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_arguments = self.parse_nonwildcard_type_arguments()
- if self.would_accept('new'):
- method_reference = tree.MemberReference(member=self.accept('new'))
+ if self.would_accept("new"):
+ method_reference = tree.MemberReference(member=self.accept("new"))
else:
method_reference = self.parse_expression()
return method_reference, type_arguments
@@ -1844,24 +1864,24 @@ def parse_method_reference(self):
def parse_lambda_expression(self):
lambda_expr = None
parameters = None
- if self.would_accept('(', Identifier, ','):
- self.accept('(')
+ if self.would_accept("(", Identifier, ","):
+ self.accept("(")
parameters = []
- while not self.would_accept(')'):
- parameters.append(tree.InferredFormalParameter(
- name=self.parse_identifier()))
- self.try_accept(',')
- self.accept(')')
+ while not self.would_accept(")"):
+ parameters.append(
+ tree.InferredFormalParameter(name=self.parse_identifier())
+ )
+ self.try_accept(",")
+ self.accept(")")
else:
parameters = self.parse_formal_parameters()
body = self.parse_lambda_method_body()
- return tree.LambdaExpression(parameters=parameters,
- body=body)
+ return tree.LambdaExpression(parameters=parameters, body=body)
@parse_debug
def parse_lambda_method_body(self):
- if self.accept('->'):
- if self.would_accept('{'):
+ if self.accept("->"):
+ if self.would_accept("{"):
return self.parse_block()
else:
return self.parse_expression()
@@ -1873,16 +1893,16 @@ def parse_infix_operator(self):
if not operator in Operator.INFIX:
self.illegal("Expected infix operator")
- if operator == '>' and self.try_accept('>'):
- operator = '>>'
+ if operator == ">" and self.try_accept(">"):
+ operator = ">>"
- if self.try_accept('>'):
- operator = '>>>'
+ if self.try_accept(">"):
+ operator = ">>>"
return operator
-# ------------------------------------------------------------------------------
-# -- Primary expressions --
+ # ------------------------------------------------------------------------------
+ # -- Primary expressions --
@parse_debug
def parse_primary(self):
@@ -1891,34 +1911,35 @@ def parse_primary(self):
if isinstance(token, Literal):
return self.parse_literal()
- elif token.value == '(':
+ elif token.value == "(":
return self.parse_par_expression()
- elif self.try_accept('this'):
+ elif self.try_accept("this"):
arguments = None
- if self.would_accept('('):
+ if self.would_accept("("):
arguments = self.parse_arguments()
return tree.ExplicitConstructorInvocation(arguments=arguments)
return tree.This()
- elif self.would_accept('super', '::'):
- self.accept('super')
+ elif self.would_accept("super", "::"):
+ self.accept("super")
return token
- elif self.try_accept('super'):
+ elif self.try_accept("super"):
super_suffix = self.parse_super_suffix()
return super_suffix
- elif self.try_accept('new'):
+ elif self.try_accept("new"):
return self.parse_creator()
- elif token.value == '<':
+ elif token.value == "<":
type_arguments = self.parse_nonwildcard_type_arguments()
- if self.try_accept('this'):
+ if self.try_accept("this"):
arguments = self.parse_arguments()
- return tree.ExplicitConstructorInvocation(type_arguments=type_arguments,
- arguments=arguments)
+ return tree.ExplicitConstructorInvocation(
+ type_arguments=type_arguments, arguments=arguments
+ )
else:
invocation = self.parse_explicit_generic_invocation_suffix()
invocation.type_arguments = type_arguments
@@ -1928,33 +1949,37 @@ def parse_primary(self):
elif isinstance(token, Identifier):
qualified_identifier = [self.parse_identifier()]
- while self.would_accept('.', Identifier):
- self.accept('.')
+ while self.would_accept(".", Identifier):
+ self.accept(".")
identifier = self.parse_identifier()
qualified_identifier.append(identifier)
identifier_suffix = self.parse_identifier_suffix()
- if isinstance(identifier_suffix, (tree.MemberReference, tree.MethodInvocation)):
+ if isinstance(
+ identifier_suffix, (tree.MemberReference, tree.MethodInvocation)
+ ):
# Take the last identifer as the member and leave the rest for the qualifier
identifier_suffix.member = qualified_identifier.pop()
elif isinstance(identifier_suffix, tree.ClassReference):
- identifier_suffix.type = tree.ReferenceType(name=qualified_identifier.pop())
+ identifier_suffix.type = tree.ReferenceType(
+ name=qualified_identifier.pop()
+ )
- identifier_suffix.qualifier = '.'.join(qualified_identifier)
+ identifier_suffix.qualifier = ".".join(qualified_identifier)
return identifier_suffix
elif isinstance(token, BasicType):
base_type = self.parse_basic_type()
base_type.dimensions = self.parse_array_dimension()
- self.accept('.', 'class')
+ self.accept(".", "class")
return tree.ClassReference(type=base_type)
- elif self.try_accept('void'):
- self.accept('.', 'class')
+ elif self.try_accept("void"):
+ self.accept(".", "class")
return tree.VoidClassReference()
self.illegal("Expected expression")
@@ -1966,9 +1991,9 @@ def parse_literal(self):
@parse_debug
def parse_par_expression(self):
- self.accept('(')
+ self.accept("(")
expression = self.parse_expression()
- self.accept(')')
+ self.accept(")")
return expression
@@ -1976,19 +2001,19 @@ def parse_par_expression(self):
def parse_arguments(self):
expressions = list()
- self.accept('(')
+ self.accept("(")
- if self.try_accept(')'):
+ if self.try_accept(")"):
return expressions
while True:
expression = self.parse_expression()
expressions.append(expression)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
- self.accept(')')
+ self.accept(")")
return expressions
@@ -1998,21 +2023,21 @@ def parse_super_suffix(self):
type_arguments = None
arguments = None
- if self.try_accept('.'):
- if self.would_accept('<'):
+ if self.try_accept("."):
+ if self.would_accept("<"):
type_arguments = self.parse_nonwildcard_type_arguments()
identifier = self.parse_identifier()
- if self.would_accept('('):
+ if self.would_accept("("):
arguments = self.parse_arguments()
else:
arguments = self.parse_arguments()
if identifier and arguments is not None:
- return tree.SuperMethodInvocation(member=identifier,
- arguments=arguments,
- type_arguments=type_arguments)
+ return tree.SuperMethodInvocation(
+ member=identifier, arguments=arguments, type_arguments=type_arguments
+ )
elif arguments is not None:
return tree.SuperConstructorInvocation(arguments=arguments)
else:
@@ -2022,16 +2047,15 @@ def parse_super_suffix(self):
def parse_explicit_generic_invocation_suffix(self):
identifier = None
arguments = None
- if self.try_accept('super'):
+ if self.try_accept("super"):
return self.parse_super_suffix()
else:
identifier = self.parse_identifier()
arguments = self.parse_arguments()
- return tree.MethodInvocation(member=identifier,
- arguments=arguments)
+ return tree.MethodInvocation(member=identifier, arguments=arguments)
-# ------------------------------------------------------------------------------
-# -- Creators --
+ # ------------------------------------------------------------------------------
+ # -- Creators --
@parse_debug
def parse_creator(self):
@@ -2043,24 +2067,28 @@ def parse_creator(self):
rest.type = created_name
return rest
- if self.would_accept('<'):
+ if self.would_accept("<"):
constructor_type_arguments = self.parse_nonwildcard_type_arguments()
created_name = self.parse_created_name()
- if self.would_accept('['):
+ if self.would_accept("["):
if constructor_type_arguments:
- self.illegal("Array creator not allowed with generic constructor type arguments")
+ self.illegal(
+ "Array creator not allowed with generic constructor type arguments"
+ )
rest = self.parse_array_creator_rest()
rest.type = created_name
return rest
else:
arguments, body = self.parse_class_creator_rest()
- return tree.ClassCreator(constructor_type_arguments=constructor_type_arguments,
- type=created_name,
- arguments=arguments,
- body=body)
+ return tree.ClassCreator(
+ constructor_type_arguments=constructor_type_arguments,
+ type=created_name,
+ arguments=arguments,
+ body=body,
+ )
@parse_debug
def parse_created_name(self):
@@ -2070,10 +2098,10 @@ def parse_created_name(self):
while True:
tail.name = self.parse_identifier()
- if self.would_accept('<'):
+ if self.would_accept("<"):
tail.arguments = self.parse_type_arguments_or_diamond()
- if self.try_accept('.'):
+ if self.try_accept("."):
tail.sub_type = tree.ReferenceType()
tail = tail.sub_type
else:
@@ -2086,57 +2114,58 @@ def parse_class_creator_rest(self):
arguments = self.parse_arguments()
class_body = None
- if self.would_accept('{'):
+ if self.would_accept("{"):
class_body = self.parse_class_body()
return (arguments, class_body)
@parse_debug
def parse_array_creator_rest(self):
- if self.would_accept('[', ']'):
+ if self.would_accept("[", "]"):
array_dimension = self.parse_array_dimension()
array_initializer = self.parse_array_initializer()
- return tree.ArrayCreator(dimensions=array_dimension,
- initializer=array_initializer)
+ return tree.ArrayCreator(
+ dimensions=array_dimension, initializer=array_initializer
+ )
else:
array_dimensions = list()
- while self.would_accept('[') and not self.would_accept('[', ']'):
- self.accept('[')
+ while self.would_accept("[") and not self.would_accept("[", "]"):
+ self.accept("[")
expression = self.parse_expression()
array_dimensions.append(expression)
- self.accept(']')
+ self.accept("]")
array_dimensions += self.parse_array_dimension()
return tree.ArrayCreator(dimensions=array_dimensions)
@parse_debug
def parse_identifier_suffix(self):
- if self.try_accept('[', ']'):
+ if self.try_accept("[", "]"):
array_dimension = [None] + self.parse_array_dimension()
- self.accept('.', 'class')
+ self.accept(".", "class")
return tree.ClassReference(type=tree.Type(dimensions=array_dimension))
- elif self.would_accept('('):
+ elif self.would_accept("("):
arguments = self.parse_arguments()
return tree.MethodInvocation(arguments=arguments)
- elif self.try_accept('.', 'class'):
+ elif self.try_accept(".", "class"):
return tree.ClassReference()
- elif self.try_accept('.', 'this'):
+ elif self.try_accept(".", "this"):
return tree.This()
- elif self.would_accept('.', '<'):
+ elif self.would_accept(".", "<"):
next(self.tokens)
return self.parse_explicit_generic_invocation()
- elif self.try_accept('.', 'new'):
+ elif self.try_accept(".", "new"):
type_arguments = None
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_arguments = self.parse_nonwildcard_type_arguments()
inner_creator = self.parse_inner_creator()
@@ -2144,8 +2173,8 @@ def parse_identifier_suffix(self):
return inner_creator
- elif self.would_accept('.', 'super', '('):
- self.accept('.', 'super')
+ elif self.would_accept(".", "super", "("):
+ self.accept(".", "super")
arguments = self.parse_arguments()
return tree.SuperConstructorInvocation(arguments=arguments)
@@ -2166,52 +2195,50 @@ def parse_inner_creator(self):
identifier = self.parse_identifier()
type_arguments = None
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_arguments = self.parse_nonwildcard_type_arguments_or_diamond()
- java_type = tree.ReferenceType(name=identifier,
- arguments=type_arguments)
+ java_type = tree.ReferenceType(name=identifier, arguments=type_arguments)
arguments, class_body = self.parse_class_creator_rest()
- return tree.InnerClassCreator(type=java_type,
- arguments=arguments,
- body=class_body)
+ return tree.InnerClassCreator(
+ type=java_type, arguments=arguments, body=class_body
+ )
@parse_debug
def parse_selector(self):
- if self.try_accept('['):
+ if self.try_accept("["):
expression = self.parse_expression()
- self.accept(']')
+ self.accept("]")
return tree.ArraySelector(index=expression)
- elif self.try_accept('.'):
+ elif self.try_accept("."):
token = self.tokens.look()
if isinstance(token, Identifier):
identifier = self.tokens.next().value
arguments = None
- if self.would_accept('('):
+ if self.would_accept("("):
arguments = self.parse_arguments()
- return tree.MethodInvocation(member=identifier,
- arguments=arguments)
+ return tree.MethodInvocation(member=identifier, arguments=arguments)
else:
return tree.MemberReference(member=identifier)
- elif self.would_accept('super', '::'):
- self.accept('super')
+ elif self.would_accept("super", "::"):
+ self.accept("super")
return token
- elif self.would_accept('<'):
+ elif self.would_accept("<"):
return self.parse_explicit_generic_invocation()
- elif self.try_accept('this'):
+ elif self.try_accept("this"):
return tree.This()
- elif self.try_accept('super'):
+ elif self.try_accept("super"):
return self.parse_super_suffix()
- elif self.try_accept('new'):
+ elif self.try_accept("new"):
type_arguments = None
- if self.would_accept('<'):
+ if self.would_accept("<"):
type_arguments = self.parse_nonwildcard_type_arguments()
inner_creator = self.parse_inner_creator()
@@ -2221,35 +2248,34 @@ def parse_selector(self):
self.illegal("Expected selector")
-# ------------------------------------------------------------------------------
-# -- Enum and annotation body --
+ # ------------------------------------------------------------------------------
+ # -- Enum and annotation body --
@parse_debug
def parse_enum_body(self):
constants = list()
body_declarations = list()
- self.accept('{')
+ self.accept("{")
- if not self.try_accept(','):
- while not (self.would_accept(';') or self.would_accept('}')):
+ if not self.try_accept(","):
+ while not (self.would_accept(";") or self.would_accept("}")):
constant = self.parse_enum_constant()
constants.append(constant)
- if not self.try_accept(','):
+ if not self.try_accept(","):
break
- if self.try_accept(';'):
- while not self.would_accept('}'):
+ if self.try_accept(";"):
+ while not self.would_accept("}"):
declaration = self.parse_class_body_declaration()
if declaration:
body_declarations.append(declaration)
- self.accept('}')
+ self.accept("}")
- return tree.EnumBody(constants=constants,
- declarations=body_declarations)
+ return tree.EnumBody(constants=constants, declarations=body_declarations)
@parse_debug
def parse_enum_constant(self):
@@ -2268,25 +2294,27 @@ def parse_enum_constant(self):
constant_name = self.parse_identifier()
- if self.would_accept('('):
+ if self.would_accept("("):
arguments = self.parse_arguments()
- if self.would_accept('{'):
+ if self.would_accept("{"):
body = self.parse_class_body()
- return tree.EnumConstantDeclaration(annotations=annotations,
- name=constant_name,
- arguments=arguments,
- body=body,
- documentation=javadoc)
+ return tree.EnumConstantDeclaration(
+ annotations=annotations,
+ name=constant_name,
+ arguments=arguments,
+ body=body,
+ documentation=javadoc,
+ )
@parse_debug
def parse_annotation_type_body(self):
declarations = None
- self.accept('{')
+ self.accept("{")
declarations = self.parse_annotation_type_element_declarations()
- self.accept('}')
+ self.accept("}")
return declarations
@@ -2294,7 +2322,7 @@ def parse_annotation_type_body(self):
def parse_annotation_type_element_declarations(self):
declarations = list()
- while not self.would_accept('}'):
+ while not self.would_accept("}"):
declaration = self.parse_annotation_type_element_declaration()
declarations.append(declaration)
@@ -2305,11 +2333,11 @@ def parse_annotation_type_element_declaration(self):
modifiers, annotations, javadoc = self.parse_modifiers()
declaration = None
- if self.would_accept('class'):
+ if self.would_accept("class"):
declaration = self.parse_normal_class_declaration()
- elif self.would_accept('interface'):
+ elif self.would_accept("interface"):
declaration = self.parse_normal_interface_declaration()
- elif self.would_accept('enum'):
+ elif self.would_accept("enum"):
declaration = self.parse_enum_declaration()
elif self.is_annotation_declaration():
declaration = self.parse_annotation_type_declaration()
@@ -2317,7 +2345,7 @@ def parse_annotation_type_element_declaration(self):
attribute_type = self.parse_type()
attribute_name = self.parse_identifier()
declaration = self.parse_annotation_method_or_constant_rest()
- self.accept(';')
+ self.accept(";")
if isinstance(declaration, tree.AnnotationMethod):
declaration.name = attribute_name
@@ -2334,20 +2362,20 @@ def parse_annotation_type_element_declaration(self):
@parse_debug
def parse_annotation_method_or_constant_rest(self):
- if self.try_accept('('):
- self.accept(')')
+ if self.try_accept("("):
+ self.accept(")")
array_dimension = self.parse_array_dimension()
default = None
- if self.try_accept('default'):
+ if self.try_accept("default"):
default = self.parse_element_value()
- return tree.AnnotationMethod(dimensions=array_dimension,
- default=default)
+ return tree.AnnotationMethod(dimensions=array_dimension, default=default)
else:
return self.parse_constant_declarators_rest()
+
def parse(tokens, debug=False):
parser = Parser(tokens)
parser.set_debug(debug)
diff --git a/baseline_tokenization/javalang/test/test_java_8_syntax.py b/baseline_tokenization/javalang/test/test_java_8_syntax.py
index 0a8c8fd..c95507f 100644
--- a/baseline_tokenization/javalang/test/test_java_8_syntax.py
+++ b/baseline_tokenization/javalang/test/test_java_8_syntax.py
@@ -5,8 +5,8 @@
def setup_java_class(content_to_add):
- """ returns an example java class with the
- given content_to_add contained within a method.
+ """returns an example java class with the
+ given content_to_add contained within a method.
"""
template = """
public class Lambda {
@@ -20,8 +20,8 @@ def setup_java_class(content_to_add):
def filter_type_in_method(clazz, the_type, method_name):
- """ yields the result of filtering the given class for the given
- type inside the given method identified by its name.
+ """yields the result of filtering the given class for the given
+ type inside the given method identified by its name.
"""
for path, node in clazz.filter(the_type):
for p in reversed(path):
@@ -32,27 +32,26 @@ def filter_type_in_method(clazz, the_type, method_name):
class LambdaSupportTest(unittest.TestCase):
- """ Contains tests for java 8 lambda syntax. """
+ """Contains tests for java 8 lambda syntax."""
- def assert_contains_lambda_expression_in_m(
- self, clazz, method_name='main'):
- """ asserts that the given tree contains a method with the supplied
- method name containing a lambda expression.
+ def assert_contains_lambda_expression_in_m(self, clazz, method_name="main"):
+ """asserts that the given tree contains a method with the supplied
+ method name containing a lambda expression.
"""
- matches = list(filter_type_in_method(
- clazz, tree.LambdaExpression, method_name))
+ matches = list(filter_type_in_method(clazz, tree.LambdaExpression, method_name))
if not matches:
- self.fail('No matching lambda expression found.')
+ self.fail("No matching lambda expression found.")
return matches
def test_lambda_support_no_parameters_no_body(self):
- """ tests support for lambda with no parameters and no body. """
+ """tests support for lambda with no parameters and no body."""
self.assert_contains_lambda_expression_in_m(
- parse.parse(setup_java_class("() -> {};")))
+ parse.parse(setup_java_class("() -> {};"))
+ )
def test_lambda_support_no_parameters_expression_body(self):
- """ tests support for lambda with no parameters and an
- expression body.
+ """tests support for lambda with no parameters and an
+ expression body.
"""
test_classes = [
setup_java_class("() -> 3;"),
@@ -65,8 +64,8 @@ def test_lambda_support_no_parameters_expression_body(self):
self.assert_contains_lambda_expression_in_m(clazz)
def test_lambda_support_no_parameters_complex_expression(self):
- """ tests support for lambda with no parameters and a
- complex expression body.
+ """tests support for lambda with no parameters and a
+ complex expression body.
"""
code = """
() -> {
@@ -77,11 +76,10 @@ def test_lambda_support_no_parameters_complex_expression(self):
return result / 2;
}
};"""
- self.assert_contains_lambda_expression_in_m(
- parse.parse(setup_java_class(code)))
+ self.assert_contains_lambda_expression_in_m(parse.parse(setup_java_class(code)))
def test_parameter_no_type_expression_body(self):
- """ tests support for lambda with parameters with inferred types. """
+ """tests support for lambda with parameters with inferred types."""
test_classes = [
setup_java_class("(bar) -> bar + 1;"),
setup_java_class("bar -> bar + 1;"),
@@ -93,123 +91,131 @@ def test_parameter_no_type_expression_body(self):
self.assert_contains_lambda_expression_in_m(clazz)
def test_parameter_with_type_expression_body(self):
- """ tests support for lambda with parameters with formal types. """
+ """tests support for lambda with parameters with formal types."""
test_classes = [
setup_java_class("(int foo) -> { return foo + 2; };"),
setup_java_class("(String s) -> s.length();"),
setup_java_class("(int foo) -> foo + 1;"),
setup_java_class("(Thread th) -> { th.start(); };"),
- setup_java_class("(String foo, String bar) -> "
- "foo + bar;"),
+ setup_java_class("(String foo, String bar) -> " "foo + bar;"),
]
for test_class in test_classes:
clazz = parse.parse(test_class)
self.assert_contains_lambda_expression_in_m(clazz)
def test_parameters_with_no_type_expression_body(self):
- """ tests support for multiple lambda parameters
- that are specified without their types.
+ """tests support for multiple lambda parameters
+ that are specified without their types.
"""
self.assert_contains_lambda_expression_in_m(
- parse.parse(setup_java_class("(x, y) -> x + y;")))
+ parse.parse(setup_java_class("(x, y) -> x + y;"))
+ )
def test_parameters_with_mixed_inferred_and_declared_types(self):
- """ this tests that lambda type specification mixing is considered
- invalid as per the specifications.
+ """this tests that lambda type specification mixing is considered
+ invalid as per the specifications.
"""
with self.assertRaises(parser.JavaSyntaxError):
parse.parse(setup_java_class("(x, int y) -> x+y;"))
def test_parameters_inferred_types_with_modifiers(self):
- """ this tests that lambda inferred type parameters with modifiers are
- considered invalid as per the specifications.
+ """this tests that lambda inferred type parameters with modifiers are
+ considered invalid as per the specifications.
"""
with self.assertRaises(parser.JavaSyntaxError):
parse.parse(setup_java_class("(x, final y) -> x+y;"))
def test_invalid_parameters_are_invalid(self):
- """ this tests that invalid lambda parameters are are
- considered invalid as per the specifications.
+ """this tests that invalid lambda parameters are are
+ considered invalid as per the specifications.
"""
with self.assertRaises(parser.JavaSyntaxError):
parse.parse(setup_java_class("(a b c) -> {};"))
def test_cast_works(self):
- """ this tests that a cast expression works as expected. """
+ """this tests that a cast expression works as expected."""
parse.parse(setup_java_class("String x = (String) A.x() ;"))
class MethodReferenceSyntaxTest(unittest.TestCase):
- """ Contains tests for java 8 method reference syntax. """
+ """Contains tests for java 8 method reference syntax."""
def assert_contains_method_reference_expression_in_m(
- self, clazz, method_name='main'):
- """ asserts that the given class contains a method with the supplied
- method name containing a method reference.
+ self, clazz, method_name="main"
+ ):
+ """asserts that the given class contains a method with the supplied
+ method name containing a method reference.
"""
- matches = list(filter_type_in_method(
- clazz, tree.MethodReference, method_name))
+ matches = list(filter_type_in_method(clazz, tree.MethodReference, method_name))
if not matches:
- self.fail('No matching method reference found.')
+ self.fail("No matching method reference found.")
return matches
def test_method_reference(self):
- """ tests that method references are supported. """
+ """tests that method references are supported."""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("String::length;")))
+ parse.parse(setup_java_class("String::length;"))
+ )
def test_method_reference_to_the_new_method(self):
- """ test support for method references to 'new'. """
+ """test support for method references to 'new'."""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("String::new;")))
+ parse.parse(setup_java_class("String::new;"))
+ )
def test_method_reference_to_the_new_method_with_explict_type(self):
- """ test support for method references to 'new' with an
- explicit type.
+ """test support for method references to 'new' with an
+ explicit type.
"""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("String:: new;")))
+ parse.parse(setup_java_class("String:: new;"))
+ )
def test_method_reference_from_super(self):
- """ test support for method references from 'super'. """
+ """test support for method references from 'super'."""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("super::toString;")))
+ parse.parse(setup_java_class("super::toString;"))
+ )
def test_method_reference_from_super_with_identifier(self):
- """ test support for method references from Identifier.super. """
+ """test support for method references from Identifier.super."""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("String.super::toString;")))
+ parse.parse(setup_java_class("String.super::toString;"))
+ )
@unittest.expectedFailure
def test_method_reference_explicit_type_arguments_for_generic_type(self):
- """ currently there is no support for method references
- for an explicit type.
+ """currently there is no support for method references
+ for an explicit type.
"""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("List::size;")))
+ parse.parse(setup_java_class("List::size;"))
+ )
def test_method_reference_explicit_type_arguments(self):
- """ test support for method references with an explicit type.
- """
+ """test support for method references with an explicit type."""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("Arrays:: sort;")))
+ parse.parse(setup_java_class("Arrays:: sort;"))
+ )
@unittest.expectedFailure
def test_method_reference_from_array_type(self):
- """ currently there is no support for method references
- from a primary type.
+ """currently there is no support for method references
+ from a primary type.
"""
self.assert_contains_method_reference_expression_in_m(
- parse.parse(setup_java_class("int[]::new;")))
+ parse.parse(setup_java_class("int[]::new;"))
+ )
class InterfaceSupportTest(unittest.TestCase):
- """ Contains tests for java 8 interface extensions. """
+ """Contains tests for java 8 interface extensions."""
def test_interface_support_static_methods(self):
- parse.parse("""
+ parse.parse(
+ """
interface Foo {
void foo();
@@ -222,20 +228,24 @@ def test_interface_support_static_methods(self):
};
}
}
- """)
+ """
+ )
def test_interface_support_default_methods(self):
- parse.parse("""
+ parse.parse(
+ """
interface Foo {
default void foo() {
System.out.println("foo");
}
}
- """)
+ """
+ )
def main():
unittest.main()
-if __name__ == '__main__':
+
+if __name__ == "__main__":
main()
diff --git a/baseline_tokenization/javalang/test/test_javadoc.py b/baseline_tokenization/javalang/test/test_javadoc.py
index 68e8aec..fdd666f 100644
--- a/baseline_tokenization/javalang/test/test_javadoc.py
+++ b/baseline_tokenization/javalang/test/test_javadoc.py
@@ -5,10 +5,11 @@
class TestJavadoc(unittest.TestCase):
def test_empty_comment(self):
- javadoc.parse('/** */')
- javadoc.parse('/***/')
- javadoc.parse('/**\n *\n */')
- javadoc.parse('/**\n *\n *\n */')
+ javadoc.parse("/** */")
+ javadoc.parse("/***/")
+ javadoc.parse("/**\n *\n */")
+ javadoc.parse("/**\n *\n *\n */")
+
if __name__ == "__main__":
unittest.main()
diff --git a/baseline_tokenization/javalang/test/test_package_declaration.py b/baseline_tokenization/javalang/test/test_package_declaration.py
index 880ac67..7b938e7 100644
--- a/baseline_tokenization/javalang/test/test_package_declaration.py
+++ b/baseline_tokenization/javalang/test/test_package_declaration.py
@@ -57,5 +57,6 @@ def get_ast(self, filename):
def main():
unittest.main()
-if __name__ == '__main__':
+
+if __name__ == "__main__":
main()
diff --git a/baseline_tokenization/javalang/test/test_util.py b/baseline_tokenization/javalang/test/test_util.py
index 08e326e..b79f474 100644
--- a/baseline_tokenization/javalang/test/test_util.py
+++ b/baseline_tokenization/javalang/test/test_util.py
@@ -27,27 +27,27 @@ def test_usage(self):
self.assertEqual(next(i), 3)
self.assertEqual(next(i), 4)
self.assertEqual(next(i), 5)
- i.pop_marker(True) # reset
+ i.pop_marker(True) # reset
self.assertEqual(i.look(), 3)
self.assertEqual(next(i), 3)
- i.push_marker() #1
+ i.push_marker() # 1
self.assertEqual(next(i), 4)
self.assertEqual(next(i), 5)
- i.push_marker() #2
+ i.push_marker() # 2
self.assertEqual(next(i), 6)
self.assertEqual(next(i), 7)
- i.push_marker() #3
+ i.push_marker() # 3
self.assertEqual(next(i), 8)
self.assertEqual(next(i), 9)
- i.pop_marker(False) #3
+ i.pop_marker(False) # 3
self.assertEqual(next(i), 10)
- i.pop_marker(True) #2
+ i.pop_marker(True) # 2
self.assertEqual(next(i), 6)
self.assertEqual(next(i), 7)
self.assertEqual(next(i), 8)
- i.pop_marker(False) #1
+ i.pop_marker(False) # 1
self.assertEqual(next(i), 9)
try:
@@ -65,5 +65,5 @@ def test_usage(self):
self.assertEqual(next(i), 14)
-if __name__=="__main__":
+if __name__ == "__main__":
unittest.main()
diff --git a/baseline_tokenization/javalang/tokenizer.py b/baseline_tokenization/javalang/tokenizer.py
index d5f6ab4..7d7f874 100644
--- a/baseline_tokenization/javalang/tokenizer.py
+++ b/baseline_tokenization/javalang/tokenizer.py
@@ -7,6 +7,7 @@
class LexerError(Exception):
pass
+
class JavaToken(object):
def __init__(self, value, position=None, javadoc=None):
self.value = value
@@ -16,8 +17,11 @@ def __init__(self, value, position=None, javadoc=None):
def __repr__(self):
if self.position:
return '%s "%s" line %d, position %d' % (
- self.__class__.__name__, self.value, self.position[0], self.position[1]
- )
+ self.__class__.__name__,
+ self.value,
+ self.position[0],
+ self.position[1],
+ )
else:
return '%s "%s"' % (self.__class__.__name__, self.value)
@@ -27,78 +31,191 @@ def __str__(self):
def __eq__(self, other):
raise Exception("Direct comparison not allowed")
+
class EndOfInput(JavaToken):
pass
+
class Keyword(JavaToken):
- VALUES = set(['abstract', 'assert', 'boolean', 'break', 'byte', 'case',
- 'catch', 'char', 'class', 'const', 'continue', 'default',
- 'do', 'double', 'else', 'enum', 'extends', 'final',
- 'finally', 'float', 'for', 'goto', 'if', 'implements',
- 'import', 'instanceof', 'int', 'interface', 'long', 'native',
- 'new', 'package', 'private', 'protected', 'public', 'return',
- 'short', 'static', 'strictfp', 'super', 'switch',
- 'synchronized', 'this', 'throw', 'throws', 'transient', 'try',
- 'void', 'volatile', 'while'])
+ VALUES = set(
+ [
+ "abstract",
+ "assert",
+ "boolean",
+ "break",
+ "byte",
+ "case",
+ "catch",
+ "char",
+ "class",
+ "const",
+ "continue",
+ "default",
+ "do",
+ "double",
+ "else",
+ "enum",
+ "extends",
+ "final",
+ "finally",
+ "float",
+ "for",
+ "goto",
+ "if",
+ "implements",
+ "import",
+ "instanceof",
+ "int",
+ "interface",
+ "long",
+ "native",
+ "new",
+ "package",
+ "private",
+ "protected",
+ "public",
+ "return",
+ "short",
+ "static",
+ "strictfp",
+ "super",
+ "switch",
+ "synchronized",
+ "this",
+ "throw",
+ "throws",
+ "transient",
+ "try",
+ "void",
+ "volatile",
+ "while",
+ ]
+ )
class Modifier(Keyword):
- VALUES = set(['abstract', 'default', 'final', 'native', 'private',
- 'protected', 'public', 'static', 'strictfp', 'synchronized',
- 'transient', 'volatile'])
+ VALUES = set(
+ [
+ "abstract",
+ "default",
+ "final",
+ "native",
+ "private",
+ "protected",
+ "public",
+ "static",
+ "strictfp",
+ "synchronized",
+ "transient",
+ "volatile",
+ ]
+ )
+
class BasicType(Keyword):
- VALUES = set(['boolean', 'byte', 'char', 'double',
- 'float', 'int', 'long', 'short'])
+ VALUES = set(["boolean", "byte", "char", "double", "float", "int", "long", "short"])
+
class Literal(JavaToken):
pass
+
class Integer(Literal):
pass
+
class DecimalInteger(Literal):
pass
+
class OctalInteger(Integer):
pass
+
class BinaryInteger(Integer):
pass
+
class HexInteger(Integer):
pass
+
class FloatingPoint(Literal):
pass
+
class DecimalFloatingPoint(FloatingPoint):
pass
+
class HexFloatingPoint(FloatingPoint):
pass
+
class Boolean(Literal):
VALUES = set(["true", "false"])
+
class Character(Literal):
pass
+
class String(Literal):
pass
+
class Null(Literal):
pass
+
class Separator(JavaToken):
- VALUES = set(['(', ')', '{', '}', '[', ']', ';', ',', '.'])
+ VALUES = set(["(", ")", "{", "}", "[", "]", ";", ",", "."])
+
class Operator(JavaToken):
MAX_LEN = 4
- VALUES = set(['>>>=', '>>=', '<<=', '%=', '^=', '|=', '&=', '/=',
- '*=', '-=', '+=', '<<', '--', '++', '||', '&&', '!=',
- '>=', '<=', '==', '%', '^', '|', '&', '/', '*', '-',
- '+', ':', '?', '~', '!', '<', '>', '=', '...', '->', '::'])
+ VALUES = set(
+ [
+ ">>>=",
+ ">>=",
+ "<<=",
+ "%=",
+ "^=",
+ "|=",
+ "&=",
+ "/=",
+ "*=",
+ "-=",
+ "+=",
+ "<<",
+ "--",
+ "++",
+ "||",
+ "&&",
+ "!=",
+ ">=",
+ "<=",
+ "==",
+ "%",
+ "^",
+ "|",
+ "&",
+ "/",
+ "*",
+ "-",
+ "+",
+ ":",
+ "?",
+ "~",
+ "!",
+ "<",
+ ">",
+ "=",
+ "...",
+ "->",
+ "::",
+ ]
+ )
# '>>>' and '>>' are excluded so that >> becomes two tokens and >>> becomes
# three. This is done because we can not distinguish the operators >> and
@@ -106,19 +223,45 @@ class Operator(JavaToken):
# lexing. The job of potentially recombining these symbols is left to the
# parser
- INFIX = set(['||', '&&', '|', '^', '&', '==', '!=', '<', '>', '<=', '>=',
- '<<', '>>', '>>>', '+', '-', '*', '/', '%'])
-
- PREFIX = set(['++', '--', '!', '~', '+', '-'])
-
- POSTFIX = set(['++', '--'])
-
- ASSIGNMENT = set(['=', '+=', '-=', '*=', '/=', '&=', '|=', '^=', '%=',
- '<<=', '>>=', '>>>='])
-
- LAMBDA = set(['->'])
-
- METHOD_REFERENCE = set(['::',])
+ INFIX = set(
+ [
+ "||",
+ "&&",
+ "|",
+ "^",
+ "&",
+ "==",
+ "!=",
+ "<",
+ ">",
+ "<=",
+ ">=",
+ "<<",
+ ">>",
+ ">>>",
+ "+",
+ "-",
+ "*",
+ "/",
+ "%",
+ ]
+ )
+
+ PREFIX = set(["++", "--", "!", "~", "+", "-"])
+
+ POSTFIX = set(["++", "--"])
+
+ ASSIGNMENT = set(
+ ["=", "+=", "-=", "*=", "/=", "&=", "|=", "^=", "%=", "<<=", ">>=", ">>>="]
+ )
+
+ LAMBDA = set(["->"])
+
+ METHOD_REFERENCE = set(
+ [
+ "::",
+ ]
+ )
def is_infix(self):
return self.value in self.INFIX
@@ -136,15 +279,18 @@ def is_assignment(self):
class Annotation(JavaToken):
pass
+
class Identifier(JavaToken):
pass
class JavaTokenizer(object):
- IDENT_START_CATEGORIES = set(['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Pc', 'Sc'])
+ IDENT_START_CATEGORIES = set(["Lu", "Ll", "Lt", "Lm", "Lo", "Nl", "Pc", "Sc"])
- IDENT_PART_CATEGORIES = set(['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Mc', 'Mn', 'Nd', 'Nl', 'Pc', 'Sc'])
+ IDENT_PART_CATEGORIES = set(
+ ["Lu", "Ll", "Lt", "Lm", "Lo", "Mc", "Mn", "Nd", "Nl", "Pc", "Sc"]
+ )
def __init__(self, data):
self.data = data
@@ -157,11 +303,10 @@ def __init__(self, data):
for v in Operator.VALUES:
self.operators[len(v) - 1].add(v)
- self.whitespace_consumer = re.compile(r'[^\s]')
+ self.whitespace_consumer = re.compile(r"[^\s]")
self.javadoc = None
-
def reset(self):
self.i = 0
self.j = 0
@@ -175,11 +320,11 @@ def consume_whitespace(self):
i = match.start()
- start_of_line = self.data.rfind('\n', self.i, i)
+ start_of_line = self.data.rfind("\n", self.i, i)
if start_of_line != -1:
self.start_of_line = start_of_line
- self.current_line += self.data.count('\n', self.i, i)
+ self.current_line += self.data.count("\n", self.i, i)
self.i = i
@@ -192,29 +337,29 @@ def read_string(self):
while True:
if j >= length:
- self.error('Unterminated character/string literal')
+ self.error("Unterminated character/string literal")
if state == 0:
- if self.data[j] == '\\':
+ if self.data[j] == "\\":
state = 1
elif self.data[j] == delim:
break
elif state == 1:
- if self.data[j] in 'btnfru"\'\\':
+ if self.data[j] in "btnfru\"'\\":
state = 0
- elif self.data[j] in '0123':
+ elif self.data[j] in "0123":
state = 2
- elif self.data[j] in '01234567':
+ elif self.data[j] in "01234567":
state = 3
else:
- self.error('Illegal escape character', self.data[j])
+ self.error("Illegal escape character", self.data[j])
elif state == 2:
# Possibly long octal
- if self.data[j] in '01234567':
+ if self.data[j] in "01234567":
state = 3
- elif self.data[j] == '\\':
+ elif self.data[j] == "\\":
state = 1
elif self.data[j] == delim:
break
@@ -222,7 +367,7 @@ def read_string(self):
elif state == 3:
state = 0
- if self.data[j] == '\\':
+ if self.data[j] == "\\":
state = 1
elif self.data[j] == delim:
break
@@ -233,14 +378,14 @@ def read_string(self):
def try_operator(self):
for l in range(min(self.length - self.i, Operator.MAX_LEN), 0, -1):
- if self.data[self.i:self.i + l] in self.operators[l - 1]:
+ if self.data[self.i : self.i + l] in self.operators[l - 1]:
self.j = self.i + l
return True
return False
def read_comment(self):
- if self.data[self.i + 1] == '/':
- i = self.data.find('\n', self.i + 2)
+ if self.data[self.i + 1] == "/":
+ i = self.data.find("\n", self.i + 2)
if i == -1:
self.i = self.length
@@ -253,7 +398,7 @@ def read_comment(self):
self.i = i
else:
- i = self.data.find('*/', self.i + 2)
+ i = self.data.find("*/", self.i + 2)
if i == -1:
self.i = self.length
@@ -262,14 +407,14 @@ def read_comment(self):
i += 2
self.start_of_line = i
- self.current_line += self.data.count('\n', self.i, i)
+ self.current_line += self.data.count("\n", self.i, i)
self.i = i
def try_javadoc_comment(self):
- if self.i + 2 >= self.length or self.data[self.i + 2] != '*':
+ if self.i + 2 >= self.length or self.data[self.i + 2] != "*":
return False
- j = self.data.find('*/', self.i + 2)
+ j = self.data.find("*/", self.i + 2)
if j == -1:
self.j = self.length
@@ -278,7 +423,7 @@ def try_javadoc_comment(self):
j += 2
self.start_of_line = j
- self.current_line += self.data.count('\n', self.i, j)
+ self.current_line += self.data.count("\n", self.i, j)
self.j = j
return True
@@ -289,23 +434,23 @@ def read_decimal_float_or_integer(self):
self.read_decimal_integer()
- if self.data[self.j] not in '.eEfFdD':
+ if self.data[self.j] not in ".eEfFdD":
return DecimalInteger
- if self.data[self.j] == '.':
+ if self.data[self.j] == ".":
self.i = self.j + 1
self.read_decimal_integer()
- if self.data[self.j] in 'eE':
+ if self.data[self.j] in "eE":
self.j = self.j + 1
- if self.data[self.j] in '-+':
+ if self.data[self.j] in "-+":
self.j = self.j + 1
self.i = self.j
self.read_decimal_integer()
- if self.data[self.j] in 'fFdD':
+ if self.data[self.j] in "fFdD":
self.j = self.j + 1
self.i = orig_i
@@ -317,25 +462,25 @@ def read_hex_integer_or_float(self):
self.read_hex_integer()
- if self.data[self.j] not in '.pP':
+ if self.data[self.j] not in ".pP":
return HexInteger
- if self.data[self.j] == '.':
+ if self.data[self.j] == ".":
self.j = self.j + 1
- self.read_digits('0123456789abcdefABCDEF')
+ self.read_digits("0123456789abcdefABCDEF")
- if self.data[self.j] in 'pP':
+ if self.data[self.j] in "pP":
self.j = self.j + 1
else:
- self.error('Invalid hex float literal')
+ self.error("Invalid hex float literal")
- if self.data[self.j] in '-+':
+ if self.data[self.j] in "-+":
self.j = self.j + 1
self.i = self.j
self.read_decimal_integer()
- if self.data[self.j] in 'fFdD':
+ if self.data[self.j] in "fFdD":
self.j = self.j + 1
self.i = orig_i
@@ -351,37 +496,37 @@ def read_digits(self, digits):
if c in digits:
self.j += 1 + tmp_i
tmp_i = 0
- elif c == '_':
+ elif c == "_":
tmp_i += 1
else:
break
- if c in 'lL':
+ if c in "lL":
self.j += 1
def read_decimal_integer(self):
self.j = self.i
- self.read_digits('0123456789')
+ self.read_digits("0123456789")
def read_hex_integer(self):
self.j = self.i + 2
- self.read_digits('0123456789abcdefABCDEF')
+ self.read_digits("0123456789abcdefABCDEF")
def read_bin_integer(self):
self.j = self.i + 2
- self.read_digits('01')
+ self.read_digits("01")
def read_octal_integer(self):
self.j = self.i + 1
- self.read_digits('01234567')
+ self.read_digits("01234567")
def read_integer_or_float(self, c, c_next):
- if c == '0' and c_next in 'xX':
+ if c == "0" and c_next in "xX":
return self.read_hex_integer_or_float()
- elif c == '0' and c_next in 'bB':
+ elif c == "0" and c_next in "bB":
self.read_bin_integer()
return BinaryInteger
- elif c == '0' and c_next in '01234567':
+ elif c == "0" and c_next in "01234567":
self.read_octal_integer()
return OctalInteger
else:
@@ -395,7 +540,7 @@ def try_separator(self):
def decode_data(self):
# Encodings to try in order
- codecs = ['utf_8', 'iso-8859-1']
+ codecs = ["utf_8", "iso-8859-1"]
# If data is already unicode don't try to redecode
if isinstance(self.data, six.text_type):
@@ -408,7 +553,7 @@ def decode_data(self):
except UnicodeDecodeError:
pass
- self.error('Could not decode input data')
+ self.error("Could not decode input data")
def is_java_identifier_start(self, c):
return unicodedata.category(c) in self.IDENT_START_CATEGORIES
@@ -419,7 +564,7 @@ def read_identifier(self):
while unicodedata.category(self.data[self.j]) in self.IDENT_PART_CATEGORIES:
self.j += 1
- ident = self.data[self.i:self.j]
+ ident = self.data[self.i : self.j]
if ident in Keyword.VALUES:
token_type = Keyword
@@ -430,7 +575,7 @@ def read_identifier(self):
elif ident in Boolean.VALUES:
token_type = Boolean
- elif ident == 'null':
+ elif ident == "null":
token_type = Null
else:
token_type = Identifier
@@ -445,15 +590,15 @@ def pre_tokenize(self):
j = 0
length = len(data)
- NONE = 0
- ELIGIBLE = 1
+ NONE = 0
+ ELIGIBLE = 1
MARKER_FOUND = 2
state = NONE
while j < length:
if state == NONE:
- j = data.find('\\', j)
+ j = data.find("\\", j)
if j == -1:
j = length
@@ -464,20 +609,20 @@ def pre_tokenize(self):
elif state == ELIGIBLE:
c = data[j]
- if c == 'u':
+ if c == "u":
state = MARKER_FOUND
- new_data.append(data[i:j - 1])
+ new_data.append(data[i : j - 1])
else:
state = NONE
elif state == MARKER_FOUND:
c = data[j]
- if c != 'u':
+ if c != "u":
try:
- escape_code = int(data[j:j+4], 16)
+ escape_code = int(data[j : j + 4], 16)
except ValueError:
- self.error('Invalid unicode escape', data[j:j+4])
+ self.error("Invalid unicode escape", data[j : j + 4])
new_data.append(six.unichr(escape_code))
@@ -492,7 +637,7 @@ def pre_tokenize(self):
new_data.append(data[i:])
- self.data = ''.join(new_data)
+ self.data = "".join(new_data)
self.length = len(self.data)
def tokenize(self):
@@ -518,24 +663,24 @@ def tokenize(self):
elif startswith in ("//", "/*"):
if startswith == "/*" and self.try_javadoc_comment():
- self.javadoc = self.data[self.i:self.j]
+ self.javadoc = self.data[self.i : self.j]
self.i = self.j
else:
self.read_comment()
continue
- elif startswith == '..' and self.try_operator():
+ elif startswith == ".." and self.try_operator():
# Ensure we don't mistake a '...' operator as a sequence of
# three '.' separators. This is done as an optimization instead
# of moving try_operator higher in the chain because operators
# aren't as common and try_operator is expensive
token_type = Operator
- elif c == '@':
+ elif c == "@":
token_type = Annotation
self.j = self.i + 1
- elif c == '.' and c_next.isdigit():
+ elif c == "." and c_next.isdigit():
token_type = self.read_decimal_float_or_integer()
elif self.try_separator():
@@ -545,7 +690,7 @@ def tokenize(self):
token_type = String
self.read_string()
- elif c in '0123456789':
+ elif c in "0123456789":
token_type = self.read_integer_or_float(c, c_next)
elif self.is_java_identifier_start(c):
@@ -555,10 +700,10 @@ def tokenize(self):
token_type = Operator
else:
- self.error('Could not process token', c)
+ self.error("Could not process token", c)
position = (self.current_line, self.i - self.start_of_line)
- token = token_type(self.data[self.i:self.j], position, self.javadoc)
+ token = token_type(self.data[self.i : self.j], position, self.javadoc)
yield token
if self.javadoc:
@@ -568,8 +713,8 @@ def tokenize(self):
def error(self, message, char=None):
# Provide additional information in the errors message
- line_start = self.data.rfind('\n', 0, self.i) + 1
- line_end = self.data.find('\n', self.i)
+ line_start = self.data.rfind("\n", 0, self.i) + 1
+ line_end = self.data.find("\n", self.i)
line = self.data[line_start:line_end].strip()
line_number = self.current_line
@@ -577,14 +722,16 @@ def error(self, message, char=None):
if not char:
char = self.data[self.j]
- message = u'%s at "%s", line %s: %s' % (message, char, line_number, line)
+ message = '%s at "%s", line %s: %s' % (message, char, line_number, line)
raise LexerError(message)
+
def tokenize(code):
tokenizer = JavaTokenizer(code)
return tokenizer.tokenize()
+
def reformat_tokens(tokens):
indent = 0
closed_block = False
@@ -597,38 +744,38 @@ def reformat_tokens(tokens):
closed_block = False
indent -= 4
- output.append('\n')
- output.append(' ' * indent)
- output.append('}')
+ output.append("\n")
+ output.append(" " * indent)
+ output.append("}")
if isinstance(token, (Literal, Keyword, Identifier)):
- output.append('\n')
- output.append(' ' * indent)
+ output.append("\n")
+ output.append(" " * indent)
- if token.value == '{':
+ if token.value == "{":
indent += 4
- output.append(' {\n')
- output.append(' ' * indent)
+ output.append(" {\n")
+ output.append(" " * indent)
- elif token.value == '}':
+ elif token.value == "}":
closed_block = True
- elif token.value == ',':
- output.append(', ')
+ elif token.value == ",":
+ output.append(", ")
elif isinstance(token, (Literal, Keyword, Identifier)):
if ident_last:
# If the last token was a literla/keyword/identifer put a space in between
- output.append(' ')
+ output.append(" ")
ident_last = True
output.append(token.value)
elif isinstance(token, Operator):
- output.append(' ' + token.value + ' ')
+ output.append(" " + token.value + " ")
- elif token.value == ';':
- output.append(';\n')
- output.append(' ' * indent)
+ elif token.value == ";":
+ output.append(";\n")
+ output.append(" " * indent)
else:
output.append(token.value)
@@ -636,8 +783,8 @@ def reformat_tokens(tokens):
ident_last = isinstance(token, (Literal, Keyword, Identifier))
if closed_block:
- output.append('\n}')
+ output.append("\n}")
- output.append('\n')
+ output.append("\n")
- return ''.join(output)
+ return "".join(output)
diff --git a/baseline_tokenization/javalang/tree.py b/baseline_tokenization/javalang/tree.py
index aea883a..0d9a2e8 100644
--- a/baseline_tokenization/javalang/tree.py
+++ b/baseline_tokenization/javalang/tree.py
@@ -1,20 +1,24 @@
-
from .ast import Node
# ------------------------------------------------------------------------------
+
class CompilationUnit(Node):
attrs = ("package", "imports", "types")
+
class Import(Node):
attrs = ("path", "static", "wildcard")
+
class Documented(Node):
attrs = ("documentation",)
+
class Declaration(Node):
attrs = ("modifiers", "annotations")
+
class TypeDeclaration(Declaration, Documented):
attrs = ("name", "body")
@@ -30,243 +34,332 @@ def methods(self):
def constructors(self):
return [decl for decl in self.body if isinstance(decl, ConstructorDeclaration)]
+
class PackageDeclaration(Declaration, Documented):
attrs = ("name",)
+
class ClassDeclaration(TypeDeclaration):
attrs = ("type_parameters", "extends", "implements")
+
class EnumDeclaration(TypeDeclaration):
attrs = ("implements",)
+
class InterfaceDeclaration(TypeDeclaration):
- attrs = ("type_parameters", "extends",)
+ attrs = (
+ "type_parameters",
+ "extends",
+ )
+
class AnnotationDeclaration(TypeDeclaration):
attrs = ()
+
# ------------------------------------------------------------------------------
+
class Type(Node):
- attrs = ("name", "dimensions",)
+ attrs = (
+ "name",
+ "dimensions",
+ )
+
class BasicType(Type):
attrs = ()
+
class ReferenceType(Type):
attrs = ("arguments", "sub_type")
+
class TypeArgument(Node):
attrs = ("type", "pattern_type")
+
# ------------------------------------------------------------------------------
+
class TypeParameter(Node):
attrs = ("name", "extends")
+
# ------------------------------------------------------------------------------
+
class Annotation(Node):
attrs = ("name", "element")
+
class ElementValuePair(Node):
attrs = ("name", "value")
+
class ElementArrayValue(Node):
attrs = ("values",)
+
# ------------------------------------------------------------------------------
+
class Member(Documented):
attrs = ()
+
class MethodDeclaration(Member, Declaration):
attrs = ("type_parameters", "return_type", "name", "parameters", "throws", "body")
+
class FieldDeclaration(Member, Declaration):
attrs = ("type", "declarators")
+
class ConstructorDeclaration(Declaration, Documented):
attrs = ("type_parameters", "name", "parameters", "throws", "body")
+
# ------------------------------------------------------------------------------
+
class ConstantDeclaration(FieldDeclaration):
attrs = ()
+
class ArrayInitializer(Node):
attrs = ("initializers",)
+
class VariableDeclaration(Declaration):
attrs = ("type", "declarators")
+
class LocalVariableDeclaration(VariableDeclaration):
attrs = ()
+
class VariableDeclarator(Node):
attrs = ("name", "dimensions", "initializer")
+
class FormalParameter(Declaration):
attrs = ("type", "name", "varargs")
+
class InferredFormalParameter(Node):
- attrs = ('name',)
+ attrs = ("name",)
+
# ------------------------------------------------------------------------------
+
class Statement(Node):
attrs = ("label",)
+
class IfStatement(Statement):
attrs = ("condition", "then_statement", "else_statement")
+
class WhileStatement(Statement):
attrs = ("condition", "body")
+
class DoStatement(Statement):
attrs = ("condition", "body")
+
class ForStatement(Statement):
attrs = ("control", "body")
+
class AssertStatement(Statement):
attrs = ("condition", "value")
+
class BreakStatement(Statement):
attrs = ("goto",)
+
class ContinueStatement(Statement):
attrs = ("goto",)
+
class ReturnStatement(Statement):
attrs = ("expression",)
+
class ThrowStatement(Statement):
attrs = ("expression",)
+
class SynchronizedStatement(Statement):
attrs = ("lock", "block")
+
class TryStatement(Statement):
attrs = ("resources", "block", "catches", "finally_block")
+
class SwitchStatement(Statement):
attrs = ("expression", "cases")
+
class BlockStatement(Statement):
attrs = ("statements",)
+
class StatementExpression(Statement):
attrs = ("expression",)
+
# ------------------------------------------------------------------------------
+
class TryResource(Declaration):
attrs = ("type", "name", "value")
+
class CatchClause(Statement):
attrs = ("parameter", "block")
+
class CatchClauseParameter(Declaration):
attrs = ("types", "name")
+
# ------------------------------------------------------------------------------
+
class SwitchStatementCase(Node):
attrs = ("case", "statements")
+
class ForControl(Node):
attrs = ("init", "condition", "update")
+
class EnhancedForControl(Node):
attrs = ("var", "iterable")
+
# ------------------------------------------------------------------------------
+
class Expression(Node):
attrs = ()
+
class Assignment(Expression):
attrs = ("expressionl", "value", "type")
+
class TernaryExpression(Expression):
attrs = ("condition", "if_true", "if_false")
+
class BinaryOperation(Expression):
attrs = ("operator", "operandl", "operandr")
+
class Cast(Expression):
attrs = ("type", "expression")
+
class MethodReference(Expression):
attrs = ("expression", "method", "type_arguments")
+
class LambdaExpression(Expression):
- attrs = ('parameters', 'body')
+ attrs = ("parameters", "body")
+
# ------------------------------------------------------------------------------
+
class Primary(Expression):
attrs = ("prefix_operators", "postfix_operators", "qualifier", "selectors")
+
class Literal(Primary):
attrs = ("value",)
+
class This(Primary):
attrs = ()
+
class MemberReference(Primary):
attrs = ("member",)
+
class Invocation(Primary):
attrs = ("type_arguments", "arguments")
+
class ExplicitConstructorInvocation(Invocation):
attrs = ()
+
class SuperConstructorInvocation(Invocation):
attrs = ()
+
class MethodInvocation(Invocation):
attrs = ("member",)
+
class SuperMethodInvocation(Invocation):
attrs = ("member",)
+
class SuperMemberReference(Primary):
attrs = ("member",)
+
class ArraySelector(Expression):
attrs = ("index",)
+
class ClassReference(Primary):
attrs = ("type",)
+
class VoidClassReference(ClassReference):
attrs = ()
+
# ------------------------------------------------------------------------------
+
class Creator(Primary):
attrs = ("type",)
+
class ArrayCreator(Creator):
attrs = ("dimensions", "initializer")
+
class ClassCreator(Creator):
attrs = ("constructor_type_arguments", "arguments", "body")
+
class InnerClassCreator(Creator):
attrs = ("constructor_type_arguments", "arguments", "body")
+
# ------------------------------------------------------------------------------
+
class EnumBody(Node):
attrs = ("constants", "declarations")
+
class EnumConstantDeclaration(Declaration, Documented):
attrs = ("name", "arguments", "body")
+
class AnnotationMethod(Declaration):
attrs = ("name", "return_type", "dimensions", "default")
-
diff --git a/baseline_tokenization/javalang/util.py b/baseline_tokenization/javalang/util.py
index b8452fd..3ea35ae 100644
--- a/baseline_tokenization/javalang/util.py
+++ b/baseline_tokenization/javalang/util.py
@@ -1,5 +1,3 @@
-
-
class LookAheadIterator(object):
def __init__(self, iterable):
self.iterable = iter(iterable)
@@ -29,7 +27,7 @@ def __next__(self):
return self.value
def look(self, i=0):
- """ Look ahead of the iterable by some number of values with advancing
+ """Look ahead of the iterable by some number of values with advancing
past them.
If the requested look ahead is past the end of the iterable then None is
@@ -41,8 +39,9 @@ def look(self, i=0):
if length <= i:
try:
- self.look_ahead.extend([next(self.iterable)
- for _ in range(length, i + 1)])
+ self.look_ahead.extend(
+ [next(self.iterable) for _ in range(length, i + 1)]
+ )
except StopIteration:
return self.default
@@ -64,11 +63,11 @@ def __exit__(self, exc_type, exc_val, exc_tb):
self.pop_marker(False)
def push_marker(self):
- """ Push a marker on to the marker stack """
+ """Push a marker on to the marker stack"""
self.markers.append(list())
def pop_marker(self, reset):
- """ Pop a marker off of the marker stack. If reset is True then the
+ """Pop a marker off of the marker stack. If reset is True then the
iterator will be returned to the state it was in before the
corresponding call to push_marker().
@@ -87,6 +86,7 @@ def pop_marker(self, reset):
# If there are not more markers in the stack then discard the values
pass
+
class LookAheadListIterator(object):
def __init__(self, iterable):
self.list = list(iterable)
@@ -116,7 +116,7 @@ def __next__(self):
return self.value
def look(self, i=0):
- """ Look ahead of the iterable by some number of values with advancing
+ """Look ahead of the iterable by some number of values with advancing
past them.
If the requested look ahead is past the end of the iterable then None is
@@ -146,11 +146,11 @@ def __exit__(self, exc_type, exc_val, exc_tb):
self.pop_marker(False)
def push_marker(self):
- """ Push a marker on to the marker stack """
+ """Push a marker on to the marker stack"""
self.saved_markers.append(self.marker)
def pop_marker(self, reset):
- """ Pop a marker off of the marker stack. If reset is True then the
+ """Pop a marker off of the marker stack. If reset is True then the
iterator will be returned to the state it was in before the
corresponding call to push_marker().
@@ -162,4 +162,3 @@ def pop_marker(self, reset):
self.marker = saved
elif self.saved_markers:
self.saved_markers[-1] = saved
-
diff --git a/baseline_tokenization/subtokenize_nmt_baseline.py b/baseline_tokenization/subtokenize_nmt_baseline.py
index 43730fe..8135dc2 100644
--- a/baseline_tokenization/subtokenize_nmt_baseline.py
+++ b/baseline_tokenization/subtokenize_nmt_baseline.py
@@ -5,46 +5,65 @@
import re
-modifiers = ['public', 'private', 'protected', 'static']
+modifiers = ["public", "private", "protected", "static"]
-RE_WORDS = re.compile(r'''
+RE_WORDS = re.compile(
+ r"""
# Find words in a string. Order matters!
[A-Z]+(?=[A-Z][a-z]) | # All upper case before a capitalized word
[A-Z]?[a-z]+ | # Capitalized words / all lower case
[A-Z]+ | # All upper case
\d+ | # Numbers
.+
-''', re.VERBOSE)
+""",
+ re.VERBOSE,
+)
+
def split_subtokens(str):
- return [subtok for subtok in RE_WORDS.findall(str) if not subtok == '_']
+ return [subtok for subtok in RE_WORDS.findall(str) if not subtok == "_"]
+
def tokenizeFile(file_path):
- lines = 0
- with open(file_path, 'r', encoding="utf-8") as file:
- with open(file_path + 'method_names.txt', 'w') as method_names_file:
- with open(file_path + 'method_subtokens_content.txt', 'w') as method_contents_file:
- for line in file:
- lines += 1
- line = line.rstrip()
- parts = line.split('|', 1)
- method_name = parts[0]
- method_content = parts[1]
- try:
- tokens = list(javalang.tokenizer.tokenize(method_content))
- except:
- print('ERROR in tokenizing: ' + method_content)
- #tokens = method_content.split(' ')
- if len(method_name) > 0 and len(tokens) > 0:
- method_names_file.write(method_name + '\n')
- method_contents_file.write(' '.join([' '.join(split_subtokens(i.value)) for i in tokens if not i.value in modifiers]) + '\n')
- else:
- print('ERROR in len of: ' + method_name + ', tokens: ' + str(tokens))
- print(str(lines))
-
-
-if __name__ == '__main__':
- file = sys.argv[1]
- tokenizeFile(file)
+ lines = 0
+ with open(file_path, "r", encoding="utf-8") as file:
+ with open(file_path + "method_names.txt", "w") as method_names_file:
+ with open(
+ file_path + "method_subtokens_content.txt", "w"
+ ) as method_contents_file:
+ for line in file:
+ lines += 1
+ line = line.rstrip()
+ parts = line.split("|", 1)
+ method_name = parts[0]
+ method_content = parts[1]
+ try:
+ tokens = list(javalang.tokenizer.tokenize(method_content))
+ except:
+ print("ERROR in tokenizing: " + method_content)
+ # tokens = method_content.split(' ')
+ if len(method_name) > 0 and len(tokens) > 0:
+ method_names_file.write(method_name + "\n")
+ method_contents_file.write(
+ " ".join(
+ [
+ " ".join(split_subtokens(i.value))
+ for i in tokens
+ if not i.value in modifiers
+ ]
+ )
+ + "\n"
+ )
+ else:
+ print(
+ "ERROR in len of: "
+ + method_name
+ + ", tokens: "
+ + str(tokens)
+ )
+ print(str(lines))
+if __name__ == "__main__":
+ file = sys.argv[1]
+ tokenizeFile(file)
diff --git a/code2seq.py b/code2seq.py
index 0225b60..c2602ab 100644
--- a/code2seq.py
+++ b/code2seq.py
@@ -6,12 +6,10 @@
from modelrunner import ModelRunner
from args import read_args
-if __name__ == '__main__':
- physical_devices = tf.config.list_physical_devices('GPU')
- if len(physical_devices):
- tf.config.experimental.set_memory_growth(physical_devices[0], True)
- # tf.config.set_visible_devices([], 'GPU')
-
+if __name__ == "__main__":
+ physical_devices = tf.config.list_physical_devices("GPU")
+ for device in physical_devices:
+ tf.config.experimental.set_memory_growth(device, True)
args = read_args()
np.random.seed(args.seed)
@@ -23,16 +21,23 @@
else:
config = Config.get_default_config(args)
- print('Created model')
+ print("Created model")
if config.TRAIN_PATH:
model = ModelRunner(config)
model.train()
if config.TEST_PATH and not args.data_path:
model = ModelRunner(config)
results, precision, recall, f1, rouge = model.evaluate()
- print('Accuracy: ' + str(results))
- print('Precision: ' + str(precision) + ', recall: ' + str(recall) + ', F1: ' + str(f1))
- print('Rouge: ', rouge)
+ print("Accuracy: " + str(results))
+ print(
+ "Precision: "
+ + str(precision)
+ + ", recall: "
+ + str(recall)
+ + ", F1: "
+ + str(f1)
+ )
+ print("Rouge: ", rouge)
if args.predict:
model = ModelRunner(config)
predictor = InteractivePredictor(config, model, args.predict)
diff --git a/common.py b/common.py
index c1f9aa7..6de2f3e 100644
--- a/common.py
+++ b/common.py
@@ -4,15 +4,15 @@
class Common:
- internal_delimiter = '|'
- SOS = ''
- EOS = ''
- PAD = ''
- UNK = ''
+ internal_delimiter = "|"
+ SOS = ""
+ EOS = ""
+ PAD = ""
+ UNK = ""
@staticmethod
def normalize_word(word):
- stripped = re.sub(r'[^a-zA-Z]', '', word)
+ stripped = re.sub(r"[^a-zA-Z]", "", word)
if len(stripped) == 0:
return word.lower()
else:
@@ -21,13 +21,16 @@ def normalize_word(word):
@staticmethod
def load_histogram(path, max_size=None):
histogram = {}
- with open(path, 'r') as file:
+ with open(path, "r") as file:
for line in file.readlines():
- parts = line.split(' ')
+ parts = line.split(" ")
if not len(parts) == 2:
continue
histogram[parts[0]] = int(parts[1])
- sorted_histogram = [(k, histogram[k]) for k in sorted(histogram, key=histogram.get, reverse=True)]
+ sorted_histogram = [
+ (k, histogram[k])
+ for k in sorted(histogram, key=histogram.get, reverse=True)
+ ]
return dict(sorted_histogram[:max_size])
@staticmethod
@@ -38,7 +41,10 @@ def load_vocab_from_dict(word_to_count, add_values=[], max_size=None):
word_to_index[value] = current_index
index_to_word[current_index] = value
current_index += 1
- sorted_counts = [(k, word_to_count[k]) for k in sorted(word_to_count, key=word_to_count.get, reverse=True)]
+ sorted_counts = [
+ (k, word_to_count[k])
+ for k in sorted(word_to_count, key=word_to_count.get, reverse=True)
+ ]
limited_sorted = dict(sorted_counts[:max_size])
for word, count in limited_sorted.items():
word_to_index[word] = current_index
@@ -82,26 +88,41 @@ def parse_results(result, pc_info_dict, topk=5):
prediction_results = {}
results_counter = 0
for single_method in result:
- original_name, top_suggestions, top_scores, attention_per_context = list(single_method)
+ original_name, top_suggestions, top_scores, attention_per_context = list(
+ single_method
+ )
current_method_prediction_results = PredictionResults(original_name)
if attention_per_context is not None:
- word_attention_pairs = [(word, attention) for word, attention in
- zip(top_suggestions, attention_per_context) if
- Common.legal_method_names_checker(word)]
+ word_attention_pairs = [
+ (word, attention)
+ for word, attention in zip(top_suggestions, attention_per_context)
+ if Common.legal_method_names_checker(word)
+ ]
for predicted_word, attention_timestep in word_attention_pairs:
current_timestep_paths = []
- for context, attention in [(key, attention_timestep[key]) for key in
- sorted(attention_timestep, key=attention_timestep.get, reverse=True)][
- :topk]:
+ for context, attention in [
+ (key, attention_timestep[key])
+ for key in sorted(
+ attention_timestep, key=attention_timestep.get, reverse=True
+ )
+ ][:topk]:
if context in pc_info_dict:
pc_info = pc_info_dict[context]
current_timestep_paths.append((attention.item(), pc_info))
- current_method_prediction_results.append_prediction(predicted_word, current_timestep_paths)
+ current_method_prediction_results.append_prediction(
+ predicted_word, current_timestep_paths
+ )
else:
for predicted_seq in top_suggestions:
- filtered_seq = [word for word in predicted_seq if Common.legal_method_names_checker(word)]
- current_method_prediction_results.append_prediction(filtered_seq, None)
+ filtered_seq = [
+ word
+ for word in predicted_seq
+ if Common.legal_method_names_checker(word)
+ ]
+ current_method_prediction_results.append_prediction(
+ filtered_seq, None
+ )
prediction_results[results_counter] = current_method_prediction_results
results_counter += 1
@@ -110,8 +131,12 @@ def parse_results(result, pc_info_dict, topk=5):
@staticmethod
def compute_bleu(ref_file_name, predicted_file_name):
with open(predicted_file_name) as predicted_file:
- pipe = subprocess.Popen(["perl", "scripts/multi-bleu.perl", ref_file_name], stdin=predicted_file,
- stdout=sys.stdout, stderr=sys.stderr)
+ pipe = subprocess.Popen(
+ ["perl", "scripts/multi-bleu.perl", ref_file_name],
+ stdin=predicted_file,
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ )
class PredictionResults:
@@ -122,26 +147,29 @@ def __init__(self, original_name):
def append_prediction(self, name, current_timestep_paths):
self.predictions.append(SingleTimeStepPrediction(name, current_timestep_paths))
+
class SingleTimeStepPrediction:
def __init__(self, prediction, attention_paths):
self.prediction = prediction
if attention_paths is not None:
paths_with_scores = []
for attention_score, pc_info in attention_paths:
- path_context_dict = {'score': attention_score,
- 'path': pc_info.longPath,
- 'token1': pc_info.token1,
- 'token2': pc_info.token2}
+ path_context_dict = {
+ "score": attention_score,
+ "path": pc_info.longPath,
+ "token1": pc_info.token1,
+ "token2": pc_info.token2,
+ }
paths_with_scores.append(path_context_dict)
self.attention_paths = paths_with_scores
class PathContextInformation:
def __init__(self, context):
- self.token1 = context['name1']
- self.longPath = context['path']
- self.shortPath = context['shortPath']
- self.token2 = context['name2']
+ self.token1 = context["name1"]
+ self.longPath = context["path"]
+ self.shortPath = context["shortPath"]
+ self.token2 = context["name2"]
def __str__(self):
- return '%s,%s,%s' % (self.token1, self.shortPath, self.token2)
+ return "%s,%s,%s" % (self.token1, self.shortPath, self.token2)
diff --git a/config.py b/config.py
index 31944cf..fbb97dc 100644
--- a/config.py
+++ b/config.py
@@ -1,11 +1,12 @@
class Config:
@staticmethod
- def get_default_config(args):
+ def get_default_config(args):
+ # Training configs
config = Config(args)
config.NUM_EPOCHS = 3000
config.SAVE_EVERY_EPOCHS = 1
config.PATIENCE = 10
- config.BATCH_SIZE = 128
+ config.BATCH_SIZE = 10
config.READER_NUM_PARALLEL_BATCHES = 1
config.SHUFFLE_BUFFER_SIZE = 10000
config.CSV_BUFFER_SIZE = 100 * 1024 * 1024 # 100 MB
@@ -43,12 +44,14 @@ def __init__(self, args):
self.MODEL_PATH = args.model_path if args.model_path is not None else None
self.SAVE_PATH = args.save_path if args.save_path is not None else None
self.LOAD_PATH = args.load_path if args.load_path is not None else None
+ self.CONTINUE_FROM_CHECKPOINT = args.continue_from_checkpoint if args.continue_from_checkpoint is not None else None
else:
self.TRAIN_PATH = None
self.TEST_PATH = None
self.MODEL_PATH = None
self.SAVE_PATH = None
self.LOAD_PATH = None
+ self.CONTINUE_FROM_CHECKPOINT = None
self.NUM_EPOCHS = 0
self.SAVE_EVERY_EPOCHS = 0
diff --git a/cpp_extractor.py b/cpp_extractor.py
index 70ecadb..be22efb 100644
--- a/cpp_extractor.py
+++ b/cpp_extractor.py
@@ -7,16 +7,21 @@
class CppExtractor:
- def __init__(self, config, ):
+ def __init__(
+ self,
+ config,
+ ):
self.config = config
- self.parser = AstParser(max_contexts_num=self.config.MAX_CONTEXTS,
- max_path_len=self.config.MAX_PATH_LENGTH,
- max_subtokens_num=self.config.MAX_NAME_PARTS,
- max_ast_depth=100,
- out_path=None)
+ self.parser = AstParser(
+ max_contexts_num=self.config.MAX_CONTEXTS,
+ max_path_len=self.config.MAX_PATH_LENGTH,
+ max_subtokens_num=self.config.MAX_NAME_PARTS,
+ max_ast_depth=100,
+ out_path=None,
+ )
def extract_paths(self, code_string):
- tmp = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.cc')
+ tmp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".cc")
try:
tmp.write(code_string)
tmp.close()
@@ -30,14 +35,18 @@ def extract_paths(self, compiler_args, file_path):
result = []
for sample in self.parser.samples:
for context in sample.contexts:
- info_context = {'name1': make_str_key(context.start_token),
- 'name2': make_str_key(context.end_token),
- 'path': make_str_key(context.path.tokens),
- 'shortPath': make_str_key(context.path.tokens)}
+ info_context = {
+ "name1": make_str_key(context.start_token),
+ "name2": make_str_key(context.end_token),
+ "path": make_str_key(context.path.tokens),
+ "shortPath": make_str_key(context.path.tokens),
+ }
pc_info = PathContextInformation(info_context)
- pc_info_dict[(pc_info.token1, pc_info.shortPath, pc_info.token2)] = pc_info
+ pc_info_dict[
+ (pc_info.token1, pc_info.shortPath, pc_info.token2)
+ ] = pc_info
result_line = str(sample)
- space_padding = ' ' * (self.config.DATA_NUM_CONTEXTS - len(sample.contexts))
+ space_padding = " " * (self.config.DATA_NUM_CONTEXTS - len(sample.contexts))
result_line += space_padding
result.append(result_line)
return result, pc_info_dict
diff --git a/dataset_scripts/codesearchnet.zip b/dataset_scripts/codesearchnet.zip
new file mode 100644
index 0000000..91c17f8
Binary files /dev/null and b/dataset_scripts/codesearchnet.zip differ
diff --git a/dataset_scripts/dataset_statistics.py b/dataset_scripts/dataset_statistics.py
new file mode 100644
index 0000000..0488747
--- /dev/null
+++ b/dataset_scripts/dataset_statistics.py
@@ -0,0 +1,40 @@
+import json
+import os
+from pathlib import Path
+
+# Will only work for codesearchnet for now.
+# For funcom json parsing is not neededself.
+# For default dataset this approach is not suitable as the data is just pure java files.
+DATASETS = [
+ "codesearchnet",
+ # "default",
+ # "funcom"
+ ]
+
+for ds in DATASETS:
+
+ path = Path("../datasets/" + ds + "/raw/")
+ file_paths = [os.path.join(dirpath,f) for (dirpath, dirnames, filenames) in os.walk(path) for f in filenames]
+
+ examples = 0
+ inline_comment_count = 0
+
+ # go over each file and collect its json
+ for path in file_paths:
+ with open(path, "r") as file:
+ lines = file.readlines()
+
+ for line in lines:
+ examples += 1
+ data = json.loads(line)
+ code = str(data["original_string"])
+ comment = str(data["docstring"])
+
+ # Count inline comments
+ if "//" in code or "/*" in code:
+ inline_comment_count += 1
+
+ print("=========" + ds + "=========")
+ print("TOTAL NUMBER OF EXAMPLES : " + str(examples))
+ print("TOTAL NUMBER OF INLINE COMMENTS: " + str(inline_comment_count))
+ print("========================================")
diff --git a/dataset_scripts/funcom_shuffle.py b/dataset_scripts/funcom_shuffle.py
new file mode 100644
index 0000000..bf3f47c
--- /dev/null
+++ b/dataset_scripts/funcom_shuffle.py
@@ -0,0 +1,117 @@
+import argparse
+import collections
+import json
+import random
+import math
+
+def load_pid():
+ f = 'fid_pid'
+ pidtofid = collections.defaultdict(list)
+ for line in open(f, 'r').readlines()[1:]:
+ t = line.split('\t')
+ fid = int(t[0])
+ pid = int(t[1])
+ pidtofid[pid].append(fid)
+
+ return pidtofid
+
+def load_data(fname):
+ data = {}
+ for line in open(fname):
+ tmp = line.split('\t')
+ fid = int(tmp[0])
+ value = tmp[1]
+ data[fid] = value
+ return data
+
+def load_json(fname):
+ f = open(fname)
+ function_json = json.load(f)
+
+ return function_json
+
+def write(data, fname):
+ fo = open(fname, 'w')
+ for fid, string in data.items():
+ fo.write("{}\t{}\n".format(fid, string))
+ fo.close()
+
+def write_function(functiondata, commentdata, fname):
+ fo = open(fname, 'w')
+ for fid, functioncode in functiondata.items():
+
+ fo.write(str({"\n/**\n{}\n*/\n\t{}\n".format(commentdata[fid], functioncode)})+ "\n")
+
+ fo.close()
+
+if __name__ == '__main__':
+ print("Making new train/valid/test split")
+ parser = argparse.ArgumentParser(description='')
+ parser.add_argument('--seed', type=int, default=None)
+ parser.add_argument('--valid-size', type=float, default=0.05)
+ parser.add_argument('--test-size', type=float, default=0.05)
+ args = parser.parse_args()
+
+ # Get args ####
+ seed = args.seed
+ valid_size = args.valid_size
+ test_size = args.test_size
+
+ # set seed for random splits
+ if seed is not None:
+ print("Using seed {}".format(seed))
+ else:
+ print("Using random seed")
+
+ random.seed(a=seed)
+
+ f1 = 'comments'
+ f2 = 'functions.json'
+
+ pidlist = load_pid()
+ coms = load_data(f1)
+
+ src = load_json(f2)
+
+ shuffle_list = list(pidlist.keys())
+ random.shuffle(shuffle_list)
+
+ testnum = math.ceil(len(shuffle_list)*test_size)
+ validnum = math.ceil(len(shuffle_list)*valid_size)
+
+ testset = shuffle_list[:testnum]
+ validset = shuffle_list[testnum:(testnum+validnum)]
+ trainset = shuffle_list[(testnum+validnum):]
+
+ print("Project counts:")
+ print("Train: {} Valid: {} Test: {}".format(len(trainset), len(validset), len(testset)))
+
+ trainfun = {}
+ validfun = {}
+ testfun = {}
+
+ traincom = {}
+ validcom = {}
+ testcom = {}
+
+ for pid in trainset:
+ for fid in pidlist[pid]:
+ traincom[fid] = coms[fid].strip()
+ trainfun[fid] = src[str(fid)].strip()
+ for pid in validset:
+ for fid in pidlist[pid]:
+ validcom[fid] = coms[fid].strip()
+ validfun[fid] = src[str(fid)].strip()
+ for pid in testset:
+ for fid in pidlist[pid]:
+ testcom[fid] = coms[fid].strip()
+ testfun[fid] = src[str(fid)].strip()
+
+
+ ftrain = './train/functions.train.jsonl'
+ fvalid = './valid/functions.val.jsonl'
+ ftest = './test/functions.test.jsonl'
+
+ write_function(trainfun, traincom, ftrain)
+ write_function(validfun, validcom, fvalid)
+ write_function(testfun, testcom, ftest)
\ No newline at end of file
diff --git a/dataset_scripts/prepare_all.sh b/dataset_scripts/prepare_all.sh
new file mode 100644
index 0000000..39b5f26
--- /dev/null
+++ b/dataset_scripts/prepare_all.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+bash prepare_codesearchnet.sh
+bash prepare_default.sh
+bash prepare_funcom.sh
+
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_all_minimal.sh b/dataset_scripts/prepare_all_minimal.sh
new file mode 100755
index 0000000..e285319
--- /dev/null
+++ b/dataset_scripts/prepare_all_minimal.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+bash prepare_codesearchnet_minimal.sh
+bash prepare_default_minimal.sh
+bash prepare_funcom_minimal.sh
+
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_codesearchnet.sh b/dataset_scripts/prepare_codesearchnet.sh
new file mode 100755
index 0000000..c6c59cb
--- /dev/null
+++ b/dataset_scripts/prepare_codesearchnet.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# This file downloads and prepares the CodeSearchNet dataset specifically.
+# Install the unzip package beforehand if your distribution does not have it
+
+mkdir -p ../datasets/codesearchnet
+
+# Copy and unzip files required for preparing the dataset
+cp codesearchnet.zip ../datasets/codesearchnet
+cd ../datasets/codesearchnet
+unzip codesearchnet.zip
+rm codesearchnet.zip
+
+rm -rf raw && mv ./dataset ./raw
+cd raw
+
+# ------------
+# Download and unzip dataset
+wget https://s3.amazonaws.com/code-search-net/CodeSearchNet/v2/java.zip
+
+# Or: copy an existing dataset
+# cp ../../../data/java.zip ./java.zip
+# ------------
+
+unzip java.zip
+
+# # Remove the now redundant archive file
+rm *.zip
+
+# # Run script to finalize the dataset unzipping
+python preprocess.py
+rm -rf */final
+rm -rf java*
+rm preprocess.py
+
+echo "Codesearchnet dataset prepared."
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_codesearchnet_minimal.sh b/dataset_scripts/prepare_codesearchnet_minimal.sh
new file mode 100755
index 0000000..1079a5e
--- /dev/null
+++ b/dataset_scripts/prepare_codesearchnet_minimal.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+FILE=../datasets/codesearchnet-minimal.tar.gz
+DATASETFOLDER=../datasets
+if [ -f "$FILE" ];
+then
+ echo "$FILE exists - unpacking it"
+ tar -xvf $FILE --directory $DATASETFOLDER
+else
+ # Download the dataset
+ bash prepare_codesearchnet.sh
+
+ cd ../datasets/codesearchnet/raw
+
+ # Trim the dataset
+ echo "Trimming the dataset..."
+
+ echo "$(head -n 50 test/test.jsonl)" > test/test.jsonl
+ echo "$(head -n 50 train/train.jsonl)" > train/train.jsonl
+ echo "$(head -n 50 valid/valid.jsonl)" > valid/valid.jsonl
+fi
+
+echo "Codesearch minimal dataset prepared."
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_default.sh b/dataset_scripts/prepare_default.sh
new file mode 100644
index 0000000..50c5f24
--- /dev/null
+++ b/dataset_scripts/prepare_default.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# This file downloads and prepares the default dataset specifically.
+
+mkdir -p ../datasets/default/raw
+
+cd ../datasets/default/raw
+
+# ------------
+# Download and unzip tokenized and filtered datasets
+# wget https://s3.amazonaws.com/code2seq/datasets/java-small.tar.gz
+
+# Or: copy an existing dataset
+cp ../../../data/java-small.tar.gz java-small.tar.gz
+# ------------
+
+# Unzip the downloaded archive
+tar xvf java-small.tar.gz
+
+# Move and rename folders
+mv java-small/test ./test
+mv java-small/training ./train
+mv java-small/validation ./valid
+
+
+# # Remove the now redundant files
+rm *.tar.gz
+rm -rf java*
+
+echo "Default dataset prepared."
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_default_minimal.sh b/dataset_scripts/prepare_default_minimal.sh
new file mode 100644
index 0000000..9273745
--- /dev/null
+++ b/dataset_scripts/prepare_default_minimal.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+FILE=../datasets/default-minimal.tar.gz
+DATASETFOLDER=../datasets
+if [ -f "$FILE" ];
+then
+ echo "$FILE exists - unpacking it"
+ tar -xvf $FILE --directory $DATASETFOLDER
+else
+ # A way to do this would be to unzip the whole archive and then just keep some particular files. Not implemented yet.
+ bash prepare_default.sh
+fi
+
+echo "Default minimal dataset prepared."
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_funcom.sh b/dataset_scripts/prepare_funcom.sh
new file mode 100644
index 0000000..60b117d
--- /dev/null
+++ b/dataset_scripts/prepare_funcom.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# This file downloads and prepares the FunCom dataset specifically.
+
+mkdir -p ../datasets/funcom/raw
+
+cd ../datasets/funcom/raw
+
+# ------------
+# Download and unzip tokenized and filtered datasets
+wget https://s3.us-east-2.amazonaws.com/leclair.tech/data/funcom/funcom_filtered.tar.gz
+wget https://s3.us-east-2.amazonaws.com/leclair.tech/data/funcom/funcom_tokenized.tar.gz
+
+# Or: copy an existing dataset
+# cp ../../../data/funcom_filtered.tar.gz funcom_filtered.tar.gz
+# cp ../../../data/funcom_tokenized.tar.gz funcom_tokenized.tar.gz
+# ------------
+
+cp ../../../dataset_scripts/funcom_shuffle.py funcom_shuffle.py
+
+# Unzip the downloaded archives
+tar xvf funcom_filtered.tar.gz
+tar xvf funcom_tokenized.tar.gz
+
+# Copy required files from both datasets
+cp ./funcom_processed/functions.json functions.json
+cp ./funcom_tokenized/comments comments
+cp ./funcom_tokenized/fid_pid fid_pid
+
+# Create training directories
+mkdir test
+mkdir train
+mkdir valid
+
+# Run the shuffle file which creates training data distributions
+python funcom_shuffle.py
+
+# Remove the now redundant files
+rm *.tar.gz
+rm -rf funcom*
+rm comments
+rm fid_pid
+rm functions.json
+
+echo "Funcom dataset prepared."
+exit
\ No newline at end of file
diff --git a/dataset_scripts/prepare_funcom_minimal.sh b/dataset_scripts/prepare_funcom_minimal.sh
new file mode 100644
index 0000000..5b2c414
--- /dev/null
+++ b/dataset_scripts/prepare_funcom_minimal.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+
+FILE=../datasets/funcom-minimal.tar.gz
+DATASETFOLDER=../datasets
+if [ -f "$FILE" ];
+then
+ echo "$FILE exists - unpacking it"
+ tar -xvf $FILE --directory $DATASETFOLDER
+else
+ # Download the dataset
+ bash prepare_funcom.sh
+
+ cd ../datasets/funcom/raw
+
+ # Trim the dataset
+ echo "Trimming the dataset..."
+
+ echo "$(head -n 50 test/functions.test.jsonl)" > test/functions.test.jsonl
+ echo "$(head -n 50 train/functions.train.jsonl)" > train/functions.train.jsonl
+ echo "$(head -n 50 valid/functions.val.jsonl)" > valid/functions.val.jsonl
+fi
+
+echo "Funcom minimal dataset prepared."
+exit
\ No newline at end of file
diff --git a/datasets/.gitignore b/datasets/.gitignore
new file mode 100644
index 0000000..b392462
--- /dev/null
+++ b/datasets/.gitignore
@@ -0,0 +1,4 @@
+*
+*/
+!.gitignore
+!*-minimal.tar.gz
\ No newline at end of file
diff --git a/datasets/codesearchnet-minimal.tar.gz b/datasets/codesearchnet-minimal.tar.gz
new file mode 100644
index 0000000..3114852
Binary files /dev/null and b/datasets/codesearchnet-minimal.tar.gz differ
diff --git a/datasets/default-minimal.tar.gz b/datasets/default-minimal.tar.gz
new file mode 100644
index 0000000..373d46b
Binary files /dev/null and b/datasets/default-minimal.tar.gz differ
diff --git a/datasets/funcom-minimal.tar.gz b/datasets/funcom-minimal.tar.gz
new file mode 100644
index 0000000..c6fcbf7
Binary files /dev/null and b/datasets/funcom-minimal.tar.gz differ
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..f3e54bd
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,35 @@
+version: '3.8'
+
+services:
+ code2seq_comments:
+ build:
+ context: ""
+ dockerfile: Dockerfile
+ image: ciselab/code2seq:latest
+ volumes:
+ # source on local machine: place in container
+ - ./models:/app/code2seq/models
+ - ./datasets/codesearchnet/raw:/app/code2seq/datasets/codesearchnet/raw:ro
+ - ./datasets/codesearchnet/preprocessed:/app/code2seq/datasets/codesearchnet/preprocessed:rw
+
+ environment:
+ dataset: "codesearchnet"
+ variant: "comments"
+ # Preprocessing variables
+ preprocess: true
+ includeComments: true
+ excludeStopwords: true
+ useTfidf: false
+ numberOfTfidfKeywords: "50"
+ # Training variables
+ train: true
+ # There has to ba an existing model for the following to work
+ continueTrainingFromCheckpoint: false
+
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: 1
+ capabilities: [gpu]
diff --git a/entrypoint.sh b/entrypoint.sh
new file mode 100644
index 0000000..2240c5f
--- /dev/null
+++ b/entrypoint.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# set -e makes the shell script exit if any command exists with non-zero exit code
+set -e
+
+if [ "$preprocess" = true ];
+then bash preprocess.sh --dataset="$dataset" --include_comments="$includeComments" --exclude_stopwords="$excludeStopwords" --include_tfidf="$useTfidf" --number_keywords="$numberOfTfidfKeywords" --variant="$variant"
+else echo "Not preprocessing."
+fi
+
+if [ "$train" = true ];
+then bash train.sh --dataset="$dataset" --continue_training_from_checkpoint="$continueTrainingFromCheckpoint" --variant="$variant"
+else echo "Not training a new model."
+fi
+
+# How to keep the container open even if it errors:
+# tail -f /dev/null
\ No newline at end of file
diff --git a/error_log.txt b/error_log.txt
new file mode 100644
index 0000000..e69de29
diff --git a/evaluation_scripts/postprocessing.py b/evaluation_scripts/postprocessing.py
new file mode 100644
index 0000000..275cc32
--- /dev/null
+++ b/evaluation_scripts/postprocessing.py
@@ -0,0 +1,98 @@
+# from nltk.tokenize import word_tokenize
+# from nltk.translate.bleu_score import sentence_bleu
+# from nltk.translate.bleu_score import SmoothingFunction
+# import scipy.stats as stats
+# from cliffs_delta import cliffs_delta
+from glob import glob
+import os
+
+# Global variable to hold the data for each experiment
+experiment_data = []
+
+class ExperimentResult:
+ """ Class to hold the output of a single experiment for easier managmenet """
+ def __init__(self, references, predictions, stats, loss) -> None:
+ self.references = references
+ self.predictions = predictions
+ self.stats = stats
+ self.loss = loss
+
+def main():
+ """ Main program """
+ # Load the data by creating an ExperimentResults object for each experiment
+ load_data()
+
+ # Process the gathered data.
+ run_statistical_tests()
+ generate_graphs()
+
+ return 0
+
+def load_data():
+
+ for dir in glob("../models/exp_*/"):
+ # ref file are the labels
+ # log file is the generated prediction
+ references = open(dir + "ref.txt").readlines() if os.path.isfile(dir + "ref.txt") else []
+ predictions = open(dir + "pred.txt").readlines() if os.path.isfile(dir + "log.txt") else []
+
+ # the format of stats is (separated by whitespace):
+ # epoch, accuracy, precision, recall, f1
+ stats = open(dir + "stats.txt").readlines() if os.path.isfile(dir + "stats.txt") else []
+
+ # the format of loss is (separated by whitespace):
+ # batch number, average loss, throughput
+ loss = open(dir + "loss.txt").readlines() if os.path.isfile(dir + "loss.txt") else []
+
+ res = ExperimentResult(references, predictions, stats, loss)
+ experiment_data.append(res)
+ print("====== LOADED EXPERIMENT DATA ======")
+
+def run_statistical_tests():
+
+ # TODO: Implement methods for statistical tests
+ # TODO: For Leonhard: this is how I calculated statistics for my results. I leave it as a reference. - Balys.
+
+ # for i, ref in enumerate(references):
+ # ref_tokens = set(word_tokenize(ref))
+ # com_pred_tokens = set(word_tokenize(predictions_com[i]))
+ # no_com_pred_tokens = set(word_tokenize(predictions_no_com[i]))
+ #
+ # # Jaccard distance
+ # jac = 1 - (len(ref_tokens & com_pred_tokens) / len(ref_tokens | com_pred_tokens))
+ # jac_no = 1 - (
+ # len(ref_tokens & no_com_pred_tokens) / len(ref_tokens | no_com_pred_tokens)
+ # )
+ # com_jac.append(jac)
+ # no_com_jac.append(jac_no)
+ #
+ # # BLEU score
+ # bleu = sentence_bleu(
+ # ref_tokens, com_pred_tokens, smoothing_function=SmoothingFunction().method1
+ # )
+ # bleu_no = sentence_bleu(
+ # ref_tokens, no_com_pred_tokens, smoothing_function=SmoothingFunction().method1
+ # )
+ # com_bleu.append(bleu)
+ # no_com_bleu.append(bleu_no)
+ #
+ # print(
+ # "Rank Sum with BLEU:\n", stats.ranksums(com_bleu, no_com_bleu, alternative="less")
+ # )
+ # print(
+ # "Rank Sum with Jaccard Distance:\n",
+ # stats.ranksums(com_jac, no_com_jac, alternative="less"),
+ # )
+ #
+ # print("BLEU Cliff Delta:", cliffs_delta(com_bleu, no_com_bleu))
+ # print("Jaccard Cliff Delta:", cliffs_delta(com_jac, no_com_jac))
+
+ pass
+
+
+def generate_graphs():
+ # TODO: implement methods for plot generation
+ pass
+
+if __name__ == "__main__":
+ main()
diff --git a/evaluation_scripts/postprocessing.sh b/evaluation_scripts/postprocessing.sh
new file mode 100644
index 0000000..e69de29
diff --git a/evaluation_scripts/requirements.txt b/evaluation_scripts/requirements.txt
new file mode 100644
index 0000000..048bf75
--- /dev/null
+++ b/evaluation_scripts/requirements.txt
@@ -0,0 +1,5 @@
+cliffs_delta==1.0.0
+nltk==3.7
+scipy==1.9.2
+argparse
+glob
diff --git a/interactive_predict.py b/interactive_predict.py
index 286e721..033abf3 100644
--- a/interactive_predict.py
+++ b/interactive_predict.py
@@ -5,56 +5,82 @@
SHOW_TOP_CONTEXTS = 10
MAX_PATH_LENGTH = 8
MAX_PATH_WIDTH = 2
-EXTRACTION_API = 'https://po3g2dx2qa.execute-api.us-east-1.amazonaws.com/production/extractmethods'
+EXTRACTION_API = (
+ "https://po3g2dx2qa.execute-api.us-east-1.amazonaws.com/production/extractmethods"
+)
class InteractivePredictor:
- exit_keywords = ['exit', 'quit', 'q']
+ exit_keywords = ["exit", "quit", "q"]
def __init__(self, config, model, language):
self.model = model
self.config = config
- if language == 'java':
- self.path_extractor = JavaExtractor(config, EXTRACTION_API, self.config.MAX_PATH_LENGTH, max_path_width=2)
- elif language == 'cpp':
+ if language == "java":
+ self.path_extractor = JavaExtractor(
+ config, EXTRACTION_API, self.config.MAX_PATH_LENGTH, max_path_width=2
+ )
+ elif language == "cpp":
self.path_extractor = CppExtractor(config)
else:
- assert False, 'Unsupported language model'
+ assert False, "Unsupported language model"
@staticmethod
def read_file(input_filename):
- with open(input_filename, 'r') as file:
+ with open(input_filename, "r") as file:
return file.readlines()
def predict(self):
- input_filename = 'Input.source'
- print('Serving')
+ input_filename = "Input.source"
+ print("Serving")
while True:
- print('Modify the file: "' + input_filename + '" and press any key when ready, or "q" / "exit" to exit')
+ print(
+ 'Modify the file: "'
+ + input_filename
+ + '" and press any key when ready, or "q" / "exit" to exit'
+ )
user_input = input()
if user_input.lower() in self.exit_keywords:
- print('Exiting...')
+ print("Exiting...")
return
- user_input = ' '.join(self.read_file(input_filename))
+ user_input = " ".join(self.read_file(input_filename))
try:
- predict_lines, pc_info_dict = self.path_extractor.extract_paths(user_input)
+ predict_lines, pc_info_dict = self.path_extractor.extract_paths(
+ user_input
+ )
except ValueError:
continue
model_results = self.model.predict(predict_lines)
- prediction_results = Common.parse_results(model_results, pc_info_dict, topk=SHOW_TOP_CONTEXTS)
+ prediction_results = Common.parse_results(
+ model_results, pc_info_dict, topk=SHOW_TOP_CONTEXTS
+ )
for index, method_prediction in prediction_results.items():
- print('Original name:\t' + method_prediction.original_name)
+ print("Original name:\t" + method_prediction.original_name)
if self.config.BEAM_WIDTH == 0:
- print('Predicted:\t%s' % [step.prediction for step in method_prediction.predictions])
- for timestep, single_timestep_prediction in enumerate(method_prediction.predictions):
- print('Attention:')
- print('TIMESTEP: %d\t: %s' % (timestep, single_timestep_prediction.prediction))
+ print(
+ "Predicted:\t%s"
+ % [step.prediction for step in method_prediction.predictions]
+ )
+ for timestep, single_timestep_prediction in enumerate(
+ method_prediction.predictions
+ ):
+ print("Attention:")
+ print(
+ "TIMESTEP: %d\t: %s"
+ % (timestep, single_timestep_prediction.prediction)
+ )
for attention_obj in single_timestep_prediction.attention_paths:
- print('%f\tcontext: %s,%s,%s' % (
- attention_obj['score'], attention_obj['token1'], attention_obj['path'],
- attention_obj['token2']))
+ print(
+ "%f\tcontext: %s,%s,%s"
+ % (
+ attention_obj["score"],
+ attention_obj["token1"],
+ attention_obj["path"],
+ attention_obj["token2"],
+ )
+ )
else:
- print('Predicted:')
+ print("Predicted:")
for predicted_seq in method_prediction.predictions:
- print('\t%s' % predicted_seq.prediction)
+ print("\t%s" % predicted_seq.prediction)
diff --git a/java_extractor.py b/java_extractor.py
index 2de621b..bf3594c 100644
--- a/java_extractor.py
+++ b/java_extractor.py
@@ -11,30 +11,37 @@ def __init__(self, config, extractor_api_url, max_path_length, max_path_width):
self.max_path_length = max_path_length
self.max_path_width = max_path_width
self.extractor_api_url = extractor_api_url
- self.bad_characters_table = str.maketrans('', '', '\t\r\n')
+ self.bad_characters_table = str.maketrans("", "", "\t\r\n")
@staticmethod
def post_request(url, code_string):
- return requests.post(url, data=json.dumps({"code": code_string, "decompose": True}, separators=(',', ':')))
+ return requests.post(
+ url,
+ data=json.dumps(
+ {"code": code_string, "decompose": True}, separators=(",", ":")
+ ),
+ )
def extract_paths(self, code_string):
response = self.post_request(self.extractor_api_url, code_string)
response_array = json.loads(response.text)
- if 'errorType' in response_array:
+ if "errorType" in response_array:
raise ValueError(response.text)
- if 'errorMessage' in response_array:
+ if "errorMessage" in response_array:
raise TimeoutError(response.text)
pc_info_dict = {}
result = []
for single_method in response_array:
- method_name = single_method['target']
+ method_name = single_method["target"]
current_result_line_parts = [method_name]
- contexts = single_method['paths']
- for context in contexts[:self.config.DATA_NUM_CONTEXTS]:
+ contexts = single_method["paths"]
+ for context in contexts[: self.config.DATA_NUM_CONTEXTS]:
pc_info = PathContextInformation(context)
current_result_line_parts += [str(pc_info)]
- pc_info_dict[(pc_info.token1, pc_info.shortPath, pc_info.token2)] = pc_info
- space_padding = ' ' * (self.config.DATA_NUM_CONTEXTS - len(contexts))
- result_line = ' '.join(current_result_line_parts) + space_padding
+ pc_info_dict[
+ (pc_info.token1, pc_info.shortPath, pc_info.token2)
+ ] = pc_info
+ space_padding = " " * (self.config.DATA_NUM_CONTEXTS - len(contexts))
+ result_line = " ".join(current_result_line_parts) + space_padding
result.append(result_line)
return result, pc_info_dict
diff --git a/model.py b/model.py
index 569d304..23aadf8 100644
--- a/model.py
+++ b/model.py
@@ -6,7 +6,14 @@
class Model(tf.Module):
- def __init__(self, config, subtoken_vocab_size, target_vocab_size, nodes_vocab_size, target_to_index):
+ def __init__(
+ self,
+ config,
+ subtoken_vocab_size,
+ target_vocab_size,
+ nodes_vocab_size,
+ target_to_index,
+ ):
super().__init__()
self.config = config
self.subtoken_vocab_shape = (subtoken_vocab_size, self.config.EMBEDDINGS_SIZE)
@@ -14,24 +21,30 @@ def __init__(self, config, subtoken_vocab_size, target_vocab_size, nodes_vocab_s
self.nodes_vocab_shape = (nodes_vocab_size, self.config.EMBEDDINGS_SIZE)
self.target_to_index = target_to_index
- initializer = tf.initializers.VarianceScaling(scale=1.0,
- mode='fan_out',
- distribution='uniform')
-
- self.subtoken_vocab = tf.Variable(name='SUBTOKENS_VOCAB',
- shape=self.subtoken_vocab_shape,
- dtype=tf.float32,
- initial_value=initializer(self.subtoken_vocab_shape))
-
- self.target_words_vocab = tf.Variable(name='TARGET_WORDS_VOCAB',
- shape=self.target_vocab_shape,
- dtype=tf.float32,
- initial_value=initializer(self.target_vocab_shape))
-
- self.nodes_vocab = tf.Variable(name='NODES_VOCAB',
- shape=self.nodes_vocab_shape,
- dtype=tf.float32,
- initial_value=initializer(self.nodes_vocab_shape))
+ initializer = tf.initializers.VarianceScaling(
+ scale=1.0, mode="fan_out", distribution="uniform"
+ )
+
+ self.subtoken_vocab = tf.Variable(
+ name="SUBTOKENS_VOCAB",
+ shape=self.subtoken_vocab_shape,
+ dtype=tf.float32,
+ initial_value=initializer(self.subtoken_vocab_shape),
+ )
+
+ self.target_words_vocab = tf.Variable(
+ name="TARGET_WORDS_VOCAB",
+ shape=self.target_vocab_shape,
+ dtype=tf.float32,
+ initial_value=initializer(self.target_vocab_shape),
+ )
+
+ self.nodes_vocab = tf.Variable(
+ name="NODES_VOCAB",
+ shape=self.nodes_vocab_shape,
+ dtype=tf.float32,
+ initial_value=initializer(self.nodes_vocab_shape),
+ )
self.rnn = None
self.embed_dense_layer = None
@@ -47,51 +60,78 @@ def __init__(self, config, subtoken_vocab_size, target_vocab_size, nodes_vocab_s
def build_encoder(self):
if self.config.BIRNN:
- rnn_cell_fw = tf.keras.layers.LSTMCell(self.config.RNN_SIZE // 2,
- dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB)
- rnn_cell_bw = tf.keras.layers.LSTMCell(self.config.RNN_SIZE // 2,
- dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB)
- self.rnn = tf.keras.layers.Bidirectional(layer=tf.keras.layers.RNN(rnn_cell_fw, return_state=True),
- backward_layer=tf.keras.layers.RNN(rnn_cell_bw, go_backwards=True,
- return_state=True),
- merge_mode="concat",
- dtype=tf.float32)
+ rnn_cell_fw = tf.keras.layers.LSTMCell(
+ self.config.RNN_SIZE // 2, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB
+ )
+ rnn_cell_bw = tf.keras.layers.LSTMCell(
+ self.config.RNN_SIZE // 2, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB
+ )
+ self.rnn = tf.keras.layers.Bidirectional(
+ layer=tf.keras.layers.RNN(rnn_cell_fw, return_state=True),
+ backward_layer=tf.keras.layers.RNN(
+ rnn_cell_bw, go_backwards=True, return_state=True
+ ),
+ merge_mode="concat",
+ dtype=tf.float32,
+ )
else:
- rnn_cell = tf.keras.layers.LSTMCell(self.config.RNN_SIZE, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB)
- self.rnn = tf.keras.layers.RNN(rnn_cell, dtype=tf.float32, return_state=True)
- self.embed_dense_layer = tf.keras.layers.Dense(units=self.config.DECODER_SIZE,
- activation=tf.nn.tanh, use_bias=False)
+ rnn_cell = tf.keras.layers.LSTMCell(
+ self.config.RNN_SIZE, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB
+ )
+ self.rnn = tf.keras.layers.RNN(
+ rnn_cell, dtype=tf.float32, return_state=True
+ )
+ self.embed_dense_layer = tf.keras.layers.Dense(
+ units=self.config.DECODER_SIZE, activation=tf.nn.tanh, use_bias=False
+ )
def build_decoder(self):
decoder_cells = [
- tf.keras.layers.LSTMCell(self.config.DECODER_SIZE, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB) for _ in
- range(self.config.NUM_DECODER_LAYERS)]
+ tf.keras.layers.LSTMCell(
+ self.config.DECODER_SIZE, dropout=1 - self.config.RNN_DROPOUT_KEEP_PROB
+ )
+ for _ in range(self.config.NUM_DECODER_LAYERS)
+ ]
self.decoder_cell = tf.keras.layers.StackedRNNCells(decoder_cells)
- self.projection_layer = tf.keras.layers.Dense(units=self.target_vocab_shape[0], use_bias=False)
- self.attention_mechanism = tfa.seq2seq.LuongAttention(units=self.config.DECODER_SIZE)
+ self.projection_layer = tf.keras.layers.Dense(
+ units=self.target_vocab_shape[0], use_bias=False
+ )
+ self.attention_mechanism = tfa.seq2seq.LuongAttention(
+ units=self.config.DECODER_SIZE
+ )
should_save_alignment_history = self.config.BEAM_WIDTH == 0
- self.decoder_cell = tfa.seq2seq.AttentionWrapper(self.decoder_cell, self.attention_mechanism,
- attention_layer_size=self.config.DECODER_SIZE,
- alignment_history=should_save_alignment_history)
+ self.decoder_cell = tfa.seq2seq.AttentionWrapper(
+ self.decoder_cell,
+ self.attention_mechanism,
+ attention_layer_size=self.config.DECODER_SIZE,
+ alignment_history=should_save_alignment_history,
+ )
if self.config.BEAM_WIDTH > 0:
self.eval_decoder = tfa.seq2seq.BeamSearchDecoder(
cell=self.decoder_cell,
- embedding_fn=lambda ids: tf.nn.embedding_lookup(self._beam_embedding, ids),
+ embedding_fn=lambda ids: tf.nn.embedding_lookup(
+ self._beam_embedding, ids
+ ),
beam_width=self.config.BEAM_WIDTH,
output_layer=self.projection_layer,
maximum_iterations=self.config.MAX_TARGET_PARTS + 1,
- length_penalty_weight=0.0)
+ length_penalty_weight=0.0,
+ )
else:
greedy_sampler = tfa.seq2seq.GreedyEmbeddingSampler()
- self.eval_decoder = tfa.seq2seq.BasicDecoder(cell=self.decoder_cell,
- sampler=greedy_sampler,
- maximum_iterations=self.config.MAX_TARGET_PARTS + 1,
- output_layer=self.projection_layer)
+ self.eval_decoder = tfa.seq2seq.BasicDecoder(
+ cell=self.decoder_cell,
+ sampler=greedy_sampler,
+ maximum_iterations=self.config.MAX_TARGET_PARTS + 1,
+ output_layer=self.projection_layer,
+ )
sampler = tfa.seq2seq.sampler.TrainingSampler()
- self.train_decoder = tfa.seq2seq.BasicDecoder(cell=self.decoder_cell,
- sampler=sampler,
- maximum_iterations=self.config.MAX_TARGET_PARTS + 1,
- output_layer=self.projection_layer)
+ self.train_decoder = tfa.seq2seq.BasicDecoder(
+ cell=self.decoder_cell,
+ sampler=sampler,
+ maximum_iterations=self.config.MAX_TARGET_PARTS + 1,
+ output_layer=self.projection_layer,
+ )
@tf.function
def run_encoder(self, input_tensors, is_training):
@@ -103,16 +143,18 @@ def run_encoder(self, input_tensors, is_training):
path_lengths = input_tensors[reader.PATH_LENGTHS_KEY]
path_target_lengths = input_tensors[reader.PATH_TARGET_LENGTHS_KEY]
- batched_contexts = self.compute_contexts(subtoken_vocab=self.subtoken_vocab,
- nodes_vocab=self.nodes_vocab,
- source_input=path_source_indices,
- nodes_input=node_indices,
- target_input=path_target_indices,
- valid_mask=valid_context_mask,
- path_source_lengths=path_source_lengths,
- path_lengths=path_lengths,
- path_target_lengths=path_target_lengths,
- is_training=is_training)
+ batched_contexts = self.compute_contexts(
+ subtoken_vocab=self.subtoken_vocab,
+ nodes_vocab=self.nodes_vocab,
+ source_input=path_source_indices,
+ nodes_input=node_indices,
+ target_input=path_target_indices,
+ valid_mask=valid_context_mask,
+ path_source_lengths=path_source_lengths,
+ path_lengths=path_lengths,
+ path_target_lengths=path_target_lengths,
+ is_training=is_training,
+ )
return batched_contexts
def setup_attention_memory(self, batched_contexts):
@@ -125,118 +167,193 @@ def run_decoder(self, batched_contexts, input_tensors, is_training):
target_index = input_tensors[reader.TARGET_INDEX_KEY]
valid_context_mask = input_tensors[reader.VALID_CONTEXT_MASK_KEY]
batch_size = tf.shape(target_index)[0]
- outputs, final_states = self.decode_outputs(target_words_vocab=self.target_words_vocab,
- target_input=target_index,
- batch_size=batch_size,
- batched_contexts=batched_contexts,
- valid_mask=valid_context_mask,
- is_training=is_training)
+ outputs, final_states = self.decode_outputs(
+ target_words_vocab=self.target_words_vocab,
+ target_input=target_index,
+ batch_size=batch_size,
+ batched_contexts=batched_contexts,
+ valid_mask=valid_context_mask,
+ is_training=is_training,
+ )
return outputs, final_states
- def path_rnn_last_state(self, path_embed, path_lengths, valid_contexts_mask, is_training):
+ def path_rnn_last_state(
+ self, path_embed, path_lengths, valid_contexts_mask, is_training
+ ):
# path_embed: (batch, max_contexts, max_path_length+1, dim)
# path_length: (batch, max_contexts)
# valid_contexts_mask: (batch, max_contexts)
max_contexts = tf.shape(path_embed)[1]
# (batch * max_contexts, max_path_length+1, dim)
- flat_paths = tf.reshape(path_embed, shape=[-1, self.config.MAX_PATH_LENGTH,
- self.config.EMBEDDINGS_SIZE])
+ flat_paths = tf.reshape(
+ path_embed,
+ shape=[-1, self.config.MAX_PATH_LENGTH, self.config.EMBEDDINGS_SIZE],
+ )
flat_valid_contexts_mask = tf.expand_dims(
- tf.sequence_mask(tf.reshape(path_lengths, [-1]), maxlen=self.config.MAX_PATH_LENGTH,
- dtype=tf.float32), axis=-1)
+ tf.sequence_mask(
+ tf.reshape(path_lengths, [-1]),
+ maxlen=self.config.MAX_PATH_LENGTH,
+ dtype=tf.float32,
+ ),
+ axis=-1,
+ )
# https://github.com/tensorflow/tensorflow/issues/26974
if self.config.BIRNN:
- res = self.rnn(inputs=flat_paths, mask=flat_valid_contexts_mask,
- training=is_training)
+ res = self.rnn(
+ inputs=flat_paths, mask=flat_valid_contexts_mask, training=is_training
+ )
_, state_fw, _, state_bw, _ = res # state = [mem, carry]
- final_rnn_state = tf.concat([state_fw, state_bw], axis=-1) # (batch * max_contexts, rnn_size)
+ final_rnn_state = tf.concat(
+ [state_fw, state_bw], axis=-1
+ ) # (batch * max_contexts, rnn_size)
else:
- _, state, _ = self.rnn(inputs=flat_paths, mask=flat_valid_contexts_mask, training=is_training)
+ _, state, _ = self.rnn(
+ inputs=flat_paths, mask=flat_valid_contexts_mask, training=is_training
+ )
final_rnn_state = state
- return tf.reshape(final_rnn_state,
- shape=[-1, max_contexts, self.config.RNN_SIZE]) # (batch, max_contexts, rnn_size)
-
- def compute_contexts(self, subtoken_vocab, nodes_vocab, source_input, nodes_input,
- target_input, valid_mask, path_source_lengths, path_lengths, path_target_lengths, is_training):
-
- source_word_embed = tf.nn.embedding_lookup(params=subtoken_vocab,
- ids=source_input) # (batch, max_contexts, max_name_parts, dim)
- path_embed = tf.nn.embedding_lookup(params=nodes_vocab,
- ids=nodes_input) # (batch, max_contexts, max_path_length+1, dim)
- target_word_embed = tf.nn.embedding_lookup(params=subtoken_vocab,
- ids=target_input) # (batch, max_contexts, max_name_parts, dim)
+ return tf.reshape(
+ final_rnn_state, shape=[-1, max_contexts, self.config.RNN_SIZE]
+ ) # (batch, max_contexts, rnn_size)
+
+ def compute_contexts(
+ self,
+ subtoken_vocab,
+ nodes_vocab,
+ source_input,
+ nodes_input,
+ target_input,
+ valid_mask,
+ path_source_lengths,
+ path_lengths,
+ path_target_lengths,
+ is_training,
+ ):
+
+ source_word_embed = tf.nn.embedding_lookup(
+ params=subtoken_vocab, ids=source_input
+ ) # (batch, max_contexts, max_name_parts, dim)
+ path_embed = tf.nn.embedding_lookup(
+ params=nodes_vocab, ids=nodes_input
+ ) # (batch, max_contexts, max_path_length+1, dim)
+ target_word_embed = tf.nn.embedding_lookup(
+ params=subtoken_vocab, ids=target_input
+ ) # (batch, max_contexts, max_name_parts, dim)
source_word_mask = tf.expand_dims(
- tf.sequence_mask(path_source_lengths, maxlen=self.config.MAX_NAME_PARTS, dtype=tf.float32),
- -1) # (batch, max_contexts, max_name_parts, 1)
+ tf.sequence_mask(
+ path_source_lengths, maxlen=self.config.MAX_NAME_PARTS, dtype=tf.float32
+ ),
+ -1,
+ ) # (batch, max_contexts, max_name_parts, 1)
target_word_mask = tf.expand_dims(
- tf.sequence_mask(path_target_lengths, maxlen=self.config.MAX_NAME_PARTS, dtype=tf.float32),
- -1) # (batch, max_contexts, max_name_parts, 1)
-
- source_words_sum = tf.reduce_sum(source_word_embed * source_word_mask,
- axis=2) # (batch, max_contexts, dim)
- path_nodes_aggregation = self.path_rnn_last_state(path_embed, path_lengths,
- valid_mask, is_training) # (batch, max_contexts, rnn_size)
- target_words_sum = tf.reduce_sum(target_word_embed * target_word_mask, axis=2) # (batch, max_contexts, dim)
-
- context_embed = tf.concat([source_words_sum, path_nodes_aggregation, target_words_sum],
- axis=-1) # (batch, max_contexts, dim * 2 + rnn_size)
+ tf.sequence_mask(
+ path_target_lengths, maxlen=self.config.MAX_NAME_PARTS, dtype=tf.float32
+ ),
+ -1,
+ ) # (batch, max_contexts, max_name_parts, 1)
+
+ source_words_sum = tf.reduce_sum(
+ source_word_embed * source_word_mask, axis=2
+ ) # (batch, max_contexts, dim)
+ path_nodes_aggregation = self.path_rnn_last_state(
+ path_embed, path_lengths, valid_mask, is_training
+ ) # (batch, max_contexts, rnn_size)
+ target_words_sum = tf.reduce_sum(
+ target_word_embed * target_word_mask, axis=2
+ ) # (batch, max_contexts, dim)
+
+ context_embed = tf.concat(
+ [source_words_sum, path_nodes_aggregation, target_words_sum], axis=-1
+ ) # (batch, max_contexts, dim * 2 + rnn_size)
if is_training:
- context_embed = tf.nn.dropout(context_embed, rate=1 - self.config.EMBEDDINGS_DROPOUT_KEEP_PROB)
+ context_embed = tf.nn.dropout(
+ context_embed, rate=1 - self.config.EMBEDDINGS_DROPOUT_KEEP_PROB
+ )
batched_embed = self.embed_dense_layer(inputs=context_embed)
if not is_training and self.config.BEAM_WIDTH > 0:
- batched_embed = tfa.seq2seq.tile_batch(batched_embed, multiplier=self.config.BEAM_WIDTH)
+ batched_embed = tfa.seq2seq.tile_batch(
+ batched_embed, multiplier=self.config.BEAM_WIDTH
+ )
return batched_embed
- def decode_outputs(self, target_words_vocab, target_input, batch_size, batched_contexts, valid_mask, is_training):
+ def decode_outputs(
+ self,
+ target_words_vocab,
+ target_input,
+ batch_size,
+ batched_contexts,
+ valid_mask,
+ is_training,
+ ):
num_contexts_per_example = tf.math.count_nonzero(valid_mask, axis=-1)
- start_fill = tf.fill([batch_size],
- self.target_to_index[Common.SOS]) # (batch, )
+ start_fill = tf.fill(
+ [batch_size], self.target_to_index[Common.SOS]
+ ) # (batch, )
- contexts_sum = tf.reduce_sum(batched_contexts * tf.expand_dims(valid_mask, -1),
- axis=1) # (batch_size, dim * 2 + rnn_size)
- contexts_average = tf.divide(contexts_sum, tf.cast(tf.expand_dims(num_contexts_per_example, -1), tf.float32))
+ contexts_sum = tf.reduce_sum(
+ batched_contexts * tf.expand_dims(valid_mask, -1), axis=1
+ ) # (batch_size, dim * 2 + rnn_size)
+ contexts_average = tf.divide(
+ contexts_sum,
+ tf.cast(tf.expand_dims(num_contexts_per_example, -1), tf.float32),
+ )
- fake_encoder_state = tuple([contexts_average, contexts_average] for _ in
- range(self.config.NUM_DECODER_LAYERS))
+ fake_encoder_state = tuple(
+ [contexts_average, contexts_average]
+ for _ in range(self.config.NUM_DECODER_LAYERS)
+ )
if not is_training:
target_words_embedding = target_words_vocab
if self.config.BEAM_WIDTH > 0:
# https://medium.com/@dhirensk/tensorflow-addons-seq2seq-example-using-attention-and-beam-search-9f463b58bc6b
- decoder_initial_state = self.decoder_cell.get_initial_state(dtype=tf.float32,
- batch_size=batch_size * self.config.BEAM_WIDTH)
+ decoder_initial_state = self.decoder_cell.get_initial_state(
+ dtype=tf.float32, batch_size=batch_size * self.config.BEAM_WIDTH
+ )
decoder_initial_state = decoder_initial_state.clone(
- cell_state=tfa.seq2seq.tile_batch(fake_encoder_state, multiplier=self.config.BEAM_WIDTH))
+ cell_state=tfa.seq2seq.tile_batch(
+ fake_encoder_state, multiplier=self.config.BEAM_WIDTH
+ )
+ )
else:
- decoder_initial_state = self.decoder_cell.get_initial_state(batch_size=batch_size, dtype=tf.float32)
- decoder_initial_state = decoder_initial_state.clone(cell_state=fake_encoder_state)
+ decoder_initial_state = self.decoder_cell.get_initial_state(
+ batch_size=batch_size, dtype=tf.float32
+ )
+ decoder_initial_state = decoder_initial_state.clone(
+ cell_state=fake_encoder_state
+ )
else:
# (batch, max_target_parts, dim * 2 + rnn_size)
- target_words_embedding = tf.nn.embedding_lookup(target_words_vocab,
- tf.concat([tf.expand_dims(start_fill, -1), target_input],
- axis=-1))
-
- decoder_initial_state = self.decoder_cell.get_initial_state(batch_size=batch_size,
- dtype=tf.float32)
- decoder_initial_state = decoder_initial_state.clone(cell_state=fake_encoder_state)
+ target_words_embedding = tf.nn.embedding_lookup(
+ target_words_vocab,
+ tf.concat([tf.expand_dims(start_fill, -1), target_input], axis=-1),
+ )
+
+ decoder_initial_state = self.decoder_cell.get_initial_state(
+ batch_size=batch_size, dtype=tf.float32
+ )
+ decoder_initial_state = decoder_initial_state.clone(
+ cell_state=fake_encoder_state
+ )
if is_training:
outputs, final_states, final_sequence_lengths = self.train_decoder(
target_words_embedding,
training=True,
initial_state=decoder_initial_state,
- sequence_length=tf.ones([batch_size], dtype=tf.int32) * (self.config.MAX_TARGET_PARTS + 1))
+ sequence_length=tf.ones([batch_size], dtype=tf.int32)
+ * (self.config.MAX_TARGET_PARTS + 1),
+ )
else:
if self.config.BEAM_WIDTH > 0:
self._beam_embedding = target_words_embedding
@@ -245,13 +362,15 @@ def decode_outputs(self, target_words_vocab, target_input, batch_size, batched_c
training=False,
initial_state=decoder_initial_state,
start_tokens=start_fill,
- end_token=self.target_to_index[Common.PAD])
+ end_token=self.target_to_index[Common.PAD],
+ )
else:
outputs, final_states, final_sequence_lengths = self.eval_decoder(
target_words_embedding,
training=False,
initial_state=decoder_initial_state,
start_tokens=start_fill,
- end_token=0)
+ end_token=0,
+ )
return outputs, final_states
diff --git a/modelrunner.py b/modelrunner.py
index 32913e8..754a5df 100644
--- a/modelrunner.py
+++ b/modelrunner.py
@@ -22,66 +22,105 @@ def __init__(self, config):
self.model = None
self.load_model(self.config.LOAD_PATH)
else:
- with open('{}.dict.c2s'.format(config.TRAIN_PATH), 'rb') as file:
+ with open("{}.dict.c2s".format(config.TRAIN_PATH), "rb") as file:
subtoken_to_count = pickle.load(file)
node_to_count = pickle.load(file)
target_to_count = pickle.load(file)
max_contexts = pickle.load(file)
self.num_training_examples = pickle.load(file)
+ if self.num_training_examples == 0:
+ print("Didn't receive any training examples!")
+ print("Please check your file-paths and file-contents.")
+ sys.exit(1)
print('Num training samples: {0}'.format(self.num_training_examples))
print('Dictionaries loaded.')
if self.config.DATA_NUM_CONTEXTS <= 0:
self.config.DATA_NUM_CONTEXTS = max_contexts
- self.subtoken_to_index, self.index_to_subtoken, self.subtoken_vocab_size = \
- Common.load_vocab_from_dict(subtoken_to_count, add_values=[Common.PAD, Common.UNK],
- max_size=config.SUBTOKENS_VOCAB_MAX_SIZE)
- print('Loaded subtoken vocab. size: %d' % self.subtoken_vocab_size)
-
- self.target_to_index, self.index_to_target, self.target_vocab_size = \
- Common.load_vocab_from_dict(target_to_count, add_values=[Common.PAD, Common.UNK, Common.SOS],
- max_size=config.TARGET_VOCAB_MAX_SIZE)
- print('Loaded target word vocab. size: %d' % self.target_vocab_size)
-
- self.node_to_index, self.index_to_node, self.nodes_vocab_size = \
- Common.load_vocab_from_dict(node_to_count, add_values=[Common.PAD, Common.UNK], max_size=None)
- print('Loaded nodes vocab. size: %d' % self.nodes_vocab_size)
-
- self.model = Model(self.config, self.subtoken_vocab_size, self.target_vocab_size, self.nodes_vocab_size,
- self.target_to_index)
+ (
+ self.subtoken_to_index,
+ self.index_to_subtoken,
+ self.subtoken_vocab_size,
+ ) = Common.load_vocab_from_dict(
+ subtoken_to_count,
+ add_values=[Common.PAD, Common.UNK],
+ max_size=config.SUBTOKENS_VOCAB_MAX_SIZE,
+ )
+ print("Loaded subtoken vocab. size: %d" % self.subtoken_vocab_size)
+
+ (
+ self.target_to_index,
+ self.index_to_target,
+ self.target_vocab_size,
+ ) = Common.load_vocab_from_dict(
+ target_to_count,
+ add_values=[Common.PAD, Common.UNK, Common.SOS],
+ max_size=config.TARGET_VOCAB_MAX_SIZE,
+ )
+ print("Loaded target word vocab. size: %d" % self.target_vocab_size)
+
+ (
+ self.node_to_index,
+ self.index_to_node,
+ self.nodes_vocab_size,
+ ) = Common.load_vocab_from_dict(
+ node_to_count, add_values=[Common.PAD, Common.UNK], max_size=None
+ )
+ print("Loaded nodes vocab. size: %d" % self.nodes_vocab_size)
+
+ self.model = Model(
+ self.config,
+ self.subtoken_vocab_size,
+ self.target_vocab_size,
+ self.nodes_vocab_size,
+ self.target_to_index,
+ )
if self.config.TRAIN_PATH:
- self.train_dataset_reader = reader.Reader(subtoken_to_index=self.subtoken_to_index,
- node_to_index=self.node_to_index,
- target_to_index=self.target_to_index,
- config=self.config,
- is_evaluating=False)
+ self.train_dataset_reader = reader.Reader(
+ subtoken_to_index=self.subtoken_to_index,
+ node_to_index=self.node_to_index,
+ target_to_index=self.target_to_index,
+ config=self.config,
+ is_evaluating=False,
+ )
else:
self.train_dataset_reader = None
- self.test_dataset_reader = reader.Reader(subtoken_to_index=self.subtoken_to_index,
- node_to_index=self.node_to_index,
- target_to_index=self.target_to_index,
- config=self.config,
- is_evaluating=True)
+ self.test_dataset_reader = reader.Reader(
+ subtoken_to_index=self.subtoken_to_index,
+ node_to_index=self.node_to_index,
+ target_to_index=self.target_to_index,
+ config=self.config,
+ is_evaluating=True,
+ )
def train(self):
- print('Starting training')
+ print("Starting training")
self.print_hyperparams()
- print('Number of trainable params:',
- np.sum([np.prod(v.get_shape().as_list()) for v in self.model.trainable_variables]))
-
- print('Start training loop...')
+ print(
+ "Number of trainable params:",
+ np.sum(
+ [
+ np.prod(v.get_shape().as_list())
+ for v in self.model.trainable_variables
+ ]
+ ),
+ )
+
+ print("Start training loop...")
dataset = self.train_dataset_reader.get_dataset()
if self.config.USE_MOMENTUM:
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=0.01,
decay_steps=self.num_training_examples,
- decay_rate=0.95
+ decay_rate=0.95,
+ )
+ optimizer = tf.keras.optimizers.SGD(
+ learning_rate=lr_schedule, momentum=0.95, nesterov=True
)
- optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.95, nesterov=True)
else:
optimizer = tf.keras.optimizers.Adam()
@@ -89,16 +128,36 @@ def train(self):
checkpoint = None
checkpoint_manager = None
if self.config.MODEL_PATH:
- print('Loading model...')
- checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=self.model)
- checkpoint_manager = tf.train.CheckpointManager(checkpoint, self.config.MODEL_PATH, max_to_keep=3)
-
- if checkpoint_manager.latest_checkpoint:
+ print("Loading model...")
+ checkpoint = tf.train.Checkpoint(
+ step=tf.Variable(1), optimizer=optimizer, model=self.model
+ )
+ checkpoint_manager = tf.train.CheckpointManager(
+ checkpoint, self.config.MODEL_PATH, max_to_keep=3
+ )
+ if checkpoint_manager.latest_checkpoint and self.config.CONTINUE_FROM_CHECKPOINT == "true":
checkpoint.restore(checkpoint_manager.latest_checkpoint)
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing model from scratch.")
+ if self.config.LOAD_PATH and not self.config.TRAIN_PATH:
+ model_dirname = self.config.LOAD_PATH
+ elif self.config.MODEL_PATH:
+ model_dirname = self.config.MODEL_PATH
+ else:
+ model_dirname = None
+ print('Model directory is missing')
+ exit(-1)
+
+ stats_file_name = os.path.join(model_dirname, "stats.txt")
+ loss_file_name = os.path.join(model_dirname, "avg_loss.txt")
+ try:
+ os.remove(stats_file_name)
+ os.remove(loss_file_name)
+ except OSError:
+ pass
+
sum_loss = 0
batch_num = 0
epochs_trained = 0
@@ -117,25 +176,42 @@ def train(self):
target_index = input_tensors[reader.TARGET_INDEX_KEY]
batch_size = tf.shape(target_index)[0]
with tf.GradientTape() as tape:
- batched_contexts = self.model.run_encoder(input_tensors, is_training=True)
- outputs, _ = self.model.run_decoder(batched_contexts, input_tensors, is_training=True)
-
- logits = outputs.rnn_output # (batch, max_output_length, dim * 2 + rnn_size)
- crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target_index, logits=logits)
- target_words_nonzero = tf.sequence_mask(target_lengths + 1,
- maxlen=self.config.MAX_TARGET_PARTS + 1, dtype=tf.float32)
- loss = tf.reduce_sum(crossent * target_words_nonzero) / tf.cast(batch_size, dtype=tf.float32)
+ batched_contexts = self.model.run_encoder(
+ input_tensors, is_training=True
+ )
+ outputs, _ = self.model.run_decoder(
+ batched_contexts, input_tensors, is_training=True
+ )
+
+ logits = (
+ outputs.rnn_output
+ ) # (batch, max_output_length, dim * 2 + rnn_size)
+ crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(
+ labels=target_index, logits=logits
+ )
+ target_words_nonzero = tf.sequence_mask(
+ target_lengths + 1,
+ maxlen=self.config.MAX_TARGET_PARTS + 1,
+ dtype=tf.float32,
+ )
+ loss = tf.reduce_sum(crossent * target_words_nonzero) / tf.cast(
+ batch_size, dtype=tf.float32
+ )
gradients = tape.gradient(loss, self.model.trainable_variables)
if self.config.USE_MOMENTUM:
- clipped_gradients, _ = tf.clip_by_global_norm(gradients, clip_norm=5)
- optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))
+ clipped_gradients, _ = tf.clip_by_global_norm(
+ gradients, clip_norm=5
+ )
+ optimizer.apply_gradients(
+ zip(gradients, self.model.trainable_variables)
+ )
sum_loss += loss
batch_num += 1
if batch_num % self.num_batches_to_log == 0:
- self.trace(pbar, sum_loss, batch_num, multi_batch_start_time)
+ self.trace(pbar, sum_loss, batch_num, multi_batch_start_time, loss_file_name)
sum_loss = 0
multi_batch_start_time = time.time()
@@ -144,7 +220,7 @@ def train(self):
# the end of an epoch
epochs_trained += 1
- print('Finished {0} epochs'.format(epochs_trained))
+ print("Finished {0} epochs".format(epochs_trained))
if epochs_trained % self.config.SAVE_EVERY_EPOCHS == 0:
if self.config.MODEL_PATH:
print("Checkpoint saved")
@@ -152,14 +228,21 @@ def train(self):
checkpoint_manager.save()
# validate model to calculate metrics or stop training
- results, precision, recall, f1, rouge = self.evaluate()
+ results, precision, recall, f1, rouge = self.evaluate(model_dirname)
+
+ # Add results to a stats file for later processing of graphs
+ with open(stats_file_name, "a+") as stats_file:
+ stats_file.write("{0}, {1}, {2}, {3}, {4}\n".format(epochs_trained, results, precision, recall, f1))
+
if self.config.BEAM_WIDTH == 0:
- print('Accuracy after %d epochs: %.5f' % (epochs_trained, results))
+ print("Accuracy after %d epochs: %.5f" % (epochs_trained, results))
else:
- print('Accuracy after {} epochs: {}'.format(epochs_trained, results))
- print('After %d epochs: Precision: %.5f, recall: %.5f, F1: %.5f' % (
- epochs_trained, precision, recall, f1))
- print('Rouge: ', rouge)
+ print("Accuracy after {} epochs: {}".format(epochs_trained, results))
+ print(
+ "After %d epochs: Precision: %.5f, recall: %.5f, F1: %.5f"
+ % (epochs_trained, precision, recall, f1)
+ )
+ print("Rouge: ", rouge)
if f1 > best_f1:
best_f1 = f1
best_f1_precision = precision
@@ -169,149 +252,217 @@ def train(self):
else:
epochs_no_improve += self.config.SAVE_EVERY_EPOCHS
if epochs_no_improve >= self.config.PATIENCE:
- print('Not improved for %d epochs, stopping training' % self.config.PATIENCE)
- print('Best scores - epoch %d: ' % best_epoch)
- print('Precision: %.5f, recall: %.5f, F1: %.5f' % (best_f1_precision, best_f1_recall, best_f1))
+ print(
+ "Not improved for %d epochs, stopping training"
+ % self.config.PATIENCE
+ )
+ print("Best scores - epoch %d: " % best_epoch)
+ print(
+ "Precision: %.5f, recall: %.5f, F1: %.5f"
+ % (best_f1_precision, best_f1_recall, best_f1)
+ )
break
# the end of training
if self.config.SAVE_PATH:
self.save_model(self.config.SAVE_PATH)
- print('Model saved into : {0}'.format(self.config.SAVE_PATH))
+ print("Model saved into : {0}".format(self.config.SAVE_PATH))
elapsed = int(time.time() - start_time)
- print("Training time: %sh%sm%ss\n" % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60))
+ print(
+ "Training time: %sh%sm%ss\n"
+ % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60)
+ )
- def evaluate(self):
+ def evaluate(self, model_dirname):
if not self.model:
- print('Model is not initialized')
+ print("Model is not initialized")
exit(-1)
print("Testing...")
eval_start_time = time.time()
- if self.config.LOAD_PATH and not self.config.TRAIN_PATH:
- model_dirname = os.path.dirname(self.config.LOAD_PATH)
- elif self.config.MODEL_PATH:
- model_dirname = os.path.dirname(self.config.MODEL_PATH)
- else:
- model_dirname = None
- print('Model directory is mossing')
- exit(-1)
+
- ref_file_name = os.path.join(model_dirname, 'ref.txt')
- predicted_file_name = os.path.join(model_dirname, 'pred.txt')
+ ref_file_name = os.path.join(model_dirname, "ref.txt")
+ predicted_file_name = os.path.join(model_dirname, "pred.txt")
if not os.path.exists(model_dirname):
os.makedirs(model_dirname)
- log_file_name = os.path.join(model_dirname, 'log.txt')
- with open(log_file_name, 'w') as output_file, open(ref_file_name, 'w') as ref_file, open(
- predicted_file_name,
- 'w') as pred_file:
- num_correct_predictions = 0 if self.config.BEAM_WIDTH == 0 \
+ log_file_name = os.path.join(model_dirname, "log.txt")
+ with open(log_file_name, "w") as output_file, open(
+ ref_file_name, "w"
+ ) as ref_file, open(predicted_file_name, "w") as pred_file:
+ num_correct_predictions = (
+ 0
+ if self.config.BEAM_WIDTH == 0
else np.zeros([self.config.BEAM_WIDTH], dtype=np.int32)
+ )
total_predictions = 0
total_prediction_batches = 0
true_positive, false_positive, false_negative = 0, 0, 0
dataset = self.test_dataset_reader.get_dataset()
+ if not any(True for _ in dataset):
+ print("Evaluation Dataset was Empty!")
+ print("Please check your file-paths and file-contents.")
+ sys.exit(1)
+
start_time = time.time()
for input_tensors in dataset:
true_target_strings = input_tensors[reader.TARGET_STRING_KEY]
- batched_contexts = self.model.run_encoder(input_tensors, is_training=False)
- outputs, final_states = self.model.run_decoder(batched_contexts, input_tensors, is_training=False)
+ batched_contexts = self.model.run_encoder(
+ input_tensors, is_training=False
+ )
+ outputs, final_states = self.model.run_decoder(
+ batched_contexts, input_tensors, is_training=False
+ )
if self.config.BEAM_WIDTH > 0:
predicted_indices = outputs.predicted_ids
else:
predicted_indices = outputs.sample_id
- true_target_strings = Common.binary_to_string_list(true_target_strings.numpy())
+ true_target_strings = Common.binary_to_string_list(
+ true_target_strings.numpy()
+ )
ref_file.write(
- '\n'.join(
- [name.replace(Common.internal_delimiter, ' ') for name in true_target_strings]) + '\n')
+ "\n".join(
+ [
+ name.replace(Common.internal_delimiter, " ")
+ for name in true_target_strings
+ ]
+ )
+ + "\n"
+ )
if self.config.BEAM_WIDTH > 0:
# predicted indices: (batch, time, beam_width)
- predicted_strings = [[[self.index_to_target[i] for i in timestep] for timestep in example] for
- example in predicted_indices.numpy()]
- predicted_strings = [list(map(list, zip(*example))) for example in
- predicted_strings] # (batch, top-k, target_length)
- pred_file.write('\n'.join(
- [' '.join(Common.filter_impossible_names(words)) for words in predicted_strings[0]]) + '\n')
+ predicted_strings = [
+ [
+ [self.index_to_target[i] for i in timestep]
+ for timestep in example
+ ]
+ for example in predicted_indices.numpy()
+ ]
+ predicted_strings = [
+ list(map(list, zip(*example))) for example in predicted_strings
+ ] # (batch, top-k, target_length)
+ pred_file.write(
+ "\n".join(
+ [
+ " ".join(Common.filter_impossible_names(words))
+ for words in predicted_strings[0]
+ ]
+ )
+ + "\n"
+ )
else:
- predicted_strings = [[self.index_to_target[i] for i in example]
- for example in predicted_indices.numpy()]
- pred_file.write('\n'.join(
- [' '.join(Common.filter_impossible_names(words)) for words in predicted_strings]) + '\n')
-
- num_correct_predictions = update_correct_predictions(self.config.BEAM_WIDTH, num_correct_predictions,
- output_file,
- zip(true_target_strings,
- predicted_strings))
- true_positive, false_positive, false_negative = update_per_subtoken_statistics(self.config.BEAM_WIDTH,
- zip(true_target_strings,
- predicted_strings),
- true_positive,
- false_positive,
- false_negative)
+ predicted_strings = [
+ [self.index_to_target[i] for i in example]
+ for example in predicted_indices.numpy()
+ ]
+ pred_file.write(
+ "\n".join(
+ [
+ " ".join(Common.filter_impossible_names(words))
+ for words in predicted_strings
+ ]
+ )
+ + "\n"
+ )
+
+ num_correct_predictions = update_correct_predictions(
+ self.config.BEAM_WIDTH,
+ num_correct_predictions,
+ output_file,
+ zip(true_target_strings, predicted_strings),
+ )
+ (
+ true_positive,
+ false_positive,
+ false_negative,
+ ) = update_per_subtoken_statistics(
+ self.config.BEAM_WIDTH,
+ zip(true_target_strings, predicted_strings),
+ true_positive,
+ false_positive,
+ false_negative,
+ )
total_predictions += len(true_target_strings)
total_prediction_batches += 1
if total_prediction_batches % self.num_batches_to_log == 0:
elapsed = time.time() - start_time
- trace_evaluation(output_file, num_correct_predictions, total_predictions, elapsed)
+ trace_evaluation(
+ output_file, num_correct_predictions, total_predictions, elapsed
+ )
- print('Done testing, epoch reached', flush=True)
- output_file.write(str(num_correct_predictions / total_predictions) + '\n')
+ print("Done testing, epoch reached", flush=True)
+ output_file.write(str(num_correct_predictions / total_predictions) + "\n")
elapsed = int(time.time() - eval_start_time)
precision, recall, f1 = calculate_results(true_positive, false_positive, false_negative)
- files_rouge = FilesRouge(predicted_file_name, ref_file_name)
- rouge = files_rouge.get_scores(avg=True, ignore_empty=True)
+ accuracy = num_correct_predictions / total_predictions
+
+ try:
+ files_rouge = FilesRouge(predicted_file_name, ref_file_name)
+ rouge = files_rouge.get_scores(avg=True, ignore_empty=True)
+ except ValueError:
+ rouge = 0
+
print("Evaluation time: %sh%sm%ss" % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60))
- return num_correct_predictions / total_predictions, precision, recall, f1, rouge
+ return accuracy, precision, recall, f1, rouge
def print_hyperparams(self):
- print('Training batch size:\t\t\t', self.config.BATCH_SIZE)
- print('Dataset path:\t\t\t\t', self.config.TRAIN_PATH)
- print('Training file path:\t\t\t', self.config.TRAIN_PATH + '.train.c2s')
- print('Validation path:\t\t\t', self.config.TEST_PATH)
- print('Taking max contexts from each example:\t', self.config.MAX_CONTEXTS)
- print('Random path sampling:\t\t\t', self.config.RANDOM_CONTEXTS)
- print('Embedding size:\t\t\t\t', self.config.EMBEDDINGS_SIZE)
+ print("Training batch size:\t\t\t", self.config.BATCH_SIZE)
+ print("Dataset path:\t\t\t\t", self.config.TRAIN_PATH)
+ print("Training file path:\t\t\t", self.config.TRAIN_PATH + ".train.c2s")
+ print("Validation path:\t\t\t", self.config.TEST_PATH)
+ print("Taking max contexts from each example:\t", self.config.MAX_CONTEXTS)
+ print("Random path sampling:\t\t\t", self.config.RANDOM_CONTEXTS)
+ print("Embedding size:\t\t\t\t", self.config.EMBEDDINGS_SIZE)
if self.config.BIRNN:
- print('Using BiLSTMs, each of size:\t\t', self.config.RNN_SIZE // 2)
+ print("Using BiLSTMs, each of size:\t\t", self.config.RNN_SIZE // 2)
else:
- print('Uni-directional LSTM of size:\t\t', self.config.RNN_SIZE)
- print('Decoder size:\t\t\t\t', self.config.DECODER_SIZE)
- print('Decoder layers:\t\t\t\t', self.config.NUM_DECODER_LAYERS)
- print('Max path lengths:\t\t\t', self.config.MAX_PATH_LENGTH)
- print('Max subtokens in a token:\t\t', self.config.MAX_NAME_PARTS)
- print('Max target length:\t\t\t', self.config.MAX_TARGET_PARTS)
- print('Embeddings dropout keep_prob:\t\t', self.config.EMBEDDINGS_DROPOUT_KEEP_PROB)
- print('LSTM dropout keep_prob:\t\t\t', self.config.RNN_DROPOUT_KEEP_PROB)
- print('============================================')
-
- def trace(self, pbar, sum_loss, batch_num, multi_batch_start_time):
+ print("Uni-directional LSTM of size:\t\t", self.config.RNN_SIZE)
+ print("Decoder size:\t\t\t\t", self.config.DECODER_SIZE)
+ print("Decoder layers:\t\t\t\t", self.config.NUM_DECODER_LAYERS)
+ print("Max path lengths:\t\t\t", self.config.MAX_PATH_LENGTH)
+ print("Max subtokens in a token:\t\t", self.config.MAX_NAME_PARTS)
+ print("Max target length:\t\t\t", self.config.MAX_TARGET_PARTS)
+ print(
+ "Embeddings dropout keep_prob:\t\t",
+ self.config.EMBEDDINGS_DROPOUT_KEEP_PROB,
+ )
+ print("LSTM dropout keep_prob:\t\t\t", self.config.RNN_DROPOUT_KEEP_PROB)
+ print("============================================")
+
+ def trace(self, pbar, sum_loss, batch_num, multi_batch_start_time, loss_file_name):
multi_batch_elapsed = time.time() - multi_batch_start_time
avg_loss = sum_loss / self.num_batches_to_log
- msg = 'Average loss at batch {0}: {1}, \tthroughput: {2} samples/sec'. \
- format(batch_num, avg_loss,
- self.config.BATCH_SIZE * self.num_batches_to_log / (
- multi_batch_elapsed if multi_batch_elapsed > 0 else 1))
+ throughput = self.config.BATCH_SIZE * self.num_batches_to_log / (multi_batch_elapsed if multi_batch_elapsed > 0 else 1)
+ msg = "Average loss at batch {0}: {1}, \tthroughput: {2} samples/sec".format(
+ batch_num,
+ avg_loss,
+ throughput
+ )
pbar.set_description(msg)
+ with open(loss_file_name, "a+") as loss_file:
+ loss_file.write("{0}, {1}, {2}\n".format(batch_num, avg_loss, throughput))
def encode(self, predict_data_lines):
if not self.model:
- print('Model is not initialized')
+ print("Model is not initialized")
exit(-1)
- predict_reader = reader.Reader(subtoken_to_index=self.subtoken_to_index,
- node_to_index=self.node_to_index,
- target_to_index=self.target_to_index,
- config=self.config,
- is_evaluating=True)
+ predict_reader = reader.Reader(
+ subtoken_to_index=self.subtoken_to_index,
+ node_to_index=self.node_to_index,
+ target_to_index=self.target_to_index,
+ config=self.config,
+ is_evaluating=True,
+ )
results = []
for line in predict_data_lines:
input_tensors = predict_reader.process_from_placeholder(line)
@@ -321,14 +472,16 @@ def encode(self, predict_data_lines):
def predict(self, predict_data_lines):
if not self.model:
- print('Model is not initialized')
+ print("Model is not initialized")
exit(-1)
- predict_reader = reader.Reader(subtoken_to_index=self.subtoken_to_index,
- node_to_index=self.node_to_index,
- target_to_index=self.target_to_index,
- config=self.config,
- is_evaluating=True)
+ predict_reader = reader.Reader(
+ subtoken_to_index=self.subtoken_to_index,
+ node_to_index=self.node_to_index,
+ target_to_index=self.target_to_index,
+ config=self.config,
+ is_evaluating=True,
+ )
results = []
for line in predict_data_lines:
input_tensors = predict_reader.process_from_placeholder(line)
@@ -339,7 +492,9 @@ def predict(self, predict_data_lines):
true_target_strings = input_tensors[reader.TARGET_STRING_KEY]
batched_contexts = self.model.run_encoder(input_tensors, is_training=False)
- outputs, final_states = self.model.run_decoder(batched_contexts, input_tensors, is_training=False)
+ outputs, final_states = self.model.run_decoder(
+ batched_contexts, input_tensors, is_training=False
+ )
if self.config.BEAM_WIDTH > 0:
predicted_indices = outputs.predicted_ids
@@ -348,54 +503,76 @@ def predict(self, predict_data_lines):
else:
predicted_indices = outputs.sample_id
top_scores = tf.constant(1, shape=(1, 1), dtype=tf.float32)
- attention_weights = tf.squeeze(final_states.alignment_history.stack(), 1)
+ attention_weights = tf.squeeze(
+ final_states.alignment_history.stack(), 1
+ )
top_scores = np.squeeze(top_scores.numpy(), axis=0)
path_source_string = path_source_string.numpy().reshape((-1))
path_strings = path_strings.numpy().reshape((-1))
path_target_string = path_target_string.numpy().reshape((-1))
predicted_indices = np.squeeze(predicted_indices.numpy(), axis=0)
- true_target_strings = Common.binary_to_string(true_target_strings.numpy()[0])
+ true_target_strings = Common.binary_to_string(
+ true_target_strings.numpy()[0]
+ )
if self.config.BEAM_WIDTH > 0:
- predicted_strings = [[self.index_to_target[sugg] for sugg in timestep]
- for timestep in predicted_indices] # (target_length, top-k)
- predicted_strings = list(map(list, zip(*predicted_strings))) # (top-k, target_length)
+ predicted_strings = [
+ [self.index_to_target[sugg] for sugg in timestep]
+ for timestep in predicted_indices
+ ] # (target_length, top-k)
+ predicted_strings = list(
+ map(list, zip(*predicted_strings))
+ ) # (top-k, target_length)
top_scores = [np.exp(np.sum(s)) for s in zip(*top_scores)]
else:
- predicted_strings = [self.index_to_target[idx]
- for idx in predicted_indices] # (batch, target_length)
+ predicted_strings = [
+ self.index_to_target[idx] for idx in predicted_indices
+ ] # (batch, target_length)
attention_per_path = None
if self.config.BEAM_WIDTH == 0:
- attention_per_path = self.get_attention_per_path(path_source_string, path_strings, path_target_string,
- attention_weights.numpy())
-
- results.append((true_target_strings, predicted_strings, top_scores, attention_per_path))
+ attention_per_path = self.get_attention_per_path(
+ path_source_string,
+ path_strings,
+ path_target_string,
+ attention_weights.numpy(),
+ )
+
+ results.append(
+ (true_target_strings, predicted_strings, top_scores, attention_per_path)
+ )
return results
@staticmethod
- def get_attention_per_path(source_strings, path_strings, target_strings, attention_weights):
+ def get_attention_per_path(
+ source_strings, path_strings, target_strings, attention_weights
+ ):
# attention_weights: (time, contexts)
results = []
for time_step in attention_weights:
attention_per_context = {}
- for source, path, target, weight in zip(source_strings, path_strings, target_strings, time_step):
+ for source, path, target, weight in zip(
+ source_strings, path_strings, target_strings, time_step
+ ):
string_triplet = (
- Common.binary_to_string(source), Common.binary_to_string(path), Common.binary_to_string(target))
+ Common.binary_to_string(source),
+ Common.binary_to_string(path),
+ Common.binary_to_string(target),
+ )
attention_per_context[string_triplet] = weight
results.append(attention_per_context)
return results
def save_model(self, path):
- path_name = os.path.dirname(path)
+ path_name = self.config.SAVE_PATH
if not os.path.exists(path_name):
os.makedirs(path_name)
checkpoint = tf.train.Checkpoint(model=self.model)
- checkpoint.save(os.path.join(path_name, 'model'))
+ checkpoint.save(os.path.join(path_name, "model"))
- dictionaries_path = os.path.join(path_name, 'model.dict')
- with open(dictionaries_path, 'wb') as file:
+ dictionaries_path = os.path.join(path_name, "model.dict")
+ with open(dictionaries_path, "wb") as file:
pickle.dump(self.subtoken_to_index, file)
pickle.dump(self.index_to_subtoken, file)
pickle.dump(self.subtoken_vocab_size, file)
@@ -412,9 +589,9 @@ def save_model(self, path):
pickle.dump(self.config, file)
def load_model(self, path):
- path_name = os.path.dirname(path)
+ path_name = self.config.SAVE_PATH
if os.path.exists(path_name):
- with open(os.path.join(path_name, 'model.dict'), 'rb') as file:
+ with open(os.path.join(path_name, "model.dict"), "rb") as file:
self.subtoken_to_index = pickle.load(file)
self.index_to_subtoken = pickle.load(file)
self.subtoken_vocab_size = pickle.load(file)
@@ -431,8 +608,13 @@ def load_model(self, path):
saved_config = pickle.load(file)
self.config.take_model_hyperparams_from(saved_config)
- self.model = Model(self.config, self.subtoken_vocab_size, self.target_vocab_size, self.nodes_vocab_size,
- self.target_to_index)
+ self.model = Model(
+ self.config,
+ self.subtoken_vocab_size,
+ self.target_vocab_size,
+ self.nodes_vocab_size,
+ self.target_to_index,
+ )
checkpoint = tf.train.Checkpoint(model=self.model)
status = checkpoint.restore(tf.train.latest_checkpoint(path))
status.expect_partial()
diff --git a/preprocess.py b/preprocess.py
index f4ef82b..fbfe502 100644
--- a/preprocess.py
+++ b/preprocess.py
@@ -5,34 +5,45 @@
import common
-'''
+"""
This script preprocesses the data from MethodPaths. It truncates methods with too many contexts,
and pads methods with less paths with spaces.
-'''
-
-
-def save_dictionaries(dataset_name, subtoken_to_count, node_to_count, target_to_count, max_contexts, num_examples):
- save_dict_file_path = '{}.dict.c2s'.format(dataset_name)
- with open(save_dict_file_path, 'wb') as file:
+"""
+
+
+def save_dictionaries(
+ dataset_name,
+ subtoken_to_count,
+ node_to_count,
+ target_to_count,
+ max_contexts,
+ num_examples,
+):
+ save_dict_file_path = "{}.dict.c2s".format(dataset_name)
+ with open(save_dict_file_path, "wb") as file:
pickle.dump(subtoken_to_count, file)
pickle.dump(node_to_count, file)
pickle.dump(target_to_count, file)
pickle.dump(max_contexts, file)
pickle.dump(num_examples, file)
- print('Dictionaries saved to: {}'.format(save_dict_file_path))
+ print("Dictionaries saved to: {}".format(save_dict_file_path))
-def process_file(file_path, data_file_role, dataset_name, max_contexts, max_data_contexts):
+def process_file(
+ file_path, data_file_role, dataset_name, max_contexts, max_data_contexts
+):
sum_total = 0
sum_sampled = 0
total = 0
max_unfiltered = 0
- max_contexts_to_sample = max_data_contexts if data_file_role == 'train' else max_contexts
- output_path = '{}.{}.c2s'.format(dataset_name, data_file_role)
- with open(output_path, 'w') as outfile:
- with open(file_path, 'r') as file:
+ max_contexts_to_sample = (
+ max_data_contexts if data_file_role == "train" else max_contexts
+ )
+ output_path = "{}.{}.c2s".format(dataset_name, data_file_role)
+ with open(output_path, "w") as outfile:
+ with open(file_path, "r") as file:
for line in file:
- parts = line.rstrip('\n').split(' ')
+ parts = line.rstrip("\n").split(" ")
target_name = parts[0]
contexts = parts[1:]
@@ -41,56 +52,129 @@ def process_file(file_path, data_file_role, dataset_name, max_contexts, max_data
sum_total += len(contexts)
if len(contexts) > max_contexts_to_sample:
- contexts = np.random.choice(contexts, max_contexts_to_sample, replace=False)
+ contexts = np.random.choice(
+ contexts, max_contexts_to_sample, replace=False
+ )
sum_sampled += len(contexts)
csv_padding = " " * (max_data_contexts - len(contexts))
total += 1
- outfile.write(target_name + ' ' + " ".join(contexts) + csv_padding + '\n')
-
- print('File: ' + data_file_path)
- print('Average total contexts: ' + str(float(sum_total) / total))
- print('Average final (after sampling) contexts: ' + str(float(sum_sampled) / total))
- print('Total examples: ' + str(total))
- print('Max number of contexts per word: ' + str(max_unfiltered))
+ outfile.write(
+ target_name + " " + " ".join(contexts) + csv_padding + "\n"
+ )
+
+ print("File: " + data_file_path)
+ print("Average total contexts: " + str(float(sum_total) / total))
+ print("Average final (after sampling) contexts: " + str(float(sum_sampled) / total))
+ print("Total examples: " + str(total))
+ print("Max number of contexts per word: " + str(max_unfiltered))
return total
def context_full_found(context_parts, word_to_count, path_to_count):
- return context_parts[0] in word_to_count \
- and context_parts[1] in path_to_count and context_parts[2] in word_to_count
+ return (
+ context_parts[0] in word_to_count
+ and context_parts[1] in path_to_count
+ and context_parts[2] in word_to_count
+ )
def context_partial_found(context_parts, word_to_count, path_to_count):
- return context_parts[0] in word_to_count \
- or context_parts[1] in path_to_count or context_parts[2] in word_to_count
+ return (
+ context_parts[0] in word_to_count
+ or context_parts[1] in path_to_count
+ or context_parts[2] in word_to_count
+ )
-if __name__ == '__main__':
+if __name__ == "__main__":
parser = ArgumentParser()
- parser.add_argument("-trd", "--train_data", dest="train_data_path",
- help="path to training data file", required=True)
- parser.add_argument("-ted", "--test_data", dest="test_data_path",
- help="path to test data file", required=True)
- parser.add_argument("-vd", "--val_data", dest="val_data_path",
- help="path to validation data file", required=True)
- parser.add_argument("-mc", "--max_contexts", dest="max_contexts", default=200,
- help="number of max contexts to keep in test+validation", required=False)
- parser.add_argument("-mdc", "--max_data_contexts", dest="max_data_contexts", default=1000,
- help="number of max contexts to keep in the dataset", required=False)
- parser.add_argument("-svs", "--subtoken_vocab_size", dest="subtoken_vocab_size", default=186277,
- help="Max number of source subtokens to keep in the vocabulary", required=False)
- parser.add_argument("-tvs", "--target_vocab_size", dest="target_vocab_size", default=26347,
- help="Max number of target words to keep in the vocabulary", required=False)
- parser.add_argument("-sh", "--subtoken_histogram", dest="subtoken_histogram",
- help="subtoken histogram file", metavar="FILE", required=True)
- parser.add_argument("-nh", "--node_histogram", dest="node_histogram",
- help="node_histogram file", metavar="FILE", required=True)
- parser.add_argument("-th", "--target_histogram", dest="target_histogram",
- help="target histogram file", metavar="FILE", required=True)
- parser.add_argument("-o", "--output_name", dest="output_name",
- help="output name - the base name for the created dataset", required=True, default='data')
+ parser.add_argument(
+ "-trd",
+ "--train_data",
+ dest="train_data_path",
+ help="path to training data file",
+ required=True,
+ )
+ parser.add_argument(
+ "-ted",
+ "--test_data",
+ dest="test_data_path",
+ help="path to test data file",
+ required=True,
+ )
+ parser.add_argument(
+ "-vd",
+ "--val_data",
+ dest="val_data_path",
+ help="path to validation data file",
+ required=True,
+ )
+ parser.add_argument(
+ "-mc",
+ "--max_contexts",
+ dest="max_contexts",
+ default=200,
+ help="number of max contexts to keep in test+validation",
+ required=False,
+ )
+ parser.add_argument(
+ "-mdc",
+ "--max_data_contexts",
+ dest="max_data_contexts",
+ default=1000,
+ help="number of max contexts to keep in the dataset",
+ required=False,
+ )
+ parser.add_argument(
+ "-svs",
+ "--subtoken_vocab_size",
+ dest="subtoken_vocab_size",
+ default=186277,
+ help="Max number of source subtokens to keep in the vocabulary",
+ required=False,
+ )
+ parser.add_argument(
+ "-tvs",
+ "--target_vocab_size",
+ dest="target_vocab_size",
+ default=26347,
+ help="Max number of target words to keep in the vocabulary",
+ required=False,
+ )
+ parser.add_argument(
+ "-sh",
+ "--subtoken_histogram",
+ dest="subtoken_histogram",
+ help="subtoken histogram file",
+ metavar="FILE",
+ required=True,
+ )
+ parser.add_argument(
+ "-nh",
+ "--node_histogram",
+ dest="node_histogram",
+ help="node_histogram file",
+ metavar="FILE",
+ required=True,
+ )
+ parser.add_argument(
+ "-th",
+ "--target_histogram",
+ dest="target_histogram",
+ help="target histogram file",
+ metavar="FILE",
+ required=True,
+ )
+ parser.add_argument(
+ "-o",
+ "--output_name",
+ dest="output_name",
+ help="output name - the base name for the created dataset",
+ required=True,
+ default="data",
+ )
args = parser.parse_args()
train_data_path = args.train_data_path
@@ -99,23 +183,36 @@ def context_partial_found(context_parts, word_to_count, path_to_count):
subtoken_histogram_path = args.subtoken_histogram
node_histogram_path = args.node_histogram
- subtoken_to_count = common.Common.load_histogram(subtoken_histogram_path,
- max_size=int(args.subtoken_vocab_size))
- node_to_count = common.Common.load_histogram(node_histogram_path,
- max_size=None)
- target_to_count = common.Common.load_histogram(args.target_histogram,
- max_size=int(args.target_vocab_size))
- print('subtoken vocab size: ', len(subtoken_to_count))
- print('node vocab size: ', len(node_to_count))
- print('target vocab size: ', len(target_to_count))
+ subtoken_to_count = common.Common.load_histogram(
+ subtoken_histogram_path, max_size=int(args.subtoken_vocab_size)
+ )
+ node_to_count = common.Common.load_histogram(node_histogram_path, max_size=None)
+ target_to_count = common.Common.load_histogram(
+ args.target_histogram, max_size=int(args.target_vocab_size)
+ )
+ print("subtoken vocab size: ", len(subtoken_to_count))
+ print("node vocab size: ", len(node_to_count))
+ print("target vocab size: ", len(target_to_count))
num_training_examples = 0
- for data_file_path, data_role in zip([test_data_path, val_data_path, train_data_path], ['test', 'val', 'train']):
- num_examples = process_file(file_path=data_file_path, data_file_role=data_role, dataset_name=args.output_name,
- max_contexts=int(args.max_contexts), max_data_contexts=int(args.max_data_contexts))
- if data_role == 'train':
+ for data_file_path, data_role in zip(
+ [test_data_path, val_data_path, train_data_path], ["test", "val", "train"]
+ ):
+ num_examples = process_file(
+ file_path=data_file_path,
+ data_file_role=data_role,
+ dataset_name=args.output_name,
+ max_contexts=int(args.max_contexts),
+ max_data_contexts=int(args.max_data_contexts),
+ )
+ if data_role == "train":
num_training_examples = num_examples
- save_dictionaries(dataset_name=args.output_name, subtoken_to_count=subtoken_to_count,
- node_to_count=node_to_count, target_to_count=target_to_count,
- max_contexts=int(args.max_data_contexts), num_examples=num_training_examples)
+ save_dictionaries(
+ dataset_name=args.output_name,
+ subtoken_to_count=subtoken_to_count,
+ node_to_count=node_to_count,
+ target_to_count=target_to_count,
+ max_contexts=int(args.max_data_contexts),
+ num_examples=num_training_examples,
+ )
diff --git a/preprocess.sh b/preprocess.sh
old mode 100644
new mode 100755
index 53644de..24eff20
--- a/preprocess.sh
+++ b/preprocess.sh
@@ -21,10 +21,56 @@
# recommended to use a multi-core machine for the preprocessing
# step and set this value to the number of cores.
# PYTHON - python3 interpreter alias.
-TRAIN_DIR=my_training_dir
-VAL_DIR=my_val_dir
-TEST_DIR=my_test_dir
-DATASET_NAME=my_dataset
+
+# set -e makes the shell script exit if any command exists with non-zero exit code
+set -e
+
+# Default preprocessing values
+DATASET_NAME=default
+VARIANT=default
+INCLUDE_COMMENTS=true
+EXCLUDE_STOPWORDS=false
+USE_TFIDF=false
+NUMBER_OF_TFIDF_KEYWORDS=45
+
+# This code block is used to get long two-dash arguments from the command line.
+die() { echo "$*" >&2; exit 2; } # complain to STDERR and exit with error
+needs_arg() { if [ -z "$OPTARG" ]; then die "No arg for --$OPT option"; fi; }
+
+while getopts ab:c:-: OPT; do
+ # support long options: https://stackoverflow.com/a/28466267/519360
+ if [ "$OPT" = "-" ]; then # long option: reformulate OPT and OPTARG
+ OPT="${OPTARG%%=*}" # extract long option name
+ OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty)
+ OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=`
+ fi
+ case "$OPT" in
+ dataset ) DATASET_NAME="$OPTARG" ;;
+ include_comments ) INCLUDE_COMMENTS="$OPTARG" ;;
+ exclude_stopwords ) EXCLUDE_STOPWORDS="$OPTARG" ;;
+ include_tfidf ) USE_TFIDF="$OPTARG" ;;
+ number_keywords ) NUMBER_OF_TFIDF_KEYWORDS="$OPTARG" ;;
+ variant ) VARIANT="$OPTARG" ;;
+ ??* ) die "Illegal option --$OPT" ;; # bad long option
+ ? ) exit 2 ;; # bad short option (error reported via getopts)
+ esac
+done
+shift $((OPTIND-1)) # remove parsed options and args from $@ list
+
+echo "Dataset: $DATASET_NAME"
+echo "Variant: $VARIANT"
+echo "Including comments: $INCLUDE_COMMENTS"
+echo "Excluding stopwords: $EXCLUDE_STOPWORDS"
+echo "Using TFIDF: $USE_TFIDF"
+echo "TFIDF keywords: $NUMBER_OF_TFIDF_KEYWORDS"
+
+
+INPUT_DIR=datasets
+TRAIN_DIR=${INPUT_DIR}/${DATASET_NAME}/raw/train
+VAL_DIR=${INPUT_DIR}/${DATASET_NAME}/raw/valid
+TEST_DIR=${INPUT_DIR}/${DATASET_NAME}/raw/test
+
+# Preprocessing configs
MAX_DATA_CONTEXTS=1000
MAX_CONTEXTS=200
SUBTOKEN_VOCAB_SIZE=186277
@@ -33,27 +79,28 @@ NUM_THREADS=64
PYTHON=python3
###########################################################
-TRAIN_DATA_FILE=${DATASET_NAME}.train.raw.txt
-VAL_DATA_FILE=${DATASET_NAME}.val.raw.txt
-TEST_DATA_FILE=${DATASET_NAME}.test.raw.txt
-EXTRACTOR_JAR=JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar
+OUTPUT_DIR=${INPUT_DIR}/${DATASET_NAME}/preprocessed/exp_${VARIANT}
-mkdir -p data
-mkdir -p data/${DATASET_NAME}
+mkdir -p ${INPUT_DIR}/${DATASET_NAME}/preprocessed/exp_${VARIANT}
+
+TRAIN_DATA_FILE=${OUTPUT_DIR}/${DATASET_NAME}.train.raw.txt
+VAL_DATA_FILE=${OUTPUT_DIR}/${DATASET_NAME}.val.raw.txt
+TEST_DATA_FILE=${OUTPUT_DIR}/${DATASET_NAME}.test.raw.txt
+EXTRACTOR_JAR=JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar
echo "Extracting paths from validation set..."
-${PYTHON} JavaExtractor/extract.py --dir ${VAL_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} --jar ${EXTRACTOR_JAR} > ${VAL_DATA_FILE} 2>> error_log.txt
+${PYTHON} JavaExtractor/extract.py --dir ${VAL_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} -d ${DATASET_NAME} --jar ${EXTRACTOR_JAR} --include_comments ${INCLUDE_COMMENTS} --exclude_stopwords ${EXCLUDE_STOPWORDS} --include_tfidf ${USE_TFIDF} --number_keywords ${NUMBER_OF_TFIDF_KEYWORDS} > ${VAL_DATA_FILE} 2>> error_log.txt
echo "Finished extracting paths from validation set"
echo "Extracting paths from test set..."
-${PYTHON} JavaExtractor/extract.py --dir ${TEST_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} --jar ${EXTRACTOR_JAR} > ${TEST_DATA_FILE} 2>> error_log.txt
+${PYTHON} JavaExtractor/extract.py --dir ${TEST_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} -d ${DATASET_NAME} --jar ${EXTRACTOR_JAR} --include_comments ${INCLUDE_COMMENTS} --exclude_stopwords ${EXCLUDE_STOPWORDS} --include_tfidf ${USE_TFIDF} --number_keywords ${NUMBER_OF_TFIDF_KEYWORDS} > ${TEST_DATA_FILE} 2>> error_log.txt
echo "Finished extracting paths from test set"
echo "Extracting paths from training set..."
-${PYTHON} JavaExtractor/extract.py --dir ${TRAIN_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} --jar ${EXTRACTOR_JAR} | shuf > ${TRAIN_DATA_FILE} 2>> error_log.txt
+${PYTHON} JavaExtractor/extract.py --dir ${TRAIN_DIR} --max_path_length 8 --max_path_width 2 --num_threads ${NUM_THREADS} -d ${DATASET_NAME} --include_comments ${INCLUDE_COMMENTS} --exclude_stopwords ${EXCLUDE_STOPWORDS} --include_tfidf ${USE_TFIDF} --number_keywords ${NUMBER_OF_TFIDF_KEYWORDS} --jar ${EXTRACTOR_JAR} | shuf > ${TRAIN_DATA_FILE} 2>> error_log.txt
echo "Finished extracting paths from training set"
-TARGET_HISTOGRAM_FILE=data/${DATASET_NAME}/${DATASET_NAME}.histo.tgt.c2s
-SOURCE_SUBTOKEN_HISTOGRAM=data/${DATASET_NAME}/${DATASET_NAME}.histo.ori.c2s
-NODE_HISTOGRAM_FILE=data/${DATASET_NAME}/${DATASET_NAME}.histo.node.c2s
+TARGET_HISTOGRAM_FILE=${OUTPUT_DIR}/${DATASET_NAME}.histo.tgt.c2s
+SOURCE_SUBTOKEN_HISTOGRAM=${OUTPUT_DIR}/${DATASET_NAME}.histo.ori.c2s
+NODE_HISTOGRAM_FILE=${OUTPUT_DIR}/${DATASET_NAME}.histo.node.c2s
echo "Creating histograms from the training data"
cat ${TRAIN_DATA_FILE} | cut -d' ' -f1 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${TARGET_HISTOGRAM_FILE}
@@ -63,7 +110,7 @@ cat ${TRAIN_DATA_FILE} | cut -d' ' -f2- | tr ' ' '\n' | cut -d',' -f2 | tr '|' '
${PYTHON} preprocess.py --train_data ${TRAIN_DATA_FILE} --test_data ${TEST_DATA_FILE} --val_data ${VAL_DATA_FILE} \
--max_contexts ${MAX_CONTEXTS} --max_data_contexts ${MAX_DATA_CONTEXTS} --subtoken_vocab_size ${SUBTOKEN_VOCAB_SIZE} \
--target_vocab_size ${TARGET_VOCAB_SIZE} --subtoken_histogram ${SOURCE_SUBTOKEN_HISTOGRAM} \
- --node_histogram ${NODE_HISTOGRAM_FILE} --target_histogram ${TARGET_HISTOGRAM_FILE} --output_name data/${DATASET_NAME}/${DATASET_NAME}
+ --node_histogram ${NODE_HISTOGRAM_FILE} --target_histogram ${TARGET_HISTOGRAM_FILE} --output_name ${OUTPUT_DIR}/${DATASET_NAME}
# If all went well, the raw data files can be deleted, because preprocess.py creates new files
# with truncated and padded number of paths for each example.
diff --git a/reader.py b/reader.py
index a2346eb..cb32ac3 100644
--- a/reader.py
+++ b/reader.py
@@ -8,19 +8,19 @@
from common import Common
from config import Config
-TARGET_INDEX_KEY = 'TARGET_INDEX_KEY'
-TARGET_STRING_KEY = 'TARGET_STRING_KEY'
-TARGET_LENGTH_KEY = 'TARGET_LENGTH_KEY'
-PATH_SOURCE_INDICES_KEY = 'PATH_SOURCE_INDICES_KEY'
-NODE_INDICES_KEY = 'NODES_INDICES_KEY'
-PATH_TARGET_INDICES_KEY = 'PATH_TARGET_INDICES_KEY'
-VALID_CONTEXT_MASK_KEY = 'VALID_CONTEXT_MASK_KEY'
-PATH_SOURCE_LENGTHS_KEY = 'PATH_SOURCE_LENGTHS_KEY'
-PATH_LENGTHS_KEY = 'PATH_LENGTHS_KEY'
-PATH_TARGET_LENGTHS_KEY = 'PATH_TARGET_LENGTHS_KEY'
-PATH_SOURCE_STRINGS_KEY = 'PATH_SOURCE_STRINGS_KEY'
-PATH_STRINGS_KEY = 'PATH_STRINGS_KEY'
-PATH_TARGET_STRINGS_KEY = 'PATH_TARGET_STRINGS_KEY'
+TARGET_INDEX_KEY = "TARGET_INDEX_KEY"
+TARGET_STRING_KEY = "TARGET_STRING_KEY"
+TARGET_LENGTH_KEY = "TARGET_LENGTH_KEY"
+PATH_SOURCE_INDICES_KEY = "PATH_SOURCE_INDICES_KEY"
+NODE_INDICES_KEY = "NODES_INDICES_KEY"
+PATH_TARGET_INDICES_KEY = "PATH_TARGET_INDICES_KEY"
+VALID_CONTEXT_MASK_KEY = "VALID_CONTEXT_MASK_KEY"
+PATH_SOURCE_LENGTHS_KEY = "PATH_SOURCE_LENGTHS_KEY"
+PATH_LENGTHS_KEY = "PATH_LENGTHS_KEY"
+PATH_TARGET_LENGTHS_KEY = "PATH_TARGET_LENGTHS_KEY"
+PATH_SOURCE_STRINGS_KEY = "PATH_SOURCE_STRINGS_KEY"
+PATH_STRINGS_KEY = "PATH_STRINGS_KEY"
+PATH_TARGET_STRINGS_KEY = "PATH_TARGET_STRINGS_KEY"
class Reader:
@@ -28,17 +28,33 @@ class Reader:
class_target_table = None
class_node_table = None
- def __init__(self, subtoken_to_index, target_to_index, node_to_index, config, is_evaluating=False):
+ def __init__(
+ self,
+ subtoken_to_index,
+ target_to_index,
+ node_to_index,
+ config,
+ is_evaluating=False,
+ ):
self.config = config
- self.file_path = config.TEST_PATH if is_evaluating else (config.TRAIN_PATH + '.train.c2s')
+ self.file_path = (
+ config.TEST_PATH if is_evaluating else (config.TRAIN_PATH + ".train.c2s")
+ )
if self.file_path is not None and not os.path.exists(self.file_path):
print(
- '%s cannot find file: %s' % ('Evaluation reader' if is_evaluating else 'Train reader', self.file_path))
+ "%s cannot find file: %s"
+ % (
+ "Evaluation reader" if is_evaluating else "Train reader",
+ self.file_path,
+ )
+ )
self.batch_size = config.BATCH_SIZE
self.is_evaluating = is_evaluating
- self.context_pad = '{},{},{}'.format(Common.PAD, Common.PAD, Common.PAD)
- self.record_defaults = [[self.context_pad]] * (self.config.DATA_NUM_CONTEXTS + 1)
+ self.context_pad = "{},{},{}".format(Common.PAD, Common.PAD, Common.PAD)
+ self.record_defaults = [[self.context_pad]] * (
+ self.config.DATA_NUM_CONTEXTS + 1
+ )
self.subtoken_table = Reader.get_subtoken_table(subtoken_to_index)
self.target_table = Reader.get_target_table(target_to_index)
@@ -48,30 +64,46 @@ def __init__(self, subtoken_to_index, target_to_index, node_to_index, config, is
@classmethod
def get_subtoken_table(cls, subtoken_to_index):
if cls.class_subtoken_table is None:
- cls.class_subtoken_table = cls.initialize_hash_map(subtoken_to_index, subtoken_to_index[Common.UNK])
+ cls.class_subtoken_table = cls.initialize_hash_map(
+ subtoken_to_index, subtoken_to_index[Common.UNK]
+ )
return cls.class_subtoken_table
@classmethod
def get_target_table(cls, target_to_index):
if cls.class_target_table is None:
- cls.class_target_table = cls.initialize_hash_map(target_to_index, target_to_index[Common.UNK])
+ cls.class_target_table = cls.initialize_hash_map(
+ target_to_index, target_to_index[Common.UNK]
+ )
return cls.class_target_table
@classmethod
def get_node_table(cls, node_to_index):
if cls.class_node_table is None:
- cls.class_node_table = cls.initialize_hash_map(node_to_index, node_to_index[Common.UNK])
+ cls.class_node_table = cls.initialize_hash_map(
+ node_to_index, node_to_index[Common.UNK]
+ )
return cls.class_node_table
@classmethod
def initialize_hash_map(cls, word_to_index, default_value):
return tf.lookup.StaticHashTable(
- tf.lookup.KeyValueTensorInitializer(list(word_to_index.keys()), list(word_to_index.values()),
- key_dtype=tf.string,
- value_dtype=tf.int32), default_value)
+ tf.lookup.KeyValueTensorInitializer(
+ list(word_to_index.keys()),
+ list(word_to_index.values()),
+ key_dtype=tf.string,
+ value_dtype=tf.int32,
+ ),
+ default_value,
+ )
def process_from_placeholder(self, row):
- parts = tf.io.decode_csv(row, record_defaults=self.record_defaults, field_delim=' ', use_quote_delim=False)
+ parts = tf.io.decode_csv(
+ row,
+ record_defaults=self.record_defaults,
+ field_delim=" ",
+ use_quote_delim=False,
+ )
res_dict = self.process_dataset(*parts)
# add batch size dimension
for key, value in res_dict.items():
@@ -85,124 +117,206 @@ def process_dataset(self, *row_parts):
if not self.is_evaluating and self.config.RANDOM_CONTEXTS:
all_contexts = tf.stack(row_parts[1:])
all_contexts_padded = tf.concat([all_contexts, [self.context_pad]], axis=-1)
- index_of_blank_context = tf.where(tf.equal(all_contexts_padded, self.context_pad))
+ index_of_blank_context = tf.where(
+ tf.equal(all_contexts_padded, self.context_pad)
+ )
num_contexts_per_example = tf.reduce_min(index_of_blank_context)
# if there are less than self.max_contexts valid contexts, still sample self.max_contexts
- safe_limit = tf.cast(tf.maximum(num_contexts_per_example, self.config.MAX_CONTEXTS), tf.int32)
- rand_indices = tf.random.shuffle(tf.range(safe_limit))[:self.config.MAX_CONTEXTS]
+ safe_limit = tf.cast(
+ tf.maximum(num_contexts_per_example, self.config.MAX_CONTEXTS), tf.int32
+ )
+ rand_indices = tf.random.shuffle(tf.range(safe_limit))[
+ : self.config.MAX_CONTEXTS
+ ]
contexts = tf.gather(all_contexts, rand_indices) # (max_contexts,)
else:
- contexts = row_parts[1:(self.config.MAX_CONTEXTS + 1)] # (max_contexts,)
+ contexts = row_parts[1 : (self.config.MAX_CONTEXTS + 1)] # (max_contexts,)
# contexts: (max_contexts, )
- split_contexts = tf.strings.split(contexts, sep=',')
+ split_contexts = tf.strings.split(contexts, sep=",")
sparse_split_contexts = split_contexts.to_sparse()
dense_split_contexts = tf.reshape(
- tf.sparse.to_dense(sp_input=sparse_split_contexts, default_value=Common.PAD),
- shape=[self.config.MAX_CONTEXTS, 3]) # (batch, max_contexts, 3)
+ tf.sparse.to_dense(
+ sp_input=sparse_split_contexts, default_value=Common.PAD
+ ),
+ shape=[self.config.MAX_CONTEXTS, 3],
+ ) # (batch, max_contexts, 3)
- split_target_labels = tf.strings.split(tf.expand_dims(word, -1), sep='|')
+ split_target_labels = tf.strings.split(tf.expand_dims(word, -1), sep="|")
sparse_target_labels = split_target_labels.to_sparse()
- sparse_target_labels = tf.sparse.reset_shape(sparse_target_labels,
- [1, tf.maximum(tf.cast(self.config.MAX_TARGET_PARTS, tf.int64),
- sparse_target_labels.dense_shape[1] + 1)])
- dense_target_label = tf.reshape(tf.sparse.to_dense(sp_input=sparse_target_labels,
- default_value=Common.PAD),
- shape=[-1])
+ sparse_target_labels = tf.sparse.reset_shape(
+ sparse_target_labels,
+ [
+ 1,
+ tf.maximum(
+ tf.cast(self.config.MAX_TARGET_PARTS, tf.int64),
+ sparse_target_labels.dense_shape[1] + 1,
+ ),
+ ],
+ )
+ dense_target_label = tf.reshape(
+ tf.sparse.to_dense(sp_input=sparse_target_labels, default_value=Common.PAD),
+ shape=[-1],
+ )
index_of_blank = tf.where(tf.equal(dense_target_label, Common.PAD))
target_length = tf.reduce_min(index_of_blank)
- dense_target_label = dense_target_label[:self.config.MAX_TARGET_PARTS]
- clipped_target_lengths = tf.clip_by_value(target_length, clip_value_min=0,
- clip_value_max=self.config.MAX_TARGET_PARTS)
- target_word_labels = tf.concat([
- self.target_table.lookup(dense_target_label), [0]], axis=-1) # (max_target_parts + 1) of int
-
- path_source_strings = tf.slice(dense_split_contexts, [0, 0], [self.config.MAX_CONTEXTS, 1]) # (max_contexts, 1)
+ dense_target_label = dense_target_label[: self.config.MAX_TARGET_PARTS]
+ clipped_target_lengths = tf.clip_by_value(
+ target_length, clip_value_min=0, clip_value_max=self.config.MAX_TARGET_PARTS
+ )
+ target_word_labels = tf.concat(
+ [self.target_table.lookup(dense_target_label), [0]], axis=-1
+ ) # (max_target_parts + 1) of int
+
+ path_source_strings = tf.slice(
+ dense_split_contexts, [0, 0], [self.config.MAX_CONTEXTS, 1]
+ ) # (max_contexts, 1)
flat_source_strings = tf.reshape(path_source_strings, [-1]) # (max_contexts)
- split_source = tf.strings.split(flat_source_strings, sep='|') # (max_contexts, max_name_parts)
+ split_source = tf.strings.split(
+ flat_source_strings, sep="|"
+ ) # (max_contexts, max_name_parts)
sparse_split_source = split_source.to_sparse()
- sparse_split_source = tf.sparse.reset_shape(sparse_split_source,
- [self.config.MAX_CONTEXTS,
- tf.maximum(
- tf.cast(self.config.MAX_NAME_PARTS, tf.int64),
- sparse_split_source.dense_shape[1])])
-
- dense_split_source = tf.sparse.to_dense(sp_input=sparse_split_source,
- default_value=Common.PAD) # (max_contexts, max_name_parts)
- dense_split_source = tf.slice(dense_split_source, [0, 0], [-1, self.config.MAX_NAME_PARTS])
- path_source_indices = self.subtoken_table.lookup(dense_split_source) # (max_contexts, max_name_parts)
- path_source_lengths = tf.reduce_sum(tf.cast(tf.not_equal(dense_split_source, Common.PAD), tf.int32),
- -1) # (max_contexts)
-
- path_strings = tf.slice(dense_split_contexts, [0, 1], [self.config.MAX_CONTEXTS, 1])
+ sparse_split_source = tf.sparse.reset_shape(
+ sparse_split_source,
+ [
+ self.config.MAX_CONTEXTS,
+ tf.maximum(
+ tf.cast(self.config.MAX_NAME_PARTS, tf.int64),
+ sparse_split_source.dense_shape[1],
+ ),
+ ],
+ )
+
+ dense_split_source = tf.sparse.to_dense(
+ sp_input=sparse_split_source, default_value=Common.PAD
+ ) # (max_contexts, max_name_parts)
+ dense_split_source = tf.slice(
+ dense_split_source, [0, 0], [-1, self.config.MAX_NAME_PARTS]
+ )
+ path_source_indices = self.subtoken_table.lookup(
+ dense_split_source
+ ) # (max_contexts, max_name_parts)
+ path_source_lengths = tf.reduce_sum(
+ tf.cast(tf.not_equal(dense_split_source, Common.PAD), tf.int32), -1
+ ) # (max_contexts)
+
+ path_strings = tf.slice(
+ dense_split_contexts, [0, 1], [self.config.MAX_CONTEXTS, 1]
+ )
flat_path_strings = tf.reshape(path_strings, [-1])
- split_path = tf.strings.split(flat_path_strings, sep='|')
+ split_path = tf.strings.split(flat_path_strings, sep="|")
sparse_split_path = split_path.to_sparse()
if self.config.MAX_PATH_LENGTH < sparse_split_path.dense_shape[1]:
- sparse_split_path = tf.sparse.slice(sparse_split_path, [0, 0],
- [sparse_split_path.dense_shape[0], self.config.MAX_PATH_LENGTH])
-
- sparse_split_path = tf.sparse.reset_shape(sparse_split_path,
- [self.config.MAX_CONTEXTS, self.config.MAX_PATH_LENGTH])
-
- dense_split_path = tf.sparse.to_dense(sp_input=sparse_split_path,
- default_value=Common.PAD) # (batch, max_contexts, max_path_length)
-
- node_indices = self.node_table.lookup(dense_split_path) # (max_contexts, max_path_length)
- path_lengths = tf.reduce_sum(tf.cast(tf.not_equal(dense_split_path, Common.PAD), tf.int32),
- -1) # (max_contexts)
-
- path_target_strings = tf.slice(dense_split_contexts, [0, 2], [self.config.MAX_CONTEXTS, 1]) # (max_contexts, 1)
+ sparse_split_path = tf.sparse.slice(
+ sparse_split_path,
+ [0, 0],
+ [sparse_split_path.dense_shape[0], self.config.MAX_PATH_LENGTH],
+ )
+
+ sparse_split_path = tf.sparse.reset_shape(
+ sparse_split_path, [self.config.MAX_CONTEXTS, self.config.MAX_PATH_LENGTH]
+ )
+
+ dense_split_path = tf.sparse.to_dense(
+ sp_input=sparse_split_path, default_value=Common.PAD
+ ) # (batch, max_contexts, max_path_length)
+
+ node_indices = self.node_table.lookup(
+ dense_split_path
+ ) # (max_contexts, max_path_length)
+ path_lengths = tf.reduce_sum(
+ tf.cast(tf.not_equal(dense_split_path, Common.PAD), tf.int32), -1
+ ) # (max_contexts)
+
+ path_target_strings = tf.slice(
+ dense_split_contexts, [0, 2], [self.config.MAX_CONTEXTS, 1]
+ ) # (max_contexts, 1)
flat_target_strings = tf.reshape(path_target_strings, [-1]) # (max_contexts)
- split_target = tf.strings.split(flat_target_strings, sep='|') # (max_contexts, max_name_parts)
+ split_target = tf.strings.split(
+ flat_target_strings, sep="|"
+ ) # (max_contexts, max_name_parts)
sparse_split_target = split_target.to_sparse()
- sparse_split_target = tf.sparse.reset_shape(sparse_split_target, [self.config.MAX_CONTEXTS,
- tf.maximum(
- tf.cast(self.config.MAX_NAME_PARTS,
- tf.int64),
- sparse_split_target.dense_shape[1])])
- dense_split_target = tf.sparse.to_dense(sp_input=sparse_split_target,
- default_value=Common.PAD) # (max_contexts, max_name_parts)
- dense_split_target = tf.slice(dense_split_target, [0, 0], [-1, self.config.MAX_NAME_PARTS])
- path_target_indices = self.subtoken_table.lookup(dense_split_target) # (max_contexts, max_name_parts)
- path_target_lengths = tf.reduce_sum(tf.cast(tf.not_equal(dense_split_target, Common.PAD), tf.int32),
- -1) # (max_contexts)
-
- valid_contexts_mask = tf.cast(tf.not_equal(
- tf.reduce_max(path_source_indices, -1) + tf.reduce_max(node_indices, -1) + tf.reduce_max(
- path_target_indices, -1), 0), tf.float32)
-
- return {TARGET_STRING_KEY: word, TARGET_INDEX_KEY: target_word_labels,
- TARGET_LENGTH_KEY: clipped_target_lengths,
- PATH_SOURCE_INDICES_KEY: path_source_indices, NODE_INDICES_KEY: node_indices,
- PATH_TARGET_INDICES_KEY: path_target_indices, VALID_CONTEXT_MASK_KEY: valid_contexts_mask,
- PATH_SOURCE_LENGTHS_KEY: path_source_lengths, PATH_LENGTHS_KEY: path_lengths,
- PATH_TARGET_LENGTHS_KEY: path_target_lengths, PATH_SOURCE_STRINGS_KEY: path_source_strings,
- PATH_STRINGS_KEY: path_strings, PATH_TARGET_STRINGS_KEY: path_target_strings
- }
+ sparse_split_target = tf.sparse.reset_shape(
+ sparse_split_target,
+ [
+ self.config.MAX_CONTEXTS,
+ tf.maximum(
+ tf.cast(self.config.MAX_NAME_PARTS, tf.int64),
+ sparse_split_target.dense_shape[1],
+ ),
+ ],
+ )
+ dense_split_target = tf.sparse.to_dense(
+ sp_input=sparse_split_target, default_value=Common.PAD
+ ) # (max_contexts, max_name_parts)
+ dense_split_target = tf.slice(
+ dense_split_target, [0, 0], [-1, self.config.MAX_NAME_PARTS]
+ )
+ path_target_indices = self.subtoken_table.lookup(
+ dense_split_target
+ ) # (max_contexts, max_name_parts)
+ path_target_lengths = tf.reduce_sum(
+ tf.cast(tf.not_equal(dense_split_target, Common.PAD), tf.int32), -1
+ ) # (max_contexts)
+
+ valid_contexts_mask = tf.cast(
+ tf.not_equal(
+ tf.reduce_max(path_source_indices, -1)
+ + tf.reduce_max(node_indices, -1)
+ + tf.reduce_max(path_target_indices, -1),
+ 0,
+ ),
+ tf.float32,
+ )
+
+ return {
+ TARGET_STRING_KEY: word,
+ TARGET_INDEX_KEY: target_word_labels,
+ TARGET_LENGTH_KEY: clipped_target_lengths,
+ PATH_SOURCE_INDICES_KEY: path_source_indices,
+ NODE_INDICES_KEY: node_indices,
+ PATH_TARGET_INDICES_KEY: path_target_indices,
+ VALID_CONTEXT_MASK_KEY: valid_contexts_mask,
+ PATH_SOURCE_LENGTHS_KEY: path_source_lengths,
+ PATH_LENGTHS_KEY: path_lengths,
+ PATH_TARGET_LENGTHS_KEY: path_target_lengths,
+ PATH_SOURCE_STRINGS_KEY: path_source_strings,
+ PATH_STRINGS_KEY: path_strings,
+ PATH_TARGET_STRINGS_KEY: path_target_strings,
+ }
def get_dataset(self):
self.init_dataset()
return self.dataset
def init_dataset(self):
- self.dataset = tf.data.experimental.CsvDataset(self.file_path, record_defaults=self.record_defaults,
- field_delim=' ',
- use_quote_delim=False, buffer_size=self.config.CSV_BUFFER_SIZE)
+ self.dataset = tf.data.experimental.CsvDataset(
+ self.file_path,
+ record_defaults=self.record_defaults,
+ field_delim=" ",
+ use_quote_delim=False,
+ buffer_size=self.config.CSV_BUFFER_SIZE,
+ )
if not self.is_evaluating:
- self.dataset = self.dataset.shuffle(self.config.SHUFFLE_BUFFER_SIZE, reshuffle_each_iteration=True)
-
- self.dataset = self.dataset \
- .map(map_func=self.process_dataset, num_parallel_calls=self.config.READER_NUM_PARALLEL_BATCHES) \
- .batch(batch_size=self.batch_size, drop_remainder=True) \
+ self.dataset = self.dataset.shuffle(
+ self.config.SHUFFLE_BUFFER_SIZE, reshuffle_each_iteration=True
+ )
+
+ self.dataset = (
+ self.dataset.map(
+ map_func=self.process_dataset,
+ num_parallel_calls=self.config.READER_NUM_PARALLEL_BATCHES,
+ )
+ .batch(batch_size=self.batch_size, drop_remainder=True)
.prefetch(tf.data.experimental.AUTOTUNE)
+ )
-if __name__ == '__main__':
+if __name__ == "__main__":
# tf.config.experimental_run_functions_eagerly(True)
print("tf executing eagerly: " + str(tf.executing_eagerly()))
@@ -210,40 +324,55 @@ def init_dataset(self):
args = read_args()
config = Config.get_default_config(args)
- with open('{}.dict.c2s'.format(config.TRAIN_PATH), 'rb') as file:
+ with open("{}.dict.c2s".format(config.TRAIN_PATH), "rb") as file:
subtoken_to_count = pickle.load(file)
node_to_count = pickle.load(file)
target_to_count = pickle.load(file)
max_contexts = pickle.load(file)
num_training_examples = pickle.load(file)
- print('Dictionaries loaded.')
+ print("Dictionaries loaded.")
if config.DATA_NUM_CONTEXTS <= 0:
config.DATA_NUM_CONTEXTS = max_contexts
- subtoken_to_index, index_to_subtoken, subtoken_vocab_size = \
- Common.load_vocab_from_dict(subtoken_to_count, add_values=[Common.PAD, Common.UNK],
- max_size=config.SUBTOKENS_VOCAB_MAX_SIZE)
- print('Loaded subtoken vocab. size: %d' % subtoken_vocab_size)
-
- target_to_index, index_to_target, target_vocab_size = \
- Common.load_vocab_from_dict(target_to_count, add_values=[Common.PAD, Common.UNK, Common.SOS],
- max_size=config.TARGET_VOCAB_MAX_SIZE)
- print('Loaded target word vocab. size: %d' % target_vocab_size)
-
- node_to_index, index_to_node, nodes_vocab_size = \
- Common.load_vocab_from_dict(node_to_count, add_values=[Common.PAD, Common.UNK], max_size=None)
- print('Loaded nodes vocab. size: %d' % nodes_vocab_size)
-
- reader = Reader(subtoken_to_index, target_to_index, node_to_index, config, False)
+ (
+ subtoken_to_index,
+ index_to_subtoken,
+ subtoken_vocab_size,
+ ) = Common.load_vocab_from_dict(
+ subtoken_to_count,
+ add_values=[Common.PAD, Common.UNK],
+ max_size=config.SUBTOKENS_VOCAB_MAX_SIZE,
+ )
+ print("Loaded subtoken vocab. size: %d" % subtoken_vocab_size)
+
+ (
+ target_to_index,
+ index_to_target,
+ target_vocab_size,
+ ) = Common.load_vocab_from_dict(
+ target_to_count,
+ add_values=[Common.PAD, Common.UNK, Common.SOS],
+ max_size=config.TARGET_VOCAB_MAX_SIZE,
+ )
+ print("Loaded target word vocab. size: %d" % target_vocab_size)
+
+ node_to_index, index_to_node, nodes_vocab_size = Common.load_vocab_from_dict(
+ node_to_count, add_values=[Common.PAD, Common.UNK], max_size=None
+ )
+ print("Loaded nodes vocab. size: %d" % nodes_vocab_size)
+
+ reader = Reader(
+ subtoken_to_index, target_to_index, node_to_index, config, False
+ )
test_manually = True
if test_manually:
- with open('{}.train.c2s'.format(config.TRAIN_PATH), 'r') as data_file:
+ with open("{}.train.c2s".format(config.TRAIN_PATH), "r") as data_file:
for test_sample in data_file.readlines():
test_sample = test_sample.strip()
contexts_num = sum(ch.isspace() for ch in test_sample)
- space_padding = ' ' * (config.DATA_NUM_CONTEXTS - contexts_num)
+ space_padding = " " * (config.DATA_NUM_CONTEXTS - contexts_num)
test_sample += space_padding
reader.process_from_placeholder(test_sample)
@@ -266,21 +395,40 @@ def init_dataset(self):
path_strings = output[PATH_STRINGS_KEY].numpy()
path_target_strings = output[PATH_TARGET_STRINGS_KEY].numpy()
- print('Target strings: ', Common.binary_to_string_list(target_strings))
- print('Context strings: ', Common.binary_to_string_3d(
- np.concatenate([path_source_strings, path_strings, path_target_strings], -1)))
- print('Target indices: ', target_indices)
- print('Target lengths: ', target_lengths)
- print('Path source strings: ', Common.binary_to_string_3d(path_source_strings))
- print('Path source indices: ', path_source_indices)
- print('Path source lengths: ', path_source_lengths)
- print('Path strings: ', Common.binary_to_string_3d(path_strings))
- print('Node indices: ', node_indices)
- print('Path lengths: ', path_lengths)
- print('Path target strings: ', Common.binary_to_string_3d(path_target_strings))
- print('Path target indices: ', path_target_indices)
- print('Path target lengths: ', path_target_lengths)
- print('Valid context mask: ', valid_context_mask)
+ print(
+ "Target strings: ", Common.binary_to_string_list(target_strings)
+ )
+ print(
+ "Context strings: ",
+ Common.binary_to_string_3d(
+ np.concatenate(
+ [
+ path_source_strings,
+ path_strings,
+ path_target_strings,
+ ],
+ -1,
+ )
+ ),
+ )
+ print("Target indices: ", target_indices)
+ print("Target lengths: ", target_lengths)
+ print(
+ "Path source strings: ",
+ Common.binary_to_string_3d(path_source_strings),
+ )
+ print("Path source indices: ", path_source_indices)
+ print("Path source lengths: ", path_source_lengths)
+ print("Path strings: ", Common.binary_to_string_3d(path_strings))
+ print("Node indices: ", node_indices)
+ print("Path lengths: ", path_lengths)
+ print(
+ "Path target strings: ",
+ Common.binary_to_string_3d(path_target_strings),
+ )
+ print("Path target indices: ", path_target_indices)
+ print("Path target lengths: ", path_target_lengths)
+ print("Valid context mask: ", valid_context_mask)
except tf.errors.OutOfRangeError:
- print('Done training, epoch reached')
+ print("Done training, epoch reached")
diff --git a/requirements_docker.txt b/requirements_docker.txt
new file mode 100644
index 0000000..03f7f8b
--- /dev/null
+++ b/requirements_docker.txt
@@ -0,0 +1,10 @@
+clang
+joblib
+libclang
+lmdb
+networkx
+numpy==1.19.2
+rouge==0.3.2
+tensorflow_addons==0.8.3
+scikit_learn
+flatbuffers
\ No newline at end of file
diff --git a/results.py b/results.py
index fedb001..301243b 100644
--- a/results.py
+++ b/results.py
@@ -3,9 +3,13 @@
def trace_evaluation(output_file, correct_predictions, total_predictions, elapsed):
- accuracy_message = "Accuracy: {0}".format(str(correct_predictions / total_predictions))
- throughput_message = "Prediction throughput: %d" % int(total_predictions / (elapsed if elapsed > 0 else 1))
- output_file.write(accuracy_message + '\n')
+ accuracy_message = "Accuracy: {0}".format(
+ str(correct_predictions / total_predictions)
+ )
+ throughput_message = "Prediction throughput: %d" % int(
+ total_predictions / (elapsed if elapsed > 0 else 1)
+ )
+ output_file.write(accuracy_message + "\n")
output_file.write(throughput_message)
print(accuracy_message)
print(throughput_message, flush=True)
@@ -27,49 +31,73 @@ def calculate_results(true_positive, false_positive, false_negative):
return precision, recall, f1
-def update_correct_predictions(beam_width, num_correct_predictions, output_file, results):
+def update_correct_predictions(
+ beam_width, num_correct_predictions, output_file, results
+):
for original_name, predicted in results:
original_name_parts = original_name.split(Common.internal_delimiter) # list
filtered_original = Common.filter_impossible_names(original_name_parts) # list
predicted_first = predicted
if beam_width > 0:
predicted_first = predicted[0]
- filtered_predicted_first_parts = Common.filter_impossible_names(predicted_first) # list
+ filtered_predicted_first_parts = Common.filter_impossible_names(
+ predicted_first
+ ) # list
if beam_width == 0:
- output_file.write('Original: ' + Common.internal_delimiter.join(original_name_parts) +
- ' , predicted 1st: ' + Common.internal_delimiter.join(
- filtered_predicted_first_parts) + '\n')
- if filtered_original == filtered_predicted_first_parts or Common.unique(
- filtered_original) == Common.unique(
- filtered_predicted_first_parts) or ''.join(filtered_original) == ''.join(
- filtered_predicted_first_parts):
+ output_file.write(
+ "Original: "
+ + Common.internal_delimiter.join(original_name_parts)
+ + " , predicted 1st: "
+ + Common.internal_delimiter.join(filtered_predicted_first_parts)
+ + "\n"
+ )
+ if (
+ filtered_original == filtered_predicted_first_parts
+ or Common.unique(filtered_original)
+ == Common.unique(filtered_predicted_first_parts)
+ or "".join(filtered_original) == "".join(filtered_predicted_first_parts)
+ ):
num_correct_predictions += 1
else:
- filtered_predicted = [Common.internal_delimiter.join(Common.filter_impossible_names(p)) for p in
- predicted]
+ filtered_predicted = [
+ Common.internal_delimiter.join(Common.filter_impossible_names(p))
+ for p in predicted
+ ]
true_ref = original_name
- output_file.write('Original: ' + ' '.join(original_name_parts) + '\n')
+ output_file.write("Original: " + " ".join(original_name_parts) + "\n")
for i, p in enumerate(filtered_predicted):
- output_file.write('\t@{}: {}'.format(i + 1, ' '.join(p.split(Common.internal_delimiter))) + '\n')
+ output_file.write(
+ "\t@{}: {}".format(
+ i + 1, " ".join(p.split(Common.internal_delimiter))
+ )
+ + "\n"
+ )
if true_ref in filtered_predicted:
index_of_correct = filtered_predicted.index(true_ref)
update = np.concatenate(
- [np.zeros(index_of_correct, dtype=np.int32),
- np.ones(beam_width - index_of_correct, dtype=np.int32)])
+ [
+ np.zeros(index_of_correct, dtype=np.int32),
+ np.ones(beam_width - index_of_correct, dtype=np.int32),
+ ]
+ )
num_correct_predictions += update
return num_correct_predictions
-def update_per_subtoken_statistics(beam_width, results, true_positive, false_positive, false_negative):
+def update_per_subtoken_statistics(
+ beam_width, results, true_positive, false_positive, false_negative
+):
for original_name, predicted in results:
if beam_width > 0:
predicted = predicted[0]
filtered_predicted_names = Common.filter_impossible_names(predicted)
- filtered_original_subtokens = Common.filter_impossible_names(original_name.split(Common.internal_delimiter))
+ filtered_original_subtokens = Common.filter_impossible_names(
+ original_name.split(Common.internal_delimiter)
+ )
- if ''.join(filtered_original_subtokens) == ''.join(filtered_predicted_names):
+ if "".join(filtered_original_subtokens) == "".join(filtered_predicted_names):
true_positive += len(filtered_original_subtokens)
continue
diff --git a/train.sh b/train.sh
index be20fef..c0548c9 100644
--- a/train.sh
+++ b/train.sh
@@ -5,13 +5,41 @@
# test_data: by default, points to the validation set, since this is the set that
# will be evaluated after each training iteration. If you wish to test
# on the final (held-out) test set, change 'val' to 'test'.
-type=java-large-model
-dataset_name=java-large
-data_dir=data/java-large
+
+dataset_name=default
+variant=default
+continue_training_from_checkpoint=true
+
+# This code block is used to get long two-dash arguments from the command line.
+die() { echo "$*" >&2; exit 2; } # complain to STDERR and exit with error
+needs_arg() { if [ -z "$OPTARG" ]; then die "No arg for --$OPT option"; fi; }
+
+while getopts ab:c:-: OPT; do
+ # support long options: https://stackoverflow.com/a/28466267/519360
+ if [ "$OPT" = "-" ]; then # long option: reformulate OPT and OPTARG
+ OPT="${OPTARG%%=*}" # extract long option name
+ OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty)
+ OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=`
+ fi
+ case "$OPT" in
+ dataset ) dataset_name="$OPTARG" ;;
+ continue_training_from_checkpoint ) continue_training_from_checkpoint="$OPTARG" ;;
+ variant ) variant="$OPTARG" ;;
+ ??* ) die "Illegal option --$OPT" ;; # bad long option
+ ? ) exit 2 ;; # bad short option (error reported via getopts)
+ esac
+done
+shift $((OPTIND-1)) # remove parsed options and args from $@ list
+
+echo "Dataset: $dataset_name"
+echo "Training from a previous checkpoint: $continue_training_from_checkpoint"
+
+type=exp_${dataset_name}_${variant}
+data_dir=datasets/${dataset_name}/preprocessed/exp_${variant}
data=${data_dir}/${dataset_name}
test_data=${data_dir}/${dataset_name}.val.c2s
model_dir=models/${type}
mkdir -p ${model_dir}
set -e
-python3 -u code2seq.py --data ${data} --test ${test_data} --save_prefix ${model_dir}/model
+python3 -u code2seq.py --data ${data} --test ${test_data} --save_path ${model_dir} --model_path ${model_dir} --continue_training_from_checkpoint ${continue_training_from_checkpoint}