diff --git a/.gitignore b/.gitignore index c3a0ade..30195d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ *.lst .idea/* *.iml -*.xml \ No newline at end of file +*.xml +*.pyc + diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..7967ed9 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,8 @@ +@inproceedings{ + alon2018codeseq, + title={code2seq: Generating Sequences from Structured Representations of Code}, + author={Uri Alon and Shaked Brody and Omer Levy and Eran Yahav}, + booktitle={International Conference on Learning Representations}, + year={2019}, + url={https://openreview.net/forum?id=H1gKYo09tX}, +} diff --git a/CSharpExtractor/CSharpExtractor/Extractor/Extractor.cs b/CSharpExtractor/CSharpExtractor/Extractor/Extractor.cs index 98a9d03..8636885 100644 --- a/CSharpExtractor/CSharpExtractor/Extractor/Extractor.cs +++ b/CSharpExtractor/CSharpExtractor/Extractor/Extractor.cs @@ -13,13 +13,10 @@ namespace Extractor { public class Extractor { - //private SemanticModel semanticModel; - public const string UpTreeChar = "^"; - public const string DownTreeChar = "_"; public const string InternalDelimiter = "|"; + public const string UpTreeChar = InternalDelimiter; + public const string DownTreeChar = InternalDelimiter; public const string MethodNameConst = "METHOD_NAME"; - // public const string UpTreeChar = InternalDelimiter; - // public const string DownTreeChar = InternalDelimiter; public static SyntaxKind[] ParentTypeToAddChildId = new SyntaxKind[] { SyntaxKind.SimpleAssignmentExpression, SyntaxKind.ElementAccessExpression, SyntaxKind.SimpleMemberAccessExpression, SyntaxKind.InvocationExpression, SyntaxKind.BracketedArgumentList, SyntaxKind.ArgumentList}; diff --git a/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs b/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs index 666eec1..ce2991e 100644 --- a/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs +++ b/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs @@ -25,7 +25,7 @@ public class Options [Option('o', "ofile_name", Default = "test.txt", HelpText = "Output file name")] public String OFileName { get; set; } - [Option('h', "no_hash", Default = false, HelpText = "When enabled, prints the whole path strings (not hashed)")] + [Option('h', "no_hash", Default = true, HelpText = "When enabled, prints the whole path strings (not hashed)")] public Boolean NoHash { get; set; } [Option('l', "max_contexts", Default = 30000, HelpText = "Max number of path contexts to sample. Affects only very large snippets")] diff --git a/JavaExtractor/JPredict/dependency-reduced-pom.xml b/JavaExtractor/JPredict/dependency-reduced-pom.xml index 53ae0f1..f1fd7c0 100644 --- a/JavaExtractor/JPredict/dependency-reduced-pom.xml +++ b/JavaExtractor/JPredict/dependency-reduced-pom.xml @@ -10,10 +10,9 @@ maven-compiler-plugin - 3.2 + 3.8.0 - 1.8 - 1.8 + 11 Test.java diff --git a/JavaExtractor/JPredict/pom.xml b/JavaExtractor/JPredict/pom.xml index 0eda87c..d92435f 100644 --- a/JavaExtractor/JPredict/pom.xml +++ b/JavaExtractor/JPredict/pom.xml @@ -11,10 +11,9 @@ maven-compiler-plugin - 3.2 + 3.8.0 - 1.8 - 1.8 + 11 Test.java @@ -56,7 +55,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.8 + 2.9.10.4 args4j @@ -68,6 +67,11 @@ commons-lang3 3.5 + + com.google.code.gson + gson + 2.8.5 + UTF-8 diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java index 51fd9c1..808c733 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java @@ -40,6 +40,9 @@ public class CommandLineValues { @Option(name = "--max_child_id", required = false) public int MaxChildId = 3; + @Option(name = "--json_output", required = false) + public boolean JsonOutput = false; + public CommandLineValues(String... args) throws CmdLineException { CmdLineParser parser = new CmdLineParser(this); try { diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java index 010405a..c85bce8 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java @@ -53,8 +53,9 @@ public static boolean isMethod(Node node, String type) { } public static ArrayList splitToSubtokens(String str1) { - String str2 = str1.trim(); - return Stream.of(str2.split("(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+")) + String str2 = str1.replace("|", " "); + String str3 = str2.trim(); + return Stream.of(str3.split("(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+")) .filter(s -> s.length() > 0).map(s -> Common.normalizeName(s, Common.EmptyString)) .filter(s -> s.length() > 0).collect(Collectors.toCollection(ArrayList::new)); } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java index 247a658..9bf407c 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java @@ -8,9 +8,12 @@ public class MethodContent { private final ArrayList leaves; private final String name; - public MethodContent(ArrayList leaves, String name) { + private final String content; + + public MethodContent(ArrayList leaves, String name, String content) { this.leaves = leaves; this.name = name; + this.content = content; } public ArrayList getLeaves() { @@ -20,4 +23,8 @@ public ArrayList getLeaves() { public String getName() { return name; } + + public String getContent() { + return content; + } } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java index 41e4481..6b32163 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java @@ -12,13 +12,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import com.google.gson.Gson; class ExtractFeaturesTask implements Callable { - private final CommandLineValues m_CommandLineValues; + private final CommandLineValues commandLineValues; private final Path filePath; public ExtractFeaturesTask(CommandLineValues commandLineValues, Path path) { - m_CommandLineValues = commandLineValues; + this.commandLineValues = commandLineValues; this.filePath = path; } @@ -49,8 +50,8 @@ public void processFile() { private ArrayList extractSingleFile() throws IOException { String code; - if (m_CommandLineValues.MaxFileLength > 0 && - Files.lines(filePath, Charset.defaultCharset()).count() > m_CommandLineValues.MaxFileLength) { + if (commandLineValues.MaxFileLength > 0 && + Files.lines(filePath, Charset.defaultCharset()).count() > commandLineValues.MaxFileLength) { return new ArrayList<>(); } try { @@ -59,7 +60,7 @@ private ArrayList extractSingleFile() throws IOException { e.printStackTrace(); code = Common.EmptyString; } - FeatureExtractor featureExtractor = new FeatureExtractor(m_CommandLineValues); + FeatureExtractor featureExtractor = new FeatureExtractor(commandLineValues, this.filePath); return featureExtractor.extractFeatures(code); } @@ -74,8 +75,14 @@ public String featuresToString(ArrayList features) { for (ProgramFeatures singleMethodFeatures : features) { StringBuilder builder = new StringBuilder(); - String toPrint = singleMethodFeatures.toString(); - if (m_CommandLineValues.PrettyPrint) { + String toPrint; + if (commandLineValues.JsonOutput) { + toPrint = new Gson().toJson(singleMethodFeatures); + } + else { + toPrint = singleMethodFeatures.toString(); + } + if (commandLineValues.PrettyPrint) { toPrint = toPrint.replace(" ", "\n\t"); } builder.append(toPrint); diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java index 782db11..aa6b20b 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java @@ -11,6 +11,8 @@ import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.Node; +import java.io.File; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; import java.util.Set; @@ -26,9 +28,11 @@ class FeatureExtractor { .of("AssignExpr", "ArrayAccessExpr", "FieldAccessExpr", "MethodCallExpr") .collect(Collectors.toCollection(HashSet::new)); private final CommandLineValues m_CommandLineValues; + private final Path filePath; - public FeatureExtractor(CommandLineValues commandLineValues) { + public FeatureExtractor(CommandLineValues commandLineValues, Path filePath) { this.m_CommandLineValues = commandLineValues; + this.filePath = filePath; } private static ArrayList getTreeStack(Node node) { @@ -90,7 +94,8 @@ private ArrayList generatePathFeatures(ArrayList private ProgramFeatures generatePathFeaturesForFunction(MethodContent methodContent) { ArrayList functionLeaves = methodContent.getLeaves(); - ProgramFeatures programFeatures = new ProgramFeatures(methodContent.getName()); + ProgramFeatures programFeatures = new ProgramFeatures( + methodContent.getName(), this.filePath, methodContent.getContent()); for (int i = 0; i < functionLeaves.size(); i++) { for (int j = i + 1; j < functionLeaves.size(); j++) { diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java index d9b9109..6194777 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java @@ -1,17 +1,22 @@ package JavaExtractor.FeaturesEntities; -import com.fasterxml.jackson.annotation.JsonIgnore; - +import java.nio.file.Path; import java.util.ArrayList; import java.util.stream.Collectors; public class ProgramFeatures { - private final String name; + String name; + + transient ArrayList features = new ArrayList<>(); + String textContent; + + String filePath; - private final ArrayList features = new ArrayList<>(); + public ProgramFeatures(String name, Path filePath, String textContent) { - public ProgramFeatures(String name) { this.name = name; + this.filePath = filePath.toAbsolutePath().toString(); + this.textContent = textContent; } @SuppressWarnings("StringBufferReplaceableByString") @@ -29,7 +34,6 @@ public void addFeature(Property source, String path, Property target) { features.add(newRelation); } - @JsonIgnore public boolean isEmpty() { return features.isEmpty(); } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java index 43fdc11..b48bcbd 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java @@ -1,18 +1,18 @@ package JavaExtractor.FeaturesEntities; public class ProgramRelation { - private final Property m_Source; - private final Property m_Target; - private final String m_Path; + Property source; + Property target; + String path; public ProgramRelation(Property sourceName, Property targetName, String path) { - m_Source = sourceName; - m_Target = targetName; - m_Path = path; + source = sourceName; + target = targetName; + this.path = path; } public String toString() { - return String.format("%s,%s,%s", m_Source.getName(), m_Path, - m_Target.getName()); + return String.format("%s,%s,%s", source.getName(), path, + target.getName()); } } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java index bf9fca5..28c2735 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java @@ -12,11 +12,11 @@ @SuppressWarnings("StringEquality") public class FunctionVisitor extends VoidVisitorAdapter { - private final ArrayList m_Methods = new ArrayList<>(); - private final CommandLineValues m_CommandLineValues; + private final ArrayList methods = new ArrayList<>(); + private final CommandLineValues commandLineValues; public FunctionVisitor(CommandLineValues commandLineValues) { - this.m_CommandLineValues = commandLineValues; + this.commandLineValues = commandLineValues; } @Override @@ -38,14 +38,13 @@ private void visitMethod(MethodDeclaration node) { splitName = String.join(Common.internalSeparator, splitNameParts); } + node.setName(Common.methodName); + if (node.getBody() != null) { long methodLength = getMethodLength(node.getBody().toString()); - if (m_CommandLineValues.MaxCodeLength > 0) { - if (methodLength >= m_CommandLineValues.MinCodeLength && methodLength <= m_CommandLineValues.MaxCodeLength) { - m_Methods.add(new MethodContent(leaves, splitName)); - } - } else { - m_Methods.add(new MethodContent(leaves, splitName)); + if (commandLineValues.MaxCodeLength <= 0 || + (methodLength >= commandLineValues.MinCodeLength && methodLength <= commandLineValues.MaxCodeLength)) { + methods.add(new MethodContent(leaves, splitName, node.toString())); } } } @@ -65,6 +64,6 @@ private long getMethodLength(String code) { } public ArrayList getMethodContents() { - return m_Methods; + return methods; } } diff --git a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar b/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar index 32daa7a..da4f7db 100644 Binary files a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar and b/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar differ diff --git a/Python150kExtractor/README.md b/Python150kExtractor/README.md new file mode 100644 index 0000000..60a4618 --- /dev/null +++ b/Python150kExtractor/README.md @@ -0,0 +1,70 @@ +# Python150k dataset + +## Steps to reproduce + +1. Download parsed python dataset from [here](https://www.sri.inf.ethz.ch/py150 +), unarchive and place under `PYTHON150K_DIR`: + +```bash +# Replace with desired path. +>>> PYTHON150K_DIR=/path/to/data/dir +>>> mkdir -p $PYTHON150K_DIR +>>> cd $PYTHON150K_DIR +>>> wget http://files.srl.inf.ethz.ch/data/py150.tar.gz +... +>>> tar -xzvf py150.tar.gz +... +``` + +2. Extract samples to `DATA_DIR`: + +```bash +# Replace with desired path. +>>> DATA_DIR=$(pwd)/data/default +>>> SEED=239 +>>> python extract.py \ + --data_dir=$PYTHON150K_DIR \ + --output_dir=$DATA_DIR \ + --seed=$SEED +... +``` + +3. Preprocess for training: + +```bash +>>> ./preprocess.sh $DATA_DIR +... +``` + +4. Train: + +```bash +>>> cd .. +>>> DESC=default +>>> CUDA=0 +>>> ./train_python150k.sh $DATA_DIR $DESC $CUDA $SEED +... +``` + +## Test results (seed=239) + +### Best scores + +**setup#2**: `batch_size=64` +**setup#3**: `embedding_size=256,use_momentum=False` +**setup#4**: `batch_size=32,embedding_size=256,embeddings_dropout_keep_prob=0.5,use_momentum=False` + +| params | Precision | Recall | F1 | ROUGE-2 | ROUGE-L | +|---|---|---|---|---|---| +| default | 0.37 | 0.27 | 0.31 | 0.06 | 0.38 | +| setup#2 | 0.40 | 0.31 | 0.34 | 0.08 | 0.41 | +| setup#3 | 0.36 | 0.31 | 0.33 | 0.09 | 0.38 | +| setup#4 | 0.33 | 0.25 | 0.28 | 0.05 | 0.34 | + +### Ablation studies + +| params | Precision | Recall | F1 | ROUGE-2 | ROUGE-L | +|---|---|---|---|---|---| +| default | 0.37 | 0.27 | 0.31 | 0.06 | 0.38 | +| no ast nodes (5th epoch) | 0.27 | 0.16 | 0.20 | 0.02 | 0.28 | +| no token split (4th epoch) | 0.60 | 0.09 | 0.15 | 0.00 | 0.60 | \ No newline at end of file diff --git a/Python150kExtractor/extract.py b/Python150kExtractor/extract.py new file mode 100644 index 0000000..cb140a0 --- /dev/null +++ b/Python150kExtractor/extract.py @@ -0,0 +1,189 @@ +import argparse +import re +import json +import multiprocessing +import itertools +import tqdm +import joblib +import numpy as np + +from pathlib import Path +from sklearn import model_selection as sklearn_model_selection + +METHOD_NAME, NUM = 'METHODNAME', 'NUM' + +parser = argparse.ArgumentParser() +parser.add_argument('--data_dir', required=True, type=str) +parser.add_argument('--valid_p', type=float, default=0.2) +parser.add_argument('--max_path_length', type=int, default=8) +parser.add_argument('--max_path_width', type=int, default=2) +parser.add_argument('--use_method_name', type=bool, default=True) +parser.add_argument('--use_nums', type=bool, default=True) +parser.add_argument('--output_dir', required=True, type=str) +parser.add_argument('--n_jobs', type=int, default=multiprocessing.cpu_count()) +parser.add_argument('--seed', type=int, default=239) + + +def __collect_asts(json_file): + with open(json_file, 'r', encoding='utf-8') as f: + for line in tqdm.tqdm(f): + yield line + + +def __terminals(ast, node_index, args): + stack, paths = [], [] + + def dfs(v): + stack.append(v) + + v_node = ast[v] + + if 'value' in v_node: + if v == node_index: # Top-level func def node. + if args.use_method_name: + paths.append((stack.copy(), METHOD_NAME)) + else: + v_type = v_node['type'] + + if v_type.startswith('Name'): + paths.append((stack.copy(), v_node['value'])) + elif args.use_nums and v_type == 'Num': + paths.append((stack.copy(), NUM)) + else: + pass + + if 'children' in v_node: + for child in v_node['children']: + dfs(child) + + stack.pop() + + dfs(node_index) + + return paths + + +def __merge_terminals2_paths(v_path, u_path): + s, n, m = 0, len(v_path), len(u_path) + while s < min(n, m) and v_path[s] == u_path[s]: + s += 1 + + prefix = list(reversed(v_path[s:])) + lca = v_path[s - 1] + suffix = u_path[s:] + + return prefix, lca, suffix + + +def __raw_tree_paths(ast, node_index, args): + tnodes = __terminals(ast, node_index, args) + + tree_paths = [] + for (v_path, v_value), (u_path, u_value) in itertools.combinations( + iterable=tnodes, + r=2, + ): + prefix, lca, suffix = __merge_terminals2_paths(v_path, u_path) + if (len(prefix) + 1 + len(suffix) <= args.max_path_length) \ + and (abs(len(prefix) - len(suffix)) <= args.max_path_width): + path = prefix + [lca] + suffix + tree_path = v_value, path, u_value + tree_paths.append(tree_path) + + return tree_paths + + +def __delim_name(name): + if name in {METHOD_NAME, NUM}: + return name + + def camel_case_split(identifier): + matches = re.finditer( + '.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', + identifier, + ) + return [m.group(0) for m in matches] + + blocks = [] + for underscore_block in name.split('_'): + blocks.extend(camel_case_split(underscore_block)) + + return '|'.join(block.lower() for block in blocks) + + +def __collect_sample(ast, fd_index, args): + root = ast[fd_index] + if root['type'] != 'FunctionDef': + raise ValueError('Wrong node type.') + + target = root['value'] + + tree_paths = __raw_tree_paths(ast, fd_index, args) + contexts = [] + for tree_path in tree_paths: + start, connector, finish = tree_path + + start, finish = __delim_name(start), __delim_name(finish) + connector = '|'.join(ast[v]['type'] for v in connector) + + context = f'{start},{connector},{finish}' + contexts.append(context) + + if len(contexts) == 0: + return None + + target = __delim_name(target) + context = ' '.join(contexts) + + return f'{target} {context}' + + +def __collect_samples(ast, args): + samples = [] + for node_index, node in enumerate(ast): + if node['type'] == 'FunctionDef': + sample = __collect_sample(ast, node_index, args) + if sample is not None: + samples.append(sample) + + return samples + + +def __collect_all_and_save(asts, args, output_file): + parallel = joblib.Parallel(n_jobs=args.n_jobs) + func = joblib.delayed(__collect_samples) + + samples = parallel(func(ast, args) for ast in tqdm.tqdm(asts)) + samples = list(itertools.chain.from_iterable(samples)) + + with open(output_file, 'w') as f: + for line_index, line in enumerate(samples): + f.write(line + ('' if line_index == len(samples) - 1 else '\n')) + + +def main(): + args = parser.parse_args() + np.random.seed(args.seed) + + data_dir = Path(args.data_dir) + trains = list(__collect_asts(data_dir / 'python100k_train.json')) + evals = list(__collect_asts(data_dir / 'python50k_eval.json')) + + train, valid = sklearn_model_selection.train_test_split( + trains, + test_size=args.valid_p, + ) + test = evals + + output_dir = Path(args.output_dir) + output_dir.mkdir(exist_ok=True) + for split_name, split in zip( + ('train', 'valid', 'test'), + (train, valid, test), + ): + output_file = output_dir / f'{split_name}_output_file.txt' + __collect_all_and_save((json.loads(line) for line in split), args, output_file) + + +if __name__ == '__main__': + main() diff --git a/Python150kExtractor/preprocess.sh b/Python150kExtractor/preprocess.sh new file mode 100644 index 0000000..d3ebefd --- /dev/null +++ b/Python150kExtractor/preprocess.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +MAX_CONTEXTS=200 +MAX_DATA_CONTEXTS=1000 +SUBTOKEN_VOCAB_SIZE=186277 +TARGET_VOCAB_SIZE=26347 + +data_dir=${1:-data} +mkdir -p "${data_dir}" +train_data_file=$data_dir/train_output_file.txt +valid_data_file=$data_dir/valid_output_file.txt +test_data_file=$data_dir/test_output_file.txt + +echo "Creating histograms from the training data..." +target_histogram_file=$data_dir/histo.tgt.c2s +source_subtoken_histogram=$data_dir/histo.ori.c2s +node_histogram_file=$data_dir/histo.node.c2s +cut <"${train_data_file}" -d' ' -f1 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' >"${target_histogram_file}" +cut <"${train_data_file}" -d' ' -f2- | tr ' ' '\n' | cut -d',' -f1,3 | tr ',|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' >"${source_subtoken_histogram}" +cut <"${train_data_file}" -d' ' -f2- | tr ' ' '\n' | cut -d',' -f2 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' >"${node_histogram_file}" + +echo "Preprocessing..." +python ../preprocess.py \ + --train_data "${train_data_file}" \ + --val_data "${valid_data_file}" \ + --test_data "${test_data_file}" \ + --max_contexts ${MAX_CONTEXTS} \ + --max_data_contexts ${MAX_DATA_CONTEXTS} \ + --subtoken_vocab_size ${SUBTOKEN_VOCAB_SIZE} \ + --target_vocab_size ${TARGET_VOCAB_SIZE} \ + --target_histogram "${target_histogram_file}" \ + --subtoken_histogram "${source_subtoken_histogram}" \ + --node_histogram "${node_histogram_file}" \ + --output_name "${data_dir}"/"$(basename "${data_dir}")" +rm \ + "${target_histogram_file}" \ + "${source_subtoken_histogram}" \ + "${node_histogram_file}" diff --git a/README.md b/README.md index 561c2e0..86d2417 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ This is an official implementation of the model described in: [Uri Alon](http://urialon.cswp.cs.technion.ac.il), [Shaked Brody](http://www.cs.technion.ac.il/people/shakedbr/), [Omer Levy](https://levyomer.wordpress.com) and [Eran Yahav](http://www.cs.technion.ac.il/~yahave/), "code2seq: Generating Sequences from Structured Representations of Code" [[PDF]](https://openreview.net/pdf?id=H1gKYo09tX) -to appear in *ICLR'2019* +Appeared in **ICLR'2019** (**poster** available [here](https://urialon.cswp.cs.technion.ac.il/wp-content/uploads/sites/83/2019/05/ICLR19_poster_code2seq.pdf)) An **online demo** is available at [https://code2seq.org](https://code2seq.org). @@ -14,6 +14,13 @@ Contributions are welcome.
+## See also: + * **Structural Language Models for Code** (ICML'2020) is a new paper that learns to generate the missing code within a larger code snippet. This is similar to code completion, but is able to predict complex expressions rather than a single token at a time. See [PDF](https://arxiv.org/pdf/1910.00577.pdf), demo at [http://AnyCodeGen.org](http://AnyCodeGen.org). + * **Adversarial Examples for Models of Code** is a new paper that shows how to slightly mutate the input code snippet of code2vec and GNNs models (thus, introducing adversarial examples), such that the model (code2vec or GNNs) will output a prediction of our choice. See [PDF](https://arxiv.org/pdf/1910.07517.pdf) (code: soon). + * **Neural Reverse Engineering of Stripped Binaries** is a new paper that learns to predict procedure names in stripped binaries, thus use neural networks for reverse engineering. See [PDF](https://arxiv.org/pdf/1902.09122) (code: soon). + * **code2vec** (POPL'2019) is our previous model. It can only generate a single label at a time (rather than a sequence as code2seq), but it is much faster to train (because of its simplicity). See [PDF](https://urialon.cswp.cs.technion.ac.il/wp-content/uploads/sites/83/2018/12/code2vec-popl19.pdf), demo at [https://code2vec.org](https://code2vec.org) and [code](https://github.com/tech-srl/code2vec/). + + Table of Contents ================= * [Requirements](#requirements) @@ -22,14 +29,17 @@ Table of Contents * [Releasing a trained mode](#releasing-a-trained-model) * [Extending to other languages](#extending-to-other-languages) * [Datasets](#datasets) + * [Baselines](#baselines) * [Citation](#citation) ## Requirements * [python3](https://www.linuxbabe.com/ubuntu/install-python-3-6-ubuntu-16-04-16-10-17-04) - * TensorFlow 1.12 or newer ([install](https://www.tensorflow.org/install/install_linux)). To check TensorFlow version: + * TensorFlow 1.12 ([install](https://www.tensorflow.org/install/install_linux)). To check TensorFlow version: > python3 -c 'import tensorflow as tf; print(tf.\_\_version\_\_)' + - For a TensorFlow 2.1 implementation by [@Kolkir](https://github.com/Kolkir/), see: [https://github.com/Kolkir/code2seq](https://github.com/Kolkir/code2seq) * For [creating a new Java dataset](#creating-and-preprocessing-a-new-java-dataset) or [manually examining a trained model](#step-4-manual-examination-of-a-trained-model) (any operation that requires parsing of a new code example): [JDK](https://openjdk.java.net/install/) * For creating a C# dataset: [dotnet-core](https://dotnet.microsoft.com/download) version 2.2 or newer. + * `pip install rouge` for computing rouge scores. ## Quickstart ### Step 0: Cloning this repository @@ -70,7 +80,7 @@ tar -xvzf java-large-model.tar.gz ``` ##### Note: -This trained model is in a "released" state, which means that we stripped it from its training parameters and can thus be used for inference, but cannot be further trained. +This trained model is in a "released" state, which means that we stripped it from its training parameters. #### Training a model from scratch To train a model from scratch: @@ -168,12 +178,26 @@ This will save a copy of the trained model with the '.release' suffix. A "released" model usually takes ~3x less disk space. ## Extending to other languages + +This project currently supports Java and C\# as the input languages. + +_**March 2020** - a code2seq extractor for **C++** based on LLVM was developed by [@Kolkir](https://github.com/Kolkir/) and is available here: [https://github.com/Kolkir/cppminer](https://github.com/Kolkir/cppminer)._ + +_**January 2020** - a code2seq extractor for Python (specifically targeting the Python150k dataset) was contributed by [@stasbel](https://github.com/stasbel). See: [https://github.com/tech-srl/code2seq/tree/master/Python150kExtractor](https://github.com/tech-srl/code2seq/tree/master/Python150kExtractor)._ + +_**January 2020** - an extractor for predicting TypeScript type annotations for JavaScript input using code2vec was developed by [@izosak](https://github.com/izosak) and Noa Cohen, and is available here: +[https://github.com/tech-srl/id2vec](https://github.com/tech-srl/id2vec)._ + +~~_**June 2019** - an extractor for **C** that is compatible with our model was developed by [CMU SEI team](https://github.com/cmu-sei/code2vec-c)._~~ - removed by CMU SEI team. + +_**June 2019** - a code2vec extractor for **Python, Java, C, C++** by JetBrains Research is available here: [PathMiner](https://github.com/JetBrains-Research/astminer)._ + To extend code2seq to other languages other than Java and C#, a new extractor (similar to the [JavaExtractor](JavaExtractor)) should be implemented, and be called by [preprocess.sh](preprocess.sh). Basically, an extractor should be able to output for each directory containing source files: * A single text file, where each row is an example. * Each example is a space-delimited list of fields, where: - 1. The first field is the target label, internally delimited by the "|" character (for example: `compare|ignore|case` + 1. The first field is the target label, internally delimited by the "|" character (for example: `compare|ignore|case`) 2. Each of the following field are contexts, where each context has three components separated by commas (","). None of these components can include spaces nor commas. We refer to these three components as a token, a path, and another token, but in general other types of ternary contexts can be considered. @@ -194,10 +218,59 @@ To download the Java-small, Java-med and Java-large datasets used in the Code Su * [Java-small](https://s3.amazonaws.com/code2seq/datasets/java-small.tar.gz) * [Java-med](https://s3.amazonaws.com/code2seq/datasets/java-med.tar.gz) * [Java-large](https://s3.amazonaws.com/code2seq/datasets/java-large.tar.gz) + +To download the preprocessed datasets, use: + * [Java-small-preprocessed](https://s3.amazonaws.com/code2seq/datasets/java-small-preprocessed.tar.gz) + * [Java-med-preprocessed](https://s3.amazonaws.com/code2seq/datasets/java-med-preprocessed.tar.gz) + * [Java-large-preprocessed](https://s3.amazonaws.com/code2seq/datasets/java-large-preprocessed.tar.gz) ### C# The C# dataset used in the Code Captioning task can be downloaded from the [CodeNN](https://github.com/sriniiyer/codenn/) repository. +## Baselines +### Using the trained model +For the NMT baselines (BiLSTM, Transformer) we used the implementation of [OpenNMT-py](http://opennmt.net/OpenNMT-py/). +The trained BiLSTM model is available here: +`https://code2seq.s3.amazonaws.com/lstm_baseline/model_acc_62.88_ppl_12.03_e16.pt` + +Test+validation sources and targets: +``` +https://code2seq.s3.amazonaws.com/lstm_baseline/test_expected_actual.txt +https://code2seq.s3.amazonaws.com/lstm_baseline/test_source.txt +https://code2seq.s3.amazonaws.com/lstm_baseline/test_target.txt +https://code2seq.s3.amazonaws.com/lstm_baseline/val_source.txt +https://code2seq.s3.amazonaws.com/lstm_baseline/val_target.txt +``` + +The command line for "translating" a "source" file to a "target" is: +`python3 translate.py -model model_acc_62.88_ppl_12.03_e16.pt -src test_source.txt -output translation_epoch16.txt -gpu 0` + +This results in a `translation_epoch16.txt` which we compare to `test_target.txt` to compute the score. +The file `test_expected_actual.txt` is a line-by-line concatenation of the true reference ("expected") with the corresponding prediction (the "actual"). + +### Creating data for the baseline +We first modified the JavaExtractor (the same one as in this) to locate the methods to train on and print them to a file where each method is a single line. This modification is currently not checked in, but instead of extracting paths, it just prints `node.toString()` and replaces "\n" with space, where `node` is the object holding the AST node of type `MethodDeclaration`. + +Then, we tokenized (including sub-tokenization of identifiers, i.e., `"ArrayList"-> ["Array","List"])` each method body using `javalang`, using [this](baseline_tokenization/subtokenize_nmt_baseline.py) script (which can be run on [this](baseline_tokenization/input_example.txt) input example). +So a program of: +``` +void methodName(String fooBar) { + System.out.println("hello world"); +} +``` + +should be printed by the modified JavaExtractor as: + +```method name|void (String fooBar){ System.out.println("hello world");}``` + +and the tokenization script would turn it into: + +```void ( String foo Bar ) { System . out . println ( " hello world " ) ; }``` + +and the label to be predicted, i.e., "method name", into a separate file. + +OpenNMT-py can then be trained over these training source and target files. + ## Citation [code2seq: Generating Sequences from Structured Representations of Code](https://arxiv.org/pdf/1808.01400) diff --git a/baseline_tokenization/input_example.txt b/baseline_tokenization/input_example.txt new file mode 100644 index 0000000..244fc1a --- /dev/null +++ b/baseline_tokenization/input_example.txt @@ -0,0 +1,10 @@ +requires landscape|boolean (){ return false; } +get parent key|Object (){ return new ContactsUiKey(); } +get parent key|Object (){ return new ContactsUiKey(); } +get layout id|int (){ return R.layout.loose_screen; } +get parent key|Object (){ return new EditContactKey(contactId); } +to contact|Contact (){ return new Contact(id, name, email); } +to string|String (){ return "Welcome!\nClick to continue."; } +get parent key|Object (){ return new EditContactKey(contactId); } +tear down services|void (@NonNull Services services){ } +get layout id|int (){ return R.layout.landscape_screen; } diff --git a/baseline_tokenization/javalang/__init__.py b/baseline_tokenization/javalang/__init__.py new file mode 100644 index 0000000..8ee0b30 --- /dev/null +++ b/baseline_tokenization/javalang/__init__.py @@ -0,0 +1,8 @@ + +from . import parser +from . import parse +from . import tokenizer +from . import javadoc + + +__version__ = "0.10.1" diff --git a/baseline_tokenization/javalang/ast.py b/baseline_tokenization/javalang/ast.py new file mode 100644 index 0000000..66f9312 --- /dev/null +++ b/baseline_tokenization/javalang/ast.py @@ -0,0 +1,78 @@ +import pickle + +import six + + +class MetaNode(type): + def __new__(mcs, name, bases, dict): + attrs = list(dict['attrs']) + dict['attrs'] = list() + + for base in bases: + if hasattr(base, 'attrs'): + dict['attrs'].extend(base.attrs) + + dict['attrs'].extend(attrs) + + return type.__new__(mcs, name, bases, dict) + + +@six.add_metaclass(MetaNode) +class Node(object): + attrs = () + + def __init__(self, **kwargs): + values = kwargs.copy() + + for attr_name in self.attrs: + value = values.pop(attr_name, None) + setattr(self, attr_name, value) + + if values: + raise ValueError('Extraneous arguments') + + def __equals__(self, other): + if type(other) is not type(self): + return False + + for attr in self.attrs: + if getattr(other, attr) != getattr(self, attr): + return False + + return True + + def __repr__(self): + return type(self).__name__ + + def __iter__(self): + return walk_tree(self) + + def filter(self, pattern): + for path, node in self: + if ((isinstance(pattern, type) and isinstance(node, pattern)) or + (node == pattern)): + yield path, node + + @property + def children(self): + return [getattr(self, attr_name) for attr_name in self.attrs] + +def walk_tree(root): + children = None + + if isinstance(root, Node): + yield (), root + children = root.children + else: + children = root + + for child in children: + if isinstance(child, (Node, list, tuple)): + for path, node in walk_tree(child): + yield (root,) + path, node + +def dump(ast, file): + pickle.dump(ast, file) + +def load(file): + return pickle.load(file) diff --git a/baseline_tokenization/javalang/javadoc.py b/baseline_tokenization/javalang/javadoc.py new file mode 100644 index 0000000..ee3635d --- /dev/null +++ b/baseline_tokenization/javalang/javadoc.py @@ -0,0 +1,120 @@ + +import re + +def join(s): + return ' '.join(l.strip() for l in s.split('\n')) + +class DocBlock(object): + def __init__(self): + self.description = '' + self.return_doc = None + self.params = [] + + self.authors = [] + self.deprecated = False + + # @exception and @throw are equivalent + self.throws = {} + self.exceptions = self.throws + + self.tags = {} + + def add_block(self, name, value): + value = value.strip() + + if name == 'param': + try: + param, description = value.split(None, 1) + except ValueError: + param, description = value, '' + self.params.append((param, join(description))) + + elif name in ('throws', 'exception'): + try: + ex, description = value.split(None, 1) + except ValueError: + ex, description = value, '' + self.throws[ex] = join(description) + + elif name == 'return': + self.return_doc = value + + elif name == 'author': + self.authors.append(value) + + elif name == 'deprecated': + self.deprecated = True + + self.tags.setdefault(name, []).append(value) + +blocks_re = re.compile('(^@)', re.MULTILINE) +leading_space_re = re.compile(r'^\s*\*', re.MULTILINE) +blocks_justify_re = re.compile(r'^\s*@', re.MULTILINE) + +def _sanitize(s): + s = s.strip() + + if not (s[:3] == '/**' and s[-2:] == '*/'): + raise ValueError('not a valid Javadoc comment') + + s = s.replace('\t', ' ') + + return s + +def _uncomment(s): + # Remove /** and */ + s = s[3:-2].strip() + + return leading_space_re.sub('', s) + +def _get_indent_level(s): + return len(s) - len(s.lstrip()) + +def _left_justify(s): + lines = s.rstrip().splitlines() + + if not lines: + return '' + + indent_levels = [] + for line in lines: + if line.strip(): + indent_levels.append(_get_indent_level(line)) + indent_levels.sort() + + common_indent = indent_levels[0] + if common_indent == 0: + return s + else: + lines = [line[common_indent:] for line in lines] + return '\n'.join(lines) + +def _force_blocks_left(s): + return blocks_justify_re.sub('@', s) + +def parse(raw): + sanitized = _sanitize(raw) + uncommented = _uncomment(sanitized) + justified = _left_justify(uncommented) + justified_fixed = _force_blocks_left(justified) + prepared = justified_fixed + + blocks = blocks_re.split(prepared) + + doc = DocBlock() + + if blocks[0] != '@': + doc.description = blocks[0].strip() + blocks = blocks[2::2] + else: + blocks = blocks[1::2] + + for block in blocks: + try: + tag, value = block.split(None, 1) + except ValueError: + tag, value = block, '' + + doc.add_block(tag, value) + + return doc diff --git a/baseline_tokenization/javalang/parse.py b/baseline_tokenization/javalang/parse.py new file mode 100644 index 0000000..0451fed --- /dev/null +++ b/baseline_tokenization/javalang/parse.py @@ -0,0 +1,53 @@ + +from .parser import Parser +from .tokenizer import tokenize + +def parse_expression(exp): + if not exp.endswith(';'): + exp = exp + ';' + + tokens = tokenize(exp) + parser = Parser(tokens) + + return parser.parse_expression() + +def parse_member_signature(sig): + if not sig.endswith(';'): + sig = sig + ';' + + tokens = tokenize(sig) + parser = Parser(tokens) + + return parser.parse_member_declaration() + +def parse_constructor_signature(sig): + # Add an empty body to the signature, replacing a ; if necessary + if sig.endswith(';'): + sig = sig[:-1] + sig = sig + '{ }' + + tokens = tokenize(sig) + parser = Parser(tokens) + + return parser.parse_member_declaration() + +def parse_type(s): + tokens = tokenize(s) + parser = Parser(tokens) + + return parser.parse_type() + +def parse_type_signature(sig): + if sig.endswith(';'): + sig = sig[:-1] + sig = sig + '{ }' + + tokens = tokenize(sig) + parser = Parser(tokens) + + return parser.parse_class_or_interface_declaration() + +def parse(s): + tokens = tokenize(s) + parser = Parser(tokens) + return parser.parse() diff --git a/baseline_tokenization/javalang/parser.py b/baseline_tokenization/javalang/parser.py new file mode 100644 index 0000000..c78a9f4 --- /dev/null +++ b/baseline_tokenization/javalang/parser.py @@ -0,0 +1,2354 @@ +import six + +from . import util +from . import tree +from .tokenizer import ( + EndOfInput, Keyword, Modifier, BasicType, Identifier, + Annotation, Literal, Operator, JavaToken, + ) + +ENABLE_DEBUG_SUPPORT = False + +def parse_debug(method): + global ENABLE_DEBUG_SUPPORT + + if ENABLE_DEBUG_SUPPORT: + def _method(self): + if not hasattr(self, 'recursion_depth'): + self.recursion_depth = 0 + + if self.debug: + depth = "%02d" % (self.recursion_depth,) + token = six.text_type(self.tokens.look()) + start_value = self.tokens.look().value + name = method.__name__ + sep = ("-" * self.recursion_depth) + e_message = "" + + print("%s %s> %s(%s)" % (depth, sep, name, token)) + + self.recursion_depth += 1 + + try: + r = method(self) + + except JavaSyntaxError as e: + e_message = e.description + raise + + except Exception as e: + e_message = six.text_type(e) + raise + + finally: + token = six.text_type(self.tokens.last()) + print("%s <%s %s(%s, %s) %s" % + (depth, sep, name, start_value, token, e_message)) + self.recursion_depth -= 1 + else: + self.recursion_depth += 1 + try: + r = method(self) + finally: + self.recursion_depth -= 1 + + return r + + return _method + + else: + return method + +# ------------------------------------------------------------------------------ +# ---- Parsing exception ---- + +class JavaParserBaseException(Exception): + def __init__(self, message=''): + super(JavaParserBaseException, self).__init__(message) + +class JavaSyntaxError(JavaParserBaseException): + def __init__(self, description, at=None): + super(JavaSyntaxError, self).__init__() + + self.description = description + self.at = at + +class JavaParserError(JavaParserBaseException): + pass + +# ------------------------------------------------------------------------------ +# ---- Parser class ---- + +class Parser(object): + operator_precedence = [ set(('||',)), + set(('&&',)), + set(('|',)), + set(('^',)), + set(('&',)), + set(('==', '!=')), + set(('<', '>', '>=', '<=', 'instanceof')), + set(('<<', '>>', '>>>')), + set(('+', '-')), + set(('*', '/', '%')) ] + + def __init__(self, tokens): + self.tokens = util.LookAheadListIterator(tokens) + self.tokens.set_default(EndOfInput(None)) + + self.debug = False + +# ------------------------------------------------------------------------------ +# ---- Debug control ---- + + def set_debug(self, debug=True): + self.debug = debug + +# ------------------------------------------------------------------------------ +# ---- Parsing entry point ---- + + def parse(self): + return self.parse_compilation_unit() + +# ------------------------------------------------------------------------------ +# ---- Helper methods ---- + + def illegal(self, description, at=None): + if not at: + at = self.tokens.look() + + raise JavaSyntaxError(description, at) + + def accept(self, *accepts): + last = None + + if len(accepts) == 0: + raise JavaParserError("Missing acceptable values") + + for accept in accepts: + token = next(self.tokens) + if isinstance(accept, six.string_types) and ( + not token.value == accept): + self.illegal("Expected '%s'" % (accept,)) + elif isinstance(accept, type) and not isinstance(token, accept): + self.illegal("Expected %s" % (accept.__name__,)) + + last = token + + return last.value + + def would_accept(self, *accepts): + if len(accepts) == 0: + raise JavaParserError("Missing acceptable values") + + for i, accept in enumerate(accepts): + token = self.tokens.look(i) + + if isinstance(accept, six.string_types) and ( + not token.value == accept): + return False + elif isinstance(accept, type) and not isinstance(token, accept): + return False + + return True + + def try_accept(self, *accepts): + if len(accepts) == 0: + raise JavaParserError("Missing acceptable values") + + for i, accept in enumerate(accepts): + token = self.tokens.look(i) + + if isinstance(accept, six.string_types) and ( + not token.value == accept): + return False + elif isinstance(accept, type) and not isinstance(token, accept): + return False + + for i in range(0, len(accepts)): + next(self.tokens) + + return True + + def build_binary_operation(self, parts, start_level=0): + if len(parts) == 1: + return parts[0] + + operands = list() + operators = list() + + i = 0 + + for level in range(start_level, len(self.operator_precedence)): + for j in range(1, len(parts) - 1, 2): + if parts[j] in self.operator_precedence[level]: + operand = self.build_binary_operation(parts[i:j], level + 1) + operator = parts[j] + i = j + 1 + + operands.append(operand) + operators.append(operator) + + if operands: + break + + operand = self.build_binary_operation(parts[i:], level + 1) + operands.append(operand) + + operation = operands[0] + + for operator, operandr in zip(operators, operands[1:]): + operation = tree.BinaryOperation(operandl=operation) + operation.operator = operator + operation.operandr = operandr + + return operation + + def is_annotation(self, i=0): + """ Returns true if the position is the start of an annotation application + (as opposed to an annotation declaration) + + """ + + return (isinstance(self.tokens.look(i), Annotation) + and not self.tokens.look(i + 1).value == 'interface') + + def is_annotation_declaration(self, i=0): + """ Returns true if the position is the start of an annotation application + (as opposed to an annotation declaration) + + """ + + return (isinstance(self.tokens.look(i), Annotation) + and self.tokens.look(i + 1).value == 'interface') + +# ------------------------------------------------------------------------------ +# ---- Parsing methods ---- + +# ------------------------------------------------------------------------------ +# -- Identifiers -- + + @parse_debug + def parse_identifier(self): + return self.accept(Identifier) + + @parse_debug + def parse_qualified_identifier(self): + qualified_identifier = list() + + while True: + identifier = self.parse_identifier() + qualified_identifier.append(identifier) + + if not self.try_accept('.'): + break + + return '.'.join(qualified_identifier) + + @parse_debug + def parse_qualified_identifier_list(self): + qualified_identifiers = list() + + while True: + qualified_identifier = self.parse_qualified_identifier() + qualified_identifiers.append(qualified_identifier) + + if not self.try_accept(','): + break + + return qualified_identifiers + +# ------------------------------------------------------------------------------ +# -- Top level units -- + + @parse_debug + def parse_compilation_unit(self): + package = None + package_annotations = None + javadoc = None + import_declarations = list() + type_declarations = list() + + self.tokens.push_marker() + next_token = self.tokens.look() + if next_token: + javadoc = next_token.javadoc + + if self.is_annotation(): + package_annotations = self.parse_annotations() + + if self.try_accept('package'): + self.tokens.pop_marker(False) + package_name = self.parse_qualified_identifier() + package = tree.PackageDeclaration(annotations=package_annotations, + name=package_name, + documentation=javadoc) + self.accept(';') + else: + self.tokens.pop_marker(True) + package_annotations = None + + while self.would_accept('import'): + import_declaration = self.parse_import_declaration() + import_declarations.append(import_declaration) + + while not isinstance(self.tokens.look(), EndOfInput): + try: + type_declaration = self.parse_type_declaration() + except StopIteration: + self.illegal("Unexpected end of input") + + if type_declaration: + type_declarations.append(type_declaration) + + return tree.CompilationUnit(package=package, + imports=import_declarations, + types=type_declarations) + + @parse_debug + def parse_import_declaration(self): + qualified_identifier = list() + static = False + import_all = False + + self.accept('import') + + if self.try_accept('static'): + static = True + + while True: + identifier = self.parse_identifier() + qualified_identifier.append(identifier) + + if self.try_accept('.'): + if self.try_accept('*'): + self.accept(';') + import_all = True + break + + else: + self.accept(';') + break + + return tree.Import(path='.'.join(qualified_identifier), + static=static, + wildcard=import_all) + + @parse_debug + def parse_type_declaration(self): + if self.try_accept(';'): + return None + else: + return self.parse_class_or_interface_declaration() + + @parse_debug + def parse_class_or_interface_declaration(self): + modifiers, annotations, javadoc = self.parse_modifiers() + type_declaration = None + + token = self.tokens.look() + if token.value == 'class': + type_declaration = self.parse_normal_class_declaration() + elif token.value == 'enum': + type_declaration = self.parse_enum_declaration() + elif token.value == 'interface': + type_declaration = self.parse_normal_interface_declaration() + elif self.is_annotation_declaration(): + type_declaration = self.parse_annotation_type_declaration() + else: + self.illegal("Expected type declaration") + + type_declaration.modifiers = modifiers + type_declaration.annotations = annotations + type_declaration.documentation = javadoc + + return type_declaration + + @parse_debug + def parse_normal_class_declaration(self): + name = None + type_params = None + extends = None + implements = None + body = None + + self.accept('class') + + name = self.parse_identifier() + + if self.would_accept('<'): + type_params = self.parse_type_parameters() + + if self.try_accept('extends'): + extends = self.parse_type() + + if self.try_accept('implements'): + implements = self.parse_type_list() + + body = self.parse_class_body() + + return tree.ClassDeclaration(name=name, + type_parameters=type_params, + extends=extends, + implements=implements, + body=body) + + @parse_debug + def parse_enum_declaration(self): + name = None + implements = None + body = None + + self.accept('enum') + name = self.parse_identifier() + + if self.try_accept('implements'): + implements = self.parse_type_list() + + body = self.parse_enum_body() + + return tree.EnumDeclaration(name=name, + implements=implements, + body=body) + + @parse_debug + def parse_normal_interface_declaration(self): + name = None + type_parameters = None + extends = None + body = None + + self.accept('interface') + name = self.parse_identifier() + + if self.would_accept('<'): + type_parameters = self.parse_type_parameters() + + if self.try_accept('extends'): + extends = self.parse_type_list() + + body = self.parse_interface_body() + + return tree.InterfaceDeclaration(name=name, + type_parameters=type_parameters, + extends=extends, + body=body) + + @parse_debug + def parse_annotation_type_declaration(self): + name = None + body = None + + self.accept('@', 'interface') + + name = self.parse_identifier() + body = self.parse_annotation_type_body() + + return tree.AnnotationDeclaration(name=name, + body=body) + +# ------------------------------------------------------------------------------ +# -- Types -- + + @parse_debug + def parse_type(self): + java_type = None + + if isinstance(self.tokens.look(), BasicType): + java_type = self.parse_basic_type() + elif isinstance(self.tokens.look(), Identifier): + java_type = self.parse_reference_type() + else: + self.illegal("Expected type") + + java_type.dimensions = self.parse_array_dimension() + + return java_type + + @parse_debug + def parse_basic_type(self): + return tree.BasicType(name=self.accept(BasicType)) + + @parse_debug + def parse_reference_type(self): + reference_type = tree.ReferenceType() + tail = reference_type + + while True: + tail.name = self.parse_identifier() + + if self.would_accept('<'): + tail.arguments = self.parse_type_arguments() + + if self.try_accept('.'): + tail.sub_type = tree.ReferenceType() + tail = tail.sub_type + else: + break + + return reference_type + + @parse_debug + def parse_type_arguments(self): + type_arguments = list() + + self.accept('<') + + while True: + type_argument = self.parse_type_argument() + type_arguments.append(type_argument) + + if self.try_accept('>'): + break + + self.accept(',') + + return type_arguments + + @parse_debug + def parse_type_argument(self): + pattern_type = None + base_type = None + + if self.try_accept('?'): + if self.tokens.look().value in ('extends', 'super'): + pattern_type = self.tokens.next().value + else: + return tree.TypeArgument(pattern_type='?') + + if self.would_accept(BasicType): + base_type = self.parse_basic_type() + self.accept('[', ']') + base_type.dimensions = [None] + else: + base_type = self.parse_reference_type() + base_type.dimensions = [] + + base_type.dimensions += self.parse_array_dimension() + + return tree.TypeArgument(type=base_type, + pattern_type=pattern_type) + + @parse_debug + def parse_nonwildcard_type_arguments(self): + self.accept('<') + type_arguments = self.parse_type_list() + self.accept('>') + + return [tree.TypeArgument(type=t) for t in type_arguments] + + @parse_debug + def parse_type_list(self): + types = list() + + while True: + if self.would_accept(BasicType): + base_type = self.parse_basic_type() + self.accept('[', ']') + base_type.dimensions = [None] + else: + base_type = self.parse_reference_type() + base_type.dimensions = [] + + base_type.dimensions += self.parse_array_dimension() + types.append(base_type) + + if not self.try_accept(','): + break + + return types + + @parse_debug + def parse_type_arguments_or_diamond(self): + if self.try_accept('<', '>'): + return list() + else: + return self.parse_type_arguments() + + @parse_debug + def parse_nonwildcard_type_arguments_or_diamond(self): + if self.try_accept('<', '>'): + return list() + else: + return self.parse_nonwildcard_type_arguments() + + @parse_debug + def parse_type_parameters(self): + type_parameters = list() + + self.accept('<') + + while True: + type_parameter = self.parse_type_parameter() + type_parameters.append(type_parameter) + + if self.try_accept('>'): + break + else: + self.accept(',') + + return type_parameters + + @parse_debug + def parse_type_parameter(self): + identifier = self.parse_identifier() + extends = None + + if self.try_accept('extends'): + extends = list() + + while True: + reference_type = self.parse_reference_type() + extends.append(reference_type) + + if not self.try_accept('&'): + break + + return tree.TypeParameter(name=identifier, + extends=extends) + + @parse_debug + def parse_array_dimension(self): + array_dimension = 0 + + while self.try_accept('[', ']'): + array_dimension += 1 + + return [None] * array_dimension + +# ------------------------------------------------------------------------------ +# -- Annotations and modifiers -- + + @parse_debug + def parse_modifiers(self): + annotations = list() + modifiers = set() + javadoc = None + + next_token = self.tokens.look() + if next_token: + javadoc = next_token.javadoc + + while True: + if self.would_accept(Modifier): + modifiers.add(self.accept(Modifier)) + + elif self.is_annotation(): + annotation = self.parse_annotation() + annotations.append(annotation) + + else: + break + + return (modifiers, annotations, javadoc) + + @parse_debug + def parse_annotations(self): + annotations = list() + + while True: + annotation = self.parse_annotation() + annotations.append(annotation) + + if not self.is_annotation(): + break + + return annotations + + @parse_debug + def parse_annotation(self): + qualified_identifier = None + annotation_element = None + + self.accept('@') + qualified_identifier = self.parse_qualified_identifier() + + if self.try_accept('('): + if not self.would_accept(')'): + annotation_element = self.parse_annotation_element() + self.accept(')') + + return tree.Annotation(name=qualified_identifier, + element=annotation_element) + + @parse_debug + def parse_annotation_element(self): + if self.would_accept(Identifier, '='): + return self.parse_element_value_pairs() + else: + return self.parse_element_value() + + @parse_debug + def parse_element_value_pairs(self): + pairs = list() + + while True: + pair = self.parse_element_value_pair() + pairs.append(pair) + + if not self.try_accept(','): + break + + return pairs + + @parse_debug + def parse_element_value_pair(self): + identifier = self.parse_identifier() + self.accept('=') + value = self.parse_element_value() + + return tree.ElementValuePair(name=identifier, + value=value) + + @parse_debug + def parse_element_value(self): + if self.is_annotation(): + return self.parse_annotation() + + elif self.would_accept('{'): + return self.parse_element_value_array_initializer() + + else: + return self.parse_expressionl() + + @parse_debug + def parse_element_value_array_initializer(self): + self.accept('{') + + if self.try_accept('}'): + return list() + + element_values = self.parse_element_values() + self.try_accept(',') + self.accept('}') + + return tree.ElementArrayValue(values=element_values) + + @parse_debug + def parse_element_values(self): + element_values = list() + + while True: + element_value = self.parse_element_value() + element_values.append(element_value) + + if self.would_accept('}') or self.would_accept(',', '}'): + break + + self.accept(',') + + return element_values + +# ------------------------------------------------------------------------------ +# -- Class body -- + + @parse_debug + def parse_class_body(self): + declarations = list() + + self.accept('{') + + while not self.would_accept('}'): + declaration = self.parse_class_body_declaration() + if declaration: + declarations.append(declaration) + + self.accept('}') + + return declarations + + @parse_debug + def parse_class_body_declaration(self): + token = self.tokens.look() + + if self.try_accept(';'): + return None + + elif self.would_accept('static', '{'): + self.accept('static') + return self.parse_block() + + elif self.would_accept('{'): + return self.parse_block() + + else: + return self.parse_member_declaration() + + @parse_debug + def parse_member_declaration(self): + modifiers, annotations, javadoc = self.parse_modifiers() + member = None + + token = self.tokens.look() + if self.try_accept('void'): + method_name = self.parse_identifier() + member = self.parse_void_method_declarator_rest() + member.name = method_name + + elif token.value == '<': + member = self.parse_generic_method_or_constructor_declaration() + + elif token.value == 'class': + member = self.parse_normal_class_declaration() + + elif token.value == 'enum': + member = self.parse_enum_declaration() + + elif token.value == 'interface': + member = self.parse_normal_interface_declaration() + + elif self.is_annotation_declaration(): + member = self.parse_annotation_type_declaration() + + elif self.would_accept(Identifier, '('): + constructor_name = self.parse_identifier() + member = self.parse_constructor_declarator_rest() + member.name = constructor_name + + else: + member = self.parse_method_or_field_declaraction() + + member._position = token.position + member.modifiers = modifiers + member.annotations = annotations + member.documentation = javadoc + + return member + + @parse_debug + def parse_method_or_field_declaraction(self): + member_type = self.parse_type() + member_name = self.parse_identifier() + + member = self.parse_method_or_field_rest() + + if isinstance(member, tree.MethodDeclaration): + member_type.dimensions += member.return_type.dimensions + + member.name = member_name + member.return_type = member_type + else: + member.type = member_type + member.declarators[0].name = member_name + + return member + + @parse_debug + def parse_method_or_field_rest(self): + if self.would_accept('('): + return self.parse_method_declarator_rest() + else: + rest = self.parse_field_declarators_rest() + self.accept(';') + return rest + + @parse_debug + def parse_field_declarators_rest(self): + array_dimension, initializer = self.parse_variable_declarator_rest() + declarators = [tree.VariableDeclarator(dimensions=array_dimension, + initializer=initializer)] + + while self.try_accept(','): + declarator = self.parse_variable_declarator() + declarators.append(declarator) + + return tree.FieldDeclaration(declarators=declarators) + + @parse_debug + def parse_method_declarator_rest(self): + formal_parameters = self.parse_formal_parameters() + additional_dimensions = self.parse_array_dimension() + throws = None + body = None + + if self.try_accept('throws'): + throws = self.parse_qualified_identifier_list() + + if self.would_accept('{'): + body = self.parse_block() + else: + self.accept(';') + + return tree.MethodDeclaration(parameters=formal_parameters, + throws=throws, + body=body, + return_type=tree.Type(dimensions=additional_dimensions)) + + @parse_debug + def parse_void_method_declarator_rest(self): + formal_parameters = self.parse_formal_parameters() + throws = None + body = None + + if self.try_accept('throws'): + throws = self.parse_qualified_identifier_list() + + if self.would_accept('{'): + body = self.parse_block() + else: + self.accept(';') + + return tree.MethodDeclaration(parameters=formal_parameters, + throws=throws, + body=body) + + @parse_debug + def parse_constructor_declarator_rest(self): + formal_parameters = self.parse_formal_parameters() + throws = None + body = None + + if self.try_accept('throws'): + throws = self.parse_qualified_identifier_list() + + body = self.parse_block() + + return tree.ConstructorDeclaration(parameters=formal_parameters, + throws=throws, + body=body) + + @parse_debug + def parse_generic_method_or_constructor_declaration(self): + type_parameters = self.parse_type_parameters() + method = None + + if self.would_accept(Identifier, '('): + constructor_name = self.parse_identifier() + method = self.parse_constructor_declarator_rest() + method.name = constructor_name + elif self.try_accept('void'): + method_name = self.parse_identifier() + method = self.parse_void_method_declarator_rest() + method.name = method_name + + else: + method_return_type = self.parse_type() + method_name = self.parse_identifier() + + method = self.parse_method_declarator_rest() + + method_return_type.dimensions += method.return_type.dimensions + method.return_type = method_return_type + method.name = method_name + + method.type_parameters = type_parameters + return method + +# ------------------------------------------------------------------------------ +# -- Interface body -- + + @parse_debug + def parse_interface_body(self): + declarations = list() + + self.accept('{') + while not self.would_accept('}'): + declaration = self.parse_interface_body_declaration() + + if declaration: + declarations.append(declaration) + self.accept('}') + + return declarations + + @parse_debug + def parse_interface_body_declaration(self): + if self.try_accept(';'): + return None + + modifiers, annotations, javadoc = self.parse_modifiers() + + declaration = self.parse_interface_member_declaration() + declaration.modifiers = modifiers + declaration.annotations = annotations + declaration.documentation = javadoc + + return declaration + + @parse_debug + def parse_interface_member_declaration(self): + declaration = None + + if self.would_accept('class'): + declaration = self.parse_normal_class_declaration() + elif self.would_accept('interface'): + declaration = self.parse_normal_interface_declaration() + elif self.would_accept('enum'): + declaration = self.parse_enum_declaration() + elif self.is_annotation_declaration(): + declaration = self.parse_annotation_type_declaration() + elif self.would_accept('<'): + declaration = self.parse_interface_generic_method_declarator() + elif self.try_accept('void'): + method_name = self.parse_identifier() + declaration = self.parse_void_interface_method_declarator_rest() + declaration.name = method_name + else: + declaration = self.parse_interface_method_or_field_declaration() + + return declaration + + @parse_debug + def parse_interface_method_or_field_declaration(self): + java_type = self.parse_type() + name = self.parse_identifier() + member = self.parse_interface_method_or_field_rest() + + if isinstance(member, tree.MethodDeclaration): + java_type.dimensions += member.return_type.dimensions + member.name = name + member.return_type = java_type + else: + member.declarators[0].name = name + member.type = java_type + + return member + + @parse_debug + def parse_interface_method_or_field_rest(self): + rest = None + + if self.would_accept('('): + rest = self.parse_interface_method_declarator_rest() + else: + rest = self.parse_constant_declarators_rest() + self.accept(';') + + return rest + + @parse_debug + def parse_constant_declarators_rest(self): + array_dimension, initializer = self.parse_constant_declarator_rest() + declarators = [tree.VariableDeclarator(dimensions=array_dimension, + initializer=initializer)] + + while self.try_accept(','): + declarator = self.parse_constant_declarator() + declarators.append(declarator) + + return tree.ConstantDeclaration(declarators=declarators) + + @parse_debug + def parse_constant_declarator_rest(self): + array_dimension = self.parse_array_dimension() + self.accept('=') + initializer = self.parse_variable_initializer() + + return (array_dimension, initializer) + + @parse_debug + def parse_constant_declarator(self): + name = self.parse_identifier() + additional_dimension, initializer = self.parse_constant_declarator_rest() + + return tree.VariableDeclarator(name=name, + dimensions=additional_dimension, + initializer=initializer) + + @parse_debug + def parse_interface_method_declarator_rest(self): + parameters = self.parse_formal_parameters() + array_dimension = self.parse_array_dimension() + throws = None + body = None + + if self.try_accept('throws'): + throws = self.parse_qualified_identifier_list() + + if self.would_accept('{'): + body = self.parse_block() + else: + self.accept(';') + + return tree.MethodDeclaration(parameters=parameters, + throws=throws, + body=body, + return_type=tree.Type(dimensions=array_dimension)) + + @parse_debug + def parse_void_interface_method_declarator_rest(self): + parameters = self.parse_formal_parameters() + throws = None + body = None + + if self.try_accept('throws'): + throws = self.parse_qualified_identifier_list() + + if self.would_accept('{'): + body = self.parse_block() + else: + self.accept(';') + + return tree.MethodDeclaration(parameters=parameters, + throws=throws, + body=body) + + @parse_debug + def parse_interface_generic_method_declarator(self): + type_parameters = self.parse_type_parameters() + return_type = None + method_name = None + + if not self.try_accept('void'): + return_type = self.parse_type() + + method_name = self.parse_identifier() + method = self.parse_interface_method_declarator_rest() + method.name = method_name + method.return_type = return_type + method.type_parameters = type_parameters + + return method + +# ------------------------------------------------------------------------------ +# -- Parameters and variables -- + + @parse_debug + def parse_formal_parameters(self): + formal_parameters = list() + + self.accept('(') + + if self.try_accept(')'): + return formal_parameters + + while True: + modifiers, annotations = self.parse_variable_modifiers() + parameter_type = self.parse_type() + varargs = False + + if self.try_accept('...'): + varargs = True + + parameter_name = self.parse_identifier() + parameter_type.dimensions += self.parse_array_dimension() + + parameter = tree.FormalParameter(modifiers=modifiers, + annotations=annotations, + type=parameter_type, + name=parameter_name, + varargs=varargs) + + formal_parameters.append(parameter) + + if varargs: + # varargs parameter must be the last + break + + if not self.try_accept(','): + break + + self.accept(')') + + return formal_parameters + + @parse_debug + def parse_variable_modifiers(self): + modifiers = set() + annotations = list() + + while True: + if self.try_accept('final'): + modifiers.add('final') + elif self.is_annotation(): + annotation = self.parse_annotation() + annotations.append(annotation) + else: + break + + return modifiers, annotations + + @parse_debug + def parse_variable_declators(self): + declarators = list() + + while True: + declarator = self.parse_variable_declator() + declarators.append(declarator) + + if not self.try_accept(','): + break + + return declarators + + @parse_debug + def parse_variable_declarators(self): + declarators = list() + + while True: + declarator = self.parse_variable_declarator() + declarators.append(declarator) + + if not self.try_accept(','): + break + + return declarators + + @parse_debug + def parse_variable_declarator(self): + identifier = self.parse_identifier() + array_dimension, initializer = self.parse_variable_declarator_rest() + + return tree.VariableDeclarator(name=identifier, + dimensions=array_dimension, + initializer=initializer) + + @parse_debug + def parse_variable_declarator_rest(self): + array_dimension = self.parse_array_dimension() + initializer = None + + if self.try_accept('='): + initializer = self.parse_variable_initializer() + + return (array_dimension, initializer) + + @parse_debug + def parse_variable_initializer(self): + if self.would_accept('{'): + return self.parse_array_initializer() + else: + return self.parse_expression() + + @parse_debug + def parse_array_initializer(self): + array_initializer = tree.ArrayInitializer(initializers=list()) + + self.accept('{') + + if self.try_accept(','): + self.accept('}') + return array_initializer + + if self.try_accept('}'): + return array_initializer + + while True: + initializer = self.parse_variable_initializer() + array_initializer.initializers.append(initializer) + + if not self.would_accept('}'): + self.accept(',') + + if self.try_accept('}'): + return array_initializer + +# ------------------------------------------------------------------------------ +# -- Blocks and statements -- + + @parse_debug + def parse_block(self): + statements = list() + + self.accept('{') + + while not self.would_accept('}'): + statement = self.parse_block_statement() + statements.append(statement) + self.accept('}') + + return statements + + @parse_debug + def parse_block_statement(self): + if self.would_accept(Identifier, ':'): + # Labeled statement + return self.parse_statement() + + if self.would_accept('synchronized'): + return self.parse_statement() + + token = None + found_annotations = False + i = 0 + + # Look past annoatations and modifiers. If we find a modifier that is not + # 'final' then the statement must be a class or interface declaration + while True: + token = self.tokens.look(i) + + if isinstance(token, Modifier): + if not token.value == 'final': + return self.parse_class_or_interface_declaration() + + elif self.is_annotation(i): + found_annotations = True + + i += 2 + while self.tokens.look(i).value == '.': + i += 2 + + if self.tokens.look(i).value == '(': + parens = 1 + i += 1 + + while parens > 0: + token = self.tokens.look(i) + if token.value == '(': + parens += 1 + elif token.value == ')': + parens -= 1 + i += 1 + continue + + else: + break + + i += 1 + + if token.value in ('class', 'enum', 'interface', '@'): + return self.parse_class_or_interface_declaration() + + if found_annotations or isinstance(token, BasicType): + return self.parse_local_variable_declaration_statement() + + # At this point, if the block statement is a variable definition the next + # token MUST be an identifier, so if it isn't we can conclude the block + # statement is a normal statement + if not isinstance(token, Identifier): + return self.parse_statement() + + # We can't easily determine the statement type. Try parsing as a variable + # declaration first and fall back to a statement + try: + with self.tokens: + return self.parse_local_variable_declaration_statement() + except JavaSyntaxError: + return self.parse_statement() + + @parse_debug + def parse_local_variable_declaration_statement(self): + modifiers, annotations = self.parse_variable_modifiers() + java_type = self.parse_type() + declarators = self.parse_variable_declarators() + self.accept(';') + + var = tree.LocalVariableDeclaration(modifiers=modifiers, + annotations=annotations, + type=java_type, + declarators=declarators) + return var + + @parse_debug + def parse_statement(self): + token = self.tokens.look() + if self.would_accept('{'): + block = self.parse_block() + return tree.BlockStatement(statements=block) + + elif self.try_accept(';'): + return tree.Statement() + + elif self.would_accept(Identifier, ':'): + identifer = self.parse_identifier() + self.accept(':') + + statement = self.parse_statement() + statement.label = identifer + + return statement + + elif self.try_accept('if'): + condition = self.parse_par_expression() + then = self.parse_statement() + else_statement = None + + if self.try_accept('else'): + else_statement = self.parse_statement() + + return tree.IfStatement(condition=condition, + then_statement=then, + else_statement=else_statement) + + elif self.try_accept('assert'): + condition = self.parse_expression() + value = None + + if self.try_accept(':'): + value = self.parse_expression() + + self.accept(';') + + return tree.AssertStatement(condition=condition, + value=value) + + elif self.try_accept('switch'): + switch_expression = self.parse_par_expression() + self.accept('{') + switch_block = self.parse_switch_block_statement_groups() + self.accept('}') + + return tree.SwitchStatement(expression=switch_expression, + cases=switch_block) + + elif self.try_accept('while'): + condition = self.parse_par_expression() + action = self.parse_statement() + + return tree.WhileStatement(condition=condition, + body=action) + + elif self.try_accept('do'): + action = self.parse_statement() + self.accept('while') + condition = self.parse_par_expression() + self.accept(';') + + return tree.DoStatement(condition=condition, + body=action) + + elif self.try_accept('for'): + self.accept('(') + for_control = self.parse_for_control() + self.accept(')') + for_statement = self.parse_statement() + + return tree.ForStatement(control=for_control, + body=for_statement) + + elif self.try_accept('break'): + label = None + + if self.would_accept(Identifier): + label = self.parse_identifier() + + self.accept(';') + + return tree.BreakStatement(goto=label) + + elif self.try_accept('continue'): + label = None + + if self.would_accept(Identifier): + label = self.parse_identifier() + + self.accept(';') + + return tree.ContinueStatement(goto=label) + + elif self.try_accept('return'): + value = None + + if not self.would_accept(';'): + value = self.parse_expression() + + self.accept(';') + + return tree.ReturnStatement(expression=value) + + elif self.try_accept('throw'): + value = self.parse_expression() + self.accept(';') + + return tree.ThrowStatement(expression=value) + + elif self.try_accept('synchronized'): + lock = self.parse_par_expression() + block = self.parse_block() + + return tree.SynchronizedStatement(lock=lock, + block=block) + + elif self.try_accept('try'): + resource_specification = None + block = None + catches = None + finally_block = None + + if self.would_accept('{'): + block = self.parse_block() + + if self.would_accept('catch'): + catches = self.parse_catches() + + if self.try_accept('finally'): + finally_block = self.parse_block() + + if catches == None and finally_block == None: + self.illegal("Expected catch/finally block") + + else: + resource_specification = self.parse_resource_specification() + block = self.parse_block() + + if self.would_accept('catch'): + catches = self.parse_catches() + + if self.try_accept('finally'): + finally_block = self.parse_block() + + return tree.TryStatement(resources=resource_specification, + block=block, + catches=catches, + finally_block=finally_block) + + else: + expression = self.parse_expression() + self.accept(';') + + return tree.StatementExpression(expression=expression) + +# ------------------------------------------------------------------------------ +# -- Try / catch -- + + @parse_debug + def parse_catches(self): + catches = list() + + while True: + catch = self.parse_catch_clause() + catches.append(catch) + + if not self.would_accept('catch'): + break + + return catches + + @parse_debug + def parse_catch_clause(self): + self.accept('catch', '(') + + modifiers, annotations = self.parse_variable_modifiers() + catch_parameter = tree.CatchClauseParameter(types=list()) + + while True: + catch_type = self.parse_qualified_identifier() + catch_parameter.types.append(catch_type) + + if not self.try_accept('|'): + break + catch_parameter.name = self.parse_identifier() + + self.accept(')') + block = self.parse_block() + + return tree.CatchClause(parameter=catch_parameter, + block=block) + + @parse_debug + def parse_resource_specification(self): + resources = list() + + self.accept('(') + + while True: + resource = self.parse_resource() + resources.append(resource) + + if not self.would_accept(')'): + self.accept(';') + + if self.try_accept(')'): + break + + return resources + + @parse_debug + def parse_resource(self): + modifiers, annotations = self.parse_variable_modifiers() + reference_type = self.parse_reference_type() + reference_type.dimensions = self.parse_array_dimension() + name = self.parse_identifier() + reference_type.dimensions += self.parse_array_dimension() + self.accept('=') + value = self.parse_expression() + + return tree.TryResource(modifiers=modifiers, + annotations=annotations, + type=reference_type, + name=name, + value=value) + +# ------------------------------------------------------------------------------ +# -- Switch and for statements --- + + @parse_debug + def parse_switch_block_statement_groups(self): + statement_groups = list() + + while self.tokens.look().value in ('case', 'default'): + statement_group = self.parse_switch_block_statement_group() + statement_groups.append(statement_group) + + return statement_groups + + @parse_debug + def parse_switch_block_statement_group(self): + labels = list() + statements = list() + + while True: + case_type = self.tokens.next().value + case_value = None + + if case_type == 'case': + if self.would_accept(Identifier, ':'): + case_value = self.parse_identifier() + else: + case_value = self.parse_expression() + + labels.append(case_value) + elif not case_type == 'default': + self.illegal("Expected switch case") + + self.accept(':') + + if self.tokens.look().value not in ('case', 'default'): + break + + while self.tokens.look().value not in ('case', 'default', '}'): + statement = self.parse_block_statement() + statements.append(statement) + + return tree.SwitchStatementCase(case=labels, + statements=statements) + + @parse_debug + def parse_for_control(self): + # Try for_var_control and fall back to normal three part for control + + try: + with self.tokens: + return self.parse_for_var_control() + except JavaSyntaxError: + pass + + init = None + if not self.would_accept(';'): + init = self.parse_for_init_or_update() + + self.accept(';') + + condition = None + if not self.would_accept(';'): + condition = self.parse_expression() + + self.accept(';') + + update = None + if not self.would_accept(')'): + update = self.parse_for_init_or_update() + + return tree.ForControl(init=init, + condition=condition, + update=update) + + @parse_debug + def parse_for_var_control(self): + modifiers, annotations = self.parse_variable_modifiers() + var_type = self.parse_type() + var_name = self.parse_identifier() + var_type.dimensions += self.parse_array_dimension() + + var = tree.VariableDeclaration(modifiers=modifiers, + annotations=annotations, + type=var_type) + + rest = self.parse_for_var_control_rest() + + if isinstance(rest, tree.Expression): + var.declarators = [tree.VariableDeclarator(name=var_name)] + return tree.EnhancedForControl(var=var, + iterable=rest) + else: + declarators, condition, update = rest + declarators[0].name = var_name + var.declarators = declarators + return tree.ForControl(init=var, + condition=condition, + update=update) + + @parse_debug + def parse_for_var_control_rest(self): + if self.try_accept(':'): + expression = self.parse_expression() + return expression + + declarators = None + if not self.would_accept(';'): + declarators = self.parse_for_variable_declarator_rest() + else: + declarators = [tree.VariableDeclarator()] + self.accept(';') + + condition = None + if not self.would_accept(';'): + condition = self.parse_expression() + self.accept(';') + + update = None + if not self.would_accept(')'): + update = self.parse_for_init_or_update() + + return (declarators, condition, update) + + @parse_debug + def parse_for_variable_declarator_rest(self): + initializer = None + + if self.try_accept('='): + initializer = self.parse_variable_initializer() + + declarators = [tree.VariableDeclarator(initializer=initializer)] + + while self.try_accept(','): + declarator = self.parse_variable_declarator() + declarators.append(declarator) + + return declarators + + @parse_debug + def parse_for_init_or_update(self): + expressions = list() + + while True: + expression = self.parse_expression() + expressions.append(expression) + + if not self.try_accept(','): + break + + return expressions + +# ------------------------------------------------------------------------------ +# -- Expressions -- + + @parse_debug + def parse_expression(self): + expressionl = self.parse_expressionl() + assignment_type = None + assignment_expression = None + + if self.tokens.look().value in Operator.ASSIGNMENT: + assignment_type = self.tokens.next().value + assignment_expression = self.parse_expression() + return tree.Assignment(expressionl=expressionl, + type=assignment_type, + value=assignment_expression) + else: + return expressionl + + @parse_debug + def parse_expressionl(self): + expression_2 = self.parse_expression_2() + true_expression = None + false_expression = None + + if self.try_accept('?'): + true_expression = self.parse_expression() + self.accept(':') + false_expression = self.parse_expressionl() + + return tree.TernaryExpression(condition=expression_2, + if_true=true_expression, + if_false=false_expression) + if self.would_accept('->'): + body = self.parse_lambda_method_body() + return tree.LambdaExpression(parameters=[expression_2], + body=body) + if self.try_accept('::'): + method_reference, type_arguments = self.parse_method_reference() + return tree.MethodReference( + expression=expression_2, + method=method_reference, + type_arguments=type_arguments) + return expression_2 + + @parse_debug + def parse_expression_2(self): + expression_3 = self.parse_expression_3() + token = self.tokens.look() + if token.value in Operator.INFIX or token.value == 'instanceof': + parts = self.parse_expression_2_rest() + parts.insert(0, expression_3) + return self.build_binary_operation(parts) + + return expression_3 + + @parse_debug + def parse_expression_2_rest(self): + parts = list() + + token = self.tokens.look() + while token.value in Operator.INFIX or token.value == 'instanceof': + if self.try_accept('instanceof'): + comparison_type = self.parse_type() + parts.extend(('instanceof', comparison_type)) + else: + operator = self.parse_infix_operator() + expression = self.parse_expression_3() + parts.extend((operator, expression)) + + token = self.tokens.look() + + return parts + +# ------------------------------------------------------------------------------ +# -- Expression operators -- + + @parse_debug + def parse_expression_3(self): + prefix_operators = list() + while self.tokens.look().value in Operator.PREFIX: + prefix_operators.append(self.tokens.next().value) + + if self.would_accept('('): + try: + with self.tokens: + lambda_exp = self.parse_lambda_expression() + if lambda_exp: + return lambda_exp + except JavaSyntaxError: + pass + try: + with self.tokens: + self.accept('(') + cast_target = self.parse_type() + self.accept(')') + expression = self.parse_expression_3() + + return tree.Cast(type=cast_target, + expression=expression) + except JavaSyntaxError: + pass + + primary = self.parse_primary() + primary.prefix_operators = prefix_operators + primary.selectors = list() + primary.postfix_operators = list() + + token = self.tokens.look() + while token.value in '[.': + selector = self.parse_selector() + primary.selectors.append(selector) + + token = self.tokens.look() + + while token.value in Operator.POSTFIX: + primary.postfix_operators.append(self.tokens.next().value) + token = self.tokens.look() + + return primary + + @parse_debug + def parse_method_reference(self): + type_arguments = list() + if self.would_accept('<'): + type_arguments = self.parse_nonwildcard_type_arguments() + if self.would_accept('new'): + method_reference = tree.MemberReference(member=self.accept('new')) + else: + method_reference = self.parse_expression() + return method_reference, type_arguments + + @parse_debug + def parse_lambda_expression(self): + lambda_expr = None + parameters = None + if self.would_accept('(', Identifier, ','): + self.accept('(') + parameters = [] + while not self.would_accept(')'): + parameters.append(tree.InferredFormalParameter( + name=self.parse_identifier())) + self.try_accept(',') + self.accept(')') + else: + parameters = self.parse_formal_parameters() + body = self.parse_lambda_method_body() + return tree.LambdaExpression(parameters=parameters, + body=body) + + @parse_debug + def parse_lambda_method_body(self): + if self.accept('->'): + if self.would_accept('{'): + return self.parse_block() + else: + return self.parse_expression() + + @parse_debug + def parse_infix_operator(self): + operator = self.accept(Operator) + + if not operator in Operator.INFIX: + self.illegal("Expected infix operator") + + if operator == '>' and self.try_accept('>'): + operator = '>>' + + if self.try_accept('>'): + operator = '>>>' + + return operator + +# ------------------------------------------------------------------------------ +# -- Primary expressions -- + + @parse_debug + def parse_primary(self): + token = self.tokens.look() + + if isinstance(token, Literal): + return self.parse_literal() + + elif token.value == '(': + return self.parse_par_expression() + + elif self.try_accept('this'): + arguments = None + + if self.would_accept('('): + arguments = self.parse_arguments() + return tree.ExplicitConstructorInvocation(arguments=arguments) + + return tree.This() + elif self.would_accept('super', '::'): + self.accept('super') + return token + elif self.try_accept('super'): + super_suffix = self.parse_super_suffix() + return super_suffix + + elif self.try_accept('new'): + return self.parse_creator() + + elif token.value == '<': + type_arguments = self.parse_nonwildcard_type_arguments() + + if self.try_accept('this'): + arguments = self.parse_arguments() + return tree.ExplicitConstructorInvocation(type_arguments=type_arguments, + arguments=arguments) + else: + invocation = self.parse_explicit_generic_invocation_suffix() + invocation.type_arguments = type_arguments + + return invocation + + elif isinstance(token, Identifier): + qualified_identifier = [self.parse_identifier()] + + while self.would_accept('.', Identifier): + self.accept('.') + identifier = self.parse_identifier() + qualified_identifier.append(identifier) + + identifier_suffix = self.parse_identifier_suffix() + + if isinstance(identifier_suffix, (tree.MemberReference, tree.MethodInvocation)): + # Take the last identifer as the member and leave the rest for the qualifier + identifier_suffix.member = qualified_identifier.pop() + + elif isinstance(identifier_suffix, tree.ClassReference): + identifier_suffix.type = tree.ReferenceType(name=qualified_identifier.pop()) + + identifier_suffix.qualifier = '.'.join(qualified_identifier) + + return identifier_suffix + + elif isinstance(token, BasicType): + base_type = self.parse_basic_type() + base_type.dimensions = self.parse_array_dimension() + self.accept('.', 'class') + + return tree.ClassReference(type=base_type) + + elif self.try_accept('void'): + self.accept('.', 'class') + return tree.VoidClassReference() + + self.illegal("Expected expression") + + @parse_debug + def parse_literal(self): + literal = self.accept(Literal) + return tree.Literal(value=literal) + + @parse_debug + def parse_par_expression(self): + self.accept('(') + expression = self.parse_expression() + self.accept(')') + + return expression + + @parse_debug + def parse_arguments(self): + expressions = list() + + self.accept('(') + + if self.try_accept(')'): + return expressions + + while True: + expression = self.parse_expression() + expressions.append(expression) + + if not self.try_accept(','): + break + + self.accept(')') + + return expressions + + @parse_debug + def parse_super_suffix(self): + identifier = None + type_arguments = None + arguments = None + + if self.try_accept('.'): + if self.would_accept('<'): + type_arguments = self.parse_nonwildcard_type_arguments() + + identifier = self.parse_identifier() + + if self.would_accept('('): + arguments = self.parse_arguments() + else: + arguments = self.parse_arguments() + + if identifier and arguments is not None: + return tree.SuperMethodInvocation(member=identifier, + arguments=arguments, + type_arguments=type_arguments) + elif arguments is not None: + return tree.SuperConstructorInvocation(arguments=arguments) + else: + return tree.SuperMemberReference(member=identifier) + + @parse_debug + def parse_explicit_generic_invocation_suffix(self): + identifier = None + arguments = None + if self.try_accept('super'): + return self.parse_super_suffix() + else: + identifier = self.parse_identifier() + arguments = self.parse_arguments() + return tree.MethodInvocation(member=identifier, + arguments=arguments) + +# ------------------------------------------------------------------------------ +# -- Creators -- + + @parse_debug + def parse_creator(self): + constructor_type_arguments = None + + if self.would_accept(BasicType): + created_name = self.parse_basic_type() + rest = self.parse_array_creator_rest() + rest.type = created_name + return rest + + if self.would_accept('<'): + constructor_type_arguments = self.parse_nonwildcard_type_arguments() + + created_name = self.parse_created_name() + + if self.would_accept('['): + if constructor_type_arguments: + self.illegal("Array creator not allowed with generic constructor type arguments") + + rest = self.parse_array_creator_rest() + rest.type = created_name + return rest + else: + arguments, body = self.parse_class_creator_rest() + return tree.ClassCreator(constructor_type_arguments=constructor_type_arguments, + type=created_name, + arguments=arguments, + body=body) + + @parse_debug + def parse_created_name(self): + created_name = tree.ReferenceType() + tail = created_name + + while True: + tail.name = self.parse_identifier() + + if self.would_accept('<'): + tail.arguments = self.parse_type_arguments_or_diamond() + + if self.try_accept('.'): + tail.sub_type = tree.ReferenceType() + tail = tail.sub_type + else: + break + + return created_name + + @parse_debug + def parse_class_creator_rest(self): + arguments = self.parse_arguments() + class_body = None + + if self.would_accept('{'): + class_body = self.parse_class_body() + + return (arguments, class_body) + + @parse_debug + def parse_array_creator_rest(self): + if self.would_accept('[', ']'): + array_dimension = self.parse_array_dimension() + array_initializer = self.parse_array_initializer() + + return tree.ArrayCreator(dimensions=array_dimension, + initializer=array_initializer) + + else: + array_dimensions = list() + + while self.would_accept('[') and not self.would_accept('[', ']'): + self.accept('[') + expression = self.parse_expression() + array_dimensions.append(expression) + self.accept(']') + + array_dimensions += self.parse_array_dimension() + return tree.ArrayCreator(dimensions=array_dimensions) + + @parse_debug + def parse_identifier_suffix(self): + if self.try_accept('[', ']'): + array_dimension = [None] + self.parse_array_dimension() + self.accept('.', 'class') + return tree.ClassReference(type=tree.Type(dimensions=array_dimension)) + + elif self.would_accept('('): + arguments = self.parse_arguments() + return tree.MethodInvocation(arguments=arguments) + + elif self.try_accept('.', 'class'): + return tree.ClassReference() + + elif self.try_accept('.', 'this'): + return tree.This() + + elif self.would_accept('.', '<'): + next(self.tokens) + return self.parse_explicit_generic_invocation() + + elif self.try_accept('.', 'new'): + type_arguments = None + + if self.would_accept('<'): + type_arguments = self.parse_nonwildcard_type_arguments() + + inner_creator = self.parse_inner_creator() + inner_creator.constructor_type_arguments = type_arguments + + return inner_creator + + elif self.would_accept('.', 'super', '('): + self.accept('.', 'super') + arguments = self.parse_arguments() + return tree.SuperConstructorInvocation(arguments=arguments) + + else: + return tree.MemberReference() + + @parse_debug + def parse_explicit_generic_invocation(self): + type_arguments = self.parse_nonwildcard_type_arguments() + + invocation = self.parse_explicit_generic_invocation_suffix() + invocation.type_arguments = type_arguments + + return invocation + + @parse_debug + def parse_inner_creator(self): + identifier = self.parse_identifier() + type_arguments = None + + if self.would_accept('<'): + type_arguments = self.parse_nonwildcard_type_arguments_or_diamond() + + java_type = tree.ReferenceType(name=identifier, + arguments=type_arguments) + + arguments, class_body = self.parse_class_creator_rest() + + return tree.InnerClassCreator(type=java_type, + arguments=arguments, + body=class_body) + + @parse_debug + def parse_selector(self): + if self.try_accept('['): + expression = self.parse_expression() + self.accept(']') + return tree.ArraySelector(index=expression) + + elif self.try_accept('.'): + + token = self.tokens.look() + if isinstance(token, Identifier): + identifier = self.tokens.next().value + arguments = None + + if self.would_accept('('): + arguments = self.parse_arguments() + + return tree.MethodInvocation(member=identifier, + arguments=arguments) + else: + return tree.MemberReference(member=identifier) + elif self.would_accept('super', '::'): + self.accept('super') + return token + elif self.would_accept('<'): + return self.parse_explicit_generic_invocation() + elif self.try_accept('this'): + return tree.This() + elif self.try_accept('super'): + return self.parse_super_suffix() + elif self.try_accept('new'): + type_arguments = None + + if self.would_accept('<'): + type_arguments = self.parse_nonwildcard_type_arguments() + + inner_creator = self.parse_inner_creator() + inner_creator.constructor_type_arguments = type_arguments + + return inner_creator + + self.illegal("Expected selector") + +# ------------------------------------------------------------------------------ +# -- Enum and annotation body -- + + @parse_debug + def parse_enum_body(self): + constants = list() + body_declarations = list() + + self.accept('{') + + if not self.try_accept(','): + while not (self.would_accept(';') or self.would_accept('}')): + constant = self.parse_enum_constant() + constants.append(constant) + + if not self.try_accept(','): + break + + if self.try_accept(';'): + while not self.would_accept('}'): + declaration = self.parse_class_body_declaration() + + if declaration: + body_declarations.append(declaration) + + self.accept('}') + + return tree.EnumBody(constants=constants, + declarations=body_declarations) + + @parse_debug + def parse_enum_constant(self): + annotations = list() + javadoc = None + constant_name = None + arguments = None + body = None + + next_token = self.tokens.look() + if next_token: + javadoc = next_token.javadoc + + if self.would_accept(Annotation): + annotations = self.parse_annotations() + + constant_name = self.parse_identifier() + + if self.would_accept('('): + arguments = self.parse_arguments() + + if self.would_accept('{'): + body = self.parse_class_body() + + return tree.EnumConstantDeclaration(annotations=annotations, + name=constant_name, + arguments=arguments, + body=body, + documentation=javadoc) + + @parse_debug + def parse_annotation_type_body(self): + declarations = None + + self.accept('{') + declarations = self.parse_annotation_type_element_declarations() + self.accept('}') + + return declarations + + @parse_debug + def parse_annotation_type_element_declarations(self): + declarations = list() + + while not self.would_accept('}'): + declaration = self.parse_annotation_type_element_declaration() + declarations.append(declaration) + + return declarations + + @parse_debug + def parse_annotation_type_element_declaration(self): + modifiers, annotations, javadoc = self.parse_modifiers() + declaration = None + + if self.would_accept('class'): + declaration = self.parse_normal_class_declaration() + elif self.would_accept('interface'): + declaration = self.parse_normal_interface_declaration() + elif self.would_accept('enum'): + declaration = self.parse_enum_declaration() + elif self.is_annotation_declaration(): + declaration = self.parse_annotation_type_declaration() + else: + attribute_type = self.parse_type() + attribute_name = self.parse_identifier() + declaration = self.parse_annotation_method_or_constant_rest() + self.accept(';') + + if isinstance(declaration, tree.AnnotationMethod): + declaration.name = attribute_name + declaration.return_type = attribute_type + else: + declaration.declarators[0].name = attribute_name + declaration.type = attribute_type + + declaration.modifiers = modifiers + declaration.annotations = annotations + declaration.documentation = javadoc + + return declaration + + @parse_debug + def parse_annotation_method_or_constant_rest(self): + if self.try_accept('('): + self.accept(')') + + array_dimension = self.parse_array_dimension() + default = None + + if self.try_accept('default'): + default = self.parse_element_value() + + return tree.AnnotationMethod(dimensions=array_dimension, + default=default) + else: + return self.parse_constant_declarators_rest() + +def parse(tokens, debug=False): + parser = Parser(tokens) + parser.set_debug(debug) + return parser.parse() diff --git a/baseline_tokenization/javalang/test/__init__.py b/baseline_tokenization/javalang/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/baseline_tokenization/javalang/test/source/package-info/AnnotationJavadoc.java b/baseline_tokenization/javalang/test/source/package-info/AnnotationJavadoc.java new file mode 100644 index 0000000..2bca1cb --- /dev/null +++ b/baseline_tokenization/javalang/test/source/package-info/AnnotationJavadoc.java @@ -0,0 +1,5 @@ +@Package +/** + Test that includes java doc first but no annotation +*/ +package org.javalang.test; \ No newline at end of file diff --git a/baseline_tokenization/javalang/test/source/package-info/AnnotationOnly.java b/baseline_tokenization/javalang/test/source/package-info/AnnotationOnly.java new file mode 100644 index 0000000..802b9fe --- /dev/null +++ b/baseline_tokenization/javalang/test/source/package-info/AnnotationOnly.java @@ -0,0 +1,2 @@ +@Package +package org.javalang.test; \ No newline at end of file diff --git a/baseline_tokenization/javalang/test/source/package-info/JavadocAnnotation.java b/baseline_tokenization/javalang/test/source/package-info/JavadocAnnotation.java new file mode 100644 index 0000000..dd81ffa --- /dev/null +++ b/baseline_tokenization/javalang/test/source/package-info/JavadocAnnotation.java @@ -0,0 +1,5 @@ +/** + Test that includes java doc first but no annotation +*/ +@Package +package org.javalang.test; \ No newline at end of file diff --git a/baseline_tokenization/javalang/test/source/package-info/JavadocOnly.java b/baseline_tokenization/javalang/test/source/package-info/JavadocOnly.java new file mode 100644 index 0000000..34b29b6 --- /dev/null +++ b/baseline_tokenization/javalang/test/source/package-info/JavadocOnly.java @@ -0,0 +1,4 @@ +/** + Test that includes java doc first but no annotation +*/ +package org.javalang.test; \ No newline at end of file diff --git a/baseline_tokenization/javalang/test/source/package-info/NoAnnotationNoJavadoc.java b/baseline_tokenization/javalang/test/source/package-info/NoAnnotationNoJavadoc.java new file mode 100644 index 0000000..9a86220 --- /dev/null +++ b/baseline_tokenization/javalang/test/source/package-info/NoAnnotationNoJavadoc.java @@ -0,0 +1 @@ +package org.javalang.test; \ No newline at end of file diff --git a/baseline_tokenization/javalang/test/test_java_8_syntax.py b/baseline_tokenization/javalang/test/test_java_8_syntax.py new file mode 100644 index 0000000..0a8c8fd --- /dev/null +++ b/baseline_tokenization/javalang/test/test_java_8_syntax.py @@ -0,0 +1,241 @@ +import unittest + +from pkg_resources import resource_string +from .. import parse, parser, tree + + +def setup_java_class(content_to_add): + """ returns an example java class with the + given content_to_add contained within a method. + """ + template = """ +public class Lambda { + + public static void main(String args[]) { + %s + } +} + """ + return template % content_to_add + + +def filter_type_in_method(clazz, the_type, method_name): + """ yields the result of filtering the given class for the given + type inside the given method identified by its name. + """ + for path, node in clazz.filter(the_type): + for p in reversed(path): + if isinstance(p, tree.MethodDeclaration): + if p.name == method_name: + yield path, node + + +class LambdaSupportTest(unittest.TestCase): + + """ Contains tests for java 8 lambda syntax. """ + + def assert_contains_lambda_expression_in_m( + self, clazz, method_name='main'): + """ asserts that the given tree contains a method with the supplied + method name containing a lambda expression. + """ + matches = list(filter_type_in_method( + clazz, tree.LambdaExpression, method_name)) + if not matches: + self.fail('No matching lambda expression found.') + return matches + + def test_lambda_support_no_parameters_no_body(self): + """ tests support for lambda with no parameters and no body. """ + self.assert_contains_lambda_expression_in_m( + parse.parse(setup_java_class("() -> {};"))) + + def test_lambda_support_no_parameters_expression_body(self): + """ tests support for lambda with no parameters and an + expression body. + """ + test_classes = [ + setup_java_class("() -> 3;"), + setup_java_class("() -> null;"), + setup_java_class("() -> { return 21; };"), + setup_java_class("() -> { System.exit(1); };"), + ] + for test_class in test_classes: + clazz = parse.parse(test_class) + self.assert_contains_lambda_expression_in_m(clazz) + + def test_lambda_support_no_parameters_complex_expression(self): + """ tests support for lambda with no parameters and a + complex expression body. + """ + code = """ + () -> { + if (true) return 21; + else + { + int result = 21; + return result / 2; + } + };""" + self.assert_contains_lambda_expression_in_m( + parse.parse(setup_java_class(code))) + + def test_parameter_no_type_expression_body(self): + """ tests support for lambda with parameters with inferred types. """ + test_classes = [ + setup_java_class("(bar) -> bar + 1;"), + setup_java_class("bar -> bar + 1;"), + setup_java_class("x -> x.length();"), + setup_java_class("y -> { y.boom(); };"), + ] + for test_class in test_classes: + clazz = parse.parse(test_class) + self.assert_contains_lambda_expression_in_m(clazz) + + def test_parameter_with_type_expression_body(self): + """ tests support for lambda with parameters with formal types. """ + test_classes = [ + setup_java_class("(int foo) -> { return foo + 2; };"), + setup_java_class("(String s) -> s.length();"), + setup_java_class("(int foo) -> foo + 1;"), + setup_java_class("(Thread th) -> { th.start(); };"), + setup_java_class("(String foo, String bar) -> " + "foo + bar;"), + ] + for test_class in test_classes: + clazz = parse.parse(test_class) + self.assert_contains_lambda_expression_in_m(clazz) + + def test_parameters_with_no_type_expression_body(self): + """ tests support for multiple lambda parameters + that are specified without their types. + """ + self.assert_contains_lambda_expression_in_m( + parse.parse(setup_java_class("(x, y) -> x + y;"))) + + def test_parameters_with_mixed_inferred_and_declared_types(self): + """ this tests that lambda type specification mixing is considered + invalid as per the specifications. + """ + with self.assertRaises(parser.JavaSyntaxError): + parse.parse(setup_java_class("(x, int y) -> x+y;")) + + def test_parameters_inferred_types_with_modifiers(self): + """ this tests that lambda inferred type parameters with modifiers are + considered invalid as per the specifications. + """ + with self.assertRaises(parser.JavaSyntaxError): + parse.parse(setup_java_class("(x, final y) -> x+y;")) + + def test_invalid_parameters_are_invalid(self): + """ this tests that invalid lambda parameters are are + considered invalid as per the specifications. + """ + with self.assertRaises(parser.JavaSyntaxError): + parse.parse(setup_java_class("(a b c) -> {};")) + + def test_cast_works(self): + """ this tests that a cast expression works as expected. """ + parse.parse(setup_java_class("String x = (String) A.x() ;")) + + +class MethodReferenceSyntaxTest(unittest.TestCase): + + """ Contains tests for java 8 method reference syntax. """ + + def assert_contains_method_reference_expression_in_m( + self, clazz, method_name='main'): + """ asserts that the given class contains a method with the supplied + method name containing a method reference. + """ + matches = list(filter_type_in_method( + clazz, tree.MethodReference, method_name)) + if not matches: + self.fail('No matching method reference found.') + return matches + + def test_method_reference(self): + """ tests that method references are supported. """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("String::length;"))) + + def test_method_reference_to_the_new_method(self): + """ test support for method references to 'new'. """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("String::new;"))) + + def test_method_reference_to_the_new_method_with_explict_type(self): + """ test support for method references to 'new' with an + explicit type. + """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("String:: new;"))) + + def test_method_reference_from_super(self): + """ test support for method references from 'super'. """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("super::toString;"))) + + def test_method_reference_from_super_with_identifier(self): + """ test support for method references from Identifier.super. """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("String.super::toString;"))) + + @unittest.expectedFailure + def test_method_reference_explicit_type_arguments_for_generic_type(self): + """ currently there is no support for method references + for an explicit type. + """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("List::size;"))) + + def test_method_reference_explicit_type_arguments(self): + """ test support for method references with an explicit type. + """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("Arrays:: sort;"))) + + @unittest.expectedFailure + def test_method_reference_from_array_type(self): + """ currently there is no support for method references + from a primary type. + """ + self.assert_contains_method_reference_expression_in_m( + parse.parse(setup_java_class("int[]::new;"))) + + +class InterfaceSupportTest(unittest.TestCase): + + """ Contains tests for java 8 interface extensions. """ + + def test_interface_support_static_methods(self): + parse.parse(""" +interface Foo { + void foo(); + + static Foo create() { + return new Foo() { + @Override + void foo() { + System.out.println("foo"); + } + }; + } +} + """) + + def test_interface_support_default_methods(self): + parse.parse(""" +interface Foo { + default void foo() { + System.out.println("foo"); + } +} + """) + + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/baseline_tokenization/javalang/test/test_javadoc.py b/baseline_tokenization/javalang/test/test_javadoc.py new file mode 100644 index 0000000..68e8aec --- /dev/null +++ b/baseline_tokenization/javalang/test/test_javadoc.py @@ -0,0 +1,14 @@ +import unittest + +from .. import javadoc + + +class TestJavadoc(unittest.TestCase): + def test_empty_comment(self): + javadoc.parse('/** */') + javadoc.parse('/***/') + javadoc.parse('/**\n *\n */') + javadoc.parse('/**\n *\n *\n */') + +if __name__ == "__main__": + unittest.main() diff --git a/baseline_tokenization/javalang/test/test_package_declaration.py b/baseline_tokenization/javalang/test/test_package_declaration.py new file mode 100644 index 0000000..880ac67 --- /dev/null +++ b/baseline_tokenization/javalang/test/test_package_declaration.py @@ -0,0 +1,61 @@ +import unittest + +from pkg_resources import resource_string +from .. import parse + + +# From my reading of the spec (http://docs.oracle.com/javase/specs/jls/se7/html/jls-7.html) the +# allowed order is javadoc, optional annotation, package declaration +class PackageInfo(unittest.TestCase): + def testPackageDeclarationOnly(self): + source_file = "source/package-info/NoAnnotationNoJavadoc.java" + ast = self.get_ast(source_file) + + self.failUnless(ast.package.name == "org.javalang.test") + self.failIf(ast.package.annotations) + self.failIf(ast.package.documentation) + + def testAnnotationOnly(self): + source_file = "source/package-info/AnnotationOnly.java" + ast = self.get_ast(source_file) + + self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.annotations) + self.failIf(ast.package.documentation) + + def testJavadocOnly(self): + source_file = "source/package-info/JavadocOnly.java" + ast = self.get_ast(source_file) + + self.failUnless(ast.package.name == "org.javalang.test") + self.failIf(ast.package.annotations) + self.failUnless(ast.package.documentation) + + def testAnnotationThenJavadoc(self): + source_file = "source/package-info/AnnotationJavadoc.java" + ast = self.get_ast(source_file) + + self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.annotations) + self.failIf(ast.package.documentation) + + def testJavadocThenAnnotation(self): + source_file = "source/package-info/JavadocAnnotation.java" + ast = self.get_ast(source_file) + + self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.annotations) + self.failUnless(ast.package.documentation) + + def get_ast(self, filename): + source = resource_string(__name__, filename) + ast = parse.parse(source) + + return ast + + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/baseline_tokenization/javalang/test/test_util.py b/baseline_tokenization/javalang/test/test_util.py new file mode 100644 index 0000000..08e326e --- /dev/null +++ b/baseline_tokenization/javalang/test/test_util.py @@ -0,0 +1,69 @@ +import unittest + +from ..util import LookAheadIterator + + +class TestLookAheadIterator(unittest.TestCase): + def test_usage(self): + i = LookAheadIterator(list(range(0, 10000))) + + self.assertEqual(next(i), 0) + self.assertEqual(next(i), 1) + self.assertEqual(next(i), 2) + + self.assertEqual(i.last(), 2) + + self.assertEqual(i.look(), 3) + self.assertEqual(i.last(), 3) + + self.assertEqual(i.look(1), 4) + self.assertEqual(i.look(2), 5) + self.assertEqual(i.look(3), 6) + self.assertEqual(i.look(4), 7) + + self.assertEqual(i.last(), 7) + + i.push_marker() + self.assertEqual(next(i), 3) + self.assertEqual(next(i), 4) + self.assertEqual(next(i), 5) + i.pop_marker(True) # reset + + self.assertEqual(i.look(), 3) + self.assertEqual(next(i), 3) + + i.push_marker() #1 + self.assertEqual(next(i), 4) + self.assertEqual(next(i), 5) + i.push_marker() #2 + self.assertEqual(next(i), 6) + self.assertEqual(next(i), 7) + i.push_marker() #3 + self.assertEqual(next(i), 8) + self.assertEqual(next(i), 9) + i.pop_marker(False) #3 + self.assertEqual(next(i), 10) + i.pop_marker(True) #2 + self.assertEqual(next(i), 6) + self.assertEqual(next(i), 7) + self.assertEqual(next(i), 8) + i.pop_marker(False) #1 + self.assertEqual(next(i), 9) + + try: + with i: + self.assertEqual(next(i), 10) + self.assertEqual(next(i), 11) + raise Exception() + except: + self.assertEqual(next(i), 10) + self.assertEqual(next(i), 11) + + with i: + self.assertEqual(next(i), 12) + self.assertEqual(next(i), 13) + self.assertEqual(next(i), 14) + + +if __name__=="__main__": + unittest.main() diff --git a/baseline_tokenization/javalang/tokenizer.py b/baseline_tokenization/javalang/tokenizer.py new file mode 100644 index 0000000..d5f6ab4 --- /dev/null +++ b/baseline_tokenization/javalang/tokenizer.py @@ -0,0 +1,643 @@ +import re +import unicodedata + +import six + + +class LexerError(Exception): + pass + +class JavaToken(object): + def __init__(self, value, position=None, javadoc=None): + self.value = value + self.position = position + self.javadoc = javadoc + + def __repr__(self): + if self.position: + return '%s "%s" line %d, position %d' % ( + self.__class__.__name__, self.value, self.position[0], self.position[1] + ) + else: + return '%s "%s"' % (self.__class__.__name__, self.value) + + def __str__(self): + return repr(self) + + def __eq__(self, other): + raise Exception("Direct comparison not allowed") + +class EndOfInput(JavaToken): + pass + +class Keyword(JavaToken): + VALUES = set(['abstract', 'assert', 'boolean', 'break', 'byte', 'case', + 'catch', 'char', 'class', 'const', 'continue', 'default', + 'do', 'double', 'else', 'enum', 'extends', 'final', + 'finally', 'float', 'for', 'goto', 'if', 'implements', + 'import', 'instanceof', 'int', 'interface', 'long', 'native', + 'new', 'package', 'private', 'protected', 'public', 'return', + 'short', 'static', 'strictfp', 'super', 'switch', + 'synchronized', 'this', 'throw', 'throws', 'transient', 'try', + 'void', 'volatile', 'while']) + + +class Modifier(Keyword): + VALUES = set(['abstract', 'default', 'final', 'native', 'private', + 'protected', 'public', 'static', 'strictfp', 'synchronized', + 'transient', 'volatile']) + +class BasicType(Keyword): + VALUES = set(['boolean', 'byte', 'char', 'double', + 'float', 'int', 'long', 'short']) + +class Literal(JavaToken): + pass + +class Integer(Literal): + pass + +class DecimalInteger(Literal): + pass + +class OctalInteger(Integer): + pass + +class BinaryInteger(Integer): + pass + +class HexInteger(Integer): + pass + +class FloatingPoint(Literal): + pass + +class DecimalFloatingPoint(FloatingPoint): + pass + +class HexFloatingPoint(FloatingPoint): + pass + +class Boolean(Literal): + VALUES = set(["true", "false"]) + +class Character(Literal): + pass + +class String(Literal): + pass + +class Null(Literal): + pass + +class Separator(JavaToken): + VALUES = set(['(', ')', '{', '}', '[', ']', ';', ',', '.']) + +class Operator(JavaToken): + MAX_LEN = 4 + VALUES = set(['>>>=', '>>=', '<<=', '%=', '^=', '|=', '&=', '/=', + '*=', '-=', '+=', '<<', '--', '++', '||', '&&', '!=', + '>=', '<=', '==', '%', '^', '|', '&', '/', '*', '-', + '+', ':', '?', '~', '!', '<', '>', '=', '...', '->', '::']) + + # '>>>' and '>>' are excluded so that >> becomes two tokens and >>> becomes + # three. This is done because we can not distinguish the operators >> and + # >>> from the closing of multipel type parameter/argument lists when + # lexing. The job of potentially recombining these symbols is left to the + # parser + + INFIX = set(['||', '&&', '|', '^', '&', '==', '!=', '<', '>', '<=', '>=', + '<<', '>>', '>>>', '+', '-', '*', '/', '%']) + + PREFIX = set(['++', '--', '!', '~', '+', '-']) + + POSTFIX = set(['++', '--']) + + ASSIGNMENT = set(['=', '+=', '-=', '*=', '/=', '&=', '|=', '^=', '%=', + '<<=', '>>=', '>>>=']) + + LAMBDA = set(['->']) + + METHOD_REFERENCE = set(['::',]) + + def is_infix(self): + return self.value in self.INFIX + + def is_prefix(self): + return self.value in self.PREFIX + + def is_postfix(self): + return self.value in self.POSTFIX + + def is_assignment(self): + return self.value in self.ASSIGNMENT + + +class Annotation(JavaToken): + pass + +class Identifier(JavaToken): + pass + + +class JavaTokenizer(object): + + IDENT_START_CATEGORIES = set(['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Pc', 'Sc']) + + IDENT_PART_CATEGORIES = set(['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Mc', 'Mn', 'Nd', 'Nl', 'Pc', 'Sc']) + + def __init__(self, data): + self.data = data + + self.current_line = 1 + self.start_of_line = 0 + + self.operators = [set() for i in range(0, Operator.MAX_LEN)] + + for v in Operator.VALUES: + self.operators[len(v) - 1].add(v) + + self.whitespace_consumer = re.compile(r'[^\s]') + + self.javadoc = None + + + def reset(self): + self.i = 0 + self.j = 0 + + def consume_whitespace(self): + match = self.whitespace_consumer.search(self.data, self.i + 1) + + if not match: + self.i = self.length + return + + i = match.start() + + start_of_line = self.data.rfind('\n', self.i, i) + + if start_of_line != -1: + self.start_of_line = start_of_line + self.current_line += self.data.count('\n', self.i, i) + + self.i = i + + def read_string(self): + delim = self.data[self.i] + + state = 0 + j = self.i + 1 + length = self.length + + while True: + if j >= length: + self.error('Unterminated character/string literal') + + if state == 0: + if self.data[j] == '\\': + state = 1 + elif self.data[j] == delim: + break + + elif state == 1: + if self.data[j] in 'btnfru"\'\\': + state = 0 + elif self.data[j] in '0123': + state = 2 + elif self.data[j] in '01234567': + state = 3 + else: + self.error('Illegal escape character', self.data[j]) + + elif state == 2: + # Possibly long octal + if self.data[j] in '01234567': + state = 3 + elif self.data[j] == '\\': + state = 1 + elif self.data[j] == delim: + break + + elif state == 3: + state = 0 + + if self.data[j] == '\\': + state = 1 + elif self.data[j] == delim: + break + + j += 1 + + self.j = j + 1 + + def try_operator(self): + for l in range(min(self.length - self.i, Operator.MAX_LEN), 0, -1): + if self.data[self.i:self.i + l] in self.operators[l - 1]: + self.j = self.i + l + return True + return False + + def read_comment(self): + if self.data[self.i + 1] == '/': + i = self.data.find('\n', self.i + 2) + + if i == -1: + self.i = self.length + return + + i += 1 + + self.start_of_line = i + self.current_line += 1 + self.i = i + + else: + i = self.data.find('*/', self.i + 2) + + if i == -1: + self.i = self.length + return + + i += 2 + + self.start_of_line = i + self.current_line += self.data.count('\n', self.i, i) + self.i = i + + def try_javadoc_comment(self): + if self.i + 2 >= self.length or self.data[self.i + 2] != '*': + return False + + j = self.data.find('*/', self.i + 2) + + if j == -1: + self.j = self.length + return False + + j += 2 + + self.start_of_line = j + self.current_line += self.data.count('\n', self.i, j) + self.j = j + + return True + + def read_decimal_float_or_integer(self): + orig_i = self.i + self.j = self.i + + self.read_decimal_integer() + + if self.data[self.j] not in '.eEfFdD': + return DecimalInteger + + if self.data[self.j] == '.': + self.i = self.j + 1 + self.read_decimal_integer() + + if self.data[self.j] in 'eE': + self.j = self.j + 1 + + if self.data[self.j] in '-+': + self.j = self.j + 1 + + self.i = self.j + self.read_decimal_integer() + + if self.data[self.j] in 'fFdD': + self.j = self.j + 1 + + self.i = orig_i + return DecimalFloatingPoint + + def read_hex_integer_or_float(self): + orig_i = self.i + self.j = self.i + 2 + + self.read_hex_integer() + + if self.data[self.j] not in '.pP': + return HexInteger + + if self.data[self.j] == '.': + self.j = self.j + 1 + self.read_digits('0123456789abcdefABCDEF') + + if self.data[self.j] in 'pP': + self.j = self.j + 1 + else: + self.error('Invalid hex float literal') + + if self.data[self.j] in '-+': + self.j = self.j + 1 + + self.i = self.j + self.read_decimal_integer() + + if self.data[self.j] in 'fFdD': + self.j = self.j + 1 + + self.i = orig_i + return HexFloatingPoint + + def read_digits(self, digits): + tmp_i = 0 + c = None + + while True: + c = self.data[self.j + tmp_i] + + if c in digits: + self.j += 1 + tmp_i + tmp_i = 0 + elif c == '_': + tmp_i += 1 + else: + break + + if c in 'lL': + self.j += 1 + + def read_decimal_integer(self): + self.j = self.i + self.read_digits('0123456789') + + def read_hex_integer(self): + self.j = self.i + 2 + self.read_digits('0123456789abcdefABCDEF') + + def read_bin_integer(self): + self.j = self.i + 2 + self.read_digits('01') + + def read_octal_integer(self): + self.j = self.i + 1 + self.read_digits('01234567') + + def read_integer_or_float(self, c, c_next): + if c == '0' and c_next in 'xX': + return self.read_hex_integer_or_float() + elif c == '0' and c_next in 'bB': + self.read_bin_integer() + return BinaryInteger + elif c == '0' and c_next in '01234567': + self.read_octal_integer() + return OctalInteger + else: + return self.read_decimal_float_or_integer() + + def try_separator(self): + if self.data[self.i] in Separator.VALUES: + self.j = self.i + 1 + return True + return False + + def decode_data(self): + # Encodings to try in order + codecs = ['utf_8', 'iso-8859-1'] + + # If data is already unicode don't try to redecode + if isinstance(self.data, six.text_type): + return self.data + + for codec in codecs: + try: + data = self.data.decode(codec) + return data + except UnicodeDecodeError: + pass + + self.error('Could not decode input data') + + def is_java_identifier_start(self, c): + return unicodedata.category(c) in self.IDENT_START_CATEGORIES + + def read_identifier(self): + self.j = self.i + 1 + + while unicodedata.category(self.data[self.j]) in self.IDENT_PART_CATEGORIES: + self.j += 1 + + ident = self.data[self.i:self.j] + if ident in Keyword.VALUES: + token_type = Keyword + + if ident in BasicType.VALUES: + token_type = BasicType + elif ident in Modifier.VALUES: + token_type = Modifier + + elif ident in Boolean.VALUES: + token_type = Boolean + elif ident == 'null': + token_type = Null + else: + token_type = Identifier + + return token_type + + def pre_tokenize(self): + new_data = list() + data = self.decode_data() + + i = 0 + j = 0 + length = len(data) + + NONE = 0 + ELIGIBLE = 1 + MARKER_FOUND = 2 + + state = NONE + + while j < length: + if state == NONE: + j = data.find('\\', j) + + if j == -1: + j = length + break + + state = ELIGIBLE + + elif state == ELIGIBLE: + c = data[j] + + if c == 'u': + state = MARKER_FOUND + new_data.append(data[i:j - 1]) + else: + state = NONE + + elif state == MARKER_FOUND: + c = data[j] + + if c != 'u': + try: + escape_code = int(data[j:j+4], 16) + except ValueError: + self.error('Invalid unicode escape', data[j:j+4]) + + new_data.append(six.unichr(escape_code)) + + i = j + 4 + j = i + + state = NONE + + continue + + j = j + 1 + + new_data.append(data[i:]) + + self.data = ''.join(new_data) + self.length = len(self.data) + + def tokenize(self): + self.reset() + + # Convert unicode escapes + self.pre_tokenize() + + while self.i < self.length: + token_type = None + + c = self.data[self.i] + c_next = None + startswith = c + + if self.i + 1 < self.length: + c_next = self.data[self.i + 1] + startswith = c + c_next + + if c.isspace(): + self.consume_whitespace() + continue + + elif startswith in ("//", "/*"): + if startswith == "/*" and self.try_javadoc_comment(): + self.javadoc = self.data[self.i:self.j] + self.i = self.j + else: + self.read_comment() + continue + + elif startswith == '..' and self.try_operator(): + # Ensure we don't mistake a '...' operator as a sequence of + # three '.' separators. This is done as an optimization instead + # of moving try_operator higher in the chain because operators + # aren't as common and try_operator is expensive + token_type = Operator + + elif c == '@': + token_type = Annotation + self.j = self.i + 1 + + elif c == '.' and c_next.isdigit(): + token_type = self.read_decimal_float_or_integer() + + elif self.try_separator(): + token_type = Separator + + elif c in ("'", '"'): + token_type = String + self.read_string() + + elif c in '0123456789': + token_type = self.read_integer_or_float(c, c_next) + + elif self.is_java_identifier_start(c): + token_type = self.read_identifier() + + elif self.try_operator(): + token_type = Operator + + else: + self.error('Could not process token', c) + + position = (self.current_line, self.i - self.start_of_line) + token = token_type(self.data[self.i:self.j], position, self.javadoc) + yield token + + if self.javadoc: + self.javadoc = None + + self.i = self.j + + def error(self, message, char=None): + # Provide additional information in the errors message + line_start = self.data.rfind('\n', 0, self.i) + 1 + line_end = self.data.find('\n', self.i) + line = self.data[line_start:line_end].strip() + + line_number = self.current_line + + if not char: + char = self.data[self.j] + + message = u'%s at "%s", line %s: %s' % (message, char, line_number, line) + + raise LexerError(message) + +def tokenize(code): + tokenizer = JavaTokenizer(code) + return tokenizer.tokenize() + +def reformat_tokens(tokens): + indent = 0 + closed_block = False + ident_last = False + + output = list() + + for token in tokens: + if closed_block: + closed_block = False + indent -= 4 + + output.append('\n') + output.append(' ' * indent) + output.append('}') + + if isinstance(token, (Literal, Keyword, Identifier)): + output.append('\n') + output.append(' ' * indent) + + if token.value == '{': + indent += 4 + output.append(' {\n') + output.append(' ' * indent) + + elif token.value == '}': + closed_block = True + + elif token.value == ',': + output.append(', ') + + elif isinstance(token, (Literal, Keyword, Identifier)): + if ident_last: + # If the last token was a literla/keyword/identifer put a space in between + output.append(' ') + ident_last = True + output.append(token.value) + + elif isinstance(token, Operator): + output.append(' ' + token.value + ' ') + + elif token.value == ';': + output.append(';\n') + output.append(' ' * indent) + + else: + output.append(token.value) + + ident_last = isinstance(token, (Literal, Keyword, Identifier)) + + if closed_block: + output.append('\n}') + + output.append('\n') + + return ''.join(output) diff --git a/baseline_tokenization/javalang/tree.py b/baseline_tokenization/javalang/tree.py new file mode 100644 index 0000000..aea883a --- /dev/null +++ b/baseline_tokenization/javalang/tree.py @@ -0,0 +1,272 @@ + +from .ast import Node + +# ------------------------------------------------------------------------------ + +class CompilationUnit(Node): + attrs = ("package", "imports", "types") + +class Import(Node): + attrs = ("path", "static", "wildcard") + +class Documented(Node): + attrs = ("documentation",) + +class Declaration(Node): + attrs = ("modifiers", "annotations") + +class TypeDeclaration(Declaration, Documented): + attrs = ("name", "body") + + @property + def fields(self): + return [decl for decl in self.body if isinstance(decl, FieldDeclaration)] + + @property + def methods(self): + return [decl for decl in self.body if isinstance(decl, MethodDeclaration)] + + @property + def constructors(self): + return [decl for decl in self.body if isinstance(decl, ConstructorDeclaration)] + +class PackageDeclaration(Declaration, Documented): + attrs = ("name",) + +class ClassDeclaration(TypeDeclaration): + attrs = ("type_parameters", "extends", "implements") + +class EnumDeclaration(TypeDeclaration): + attrs = ("implements",) + +class InterfaceDeclaration(TypeDeclaration): + attrs = ("type_parameters", "extends",) + +class AnnotationDeclaration(TypeDeclaration): + attrs = () + +# ------------------------------------------------------------------------------ + +class Type(Node): + attrs = ("name", "dimensions",) + +class BasicType(Type): + attrs = () + +class ReferenceType(Type): + attrs = ("arguments", "sub_type") + +class TypeArgument(Node): + attrs = ("type", "pattern_type") + +# ------------------------------------------------------------------------------ + +class TypeParameter(Node): + attrs = ("name", "extends") + +# ------------------------------------------------------------------------------ + +class Annotation(Node): + attrs = ("name", "element") + +class ElementValuePair(Node): + attrs = ("name", "value") + +class ElementArrayValue(Node): + attrs = ("values",) + +# ------------------------------------------------------------------------------ + +class Member(Documented): + attrs = () + +class MethodDeclaration(Member, Declaration): + attrs = ("type_parameters", "return_type", "name", "parameters", "throws", "body") + +class FieldDeclaration(Member, Declaration): + attrs = ("type", "declarators") + +class ConstructorDeclaration(Declaration, Documented): + attrs = ("type_parameters", "name", "parameters", "throws", "body") + +# ------------------------------------------------------------------------------ + +class ConstantDeclaration(FieldDeclaration): + attrs = () + +class ArrayInitializer(Node): + attrs = ("initializers",) + +class VariableDeclaration(Declaration): + attrs = ("type", "declarators") + +class LocalVariableDeclaration(VariableDeclaration): + attrs = () + +class VariableDeclarator(Node): + attrs = ("name", "dimensions", "initializer") + +class FormalParameter(Declaration): + attrs = ("type", "name", "varargs") + +class InferredFormalParameter(Node): + attrs = ('name',) + +# ------------------------------------------------------------------------------ + +class Statement(Node): + attrs = ("label",) + +class IfStatement(Statement): + attrs = ("condition", "then_statement", "else_statement") + +class WhileStatement(Statement): + attrs = ("condition", "body") + +class DoStatement(Statement): + attrs = ("condition", "body") + +class ForStatement(Statement): + attrs = ("control", "body") + +class AssertStatement(Statement): + attrs = ("condition", "value") + +class BreakStatement(Statement): + attrs = ("goto",) + +class ContinueStatement(Statement): + attrs = ("goto",) + +class ReturnStatement(Statement): + attrs = ("expression",) + +class ThrowStatement(Statement): + attrs = ("expression",) + +class SynchronizedStatement(Statement): + attrs = ("lock", "block") + +class TryStatement(Statement): + attrs = ("resources", "block", "catches", "finally_block") + +class SwitchStatement(Statement): + attrs = ("expression", "cases") + +class BlockStatement(Statement): + attrs = ("statements",) + +class StatementExpression(Statement): + attrs = ("expression",) + +# ------------------------------------------------------------------------------ + +class TryResource(Declaration): + attrs = ("type", "name", "value") + +class CatchClause(Statement): + attrs = ("parameter", "block") + +class CatchClauseParameter(Declaration): + attrs = ("types", "name") + +# ------------------------------------------------------------------------------ + +class SwitchStatementCase(Node): + attrs = ("case", "statements") + +class ForControl(Node): + attrs = ("init", "condition", "update") + +class EnhancedForControl(Node): + attrs = ("var", "iterable") + +# ------------------------------------------------------------------------------ + +class Expression(Node): + attrs = () + +class Assignment(Expression): + attrs = ("expressionl", "value", "type") + +class TernaryExpression(Expression): + attrs = ("condition", "if_true", "if_false") + +class BinaryOperation(Expression): + attrs = ("operator", "operandl", "operandr") + +class Cast(Expression): + attrs = ("type", "expression") + +class MethodReference(Expression): + attrs = ("expression", "method", "type_arguments") + +class LambdaExpression(Expression): + attrs = ('parameters', 'body') + +# ------------------------------------------------------------------------------ + +class Primary(Expression): + attrs = ("prefix_operators", "postfix_operators", "qualifier", "selectors") + +class Literal(Primary): + attrs = ("value",) + +class This(Primary): + attrs = () + +class MemberReference(Primary): + attrs = ("member",) + +class Invocation(Primary): + attrs = ("type_arguments", "arguments") + +class ExplicitConstructorInvocation(Invocation): + attrs = () + +class SuperConstructorInvocation(Invocation): + attrs = () + +class MethodInvocation(Invocation): + attrs = ("member",) + +class SuperMethodInvocation(Invocation): + attrs = ("member",) + +class SuperMemberReference(Primary): + attrs = ("member",) + +class ArraySelector(Expression): + attrs = ("index",) + +class ClassReference(Primary): + attrs = ("type",) + +class VoidClassReference(ClassReference): + attrs = () + +# ------------------------------------------------------------------------------ + +class Creator(Primary): + attrs = ("type",) + +class ArrayCreator(Creator): + attrs = ("dimensions", "initializer") + +class ClassCreator(Creator): + attrs = ("constructor_type_arguments", "arguments", "body") + +class InnerClassCreator(Creator): + attrs = ("constructor_type_arguments", "arguments", "body") + +# ------------------------------------------------------------------------------ + +class EnumBody(Node): + attrs = ("constants", "declarations") + +class EnumConstantDeclaration(Declaration, Documented): + attrs = ("name", "arguments", "body") + +class AnnotationMethod(Declaration): + attrs = ("name", "return_type", "dimensions", "default") + diff --git a/baseline_tokenization/javalang/util.py b/baseline_tokenization/javalang/util.py new file mode 100644 index 0000000..b8452fd --- /dev/null +++ b/baseline_tokenization/javalang/util.py @@ -0,0 +1,165 @@ + + +class LookAheadIterator(object): + def __init__(self, iterable): + self.iterable = iter(iterable) + self.look_ahead = list() + self.markers = list() + self.default = None + self.value = None + + def __iter__(self): + return self + + def set_default(self, value): + self.default = value + + def next(self): + return self.__next__() + + def __next__(self): + if self.look_ahead: + self.value = self.look_ahead.pop(0) + else: + self.value = next(self.iterable) + + if self.markers: + self.markers[-1].append(self.value) + + return self.value + + def look(self, i=0): + """ Look ahead of the iterable by some number of values with advancing + past them. + + If the requested look ahead is past the end of the iterable then None is + returned. + + """ + + length = len(self.look_ahead) + + if length <= i: + try: + self.look_ahead.extend([next(self.iterable) + for _ in range(length, i + 1)]) + except StopIteration: + return self.default + + self.value = self.look_ahead[i] + return self.value + + def last(self): + return self.value + + def __enter__(self): + self.push_marker() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Reset the iterator if there was an error + if exc_type or exc_val or exc_tb: + self.pop_marker(True) + else: + self.pop_marker(False) + + def push_marker(self): + """ Push a marker on to the marker stack """ + self.markers.append(list()) + + def pop_marker(self, reset): + """ Pop a marker off of the marker stack. If reset is True then the + iterator will be returned to the state it was in before the + corresponding call to push_marker(). + + """ + + marker = self.markers.pop() + + if reset: + # Make the values available to be read again + marker.extend(self.look_ahead) + self.look_ahead = marker + elif self.markers: + # Otherwise, reassign the values to the top marker + self.markers[-1].extend(marker) + else: + # If there are not more markers in the stack then discard the values + pass + +class LookAheadListIterator(object): + def __init__(self, iterable): + self.list = list(iterable) + + self.marker = 0 + self.saved_markers = [] + + self.default = None + self.value = None + + def __iter__(self): + return self + + def set_default(self, value): + self.default = value + + def next(self): + return self.__next__() + + def __next__(self): + try: + self.value = self.list[self.marker] + self.marker += 1 + except IndexError: + raise StopIteration() + + return self.value + + def look(self, i=0): + """ Look ahead of the iterable by some number of values with advancing + past them. + + If the requested look ahead is past the end of the iterable then None is + returned. + + """ + + try: + self.value = self.list[self.marker + i] + except IndexError: + return self.default + + return self.value + + def last(self): + return self.value + + def __enter__(self): + self.push_marker() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Reset the iterator if there was an error + if exc_type or exc_val or exc_tb: + self.pop_marker(True) + else: + self.pop_marker(False) + + def push_marker(self): + """ Push a marker on to the marker stack """ + self.saved_markers.append(self.marker) + + def pop_marker(self, reset): + """ Pop a marker off of the marker stack. If reset is True then the + iterator will be returned to the state it was in before the + corresponding call to push_marker(). + + """ + + saved = self.saved_markers.pop() + + if reset: + self.marker = saved + elif self.saved_markers: + self.saved_markers[-1] = saved + diff --git a/baseline_tokenization/subtokenize_nmt_baseline.py b/baseline_tokenization/subtokenize_nmt_baseline.py new file mode 100644 index 0000000..43730fe --- /dev/null +++ b/baseline_tokenization/subtokenize_nmt_baseline.py @@ -0,0 +1,50 @@ +#!/usr/bin/python + +import javalang +import sys +import re + + +modifiers = ['public', 'private', 'protected', 'static'] + +RE_WORDS = re.compile(r''' + # Find words in a string. Order matters! + [A-Z]+(?=[A-Z][a-z]) | # All upper case before a capitalized word + [A-Z]?[a-z]+ | # Capitalized words / all lower case + [A-Z]+ | # All upper case + \d+ | # Numbers + .+ +''', re.VERBOSE) + +def split_subtokens(str): + return [subtok for subtok in RE_WORDS.findall(str) if not subtok == '_'] + +def tokenizeFile(file_path): + lines = 0 + with open(file_path, 'r', encoding="utf-8") as file: + with open(file_path + 'method_names.txt', 'w') as method_names_file: + with open(file_path + 'method_subtokens_content.txt', 'w') as method_contents_file: + for line in file: + lines += 1 + line = line.rstrip() + parts = line.split('|', 1) + method_name = parts[0] + method_content = parts[1] + try: + tokens = list(javalang.tokenizer.tokenize(method_content)) + except: + print('ERROR in tokenizing: ' + method_content) + #tokens = method_content.split(' ') + if len(method_name) > 0 and len(tokens) > 0: + method_names_file.write(method_name + '\n') + method_contents_file.write(' '.join([' '.join(split_subtokens(i.value)) for i in tokens if not i.value in modifiers]) + '\n') + else: + print('ERROR in len of: ' + method_name + ', tokens: ' + str(tokens)) + print(str(lines)) + + +if __name__ == '__main__': + file = sys.argv[1] + tokenizeFile(file) + + diff --git a/code2seq.py b/code2seq.py index cb4df31..e42aa48 100644 --- a/code2seq.py +++ b/code2seq.py @@ -1,4 +1,6 @@ from argparse import ArgumentParser +import numpy as np +import tensorflow as tf from config import Config from interactive_predict import InteractivePredictor @@ -20,8 +22,12 @@ 'size.') parser.add_argument('--predict', action='store_true') parser.add_argument('--debug', action='store_true') + parser.add_argument('--seed', type=int, default=239) args = parser.parse_args() + np.random.seed(args.seed) + tf.set_random_seed(args.seed) + if args.debug: config = Config.get_debug_config(args) else: @@ -32,9 +38,10 @@ if config.TRAIN_PATH: model.train() if config.TEST_PATH and not args.data_path: - results, precision, recall, f1 = model.evaluate() + results, precision, recall, f1, rouge = model.evaluate() print('Accuracy: ' + str(results)) print('Precision: ' + str(precision) + ', recall: ' + str(recall) + ', F1: ' + str(f1)) + print('Rouge: ', rouge) if args.predict: predictor = InteractivePredictor(config, model) predictor.predict() diff --git a/common.py b/common.py index c1f9aa7..e1eb3f8 100644 --- a/common.py +++ b/common.py @@ -73,9 +73,7 @@ def filter_impossible_names(top_words): @staticmethod def unique(sequence): - unique = [] - [unique.append(item) for item in sequence if item not in unique] - return unique + return list(set(sequence)) @staticmethod def parse_results(result, pc_info_dict, topk=5): diff --git a/interactive_predict.py b/interactive_predict.py index 59e9272..039c037 100644 --- a/interactive_predict.py +++ b/interactive_predict.py @@ -4,7 +4,7 @@ SHOW_TOP_CONTEXTS = 10 MAX_PATH_LENGTH = 8 MAX_PATH_WIDTH = 2 -EXTRACTION_API = 'https://ff655m4ut8.execute-api.us-east-1.amazonaws.com/production/extractmethods' +EXTRACTION_API = 'https://po3g2dx2qa.execute-api.us-east-1.amazonaws.com/production/extractmethods' class InteractivePredictor: diff --git a/model.py b/model.py index 6f77c97..278cfc1 100644 --- a/model.py +++ b/model.py @@ -8,6 +8,7 @@ import reader from common import Common +from rouge import FilesRouge class Model: @@ -94,6 +95,7 @@ def train(self): batch_num += 1 _, batch_loss = self.sess.run([optimizer, train_loss]) sum_loss += batch_loss + # print('SINGLE BATCH LOSS', batch_loss) if batch_num % self.num_batches_to_log == 0: self.trace(sum_loss, batch_num, multi_batch_start_time) sum_loss = 0 @@ -103,10 +105,14 @@ def train(self): except tf.errors.OutOfRangeError: self.epochs_trained += self.config.SAVE_EVERY_EPOCHS print('Finished %d epochs' % self.config.SAVE_EVERY_EPOCHS) - results, precision, recall, f1 = self.evaluate() - print('Accuracy after %d epochs: %.5f' % (self.epochs_trained, results)) + results, precision, recall, f1, rouge = self.evaluate() + if self.config.BEAM_WIDTH == 0: + print('Accuracy after %d epochs: %.5f' % (self.epochs_trained, results)) + else: + print('Accuracy after {} epochs: {}'.format(self.epochs_trained, results)) print('After %d epochs: Precision: %.5f, recall: %.5f, F1: %.5f' % ( self.epochs_trained, precision, recall, f1)) + print('Rouge: ', rouge) if f1 > best_f1: best_f1 = f1 best_f1_precision = precision @@ -131,7 +137,7 @@ def train(self): def trace(self, sum_loss, batch_num, multi_batch_start_time): multi_batch_elapsed = time.time() - multi_batch_start_time - avg_loss = sum_loss / (self.num_batches_to_log * self.config.BATCH_SIZE) + avg_loss = sum_loss / self.num_batches_to_log print('Average loss at batch %d: %f, \tthroughput: %d samples/sec' % (batch_num, avg_loss, self.config.BATCH_SIZE * self.num_batches_to_log / ( multi_batch_elapsed if multi_batch_elapsed > 0 else 1))) @@ -167,7 +173,8 @@ def evaluate(self, release=False): with open(model_dirname + '/log.txt', 'w') as output_file, open(ref_file_name, 'w') as ref_file, open( predicted_file_name, 'w') as pred_file: - num_correct_predictions = 0 + num_correct_predictions = 0 if self.config.BEAM_WIDTH == 0 \ + else np.zeros([self.config.BEAM_WIDTH], dtype=np.int32) total_predictions = 0 total_prediction_batches = 0 true_positive, false_positive, false_negative = 0, 0, 0 @@ -218,22 +225,44 @@ def evaluate(self, release=False): elapsed = int(time.time() - eval_start_time) precision, recall, f1 = self.calculate_results(true_positive, false_positive, false_negative) + try: + files_rouge = FilesRouge() + rouge = files_rouge.get_scores( + hyp_path=predicted_file_name, ref_path=ref_file_name, avg=True, ignore_empty=True) + except ValueError: + rouge = 0 print("Evaluation time: %sh%sm%ss" % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60)) - return num_correct_predictions / total_predictions, precision, recall, f1 + return num_correct_predictions / total_predictions, \ + precision, recall, f1, rouge def update_correct_predictions(self, num_correct_predictions, output_file, results): for original_name, predicted in results: + original_name_parts = original_name.split(Common.internal_delimiter) # list + filtered_original = Common.filter_impossible_names(original_name_parts) # list + predicted_first = predicted if self.config.BEAM_WIDTH > 0: - predicted = predicted[0] - original_name_parts = original_name.split(Common.internal_delimiter) - filtered_original = Common.filter_impossible_names(original_name_parts) - filtered_predicted_parts = Common.filter_impossible_names(predicted) - output_file.write('Original: ' + Common.internal_delimiter.join(original_name_parts) + - ' , predicted 1st: ' + Common.internal_delimiter.join( - [target for target in filtered_predicted_parts]) + '\n') - if filtered_original == filtered_predicted_parts or Common.unique(filtered_original) == Common.unique( - filtered_predicted_parts) or ''.join(filtered_original) == ''.join(filtered_predicted_parts): - num_correct_predictions += 1 + predicted_first = predicted[0] + filtered_predicted_first_parts = Common.filter_impossible_names(predicted_first) # list + + if self.config.BEAM_WIDTH == 0: + output_file.write('Original: ' + Common.internal_delimiter.join(original_name_parts) + + ' , predicted 1st: ' + Common.internal_delimiter.join(filtered_predicted_first_parts) + '\n') + if filtered_original == filtered_predicted_first_parts or Common.unique(filtered_original) == Common.unique( + filtered_predicted_first_parts) or ''.join(filtered_original) == ''.join(filtered_predicted_first_parts): + num_correct_predictions += 1 + else: + filtered_predicted = [Common.internal_delimiter.join(Common.filter_impossible_names(p)) for p in predicted] + + true_ref = original_name + output_file.write('Original: ' + ' '.join(original_name_parts) + '\n') + for i, p in enumerate(filtered_predicted): + output_file.write('\t@{}: {}'.format(i + 1, ' '.join(p.split(Common.internal_delimiter)))+ '\n') + if true_ref in filtered_predicted: + index_of_correct = filtered_predicted.index(true_ref) + update = np.concatenate( + [np.zeros(index_of_correct, dtype=np.int32), + np.ones(self.config.BEAM_WIDTH - index_of_correct, dtype=np.int32)]) + num_correct_predictions += update return num_correct_predictions def update_per_subtoken_statistics(self, results, true_positive, false_positive, false_negative): @@ -599,7 +628,7 @@ def predict(self, predict_data_lines): predicted_strings = [[self.index_to_target[sugg] for sugg in timestep] for timestep in predicted_indices] # (target_length, top-k) predicted_strings = list(map(list, zip(*predicted_strings))) # (top-k, target_length) - top_scores = [np.exp(np.sum(s, 0)) for s in top_scores] + top_scores = [np.exp(np.sum(s)) for s in zip(*top_scores)] else: predicted_strings = [self.index_to_target[idx] for idx in predicted_indices] # (batch, target_length) diff --git a/preprocess.py b/preprocess.py index f4ef82b..2a6351b 100644 --- a/preprocess.py +++ b/preprocess.py @@ -49,7 +49,7 @@ def process_file(file_path, data_file_role, dataset_name, max_contexts, max_data total += 1 outfile.write(target_name + ' ' + " ".join(contexts) + csv_padding + '\n') - print('File: ' + data_file_path) + print('File: ' + file_path) print('Average total contexts: ' + str(float(sum_total) / total)) print('Average final (after sampling) contexts: ' + str(float(sum_sampled) / total)) print('Total examples: ' + str(total)) diff --git a/train_python150k.sh b/train_python150k.sh new file mode 100644 index 0000000..e08a975 --- /dev/null +++ b/train_python150k.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +data_dir=$1 +data_name=$(basename "${data_dir}") +data=${data_dir}/${data_name} +test=${data_dir}/${data_name}.val.c2s +run_name=$2 +model_dir=models/python150k-${run_name} +save_prefix=${model_dir}/model +cuda=${3:-0} +seed=${4:-239} + +mkdir -p "${model_dir}" +set -e +CUDA_VISIBLE_DEVICES=$cuda python -u code2seq.py \ + --data="${data}" \ + --test="${test}" \ + --save_prefix="${save_prefix}" \ + --seed="${seed}"