diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..7967ed9 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,8 @@ +@inproceedings{ + alon2018codeseq, + title={code2seq: Generating Sequences from Structured Representations of Code}, + author={Uri Alon and Shaked Brody and Omer Levy and Eran Yahav}, + booktitle={International Conference on Learning Representations}, + year={2019}, + url={https://openreview.net/forum?id=H1gKYo09tX}, +} diff --git a/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs b/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs index 666eec1..ce2991e 100644 --- a/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs +++ b/CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs @@ -25,7 +25,7 @@ public class Options [Option('o', "ofile_name", Default = "test.txt", HelpText = "Output file name")] public String OFileName { get; set; } - [Option('h', "no_hash", Default = false, HelpText = "When enabled, prints the whole path strings (not hashed)")] + [Option('h', "no_hash", Default = true, HelpText = "When enabled, prints the whole path strings (not hashed)")] public Boolean NoHash { get; set; } [Option('l', "max_contexts", Default = 30000, HelpText = "Max number of path contexts to sample. Affects only very large snippets")] diff --git a/JavaExtractor/JPredict/dependency-reduced-pom.xml b/JavaExtractor/JPredict/dependency-reduced-pom.xml index 53ae0f1..f1fd7c0 100644 --- a/JavaExtractor/JPredict/dependency-reduced-pom.xml +++ b/JavaExtractor/JPredict/dependency-reduced-pom.xml @@ -10,10 +10,9 @@ maven-compiler-plugin - 3.2 + 3.8.0 - 1.8 - 1.8 + 11 Test.java diff --git a/JavaExtractor/JPredict/pom.xml b/JavaExtractor/JPredict/pom.xml index 68f6da1..d92435f 100644 --- a/JavaExtractor/JPredict/pom.xml +++ b/JavaExtractor/JPredict/pom.xml @@ -11,10 +11,9 @@ maven-compiler-plugin - 3.2 + 3.8.0 - 1.8 - 1.8 + 11 Test.java @@ -68,6 +67,11 @@ commons-lang3 3.5 + + com.google.code.gson + gson + 2.8.5 + UTF-8 diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java index 51fd9c1..808c733 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java @@ -40,6 +40,9 @@ public class CommandLineValues { @Option(name = "--max_child_id", required = false) public int MaxChildId = 3; + @Option(name = "--json_output", required = false) + public boolean JsonOutput = false; + public CommandLineValues(String... args) throws CmdLineException { CmdLineParser parser = new CmdLineParser(this); try { diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java index 247a658..9bf407c 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java @@ -8,9 +8,12 @@ public class MethodContent { private final ArrayList leaves; private final String name; - public MethodContent(ArrayList leaves, String name) { + private final String content; + + public MethodContent(ArrayList leaves, String name, String content) { this.leaves = leaves; this.name = name; + this.content = content; } public ArrayList getLeaves() { @@ -20,4 +23,8 @@ public ArrayList getLeaves() { public String getName() { return name; } + + public String getContent() { + return content; + } } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java index 41e4481..6b32163 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java @@ -12,13 +12,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import com.google.gson.Gson; class ExtractFeaturesTask implements Callable { - private final CommandLineValues m_CommandLineValues; + private final CommandLineValues commandLineValues; private final Path filePath; public ExtractFeaturesTask(CommandLineValues commandLineValues, Path path) { - m_CommandLineValues = commandLineValues; + this.commandLineValues = commandLineValues; this.filePath = path; } @@ -49,8 +50,8 @@ public void processFile() { private ArrayList extractSingleFile() throws IOException { String code; - if (m_CommandLineValues.MaxFileLength > 0 && - Files.lines(filePath, Charset.defaultCharset()).count() > m_CommandLineValues.MaxFileLength) { + if (commandLineValues.MaxFileLength > 0 && + Files.lines(filePath, Charset.defaultCharset()).count() > commandLineValues.MaxFileLength) { return new ArrayList<>(); } try { @@ -59,7 +60,7 @@ private ArrayList extractSingleFile() throws IOException { e.printStackTrace(); code = Common.EmptyString; } - FeatureExtractor featureExtractor = new FeatureExtractor(m_CommandLineValues); + FeatureExtractor featureExtractor = new FeatureExtractor(commandLineValues, this.filePath); return featureExtractor.extractFeatures(code); } @@ -74,8 +75,14 @@ public String featuresToString(ArrayList features) { for (ProgramFeatures singleMethodFeatures : features) { StringBuilder builder = new StringBuilder(); - String toPrint = singleMethodFeatures.toString(); - if (m_CommandLineValues.PrettyPrint) { + String toPrint; + if (commandLineValues.JsonOutput) { + toPrint = new Gson().toJson(singleMethodFeatures); + } + else { + toPrint = singleMethodFeatures.toString(); + } + if (commandLineValues.PrettyPrint) { toPrint = toPrint.replace(" ", "\n\t"); } builder.append(toPrint); diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java index 782db11..aa6b20b 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java @@ -11,6 +11,8 @@ import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.Node; +import java.io.File; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; import java.util.Set; @@ -26,9 +28,11 @@ class FeatureExtractor { .of("AssignExpr", "ArrayAccessExpr", "FieldAccessExpr", "MethodCallExpr") .collect(Collectors.toCollection(HashSet::new)); private final CommandLineValues m_CommandLineValues; + private final Path filePath; - public FeatureExtractor(CommandLineValues commandLineValues) { + public FeatureExtractor(CommandLineValues commandLineValues, Path filePath) { this.m_CommandLineValues = commandLineValues; + this.filePath = filePath; } private static ArrayList getTreeStack(Node node) { @@ -90,7 +94,8 @@ private ArrayList generatePathFeatures(ArrayList private ProgramFeatures generatePathFeaturesForFunction(MethodContent methodContent) { ArrayList functionLeaves = methodContent.getLeaves(); - ProgramFeatures programFeatures = new ProgramFeatures(methodContent.getName()); + ProgramFeatures programFeatures = new ProgramFeatures( + methodContent.getName(), this.filePath, methodContent.getContent()); for (int i = 0; i < functionLeaves.size(); i++) { for (int j = i + 1; j < functionLeaves.size(); j++) { diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java index d9b9109..6194777 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java @@ -1,17 +1,22 @@ package JavaExtractor.FeaturesEntities; -import com.fasterxml.jackson.annotation.JsonIgnore; - +import java.nio.file.Path; import java.util.ArrayList; import java.util.stream.Collectors; public class ProgramFeatures { - private final String name; + String name; + + transient ArrayList features = new ArrayList<>(); + String textContent; + + String filePath; - private final ArrayList features = new ArrayList<>(); + public ProgramFeatures(String name, Path filePath, String textContent) { - public ProgramFeatures(String name) { this.name = name; + this.filePath = filePath.toAbsolutePath().toString(); + this.textContent = textContent; } @SuppressWarnings("StringBufferReplaceableByString") @@ -29,7 +34,6 @@ public void addFeature(Property source, String path, Property target) { features.add(newRelation); } - @JsonIgnore public boolean isEmpty() { return features.isEmpty(); } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java index 43fdc11..b48bcbd 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java @@ -1,18 +1,18 @@ package JavaExtractor.FeaturesEntities; public class ProgramRelation { - private final Property m_Source; - private final Property m_Target; - private final String m_Path; + Property source; + Property target; + String path; public ProgramRelation(Property sourceName, Property targetName, String path) { - m_Source = sourceName; - m_Target = targetName; - m_Path = path; + source = sourceName; + target = targetName; + this.path = path; } public String toString() { - return String.format("%s,%s,%s", m_Source.getName(), m_Path, - m_Target.getName()); + return String.format("%s,%s,%s", source.getName(), path, + target.getName()); } } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java index bf9fca5..28c2735 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java @@ -12,11 +12,11 @@ @SuppressWarnings("StringEquality") public class FunctionVisitor extends VoidVisitorAdapter { - private final ArrayList m_Methods = new ArrayList<>(); - private final CommandLineValues m_CommandLineValues; + private final ArrayList methods = new ArrayList<>(); + private final CommandLineValues commandLineValues; public FunctionVisitor(CommandLineValues commandLineValues) { - this.m_CommandLineValues = commandLineValues; + this.commandLineValues = commandLineValues; } @Override @@ -38,14 +38,13 @@ private void visitMethod(MethodDeclaration node) { splitName = String.join(Common.internalSeparator, splitNameParts); } + node.setName(Common.methodName); + if (node.getBody() != null) { long methodLength = getMethodLength(node.getBody().toString()); - if (m_CommandLineValues.MaxCodeLength > 0) { - if (methodLength >= m_CommandLineValues.MinCodeLength && methodLength <= m_CommandLineValues.MaxCodeLength) { - m_Methods.add(new MethodContent(leaves, splitName)); - } - } else { - m_Methods.add(new MethodContent(leaves, splitName)); + if (commandLineValues.MaxCodeLength <= 0 || + (methodLength >= commandLineValues.MinCodeLength && methodLength <= commandLineValues.MaxCodeLength)) { + methods.add(new MethodContent(leaves, splitName, node.toString())); } } } @@ -65,6 +64,6 @@ private long getMethodLength(String code) { } public ArrayList getMethodContents() { - return m_Methods; + return methods; } } diff --git a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar b/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar index 8972be5..da4f7db 100644 Binary files a/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar and b/JavaExtractor/JPredict/target/JavaExtractor-0.0.1-SNAPSHOT.jar differ diff --git a/Python150kExtractor/extract.py b/Python150kExtractor/extract.py index ce38bce..cb140a0 100644 --- a/Python150kExtractor/extract.py +++ b/Python150kExtractor/extract.py @@ -25,13 +25,9 @@ def __collect_asts(json_file): - asts = [] with open(json_file, 'r', encoding='utf-8') as f: - for line in f: - ast = json.loads(line.strip()) - asts.append(ast) - - return asts + for line in tqdm.tqdm(f): + yield line def __terminals(ast, node_index, args): @@ -170,8 +166,8 @@ def main(): np.random.seed(args.seed) data_dir = Path(args.data_dir) - trains = __collect_asts(data_dir / 'python100k_train.json') - evals = __collect_asts(data_dir / 'python50k_eval.json') + trains = list(__collect_asts(data_dir / 'python100k_train.json')) + evals = list(__collect_asts(data_dir / 'python50k_eval.json')) train, valid = sklearn_model_selection.train_test_split( trains, @@ -186,7 +182,7 @@ def main(): (train, valid, test), ): output_file = output_dir / f'{split_name}_output_file.txt' - __collect_all_and_save(split, args, output_file) + __collect_all_and_save((json.loads(line) for line in split), args, output_file) if __name__ == '__main__': diff --git a/README.md b/README.md index 211820d..86d2417 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Contributions are welcome.
## See also: - * **Structural Language Models for Any-Code Generation** is a new paper that learns to generate the missing code within a larger code snippet. This is similar to code completion, but is able to predict complex expressions rather than a single token at a time. See [PDF](https://arxiv.org/pdf/1910.00577.pdf) (demo: soon). + * **Structural Language Models for Code** (ICML'2020) is a new paper that learns to generate the missing code within a larger code snippet. This is similar to code completion, but is able to predict complex expressions rather than a single token at a time. See [PDF](https://arxiv.org/pdf/1910.00577.pdf), demo at [http://AnyCodeGen.org](http://AnyCodeGen.org). * **Adversarial Examples for Models of Code** is a new paper that shows how to slightly mutate the input code snippet of code2vec and GNNs models (thus, introducing adversarial examples), such that the model (code2vec or GNNs) will output a prediction of our choice. See [PDF](https://arxiv.org/pdf/1910.07517.pdf) (code: soon). * **Neural Reverse Engineering of Stripped Binaries** is a new paper that learns to predict procedure names in stripped binaries, thus use neural networks for reverse engineering. See [PDF](https://arxiv.org/pdf/1902.09122) (code: soon). * **code2vec** (POPL'2019) is our previous model. It can only generate a single label at a time (rather than a sequence as code2seq), but it is much faster to train (because of its simplicity). See [PDF](https://urialon.cswp.cs.technion.ac.il/wp-content/uploads/sites/83/2018/12/code2vec-popl19.pdf), demo at [https://code2vec.org](https://code2vec.org) and [code](https://github.com/tech-srl/code2vec/). @@ -34,7 +34,7 @@ Table of Contents ## Requirements * [python3](https://www.linuxbabe.com/ubuntu/install-python-3-6-ubuntu-16-04-16-10-17-04) - * TensorFlow 1.12 or newer ([install](https://www.tensorflow.org/install/install_linux)). To check TensorFlow version: + * TensorFlow 1.12 ([install](https://www.tensorflow.org/install/install_linux)). To check TensorFlow version: > python3 -c 'import tensorflow as tf; print(tf.\_\_version\_\_)' - For a TensorFlow 2.1 implementation by [@Kolkir](https://github.com/Kolkir/), see: [https://github.com/Kolkir/code2seq](https://github.com/Kolkir/code2seq) * For [creating a new Java dataset](#creating-and-preprocessing-a-new-java-dataset) or [manually examining a trained model](#step-4-manual-examination-of-a-trained-model) (any operation that requires parsing of a new code example): [JDK](https://openjdk.java.net/install/) @@ -80,7 +80,7 @@ tar -xvzf java-large-model.tar.gz ``` ##### Note: -This trained model is in a "released" state, which means that we stripped it from its training parameters and can thus be used for inference, but cannot be further trained. +This trained model is in a "released" state, which means that we stripped it from its training parameters. #### Training a model from scratch To train a model from scratch: @@ -197,7 +197,7 @@ should be implemented, and be called by [preprocess.sh](preprocess.sh). Basically, an extractor should be able to output for each directory containing source files: * A single text file, where each row is an example. * Each example is a space-delimited list of fields, where: - 1. The first field is the target label, internally delimited by the "|" character (for example: `compare|ignore|case` + 1. The first field is the target label, internally delimited by the "|" character (for example: `compare|ignore|case`) 2. Each of the following field are contexts, where each context has three components separated by commas (","). None of these components can include spaces nor commas. We refer to these three components as a token, a path, and another token, but in general other types of ternary contexts can be considered. diff --git a/common.py b/common.py index c1f9aa7..e1eb3f8 100644 --- a/common.py +++ b/common.py @@ -73,9 +73,7 @@ def filter_impossible_names(top_words): @staticmethod def unique(sequence): - unique = [] - [unique.append(item) for item in sequence if item not in unique] - return unique + return list(set(sequence)) @staticmethod def parse_results(result, pc_info_dict, topk=5): diff --git a/model.py b/model.py index df1fed3..278cfc1 100644 --- a/model.py +++ b/model.py @@ -225,8 +225,12 @@ def evaluate(self, release=False): elapsed = int(time.time() - eval_start_time) precision, recall, f1 = self.calculate_results(true_positive, false_positive, false_negative) - files_rouge = FilesRouge(predicted_file_name, ref_file_name) - rouge = files_rouge.get_scores(avg=True, ignore_empty=True) + try: + files_rouge = FilesRouge() + rouge = files_rouge.get_scores( + hyp_path=predicted_file_name, ref_path=ref_file_name, avg=True, ignore_empty=True) + except ValueError: + rouge = 0 print("Evaluation time: %sh%sm%ss" % ((elapsed // 60 // 60), (elapsed // 60) % 60, elapsed % 60)) return num_correct_predictions / total_predictions, \ precision, recall, f1, rouge diff --git a/preprocess.py b/preprocess.py index f4ef82b..2a6351b 100644 --- a/preprocess.py +++ b/preprocess.py @@ -49,7 +49,7 @@ def process_file(file_path, data_file_role, dataset_name, max_contexts, max_data total += 1 outfile.write(target_name + ' ' + " ".join(contexts) + csv_padding + '\n') - print('File: ' + data_file_path) + print('File: ' + file_path) print('Average total contexts: ' + str(float(sum_total) / total)) print('Average final (after sampling) contexts: ' + str(float(sum_sampled) / total)) print('Total examples: ' + str(total))