From f2cb5d9a90a0d939ef0d6fb0fce7fd4c177eee97 Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:41:39 +0000 Subject: [PATCH 1/6] =?UTF-8?q?`*.c2s`=20=E3=81=AB=20`qualifiedName`=20?= =?UTF-8?q?=E3=82=92=E5=90=AB=E3=82=81=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JavaExtractor/Common/MethodContent.java | 8 +++++- .../java/JavaExtractor/FeatureExtractor.java | 26 +++++++++---------- .../FeaturesEntities/ProgramFeatures.java | 5 +++- .../Visitors/FunctionVisitor.java | 20 +++++++++----- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java index 9bf407c..078a746 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java @@ -9,11 +9,13 @@ public class MethodContent { private final String name; private final String content; + private final String qualifiedName; - public MethodContent(ArrayList leaves, String name, String content) { + public MethodContent(ArrayList leaves, String name, String content, String qualifiedName) { this.leaves = leaves; this.name = name; this.content = content; + this.qualifiedName = qualifiedName; } public ArrayList getLeaves() { @@ -27,4 +29,8 @@ public String getName() { public String getContent() { return content; } + + public String getQualifiedName() { + return qualifiedName; + } } diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java index aa6b20b..bd9e1a7 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java @@ -10,6 +10,7 @@ import com.github.javaparser.ParseProblemException; import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.PackageDeclaration; import java.io.File; import java.nio.file.Path; @@ -49,7 +50,9 @@ public ArrayList extractFeatures(String code) { CompilationUnit m_CompilationUnit = parseFileWithRetries(code); FunctionVisitor functionVisitor = new FunctionVisitor(m_CommandLineValues); - functionVisitor.visit(m_CompilationUnit, null); + PackageDeclaration packageDeclaration = m_CompilationUnit.getPackage(); + functionVisitor.visit(m_CompilationUnit, + packageDeclaration != null ? packageDeclaration.getPackageName() : null); ArrayList methods = functionVisitor.getMethodContents(); @@ -94,8 +97,8 @@ private ArrayList generatePathFeatures(ArrayList private ProgramFeatures generatePathFeaturesForFunction(MethodContent methodContent) { ArrayList functionLeaves = methodContent.getLeaves(); - ProgramFeatures programFeatures = new ProgramFeatures( - methodContent.getName(), this.filePath, methodContent.getContent()); + ProgramFeatures programFeatures = new ProgramFeatures(methodContent.getName(), this.filePath, + methodContent.getContent(), methodContent.getQualifiedName()); for (int i = 0; i < functionLeaves.size(); i++) { for (int j = i + 1; j < functionLeaves.size(); j++) { @@ -146,11 +149,10 @@ private String generatePath(Node source, Node target, String separator) { String childId = Common.EmptyString; String parentRawType = currentNode.getParentNode().getUserData(Common.PropertyKey).getRawType(); if (i == 0 || s_ParentTypeToAddChildId.contains(parentRawType)) { - childId = saturateChildId(currentNode.getUserData(Common.ChildId)) - .toString(); + childId = saturateChildId(currentNode.getUserData(Common.ChildId)).toString(); } - stringBuilder.add(String.format("%s%s%s", - currentNode.getUserData(Common.PropertyKey).getType(true), childId, upSymbol)); + stringBuilder.add(String.format("%s%s%s", currentNode.getUserData(Common.PropertyKey).getType(true), + childId, upSymbol)); } Node commonNode = sourceStack.get(sourceStack.size() - commonPrefix); @@ -161,18 +163,16 @@ private String generatePath(Node source, Node target, String separator) { commonNodeParentRawType = parentNodeProperty.getRawType(); } if (s_ParentTypeToAddChildId.contains(commonNodeParentRawType)) { - commonNodeChildId = saturateChildId(commonNode.getUserData(Common.ChildId)) - .toString(); + commonNodeChildId = saturateChildId(commonNode.getUserData(Common.ChildId)).toString(); } - stringBuilder.add(String.format("%s%s", - commonNode.getUserData(Common.PropertyKey).getType(true), commonNodeChildId)); + stringBuilder.add( + String.format("%s%s", commonNode.getUserData(Common.PropertyKey).getType(true), commonNodeChildId)); for (int i = targetStack.size() - commonPrefix - 1; i >= 0; i--) { Node currentNode = targetStack.get(i); String childId = Common.EmptyString; if (i == 0 || s_ParentTypeToAddChildId.contains(currentNode.getUserData(Common.PropertyKey).getRawType())) { - childId = saturateChildId(currentNode.getUserData(Common.ChildId)) - .toString(); + childId = saturateChildId(currentNode.getUserData(Common.ChildId)).toString(); } stringBuilder.add(String.format("%s%s%s", downSymbol, currentNode.getUserData(Common.PropertyKey).getType(true), childId)); diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java index 6194777..15ff5a0 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java @@ -11,18 +11,21 @@ public class ProgramFeatures { String textContent; String filePath; + String qualifiedName; - public ProgramFeatures(String name, Path filePath, String textContent) { + public ProgramFeatures(String name, Path filePath, String textContent, String qualifiedName) { this.name = name; this.filePath = filePath.toAbsolutePath().toString(); this.textContent = textContent; + this.qualifiedName = qualifiedName; } @SuppressWarnings("StringBufferReplaceableByString") @Override public String toString() { StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(qualifiedName).append(" "); stringBuilder.append(name).append(" "); stringBuilder.append(features.stream().map(ProgramRelation::toString).collect(Collectors.joining(" "))); diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java index 28c2735..b712167 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java @@ -4,6 +4,7 @@ import JavaExtractor.Common.Common; import JavaExtractor.Common.MethodContent; import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.visitor.VoidVisitorAdapter; @@ -11,7 +12,7 @@ import java.util.Arrays; @SuppressWarnings("StringEquality") -public class FunctionVisitor extends VoidVisitorAdapter { +public class FunctionVisitor extends VoidVisitorAdapter { private final ArrayList methods = new ArrayList<>(); private final CommandLineValues commandLineValues; @@ -20,13 +21,18 @@ public FunctionVisitor(CommandLineValues commandLineValues) { } @Override - public void visit(MethodDeclaration node, Object arg) { - visitMethod(node); + public void visit(ClassOrInterfaceDeclaration node, String arg) { + super.visit(node, arg != null ? arg + "." + node.getName() : node.getName()); + } + + @Override + public void visit(MethodDeclaration node, String arg) { + visitMethod(node, arg != null ? arg + "." + node.getName() : node.getName()); super.visit(node, arg); } - private void visitMethod(MethodDeclaration node) { + private void visitMethod(MethodDeclaration node, String qualifiedName) { LeavesCollectorVisitor leavesCollectorVisitor = new LeavesCollectorVisitor(); leavesCollectorVisitor.visitDepthFirst(node); ArrayList leaves = leavesCollectorVisitor.getLeaves(); @@ -42,9 +48,9 @@ private void visitMethod(MethodDeclaration node) { if (node.getBody() != null) { long methodLength = getMethodLength(node.getBody().toString()); - if (commandLineValues.MaxCodeLength <= 0 || - (methodLength >= commandLineValues.MinCodeLength && methodLength <= commandLineValues.MaxCodeLength)) { - methods.add(new MethodContent(leaves, splitName, node.toString())); + if (commandLineValues.MaxCodeLength <= 0 || (methodLength >= commandLineValues.MinCodeLength + && methodLength <= commandLineValues.MaxCodeLength)) { + methods.add(new MethodContent(leaves, splitName, node.toString(), qualifiedName)); } } } From c7797cb9954b9765f1ce06972258968154973734 Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Mon, 27 Dec 2021 10:10:43 +0000 Subject: [PATCH 2/6] =?UTF-8?q?`*.{train,val,test}.c2s`=20=E3=81=AE2?= =?UTF-8?q?=E5=88=97=E7=9B=AE=E3=81=AB=20`target=5Fname`=20=E3=81=8C?= =?UTF-8?q?=E6=9D=A5=E3=81=AA=E3=81=84=E5=A0=B4=E5=90=88=E3=81=8C=E3=81=82?= =?UTF-8?q?=E3=82=8B=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- preprocess.py | 238 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 171 insertions(+), 67 deletions(-) diff --git a/preprocess.py b/preprocess.py index 2a6351b..9bd03cf 100644 --- a/preprocess.py +++ b/preprocess.py @@ -5,92 +5,183 @@ import common -''' +""" This script preprocesses the data from MethodPaths. It truncates methods with too many contexts, and pads methods with less paths with spaces. -''' - - -def save_dictionaries(dataset_name, subtoken_to_count, node_to_count, target_to_count, max_contexts, num_examples): - save_dict_file_path = '{}.dict.c2s'.format(dataset_name) - with open(save_dict_file_path, 'wb') as file: +""" + + +def save_dictionaries( + dataset_name, + subtoken_to_count, + node_to_count, + target_to_count, + max_contexts, + num_examples, +): + save_dict_file_path = "{}.dict.c2s".format(dataset_name) + with open(save_dict_file_path, "wb") as file: pickle.dump(subtoken_to_count, file) pickle.dump(node_to_count, file) pickle.dump(target_to_count, file) pickle.dump(max_contexts, file) pickle.dump(num_examples, file) - print('Dictionaries saved to: {}'.format(save_dict_file_path)) + print("Dictionaries saved to: {}".format(save_dict_file_path)) -def process_file(file_path, data_file_role, dataset_name, max_contexts, max_data_contexts): +def process_file( + file_path, data_file_role, dataset_name, max_contexts, max_data_contexts +): sum_total = 0 sum_sampled = 0 total = 0 max_unfiltered = 0 - max_contexts_to_sample = max_data_contexts if data_file_role == 'train' else max_contexts - output_path = '{}.{}.c2s'.format(dataset_name, data_file_role) - with open(output_path, 'w') as outfile: - with open(file_path, 'r') as file: + max_contexts_to_sample = ( + max_data_contexts if data_file_role == "train" else max_contexts + ) + output_path = "{}.{}.c2s".format(dataset_name, data_file_role) + with open(output_path, "w") as outfile: + with open(file_path, "r") as file: for line in file: - parts = line.rstrip('\n').split(' ') - target_name = parts[0] - contexts = parts[1:] + parts = line.rstrip("\n").split(" ") + qualified_name = parts[0] + target_name = parts[1] + contexts = parts[2:] if len(contexts) > max_unfiltered: max_unfiltered = len(contexts) sum_total += len(contexts) if len(contexts) > max_contexts_to_sample: - contexts = np.random.choice(contexts, max_contexts_to_sample, replace=False) + contexts = np.random.choice( + contexts, max_contexts_to_sample, replace=False + ) sum_sampled += len(contexts) csv_padding = " " * (max_data_contexts - len(contexts)) total += 1 - outfile.write(target_name + ' ' + " ".join(contexts) + csv_padding + '\n') - - print('File: ' + file_path) - print('Average total contexts: ' + str(float(sum_total) / total)) - print('Average final (after sampling) contexts: ' + str(float(sum_sampled) / total)) - print('Total examples: ' + str(total)) - print('Max number of contexts per word: ' + str(max_unfiltered)) + outfile.write( + qualified_name + + " " + + target_name + + " " + + " ".join(contexts) + + csv_padding + + "\n" + ) + + print("File: " + file_path) + print("Average total contexts: " + str(float(sum_total) / total)) + print("Average final (after sampling) contexts: " + str(float(sum_sampled) / total)) + print("Total examples: " + str(total)) + print("Max number of contexts per word: " + str(max_unfiltered)) return total def context_full_found(context_parts, word_to_count, path_to_count): - return context_parts[0] in word_to_count \ - and context_parts[1] in path_to_count and context_parts[2] in word_to_count + return ( + context_parts[0] in word_to_count + and context_parts[1] in path_to_count + and context_parts[2] in word_to_count + ) def context_partial_found(context_parts, word_to_count, path_to_count): - return context_parts[0] in word_to_count \ - or context_parts[1] in path_to_count or context_parts[2] in word_to_count + return ( + context_parts[0] in word_to_count + or context_parts[1] in path_to_count + or context_parts[2] in word_to_count + ) -if __name__ == '__main__': +if __name__ == "__main__": parser = ArgumentParser() - parser.add_argument("-trd", "--train_data", dest="train_data_path", - help="path to training data file", required=True) - parser.add_argument("-ted", "--test_data", dest="test_data_path", - help="path to test data file", required=True) - parser.add_argument("-vd", "--val_data", dest="val_data_path", - help="path to validation data file", required=True) - parser.add_argument("-mc", "--max_contexts", dest="max_contexts", default=200, - help="number of max contexts to keep in test+validation", required=False) - parser.add_argument("-mdc", "--max_data_contexts", dest="max_data_contexts", default=1000, - help="number of max contexts to keep in the dataset", required=False) - parser.add_argument("-svs", "--subtoken_vocab_size", dest="subtoken_vocab_size", default=186277, - help="Max number of source subtokens to keep in the vocabulary", required=False) - parser.add_argument("-tvs", "--target_vocab_size", dest="target_vocab_size", default=26347, - help="Max number of target words to keep in the vocabulary", required=False) - parser.add_argument("-sh", "--subtoken_histogram", dest="subtoken_histogram", - help="subtoken histogram file", metavar="FILE", required=True) - parser.add_argument("-nh", "--node_histogram", dest="node_histogram", - help="node_histogram file", metavar="FILE", required=True) - parser.add_argument("-th", "--target_histogram", dest="target_histogram", - help="target histogram file", metavar="FILE", required=True) - parser.add_argument("-o", "--output_name", dest="output_name", - help="output name - the base name for the created dataset", required=True, default='data') + parser.add_argument( + "-trd", + "--train_data", + dest="train_data_path", + help="path to training data file", + required=True, + ) + parser.add_argument( + "-ted", + "--test_data", + dest="test_data_path", + help="path to test data file", + required=True, + ) + parser.add_argument( + "-vd", + "--val_data", + dest="val_data_path", + help="path to validation data file", + required=True, + ) + parser.add_argument( + "-mc", + "--max_contexts", + dest="max_contexts", + default=200, + help="number of max contexts to keep in test+validation", + required=False, + ) + parser.add_argument( + "-mdc", + "--max_data_contexts", + dest="max_data_contexts", + default=1000, + help="number of max contexts to keep in the dataset", + required=False, + ) + parser.add_argument( + "-svs", + "--subtoken_vocab_size", + dest="subtoken_vocab_size", + default=186277, + help="Max number of source subtokens to keep in the vocabulary", + required=False, + ) + parser.add_argument( + "-tvs", + "--target_vocab_size", + dest="target_vocab_size", + default=26347, + help="Max number of target words to keep in the vocabulary", + required=False, + ) + parser.add_argument( + "-sh", + "--subtoken_histogram", + dest="subtoken_histogram", + help="subtoken histogram file", + metavar="FILE", + required=True, + ) + parser.add_argument( + "-nh", + "--node_histogram", + dest="node_histogram", + help="node_histogram file", + metavar="FILE", + required=True, + ) + parser.add_argument( + "-th", + "--target_histogram", + dest="target_histogram", + help="target histogram file", + metavar="FILE", + required=True, + ) + parser.add_argument( + "-o", + "--output_name", + dest="output_name", + help="output name - the base name for the created dataset", + required=True, + default="data", + ) args = parser.parse_args() train_data_path = args.train_data_path @@ -99,23 +190,36 @@ def context_partial_found(context_parts, word_to_count, path_to_count): subtoken_histogram_path = args.subtoken_histogram node_histogram_path = args.node_histogram - subtoken_to_count = common.Common.load_histogram(subtoken_histogram_path, - max_size=int(args.subtoken_vocab_size)) - node_to_count = common.Common.load_histogram(node_histogram_path, - max_size=None) - target_to_count = common.Common.load_histogram(args.target_histogram, - max_size=int(args.target_vocab_size)) - print('subtoken vocab size: ', len(subtoken_to_count)) - print('node vocab size: ', len(node_to_count)) - print('target vocab size: ', len(target_to_count)) + subtoken_to_count = common.Common.load_histogram( + subtoken_histogram_path, max_size=int(args.subtoken_vocab_size) + ) + node_to_count = common.Common.load_histogram(node_histogram_path, max_size=None) + target_to_count = common.Common.load_histogram( + args.target_histogram, max_size=int(args.target_vocab_size) + ) + print("subtoken vocab size: ", len(subtoken_to_count)) + print("node vocab size: ", len(node_to_count)) + print("target vocab size: ", len(target_to_count)) num_training_examples = 0 - for data_file_path, data_role in zip([test_data_path, val_data_path, train_data_path], ['test', 'val', 'train']): - num_examples = process_file(file_path=data_file_path, data_file_role=data_role, dataset_name=args.output_name, - max_contexts=int(args.max_contexts), max_data_contexts=int(args.max_data_contexts)) - if data_role == 'train': + for data_file_path, data_role in zip( + [test_data_path, val_data_path, train_data_path], ["test", "val", "train"] + ): + num_examples = process_file( + file_path=data_file_path, + data_file_role=data_role, + dataset_name=args.output_name, + max_contexts=int(args.max_contexts), + max_data_contexts=int(args.max_data_contexts), + ) + if data_role == "train": num_training_examples = num_examples - save_dictionaries(dataset_name=args.output_name, subtoken_to_count=subtoken_to_count, - node_to_count=node_to_count, target_to_count=target_to_count, - max_contexts=int(args.max_data_contexts), num_examples=num_training_examples) + save_dictionaries( + dataset_name=args.output_name, + subtoken_to_count=subtoken_to_count, + node_to_count=node_to_count, + target_to_count=target_to_count, + max_contexts=int(args.max_data_contexts), + num_examples=num_training_examples, + ) From 350150af6390a5ef92c1ca5d8ca0b28d13b47729 Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Mon, 27 Dec 2021 10:10:57 +0000 Subject: [PATCH 3/6] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 30195d3..1a83cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.xml *.pyc +data \ No newline at end of file From fb5600ca24f5dd345f506ccec43ff3a66dc2a519 Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Tue, 4 Jan 2022 00:10:33 +0900 Subject: [PATCH 4/6] =?UTF-8?q?`PackageDeclaration`=20=E3=81=AE=E9=80=94?= =?UTF-8?q?=E4=B8=AD=E3=81=AB=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=8C?= =?UTF-8?q?=E5=90=AB=E3=81=BE=E3=82=8C=E3=82=8B=E5=A0=B4=E5=90=88=EF=BC=8C?= =?UTF-8?q?=E3=81=9D=E3=82=8C=E3=82=89=E3=81=8C=20`*.raw.txt`=20=E3=81=B8?= =?UTF-8?q?=E3=81=9D=E3=81=AE=E3=81=BE=E3=81=BE=E5=87=BA=E5=8A=9B=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=81=9F=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/JavaExtractor/FeatureExtractor.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java index bd9e1a7..0f02e69 100644 --- a/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java +++ b/JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java @@ -1,18 +1,5 @@ package JavaExtractor; -import JavaExtractor.Common.CommandLineValues; -import JavaExtractor.Common.Common; -import JavaExtractor.Common.MethodContent; -import JavaExtractor.FeaturesEntities.ProgramFeatures; -import JavaExtractor.FeaturesEntities.Property; -import JavaExtractor.Visitors.FunctionVisitor; -import com.github.javaparser.JavaParser; -import com.github.javaparser.ParseProblemException; -import com.github.javaparser.ast.CompilationUnit; -import com.github.javaparser.ast.Node; -import com.github.javaparser.ast.PackageDeclaration; - -import java.io.File; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; @@ -21,6 +8,21 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParseProblemException; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.PackageDeclaration; +import com.github.javaparser.ast.expr.QualifiedNameExpr; +import com.github.javaparser.ast.visitor.GenericVisitorAdapter; + +import JavaExtractor.Common.CommandLineValues; +import JavaExtractor.Common.Common; +import JavaExtractor.Common.MethodContent; +import JavaExtractor.FeaturesEntities.ProgramFeatures; +import JavaExtractor.FeaturesEntities.Property; +import JavaExtractor.Visitors.FunctionVisitor; + @SuppressWarnings("StringEquality") class FeatureExtractor { private final static String upSymbol = "|"; @@ -51,8 +53,19 @@ public ArrayList extractFeatures(String code) { FunctionVisitor functionVisitor = new FunctionVisitor(m_CommandLineValues); PackageDeclaration packageDeclaration = m_CompilationUnit.getPackage(); - functionVisitor.visit(m_CompilationUnit, - packageDeclaration != null ? packageDeclaration.getPackageName() : null); + if (packageDeclaration != null) { + QualifiedNameExpr qualifiedNameExpr = new GenericVisitorAdapter() { + @Override + public QualifiedNameExpr visit(QualifiedNameExpr n, Void arg) { + return n; + } + }.visit(packageDeclaration, null); + + functionVisitor.visit(m_CompilationUnit, + qualifiedNameExpr.getQualifiedName()); + } else { + functionVisitor.visit(m_CompilationUnit, null); + } ArrayList methods = functionVisitor.getMethodContents(); From aa2c818c6a69c8d8bb12c9cc564479fbdab8051a Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Wed, 28 Sep 2022 21:02:19 +0900 Subject: [PATCH 5/6] =?UTF-8?q?`node=5Fto=5Fcount`=20=E3=81=AB=20subtoken?= =?UTF-8?q?=20=E3=81=8C=E6=B7=B7=E5=85=A5=E3=81=99=E3=82=8B=E5=95=8F?= =?UTF-8?q?=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- preprocess.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/preprocess.sh b/preprocess.sh index 53644de..dd0aa04 100644 --- a/preprocess.sh +++ b/preprocess.sh @@ -56,9 +56,9 @@ SOURCE_SUBTOKEN_HISTOGRAM=data/${DATASET_NAME}/${DATASET_NAME}.histo.ori.c2s NODE_HISTOGRAM_FILE=data/${DATASET_NAME}/${DATASET_NAME}.histo.node.c2s echo "Creating histograms from the training data" -cat ${TRAIN_DATA_FILE} | cut -d' ' -f1 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${TARGET_HISTOGRAM_FILE} -cat ${TRAIN_DATA_FILE} | cut -d' ' -f2- | tr ' ' '\n' | cut -d',' -f1,3 | tr ',|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${SOURCE_SUBTOKEN_HISTOGRAM} -cat ${TRAIN_DATA_FILE} | cut -d' ' -f2- | tr ' ' '\n' | cut -d',' -f2 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${NODE_HISTOGRAM_FILE} +cat ${TRAIN_DATA_FILE} | cut -d' ' -f2 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${TARGET_HISTOGRAM_FILE} +cat ${TRAIN_DATA_FILE} | cut -d' ' -f3- | tr ' ' '\n' | cut -d',' -f1,3 | tr ',|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${SOURCE_SUBTOKEN_HISTOGRAM} +cat ${TRAIN_DATA_FILE} | cut -d' ' -f3- | tr ' ' '\n' | cut -d',' -f2 | tr '|' '\n' | awk '{n[$0]++} END {for (i in n) print i,n[i]}' > ${NODE_HISTOGRAM_FILE} ${PYTHON} preprocess.py --train_data ${TRAIN_DATA_FILE} --test_data ${TEST_DATA_FILE} --val_data ${VAL_DATA_FILE} \ --max_contexts ${MAX_CONTEXTS} --max_data_contexts ${MAX_DATA_CONTEXTS} --subtoken_vocab_size ${SUBTOKEN_VOCAB_SIZE} \ From dd566d3952aa6a045be8a67a0a71b8d264a0deeb Mon Sep 17 00:00:00 2001 From: hiroto7 <32275337+hiroto7@users.noreply.github.com> Date: Wed, 1 Jun 2022 08:55:25 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=E3=82=B7=E3=82=A7=E3=83=AB=E3=82=B9?= =?UTF-8?q?=E3=82=AF=E3=83=AA=E3=83=97=E3=83=88=E3=81=AE=E9=80=94=E4=B8=AD?= =?UTF-8?q?=E3=81=A7=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=8C=E7=99=BA=E7=94=9F?= =?UTF-8?q?=E3=81=97=E3=81=9F=E5=A0=B4=E5=90=88=EF=BC=8C=E3=81=9D=E3=81=AE?= =?UTF-8?q?=E6=99=82=E7=82=B9=E3=81=A7=E7=B5=82=E4=BA=86=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit aa9860d42a96b36731749c0c873f026f863f3696) --- preprocess.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocess.sh b/preprocess.sh index dd0aa04..88815ee 100644 --- a/preprocess.sh +++ b/preprocess.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash -eu ########################################################### # Change the following values to preprocess a new dataset. # TRAIN_DIR, VAL_DIR and TEST_DIR should be paths to