diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..c3e593d --- /dev/null +++ b/.classpath @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..5c98b42 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,2 @@ +# Default ignored files +/workspace.xml \ No newline at end of file diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..1a4eb19 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +JFastText \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..e04ecec --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..b26911b --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..4b661a5 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.project b/.project new file mode 100644 index 0000000..ba9a296 --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + fasttext + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..f9fe345 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 0000000..d4313d4 --- /dev/null +++ b/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..3fc7e86 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.processAnnotations=disabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/PATENTS b/PATENTS old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/classification-example.sh b/classification-example.sh old mode 100644 new mode 100755 diff --git a/classification-results.sh b/classification-results.sh old mode 100644 new mode 100755 diff --git a/eval.py b/eval.py old mode 100644 new mode 100755 diff --git a/pom.xml b/pom.xml index f6e50c8..d1e07fb 100644 --- a/pom.xml +++ b/pom.xml @@ -2,27 +2,64 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - fasttext - fasttext - 0.0.1-SNAPSHOT + ai.searchbox + FastText4J + 0.9.43 jar - fasttext + ai.searchbox.FastText4J http://maven.apache.org UTF-8 - 1.6 - 1.6 + 1.8 + 1.8 + 7.7.1 + + com.google.guava + guava + 19.0 + + + + org.apache.lucene + lucene-core + ${lucene.version} + + + + org.apache.lucene + lucene-backward-codecs + ${lucene.version} + + + + org.apache.lucene + lucene-sandbox + ${lucene.version} + + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + junit junit 4.12 test + + + log4j + log4j + 1.2.17 + @@ -35,11 +72,12 @@ lib 1.6 - 1.6 - 1.6 + 1.8 + 1.8 3.3 + org.apache.maven.plugins maven-resources-plugin @@ -48,6 +86,7 @@ 3.0.1 + org.apache.maven.plugins maven-source-plugin @@ -71,7 +110,7 @@ - fasttext.Main + ai.searchbox.FastText4J.Main diff --git a/src/main/java/ai/searchbox/FastText4J/Args.java b/src/main/java/ai/searchbox/FastText4J/Args.java new file mode 100644 index 0000000..b9ff8db --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Args.java @@ -0,0 +1,302 @@ +package ai.searchbox.FastText4J; + +import ai.searchbox.FastText4J.io.IOUtil; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class Args { + public String input; + + public String output; + + public String test; + + public double lr = 0.05D; + + public int lrUpdateRate = 100; + + public int dim = 100; + + public int ws = 5; + + public int epoch = 5; + + public int minCount = 5; + + public int minCountLabel = 0; + + public int neg = 5; + + public int wordNgrams = 1; + + public LossType loss = LossType.ns; + + public ModelType model = ModelType.sg; + + public int bucket = 2000000; + + public int minn = 3; + + public int maxn = 6; + + public int thread = 1; + + public double t = 1.0E-4D; + + public String label = "__label__"; + + public int verbose = 2; + + public String pretrainedVectors = ""; + + public void printHelp() { + System.out.println("\nThe following arguments are mandatory:\n -input training file path\n -output output file path\n\nThe following arguments are optional:\n -lr learning rate [" + + + this.lr + "]\n" + + " -lrUpdateRate change the rate of updates for the learning rate [" + this.lrUpdateRate + "]\n" + + " -dim size of word vectors [" + this.dim + "]\n" + + " -ws size of the context window [" + this.ws + "]\n" + + " -epoch number of epochs [" + this.epoch + "]\n" + + " -minCount minimal number of word occurences [" + this.minCount + "]\n" + + " -minCountLabel minimal number of label occurences [" + this.minCountLabel + "]\n" + + " -neg number of negatives sampled [" + this.neg + "]\n" + + " -wordNgrams max length of word ngram [" + this.wordNgrams + "]\n" + + " -loss loss function {ns, hs, softmax} [ns]\n" + + " -bucket number of buckets [" + this.bucket + "]\n" + + " -minn min length of char ngram [" + this.minn + "]\n" + + " -maxn max length of char ngram [" + this.maxn + "]\n" + + " -thread number of threads [" + this.thread + "]\n" + + " -t sampling threshold [" + this.t + "]\n" + + " -label labels prefix [" + this.label + "]\n" + + " -verbose verbosity level [" + this.verbose + "]\n" + + " -pretrainedVectors pretrained word vectors for supervised learning []"); + } + + public void save(OutputStream ofs) throws IOException { + IOUtil ioutil = new IOUtil(); + ofs.write(ioutil.intToByteArray(this.dim)); + ofs.write(ioutil.intToByteArray(this.ws)); + ofs.write(ioutil.intToByteArray(this.epoch)); + ofs.write(ioutil.intToByteArray(this.minCount)); + ofs.write(ioutil.intToByteArray(this.neg)); + ofs.write(ioutil.intToByteArray(this.wordNgrams)); + ofs.write(ioutil.intToByteArray(this.loss.value)); + ofs.write(ioutil.intToByteArray(this.model.value)); + ofs.write(ioutil.intToByteArray(this.bucket)); + ofs.write(ioutil.intToByteArray(this.minn)); + ofs.write(ioutil.intToByteArray(this.maxn)); + ofs.write(ioutil.intToByteArray(this.lrUpdateRate)); + ofs.write(ioutil.doubleToByteArray(this.t)); + } + + public void load(InputStream input) throws IOException { + IOUtil ioutil = new IOUtil(); + this.dim = ioutil.readInt(input); + this.ws = ioutil.readInt(input); + this.epoch = ioutil.readInt(input); + this.minCount = ioutil.readInt(input); + this.neg = ioutil.readInt(input); + this.wordNgrams = ioutil.readInt(input); + this.loss = LossType.fromValue(ioutil.readInt(input)); + this.model = ModelType.fromValue(ioutil.readInt(input)); + this.bucket = ioutil.readInt(input); + this.minn = ioutil.readInt(input); + this.maxn = ioutil.readInt(input); + this.lrUpdateRate = ioutil.readInt(input); + this.t = ioutil.readDouble(input); + } + + public void parseArgs(String[] args) { + String command = args[0]; + if ("supervised".equalsIgnoreCase(command)) { + this.model = ModelType.sup; + this.loss = LossType.softmax; + this.minCount = 1; + this.minn = 0; + this.maxn = 0; + this.lr = 0.1D; + } + if ("cbow".equalsIgnoreCase(command)) + this.model = ModelType.cbow; + if ("skipgram".equalsIgnoreCase(command)) + this.model = ModelType.sg; + int ai = 1; + while (ai < args.length) { + if (args[ai].charAt(0) != '-') { + System.out.println("Provided argument without a dash! Usage:"); + printHelp(); + System.exit(1); + } + if ("-h".equals(args[ai])) { + System.out.println("Here is the help! Usage:"); + printHelp(); + System.exit(1); + } else if ("-input".equals(args[ai])) { + this.input = args[ai + 1]; + } else if ("-test".equals(args[ai])) { + this.test = args[ai + 1]; + } else if ("-output".equals(args[ai])) { + this.output = args[ai + 1]; + } else if ("-lr".equals(args[ai])) { + this.lr = Double.parseDouble(args[ai + 1]); + } else if ("-lrUpdateRate".equals(args[ai])) { + this.lrUpdateRate = Integer.parseInt(args[ai + 1]); + } else if ("-dim".equals(args[ai])) { + this.dim = Integer.parseInt(args[ai + 1]); + } else if ("-ws".equals(args[ai])) { + this.ws = Integer.parseInt(args[ai + 1]); + } else if ("-epoch".equals(args[ai])) { + this.epoch = Integer.parseInt(args[ai + 1]); + } else if ("-minCount".equals(args[ai])) { + this.minCount = Integer.parseInt(args[ai + 1]); + } else if ("-minCountLabel".equals(args[ai])) { + this.minCountLabel = Integer.parseInt(args[ai + 1]); + } else if ("-neg".equals(args[ai])) { + this.neg = Integer.parseInt(args[ai + 1]); + } else if ("-wordNgrams".equals(args[ai])) { + this.wordNgrams = Integer.parseInt(args[ai + 1]); + } else if ("-loss".equals(args[ai])) { + if ("hs".equalsIgnoreCase(args[ai + 1])) { + this.loss = LossType.hs; + } else if ("ns".equalsIgnoreCase(args[ai + 1])) { + this.loss = LossType.ns; + } else if ("softmax".equalsIgnoreCase(args[ai + 1])) { + this.loss = LossType.softmax; + } else { + System.out.println("Unknown loss: " + args[ai + 1]); + printHelp(); + System.exit(1); + } + } else if ("-bucket".equals(args[ai])) { + this.bucket = Integer.parseInt(args[ai + 1]); + } else if ("-minn".equals(args[ai])) { + this.minn = Integer.parseInt(args[ai + 1]); + } else if ("-maxn".equals(args[ai])) { + this.maxn = Integer.parseInt(args[ai + 1]); + } else if ("-thread".equals(args[ai])) { + this.thread = Integer.parseInt(args[ai + 1]); + } else if ("-t".equals(args[ai])) { + this.t = Double.parseDouble(args[ai + 1]); + } else if ("-label".equals(args[ai])) { + this.label = args[ai + 1]; + } else if ("-verbose".equals(args[ai])) { + this.verbose = Integer.parseInt(args[ai + 1]); + } else if ("-pretrainedVectors".equals(args[ai])) { + this.pretrainedVectors = args[ai + 1]; + } else { + System.out.println("Unknown argument: " + args[ai]); + printHelp(); + System.exit(1); + } + ai += 2; + } + if (Utils.isEmpty(this.input) || Utils.isEmpty(this.output)) { + System.out.println("Empty input or output path."); + printHelp(); + System.exit(1); + } + if (this.wordNgrams <= 1 && this.maxn == 0) + this.bucket = 0; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Args [input="); + builder.append(this.input); + builder.append(", output="); + builder.append(this.output); + builder.append(", test="); + builder.append(this.test); + builder.append(", lr="); + builder.append(this.lr); + builder.append(", lrUpdateRate="); + builder.append(this.lrUpdateRate); + builder.append(", dim="); + builder.append(this.dim); + builder.append(", ws="); + builder.append(this.ws); + builder.append(", epoch="); + builder.append(this.epoch); + builder.append(", minCount="); + builder.append(this.minCount); + builder.append(", minCountLabel="); + builder.append(this.minCountLabel); + builder.append(", neg="); + builder.append(this.neg); + builder.append(", wordNgrams="); + builder.append(this.wordNgrams); + builder.append(", loss="); + builder.append(this.loss); + builder.append(", model="); + builder.append(this.model); + builder.append(", bucket="); + builder.append(this.bucket); + builder.append(", minn="); + builder.append(this.minn); + builder.append(", maxn="); + builder.append(this.maxn); + builder.append(", thread="); + builder.append(this.thread); + builder.append(", t="); + builder.append(this.t); + builder.append(", label="); + builder.append(this.label); + builder.append(", verbose="); + builder.append(this.verbose); + builder.append(", pretrainedVectors="); + builder.append(this.pretrainedVectors); + builder.append("]"); + return builder.toString(); + } + + public enum ModelType { + cbow(1), + sg(2), + sup(3); + + private int value; + + public int getValue() { + return this.value; + } + + public static ModelType fromValue(int value) throws IllegalArgumentException { + try { + value--; + return values()[value]; + } catch (ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Unknown model_name enum value :" + value); + } + } + + ModelType(int value) { + this.value = value; + } + } + + public enum LossType { + hs(1), + ns(2), + softmax(3); + + private int value; + + public int getValue() { + return this.value; + } + + public static LossType fromValue(int value) throws IllegalArgumentException { + try { + value--; + return values()[value]; + } catch (ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Unknown loss_name enum value :" + value); + } + } + + LossType(int value) { + this.value = value; + } + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/Dictionary.java b/src/main/java/ai/searchbox/FastText4J/Dictionary.java new file mode 100644 index 0000000..4adf706 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Dictionary.java @@ -0,0 +1,508 @@ +package ai.searchbox.FastText4J; + +import ai.searchbox.FastText4J.io.IOUtil; +import ai.searchbox.FastText4J.io.LineReader; +import ai.searchbox.FastText4J.io.MappedByteBufferLineReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.log4j.Logger; + +public class Dictionary { + private static final Logger logger = Logger.getLogger(Dictionary.class.getName()); + + private static final int MAX_VOCAB_SIZE = 30000000; + + private static final int MAX_LINE_SIZE = 1024; + + private static final Integer WORDID_DEFAULT = Integer.valueOf(-1); + + private static final String EOS = ""; + + private static final String BOW = "<"; + + private static final String EOW = ">"; + + private int size = 0; + + private int nwords = 0; + + private long ntokens = 0L; + + private int nlabels = 0; + + protected long pruneIdxSize = -1L; + + Map pruneIdx; + + private Args args_; + + private List words; + + private Map word2int; + + private List pdiscard; + + private String charsetName_ = "UTF-8"; + + private Class lineReaderClass_ = (Class)MappedByteBufferLineReader.class; + + private transient Comparator entry_comparator; + + public List getWords() { + return this.words; + } + + public int nwords() { + return this.nwords; + } + + public int nlabels() { + return this.nlabels; + } + + public long ntokens() { + return this.ntokens; + } + + public Map getWord2int() { + return this.word2int; + } + + public List getPdiscard() { + return this.pdiscard; + } + + public int getSize() { + return this.size; + } + + public Args getArgs() { + return this.args_; + } + + public boolean isPruned() { + return (this.pruneIdxSize >= 0L); + } + + public String getCharsetName() { + return this.charsetName_; + } + + public void setCharsetName(String charsetName) { + this.charsetName_ = charsetName; + } + + public Class getLineReaderClass() { + return this.lineReaderClass_; + } + + public void setLineReaderClass(Class lineReaderClass) { + this.lineReaderClass_ = lineReaderClass; + } + + public int getId(String w) { + long h = find(w); + return ((Integer)Utils.mapGetOrDefault(this.word2int, Long.valueOf(h), WORDID_DEFAULT)).intValue(); + } + + public EntryType getType(int id) { + Utils.checkArgument((id >= 0)); + Utils.checkArgument((id < this.size)); + return ((Entry)this.words.get(id)).type; + } + + public String getWord(int id) { + Utils.checkArgument((id >= 0)); + Utils.checkArgument((id < this.size)); + return ((Entry)this.words.get(id)).word; + } + + public String getLabel(int lid) { + Utils.checkArgument((lid >= 0)); + Utils.checkArgument((lid < this.nlabels)); + return ((Entry)this.words.get(lid + this.nwords)).word; + } + + public long find(String w) { + long h = hash(w) % 30000000L; + Entry e = null; + while (Utils.mapGetOrDefault(this.word2int, Long.valueOf(h), WORDID_DEFAULT) != WORDID_DEFAULT && ( + e = this.words.get(((Integer)this.word2int.get(Long.valueOf(h))).intValue())) != null && + !w.equals(e.word)) + h = (h + 1L) % 30000000L; + return h; + } + + public void add(String w) { + long h = find(w); + if (Utils.mapGetOrDefault(this.word2int, Long.valueOf(h), WORDID_DEFAULT) == WORDID_DEFAULT) { + Entry e = new Entry(); + e.word = w; + e.count = 1L; + e.type = w.startsWith(this.args_.label) ? EntryType.label : EntryType.word; + this.words.add(e); + this.word2int.put(Long.valueOf(h), Integer.valueOf(this.size++)); + } else { + ((Entry)this.words.get(((Integer)this.word2int.get(Long.valueOf(h))).intValue())).count++; + } + this.ntokens++; + } + + public final List getNgrams(String word) { + if (!word.equals("")) + return computeNgrams("<" + word + ">"); + int id = getId(word); + if (id != WORDID_DEFAULT.intValue()) + return getNgrams(id); + return new ArrayList<>(); + } + + public final List getNgrams(int i) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < this.nwords)); + return ((Entry)this.words.get(i)).subwords; + } + + public void addNgrams(List line, int n) { + Utils.checkArgument((n > 0)); + int line_size = line.size(); + for (int i = 0; i < line_size; i++) { + BigInteger h = BigInteger.valueOf(((Integer)line.get(i)).intValue()); + BigInteger r = BigInteger.valueOf(116049371L); + BigInteger b = BigInteger.valueOf(this.args_.bucket); + for (int j = i + 1; j < line_size && j < i + n; j++) { + h = h.multiply(r).add(BigInteger.valueOf(((Integer)line.get(j)).intValue())); + line.add(Integer.valueOf(this.nwords + h.remainder(b).intValue())); + } + } + } + + public int getLine(String[] tokens, List words, List labels, Random urd) { + words.clear(); + labels.clear(); + int ntokens = 0; + if (tokens != null) + for (int i = 0; i <= tokens.length; i++) { + if (i >= tokens.length || !Utils.isEmpty(tokens[i])) { + int wid = (i == tokens.length) ? getId("") : getId(tokens[i]); + if (wid >= 0) { + ntokens++; + EntryType type = getType(wid); + if (type == EntryType.word && !discard(wid, Utils.randomFloat(urd, 0.0F, 1.0F))) + words.add(Integer.valueOf(wid)); + if (type == EntryType.label) + labels.add(Integer.valueOf(wid - this.nwords)); + if (words.size() > 1024 && this.args_.model != Args.ModelType.sup) + break; + } + } + } + return ntokens; + } + + public List countType(EntryType type) { + int size = (EntryType.label == type) ? nlabels() : nwords(); + List counts = new ArrayList<>(size); + for (Entry w : this.words) { + if (w.type == type) + counts.add(Long.valueOf(w.count)); + } + return counts; + } + + public void readFromFile(String file) throws Exception { + Exception exception1 = null, exception2 = null; + try { + LineReader lineReader = this.lineReaderClass_ + .getConstructor(new Class[] { String.class, String.class }).newInstance(new Object[] { file, this.charsetName_ }); + try { + long minThreshold = 1L; + String[] lineTokens; + while ((lineTokens = lineReader.readLineTokens()) != null) { + for (int i = 0; i <= lineTokens.length; i++) { + if (i == lineTokens.length) { + add(""); + } else { + if (Utils.isEmpty(lineTokens[i])) + continue; + add(lineTokens[i]); + } + if (this.size > 2.25E7D) { + minThreshold++; + threshold(minThreshold, minThreshold); + } + if (this.ntokens % 1000000L == 0L && this.args_.verbose > 1) + System.out.printf("\rRead %dM words", new Object[] { Long.valueOf(this.ntokens / 1000000L) }); + continue; + } + } + } finally { + if (lineReader != null) + lineReader.close(); + } + } finally { + exception2 = null; + if (exception1 == null) { + exception1 = exception2; + } else if (exception1 != exception2) { + exception1.addSuppressed(exception2); + } + } + if (Args.ModelType.cbow == this.args_.model || Args.ModelType.sg == this.args_.model) + initNgrams(); + if (this.args_.verbose > 0) { + System.out.printf("\rRead %dM words\n", new Object[] { Long.valueOf(this.ntokens / 1000000L) }); + System.out.println("Number of words: " + this.nwords); + System.out.println("Number of labels: " + this.nlabels); + } + if (this.size == 0) + throw new Exception("Empty vocabulary. Try a smaller -minCount value."); + } + + public void load(InputStream is) throws IOException { + IOUtil io = new IOUtil(); + this.size = io.readInt(is); + this.nwords = io.readInt(is); + this.nlabels = io.readInt(is); + this.ntokens = io.readLong(is); + this.pruneIdxSize = io.readLong(is); + this.words = new ArrayList<>(this.size); + this.word2int = new HashMap<>(this.size); + int i; + for (i = 0; i < this.size; i++) { + Entry e = new Entry(); + e.word = io.readString(is); + e.count = io.readLong(is); + e.type = EntryType.fromValue(io.readByteAsInt(is)); + this.words.add(e); + this.word2int.put(Long.valueOf(find(e.word)), Integer.valueOf(i)); + } + this.pruneIdx = new HashMap<>((int)Math.max(0L, this.pruneIdxSize)); + if (this.pruneIdxSize > 0L) + for (i = 0; i < this.pruneIdxSize; i++) { + int first = io.readInt(is); + int second = io.readInt(is); + this.pruneIdx.put(Integer.valueOf(first), Integer.valueOf(second)); + } + initTableDiscard(); + initNgrams(); + } + + public void save(OutputStream ofs) throws IOException { + IOUtil io = new IOUtil(); + ofs.write(io.intToByteArray(this.size)); + ofs.write(io.intToByteArray(this.nwords)); + ofs.write(io.intToByteArray(this.nlabels)); + ofs.write(io.longToByteArray(this.ntokens)); + ofs.write(io.longToByteArray(this.pruneIdxSize)); + for (int i = 0; i < this.size; i++) { + Entry e = this.words.get(i); + ofs.write(e.word.getBytes()); + ofs.write(0); + ofs.write(io.longToByteArray(e.count)); + ofs.write(io.intToByte(e.type.value)); + } + } + + public void threshold(long t, long tl) { + Collections.sort(this.words, this.entry_comparator); + Iterator iterator = this.words.iterator(); + while (iterator.hasNext()) { + Entry _entry = iterator.next(); + if ((EntryType.word == _entry.type && _entry.count < t) || ( + EntryType.label == _entry.type && _entry.count < tl)) + iterator.remove(); + } + ((ArrayList)this.words).trimToSize(); + this.size = 0; + this.nwords = 0; + this.nlabels = 0; + this.word2int = new HashMap<>(this.words.size()); + for (Entry _entry : this.words) { + long h = find(_entry.word); + this.word2int.put(Long.valueOf(h), Integer.valueOf(this.size++)); + if (EntryType.word == _entry.type) { + this.nwords++; + continue; + } + this.nlabels++; + } + } + + public long hash(String str) { + int h = -2128831035; + byte b; + int i; + byte[] arrayOfByte; + for (i = (arrayOfByte = str.getBytes()).length, b = 0; b < i; ) { + byte strByte = arrayOfByte[b]; + h = (h ^ strByte) * 16777619; + b++; + } + return h & 0xFFFFFFFFL; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Dictionary [words_="); + builder.append(this.words); + builder.append(", pdiscard_="); + builder.append(this.pdiscard); + builder.append(", word2int_="); + builder.append(this.word2int); + builder.append(", size_="); + builder.append(this.size); + builder.append(", nwords_="); + builder.append(this.nwords); + builder.append(", nlabels_="); + builder.append(this.nlabels); + builder.append(", ntokens_="); + builder.append(this.ntokens); + builder.append("]"); + return builder.toString(); + } + + private void initTableDiscard() { + this.pdiscard = new ArrayList<>(this.size); + for (int i = 0; i < this.size; i++) { + float f = (float)((Entry)this.words.get(i)).count / (float)this.ntokens; + this.pdiscard.add(Float.valueOf((float)(Math.sqrt(this.args_.t / f) + this.args_.t / f))); + } + } + + private void initNgrams() { + for (int i = 0; i < this.size; i++) { + String word = "<" + ((Entry)this.words.get(i)).word + ">"; + Entry e = this.words.get(i); + e.subwords = new ArrayList<>(); + if (!((Entry)this.words.get(i)).word.equals("")) + e.subwords = computeNgrams(word); + e.subwords.add(Integer.valueOf(i)); + } + } + + private boolean charMatches(char ch) { + return !(ch != ' ' && ch != '\t' && ch != '\n' && ch != '\f' && ch != '\r'); + } + + private boolean discard(int id, float rand) { + Utils.checkArgument((id >= 0)); + Utils.checkArgument((id < this.nwords)); + return (this.args_.model == Args.ModelType.sup) ? false : ((rand > ((Float)this.pdiscard.get(id)).floatValue())); + } + + private List computeNgrams(String word) { + List ngrams = new ArrayList<>(); + if (word.equals("")) + return ngrams; + for (int i = 0; i < word.length(); i++) { + StringBuilder ngram = new StringBuilder(); + if (!charMatches(word.charAt(i))) + for (int j = i, n = 1; j < word.length() && n <= this.args_.maxn; n++) { + ngram.append(word.charAt(j++)); + while (j < word.length() && charMatches(word.charAt(j))) + ngram.append(word.charAt(j++)); + if (n >= this.args_.minn && (n != 1 || (i != 0 && j != word.length()))) { + int h = (int)(this.nwords + hash(ngram.toString()) % this.args_.bucket); + if (h < 0) + logger.error("computeNgrams h<0: " + h + " on word: " + word); + pushHash(ngrams, h); + } + } + } + return ngrams; + } + + private void pushHash(List hashes, int id) { + if (this.pruneIdxSize == 0L || id < 0) + return; + if (this.pruneIdxSize > 0L) { + int pruneId = getPruning(id); + if (pruneId >= 0) { + id = pruneId; + } else { + return; + } + } + hashes.add(Integer.valueOf(id)); + } + + private int getPruning(int id) { + return ((Integer)this.pruneIdx.getOrDefault(Integer.valueOf(id), Integer.valueOf(-1))).intValue(); + } + + public Dictionary(Args args) { + this.entry_comparator = new Comparator() { + public int compare(Dictionary.Entry o1, Dictionary.Entry o2) { + int cmp = (o1.type.value < o2.type.value) ? -1 : ((o1.type.value == o2.type.value) ? 0 : 1); + if (cmp == 0) + cmp = (o2.count < o1.count) ? -1 : ((o2.count == o1.count) ? 0 : 1); + return cmp; + } + }; + this.args_ = args; + this.words = new ArrayList<>(30000000); + this.word2int = new HashMap<>(30000000); + } + + public enum EntryType { + word(0), + label(1); + + private int value; + + public int getValue() { + return this.value; + } + + public static EntryType fromValue(int value) throws IllegalArgumentException { + try { + return values()[value]; + } catch (ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Unknown entry_type enum value :" + value); + } + } + + public String toString() { + return (this.value == 0) ? "word" : ((this.value == 1) ? "label" : "unknown"); + } + + EntryType(int value) { + this.value = value; + } + } + + public class Entry { + public String word; + + public Dictionary.EntryType type; + + public long count; + + public List subwords; + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("entry [word="); + builder.append(this.word); + builder.append(", count="); + builder.append(this.count); + builder.append(", type="); + builder.append(this.type); + builder.append(", subwords="); + builder.append(this.subwords); + builder.append("]"); + return builder.toString(); + } + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/FastText.java b/src/main/java/ai/searchbox/FastText4J/FastText.java new file mode 100644 index 0000000..de3a365 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/FastText.java @@ -0,0 +1,460 @@ +package ai.searchbox.FastText4J; + +import ai.searchbox.FastText4J.io.LineReader; +import ai.searchbox.FastText4J.io.MappedByteBufferLineReader; +import ai.searchbox.FastText4J.math.Matrix; +import ai.searchbox.FastText4J.math.Vector; +import com.google.common.collect.MinMaxPriorityQueue; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.log4j.Logger; + +public class FastText { + private static final Logger logger = Logger.getLogger(FastText.class.getName()); + + public static int FASTTEXT_VERSION = 12; + + public static int FASTTEXT_FILEFORMAT_MAGIC_INT = 793712314; + + private long start_; + + int threadCount; + + long threadFileSize; + + private Args args_; + + private Dictionary dict_; + + private Model model_; + + private Matrix input_; + + private Matrix output_; + + private Matrix wordVectors = null; + + private Matrix wordVectorsOut = null; + + private boolean isQuant = false; + + private AtomicLong tokenCount_; + + private String charsetName_ = "UTF-8"; + + private Class lineReaderClass_ = (Class)MappedByteBufferLineReader.class; + + public Args getArgs() { + return this.args_; + } + + public void setArgs(Args args) { + this.args_ = args; + this.dict_ = new Dictionary(args); + } + + public Dictionary dict() { + return this.dict_; + } + + public Vector getWordVectorIn(String word) { + Vector vec = new Vector(this.args_.dim); + vec.zero(); + List ngrams = this.dict_.getNgrams(word); + for (Integer it : ngrams) + vec.addRow(this.input_, it.intValue()); + if (ngrams.size() > 0) + vec.mul(1.0F / ngrams.size()); + return vec; + } + + public Vector getWordVectorOut(String word) { + int id = this.dict_.getId(word); + Vector vec = new Vector(this.args_.dim); + vec.zero(); + if (this.isQuant) + return vec; + vec.addRow(this.output_, id); + return vec; + } + + public Vector getSentenceVector(List sentence) { + Vector svec = new Vector(this.args_.dim); + svec.zero(); + if (this.args_.model == Args.ModelType.sup) { + List tokens = new ArrayList<>(); + List labels = new ArrayList<>(); + this.dict_.getLine(sentence.toArray(new String[sentence.size()]), tokens, labels, this.model_.rng); + for (int i = 0; i < tokens.size(); i++) + svec.addRow(this.input_, ((Integer)tokens.get(i)).intValue()); + if (!tokens.isEmpty()) + svec.mul(1.0F / tokens.size()); + } else { + int count = 0; + for (String word : sentence) { + Vector vec = getWordVectorIn(word); + svec.addVector(vec); + count++; + } + if (count > 0) + svec.mul(1.0F / count); + } + return svec; + } + + public Vector getSentenceVectorOut(List sentence) { + Vector svec = new Vector(this.args_.dim); + svec.zero(); + int count = 0; + for (String word : sentence) { + Vector vec = getWordVectorOut(word); + svec.addVector(vec); + count++; + } + if (count > 0) + svec.mul(1.0F / count); + return svec; + } + + public List> predict(String[] lineTokens, int k) { + List> predictions = new ArrayList<>(); + List words = new ArrayList<>(); + List labels = new ArrayList<>(); + this.dict_.getLine(lineTokens, words, labels, this.model_.rng); + this.dict_.addNgrams(words, this.args_.wordNgrams); + if (words.isEmpty()) + return predictions; + List> modelPredictions = new ArrayList<>(k + 1); + this.model_.predict(words, k, modelPredictions); + for (Pair pair : modelPredictions) + predictions.add(new Pair<>(pair.getKey(), this.dict_.getLabel(((Integer)pair.getValue()).intValue()))); + return predictions; + } + + public List findNN(Vector queryVec, int k, Set banSet) { + return findNN(this.wordVectors, queryVec, k, banSet); + } + + public List findNNOut(Vector queryVec, int k, Set banSet) { + return findNN(this.wordVectorsOut, queryVec, k, banSet); + } + + public List findNN(Matrix wordVectors, Vector queryVec, int k, Set banSet) { + MinMaxPriorityQueue> heap = + MinMaxPriorityQueue.orderedBy(new HeapComparator()) + .expectedSize(this.dict_.nlabels()) + .create(); + float queryNorm = queryVec.norm(); + if (queryNorm > 0.0F) + queryVec.mul(1.0F / queryNorm); + for (int i = 0; i < this.dict_.nwords(); i++) { + String word = this.dict_.getWord(i); + float dp = wordVectors.dotRow(queryVec, i); + heap.add(new Pair<>(Float.valueOf(dp), word)); + } + List syns = new ArrayList<>(); + int j = 0; + while (j < k && heap.size() > 0) { + Pair synonym = (Pair)heap.pollFirst(); + boolean banned = banSet.contains(synonym.getValue()); + if (!banned) { + syns.add(new FastTextSynonym(synonym.getValue(), ((Float)synonym.getKey()).floatValue())); + j++; + } + } + return syns; + } + + public void saveModel() throws IOException { + if (Utils.isEmpty(this.args_.output)) { + if (this.args_.verbose > 1) + System.out.println("output is empty, skip save model file"); + return; + } + File file = new File(String.valueOf(this.args_.output) + ".bin"); + if (file.exists()) + file.delete(); + if (file.getParentFile() != null) + file.getParentFile().mkdirs(); + if (this.args_.verbose > 1) + System.out.println("Saving model to " + file.getCanonicalPath().toString()); + Exception exception1 = null, exception2 = null; + } + + public void loadModel(String filename) throws IOException { + logger.info("Loading " + filename); + File file = new File(filename); + if (!file.exists() || !file.isFile() || !file.canRead()) + throw new IOException("Model file cannot be opened for loading!"); + Exception exception1 = null, exception2 = null; + } + + public void train() throws Exception { + if ("-".equals(this.args_.input)) + throw new IOException("Cannot use stdin for training!"); + File file = new File(this.args_.input); + if (!file.exists() || !file.isFile() || !file.canRead()) + throw new IOException("Input file cannot be opened! " + this.args_.input); + logger.debug("Building dict"); + this.dict_ = new Dictionary(this.args_); + this.dict_.setCharsetName(this.charsetName_); + this.dict_.setLineReaderClass(this.lineReaderClass_); + this.dict_.readFromFile(this.args_.input); + logger.debug("Building input matrix"); + if (!Utils.isEmpty(this.args_.pretrainedVectors)) { + loadVecFile(); + } else { + this.input_ = new Matrix(this.dict_.nwords() + this.args_.bucket, this.args_.dim); + this.input_.uniform(1.0F / this.args_.dim); + } + logger.debug("Building output matrix"); + int m = (this.args_.model == Args.ModelType.sup) ? this.dict_.nlabels() : this.dict_.nwords(); + this.output_ = new Matrix(m, this.args_.dim); + this.output_.zero(); + this.start_ = System.currentTimeMillis(); + this.tokenCount_ = new AtomicLong(0L); + long t0 = System.currentTimeMillis(); + this.threadFileSize = Utils.sizeLine(this.args_.input); + this.threadCount = this.args_.thread; + for (int i = 0; i < this.args_.thread; i++) { + logger.debug("Spawning training thread"); + Thread t = new TrainThread(this, i); + t.setUncaughtExceptionHandler(this.trainThreadExcpetionHandler); + t.start(); + } + synchronized (this) { + while (this.threadCount > 0) { + try { + wait(); + } catch (InterruptedException interruptedException) {} + } + } + this.model_ = new Model(this.input_, this.output_, this.args_, 0); + if (this.args_.verbose > 1) { + long trainTime = (System.currentTimeMillis() - t0) / 1000L; + System.out.printf("\nTrain time used: %d sec\n", new Object[] { Long.valueOf(trainTime) }); + } + logger.debug("Saving fasttext"); + saveModel(); + if (this.args_.model != Args.ModelType.sup) + saveVecFile(); + } + + public void test(InputStream in, int k) throws IOException, Exception { + int nexamples = 0, nlabels = 0; + double precision = 0.0D; + List line = new ArrayList<>(); + List labels = new ArrayList<>(); + LineReader lineReader = null; + try { + lineReader = this.lineReaderClass_.getConstructor(new Class[] { InputStream.class, String.class }).newInstance(new Object[] { in, this.charsetName_ }); + String[] lineTokens; + while ((lineTokens = lineReader.readLineTokens()) != null && ( + lineTokens.length != 1 || !"quit".equals(lineTokens[0]))) { + this.dict_.getLine(lineTokens, line, labels, this.model_.rng); + this.dict_.addNgrams(line, this.args_.wordNgrams); + if (labels.size() > 0 && line.size() > 0) { + List> modelPredictions = new ArrayList<>(); + this.model_.predict(line, k, modelPredictions); + for (Pair pair : modelPredictions) { + if (labels.contains(pair.getValue())) + precision++; + } + nexamples++; + nlabels += labels.size(); + } + } + } finally { + if (lineReader != null) + lineReader.close(); + } + System.out.printf("P@%d: %.3f%n", new Object[] { Integer.valueOf(k), Double.valueOf(precision / (k * nexamples)) }); + System.out.printf("R@%d: %.3f%n", new Object[] { Integer.valueOf(k), Double.valueOf(precision / nlabels) }); + System.out.println("Number of examples: " + nexamples); + } + + void cbow(Model model, float lr, List line) { + List bow = new ArrayList<>(); + for (int w = 0; w < line.size(); w++) { + bow.clear(); + int boundary = Utils.randomInt(model.rng, 1, this.args_.ws); + for (int c = -boundary; c <= boundary; c++) { + if (c != 0 && w + c >= 0 && w + c < line.size()) { + List ngrams = this.dict_.getNgrams(((Integer)line.get(w + c)).intValue()); + bow.addAll(ngrams); + } + } + model.update(bow, ((Integer)line.get(w)).intValue(), lr); + } + } + + void skipgram(Model model, float lr, List line) { + for (int w = 0; w < line.size(); w++) { + int boundary = Utils.randomInt(model.rng, 1, this.args_.ws); + List ngrams = this.dict_.getNgrams(((Integer)line.get(w)).intValue()); + for (int c = -boundary; c <= boundary; c++) { + if (c != 0 && w + c >= 0 && w + c < line.size()) + model.update(ngrams, ((Integer)line.get(w + c)).intValue(), lr); + } + } + } + + void supervised(Model model, float lr, List line, List labels) { + if (labels.size() == 0 || line.size() == 0) + return; + int i = Utils.randomInt(model.rng, 1, labels.size()) - 1; + model.update(line, ((Integer)labels.get(i)).intValue(), lr); + } + + void checkModel(int magic, int version) { + if (magic != FASTTEXT_FILEFORMAT_MAGIC_INT) + throw new IllegalArgumentException("Unhandled file format"); + if (version > FASTTEXT_VERSION) + throw new IllegalArgumentException( + "Input model version (" + version + ") doesn't match current version (" + FASTTEXT_VERSION + ")"); + } + + public void loadVecFile() throws IOException { + loadVecFile(this.args_.pretrainedVectors); + } + + public void loadVecFile(String path) throws IOException { + Exception exception1 = null, exception2 = null; + } + + public void saveVecFile() throws IOException { + saveVecFile(String.valueOf(this.args_.output) + ".vec", true); + } + + public void saveVecFile(String path, boolean in) throws IOException { + File file = new File(path); + if (file.exists()) + file.delete(); + if (file.getParentFile() != null) + file.getParentFile().mkdirs(); + if (this.args_.verbose > 1) + System.out.println("Saving Vectors to " + file.getCanonicalPath().toString()); + Exception exception1 = null, exception2 = null; + } + + Thread.UncaughtExceptionHandler trainThreadExcpetionHandler = new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread th, Throwable ex) { + ex.printStackTrace(); + } + }; + + private Matrix precomputeWordVectors(boolean in) { + Matrix wordVectors = new Matrix(this.dict_.nwords(), this.args_.dim); + wordVectors.zero(); + for (int i = 0; i < this.dict_.nwords(); i++) { + String word = this.dict_.getWord(i); + try { + Vector vec = in ? getWordVectorIn(word) : getWordVectorOut(word); + float norm = vec.norm(); + if (norm > 0.0F) + wordVectors.addRow(vec, i, 1.0F / norm); + } catch (Exception e) { + logger.error("Failed precomputing word vectors for " + word + " in in:" + in); + } + } + return wordVectors; + } + + public static class HeapComparator implements Comparator> { + public int compare(Pair p1, Pair p2) { + if (((Float)p1.getKey()).equals(p2.getKey())) + return 0; + if (((Float)p1.getKey()).floatValue() < ((Float)p2.getKey()).floatValue()) + return 1; + return -1; + } + } + + public class TrainThread extends Thread { + final FastText ft; + + int threadId; + + public TrainThread(FastText ft, int threadId) { + super("FT-TrainThread-" + threadId); + this.ft = ft; + this.threadId = threadId; + } + + public void run() { + if (FastText.this.args_.verbose > 2) + System.out.println("thread: " + this.threadId + " RUNNING!"); + try { + Exception exception2, exception1 = null; + } catch (Exception e) { + FastText.logger.error(e); + } + synchronized (this.ft) { + if (FastText.this.args_.verbose > 2) + System.out.println("\nthread: " + this.threadId + " EXIT!"); + this.ft.threadCount--; + this.ft.notify(); + } + } + + private void printInfo(float progress, float loss) throws Exception { + float t = (float)(System.currentTimeMillis() - FastText.this.start_) / 1000.0F; + float ws = (float)FastText.this.tokenCount_.get() / t; + float wst = (float)FastText.this.tokenCount_.get() / t / FastText.this.args_.thread; + float lr = (float)(FastText.this.args_.lr * (1.0F - progress)); + int eta = (int)(t / progress * (1.0F - progress)); + int etah = eta / 3600; + int etam = (eta - etah * 3600) / 60; + System.out.printf("\rProgress: %.1f%% words/sec: %d words/sec/thread: %d lr: %.6f loss: %.6f eta: %d h %d m", new Object[] { Float.valueOf(100.0F * progress), Integer.valueOf((int)ws), Integer.valueOf((int)wst), Float.valueOf(lr), Float.valueOf(loss), Integer.valueOf(etah), Integer.valueOf(etam) }); + System.out.println("ss"); + } + } + + public class FastTextSynonym { + private final String word; + + private final double cosineSimilarity; + + public FastTextSynonym(String word, double cosineSimilarity) { + this.word = word; + this.cosineSimilarity = cosineSimilarity; + } + + public String word() { + return this.word; + } + + public double cosineSimilarity() { + return this.cosineSimilarity; + } + } + + public class FastTextPrediction { + private final String label; + + private final double logProbability; + + public FastTextPrediction(String label, double logProbability) { + this.label = label; + this.logProbability = logProbability; + } + + public String label() { + return this.label; + } + + public double logProbability() { + return this.logProbability; + } + + public double probability() { + return Math.exp(this.logProbability); + } + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/Main.java b/src/main/java/ai/searchbox/FastText4J/Main.java new file mode 100644 index 0000000..5dd14e5 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Main.java @@ -0,0 +1,83 @@ +package ai.searchbox.FastText4J; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +public class Main { + public static void printUsage() { + System.out.print("usage: java -jar fasttext.jar \n\nThe commands supported by fasttext are:\n\n supervised train a supervised classifier\n skipgram train a skipgram model\n cbow train a cbow model\n predict predict most likely labels\n predict-prob predict most likely labels with probabilities\n test evaluate a supervised classifier\n"); + } + + public static void printPredictUsage() { + System.out.print("usage: java -jar fasttext.jar predict[-prob] []\n\n model filename\n test data filename (if -, read from stdin)\n (optional; 1 by default) predict top k labels\n"); + } + + public static void printTestUsage() { + System.out.print("usage: java -jar fasttext.jar test []\n\n model filename\n test data filename (if -, read from stdin)\n (optional; 1 by default) predict top k labels\n"); + } + + private void predict(String[] args) throws Exception {} + + private void train(String[] args) throws Exception { + Args a = new Args(); + a.parseArgs(args); + FastText fasttext = new FastText(); + fasttext.setArgs(a); + fasttext.train(); + } + + private void test(String[] args) throws Exception { + int k = 1; + if (args.length == 3) { + k = 1; + } else if (args.length == 4) { + k = Integer.parseInt(args[3]); + } else { + printTestUsage(); + System.exit(1); + } + FastText fasttext = new FastText(); + fasttext.loadModel(args[1]); + String infile = args[2]; + if ("-".equals(infile)) { + fasttext.test(System.in, k); + } else { + File file = new File(infile); + if (!file.exists() || !file.isFile() || !file.canRead()) + throw new IOException("Test file cannot be opened!"); + fasttext.test(new FileInputStream(file), k); + } + } + + public static void main(String[] args) { + args = new String[] { "skipgram", + "-input", "/Users/davidgortega/Projects/tmp/fastText-0.9.1/data/file9short", + "-output", "/Users/davidgortega/Projects/tmp/fastText-0.9.1/result/fil9Java", + "-thread", "8" }; + Main op = new Main(); + if (args.length == 0) { + printUsage(); + System.exit(1); + } + try { + String command = args[0]; + if ("predict".equalsIgnoreCase(command) || "predict-prob".equalsIgnoreCase(command)) { + op.predict(args); + } else if ("skipgram".equalsIgnoreCase(command) || + "cbow".equalsIgnoreCase(command) || + "supervised".equalsIgnoreCase(command)) { + op.train(args); + } else if ("test".equalsIgnoreCase(command)) { + op.test(args); + } else { + printUsage(); + System.exit(1); + } + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + System.exit(0); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/Model.java b/src/main/java/ai/searchbox/FastText4J/Model.java new file mode 100644 index 0000000..070e988 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Model.java @@ -0,0 +1,349 @@ +package ai.searchbox.FastText4J; + +import ai.searchbox.FastText4J.math.Matrix; +import ai.searchbox.FastText4J.math.Vector; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +public class Model { + static final int SIGMOID_TABLE_SIZE = 512; + + static final int MAX_SIGMOID = 8; + + static final int LOG_TABLE_SIZE = 512; + + static final int NEGATIVE_TABLE_SIZE = 10000000; + + private Args args_; + + private Matrix wi_; + + private Matrix wo_; + + private Vector hidden_; + + private Vector output_; + + private Vector grad_; + + private int hsz_; + + private int isz_; + + private int osz_; + + private float loss_; + + private long nexamples_; + + private float[] t_sigmoid; + + private float[] t_log; + + private List negatives; + + private int negpos; + + private List> paths; + + private List> codes; + + private List tree; + + public transient Random rng; + + private Comparator> comparePairs; + + public Model(Matrix wi, Matrix wo, Args args, int seed) { + this.comparePairs = new Comparator>() { + public int compare(Pair o1, Pair o2) { + return ((Float)o2.getKey()).compareTo(o1.getKey()); + } + }; + this.negpos = 0; + this.loss_ = 0.0F; + this.nexamples_ = 1L; + this.wi_ = wi; + this.wo_ = wo; + this.args_ = args; + this.isz_ = wi.m; + this.osz_ = wo.m; + this.hsz_ = args.dim; + this.hidden_ = new Vector(args.dim); + this.output_ = new Vector(wo.m); + this.grad_ = new Vector(args.dim); + this.rng = new Random(seed); + initSigmoid(); + initLog(); + } + + public float binaryLogistic(int target, boolean label, float lr) { + float score = sigmoid(this.wo_.dotRow(this.hidden_, target)); + float alpha = lr * ((label ? 1.0F : 0.0F) - score); + this.grad_.addRow(this.wo_, target, alpha); + this.wo_.addRow(this.hidden_, target, alpha); + if (label) + return -log(score); + return -log(1.0F - score); + } + + public float negativeSampling(int target, float lr) { + float loss = 0.0F; + this.grad_.zero(); + for (int n = 0; n <= this.args_.neg; n++) { + if (n == 0) { + loss += binaryLogistic(target, true, lr); + } else { + loss += binaryLogistic(getNegative(target), false, lr); + } + } + return loss; + } + + public float hierarchicalSoftmax(int target, float lr) { + float loss = 0.0F; + this.grad_.zero(); + List binaryCode = this.codes.get(target); + List pathToRoot = this.paths.get(target); + for (int i = 0; i < pathToRoot.size(); i++) + loss += binaryLogistic(((Integer)pathToRoot.get(i)).intValue(), ((Boolean)binaryCode.get(i)).booleanValue(), lr); + return loss; + } + + public float softmax(int target, float lr) { + this.grad_.zero(); + computeOutputSoftmax(); + for (int i = 0; i < this.osz_; i++) { + float label = (i == target) ? 1.0F : 0.0F; + float alpha = lr * (label - this.output_.get(i)); + this.grad_.addRow(this.wo_, i, alpha); + this.wo_.addRow(this.hidden_, i, alpha); + } + return -log(this.output_.get(target)); + } + + public void computeOutputSoftmax() { + computeOutputSoftmax(this.hidden_, this.output_); + } + + public void computeOutputSoftmax(Vector hidden, Vector output) { + output.mul(this.wo_, hidden); + float max = output.get(0), z = 0.0F; + int i; + for (i = 1; i < this.osz_; i++) + max = Math.max(output.get(i), max); + for (i = 0; i < this.osz_; i++) { + output.set(i, (float)Math.exp((output.get(i) - max))); + z += output.get(i); + } + for (i = 0; i < this.osz_; i++) + output.set(i, output.get(i) / z); + } + + public void computeHidden(List input, Vector hidden) { + Utils.checkArgument((hidden.size() == this.hsz_)); + hidden.zero(); + for (Integer it : input) + hidden.addRow(this.wi_, it.intValue()); + hidden.mul(1.0F / input.size()); + } + + public void predict(List input, int k, List> heap, Vector hidden, Vector output) { + Utils.checkArgument((k > 0)); + if (heap instanceof ArrayList) + ((ArrayList)heap).ensureCapacity(k + 1); + computeHidden(input, hidden); + if (this.args_.loss == Args.LossType.hs) { + dfs(k, 2 * this.osz_ - 2, 0.0F, heap, hidden); + } else { + findKBest(k, heap, hidden, output); + } + Collections.sort(heap, this.comparePairs); + } + + public void predict(List input, int k, List> heap) { + predict(input, k, heap, this.hidden_, this.output_); + } + + public void findKBest(int k, List> heap, Vector hidden, Vector output) { + computeOutputSoftmax(hidden, output); + for (int i = 0; i < this.osz_; i++) { + if (heap.size() != k || log(output.get(i)) >= ((Float)((Pair)heap.get(heap.size() - 1)).getKey()).floatValue()) { + heap.add(new Pair<>(Float.valueOf(log(output.get(i))), Integer.valueOf(i))); + Collections.sort(heap, this.comparePairs); + if (heap.size() > k) { + Collections.sort(heap, this.comparePairs); + heap.remove(heap.size() - 1); + } + } + } + } + + public void dfs(int k, int node, float score, List> heap, Vector hidden) { + if (heap.size() == k && score < ((Float)((Pair)heap.get(heap.size() - 1)).getKey()).floatValue()) + return; + if (((Node)this.tree.get(node)).left == -1 && ((Node)this.tree.get(node)).right == -1) { + heap.add(new Pair<>(Float.valueOf(score), Integer.valueOf(node))); + Collections.sort(heap, this.comparePairs); + if (heap.size() > k) { + Collections.sort(heap, this.comparePairs); + heap.remove(heap.size() - 1); + } + return; + } + float f = sigmoid(this.wo_.dotRow(hidden, node - this.osz_)); + dfs(k, ((Node)this.tree.get(node)).left, score + log(1.0F - f), heap, hidden); + dfs(k, ((Node)this.tree.get(node)).right, score + log(f), heap, hidden); + } + + public void update(List input, int target, float lr) { + Utils.checkArgument((target >= 0)); + Utils.checkArgument((target < this.osz_)); + if (input.size() == 0) + return; + computeHidden(input, this.hidden_); + if (this.args_.loss == Args.LossType.ns) { + this.loss_ += negativeSampling(target, lr); + } else if (this.args_.loss == Args.LossType.hs) { + this.loss_ += hierarchicalSoftmax(target, lr); + } else { + this.loss_ += softmax(target, lr); + } + this.nexamples_++; + if (this.args_.model == Args.ModelType.sup) + this.grad_.mul(1.0F / input.size()); + for (Integer it : input) + this.wi_.addRow(this.grad_, it.intValue(), 1.0F); + } + + public void setTargetCounts(List counts) { + Utils.checkArgument((counts.size() == this.osz_)); + if (this.args_.loss == Args.LossType.ns) + initTableNegatives(counts); + if (this.args_.loss == Args.LossType.hs) + buildTree(counts); + } + + public void initTableNegatives(List counts) { + this.negatives = new ArrayList<>(counts.size()); + float z = 0.0F; + int i; + for (i = 0; i < counts.size(); i++) + z += (float)Math.pow(((Long)counts.get(i)).longValue(), 0.5D); + for (i = 0; i < counts.size(); i++) { + float c = (float)Math.pow(((Long)counts.get(i)).longValue(), 0.5D); + for (int j = 0; j < c * 1.0E7F / z; j++) + this.negatives.add(Integer.valueOf(i)); + } + Utils.shuffle(this.negatives, this.rng); + } + + public int getNegative(int target) { + while (true) { + int negative = ((Integer)this.negatives.get(this.negpos)).intValue(); + this.negpos = (this.negpos + 1) % this.negatives.size(); + if (target != negative) + return negative; + } + } + + public void buildTree(List counts) { + this.paths = new ArrayList<>(this.osz_); + this.codes = new ArrayList<>(this.osz_); + this.tree = new ArrayList<>(2 * this.osz_ - 1); + int i; + for (i = 0; i < 2 * this.osz_ - 1; i++) { + Node node1 = new Node(); + node1.parent = -1; + node1.left = -1; + node1.right = -1; + node1.count = 1000000000000000L; + node1.binary = false; + this.tree.add(i, node1); + } + for (i = 0; i < this.osz_; i++) + ((Node)this.tree.get(i)).count = ((Long)counts.get(i)).longValue(); + int leaf = this.osz_ - 1; + int node = this.osz_; + int j; + for (j = this.osz_; j < 2 * this.osz_ - 1; j++) { + int[] mini = new int[2]; + for (int k = 0; k < 2; k++) { + if (leaf >= 0 && ((Node)this.tree.get(leaf)).count < ((Node)this.tree.get(node)).count) { + mini[k] = leaf--; + } else { + mini[k] = node++; + } + } + ((Node)this.tree.get(j)).left = mini[0]; + ((Node)this.tree.get(j)).right = mini[1]; + ((Node)this.tree.get(mini[0])).count += ((Node)this.tree.get(mini[1])).count; + ((Node)this.tree.get(mini[0])).parent = j; + ((Node)this.tree.get(mini[1])).parent = j; + ((Node)this.tree.get(mini[1])).binary = true; + } + for (j = 0; j < this.osz_; j++) { + List path = new ArrayList<>(); + List code = new ArrayList<>(); + int k = j; + while (((Node)this.tree.get(k)).parent != -1) { + path.add(Integer.valueOf(((Node)this.tree.get(k)).parent - this.osz_)); + code.add(Boolean.valueOf(((Node)this.tree.get(k)).binary)); + k = ((Node)this.tree.get(k)).parent; + } + this.paths.add(path); + this.codes.add(code); + } + } + + public float getLoss() { + return this.loss_ / (float)this.nexamples_; + } + + public float log(float x) { + if (x > 1.0F) + return 0.0F; + int i = (int)(x * 512.0F); + return this.t_log[i]; + } + + public float sigmoid(float x) { + if (x < -8.0F) + return 0.0F; + if (x > 8.0F) + return 1.0F; + int i = (int)((x + 8.0F) * 512.0F / 8.0F / 2.0F); + return this.t_sigmoid[i]; + } + + private void initSigmoid() { + this.t_sigmoid = new float[513]; + for (int i = 0; i < 513; i++) { + float x = (i * 2 * 8) / 512.0F - 8.0F; + this.t_sigmoid[i] = (float)(1.0D / (1.0D + Math.exp(-x))); + } + } + + private void initLog() { + this.t_log = new float[513]; + for (int i = 0; i < 513; i++) { + float x = (i + 1.0E-5F) / 512.0F; + this.t_log[i] = (float)Math.log(x); + } + } + + public class Node { + int parent; + + int left; + + int right; + + long count; + + boolean binary; + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/Pair.java b/src/main/java/ai/searchbox/FastText4J/Pair.java new file mode 100644 index 0000000..9abe0b2 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Pair.java @@ -0,0 +1,28 @@ +package ai.searchbox.FastText4J; + +public class Pair { + private K key_; + + private V value_; + + public Pair(K key, V value) { + this.key_ = key; + this.value_ = value; + } + + public K getKey() { + return this.key_; + } + + public V getValue() { + return this.value_; + } + + public void setKey(K key) { + this.key_ = key; + } + + public void setValue(V value) { + this.value_ = value; + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/Utils.java b/src/main/java/ai/searchbox/FastText4J/Utils.java new file mode 100644 index 0000000..8a1c61d --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/Utils.java @@ -0,0 +1,80 @@ +package ai.searchbox.FastText4J; + +import java.io.IOException; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Random; + +public class Utils { + private static final int SHUFFLE_THRESHOLD = 5; + + public static void checkArgument(boolean expression) { + if (!expression) + throw new IllegalArgumentException(); + } + + public static boolean isEmpty(String str) { + return !(str != null && !str.isEmpty()); + } + + public static V mapGetOrDefault(Map map, K key, V defaultValue) { + return map.containsKey(key) ? map.get(key) : defaultValue; + } + + public static int randomInt(Random rnd, int lower, int upper) { + checkArgument(((lower <= upper)) & ((lower > 0))); + if (lower == upper) + return lower; + return rnd.nextInt(upper - lower) + lower; + } + + public static float randomFloat(Random rnd, float lower, float upper) { + checkArgument((lower <= upper)); + if (lower == upper) + return lower; + return rnd.nextFloat() * (upper - lower) + lower; + } + + public static long sizeLine(String filename) throws IOException { + Exception exception1 = null, exception2 = null; + try { + + } finally { + exception2 = null; + if (exception1 == null) { + exception1 = exception2; + } else if (exception1 != exception2) { + exception1.addSuppressed(exception2); + } + } + } + + public static void shuffle(List list, Random rnd) { + int size = list.size(); + if (size < 5 || list instanceof java.util.RandomAccess) { + for (int i = size; i > 1; i--) + swap(list, i - 1, rnd.nextInt(i)); + } else { + Object[] arr = list.toArray(); + for (int i = size; i > 1; i--) + swap(arr, i - 1, rnd.nextInt(i)); + ListIterator it = list.listIterator(); + for (int j = 0; j < arr.length; j++) { + it.next(); + it.set(arr[j]); + } + } + } + + public static void swap(Object[] arr, int i, int j) { + Object tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; + } + + public static void swap(List list, int i, int j) { + List l = list; + l.set(i, l.set(j, l.get(i))); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/io/BufferedLineReader.java b/src/main/java/ai/searchbox/FastText4J/io/BufferedLineReader.java new file mode 100644 index 0000000..5c15e74 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/io/BufferedLineReader.java @@ -0,0 +1,122 @@ +package ai.searchbox.FastText4J.io; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import org.apache.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + +public class BufferedLineReader extends LineReader { + private static final Logger logger = Logger.getLogger(BufferedLineReader.class.getName()); + + private String lineDelimitingRegex_ = " |\r|\t|\\v|\f|\000"; + + private BufferedReader br_; + + public BufferedLineReader(String filename, String charsetName) throws IOException, UnsupportedEncodingException { + super(filename, charsetName); + FileInputStream fis = new FileInputStream(this.file_); + this.br_ = new BufferedReader(new InputStreamReader(fis, this.charset_)); + } + + public BufferedLineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { + super(inputStream, charsetName); + this.br_ = new BufferedReader(new InputStreamReader(inputStream, this.charset_)); + } + + public long skipLine(long n) throws IOException { + if (n < 0L) + throw new IllegalArgumentException("skip value is negative"); + long currentLine = 0L; + long readLine = 0L; + synchronized (this.lock) { + String line; + while (currentLine < n && (line = this.br_.readLine()) != null) { + readLine++; + if (line == null || line.isEmpty() || line.startsWith("#")) + continue; + currentLine++; + } + return readLine; + } + } + + public String readLine() throws IOException { + synchronized (this.lock) { + String lineString = this.br_.readLine(); + while (lineString != null && (lineString.isEmpty() || lineString.startsWith("#"))) + lineString = this.br_.readLine(); + return lineString; + } + } + + public String[] readLineTokens() throws IOException { + logger.debug("reading tokens"); + String line = readLine(); + logger.debug("line readed"); + if (line == null) + return null; + return line.split(this.lineDelimitingRegex_, -1); + } + + private String[] analyze(String text, Analyzer analyzer) throws IOException { + int size = 0; + TokenStream tokenStream1 = analyzer.tokenStream("all", text); + tokenStream1.reset(); + while (tokenStream1.incrementToken()) + size++; + tokenStream1.close(); + logger.debug("size calculated"); + String[] result = new String[size]; + int index = 0; + TokenStream tokenStream = analyzer.tokenStream("all", text); + CharTermAttribute attr = (CharTermAttribute)tokenStream.addAttribute(CharTermAttribute.class); + tokenStream.reset(); + while (tokenStream.incrementToken()) { + result[index] = attr.toString(); + index++; + } + tokenStream.close(); + logger.debug("tokens readed"); + return result; + } + + public int read(char[] cbuf, int off, int len) throws IOException { + synchronized (this.lock) { + return this.br_.read(cbuf, off, len); + } + } + + public void close() throws IOException { + synchronized (this.lock) { + if (this.br_ != null) + this.br_.close(); + } + } + + public void rewind() throws IOException { + synchronized (this.lock) { + if (this.br_ != null) + this.br_.close(); + if (this.file_ != null) { + FileInputStream fis = new FileInputStream(this.file_); + this.br_ = new BufferedReader(new InputStreamReader(fis, this.charset_)); + } else { + throw new UnsupportedOperationException("InputStream rewind not supported"); + } + } + } + + public String getLineDelimitingRege() { + return this.lineDelimitingRegex_; + } + + public void setLineDelimitingRegex(String lineDelimitingRegex) { + this.lineDelimitingRegex_ = lineDelimitingRegex; + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/io/IOUtil.java b/src/main/java/ai/searchbox/FastText4J/io/IOUtil.java new file mode 100644 index 0000000..9d0fcda --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/io/IOUtil.java @@ -0,0 +1,157 @@ +package ai.searchbox.FastText4J.io; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class IOUtil { + private int string_buf_size_ = 128; + + private byte[] bool_bytes_ = new byte[1]; + + private byte[] int_bytes_ = new byte[4]; + + private byte[] long_bytes_ = new byte[8]; + + private byte[] float_bytes_ = new byte[4]; + + private byte[] double_bytes_ = new byte[8]; + + private byte[] string_bytes_ = new byte[this.string_buf_size_]; + + private StringBuilder stringBuilder_ = new StringBuilder(); + + private ByteBuffer float_array_bytebuffer_ = null; + + private byte[] float_array_bytes_ = null; + + public void setStringBufferSize(int size) { + this.string_buf_size_ = size; + this.string_bytes_ = new byte[this.string_buf_size_]; + } + + public void setFloatArrayBufferSize(int itemSize) { + this.float_array_bytebuffer_ = ByteBuffer.allocate(itemSize * 4).order(ByteOrder.LITTLE_ENDIAN); + this.float_array_bytes_ = new byte[itemSize * 4]; + } + + public byte readByte(InputStream is) throws IOException { + return (byte)is.read(); + } + + public int readByteAsInt(InputStream is) throws IOException { + return readByte(is) & 0xFF; + } + + public boolean readBool(InputStream is) throws IOException { + int ch = readByte(is); + return (ch != 0); + } + + public int readInt(InputStream is) throws IOException { + is.read(this.int_bytes_); + return getInt(this.int_bytes_); + } + + public int getInt(byte[] b) { + return (b[0] & 0xFF) << 0 | (b[1] & 0xFF) << 8 | (b[2] & 0xFF) << 16 | (b[3] & 0xFF) << 24; + } + + public long readLong(InputStream is) throws IOException { + is.read(this.long_bytes_); + return getLong(this.long_bytes_); + } + + public long getLong(byte[] b) { + return (b[0] & 0xFFL) << 0L | (b[1] & 0xFFL) << 8L | ( + b[2] & 0xFFL) << 16L | (b[3] & 0xFFL) << 24L | ( + b[4] & 0xFFL) << 32L | (b[5] & 0xFFL) << 40L | ( + b[6] & 0xFFL) << 48L | (b[7] & 0xFFL) << 56L; + } + + public float readFloat(InputStream is) throws IOException { + is.read(this.float_bytes_); + return getFloat(this.float_bytes_); + } + + public void readFloat(InputStream is, float[] data) throws IOException { + is.read(this.float_array_bytes_); + this.float_array_bytebuffer_.clear(); + ((ByteBuffer)this.float_array_bytebuffer_.put(this.float_array_bytes_).flip()).asFloatBuffer().get(data); + } + + public float getFloat(byte[] b) { + return + Float.intBitsToFloat((b[0] & 0xFF) << 0 | (b[1] & 0xFF) << 8 | (b[2] & 0xFF) << 16 | (b[3] & 0xFF) << 24); + } + + public double readDouble(InputStream is) throws IOException { + is.read(this.double_bytes_); + return getDouble(this.double_bytes_); + } + + public double getDouble(byte[] b) { + return Double.longBitsToDouble(getLong(b)); + } + + public String readString(InputStream is) throws IOException { + int b = is.read(); + if (b < 0) + return null; + int i = -1; + this.stringBuilder_.setLength(0); + while (b > -1 && b != 32 && b != 10 && b != 0) { + this.string_bytes_[++i] = (byte)b; + b = is.read(); + if (i == this.string_buf_size_ - 1) { + this.stringBuilder_.append(new String(this.string_bytes_)); + i = -1; + } + } + this.stringBuilder_.append(new String(this.string_bytes_, 0, i + 1)); + return this.stringBuilder_.toString(); + } + + public byte intToByte(int i) { + return (byte)(i & 0xFF); + } + + public byte[] intToByteArray(int i) { + this.int_bytes_[0] = (byte)(i >> 0 & 0xFF); + this.int_bytes_[1] = (byte)(i >> 8 & 0xFF); + this.int_bytes_[2] = (byte)(i >> 16 & 0xFF); + this.int_bytes_[3] = (byte)(i >> 24 & 0xFF); + return this.int_bytes_; + } + + public byte[] longToByteArray(long l) { + this.long_bytes_[0] = (byte)(int)(l >> 0L & 0xFFL); + this.long_bytes_[1] = (byte)(int)(l >> 8L & 0xFFL); + this.long_bytes_[2] = (byte)(int)(l >> 16L & 0xFFL); + this.long_bytes_[3] = (byte)(int)(l >> 24L & 0xFFL); + this.long_bytes_[4] = (byte)(int)(l >> 32L & 0xFFL); + this.long_bytes_[5] = (byte)(int)(l >> 40L & 0xFFL); + this.long_bytes_[6] = (byte)(int)(l >> 48L & 0xFFL); + this.long_bytes_[7] = (byte)(int)(l >> 56L & 0xFFL); + return this.long_bytes_; + } + + public byte[] floatToByteArray(float f) { + return intToByteArray(Float.floatToIntBits(f)); + } + + public byte[] floatToByteArray(float[] f) { + this.float_array_bytebuffer_.clear(); + this.float_array_bytebuffer_.asFloatBuffer().put(f); + return this.float_array_bytebuffer_.array(); + } + + public byte[] doubleToByteArray(double d) { + return longToByteArray(Double.doubleToRawLongBits(d)); + } + + public byte[] booleanToByteArray(boolean b) { + return new byte[] { (byte)(b ? 1 : 0) }; + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/io/LineReader.java b/src/main/java/ai/searchbox/FastText4J/io/LineReader.java new file mode 100644 index 0000000..80d7580 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/io/LineReader.java @@ -0,0 +1,42 @@ +package ai.searchbox.FastText4J.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; + +public abstract class LineReader extends Reader { + protected InputStream inputStream_ = null; + + protected File file_ = null; + + protected Charset charset_ = null; + + protected LineReader() {} + + protected LineReader(Object lock) { + super(lock); + } + + public LineReader(String filename, String charsetName) throws IOException, UnsupportedEncodingException { + this(); + this.file_ = new File(filename); + this.charset_ = Charset.forName(charsetName); + } + + public LineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { + this(); + this.inputStream_ = inputStream; + this.charset_ = Charset.forName(charsetName); + } + + public abstract long skipLine(long paramLong) throws IOException; + + public abstract String readLine() throws IOException; + + public abstract String[] readLineTokens() throws IOException; + + public abstract void rewind() throws IOException; +} diff --git a/src/main/java/ai/searchbox/FastText4J/io/MappedByteBufferLineReader.java b/src/main/java/ai/searchbox/FastText4J/io/MappedByteBufferLineReader.java new file mode 100644 index 0000000..635b789 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/io/MappedByteBufferLineReader.java @@ -0,0 +1,214 @@ +package ai.searchbox.FastText4J.io; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +public class MappedByteBufferLineReader extends LineReader { + private static int DEFAULT_BUFFER_SIZE = 1024; + + private volatile ByteBuffer byteBuffer_ = null; + + private RandomAccessFile raf_ = null; + + private FileChannel channel_ = null; + + private byte[] bytes_ = null; + + private int string_buf_size_ = DEFAULT_BUFFER_SIZE; + + private boolean fillLine_ = false; + + private StringBuilder sb_ = new StringBuilder(); + + private List tokens_ = new ArrayList<>(); + + public MappedByteBufferLineReader(String filename, String charsetName) throws IOException, UnsupportedEncodingException { + super(filename, charsetName); + this.raf_ = new RandomAccessFile(this.file_, "r"); + this.channel_ = this.raf_.getChannel(); + this.byteBuffer_ = this.channel_.map(FileChannel.MapMode.READ_ONLY, 0L, this.channel_.size()); + this.bytes_ = new byte[this.string_buf_size_]; + } + + public MappedByteBufferLineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { + this(inputStream, charsetName, DEFAULT_BUFFER_SIZE); + } + + public MappedByteBufferLineReader(InputStream inputStream, String charsetName, int buf_size) throws UnsupportedEncodingException { + super((inputStream instanceof BufferedInputStream) ? inputStream : new BufferedInputStream(inputStream), charsetName); + this.string_buf_size_ = buf_size; + this.byteBuffer_ = ByteBuffer.allocateDirect(this.string_buf_size_); + this.bytes_ = new byte[this.string_buf_size_]; + if (inputStream == System.in) + this.fillLine_ = true; + } + + public long skipLine(long n) throws IOException { + if (n < 0L) + throw new IllegalArgumentException("skip value is negative"); + long currentLine = 0L; + long readLine = 0L; + synchronized (this.lock) { + ensureOpen(); + String line; + while (currentLine < n && (line = getLine()) != null) { + readLine++; + if (line == null || line.isEmpty() || line.startsWith("#")) + continue; + currentLine++; + } + } + return readLine; + } + + public String readLine() throws IOException { + synchronized (this.lock) { + ensureOpen(); + String lineString = getLine(); + while (lineString != null && (lineString.isEmpty() || lineString.startsWith("#"))) + lineString = getLine(); + return lineString; + } + } + + public String[] readLineTokens() throws IOException { + synchronized (this.lock) { + ensureOpen(); + String[] tokens = getLineTokens(); + while (tokens != null && ((tokens.length == 1 && tokens[0].isEmpty()) || tokens[0].startsWith("#"))) + tokens = getLineTokens(); + return tokens; + } + } + + public void rewind() throws IOException { + synchronized (this.lock) { + ensureOpen(); + if (this.raf_ != null) { + this.raf_.seek(0L); + this.channel_.position(0L); + } + this.byteBuffer_.position(0); + } + } + + public int read(char[] cbuf, int off, int len) throws IOException { + synchronized (this.lock) { + ensureOpen(); + if (off < 0 || off > cbuf.length || len < 0 || off + len > cbuf.length || off + len < 0) + throw new IndexOutOfBoundsException(); + if (len == 0) + return 0; + CharBuffer charBuffer = this.byteBuffer_.asCharBuffer(); + int length = Math.min(len, charBuffer.remaining()); + charBuffer.get(cbuf, off, length); + if (this.inputStream_ != null) { + off += length; + while (off < len) { + fillByteBuffer(); + if (!this.byteBuffer_.hasRemaining()) + break; + charBuffer = this.byteBuffer_.asCharBuffer(); + length = Math.min(len, charBuffer.remaining()); + charBuffer.get(cbuf, off, length); + off += length; + } + } + return (length == len) ? len : -1; + } + } + + public void close() throws IOException { + synchronized (this.lock) { + if (this.raf_ != null) { + this.raf_.close(); + } else if (this.inputStream_ != null) { + this.inputStream_.close(); + } + this.channel_ = null; + this.byteBuffer_ = null; + } + } + + private void ensureOpen() throws IOException { + if (this.byteBuffer_ == null) + throw new IOException("Stream closed"); + } + + protected String getLine() throws IOException { + fillByteBuffer(); + if (!this.byteBuffer_.hasRemaining()) + return null; + this.sb_.setLength(0); + int b = -1; + int i = -1; + do { + b = this.byteBuffer_.get(); + if ((b >= 10 && b <= 13) || b == 0) + break; + this.bytes_[++i] = (byte)b; + if (i == this.string_buf_size_ - 1) { + this.sb_.append(new String(this.bytes_, this.charset_)); + i = -1; + } + fillByteBuffer(); + } while (this.byteBuffer_.hasRemaining()); + this.sb_.append(new String(this.bytes_, 0, i + 1, this.charset_)); + return this.sb_.toString(); + } + + protected String[] getLineTokens() throws IOException { + fillByteBuffer(); + if (!this.byteBuffer_.hasRemaining()) + return null; + this.tokens_.clear(); + this.sb_.setLength(0); + int b = -1; + int i = -1; + do { + b = this.byteBuffer_.get(); + if ((b >= 10 && b <= 13) || b == 0) + break; + if (b == 9 || b == 32) { + this.sb_.append(new String(this.bytes_, 0, i + 1, this.charset_)); + this.tokens_.add(this.sb_.toString()); + this.sb_.setLength(0); + i = -1; + } else { + this.bytes_[++i] = (byte)b; + if (i == this.string_buf_size_ - 1) { + this.sb_.append(new String(this.bytes_, this.charset_)); + i = -1; + } + } + fillByteBuffer(); + } while (this.byteBuffer_.hasRemaining()); + this.sb_.append(new String(this.bytes_, 0, i + 1, this.charset_)); + this.tokens_.add(this.sb_.toString()); + return this.tokens_.toArray(new String[this.tokens_.size()]); + } + + private void fillByteBuffer() throws IOException { + if (this.inputStream_ == null || this.byteBuffer_.hasRemaining()) + return; + this.byteBuffer_.clear(); + for (int i = 0; i < this.string_buf_size_; i++) { + int b = this.inputStream_.read(); + if (b < 0) + break; + this.byteBuffer_.put((byte)b); + if (this.fillLine_ && (( + b >= 10 && b <= 13) || b == 0)) + break; + } + this.byteBuffer_.flip(); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/Matrix.java b/src/main/java/ai/searchbox/FastText4J/math/Matrix.java new file mode 100644 index 0000000..b73543c --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/Matrix.java @@ -0,0 +1,103 @@ +package ai.searchbox.FastText4J.math; + +import ai.searchbox.FastText4J.Utils; +import ai.searchbox.FastText4J.io.IOUtil; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Random; + +public class Matrix { + public int m = 0; + + public int n = 0; + + public float[][] data = null; + + public Matrix() {} + + public Matrix(int m, int n) { + this.m = m; + this.n = n; + this.data = new float[m][n]; + } + + public void zero() { + for (int i = 0; i < this.m; i++) { + for (int j = 0; j < this.n; j++) + this.data[i][j] = 0.0F; + } + } + + public void uniform(float a) { + Random random = new Random(1L); + for (int i = 0; i < this.m; i++) { + for (int j = 0; j < this.n; j++) + this.data[i][j] = Utils.randomFloat(random, -a, a); + } + } + + public void addToVector(Vector x, int t) { + for (int j = 0; j < this.n; j++) + x.data[j] = x.data[j] + this.data[t][j]; + } + + public void addRow(Vector vec, int i, float a) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < this.m)); + Utils.checkArgument((vec.m == this.n)); + for (int j = 0; j < this.n; j++) + this.data[i][j] = this.data[i][j] + a * vec.data[j]; + } + + public float dotRow(Vector vec, int i) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < this.m)); + Utils.checkArgument((vec.m == this.n)); + float d = 0.0F; + for (int j = 0; j < this.n; j++) + d += this.data[i][j] * vec.data[j]; + return d; + } + + public void load(InputStream input) throws IOException { + IOUtil ioutil = new IOUtil(); + this.m = (int)ioutil.readLong(input); + this.n = (int)ioutil.readLong(input); + ioutil.setFloatArrayBufferSize(this.n); + this.data = new float[this.m][this.n]; + for (int i = 0; i < this.m; i++) + ioutil.readFloat(input, this.data[i]); + } + + public void save(OutputStream ofs) throws IOException { + IOUtil ioutil = new IOUtil(); + ioutil.setFloatArrayBufferSize(this.n); + ofs.write(ioutil.longToByteArray(this.m)); + ofs.write(ioutil.longToByteArray(this.n)); + for (int i = 0; i < this.m; i++) + ofs.write(ioutil.floatToByteArray(this.data[i])); + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Matrix [data_="); + if (this.data != null) { + builder.append("["); + for (int i = 0; i < this.m && i < 10; i++) { + for (int j = 0; j < this.n && j < 10; j++) + builder.append(this.data[i][j]).append(","); + } + builder.setLength(builder.length() - 1); + builder.append("]"); + } else { + builder.append("null"); + } + builder.append(", m_="); + builder.append(this.m); + builder.append(", n_="); + builder.append(this.n); + builder.append("]"); + return builder.toString(); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/MatrixQ.java b/src/main/java/ai/searchbox/FastText4J/math/MatrixQ.java new file mode 100644 index 0000000..3c9d733 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/MatrixQ.java @@ -0,0 +1,84 @@ +package ai.searchbox.FastText4J.math; + +import ai.searchbox.FastText4J.Utils; +import ai.searchbox.FastText4J.io.IOUtil; +import ai.searchbox.FastText4J.math.quant.ProductQuantizer; +import ai.searchbox.FastText4J.math.quant.QCodeArray; +import ai.searchbox.FastText4J.math.quant.QCodes; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class MatrixQ extends Matrix { + QCodeArray codes; + + ProductQuantizer pq = new ProductQuantizer(); + + QCodeArray normCodes; + + ProductQuantizer npq = new ProductQuantizer(); + + boolean qnorm; + + public void addToVector(Vector x, int t) { + float norm = 1.0F; + if (this.qnorm) { + int cPosition = this.npq.getCentroidsPosition(0, this.normCodes.get(t)); + norm = this.npq.getCentroid(cPosition); + } + this.pq.addCode(x, (QCodes)this.codes, t, norm); + } + + public float dotRow(Vector vec, int i) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < this.m)); + Utils.checkArgument((vec.m == this.n)); + float norm = 1.0F; + if (this.qnorm) { + int cPosition = this.npq.getCentroidsPosition(0, this.normCodes.get(i)); + norm = this.npq.getCentroid(cPosition); + } + return this.pq.mulCode(vec, (QCodes)this.codes, i, norm); + } + + public void load(InputStream is) throws IOException { + IOUtil ioutil = new IOUtil(); + this.qnorm = ioutil.readBool(is); + this.m = (int)ioutil.readLong(is); + this.n = (int)ioutil.readLong(is); + int codeSize = ioutil.readInt(is); + int[] rawCodes = new int[codeSize]; + for (int i = 0; i < codeSize; i++) { + int c = ioutil.readByteAsInt(is); + rawCodes[i] = c; + } + this.codes = new QCodeArray(rawCodes); + this.pq.load(is); + if (this.qnorm) { + int[] rawNormCodes = new int[this.m]; + for (int j = 0; j < this.m; j++) { + int c = ioutil.readByteAsInt(is); + rawNormCodes[j] = c; + } + this.normCodes = new QCodeArray(rawNormCodes); + this.npq.load(is); + } + } + + public void save(OutputStream os) throws IOException { + IOUtil ioutil = new IOUtil(); + os.write(ioutil.booleanToByteArray(this.qnorm)); + os.write(ioutil.longToByteArray(this.m)); + os.write(ioutil.longToByteArray(this.n)); + os.write(ioutil.intToByteArray(this.codes.size())); + int i; + for (i = 0; i < this.codes.size(); i++) + os.write(ioutil.intToByte(this.codes.get(i))); + this.pq.save(os); + if (this.qnorm) { + for (i = 0; i < this.m; i++) + os.write(ioutil.intToByte(this.normCodes.get(i))); + this.npq.save(os); + } + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/Vector.java b/src/main/java/ai/searchbox/FastText4J/math/Vector.java new file mode 100644 index 0000000..85d172b --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/Vector.java @@ -0,0 +1,108 @@ +package ai.searchbox.FastText4J.math; + +import ai.searchbox.FastText4J.Utils; + +public class Vector { + public int m; + + public float[] data; + + public Vector(int size) { + this.m = size; + this.data = new float[this.m]; + } + + public Vector(float[] v) { + this.m = v.length; + this.data = new float[this.m]; + for (int i = 0; i < this.m; i++) + set(i, v[i]); + } + + public int size() { + return this.m; + } + + public void zero() { + for (int i = 0; i < this.m; i++) + this.data[i] = 0.0F; + } + + public void mul(float a) { + for (int i = 0; i < this.m; i++) + this.data[i] = this.data[i] * a; + } + + public void addRow(Matrix A, int i) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < A.m)); + Utils.checkArgument((this.m == A.n)); + A.addToVector(this, i); + } + + public void addRow(Matrix A, int i, float a) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < A.m)); + Utils.checkArgument((this.m == A.n)); + for (int j = 0; j < A.n; j++) + this.data[j] = this.data[j] + a * A.data[i][j]; + } + + public void addVector(Vector source) { + Utils.checkArgument((this.m == source.m)); + for (int i = 0; i < this.m; i++) + this.data[i] = this.data[i] + source.get(i); + } + + public float norm() { + float sum = 0.0F; + for (int i = 0; i < this.m; i++) + sum += this.data[i] * this.data[i]; + return (float)Math.sqrt(sum); + } + + public void mul(Matrix A, Vector vec) { + Utils.checkArgument((A.m == this.m)); + Utils.checkArgument((A.n == vec.m)); + for (int i = 0; i < this.m; i++) { + this.data[i] = 0.0F; + for (int j = 0; j < A.n; j++) + this.data[i] = A.dotRow(vec, i); + } + } + + public int argmax() { + float max = this.data[0]; + int argmax = 0; + for (int i = 1; i < this.m; i++) { + if (this.data[i] > max) { + max = this.data[i]; + argmax = i; + } + } + return argmax; + } + + public float get(int i) { + return this.data[i]; + } + + public void set(int i, float value) { + this.data[i] = value; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + byte b; + int i; + float[] arrayOfFloat; + for (i = (arrayOfFloat = this.data).length, b = 0; b < i; ) { + float data = arrayOfFloat[b]; + builder.append(data).append(' '); + b++; + } + if (builder.length() > 1) + builder.setLength(builder.length() - 1); + return builder.toString(); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/quant/ProductQuantizer.java b/src/main/java/ai/searchbox/FastText4J/math/quant/ProductQuantizer.java new file mode 100644 index 0000000..8364ba3 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/quant/ProductQuantizer.java @@ -0,0 +1,158 @@ +package ai.searchbox.FastText4J.math.quant; + +import ai.searchbox.FastText4J.io.IOUtil; +import ai.searchbox.FastText4J.math.Vector; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Random; + +public class ProductQuantizer { + private static final int SEED = 1234; + + private static final int NUM_BITS = 8; + + private static final int KSUB = 256; + + private static final int MAX_POINTS_PER_CLUSTER = 256; + + private static final int MAX_POINTS = 65536; + + private static final int NUM_ITER = 25; + + private static final double EPS = 1.0E-7D; + + int dim; + + int nsubq; + + int dsub; + + int lastdsub; + + float[] centroids; + + final Random rng = new Random(1234L); + + public float distL2(float[] x, float[] y, int d) { + return distL2(x, y, d, 0, 0); + } + + public float distL2(float[] x, float[] y, int d, int xpos, int ypos) { + float dist = 0.0F; + for (int i = 0; i < d; i++) { + float tmp = x[i + xpos] - y[i + ypos]; + dist += tmp * tmp; + } + return dist; + } + + public int dim() { + return this.dim; + } + + public int dsub() { + return this.dsub; + } + + public int nsubq() { + return this.nsubq; + } + + public int lastdsub() { + return this.lastdsub; + } + + public float[] centroids() { + return this.centroids; + } + + public float getCentroid(int position) { + return this.centroids[position]; + } + + public int getCentroidsPosition(int m, int i) { + if (m == this.nsubq - 1) + return m * 256 * this.dsub + i * this.lastdsub; + return (m * 256 + i) * this.dsub; + } + + public void train(int n, float[] x) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public void computeCode(float[] x, QCodes codes, int xBeginPosition, int codeBeginPosition) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public void computeCodes(float[] x, QCodes codes, int m) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public float mulCode(Vector x, QCodes codes, int t, float alpha) { + float res = 0.0F; + int d = this.dsub; + int codePos = this.nsubq + t; + for (int m = 0; m < this.nsubq; m++) { + int c = getCentroidsPosition(m, codes.get(m + codePos)); + if (m == this.nsubq - 1) + d = this.lastdsub; + for (int n = 0; n < d; n++) + res += x.data[m * this.dsub + n] * this.centroids[c * n]; + } + return res * alpha; + } + + public void addCode(Vector x, QCodes codes, int t, float alpha) { + int d = this.dsub; + int codePos = this.nsubq * t; + for (int m = 0; m < this.nsubq; m++) { + int c = getCentroidsPosition(m, codes.get(m + codePos)); + if (m == this.nsubq - 1) + d = this.lastdsub; + for (int n = 0; n < d; n++) + x.data[m * this.dsub + n] = x.data[m * this.dsub + n] + alpha * this.centroids[c + n]; + } + } + + public void save(OutputStream os) throws IOException { + IOUtil io = new IOUtil(); + os.write(io.intToByteArray(this.dim)); + os.write(io.intToByteArray(this.nsubq)); + os.write(io.intToByteArray(this.dsub)); + os.write(io.intToByteArray(this.lastdsub)); + for (int i = 0; i < this.centroids.length; i++) + os.write(io.floatToByteArray(this.centroids[i])); + } + + public void load(InputStream is) throws IOException { + IOUtil io = new IOUtil(); + this.dim = io.readInt(is); + this.nsubq = io.readInt(is); + this.dsub = io.readInt(is); + this.lastdsub = io.readInt(is); + this.centroids = new float[this.dim * 256]; + for (int i = 0; i < this.centroids.length; i++) + this.centroids[i] = io.readFloat(is); + } + + public static int findCentroidsSize(int dimension) { + return dimension * 256; + } + + private float assignCentroid(float[] x, int xStartPosition, int c0Position, QCodes codes, int codeStartPosition, int d) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + private void eStep(float[] x, int cPosition, QCodes codes, int d, int n) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + private void mStep(float[] x0, int cPosition, QCodes codes, int d, int n) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + private void kmeans(float[] x, int cPosition, int n, int d) { + throw new UnsupportedOperationException("Not implemented yet"); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/quant/QCodeArray.java b/src/main/java/ai/searchbox/FastText4J/math/quant/QCodeArray.java new file mode 100644 index 0000000..4f3a54b --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/quant/QCodeArray.java @@ -0,0 +1,42 @@ +package ai.searchbox.FastText4J.math.quant; + +import ai.searchbox.FastText4J.Utils; + +public class QCodeArray implements QCodes { + private final int[] codes; + + public QCodeArray(QCodeArray qcodes) { + this.codes = qcodes.codes; + } + + public QCodeArray(int[] codes) { + this.codes = codes; + } + + public QCodeArray(int size) { + this.codes = new int[size]; + } + + public int get(int i) { + Utils.checkArgument((i >= 0)); + Utils.checkArgument((i < this.codes.length)); + return this.codes[i]; + } + + public int size() { + return this.codes.length; + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("QCodeArray(size="); + builder.append(size()); + builder.append(", ["); + for (int i = 0; i < size(); i++) + builder.append(get(i)).append(' '); + if (builder.length() > 1) + builder.setLength(builder.length() - 1); + builder.append("])"); + return builder.toString(); + } +} diff --git a/src/main/java/ai/searchbox/FastText4J/math/quant/QCodes.java b/src/main/java/ai/searchbox/FastText4J/math/quant/QCodes.java new file mode 100644 index 0000000..c7b3fe2 --- /dev/null +++ b/src/main/java/ai/searchbox/FastText4J/math/quant/QCodes.java @@ -0,0 +1,7 @@ +package ai.searchbox.FastText4J.math.quant; + +public interface QCodes { + int get(int paramInt); + + int size(); +} diff --git a/src/main/java/fasttext/Args.java b/src/main/java/fasttext/Args.java deleted file mode 100644 index 2d231c9..0000000 --- a/src/main/java/fasttext/Args.java +++ /dev/null @@ -1,279 +0,0 @@ -package fasttext; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class Args { - - public enum model_name { - cbow(1), sg(2), sup(3); - - private int value; - - private model_name(int value) { - this.value = value; - } - - public int getValue() { - return this.value; - } - - public static model_name fromValue(int value) throws IllegalArgumentException { - try { - value -= 1; - return model_name.values()[value]; - } catch (ArrayIndexOutOfBoundsException e) { - throw new IllegalArgumentException("Unknown model_name enum value :" + value); - } - } - } - - public enum loss_name { - hs(1), ns(2), softmax(3); - private int value; - - private loss_name(int value) { - this.value = value; - } - - public int getValue() { - return this.value; - } - - public static loss_name fromValue(int value) throws IllegalArgumentException { - try { - value -= 1; - return loss_name.values()[value]; - } catch (ArrayIndexOutOfBoundsException e) { - throw new IllegalArgumentException("Unknown loss_name enum value :" + value); - } - } - } - - public String input; - public String output; - public String test; - public double lr = 0.05; - public int lrUpdateRate = 100; - public int dim = 100; - public int ws = 5; - public int epoch = 5; - public int minCount = 5; - public int minCountLabel = 0; - public int neg = 5; - public int wordNgrams = 1; - public loss_name loss = loss_name.ns; - public model_name model = model_name.sg; - public int bucket = 2000000; - public int minn = 3; - public int maxn = 6; - public int thread = 1; - public double t = 1e-4; - public String label = "__label__"; - public int verbose = 2; - public String pretrainedVectors = ""; - - public void printHelp() { - System.out.println("\n" + "The following arguments are mandatory:\n" - + " -input training file path\n" - + " -output output file path\n\n" - + "The following arguments are optional:\n" - + " -lr learning rate [" + lr + "]\n" - + " -lrUpdateRate change the rate of updates for the learning rate [" + lrUpdateRate + "]\n" - + " -dim size of word vectors [" + dim + "]\n" - + " -ws size of the context window [" + ws + "]\n" - + " -epoch number of epochs [" + epoch + "]\n" - + " -minCount minimal number of word occurences [" + minCount + "]\n" - + " -minCountLabel minimal number of label occurences [" + minCountLabel + "]\n" - + " -neg number of negatives sampled [" + neg + "]\n" - + " -wordNgrams max length of word ngram [" + wordNgrams + "]\n" - + " -loss loss function {ns, hs, softmax} [ns]\n" - + " -bucket number of buckets [" + bucket + "]\n" - + " -minn min length of char ngram [" + minn + "]\n" - + " -maxn max length of char ngram [" + maxn + "]\n" - + " -thread number of threads [" + thread + "]\n" - + " -t sampling threshold [" + t + "]\n" - + " -label labels prefix [" + label + "]\n" - + " -verbose verbosity level [" + verbose + "]\n" - + " -pretrainedVectors pretrained word vectors for supervised learning []"); - } - - public void save(OutputStream ofs) throws IOException { - IOUtil ioutil = new IOUtil(); - ofs.write(ioutil.intToByteArray(dim)); - ofs.write(ioutil.intToByteArray(ws)); - ofs.write(ioutil.intToByteArray(epoch)); - ofs.write(ioutil.intToByteArray(minCount)); - ofs.write(ioutil.intToByteArray(neg)); - ofs.write(ioutil.intToByteArray(wordNgrams)); - ofs.write(ioutil.intToByteArray(loss.value)); - ofs.write(ioutil.intToByteArray(model.value)); - ofs.write(ioutil.intToByteArray(bucket)); - ofs.write(ioutil.intToByteArray(minn)); - ofs.write(ioutil.intToByteArray(maxn)); - ofs.write(ioutil.intToByteArray(lrUpdateRate)); - ofs.write(ioutil.doubleToByteArray(t)); - } - - public void load(InputStream input) throws IOException { - IOUtil ioutil = new IOUtil(); - dim = ioutil.readInt(input); - ws = ioutil.readInt(input); - epoch = ioutil.readInt(input); - minCount = ioutil.readInt(input); - neg = ioutil.readInt(input); - wordNgrams = ioutil.readInt(input); - loss = loss_name.fromValue(ioutil.readInt(input)); - model = model_name.fromValue(ioutil.readInt(input)); - bucket = ioutil.readInt(input); - minn = ioutil.readInt(input); - maxn = ioutil.readInt(input); - lrUpdateRate = ioutil.readInt(input); - t = ioutil.readDouble(input); - } - - public void parseArgs(String[] args) { - String command = args[0]; - if ("supervised".equalsIgnoreCase(command)) { - model = model_name.sup; - loss = loss_name.softmax; - minCount = 1; - minn = 0; - maxn = 0; - lr = 0.1; - } else if ("cbow".equalsIgnoreCase(command)) { - model = model_name.cbow; - } - int ai = 1; - while (ai < args.length) { - if (args[ai].charAt(0) != '-') { - System.out.println("Provided argument without a dash! Usage:"); - printHelp(); - System.exit(1); - } - if ("-h".equals(args[ai])) { - System.out.println("Here is the help! Usage:"); - printHelp(); - System.exit(1); - } else if ("-input".equals(args[ai])) { - input = args[ai + 1]; - } else if ("-test".equals(args[ai])) { - test = args[ai + 1]; - } else if ("-output".equals(args[ai])) { - output = args[ai + 1]; - } else if ("-lr".equals(args[ai])) { - lr = Double.parseDouble(args[ai + 1]); - } else if ("-lrUpdateRate".equals(args[ai])) { - lrUpdateRate = Integer.parseInt(args[ai + 1]); - } else if ("-dim".equals(args[ai])) { - dim = Integer.parseInt(args[ai + 1]); - } else if ("-ws".equals(args[ai])) { - ws = Integer.parseInt(args[ai + 1]); - } else if ("-epoch".equals(args[ai])) { - epoch = Integer.parseInt(args[ai + 1]); - } else if ("-minCount".equals(args[ai])) { - minCount = Integer.parseInt(args[ai + 1]); - } else if ("-minCountLabel".equals(args[ai])) { - minCountLabel = Integer.parseInt(args[ai + 1]); - } else if ("-neg".equals(args[ai])) { - neg = Integer.parseInt(args[ai + 1]); - } else if ("-wordNgrams".equals(args[ai])) { - wordNgrams = Integer.parseInt(args[ai + 1]); - } else if ("-loss".equals(args[ai])) { - if ("hs".equalsIgnoreCase(args[ai + 1])) { - loss = loss_name.hs; - } else if ("ns".equalsIgnoreCase(args[ai + 1])) { - loss = loss_name.ns; - } else if ("softmax".equalsIgnoreCase(args[ai + 1])) { - loss = loss_name.softmax; - } else { - System.out.println("Unknown loss: " + args[ai + 1]); - printHelp(); - System.exit(1); - } - } else if ("-bucket".equals(args[ai])) { - bucket = Integer.parseInt(args[ai + 1]); - } else if ("-minn".equals(args[ai])) { - minn = Integer.parseInt(args[ai + 1]); - } else if ("-maxn".equals(args[ai])) { - maxn = Integer.parseInt(args[ai + 1]); - } else if ("-thread".equals(args[ai])) { - thread = Integer.parseInt(args[ai + 1]); - } else if ("-t".equals(args[ai])) { - t = Double.parseDouble(args[ai + 1]); - } else if ("-label".equals(args[ai])) { - label = args[ai + 1]; - } else if ("-verbose".equals(args[ai])) { - verbose = Integer.parseInt(args[ai + 1]); - } else if ("-pretrainedVectors".equals(args[ai])) { - pretrainedVectors = args[ai + 1]; - } else { - System.out.println("Unknown argument: " + args[ai]); - printHelp(); - System.exit(1); - } - ai += 2; - } - if (Utils.isEmpty(input) || Utils.isEmpty(output)) { - System.out.println("Empty input or output path."); - printHelp(); - System.exit(1); - } - if (wordNgrams <= 1 && maxn == 0) { - bucket = 0; - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Args [input="); - builder.append(input); - builder.append(", output="); - builder.append(output); - builder.append(", test="); - builder.append(test); - builder.append(", lr="); - builder.append(lr); - builder.append(", lrUpdateRate="); - builder.append(lrUpdateRate); - builder.append(", dim="); - builder.append(dim); - builder.append(", ws="); - builder.append(ws); - builder.append(", epoch="); - builder.append(epoch); - builder.append(", minCount="); - builder.append(minCount); - builder.append(", minCountLabel="); - builder.append(minCountLabel); - builder.append(", neg="); - builder.append(neg); - builder.append(", wordNgrams="); - builder.append(wordNgrams); - builder.append(", loss="); - builder.append(loss); - builder.append(", model="); - builder.append(model); - builder.append(", bucket="); - builder.append(bucket); - builder.append(", minn="); - builder.append(minn); - builder.append(", maxn="); - builder.append(maxn); - builder.append(", thread="); - builder.append(thread); - builder.append(", t="); - builder.append(t); - builder.append(", label="); - builder.append(label); - builder.append(", verbose="); - builder.append(verbose); - builder.append(", pretrainedVectors="); - builder.append(pretrainedVectors); - builder.append("]"); - return builder.toString(); - } - -} diff --git a/src/main/java/fasttext/Dictionary.java b/src/main/java/fasttext/Dictionary.java deleted file mode 100644 index b962ce3..0000000 --- a/src/main/java/fasttext/Dictionary.java +++ /dev/null @@ -1,494 +0,0 @@ -package fasttext; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.math.BigInteger; - -import fasttext.Args.model_name; -import fasttext.io.BufferedLineReader; -import fasttext.io.LineReader; - -public class Dictionary { - - private static final int MAX_VOCAB_SIZE = 30000000; - private static final int MAX_LINE_SIZE = 1024; - private static final Integer WORDID_DEFAULT = -1; - - private static final String EOS = ""; - private static final String BOW = "<"; - private static final String EOW = ">"; - - public enum entry_type { - word(0), label(1); - - private int value; - - private entry_type(int value) { - this.value = value; - } - - public int getValue() { - return this.value; - } - - public static entry_type fromValue(int value) throws IllegalArgumentException { - try { - return entry_type.values()[value]; - } catch (ArrayIndexOutOfBoundsException e) { - throw new IllegalArgumentException("Unknown entry_type enum value :" + value); - } - } - - @Override - public String toString() { - return value == 0 ? "word" : value == 1 ? "label" : "unknown"; - } - } - - public class entry { - String word; - long count; - entry_type type; - List subwords; - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("entry [word="); - builder.append(word); - builder.append(", count="); - builder.append(count); - builder.append(", type="); - builder.append(type); - builder.append(", subwords="); - builder.append(subwords); - builder.append("]"); - return builder.toString(); - } - - } - - private List words_; - private List pdiscard_; - private Map word2int_; - private int size_; - private int nwords_; - private int nlabels_; - private long ntokens_; - - private Args args_; - - private String charsetName_ = "UTF-8"; - private Class lineReaderClass_ = BufferedLineReader.class; - - public Dictionary(Args args) { - args_ = args; - size_ = 0; - nwords_ = 0; - nlabels_ = 0; - ntokens_ = 0; - word2int_ = new HashMap(MAX_VOCAB_SIZE); - words_ = new ArrayList(MAX_VOCAB_SIZE); - } - - public long find(final String w) { - long h = hash(w) % MAX_VOCAB_SIZE; - entry e = null; - while (Utils.mapGetOrDefault(word2int_, h, WORDID_DEFAULT) != WORDID_DEFAULT - && ((e = words_.get(word2int_.get(h))) != null && !w.equals(e.word))) { - h = (h + 1) % MAX_VOCAB_SIZE; - } - return h; - } - - public void add(final String w) { - long h = find(w); - ntokens_++; - if (Utils.mapGetOrDefault(word2int_, h, WORDID_DEFAULT) == WORDID_DEFAULT) { - entry e = new entry(); - e.word = w; - e.count = 1; - e.type = w.startsWith(args_.label) ? entry_type.label : entry_type.word; - words_.add(e); - word2int_.put(h, size_++); - } else { - words_.get(word2int_.get(h)).count++; - } - } - - public int nwords() { - return nwords_; - } - - public int nlabels() { - return nlabels_; - } - - public long ntokens() { - return ntokens_; - } - - public final List getNgrams(int i) { - Utils.checkArgument(i >= 0); - Utils.checkArgument(i < nwords_); - return words_.get(i).subwords; - } - - public final List getNgrams(final String word) { - List ngrams = new ArrayList(); - int i = getId(word); - if (i >= 0) { - ngrams = words_.get(i).subwords; - } else { - computeNgrams(BOW + word + EOW, ngrams); - } - return ngrams; - } - - public boolean discard(int id, float rand) { - Utils.checkArgument(id >= 0); - Utils.checkArgument(id < nwords_); - if (args_.model == model_name.sup) - return false; - return rand > pdiscard_.get(id); - } - - public int getId(final String w) { - long h = find(w); - return Utils.mapGetOrDefault(word2int_, h, WORDID_DEFAULT); - } - - public entry_type getType(int id) { - Utils.checkArgument(id >= 0); - Utils.checkArgument(id < size_); - return words_.get(id).type; - } - - public String getWord(int id) { - Utils.checkArgument(id >= 0); - Utils.checkArgument(id < size_); - return words_.get(id).word; - } - - /** - * String FNV-1a Hash - * - * @param str - * @return - */ - public long hash(final String str) { - int h = (int) 2166136261L;// 0xffffffc5; - for (byte strByte : str.getBytes()) { - h = (h ^ strByte) * 16777619; // FNV-1a - // h = (h * 16777619) ^ strByte; //FNV-1 - } - return h & 0xffffffffL; - } - - public void computeNgrams(final String word, List ngrams) { - for (int i = 0; i < word.length(); i++) { - StringBuilder ngram = new StringBuilder(); - if (charMatches(word.charAt(i))) { - continue; - } - for (int j = i, n = 1; j < word.length() && n <= args_.maxn; n++) { - ngram.append(word.charAt(j++)); - while (j < word.length() && charMatches(word.charAt(j))) { - ngram.append(word.charAt(j++)); - } - if (n >= args_.minn && !(n == 1 && (i == 0 || j == word.length()))) { - int h = (int) (nwords_ + (hash(ngram.toString()) % args_.bucket)); - if (h < 0) { - System.err.println("computeNgrams h<0: " + h + " on word: " + word); - } - ngrams.add(h); - } - } - } - } - - private boolean charMatches(char ch) { - if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r') { - return true; - } - return false; - } - - public void initNgrams() { - for (int i = 0; i < size_; i++) { - String word = BOW + words_.get(i).word + EOW; - entry e = words_.get(i); - if (e.subwords == null) { - e.subwords = new ArrayList(); - } - e.subwords.add(i); - computeNgrams(word, e.subwords); - } - } - - public void readFromFile(String file) throws IOException, Exception { - LineReader lineReader = null; - - try { - lineReader = lineReaderClass_.getConstructor(String.class, String.class).newInstance(file, charsetName_); - long minThreshold = 1; - String[] lineTokens; - while ((lineTokens = lineReader.readLineTokens()) != null) { - for (int i = 0; i <= lineTokens.length; i++) { - if (i == lineTokens.length) { - add(EOS); - } else { - if (Utils.isEmpty(lineTokens[i])) { - continue; - } - add(lineTokens[i]); - } - if (ntokens_ % 1000000 == 0 && args_.verbose > 1) { - System.out.printf("\rRead %dM words", ntokens_ / 1000000); - } - if (size_ > 0.75 * MAX_VOCAB_SIZE) { - minThreshold++; - threshold(minThreshold, minThreshold); - } - } - } - } finally { - if (lineReader != null) { - lineReader.close(); - } - } - threshold(args_.minCount, args_.minCountLabel); - initTableDiscard(); - if (model_name.cbow == args_.model || model_name.sg == args_.model) { - initNgrams(); - } - if (args_.verbose > 0) { - System.out.printf("\rRead %dM words\n", ntokens_ / 1000000); - System.out.println("Number of words: " + nwords_); - System.out.println("Number of labels: " + nlabels_); - } - if (size_ == 0) { - System.err.println("Empty vocabulary. Try a smaller -minCount value."); - System.exit(1); - } - } - - public void threshold(long t, long tl) { - Collections.sort(words_, entry_comparator); - Iterator iterator = words_.iterator(); - while (iterator.hasNext()) { - entry _entry = iterator.next(); - if ((entry_type.word == _entry.type && _entry.count < t) - || (entry_type.label == _entry.type && _entry.count < tl)) { - iterator.remove(); - } - } - ((ArrayList) words_).trimToSize(); - size_ = 0; - nwords_ = 0; - nlabels_ = 0; - // word2int_.clear(); - word2int_ = new HashMap(words_.size()); - for (entry _entry : words_) { - long h = find(_entry.word); - word2int_.put(h, size_++); - if (entry_type.word == _entry.type) { - nwords_++; - } else if (entry_type.label == _entry.type) { - nlabels_++; - } - } - } - - private transient Comparator entry_comparator = new Comparator() { - @Override - public int compare(entry o1, entry o2) { - int cmp = (o1.type.value < o2.type.value) ? -1 : ((o1.type.value == o2.type.value) ? 0 : 1); - if (cmp == 0) { - cmp = (o2.count < o1.count) ? -1 : ((o2.count == o1.count) ? 0 : 1); - } - return cmp; - } - }; - - public void initTableDiscard() { - pdiscard_ = new ArrayList(size_); - for (int i = 0; i < size_; i++) { - float f = (float) (words_.get(i).count) / (float) ntokens_; - pdiscard_.add((float) (Math.sqrt(args_.t / f) + args_.t / f)); - } - } - - public List getCounts(entry_type type) { - List counts = entry_type.label == type ? new ArrayList(nlabels()) : new ArrayList(nwords()); - for (entry w : words_) { - if (w.type == type) - counts.add(w.count); - } - return counts; - } - - public void addNgrams(List line, int n) { - if (n <= 1) { - return; - } - int line_size = line.size(); - for (int i = 0; i < line_size; i++) { - BigInteger h = BigInteger.valueOf(line.get(i)); - BigInteger r = BigInteger.valueOf(116049371l); - BigInteger b = BigInteger.valueOf(args_.bucket); - - for (int j = i + 1; j < line_size && j < i + n; j++) { - h = h.multiply(r).add(BigInteger.valueOf(line.get(j)));; - line.add(nwords_ + h.remainder(b).intValue()); - } - } - } - - public int getLine(String[] tokens, List words, List labels, Random urd) { - int ntokens = 0; - words.clear(); - labels.clear(); - if (tokens != null) { - for (int i = 0; i <= tokens.length; i++) { - if (i < tokens.length && Utils.isEmpty(tokens[i])) { - continue; - } - int wid = i == tokens.length ? getId(EOS) : getId(tokens[i]); - if (wid < 0) { - continue; - } - entry_type type = getType(wid); - ntokens++; - if (type == entry_type.word && !discard(wid, Utils.randomFloat(urd, 0, 1))) { - words.add(wid); - } - if (type == entry_type.label) { - labels.add(wid - nwords_); - } - if (words.size() > MAX_LINE_SIZE && args_.model != model_name.sup) { - break; - } - // if (EOS == tokens[i]){ - // break; - // } - } - } - return ntokens; - } - - public String getLabel(int lid) { - Utils.checkArgument(lid >= 0); - Utils.checkArgument(lid < nlabels_); - return words_.get(lid + nwords_).word; - } - - public void save(OutputStream ofs) throws IOException { - IOUtil ioutil = new IOUtil(); - ofs.write(ioutil.intToByteArray(size_)); - ofs.write(ioutil.intToByteArray(nwords_)); - ofs.write(ioutil.intToByteArray(nlabels_)); - ofs.write(ioutil.longToByteArray(ntokens_)); - // Charset charset = Charset.forName("UTF-8"); - for (int i = 0; i < size_; i++) { - entry e = words_.get(i); - ofs.write(e.word.getBytes()); - ofs.write(0); - ofs.write(ioutil.longToByteArray(e.count)); - ofs.write(ioutil.intToByte(e.type.value)); - } - } - - public void load(InputStream ifs) throws IOException { - // words_.clear(); - // word2int_.clear(); - IOUtil ioutil = new IOUtil(); - size_ = ioutil.readInt(ifs); - nwords_ = ioutil.readInt(ifs); - nlabels_ = ioutil.readInt(ifs); - ntokens_ = ioutil.readLong(ifs); - - word2int_ = new HashMap(size_); - words_ = new ArrayList(size_); - - for (int i = 0; i < size_; i++) { - entry e = new entry(); - e.word = ioutil.readString(ifs); - e.count = ioutil.readLong(ifs); - e.type = entry_type.fromValue(ioutil.readByte(ifs)); - words_.add(e); - word2int_.put(find(e.word), i); - } - initTableDiscard(); - if (model_name.cbow == args_.model || model_name.sg == args_.model) { - initNgrams(); - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Dictionary [words_="); - builder.append(words_); - builder.append(", pdiscard_="); - builder.append(pdiscard_); - builder.append(", word2int_="); - builder.append(word2int_); - builder.append(", size_="); - builder.append(size_); - builder.append(", nwords_="); - builder.append(nwords_); - builder.append(", nlabels_="); - builder.append(nlabels_); - builder.append(", ntokens_="); - builder.append(ntokens_); - builder.append("]"); - return builder.toString(); - } - - public List getWords() { - return words_; - } - - public List getPdiscard() { - return pdiscard_; - } - - public Map getWord2int() { - return word2int_; - } - - public int getSize() { - return size_; - } - - public Args getArgs() { - return args_; - } - - public String getCharsetName() { - return charsetName_; - } - - public Class getLineReaderClass() { - return lineReaderClass_; - } - - public void setCharsetName(String charsetName) { - this.charsetName_ = charsetName; - } - - public void setLineReaderClass(Class lineReaderClass) { - this.lineReaderClass_ = lineReaderClass; - } - -} diff --git a/src/main/java/fasttext/FastText.java b/src/main/java/fasttext/FastText.java deleted file mode 100644 index d716155..0000000 --- a/src/main/java/fasttext/FastText.java +++ /dev/null @@ -1,694 +0,0 @@ -package fasttext; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import fasttext.Args.model_name; -import fasttext.Dictionary.entry_type; -import fasttext.io.*; - -/** - * FastText class, can be used as a lib in other projects - * - * @author Ivan - * - */ -public class FastText { - - private Args args_; - private Dictionary dict_; - private Matrix input_; - private Matrix output_; - private Model model_; - - private AtomicLong tokenCount_; - private long start_; - - private String charsetName_ = "UTF-8"; - private Class lineReaderClass_ = BufferedLineReader.class; - - public void getVector(Vector vec, final String word) { - final List ngrams = dict_.getNgrams(word); - vec.zero(); - for (Integer it : ngrams) { - vec.addRow(input_, it); - } - if (ngrams.size() > 0) { - vec.mul(1.0f / ngrams.size()); - } - } - - public void saveVectors() throws IOException { - if (Utils.isEmpty(args_.output)) { - if (args_.verbose > 1) { - System.out.println("output is empty, skip save vector file"); - } - return; - } - - File file = new File(args_.output + ".vec"); - if (file.exists()) { - file.delete(); - } - if (file.getParentFile() != null) { - file.getParentFile().mkdirs(); - } - if (args_.verbose > 1) { - System.out.println("Saving Vectors to " + file.getCanonicalPath().toString()); - } - Vector vec = new Vector(args_.dim); - DecimalFormat df = new DecimalFormat("0.#####"); - Writer writer = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(file)), "UTF-8"); - try { - writer.write(dict_.nwords() + " " + args_.dim + "\n"); - for (int i = 0; i < dict_.nwords(); i++) { - String word = dict_.getWord(i); - getVector(vec, word); - writer.write(word); - for (int j = 0; j < vec.m_; j++) { - writer.write(" "); - writer.write(df.format(vec.data_[j])); - } - writer.write("\n"); - } - } finally { - writer.flush(); - writer.close(); - } - } - - public void saveModel() throws IOException { - if (Utils.isEmpty(args_.output)) { - if (args_.verbose > 1) { - System.out.println("output is empty, skip save model file"); - } - return; - } - - File file = new File(args_.output + ".bin"); - if (file.exists()) { - file.delete(); - } - if (file.getParentFile() != null) { - file.getParentFile().mkdirs(); - } - if (args_.verbose > 1) { - System.out.println("Saving model to " + file.getCanonicalPath().toString()); - } - OutputStream ofs = new BufferedOutputStream(new FileOutputStream(file)); - try { - args_.save(ofs); - dict_.save(ofs); - input_.save(ofs); - output_.save(ofs); - } finally { - ofs.flush(); - ofs.close(); - } - } - - /** - * Load binary model file. - * - * @param filename - * @throws IOException - */ - public void loadModel(String filename) throws IOException { - DataInputStream dis = null; - BufferedInputStream bis = null; - try { - File file = new File(filename); - if (!(file.exists() && file.isFile() && file.canRead())) { - throw new IOException("Model file cannot be opened for loading!"); - } - bis = new BufferedInputStream(new FileInputStream(file)); - dis = new DataInputStream(bis); - - args_ = new Args(); - dict_ = new Dictionary(args_); - input_ = new Matrix(); - output_ = new Matrix(); - - args_.load(dis); - dict_.load(dis); - input_.load(dis); - output_.load(dis); - - model_ = new Model(input_, output_, args_, 0); - if (args_.model == model_name.sup) { - model_.setTargetCounts(dict_.getCounts(entry_type.label)); - } else { - model_.setTargetCounts(dict_.getCounts(entry_type.word)); - } - } finally { - if (bis != null) { - bis.close(); - } - if (dis != null) { - dis.close(); - } - } - } - - public void printInfo(float progress, float loss) { - float t = (float) (System.currentTimeMillis() - start_) / 1000; - float ws = (float) (tokenCount_.get()) / t; - float wst = (float) (tokenCount_.get()) / t / args_.thread; - float lr = (float) (args_.lr * (1.0f - progress)); - int eta = (int) (t / progress * (1 - progress)); - int etah = eta / 3600; - int etam = (eta - etah * 3600) / 60; - System.out.printf("\rProgress: %.1f%% words/sec: %d words/sec/thread: %d lr: %.6f loss: %.6f eta: %d h %d m", - 100 * progress, (int) ws, (int) wst, lr, loss, etah, etam); - } - - public void supervised(Model model, float lr, final List line, final List labels) { - if (labels.size() == 0 || line.size() == 0) - return; - int i = Utils.randomInt(model.rng, 1, labels.size()) - 1; - model.update(line, labels.get(i), lr); - } - - public void cbow(Model model, float lr, final List line) { - List bow = new ArrayList(); - for (int w = 0; w < line.size(); w++) { - int boundary = Utils.randomInt(model.rng, 1, args_.ws); - bow.clear(); - for (int c = -boundary; c <= boundary; c++) { - if (c != 0 && w + c >= 0 && w + c < line.size()) { - final List ngrams = dict_.getNgrams(line.get(w + c)); - bow.addAll(ngrams); - } - } - model.update(bow, line.get(w), lr); - } - } - - public void skipgram(Model model, float lr, final List line) { - for (int w = 0; w < line.size(); w++) { - int boundary = Utils.randomInt(model.rng, 1, args_.ws); - final List ngrams = dict_.getNgrams(line.get(w)); - for (int c = -boundary; c <= boundary; c++) { - if (c != 0 && w + c >= 0 && w + c < line.size()) { - model.update(ngrams, line.get(w + c), lr); - } - } - } - } - - public void test(InputStream in, int k) throws IOException, Exception { - int nexamples = 0, nlabels = 0; - double precision = 0.0f; - List line = new ArrayList(); - List labels = new ArrayList(); - - LineReader lineReader = null; - try { - lineReader = lineReaderClass_.getConstructor(InputStream.class, String.class).newInstance(in, charsetName_); - String[] lineTokens; - while ((lineTokens = lineReader.readLineTokens()) != null) { - if (lineTokens.length == 1 && "quit".equals(lineTokens[0])) { - break; - } - dict_.getLine(lineTokens, line, labels, model_.rng); - dict_.addNgrams(line, args_.wordNgrams); - if (labels.size() > 0 && line.size() > 0) { - List> modelPredictions = new ArrayList>(); - model_.predict(line, k, modelPredictions); - for (Pair pair : modelPredictions) { - if (labels.contains(pair.getValue())) { - precision += 1.0f; - } - } - nexamples++; - nlabels += labels.size(); - // } else { - // System.out.println("FAIL Test line: " + lineTokens + - // "labels: " + labels + " line: " + line); - } - } - } finally { - if (lineReader != null) { - lineReader.close(); - } - } - - System.out.printf("P@%d: %.3f%n", k, precision / (k * nexamples)); - System.out.printf("R@%d: %.3f%n", k, precision / nlabels); - System.out.println("Number of examples: " + nexamples); - } - - /** - * Thread-safe predict api - * - * @param lineTokens - * @param k - * @return - */ - public List> predict(String[] lineTokens, int k) { - List words = new ArrayList(); - List labels = new ArrayList(); - dict_.getLine(lineTokens, words, labels, model_.rng); - dict_.addNgrams(words, args_.wordNgrams); - - if (words.isEmpty()) { - return null; - } - - Vector hidden = new Vector(args_.dim); - Vector output = new Vector(dict_.nlabels()); - List> modelPredictions = new ArrayList>(k + 1); - - model_.predict(words, k, modelPredictions, hidden, output); - - List> predictions = new ArrayList>(k); - for (Pair pair : modelPredictions) { - predictions.add(new Pair(pair.getKey(), dict_.getLabel(pair.getValue()))); - } - return predictions; - } - - public void predict(String[] lineTokens, int k, List> predictions) throws IOException { - List words = new ArrayList(); - List labels = new ArrayList(); - dict_.getLine(lineTokens, words, labels, model_.rng); - dict_.addNgrams(words, args_.wordNgrams); - - if (words.isEmpty()) { - return; - } - List> modelPredictions = new ArrayList>(k + 1); - model_.predict(words, k, modelPredictions); - predictions.clear(); - for (Pair pair : modelPredictions) { - predictions.add(new Pair(pair.getKey(), dict_.getLabel(pair.getValue()))); - } - } - - public void predict(InputStream in, int k, boolean print_prob) throws IOException, Exception { - List> predictions = new ArrayList>(k); - - LineReader lineReader = null; - - try { - lineReader = lineReaderClass_.getConstructor(InputStream.class, String.class).newInstance(in, charsetName_); - String[] lineTokens; - while ((lineTokens = lineReader.readLineTokens()) != null) { - if (lineTokens.length == 1 && "quit".equals(lineTokens[0])) { - break; - } - predictions.clear(); - predict(lineTokens, k, predictions); - if (predictions.isEmpty()) { - System.out.println("n/a"); - continue; - } - for (Pair pair : predictions) { - System.out.print(pair.getValue()); - if (print_prob) { - System.out.printf(" %f", Math.exp(pair.getKey())); - } - } - System.out.println(); - } - } finally { - if (lineReader != null) { - lineReader.close(); - } - } - } - - public void wordVectors() { - Vector vec = new Vector(args_.dim); - LineReader lineReader = null; - try { - lineReader = lineReaderClass_.getConstructor(InputStream.class, String.class).newInstance(System.in, - charsetName_); - String word; - while (!Utils.isEmpty((word = lineReader.readLine()))) { - getVector(vec, word); - System.out.println(word + " " + vec); - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (lineReader != null) { - try { - lineReader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - public void textVectors() { - List line = new ArrayList(); - List labels = new ArrayList(); - Vector vec = new Vector(args_.dim); - LineReader lineReader = null; - try { - lineReader = lineReaderClass_.getConstructor(InputStream.class, String.class).newInstance(System.in, - charsetName_); - String[] lineTokens; - while ((lineTokens = lineReader.readLineTokens()) != null) { - if (lineTokens.length == 1 && "quit".equals(lineTokens[0])) { - break; - } - dict_.getLine(lineTokens, line, labels, model_.rng); - dict_.addNgrams(line, args_.wordNgrams); - vec.zero(); - for (Integer it : line) { - vec.addRow(input_, it); - } - if (!line.isEmpty()) { - vec.mul(1.0f / line.size()); - } - System.out.println(vec); - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (lineReader != null) { - try { - lineReader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - public void printVectors() { - if (args_.model == model_name.sup) { - textVectors(); - } else { - wordVectors(); - } - } - - public class TrainThread extends Thread { - final FastText ft; - int threadId; - - public TrainThread(FastText ft, int threadId) { - super("FT-TrainThread-" + threadId); - this.ft = ft; - this.threadId = threadId; - } - - public void run() { - if (args_.verbose > 2) { - System.out.println("thread: " + threadId + " RUNNING!"); - } - Exception catchedException = null; - LineReader lineReader = null; - try { - lineReader = lineReaderClass_.getConstructor(String.class, String.class).newInstance(args_.input, - charsetName_); - lineReader.skipLine(threadId * threadFileSize / args_.thread); - Model model = new Model(input_, output_, args_, threadId); - if (args_.model == model_name.sup) { - model.setTargetCounts(dict_.getCounts(entry_type.label)); - } else { - model.setTargetCounts(dict_.getCounts(entry_type.word)); - } - - final long ntokens = dict_.ntokens(); - long localTokenCount = 0; - - List line = new ArrayList(); - List labels = new ArrayList(); - - String[] lineTokens; - while (tokenCount_.get() < args_.epoch * ntokens) { - lineTokens = lineReader.readLineTokens(); - if (lineTokens == null) { - try { - lineReader.rewind(); - if (args_.verbose > 2) { - System.out.println("Input file reloaded!"); - } - } catch (Exception e) { - e.printStackTrace(); - } - lineTokens = lineReader.readLineTokens(); - } - - float progress = (float) (tokenCount_.get()) / (args_.epoch * ntokens); - float lr = (float) (args_.lr * (1.0 - progress)); - localTokenCount += dict_.getLine(lineTokens, line, labels, model.rng); - if (args_.model == model_name.sup) { - dict_.addNgrams(line, args_.wordNgrams); - if (labels.size() == 0 || line.size() == 0) { - continue; - } - supervised(model, lr, line, labels); - } else if (args_.model == model_name.cbow) { - cbow(model, lr, line); - } else if (args_.model == model_name.sg) { - skipgram(model, lr, line); - } - if (localTokenCount > args_.lrUpdateRate) { - tokenCount_.addAndGet(localTokenCount); - localTokenCount = 0; - if (threadId == 0 && args_.verbose > 1 && (System.currentTimeMillis() - start_) % 1000 == 0) { - printInfo(progress, model.getLoss()); - } - } - } - - if (threadId == 0 && args_.verbose > 1) { - printInfo(1.0f, model.getLoss()); - } - } catch (Exception e) { - catchedException = e; - } finally { - if (lineReader != null) - try { - lineReader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // exit from thread - synchronized (ft) { - if (args_.verbose > 2) { - System.out.println("\nthread: " + threadId + " EXIT!"); - } - ft.threadCount--; - ft.notify(); - if (catchedException != null) { - throw new RuntimeException(catchedException); - } - } - } - } - - public void loadVectors(String filename) throws IOException { - List words; - Matrix mat; // temp. matrix for pretrained vectors - int n, dim; - - BufferedReader dis = null; - String line; - String[] lineParts; - try { - dis = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8")); - - line = dis.readLine(); - lineParts = line.split(" "); - n = Integer.parseInt(lineParts[0]); - dim = Integer.parseInt(lineParts[1]); - - words = new ArrayList(n); - - if (dim != args_.dim) { - throw new IllegalArgumentException( - "Dimension of pretrained vectors does not match args -dim option, pretrain dim is " + dim - + ", args dim is " + args_.dim); - } - - mat = new Matrix(n, dim); - for (int i = 0; i < n; i++) { - line = dis.readLine(); - lineParts = line.split(" "); - String word = lineParts[0]; - for (int j = 1; j <= dim; j++) { - mat.data_[i][j - 1] = Float.parseFloat(lineParts[j]); - } - words.add(word); - dict_.add(word); - } - - dict_.threshold(1, 0); - input_ = new Matrix(dict_.nwords() + args_.bucket, args_.dim); - input_.uniform(1.0f / args_.dim); - for (int i = 0; i < n; i++) { - int idx = dict_.getId(words.get(i)); - if (idx < 0 || idx >= dict_.nwords()) - continue; - for (int j = 0; j < dim; j++) { - input_.data_[idx][j] = mat.data_[i][j]; - } - } - - } catch (IOException e) { - throw new IOException("Pretrained vectors file cannot be opened!", e); - } finally { - try { - if (dis != null) { - dis.close(); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - int threadCount; - long threadFileSize; - - public void train(Args args) throws IOException, Exception { - args_ = args; - dict_ = new Dictionary(args_); - dict_.setCharsetName(charsetName_); - dict_.setLineReaderClass(lineReaderClass_); - - if ("-".equals(args_.input)) { - throw new IOException("Cannot use stdin for training!"); - } - - File file = new File(args_.input); - if (!(file.exists() && file.isFile() && file.canRead())) { - throw new IOException("Input file cannot be opened! " + args_.input); - } - - dict_.readFromFile(args_.input); - threadFileSize = Utils.sizeLine(args_.input); - - if (!Utils.isEmpty(args_.pretrainedVectors)) { - loadVectors(args_.pretrainedVectors); - } else { - input_ = new Matrix(dict_.nwords() + args_.bucket, args_.dim); - input_.uniform(1.0f / args_.dim); - } - - if (args_.model == model_name.sup) { - output_ = new Matrix(dict_.nlabels(), args_.dim); - } else { - output_ = new Matrix(dict_.nwords(), args_.dim); - } - output_.zero(); - - start_ = System.currentTimeMillis(); - tokenCount_ = new AtomicLong(0); - long t0 = System.currentTimeMillis(); - threadCount = args_.thread; - for (int i = 0; i < args_.thread; i++) { - Thread t = new TrainThread(this, i); - t.setUncaughtExceptionHandler(trainThreadExcpetionHandler); - t.start(); - } - - synchronized (this) { - while (threadCount > 0) { - try { - wait(); - } catch (InterruptedException ignored) { - } - } - } - - model_ = new Model(input_, output_, args_, 0); - - if (args.verbose > 1) { - long trainTime = (System.currentTimeMillis() - t0) / 1000; - System.out.printf("\nTrain time used: %d sec\n", trainTime); - } - - saveModel(); - if (args_.model != model_name.sup) { - saveVectors(); - } - } - - protected Thread.UncaughtExceptionHandler trainThreadExcpetionHandler = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { - ex.printStackTrace(); - } - }; - - public Args getArgs() { - return args_; - } - - public Dictionary getDict() { - return dict_; - } - - public Matrix getInput() { - return input_; - } - - public Matrix getOutput() { - return output_; - } - - public Model getModel() { - return model_; - } - - public void setArgs(Args args) { - this.args_ = args; - } - - public void setDict(Dictionary dict) { - this.dict_ = dict; - } - - public void setInput(Matrix input) { - this.input_ = input; - } - - public void setOutput(Matrix output) { - this.output_ = output; - } - - public void setModel(Model model) { - this.model_ = model; - } - - public String getCharsetName() { - return charsetName_; - } - - public Class getLineReaderClass() { - return lineReaderClass_; - } - - public void setCharsetName(String charsetName) { - this.charsetName_ = charsetName; - } - - public void setLineReaderClass(Class lineReaderClass) { - this.lineReaderClass_ = lineReaderClass; - } - -} diff --git a/src/main/java/fasttext/IOUtil.java b/src/main/java/fasttext/IOUtil.java deleted file mode 100644 index beb76e2..0000000 --- a/src/main/java/fasttext/IOUtil.java +++ /dev/null @@ -1,146 +0,0 @@ -package fasttext; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * Read/write cpp primitive type - * - * @author Ivan - * - */ -public class IOUtil { - - public IOUtil() { - } - - private int string_buf_size_ = 128; - private byte[] int_bytes_ = new byte[4]; - private byte[] long_bytes_ = new byte[8]; - private byte[] float_bytes_ = new byte[4]; - private byte[] double_bytes_ = new byte[8]; - private byte[] string_bytes_ = new byte[string_buf_size_]; - private StringBuilder stringBuilder_ = new StringBuilder(); - private ByteBuffer float_array_bytebuffer_ = null; - private byte[] float_array_bytes_ = null; - - public void setStringBufferSize(int size) { - string_buf_size_ = size; - string_bytes_ = new byte[string_buf_size_]; - } - - public void setFloatArrayBufferSize(int itemSize) { - float_array_bytebuffer_ = ByteBuffer.allocate(itemSize * 4).order(ByteOrder.LITTLE_ENDIAN); - float_array_bytes_ = new byte[itemSize * 4]; - } - - public int readByte(InputStream is) throws IOException { - return is.read() & 0xFF; - } - - public int readInt(InputStream is) throws IOException { - is.read(int_bytes_); - return getInt(int_bytes_); - } - - public int getInt(byte[] b) { - return (b[0] & 0xFF) << 0 | (b[1] & 0xFF) << 8 | (b[2] & 0xFF) << 16 | (b[3] & 0xFF) << 24; - } - - public long readLong(InputStream is) throws IOException { - is.read(long_bytes_); - return getLong(long_bytes_); - } - - public long getLong(byte[] b) { - return (b[0] & 0xFFL) << 0 | (b[1] & 0xFFL) << 8 | (b[2] & 0xFFL) << 16 | (b[3] & 0xFFL) << 24 - | (b[4] & 0xFFL) << 32 | (b[5] & 0xFFL) << 40 | (b[6] & 0xFFL) << 48 | (b[7] & 0xFFL) << 56; - } - - public float readFloat(InputStream is) throws IOException { - is.read(float_bytes_); - return getFloat(float_bytes_); - } - - public void readFloat(InputStream is, float[] data) throws IOException { - is.read(float_array_bytes_); - float_array_bytebuffer_.clear(); - ((ByteBuffer) float_array_bytebuffer_.put(float_array_bytes_).flip()).asFloatBuffer().get(data); - } - - public float getFloat(byte[] b) { - return Float - .intBitsToFloat((b[0] & 0xFF) << 0 | (b[1] & 0xFF) << 8 | (b[2] & 0xFF) << 16 | (b[3] & 0xFF) << 24); - } - - public double readDouble(InputStream is) throws IOException { - is.read(double_bytes_); - return getDouble(double_bytes_); - } - - public double getDouble(byte[] b) { - return Double.longBitsToDouble(getLong(b)); - } - - public String readString(InputStream is) throws IOException { - int b = is.read(); - if (b < 0) { - return null; - } - int i = -1; - stringBuilder_.setLength(0); - // ascii space, \n, \0 - while (b > -1 && b != 32 && b != 10 && b != 0) { - string_bytes_[++i] = (byte) b; - b = is.read(); - if (i == string_buf_size_ - 1) { - stringBuilder_.append(new String(string_bytes_)); - i = -1; - } - } - stringBuilder_.append(new String(string_bytes_, 0, i + 1)); - return stringBuilder_.toString(); - } - - public int intToByte(int i) { - return (i & 0xFF); - } - - public byte[] intToByteArray(int i) { - int_bytes_[0] = (byte) ((i >> 0) & 0xff); - int_bytes_[1] = (byte) ((i >> 8) & 0xff); - int_bytes_[2] = (byte) ((i >> 16) & 0xff); - int_bytes_[3] = (byte) ((i >> 24) & 0xff); - return int_bytes_; - } - - public byte[] longToByteArray(long l) { - long_bytes_[0] = (byte) ((l >> 0) & 0xff); - long_bytes_[1] = (byte) ((l >> 8) & 0xff); - long_bytes_[2] = (byte) ((l >> 16) & 0xff); - long_bytes_[3] = (byte) ((l >> 24) & 0xff); - long_bytes_[4] = (byte) ((l >> 32) & 0xff); - long_bytes_[5] = (byte) ((l >> 40) & 0xff); - long_bytes_[6] = (byte) ((l >> 48) & 0xff); - long_bytes_[7] = (byte) ((l >> 56) & 0xff); - - return long_bytes_; - } - - public byte[] floatToByteArray(float f) { - return intToByteArray(Float.floatToIntBits(f)); - } - - public byte[] floatToByteArray(float[] f) { - float_array_bytebuffer_.clear(); - float_array_bytebuffer_.asFloatBuffer().put(f); - return float_array_bytebuffer_.array(); - } - - public byte[] doubleToByteArray(double d) { - return longToByteArray(Double.doubleToRawLongBits(d)); - } - -} diff --git a/src/main/java/fasttext/Main.java b/src/main/java/fasttext/Main.java deleted file mode 100644 index 2607704..0000000 --- a/src/main/java/fasttext/Main.java +++ /dev/null @@ -1,138 +0,0 @@ -package fasttext; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; - -public class Main { - - public static void printUsage() { - System.out.print("usage: java -jar fasttext.jar \n\n" - + "The commands supported by fasttext are:\n\n" - + " supervised train a supervised classifier\n" - + " test evaluate a supervised classifier\n" - + " predict predict most likely labels\n" - + " predict-prob predict most likely labels with probabilities\n" - + " skipgram train a skipgram model\n" - + " cbow train a cbow model\n" - + " print-vectors print vectors given a trained model\n"); - } - - public static void printTestUsage() { - System.out.print("usage: java -jar fasttext.jar test []\n\n" - + " model filename\n" - + " test data filename (if -, read from stdin)\n" - + " (optional; 1 by default) predict top k labels\n"); - } - - public static void printPredictUsage() { - System.out.print("usage: java -jar fasttext.jar predict[-prob] []\n\n" - + " model filename\n" - + " test data filename (if -, read from stdin)\n" - + " (optional; 1 by default) predict top k labels\n"); - } - - public static void printPrintVectorsUsage() { - System.out.print("usage: java -jar fasttext.jar print-vectors \n\n" - + " model filename\n"); - } - - public void test(String[] args) throws IOException, Exception { - int k = 1; - if (args.length == 3) { - k = 1; - } else if (args.length == 4) { - k = Integer.parseInt(args[3]); - } else { - printTestUsage(); - System.exit(1); - } - FastText fasttext = new FastText(); - fasttext.loadModel(args[1]); - String infile = args[2]; - if ("-".equals(infile)) { - fasttext.test(System.in, k); - } else { - File file = new File(infile); - if (!(file.exists() && file.isFile() && file.canRead())) { - throw new IOException("Test file cannot be opened!"); - } - fasttext.test(new FileInputStream(file), k); - } - } - - public void predict(String[] args) throws IOException, Exception { - int k = 1; - if (args.length == 3) { - k = 1; - } else if (args.length == 4) { - k = Integer.parseInt(args[3]); - } else { - printPredictUsage(); - System.exit(1); - } - boolean print_prob = "predict-prob".equalsIgnoreCase(args[0]); - FastText fasttext = new FastText(); - fasttext.loadModel(args[1]); - - String infile = args[2]; - if ("-".equals(infile)) { - fasttext.predict(System.in, k, print_prob); - } else { - File file = new File(infile); - if (!(file.exists() && file.isFile() && file.canRead())) { - throw new IOException("Input file cannot be opened!"); - } - fasttext.predict(new FileInputStream(file), k, print_prob); - } - } - - public void printVectors(String[] args) throws IOException { - if (args.length != 2) { - printPrintVectorsUsage(); - System.exit(1); - } - FastText fasttext = new FastText(); - fasttext.loadModel(args[1]); - fasttext.printVectors(); - } - - public void train(String[] args) throws IOException, Exception { - Args a = new Args(); - a.parseArgs(args); - FastText fasttext = new FastText(); - fasttext.train(a); - } - - public static void main(String[] args) { - Main op = new Main(); - - if (args.length == 0) { - printUsage(); - System.exit(1); - } - - try { - String command = args[0]; - if ("skipgram".equalsIgnoreCase(command) || "cbow".equalsIgnoreCase(command) - || "supervised".equalsIgnoreCase(command)) { - op.train(args); - } else if ("test".equalsIgnoreCase(command)) { - op.test(args); - } else if ("print-vectors".equalsIgnoreCase(command)) { - op.printVectors(args); - } else if ("predict".equalsIgnoreCase(command) || "predict-prob".equalsIgnoreCase(command)) { - op.predict(args); - } else { - printUsage(); - System.exit(1); - } - } catch (Exception e) { - e.printStackTrace(); - System.exit(1); - } - - System.exit(0); - } - -} diff --git a/src/main/java/fasttext/Matrix.java b/src/main/java/fasttext/Matrix.java deleted file mode 100644 index f93b66c..0000000 --- a/src/main/java/fasttext/Matrix.java +++ /dev/null @@ -1,118 +0,0 @@ -package fasttext; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Random; - -public class Matrix { - - public float[][] data_ = null; - public int m_ = 0; // vocabSize - public int n_ = 0; // layer1Size - - public Matrix() { - } - - public Matrix(int m, int n) { - m_ = m; - n_ = n; - data_ = new float[m][n]; - } - - public Matrix(final Matrix other) { - m_ = other.m_; - n_ = other.n_; - data_ = new float[m_][n_]; - for (int i = 0; i < m_; i++) { - for (int j = 0; j < n_; j++) { - data_[i][j] = other.data_[i][j]; - } - } - } - - public void zero() { - for (int i = 0; i < m_; i++) { - for (int j = 0; j < n_; j++) { - data_[i][j] = 0.0f; - } - } - } - - public void uniform(float a) { - Random random = new Random(1l); - for (int i = 0; i < m_; i++) { - for (int j = 0; j < n_; j++) { - data_[i][j] = Utils.randomFloat(random, -a, a); - } - } - } - - public void addRow(final Vector vec, int i, float a) { - Utils.checkArgument(i >= 0); - Utils.checkArgument(i < m_); - Utils.checkArgument(vec.m_ == n_); - for (int j = 0; j < n_; j++) { - data_[i][j] += a * vec.data_[j]; - } - } - - public float dotRow(final Vector vec, int i) { - Utils.checkArgument(i >= 0); - Utils.checkArgument(i < m_); - Utils.checkArgument(vec.m_ == n_); - float d = 0.0f; - for (int j = 0; j < n_; j++) { - d += data_[i][j] * vec.data_[j]; - } - return d; - } - - public void load(InputStream input) throws IOException { - IOUtil ioutil = new IOUtil(); - - m_ = (int) ioutil.readLong(input); - n_ = (int) ioutil.readLong(input); - - ioutil.setFloatArrayBufferSize(n_); - data_ = new float[m_][n_]; - for (int i = 0; i < m_; i++) { - ioutil.readFloat(input, data_[i]); - } - } - - public void save(OutputStream ofs) throws IOException { - IOUtil ioutil = new IOUtil(); - ioutil.setFloatArrayBufferSize(n_); - ofs.write(ioutil.longToByteArray(m_)); - ofs.write(ioutil.longToByteArray(n_)); - for (int i = 0; i < m_; i++) { - ofs.write(ioutil.floatToByteArray(data_[i])); - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Matrix [data_="); - if (data_ != null) { - builder.append("["); - for (int i = 0; i < m_ && i < 10; i++) { - for (int j = 0; j < n_ && j < 10; j++) { - builder.append(data_[i][j]).append(","); - } - } - builder.setLength(builder.length() - 1); - builder.append("]"); - } else { - builder.append("null"); - } - builder.append(", m_="); - builder.append(m_); - builder.append(", n_="); - builder.append(n_); - builder.append("]"); - return builder.toString(); - } - -} diff --git a/src/main/java/fasttext/Model.java b/src/main/java/fasttext/Model.java deleted file mode 100644 index f09e149..0000000 --- a/src/main/java/fasttext/Model.java +++ /dev/null @@ -1,355 +0,0 @@ -package fasttext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Random; - -import fasttext.Args.loss_name; -import fasttext.Args.model_name; - -public class Model { - - static final int SIGMOID_TABLE_SIZE = 512; - static final int MAX_SIGMOID = 8; - static final int LOG_TABLE_SIZE = 512; - - static final int NEGATIVE_TABLE_SIZE = 10000000; - - public class Node { - int parent; - int left; - int right; - long count; - boolean binary; - } - - private Matrix wi_; // input - private Matrix wo_; // output - private Args args_; - private Vector hidden_; - private Vector output_; - private Vector grad_; - private int hsz_; // dim - @SuppressWarnings("unused") - private int isz_; // input vocabSize - private int osz_; // output vocabSize - private float loss_; - private long nexamples_; - private float[] t_sigmoid; - private float[] t_log; - // used for negative sampling: - private List negatives; - private int negpos; - // used for hierarchical softmax: - private List> paths; - private List> codes; - private List tree; - - public transient Random rng; - - public Model(Matrix wi, Matrix wo, Args args, int seed) { - hidden_ = new Vector(args.dim); - output_ = new Vector(wo.m_); - grad_ = new Vector(args.dim); - rng = new Random((long) seed); - - wi_ = wi; - wo_ = wo; - args_ = args; - isz_ = wi.m_; - osz_ = wo.m_; - hsz_ = args.dim; - negpos = 0; - loss_ = 0.0f; - nexamples_ = 1l; - initSigmoid(); - initLog(); - } - - public float binaryLogistic(int target, boolean label, float lr) { - float score = sigmoid(wo_.dotRow(hidden_, target)); - float alpha = lr * ((label ? 1.0f : 0.0f) - score); - grad_.addRow(wo_, target, alpha); - wo_.addRow(hidden_, target, alpha); - if (label) { - return -log(score); - } else { - return -log(1.0f - score); - } - } - - public float negativeSampling(int target, float lr) { - float loss = 0.0f; - grad_.zero(); - for (int n = 0; n <= args_.neg; n++) { - if (n == 0) { - loss += binaryLogistic(target, true, lr); - } else { - loss += binaryLogistic(getNegative(target), false, lr); - } - } - return loss; - } - - public float hierarchicalSoftmax(int target, float lr) { - float loss = 0.0f; - grad_.zero(); - final List binaryCode = codes.get(target); - final List pathToRoot = paths.get(target); - for (int i = 0; i < pathToRoot.size(); i++) { - loss += binaryLogistic(pathToRoot.get(i), binaryCode.get(i), lr); - } - return loss; - } - - public void computeOutputSoftmax(Vector hidden, Vector output) { - output.mul(wo_, hidden); - float max = output.get(0), z = 0.0f; - for (int i = 1; i < osz_; i++) { - max = Math.max(output.get(i), max); - } - for (int i = 0; i < osz_; i++) { - output.set(i, (float) Math.exp(output.get(i) - max)); - z += output.get(i); - } - for (int i = 0; i < osz_; i++) { - output.set(i, output.get(i) / z); - } - } - - public void computeOutputSoftmax() { - computeOutputSoftmax(hidden_, output_); - } - - public float softmax(int target, float lr) { - grad_.zero(); - computeOutputSoftmax(); - for (int i = 0; i < osz_; i++) { - float label = (i == target) ? 1.0f : 0.0f; - float alpha = lr * (label - output_.get(i)); - grad_.addRow(wo_, i, alpha); - wo_.addRow(hidden_, i, alpha); - } - return -log(output_.get(target)); - } - - public void computeHidden(final List input, Vector hidden) { - Utils.checkArgument(hidden.size() == hsz_); - hidden.zero(); - for (Integer it : input) { - hidden.addRow(wi_, it); - } - hidden.mul(1.0f / input.size()); - } - - private Comparator> comparePairs = new Comparator>() { - - @Override - public int compare(Pair o1, Pair o2) { - return o2.getKey().compareTo(o1.getKey()); - } - }; - - public void predict(final List input, int k, List> heap, Vector hidden, - Vector output) { - Utils.checkArgument(k > 0); - if (heap instanceof ArrayList) { - ((ArrayList>) heap).ensureCapacity(k + 1); - } - computeHidden(input, hidden); - if (args_.loss == loss_name.hs) { - dfs(k, 2 * osz_ - 2, 0.0f, heap, hidden); - } else { - findKBest(k, heap, hidden, output); - } - Collections.sort(heap, comparePairs); - } - - public void predict(final List input, int k, List> heap) { - predict(input, k, heap, hidden_, output_); - } - - public void findKBest(int k, List> heap, Vector hidden, Vector output) { - computeOutputSoftmax(hidden, output); - for (int i = 0; i < osz_; i++) { - if (heap.size() == k && log(output.get(i)) < heap.get(heap.size() - 1).getKey()) { - continue; - } - heap.add(new Pair(log(output.get(i)), i)); - Collections.sort(heap, comparePairs); - if (heap.size() > k) { - Collections.sort(heap, comparePairs); - heap.remove(heap.size() - 1); // pop last - } - } - } - - public void dfs(int k, int node, float score, List> heap, Vector hidden) { - if (heap.size() == k && score < heap.get(heap.size() - 1).getKey()) { - return; - } - - if (tree.get(node).left == -1 && tree.get(node).right == -1) { - heap.add(new Pair(score, node)); - Collections.sort(heap, comparePairs); - if (heap.size() > k) { - Collections.sort(heap, comparePairs); - heap.remove(heap.size() - 1); // pop last - } - return; - } - - float f = sigmoid(wo_.dotRow(hidden, node - osz_)); - dfs(k, tree.get(node).left, score + log(1.0f - f), heap, hidden); - dfs(k, tree.get(node).right, score + log(f), heap, hidden); - } - - public void update(final List input, int target, float lr) { - Utils.checkArgument(target >= 0); - Utils.checkArgument(target < osz_); - if (input.size() == 0) { - return; - } - computeHidden(input, hidden_); - - if (args_.loss == loss_name.ns) { - loss_ += negativeSampling(target, lr); - } else if (args_.loss == loss_name.hs) { - loss_ += hierarchicalSoftmax(target, lr); - } else { - loss_ += softmax(target, lr); - } - nexamples_ += 1; - - if (args_.model == model_name.sup) { - grad_.mul(1.0f / input.size()); - } - for (Integer it : input) { - wi_.addRow(grad_, it, 1.0f); - } - } - - public void setTargetCounts(final List counts) { - Utils.checkArgument(counts.size() == osz_); - if (args_.loss == loss_name.ns) { - initTableNegatives(counts); - } - if (args_.loss == loss_name.hs) { - buildTree(counts); - } - } - - public void initTableNegatives(final List counts) { - negatives = new ArrayList(counts.size()); - float z = 0.0f; - for (int i = 0; i < counts.size(); i++) { - z += (float) Math.pow(counts.get(i), 0.5f); - } - for (int i = 0; i < counts.size(); i++) { - float c = (float) Math.pow(counts.get(i), 0.5f); - for (int j = 0; j < c * NEGATIVE_TABLE_SIZE / z; j++) { - negatives.add(i); - } - } - Utils.shuffle(negatives, rng); - } - - public int getNegative(int target) { - int negative; - do { - negative = negatives.get(negpos); - negpos = (negpos + 1) % negatives.size(); - } while (target == negative); - return negative; - } - - public void buildTree(final List counts) { - paths = new ArrayList>(osz_); - codes = new ArrayList>(osz_); - tree = new ArrayList(2 * osz_ - 1); - - for (int i = 0; i < 2 * osz_ - 1; i++) { - Node node = new Node(); - node.parent = -1; - node.left = -1; - node.right = -1; - node.count = 1000000000000000L;// 1e15f; - node.binary = false; - tree.add(i, node); - } - for (int i = 0; i < osz_; i++) { - tree.get(i).count = counts.get(i); - } - int leaf = osz_ - 1; - int node = osz_; - for (int i = osz_; i < 2 * osz_ - 1; i++) { - int[] mini = new int[2]; - for (int j = 0; j < 2; j++) { - if (leaf >= 0 && tree.get(leaf).count < tree.get(node).count) { - mini[j] = leaf--; - } else { - mini[j] = node++; - } - } - tree.get(i).left = mini[0]; - tree.get(i).right = mini[1]; - tree.get(i).count = tree.get(mini[0]).count + tree.get(mini[1]).count; - tree.get(mini[0]).parent = i; - tree.get(mini[1]).parent = i; - tree.get(mini[1]).binary = true; - } - for (int i = 0; i < osz_; i++) { - List path = new ArrayList(); - List code = new ArrayList(); - int j = i; - while (tree.get(j).parent != -1) { - path.add(tree.get(j).parent - osz_); - code.add(tree.get(j).binary); - j = tree.get(j).parent; - } - paths.add(path); - codes.add(code); - } - } - - public float getLoss() { - return loss_ / nexamples_; - } - - private void initSigmoid() { - t_sigmoid = new float[SIGMOID_TABLE_SIZE + 1]; - for (int i = 0; i < SIGMOID_TABLE_SIZE + 1; i++) { - float x = (float) (i * 2 * MAX_SIGMOID) / SIGMOID_TABLE_SIZE - MAX_SIGMOID; - t_sigmoid[i] = (float) (1.0f / (1.0f + Math.exp(-x))); - } - } - - private void initLog() { - t_log = new float[LOG_TABLE_SIZE + 1]; - for (int i = 0; i < LOG_TABLE_SIZE + 1; i++) { - float x = (float) (((float) (i) + 1e-5f) / LOG_TABLE_SIZE); - t_log[i] = (float) Math.log(x); - } - } - - public float log(float x) { - if (x > 1.0f) { - return 0.0f; - } - int i = (int) (x * LOG_TABLE_SIZE); - return t_log[i]; - } - - public float sigmoid(float x) { - if (x < -MAX_SIGMOID) { - return 0.0f; - } else if (x > MAX_SIGMOID) { - return 1.0f; - } else { - int i = (int) ((x + MAX_SIGMOID) * SIGMOID_TABLE_SIZE / MAX_SIGMOID / 2); - return t_sigmoid[i]; - } - } -} diff --git a/src/main/java/fasttext/Pair.java b/src/main/java/fasttext/Pair.java deleted file mode 100644 index 86cd1b2..0000000 --- a/src/main/java/fasttext/Pair.java +++ /dev/null @@ -1,29 +0,0 @@ -package fasttext; - -public class Pair { - - private K key_; - private V value_; - - public Pair(K key, V value) { - this.key_ = key; - this.value_ = value; - } - - public K getKey() { - return key_; - } - - public V getValue() { - return value_; - } - - public void setKey(K key) { - this.key_ = key; - } - - public void setValue(V value) { - this.value_ = value; - } - -} diff --git a/src/main/java/fasttext/Utils.java b/src/main/java/fasttext/Utils.java deleted file mode 100644 index 49e510b..0000000 --- a/src/main/java/fasttext/Utils.java +++ /dev/null @@ -1,148 +0,0 @@ -package fasttext; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Random; -import java.util.RandomAccess; - -public class Utils { - - /** - * Ensures the truth of an expression involving one or more parameters to - * the calling method. - * - * @param expression - * a boolean expression - * @throws IllegalArgumentException - * if {@code expression} is false - */ - public static void checkArgument(boolean expression) { - if (!expression) { - throw new IllegalArgumentException(); - } - } - - public static void checkArgument(boolean expression, String message) { - if (!expression) { - throw new IllegalArgumentException(message); - } - } - - public static boolean isEmpty(String str) { - return (str == null || str.isEmpty()); - } - - public static V mapGetOrDefault(Map map, K key, V defaultValue) { - return map.containsKey(key) ? map.get(key) : defaultValue; - } - - public static int randomInt(Random rnd, int lower, int upper) { - checkArgument(lower <= upper & lower > 0); - if (lower == upper) { - return lower; - } - return rnd.nextInt(upper - lower) + lower; - } - - public static float randomFloat(Random rnd, float lower, float upper) { - checkArgument(lower <= upper); - if (lower == upper) { - return lower; - } - return (rnd.nextFloat() * (upper - lower)) + lower; - } - - public static long sizeLine(String filename) throws IOException { - InputStream is = new BufferedInputStream(new FileInputStream(filename)); - try { - byte[] c = new byte[1024]; - long count = 0; - int readChars = 0; - boolean endsWithoutNewLine = false; - while ((readChars = is.read(c)) != -1) { - for (int i = 0; i < readChars; ++i) { - if (c[i] == '\n') - ++count; - } - endsWithoutNewLine = (c[readChars - 1] != '\n'); - } - if (endsWithoutNewLine) { - ++count; - } - return count; - } finally { - is.close(); - } - } - - /** - * - * @param br - * @param pos - * line numbers start from 1 - * @throws IOException - */ - public static void seekLine(BufferedReader br, long pos) throws IOException { - // br.reset(); - String line; - int currentLine = 1; - while (currentLine < pos && (line = br.readLine()) != null) { - if (Utils.isEmpty(line) || line.startsWith("#")) { - continue; - } - currentLine++; - } - } - - private static final int SHUFFLE_THRESHOLD = 5; - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static void shuffle(List list, Random rnd) { - int size = list.size(); - if (size < SHUFFLE_THRESHOLD || list instanceof RandomAccess) { - for (int i = size; i > 1; i--) - swap(list, i - 1, rnd.nextInt(i)); - } else { - Object arr[] = list.toArray(); - - // Shuffle array - for (int i = size; i > 1; i--) - swap(arr, i - 1, rnd.nextInt(i)); - - // Dump array back into list - // instead of using a raw type here, it's possible to capture - // the wildcard but it will require a call to a supplementary - // private method - ListIterator it = list.listIterator(); - for (int i = 0; i < arr.length; i++) { - it.next(); - it.set(arr[i]); - } - } - } - - /** - * Swaps the two specified elements in the specified array. - */ - public static void swap(Object[] arr, int i, int j) { - Object tmp = arr[i]; - arr[i] = arr[j]; - arr[j] = tmp; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static void swap(List list, int i, int j) { - // instead of using a raw type here, it's possible to capture - // the wildcard but it will require a call to a supplementary - // private method - final List l = list; - l.set(i, l.set(j, l.get(i))); - } - -} diff --git a/src/main/java/fasttext/Vector.java b/src/main/java/fasttext/Vector.java deleted file mode 100644 index ccd5d52..0000000 --- a/src/main/java/fasttext/Vector.java +++ /dev/null @@ -1,90 +0,0 @@ -package fasttext; - -public class Vector { - - public int m_; - public float[] data_; - - public Vector(int size) { - m_ = size; - data_ = new float[size]; - } - - public int size() { - return m_; - } - - public void zero() { - for (int i = 0; i < m_; i++) { - data_[i] = 0.0f; - } - } - - public void mul(float a) { - for (int i = 0; i < m_; i++) { - data_[i] *= a; - } - } - - public void addRow(final Matrix A, int i) { - Utils.checkArgument(i >= 0); - Utils.checkArgument(i < A.m_); - Utils.checkArgument(m_ == A.n_); - for (int j = 0; j < A.n_; j++) { // layer size - data_[j] += A.data_[i][j]; - } - } - - public void addRow(final Matrix A, int i, float a) { - Utils.checkArgument(i >= 0); - Utils.checkArgument(i < A.m_); - Utils.checkArgument(m_ == A.n_); - for (int j = 0; j < A.n_; j++) { - data_[j] += a * A.data_[i][j]; - } - } - - public void mul(final Matrix A, final Vector vec) { - Utils.checkArgument(A.m_ == m_); - Utils.checkArgument(A.n_ == vec.m_); - for (int i = 0; i < m_; i++) { - data_[i] = 0.0f; - for (int j = 0; j < A.n_; j++) { - data_[i] += A.data_[i][j] * vec.data_[j]; - } - } - } - - public int argmax() { - float max = data_[0]; - int argmax = 0; - for (int i = 1; i < m_; i++) { - if (data_[i] > max) { - max = data_[i]; - argmax = i; - } - } - return argmax; - } - - public float get(int i) { - return data_[i]; - } - - public void set(int i, float value) { - data_[i] = value; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - for (float data : data_) { - builder.append(data).append(' '); - } - if (builder.length() > 1) { - builder.setLength(builder.length() - 1); - } - return builder.toString(); - } - -} diff --git a/src/main/java/fasttext/io/BufferedLineReader.java b/src/main/java/fasttext/io/BufferedLineReader.java deleted file mode 100644 index 0f86edf..0000000 --- a/src/main/java/fasttext/io/BufferedLineReader.java +++ /dev/null @@ -1,108 +0,0 @@ -package fasttext.io; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; - -public class BufferedLineReader extends LineReader { - - private String lineDelimitingRegex_ = " |\r|\t|\\v|\f|\0"; - - private BufferedReader br_; - - public BufferedLineReader(String filename, String charsetName) throws IOException, UnsupportedEncodingException { - super(filename, charsetName); - FileInputStream fis = new FileInputStream(file_); - br_ = new BufferedReader(new InputStreamReader(fis, charset_)); - } - - public BufferedLineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { - super(inputStream, charsetName); - br_ = new BufferedReader(new InputStreamReader(inputStream, charset_)); - } - - @Override - public long skipLine(long n) throws IOException { - if (n < 0L) { - throw new IllegalArgumentException("skip value is negative"); - } - String line; - long currentLine = 0; - long readLine = 0; - synchronized (lock) { - while (currentLine < n && (line = br_.readLine()) != null) { - readLine++; - if (line == null || line.isEmpty() || line.startsWith("#")) { - continue; - } - currentLine++; - } - return readLine; - } - } - - @Override - public String readLine() throws IOException { - synchronized (lock) { - String lineString = br_.readLine(); - while (lineString != null && (lineString.isEmpty() || lineString.startsWith("#"))) { - lineString = br_.readLine(); - } - return lineString; - } - } - - @Override - public String[] readLineTokens() throws IOException { - String line = readLine(); - if (line == null) - return null; - else - return line.split(lineDelimitingRegex_, -1); - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - synchronized (lock) { - return br_.read(cbuf, off, len); - } - } - - @Override - public void close() throws IOException { - synchronized (lock) { - if (br_ != null) { - br_.close(); - } - } - } - - @Override - public void rewind() throws IOException { - synchronized (lock) { - if (br_ != null) { - br_.close(); - } - if (file_ != null) { - FileInputStream fis = new FileInputStream(file_); - br_ = new BufferedReader(new InputStreamReader(fis, charset_)); - } else { - // br = new BufferedReader(new InputStreamReader(inputStream, - // charset)); - throw new UnsupportedOperationException("InputStream rewind not supported"); - } - } - } - - public String getLineDelimitingRege() { - return lineDelimitingRegex_; - } - - public void setLineDelimitingRegex(String lineDelimitingRegex) { - this.lineDelimitingRegex_ = lineDelimitingRegex; - } - -} diff --git a/src/main/java/fasttext/io/LineReader.java b/src/main/java/fasttext/io/LineReader.java deleted file mode 100644 index d1bbe71..0000000 --- a/src/main/java/fasttext/io/LineReader.java +++ /dev/null @@ -1,54 +0,0 @@ -package fasttext.io; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.UnsupportedEncodingException; -import java.nio.charset.Charset; - -public abstract class LineReader extends Reader { - - protected InputStream inputStream_ = null; - protected File file_ = null; - protected Charset charset_ = null; - - protected LineReader() { - super(); - } - - protected LineReader(Object lock) { - super(lock); - } - - public LineReader(String filename, String charsetName) throws IOException, UnsupportedEncodingException { - this(); - this.file_ = new File(filename); - this.charset_ = Charset.forName(charsetName); - } - - public LineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { - this(); - this.inputStream_ = inputStream; - this.charset_ = Charset.forName(charsetName); - } - - /** - * Skips lines. - * - * @param n - * The number of lines to skip - * @return The number of lines actually skipped - * @exception IOException - * If an I/O error occurs - * @exception IllegalArgumentException - * If n is negative. - */ - public abstract long skipLine(long n) throws IOException; - - public abstract String readLine() throws IOException; - - public abstract String[] readLineTokens() throws IOException; - - public abstract void rewind() throws IOException; -} diff --git a/src/main/java/fasttext/io/MappedByteBufferLineReader.java b/src/main/java/fasttext/io/MappedByteBufferLineReader.java deleted file mode 100644 index c1899b1..0000000 --- a/src/main/java/fasttext/io/MappedByteBufferLineReader.java +++ /dev/null @@ -1,248 +0,0 @@ -package fasttext.io; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.List; - -public class MappedByteBufferLineReader extends LineReader { - - private static int DEFAULT_BUFFER_SIZE = 1024; - - private volatile ByteBuffer byteBuffer_ = null; // MappedByteBuffer - private RandomAccessFile raf_ = null; - private FileChannel channel_ = null; - private byte[] bytes_ = null; - - private int string_buf_size_ = DEFAULT_BUFFER_SIZE; - private boolean fillLine_ = false; - - private StringBuilder sb_ = new StringBuilder(); - private List tokens_ = new ArrayList(); - - public MappedByteBufferLineReader(String filename, String charsetName) - throws IOException, UnsupportedEncodingException { - super(filename, charsetName); - raf_ = new RandomAccessFile(file_, "r"); - channel_ = raf_.getChannel(); - byteBuffer_ = channel_.map(FileChannel.MapMode.READ_ONLY, 0, channel_.size()); - bytes_ = new byte[string_buf_size_]; - } - - public MappedByteBufferLineReader(InputStream inputStream, String charsetName) throws UnsupportedEncodingException { - this(inputStream, charsetName, DEFAULT_BUFFER_SIZE); - } - - public MappedByteBufferLineReader(InputStream inputStream, String charsetName, int buf_size) - throws UnsupportedEncodingException { - super(inputStream instanceof BufferedInputStream ? inputStream : new BufferedInputStream(inputStream), - charsetName); - string_buf_size_ = buf_size; - byteBuffer_ = ByteBuffer.allocateDirect(string_buf_size_); // ByteBuffer.allocate(string_buf_size_); - bytes_ = new byte[string_buf_size_]; - if (inputStream == System.in) { - fillLine_ = true; - } - } - - @Override - public long skipLine(long n) throws IOException { - if (n < 0L) { - throw new IllegalArgumentException("skip value is negative"); - } - String line; - long currentLine = 0; - long readLine = 0; - synchronized (lock) { - ensureOpen(); - while (currentLine < n && (line = getLine()) != null) { - readLine++; - if (line == null || line.isEmpty() || line.startsWith("#")) { - continue; - } - currentLine++; - } - } - return readLine; - } - - @Override - public String readLine() throws IOException { - synchronized (lock) { - ensureOpen(); - String lineString = getLine(); - while (lineString != null && (lineString.isEmpty() || lineString.startsWith("#"))) { - lineString = getLine(); - } - return lineString; - } - } - - @Override - public String[] readLineTokens() throws IOException { - synchronized (lock) { - ensureOpen(); - String[] tokens = getLineTokens(); - while (tokens != null && ((tokens.length == 1 && tokens[0].isEmpty()) || tokens[0].startsWith("#"))) { - tokens = getLineTokens(); - } - return tokens; - } - } - - @Override - public void rewind() throws IOException { - synchronized (lock) { - ensureOpen(); - if (raf_ != null) { - raf_.seek(0); - channel_.position(0); - } - byteBuffer_.position(0); - } - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - synchronized (lock) { - ensureOpen(); - if ((off < 0) || (off > cbuf.length) || (len < 0) || ((off + len) > cbuf.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } - - CharBuffer charBuffer = byteBuffer_.asCharBuffer(); - int length = Math.min(len, charBuffer.remaining()); - charBuffer.get(cbuf, off, length); - - if (inputStream_ != null) { - off += length; - - while (off < len) { - fillByteBuffer(); - if (!byteBuffer_.hasRemaining()) { - break; - } - charBuffer = byteBuffer_.asCharBuffer(); - length = Math.min(len, charBuffer.remaining()); - charBuffer.get(cbuf, off, length); - off += length; - } - } - return length == len ? len : -1; - } - } - - @Override - public void close() throws IOException { - synchronized (lock) { - if (raf_ != null) { - raf_.close(); - } else if (inputStream_ != null) { - inputStream_.close(); - } - channel_ = null; - byteBuffer_ = null; - } - } - - /** Checks to make sure that the stream has not been closed */ - private void ensureOpen() throws IOException { - if (byteBuffer_ == null) - throw new IOException("Stream closed"); - } - - protected String getLine() throws IOException { - fillByteBuffer(); - if (!byteBuffer_.hasRemaining()) { - return null; - } - sb_.setLength(0); - int b = -1; - int i = -1; - do { - b = byteBuffer_.get(); - if ((b >= 10 && b <= 13) || b == 0) { - break; - } - bytes_[++i] = (byte) b; - if (i == string_buf_size_ - 1) { - sb_.append(new String(bytes_, charset_)); - i = -1; - } - fillByteBuffer(); - } while (byteBuffer_.hasRemaining()); - - sb_.append(new String(bytes_, 0, i + 1, charset_)); - return sb_.toString(); - } - - // " |\r|\t|\\v|\f|\0" - // 32 ' ', 9 \t, 10 \n, 11 \\v, 12 \f, 13 \r, 0 \0 - protected String[] getLineTokens() throws IOException { - fillByteBuffer(); - if (!byteBuffer_.hasRemaining()) { - return null; - } - tokens_.clear(); - sb_.setLength(0); - - int b = -1; - int i = -1; - do { - b = byteBuffer_.get(); - - if ((b >= 10 && b <= 13) || b == 0) { - break; - } else if (b == 9 || b == 32) { - sb_.append(new String(bytes_, 0, i + 1, charset_)); - tokens_.add(sb_.toString()); - sb_.setLength(0); - i = -1; - } else { - bytes_[++i] = (byte) b; - if (i == string_buf_size_ - 1) { - sb_.append(new String(bytes_, charset_)); - i = -1; - } - } - fillByteBuffer(); - } while (byteBuffer_.hasRemaining()); - - sb_.append(new String(bytes_, 0, i + 1, charset_)); - tokens_.add(sb_.toString()); - return tokens_.toArray(new String[tokens_.size()]); - } - - private void fillByteBuffer() throws IOException { - if (inputStream_ == null || byteBuffer_.hasRemaining()) { - return; - } - - byteBuffer_.clear(); - - int b; - for (int i = 0; i < string_buf_size_; i++) { - b = inputStream_.read(); - if (b < 0) { // END OF STREAM - break; - } - byteBuffer_.put((byte) b); - if (fillLine_) { - if ((b >= 10 && b <= 13) || b == 0) { - break; - } - } - } - - byteBuffer_.flip(); - } - -} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..0eec6cd --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,12 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=DEBUG, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +log4j.logger.org.nd4j=OFF +log4j.logger.org.reflections=OFF \ No newline at end of file diff --git a/src/test/java/ai/searchbox/Fasttext4J/FastTextTest.java b/src/test/java/ai/searchbox/Fasttext4J/FastTextTest.java new file mode 100644 index 0000000..d882b32 --- /dev/null +++ b/src/test/java/ai/searchbox/Fasttext4J/FastTextTest.java @@ -0,0 +1,67 @@ +package ai.searchbox.Fasttext4J; + +import ai.searchbox.FastText4J.Args; +import ai.searchbox.FastText4J.FastText; +import ai.searchbox.FastText4J.Pair; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class FastTextTest { + + @Ignore + @Test + public void train() throws Exception { + Args args = new Args(); + args.input = "/Users/davidgortega/Projects/tmp/fastText-0.9.1/data/file9short"; + args.output = "/Users/davidgortega/Projects/tmp/fastText-0.9.1/result/fil9Java"; + args.thread = 4; + + FastText fasttext = new FastText(); + fasttext.setArgs(args); + fasttext.train(); + } + + @Ignore + @Test + public void unsupervised() throws IOException { + String path = "/Users/davidgortega/Projects/tmp/fastText-0.9.1/result/fil9_skip_ns_64.bin"; + path = "/Users/davidgortega/Projects/tmp/fastText-0.9.1/result/fil9Java.bin"; + FastText fasttext = new FastText(); + fasttext.loadModel(path); + + assertEquals( fasttext.getWordVectorIn("man").size(), fasttext.getArgs().dim); + assertEquals( fasttext.getWordVectorOut("man").size(), fasttext.getArgs().dim); + } + + @Ignore + @Test + public void vecsLoad() throws Exception { + Args args = new Args(); + args.pretrainedVectors = "/Users/davidgortega/Projects/tmp/fastText-0.9.1/result/fil9Java.vec"; + args.dim = 100; + + FastText fasttext = new FastText(); + fasttext.setArgs(args); + fasttext.loadVecFile(); + } + + @Test + public void supervisedQuantized() throws IOException { + String path = "src/test/resources/ftlang.ftz"; + FastText fasttext = new FastText(); + fasttext.loadModel(path); + + List> preds = fasttext.predict("this is a test".split(" "), 1); + assertEquals( preds.size() , 1); + assertEquals( preds.get(0).getValue() , "__label__en"); + + preds = fasttext.predict("esto es un test".split(" "), 2); + assertEquals( preds.size() , 2); + assertEquals( preds.get(0).getValue() , "__label__es"); + } +} diff --git a/src/test/java/fasttext/TestDictionary.java b/src/test/java/ai/searchbox/Fasttext4J/TestDictionary.java old mode 100644 new mode 100755 similarity index 86% rename from src/test/java/fasttext/TestDictionary.java rename to src/test/java/ai/searchbox/Fasttext4J/TestDictionary.java index 7ec2ab2..52056d9 --- a/src/test/java/fasttext/TestDictionary.java +++ b/src/test/java/ai/searchbox/Fasttext4J/TestDictionary.java @@ -1,11 +1,13 @@ -package fasttext; - -import static org.junit.Assert.*; +package ai.searchbox.Fasttext4J; import java.util.Map; +import ai.searchbox.FastText4J.Args; +import ai.searchbox.FastText4J.Dictionary; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class TestDictionary { Dictionary dictionary = new Dictionary(new Args()); @@ -29,6 +31,7 @@ public void testAdd() { dictionary.add(","); dictionary.add("is"); dictionary.add("is"); + String w = ""; dictionary.add(w); dictionary.add(w); diff --git a/src/test/resources/ftlang.ftz b/src/test/resources/ftlang.ftz new file mode 100644 index 0000000..1fb85b3 Binary files /dev/null and b/src/test/resources/ftlang.ftz differ diff --git a/wikifil.pl b/wikifil.pl old mode 100644 new mode 100755 diff --git a/word-vector-example.sh b/word-vector-example.sh old mode 100644 new mode 100755 index fd066dc..fe3afb8 --- a/word-vector-example.sh +++ b/word-vector-example.sh @@ -14,16 +14,23 @@ DATADIR=data mkdir -p "${RESULTDIR}" mkdir -p "${DATADIR}" -if [ ! -f "${DATADIR}/text9" ] +if [ ! -f "${DATADIR}/enwik9.zip" ] then wget -c http://mattmahoney.net/dc/enwik9.zip -P "${DATADIR}" +fi + + +if [ ! -f "${DATADIR}/text9" ] +then unzip "${DATADIR}/enwik9.zip" -d "${DATADIR}" perl wikifil.pl "${DATADIR}/enwik9" > "${DATADIR}"/text9 fi + + if [ ! -f "${DATADIR}/rw/rw.txt" ] then - wget -c http://stanford.edu/~lmthang/morphoNLM/rw.zip -P "${DATADIR}" + wget -c http://www-nlp.stanford.edu/~lmthang/morphoNLM/rw.zip -P "${DATADIR}" unzip "${DATADIR}/rw.zip" -d "${DATADIR}" fi @@ -31,8 +38,8 @@ mvn package JAR=./target/fasttext-0.0.1-SNAPSHOT-jar-with-dependencies.jar -java -jar ${JAR} skipgram -input "${DATADIR}"/text9 -output "${RESULTDIR}"/text9 -lr 0.025 -dim 100 \ - -ws 5 -epoch 1 -minCount 5 -neg 5 -loss ns -bucket 2000000 \ +java -Xmx8G -jar ${JAR} skipgram -input "${DATADIR}"/text9 -output "${RESULTDIR}"/text9 -lr 0.025 -dim 100 \ + -ws 5 -epoch 1 -minCount 5 -neg 5 -loss ns -bucket 200000 \ -minn 3 -maxn 6 -thread 4 -t 1e-4 -lrUpdateRate 100 cut -f 1,2 "${DATADIR}"/rw/rw.txt | awk '{print tolower($0)}' | tr '\t' '\n' > "${DATADIR}"/queries.txt