From a685679cabac4ae26cb87236f1c0209ed898e3d0 Mon Sep 17 00:00:00 2001 From: ivanhk Date: Sat, 4 Feb 2017 10:47:55 +0800 Subject: [PATCH 1/6] skip unnecessary Dictionary addNgrams operation --- README.md | 2 +- src/main/java/fasttext/Dictionary.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c62bab..e16cc76 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # fasttext_java -Java port of c++ version of facebook fasttext [UPDATED 2017-01-06] +Java port of c++ version of facebook fasttext [UPDATED 2017-01-29] Support Load/Save facebook fasttext binary model file diff --git a/src/main/java/fasttext/Dictionary.java b/src/main/java/fasttext/Dictionary.java index 6102681..f240404 100644 --- a/src/main/java/fasttext/Dictionary.java +++ b/src/main/java/fasttext/Dictionary.java @@ -337,6 +337,9 @@ public List getCounts(entry_type type) { } public void addNgrams(List line, int n) { + if (n <= 1) { + return; + } int line_size = line.size(); for (int i = 0; i < line_size; i++) { long h = (long) line.get(i); From 5b495c0ec463301d5282ec2db7ca4db3c0fe79ab Mon Sep 17 00:00:00 2001 From: ivanhk Date: Tue, 7 Feb 2017 14:35:52 +0800 Subject: [PATCH 2/6] skip save if output is null or empty --- src/main/java/fasttext/FastText.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/fasttext/FastText.java b/src/main/java/fasttext/FastText.java index 6c5e4d2..9d972e3 100644 --- a/src/main/java/fasttext/FastText.java +++ b/src/main/java/fasttext/FastText.java @@ -54,15 +54,23 @@ public void getVector(Vector vec, final String word) { } public void saveVectors() throws IOException { + if (Utils.isEmpty(args_.output)) { + if (args_.verbose > 1) { + System.out.println("output is empty, skip save vector file"); + } + return; + } + File file = new File(args_.output + ".vec"); if (file.exists()) { file.delete(); } - file.getParentFile().mkdirs(); + if (file.getParentFile() != null) { + file.getParentFile().mkdirs(); + } if (args_.verbose > 1) { System.out.println("Saving Vectors to " + file.getCanonicalPath().toString()); } - Vector vec = new Vector(args_.dim); DecimalFormat df = new DecimalFormat("0.#####"); Writer writer = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(file)), "UTF-8"); @@ -85,6 +93,13 @@ public void saveVectors() throws IOException { } public void saveModel() throws IOException { + if (Utils.isEmpty(args_.output)) { + if (args_.verbose > 1) { + System.out.println("output is empty, skip save model file"); + } + return; + } + File file = new File(args_.output + ".bin"); if (file.exists()) { file.delete(); From ed435f7625a7727989bc387e591a84714de4ae1d Mon Sep 17 00:00:00 2001 From: ivanhk Date: Wed, 22 Feb 2017 18:11:18 +0800 Subject: [PATCH 3/6] fix printInfo mistake, reduce printInfo rate --- src/main/java/fasttext/FastText.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/fasttext/FastText.java b/src/main/java/fasttext/FastText.java index 9d972e3..f679001 100644 --- a/src/main/java/fasttext/FastText.java +++ b/src/main/java/fasttext/FastText.java @@ -163,13 +163,14 @@ public void loadModel(String filename) throws IOException { public void printInfo(float progress, float loss) { float t = (float) (System.currentTimeMillis() - start_) / 1000; - float wst = (float) (tokenCount_.get()) / t; + float ws = (float) (tokenCount_.get()) / t; + float wst = (float) (tokenCount_.get()) / t / args_.thread; float lr = (float) (args_.lr * (1.0f - progress)); - int eta = (int) (t / progress * (1 - progress) / args_.thread); + int eta = (int) (t / progress * (1 - progress)); int etah = eta / 3600; int etam = (eta - etah * 3600) / 60; - System.out.printf("\rProgress: %.1f%% words/sec/thread: %d lr: %.6f loss: %.6f eta: %d h %d m", 100 * progress, - (int) wst, lr, loss, etah, etam); + System.out.printf("\rProgress: %.1f%% words/sec: %d words/sec/thread: %d lr: %.6f loss: %.6f eta: %d h %d m", + 100 * progress, (int) ws, (int) wst, lr, loss, etah, etam); } public void supervised(Model model, float lr, final List line, final List labels) { @@ -402,6 +403,7 @@ public class TrainThread extends Thread { int threadId; public TrainThread(FastText ft, int threadId) { + super("FT-TrainThread-" + threadId); this.ft = ft; this.threadId = threadId; } @@ -460,7 +462,7 @@ public void run() { if (localTokenCount > args_.lrUpdateRate) { tokenCount_.addAndGet(localTokenCount); localTokenCount = 0; - if (threadId == 0 && args_.verbose > 1) { + if (threadId == 0 && args_.verbose > 1 && (System.currentTimeMillis() - start_) % 1000 == 0) { printInfo(progress, model.getLoss()); } } From 6dc493dea065d5f92fcd912cbe63f0d18c39b9fc Mon Sep 17 00:00:00 2001 From: ivanhk Date: Thu, 23 Feb 2017 11:00:46 +0800 Subject: [PATCH 4/6] replace System.exit in FastText.java --- src/main/java/fasttext/FastText.java | 39 +++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/fasttext/FastText.java b/src/main/java/fasttext/FastText.java index f679001..d716155 100644 --- a/src/main/java/fasttext/FastText.java +++ b/src/main/java/fasttext/FastText.java @@ -156,8 +156,12 @@ public void loadModel(String filename) throws IOException { model_.setTargetCounts(dict_.getCounts(entry_type.word)); } } finally { - bis.close(); - dis.close(); + if (bis != null) { + bis.close(); + } + if (dis != null) { + dis.close(); + } } } @@ -412,6 +416,7 @@ public void run() { if (args_.verbose > 2) { System.out.println("thread: " + threadId + " RUNNING!"); } + Exception catchedException = null; LineReader lineReader = null; try { lineReader = lineReaderClass_.getConstructor(String.class, String.class).newInstance(args_.input, @@ -471,12 +476,8 @@ public void run() { if (threadId == 0 && args_.verbose > 1) { printInfo(1.0f, model.getLoss()); } - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); } catch (Exception e) { - e.printStackTrace(); - System.exit(1); + catchedException = e; } finally { if (lineReader != null) try { @@ -493,6 +494,9 @@ public void run() { } ft.threadCount--; ft.notify(); + if (catchedException != null) { + throw new RuntimeException(catchedException); + } } } } @@ -516,8 +520,9 @@ public void loadVectors(String filename) throws IOException { words = new ArrayList(n); if (dim != args_.dim) { - System.err.println("Dimension of pretrained vectors does not match -dim option"); - System.exit(1); + throw new IllegalArgumentException( + "Dimension of pretrained vectors does not match args -dim option, pretrain dim is " + dim + + ", args dim is " + args_.dim); } mat = new Matrix(n, dim); @@ -597,7 +602,9 @@ public void train(Args args) throws IOException, Exception { long t0 = System.currentTimeMillis(); threadCount = args_.thread; for (int i = 0; i < args_.thread; i++) { - new TrainThread(this, i).start(); + Thread t = new TrainThread(this, i); + t.setUncaughtExceptionHandler(trainThreadExcpetionHandler); + t.start(); } synchronized (this) { @@ -611,8 +618,10 @@ public void train(Args args) throws IOException, Exception { model_ = new Model(input_, output_, args_, 0); - long trainTime = (System.currentTimeMillis() - t0) / 1000; - System.out.printf("\nTrain time used: %d sec\n", trainTime); + if (args.verbose > 1) { + long trainTime = (System.currentTimeMillis() - t0) / 1000; + System.out.printf("\nTrain time used: %d sec\n", trainTime); + } saveModel(); if (args_.model != model_name.sup) { @@ -620,6 +629,12 @@ public void train(Args args) throws IOException, Exception { } } + protected Thread.UncaughtExceptionHandler trainThreadExcpetionHandler = new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread th, Throwable ex) { + ex.printStackTrace(); + } + }; + public Args getArgs() { return args_; } From df95ff4eafc0664ad9ecf9bc6f18f22c7c1aa40d Mon Sep 17 00:00:00 2001 From: faye young <694124779@qq.com> Date: Wed, 5 Apr 2017 20:21:04 +0800 Subject: [PATCH 5/6] replace hidden_ -> hidden in dfs function if loss=hs, wrong prediction happens. --- src/main/java/fasttext/Model.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fasttext/Model.java b/src/main/java/fasttext/Model.java index cab62d4..f09e149 100644 --- a/src/main/java/fasttext/Model.java +++ b/src/main/java/fasttext/Model.java @@ -201,7 +201,7 @@ public void dfs(int k, int node, float score, List> heap, V return; } - float f = sigmoid(wo_.dotRow(hidden_, node - osz_)); + float f = sigmoid(wo_.dotRow(hidden, node - osz_)); dfs(k, tree.get(node).left, score + log(1.0f - f), heap, hidden); dfs(k, tree.get(node).right, score + log(f), heap, hidden); } From b919bdc2167f12fc863a6166c48e032083c2cf04 Mon Sep 17 00:00:00 2001 From: faye young <694124779@qq.com> Date: Thu, 6 Apr 2017 16:17:06 +0800 Subject: [PATCH 6/6] fix addNgrams bug : long -> BigInteger Long in java is not equal to uint64_t in c++ , when wordNgrams is big, unexpected results happens. `h = (h * 116049371l + line.get(j)) & 0xffffffffl` , 0xffffffffl only have 4 bytes --- src/main/java/fasttext/Dictionary.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/fasttext/Dictionary.java b/src/main/java/fasttext/Dictionary.java index f240404..b962ce3 100644 --- a/src/main/java/fasttext/Dictionary.java +++ b/src/main/java/fasttext/Dictionary.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.math.BigInteger; import fasttext.Args.model_name; import fasttext.io.BufferedLineReader; @@ -342,10 +343,13 @@ public void addNgrams(List line, int n) { } int line_size = line.size(); for (int i = 0; i < line_size; i++) { - long h = (long) line.get(i); + BigInteger h = BigInteger.valueOf(line.get(i)); + BigInteger r = BigInteger.valueOf(116049371l); + BigInteger b = BigInteger.valueOf(args_.bucket); + for (int j = i + 1; j < line_size && j < i + n; j++) { - h = (h * 116049371l + line.get(j)) & 0xffffffffl; - line.add(nwords_ + (int) (h % args_.bucket)); + h = h.multiply(r).add(BigInteger.valueOf(line.get(j)));; + line.add(nwords_ + h.remainder(b).intValue()); } } }