diff --git a/src/main/java/fasttext/Dictionary.java b/src/main/java/fasttext/Dictionary.java index f240404..b962ce3 100644 --- a/src/main/java/fasttext/Dictionary.java +++ b/src/main/java/fasttext/Dictionary.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.math.BigInteger; import fasttext.Args.model_name; import fasttext.io.BufferedLineReader; @@ -342,10 +343,13 @@ public void addNgrams(List line, int n) { } int line_size = line.size(); for (int i = 0; i < line_size; i++) { - long h = (long) line.get(i); + BigInteger h = BigInteger.valueOf(line.get(i)); + BigInteger r = BigInteger.valueOf(116049371l); + BigInteger b = BigInteger.valueOf(args_.bucket); + for (int j = i + 1; j < line_size && j < i + n; j++) { - h = (h * 116049371l + line.get(j)) & 0xffffffffl; - line.add(nwords_ + (int) (h % args_.bucket)); + h = h.multiply(r).add(BigInteger.valueOf(line.get(j)));; + line.add(nwords_ + h.remainder(b).intValue()); } } } diff --git a/src/main/java/fasttext/Model.java b/src/main/java/fasttext/Model.java index cab62d4..f09e149 100644 --- a/src/main/java/fasttext/Model.java +++ b/src/main/java/fasttext/Model.java @@ -201,7 +201,7 @@ public void dfs(int k, int node, float score, List> heap, V return; } - float f = sigmoid(wo_.dotRow(hidden_, node - osz_)); + float f = sigmoid(wo_.dotRow(hidden, node - osz_)); dfs(k, tree.get(node).left, score + log(1.0f - f), heap, hidden); dfs(k, tree.get(node).right, score + log(f), heap, hidden); }