number of words can be more than 2 billion pretty easily

lstavr · Nov 11, 2014 · c537025 · c537025
1 parent 84b9cf7
commit c537025
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/...caleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java b/...caleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java
@@ -53,7 +53,7 @@ public class Word2Vec implements Persistable {
     private int batchSize = 1000;
     private int topNSize = 40;
     private double sample = 0;
-    private int totalWords = 1;
+    private long totalWords = 1;
     private AtomicInteger rateOfChange = new AtomicInteger(0);
     //learning rate
     private AtomicDouble alpha = new AtomicDouble(0.025);

diff --git a/.../main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java b/.../main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java
@@ -43,7 +43,7 @@ public class InMemoryLookupCache implements VocabCache,Serializable {
     private INDArray syn0,syn1;
     private int vectorLength = 50;
     private transient RandomGenerator rng = new XorShift64StarRandomGenerator(123);
-    private AtomicInteger totalWordOccurrences = new AtomicInteger(0);
+    private AtomicLong totalWordOccurrences = new AtomicLong(0);
     private AtomicDouble lr = new AtomicDouble(1e-1);
     double[] expTable = new double[1000];
     static double MAX_EXP = 6;