fix #12: recognize end of sentence

src/java/com/rondhuit/w2v/Corpus.java

@@ -101,9 +101,20 @@ public void rewind(int numThreads, int id){
     eoc = false;
   }
 
+  /**
+   * 
+   * @return -3 if end of sentence, -2 if end of corpus, -1 if word not found or index value of the word
+   * @throws IOException
+   */
   public int readWordIndex() throws IOException {
     String word = nextWord();
-    return word == null ? -2 : searchVocab(word);
+    if(word == null){
+      if(eoc) return -2;    // end of corpus
+      else return -3;       // end of sentence
+    }
+    else{
+      return searchVocab(word);     // index value of the word
+    }
   }
 
   /**

src/java/com/rondhuit/w2v/Word2vec.java

@@ -134,7 +134,7 @@ public void run(){
               if(word == -2) break;                // EOF
               if(word == -1) continue;
               word_count++;
-              if (word == 0) break;
+              if (word == -3) break;
               // The subsampling randomly discards frequent words while keeping the ranking same
               if (sample > 0) {
                 double ran = (Math.sqrt(vocab[word].cn / (sample * trainWords)) + 1) * (sample * trainWords) / vocab[word].cn;

src/java/com/rondhuit/w2v/lucene/LuceneIndexCorpus.java

@@ -127,11 +127,14 @@ public String nextWord() throws IOException {
         tokenStream = analyzer.tokenStream(field, values[valPos++]);
         termAtt = tokenStream.getAttribute(CharTermAttribute.class);
         tokenStream.reset();
+        eoc = false;
+        return null;
       }
       else{
         if(tdPos >= topDocs.totalHits){
           tokenStream = null;
-          return null;   // end of index
+          eoc = true;
+          return null;   // end of index == end of corpus
         }
         Document doc = reader.document(topDocs.scoreDocs[tdPos++].doc);
         values = doc.getValues(field);   // This method returns an empty array when there are no matching fields.