Merge pull request #688 from mssammon/master

update core-utilities readme with configurator info

Author: Daniel Khashabi

commasrl/pom.xml

@@ -20,7 +20,7 @@
         <pluginRepository>
             <id>CogcompSoftware</id>
             <name>CogcompSoftware</name>
-            <url>http://cogcomp.cs.illinois.edu/m2repo/</url>
+            <url>http://macniece.seas.upenn.edu/m2repo/</url>
         </pluginRepository>
     </pluginRepositories>
 
@@ -164,16 +164,4 @@
             </plugin>
         </plugins>
     </reporting>
-
-    <distributionManagement>
-        <repository>
-            <id>CogcompSoftware</id>
-            <name>CogcompSoftware</name>
-            <url>scp://legolas.cs.illinois.edu:/srv/data/cogcomp/html/m2repo</url>
-        </repository>
-        <site>
-            <id>CogcompSoftwareDoc</id>
-            <url>scp://legolas.cs.illinois.edu:/srv/data/cogcomp/html/software/doc/${project.artifactId}</url>
-        </site>
-    </distributionManagement>
 </project>

core-utilities/README.md

@@ -401,6 +401,64 @@ to specify only non-default configuration options when you instantiate
 one or more classes that use `ResourceManager` and `Configurator`
 to manage configuration options.
 
+A configurator must inherit from `edu.illinois.cs.cogcomp.core.utilities.configuration.Configurator.java`
+and override the `getDefaultConfig()` method. Specify any configuration parameters as public static
+`edu.illinois.cs.cogcomp.core.utilities.configuration.Property` fields in the Configurator subclass you
+create. This specifies configuration parameter keys and default values, and makes the names available
+to clients. 
+
+The `getDefaultConfig()` method should simply populate a `ResourceManager` object with all default values --
+there are convenience methods in `Configurator` to make this easier. 
+
+When implementing your Annotator's simplest constructor, instantiate your configurator subclass and
+call its `getDefaultConfig()` method. Implement a second Constructor that takes a `ResourceManager`
+as its argument, instantiate your configurator, and call its `getConfig(ResourceManager)` method.
+
+Here's the constructor code from ChunkAnnotator:
+```
+    public ChunkerAnnotator(boolean lazilyInitialize) {
+        this(lazilyInitialize, new ChunkerConfigurator().getDefaultConfig());
+    }
+
+    public ChunkerAnnotator(boolean lazilyInitialize, ResourceManager rm) {
+        super(ViewNames.SHALLOW_PARSE, new String[] {ViewNames.POS}, lazilyInitialize, new ChunkerConfigurator().getConfig(rm));
+    }
+``` 
+
+This allows you to only specify configuration parameters you wish to modify from their default values, using the keys specified
+in the relevant Configurator's Property fields. You can
+either directly populate a java `Properties` object with these key/value pairs, then instantiate a `ResourceManager` and call
+your Configurator's `getConfig()` method with that; or you can write these non-default values into a text file with each 
+key/value pair on a new line, with the key and value separated by a tab character. 
+
+For the Chunker example, suppose you want to change the model path. In `ChunkerConfigurator` this is specified as:
+```
+    public static final Property MODEL_PATH = new Property("modelPath", MODEL_DIR_PATH.value
+            + MODEL_NAME.value + ".lc");
+```
+
+To directly populate a Properties object:
+```
+    import edu.illinois.cs.cogcomp.core.resources.ResourceManager;
+    import java.util.Properties;
+    import edu.illinois.cs.cogcomp.chunker.main.ChunkerAnnotator;
+
+    Properties props = new Properties;
+    props.setProperty(ChunkerConfigurator.MODEL_PATH.key, "/some/other/path");
+    ChunkerAnnotator ca = new ChunkerAnnotator(new ResourceManager(props));
+```
+
+To use a text file instead, create a text file (for this example, "config/altChunker.config") with the single line:
+```
+modelPath	/some/other/path
+```
+
+...and use it to instantiate a ResourceManager object:
+
+```
+    ChunkerAnnotator ca = new ChunkerAnnotator(new ResourceManager("config/altChunker.config"));
+```
+
 ### Serialization and Deserialization
 
 To store `TextAnnotation` objects on disk, serialization/deserialization is supported in the following formats:

core-utilities/src/test/java/edu/illinois/cs/cogcomp/utilities/JsonSerializerTest.java

@@ -18,6 +18,7 @@
 import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Relation;
 import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation;
 import edu.illinois.cs.cogcomp.core.datastructures.textannotation.View;
+import edu.illinois.cs.cogcomp.core.io.IOUtils;
 import edu.illinois.cs.cogcomp.core.utilities.DummyTextAnnotationGenerator;
 import edu.illinois.cs.cogcomp.core.utilities.JsonSerializer;
 import edu.illinois.cs.cogcomp.core.utilities.SerializationHelper;
@@ -26,20 +27,21 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
 
 /**
  * Simple sanity tests for JsonSerializer.
  * 
  * @author mssammon
  */
 public class JsonSerializerTest {
+    private static final String RHYME_VIEW_NAME = "rhyme";
     private static Logger logger = LoggerFactory.getLogger(JsonSerializerTest.class);
 
     TextAnnotation ta = DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(new String[] {
@@ -122,7 +124,19 @@ public static void verifyDeserializedJsonString(String json, TextAnnotation ta)
 
     @Test
     public void testSerializerWithCharOffsets() {
-        View rhymeView = new View("rhyme", "test", ta, 0.4 );
+
+        addRhymeViewToTa(ta);
+
+        String taJson = SerializationHelper.serializeToJson(ta, true);
+        logger.info(taJson);
+
+        JsonObject jobj = (JsonObject) new JsonParser().parse(taJson);
+        JsonSerializerTest.verifySerializedJSONObject(jobj, ta);
+    }
+
+
+    private static void addRhymeViewToTa(TextAnnotation someTa) {
+        View rhymeView = new View(RHYME_VIEW_NAME, "test", someTa, 0.4);
 
         Map< String, Double > newLabelsToScores = new TreeMap< String, Double >();
         String[] labels = { "eeny", "meeny", "miny", "mo" };
@@ -131,32 +145,26 @@ public void testSerializerWithCharOffsets() {
         for ( int i = 0; i < labels.length; ++i )
             newLabelsToScores.put(labels[i], scores[i]);
 
-        Constituent first = new Constituent( newLabelsToScores, "rhyme", ta, 2, 4 );
+        Constituent first = new Constituent(newLabelsToScores, RHYME_VIEW_NAME, someTa, 2, 4);
         rhymeView.addConstituent(first);
 
         /**
          * no constraint on scores -- don't have to sum to 1.0
          */
         for ( int i = labels.length -1; i > 0; --i )
-            newLabelsToScores.put( labels[i], scores[3-i] );
+            newLabelsToScores.put(labels[i], scores[3-i]);
 
-        Constituent second = new Constituent( newLabelsToScores, "rhyme", ta, 2, 4 );
+        Constituent second = new Constituent(newLabelsToScores, RHYME_VIEW_NAME, someTa, 2, 4);
         rhymeView.addConstituent(second);
 
         Map<String, Double> relLabelsToScores = new TreeMap<>();
-        relLabelsToScores.put( "Yes", 0.8 );
-        relLabelsToScores.put( "No", 0.2 );
+        relLabelsToScores.put("Yes", 0.8);
+        relLabelsToScores.put("No", 0.2);
 
         Relation rel = new Relation( relLabelsToScores, first, second );
         rhymeView.addRelation(rel);
 
-        ta.addView("rhyme", rhymeView);
-
-        String taJson = SerializationHelper.serializeToJson(ta, true);
-        logger.info(taJson);
-
-        JsonObject jobj = (JsonObject) new JsonParser().parse(taJson);
-        JsonSerializerTest.verifySerializedJSONObject(jobj, ta);
+        someTa.addView(RHYME_VIEW_NAME, rhymeView);
     }
 
     @Test
@@ -171,4 +179,71 @@ public void testJsonSerializabilityWithOffsets() throws Exception {
 
         JsonSerializerTest.verifyDeserializedJsonString(json, ta);
     }
+
+    /**
+     * make sure that if an already serialized TextAnnotation object is modified and reserialized,
+     *    (and written to the same target file), that the file is updated correctly
+     */
+    @Test
+    public void testJsonSerializedTaUpdate() {
+
+        // make sure we aren't using a TA already updated with "rhyme" view
+        TextAnnotation localTa = DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(new String[] {
+                ViewNames.POS, ViewNames.NER_CONLL, ViewNames.SRL_VERB}, false, 3); // no noise
+
+        String serTestDir = "serTestDir";
+        if(!IOUtils.exists(serTestDir))
+            IOUtils.mkdir(serTestDir);
+        else if (IOUtils.isFile(serTestDir))
+            throw new IllegalStateException("ERROR: test directory " + serTestDir + " already exists as file.");
+        else
+            try {
+                IOUtils.cleanDir(serTestDir);
+            } catch (IOException e) {
+                e.printStackTrace();
+                throw new IllegalStateException("ERROR: test directory " + serTestDir + " could not be cleaned. Permissions?");
+            }
+        if (!IOUtils.getListOfFilesInDir(serTestDir).isEmpty())
+            throw new IllegalStateException("ERROR: test directory " + serTestDir + " already contains files even after cleaning.");
+
+        String fileName = serTestDir + "/arbitrary.json";
+        boolean forceOverwrite = true;
+        boolean useJson = true;
+        try {
+            SerializationHelper.serializeTextAnnotationToFile(localTa, fileName, forceOverwrite, useJson);
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail("error trying to serialize json file " + fileName + ".");
+        }
+
+        TextAnnotation taDeser = null;
+        try {
+            taDeser = SerializationHelper.deserializeTextAnnotationFromFile(fileName, useJson);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail("error trying to deserialize json file " + fileName + ".");
+        }
+        assertTrue(taDeser.hasView(ViewNames.SRL_VERB));
+        assertFalse(taDeser.hasView(RHYME_VIEW_NAME));
+        addRhymeViewToTa(taDeser);
+        assertTrue(taDeser.hasView(RHYME_VIEW_NAME));
+
+        try {
+            SerializationHelper.serializeTextAnnotationToFile(taDeser, fileName, forceOverwrite, useJson);
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail("error trying to serialize json file " + fileName + " for second time.");
+        }
+
+        TextAnnotation taDeserDeser = null;
+        try {
+            taDeserDeser = SerializationHelper.deserializeTextAnnotationFromFile(fileName, useJson);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail("error trying to deserialize json file " + fileName + " for second time.");
+        }
+
+        assertTrue(taDeserDeser.hasView(RHYME_VIEW_NAME));
+        assertTrue(taDeserDeser.getView(RHYME_VIEW_NAME).getConstituents().size() > 0);
+    }
 }

pom.xml

@@ -53,19 +53,19 @@
         <repository>
             <id>CogcompSoftware</id>
             <name>CogcompSoftware</name>
-            <url>http://cogcomp.org/m2repo/</url>
+            <url>http://macniece.seas.upenn.edu/m2repo/</url>
         </repository>
     </repositories>
 
     <distributionManagement>
         <repository>
             <id>CogcompSoftware</id>
             <name>CogcompSoftware</name>
-            <url>scp://legolas.cs.illinois.edu:/srv/data/cogcomp/html/m2repo</url>
+            <url>scp://macniece.seas.upenn.edu:/pool0/webserver/htdocs/public/m2repo</url>
         </repository>
         <site>
             <id>CogcompSoftwareDoc</id>
-            <url>scp://legolas.cs.illinois.edu:/srv/data/cogcomp/html/software/doc/</url>
+            <url>scp://macniece.seas.upenn.edu:/pool0/webserver/htdocs/public/software/doc/</url>
         </site>
     </distributionManagement>
 

pos/scripts/generateParenPosFormat.pl

@@ -29,6 +29,9 @@
 use strict;
 use Carp;
 
+
+# wsj-dir must be the path to the 'parsed/' directory of a Penn Treebank
+#   distribution.
 croak "Usage: $0 wsj-dir out-dir" unless @ARGV == 2;
 
 my $wsjDir = shift;

pos/src/main/java/edu/illinois/cs/cogcomp/pos/POSConfigurator.java

@@ -21,7 +21,9 @@
 public class POSConfigurator extends Configurator {
     /** A configurable prefix. */
     public static final Property CORPUS_PREFIX = new Property("corpusPrefix",
-            "/shared/corpora/corporaWeb/written/eng/POS/");
+            "/shared/experiments/mssammon/workspace-github/cogcomp-nlp/pos/corpus-paren-format/");
+							      //            "/shared/corpora/corporaWeb/written/eng/POS/");
+
     /** The file containing the training set. */
     public static final Property TRAINING_DATA = new Property("trainingData", CORPUS_PREFIX.value
             + "00-18.br");