Add support for multiline pica records.

The pica parser accepts new lines (\x0a) in the input as a field
delimiter. This makes it possible to parse not only pica plain but also
multi line pica records.

The test cases were refactored to make the input being tested more
visible.

Author: cboehme

src/main/java/org/culturegraph/mf/stream/converter/bib/PicaConstants.java

@@ -23,8 +23,10 @@
  */
 final class PicaConstants {
 
-	public static final char FIELD_DELIMITER = '\u001e';
-	public static final char SUBFIELD_DELIMITER = '\u001f';
+	public static final char RECORD_MARKER = '\u001d';
+	public static final char FIELD_MARKER = '\u001e';
+	public static final char SUBFIELD_MARKER = '\u001f';
+	public static final char FIELD_END_MARKER = '\n';
 
 	private PicaConstants() {
 		// No instances allowed

src/main/java/org/culturegraph/mf/stream/converter/bib/PicaDecoder.java

@@ -50,7 +50,7 @@
 public final class PicaDecoder
 		extends DefaultObjectPipe<String, StreamReceiver> {
 
-	private static final char[] ID_FIELD = {'0', '0', '3', '@', ' ', PicaConstants.SUBFIELD_DELIMITER, '0'};
+	private static final char[] ID_FIELD = {'0', '0', '3', '@', ' ', PicaConstants.SUBFIELD_MARKER, '0'};
 
 	private static final int BUFFER_SIZE = 1024 * 1024;
 
@@ -144,10 +144,10 @@ private boolean recordIsEmpty() {
 	/**
 	 * Searches the record for the sequence specified in {@code ID_FIELD}
 	 * and returns all characters following this sequence until the next
-	 * {@link PicaConstants.FIELD_DELIMITER},
-	 * {@link PicaConstants.SUBFIELD_DELIMITER} or the end of the record
-	 * is reached. Only the first occurrence of the sequence is processed,
-	 * later occurrences are ignored.
+	 * {@link PicaConstants.FIELD_MARKER},
+	 * {@link PicaConstants.SUBFIELD_MARKER}, {@link PicaConstants.LINE_END_MARKER}
+	 * or the end of the record is reached. Only the first occurrence of the
+	 * sequence is processed, later occurrences are ignored.
 	 * 
 	 * If the sequence is not found in the string or if it is not followed
 	 * by any characters then {@code null} is returned.
@@ -161,7 +161,8 @@ private String extractRecordId() {
 		int fieldPos = 0;
 		boolean skip = false;
 		for (int i = 0; i < recordLen; ++i) {
-			if (buffer[i] == PicaConstants.FIELD_DELIMITER) {
+			if (buffer[i] == PicaConstants.FIELD_MARKER
+					|| buffer[i] == PicaConstants.FIELD_END_MARKER) {
 				if (idBuilder.length() > 0) {
 					break;
 				}
@@ -176,7 +177,7 @@ private String extractRecordId() {
 							skip = true;
 						}
 					} else {
-						if (buffer[i] == PicaConstants.SUBFIELD_DELIMITER) {
+						if (buffer[i] == PicaConstants.SUBFIELD_MARKER) {
 							break;
 						}
 						idBuilder.append(buffer[i]);

src/main/java/org/culturegraph/mf/stream/converter/bib/PicaParserState.java

@@ -38,14 +38,18 @@ enum PicaParserState {
 		@Override
 		protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
 			final PicaParserState next;
-			if (ch == PicaConstants.FIELD_DELIMITER) {
+			switch (ch) {
+			case PicaConstants.FIELD_MARKER:
+			case PicaConstants.FIELD_END_MARKER:
 				ctx.emitStartEntity();
 				ctx.emitEndEntity();
 				next = FIELD_NAME;
-			} else if (ch == PicaConstants.SUBFIELD_DELIMITER) {
+				break;
+			case PicaConstants.SUBFIELD_MARKER:
 				ctx.emitStartEntity();
 				next = SUBFIELD_NAME;
-			} else {
+				break;
+			default:
 				if (ch != ' ') {
 					ctx.appendText(ch);
 				}
@@ -64,12 +68,16 @@ protected void endOfInput(final PicaParserContext ctx) {
 		@Override
 		protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
 			final PicaParserState next;
-			if (ch == PicaConstants.FIELD_DELIMITER) {
+			switch (ch) {
+			case PicaConstants.FIELD_MARKER:
+			case PicaConstants.FIELD_END_MARKER:
 				ctx.emitEndEntity();
 				next = FIELD_NAME;
-			} else if (ch == PicaConstants.SUBFIELD_DELIMITER) {
+				break;
+			case PicaConstants.SUBFIELD_MARKER:
 				next = this;
-			} else {
+				break;
+			default:
 				ctx.setSubfieldName(ch);
 				next = SUBFIELD_VALUE;
 			}
@@ -85,14 +93,18 @@ protected void endOfInput(final PicaParserContext ctx) {
 		@Override
 		protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
 			final PicaParserState next;
-			if (ch == PicaConstants.FIELD_DELIMITER) {
+			switch (ch) {
+			case PicaConstants.FIELD_MARKER:
+			case PicaConstants.FIELD_END_MARKER:
 				ctx.emitLiteral();
 				ctx.emitEndEntity();
 				next = FIELD_NAME;
-			} else if (ch == PicaConstants.SUBFIELD_DELIMITER) {
+				break;
+			case PicaConstants.SUBFIELD_MARKER:
 				ctx.emitLiteral();
 				next = SUBFIELD_NAME;
-			} else {
+				break;
+			default:
 				ctx.appendText(ch);
 				next = this;
 			}