Better logging of Option[input]

A30002503 · Jul 18, 2019 · caa9fbf · caa9fbf
1 parent cbe3127
commit caa9fbf
Show file tree

Hide file tree

Showing 10 changed files with 27 additions and 13 deletions.
diff --git a/src/main/scala/ai/tripl/arc/extract/AvroExtract.scala b/src/main/scala/ai/tripl/arc/extract/AvroExtract.scala
@@ -74,7 +74,10 @@ class AvroExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         for (inputField <- inputField) {

diff --git a/src/main/scala/ai/tripl/arc/extract/BytesExtract.scala b/src/main/scala/ai/tripl/arc/extract/BytesExtract.scala
@@ -63,7 +63,10 @@ class BytesExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("failMode", stage.failMode.sparkString)
-        stage.stageDetail.put("input", if (c.hasPath("inputView")) inputView else parsedGlob)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(stage.persist))
         stage.stageDetail.put("params", params.asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/DelimitedExtract.scala b/src/main/scala/ai/tripl/arc/extract/DelimitedExtract.scala
@@ -84,11 +84,10 @@ class DelimitedExtract extends PipelineStagePlugin {
           case Some(basePath) => Delimited.toSparkOptions(stage.settings) + ("basePath" -> basePath)
           case None => Delimited.toSparkOptions(stage.settings)
         }
-        val inputValue = stage.input match {
-          case Left(view) => view
-          case Right(glob) => glob
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
         }
-        stage.stageDetail.put("input", inputValue)
         stage.stageDetail.put("options", options.asJava)
         for (inputField <- inputField) {
           stage.stageDetail.put("inputField", inputField)

diff --git a/src/main/scala/ai/tripl/arc/extract/HTTPExtract.scala b/src/main/scala/ai/tripl/arc/extract/HTTPExtract.scala
@@ -74,7 +74,10 @@ class HTTPExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("headers", HTTPUtils.maskHeaders("Authorization" :: Nil)(stage.headers).asJava)
-        stage.stageDetail.put("input", if(c.hasPath("inputView")) inputView else parsedURI)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("method", method)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))

diff --git a/src/main/scala/ai/tripl/arc/extract/ImageExtract.scala b/src/main/scala/ai/tripl/arc/extract/ImageExtract.scala
@@ -59,7 +59,7 @@ class ImageExtract extends PipelineStagePlugin {
           stage.stageDetail.put("basePath", basePath)
         }
         stage.stageDetail.put("dropInvalid", java.lang.Boolean.valueOf(dropInvalid))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("params", params.asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/JSONExtract.scala b/src/main/scala/ai/tripl/arc/extract/JSONExtract.scala
@@ -69,7 +69,10 @@ class JSONExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("options", JSON.toSparkOptions(stage.settings).asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/ORCExtract.scala b/src/main/scala/ai/tripl/arc/extract/ORCExtract.scala
@@ -61,7 +61,7 @@ class ORCExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         for (basePath <- basePath) {

diff --git a/src/main/scala/ai/tripl/arc/extract/ParquetExtract.scala b/src/main/scala/ai/tripl/arc/extract/ParquetExtract.scala
@@ -60,7 +60,7 @@ class ParquetExtract extends PipelineStagePlugin {
           contiguousIndex=contiguousIndex
         )
 
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))

diff --git a/src/main/scala/ai/tripl/arc/extract/TextExtract.scala b/src/main/scala/ai/tripl/arc/extract/TextExtract.scala
@@ -58,7 +58,7 @@ class TextExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("multiLine", java.lang.Boolean.valueOf(multiLine))
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))

diff --git a/src/main/scala/ai/tripl/arc/extract/XMLExtract.scala b/src/main/scala/ai/tripl/arc/extract/XMLExtract.scala
@@ -63,7 +63,10 @@ class XMLExtract extends PipelineStagePlugin {
           contiguousIndex=contiguousIndex
         )
 
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))