Merge branch 'release/2.0.1'

A30002503 · Sep 1, 2019 · 58fc282 · 58fc282
2 parents 2fb0357 + 363c29e
commit 58fc282
Show file tree

Hide file tree

Showing 17 changed files with 39 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 ## Change Log
 
+# 2.0.1
+
+- update to [Spark 2.4.4](https://spark.apache.org/releases/spark-release-2-4-4.html).
+- update to Scala `2.12.9`
+
 # 2.0.0
 
 Arc 2.0.0 is a major (breaking) change which has been done for multiple reasons:

diff --git a/build.sbt b/build.sbt
@@ -1,7 +1,7 @@
 import Dependencies._
 
 lazy val scala211 = "2.11.12"
-lazy val scala212 = "2.12.8"
+lazy val scala212 = "2.12.9"
 lazy val supportedScalaVersions = List(scala211, scala212)
 
 lazy val root = (project in file(".")).

diff --git a/docs-src/config.toml b/docs-src/config.toml
@@ -20,10 +20,10 @@ pygmentsUseClasses=true
 	repo_url = "https://github.com/tripl-ai/arc"
 
 	image = "triplai/arc"
+	version = "2.0.1"
 	arc_jupyter_image= "triplai/arc-jupyter"
-
-	version = "2.0.0"
-	spark_version = "2.4.3"
+	arc_jupyter_version = "1.3.1"
+	spark_version = "2.4.4"
 	scala_version = "2.11"
 	hadoop_version = "2.9.2"
 	arc_image_version = "1.1.0"

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   // versions
-  lazy val sparkVersion = "2.4.3"
+  lazy val sparkVersion = "2.4.4"
   lazy val hadoopVersion = "2.9.2"
 
   // arc

diff --git a/src/main/scala/ai/tripl/arc/config/ArcPipeline.scala b/src/main/scala/ai/tripl/arc/config/ArcPipeline.scala
@@ -78,7 +78,6 @@ object ArcPipeline {
           val lifecyclePluginsOrErrors = resolveConfigPlugins(resolvedConfig, "plugins.lifecycle", arcContext.lifecyclePlugins)(spark, logger, arcContext)
           val pipelinePluginsOrErrors = resolveConfigPlugins(resolvedConfig, "stages", arcContext.pipelineStagePlugins)(spark, logger, arcContext)
 
-
           (lifecyclePluginsOrErrors, pipelinePluginsOrErrors) match {
             case (Left(lifecycleErrors), Left(pipelineErrors)) => Left(lifecycleErrors.reverse ::: pipelineErrors.reverse)
             case (Right(_), Left(pipelineErrors)) => Left(pipelineErrors.reverse)

diff --git a/src/main/scala/ai/tripl/arc/config/Plugins.scala b/src/main/scala/ai/tripl/arc/config/Plugins.scala
@@ -34,7 +34,7 @@ object Plugins {
 
         // skip stage if not in environment
         if (!arcContext.ignoreEnvironments && !environments.contains(arcContext.environment.get)) {
-          logger.trace()
+          logger.info()
             .field("event", "validateConfig")
             .field("type", pluginType.right.getOrElse("unknown"))
             .field("stageIndex", index)

diff --git a/src/main/scala/ai/tripl/arc/extract/AvroExtract.scala b/src/main/scala/ai/tripl/arc/extract/AvroExtract.scala
@@ -74,7 +74,10 @@ class AvroExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         for (inputField <- inputField) {

diff --git a/src/main/scala/ai/tripl/arc/extract/BytesExtract.scala b/src/main/scala/ai/tripl/arc/extract/BytesExtract.scala
@@ -63,7 +63,10 @@ class BytesExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("failMode", stage.failMode.sparkString)
-        stage.stageDetail.put("input", if (c.hasPath("inputView")) inputView else parsedGlob)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(stage.persist))
         stage.stageDetail.put("params", params.asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/DelimitedExtract.scala b/src/main/scala/ai/tripl/arc/extract/DelimitedExtract.scala
@@ -84,11 +84,10 @@ class DelimitedExtract extends PipelineStagePlugin {
           case Some(basePath) => Delimited.toSparkOptions(stage.settings) + ("basePath" -> basePath)
           case None => Delimited.toSparkOptions(stage.settings)
         }
-        val inputValue = stage.input match {
-          case Left(view) => view
-          case Right(glob) => glob
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
         }
-        stage.stageDetail.put("input", inputValue)
         stage.stageDetail.put("options", options.asJava)
         for (inputField <- inputField) {
           stage.stageDetail.put("inputField", inputField)

diff --git a/src/main/scala/ai/tripl/arc/extract/HTTPExtract.scala b/src/main/scala/ai/tripl/arc/extract/HTTPExtract.scala
@@ -74,7 +74,10 @@ class HTTPExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("headers", HTTPUtils.maskHeaders("Authorization" :: Nil)(stage.headers).asJava)
-        stage.stageDetail.put("input", if(c.hasPath("inputView")) inputView else parsedURI)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("method", method)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))

diff --git a/src/main/scala/ai/tripl/arc/extract/ImageExtract.scala b/src/main/scala/ai/tripl/arc/extract/ImageExtract.scala
@@ -59,7 +59,7 @@ class ImageExtract extends PipelineStagePlugin {
           stage.stageDetail.put("basePath", basePath)
         }
         stage.stageDetail.put("dropInvalid", java.lang.Boolean.valueOf(dropInvalid))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("params", params.asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/JSONExtract.scala b/src/main/scala/ai/tripl/arc/extract/JSONExtract.scala
@@ -69,7 +69,10 @@ class JSONExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("options", JSON.toSparkOptions(stage.settings).asJava)

diff --git a/src/main/scala/ai/tripl/arc/extract/ORCExtract.scala b/src/main/scala/ai/tripl/arc/extract/ORCExtract.scala
@@ -61,7 +61,7 @@ class ORCExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         for (basePath <- basePath) {

diff --git a/src/main/scala/ai/tripl/arc/extract/ParquetExtract.scala b/src/main/scala/ai/tripl/arc/extract/ParquetExtract.scala
@@ -60,7 +60,7 @@ class ParquetExtract extends PipelineStagePlugin {
           contiguousIndex=contiguousIndex
         )
 
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))

diff --git a/src/main/scala/ai/tripl/arc/extract/TextExtract.scala b/src/main/scala/ai/tripl/arc/extract/TextExtract.scala
@@ -58,7 +58,7 @@ class TextExtract extends PipelineStagePlugin {
         )
 
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))
-        stage.stageDetail.put("input", parsedGlob)
+        stage.stageDetail.put("inputURI", parsedGlob)
         stage.stageDetail.put("multiLine", java.lang.Boolean.valueOf(multiLine))
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))

diff --git a/src/main/scala/ai/tripl/arc/extract/XMLExtract.scala b/src/main/scala/ai/tripl/arc/extract/XMLExtract.scala
@@ -63,7 +63,10 @@ class XMLExtract extends PipelineStagePlugin {
           contiguousIndex=contiguousIndex
         )
 
-        stage.stageDetail.put("input", input)
+        input match {
+          case Left(inputView) => stage.stageDetail.put("inputView", inputView)
+          case Right(parsedGlob) =>stage.stageDetail.put("inputURI", parsedGlob)
+        }
         stage.stageDetail.put("outputView", outputView)
         stage.stageDetail.put("persist", java.lang.Boolean.valueOf(persist))
         stage.stageDetail.put("contiguousIndex", java.lang.Boolean.valueOf(contiguousIndex))

diff --git a/version.sbt b/version.sbt
@@ -1 +1 @@
-version := "2.0.0"
+version := "2.0.1"