Merge remote-tracking branch 'origin/master' into SNAP-2366

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2017 SnappyData Inc.
+   Copyright 2018 SnappyData Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

build.gradle

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+ * Copyright (c) 2018 SnappyData, Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You
@@ -41,7 +41,7 @@ buildscript {
   }
   dependencies {
     classpath 'io.snappydata:gradle-scalatest:0.16'
-    classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:0.8.2'
+    classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:0.9.0'
     classpath "net.rdrei.android.buildtimetracker:gradle-plugin:0.8.+"
     classpath 'com.netflix.nebula:gradle-ospackage-plugin:4.4.+'
   }
@@ -129,7 +129,7 @@ allprojects {
     snappyStoreVersion = '1.6.2.1'
     snappydataVersion = version
     pulseVersion = '1.5.1'
-    zeppelinInterpreterVersion = '0.7.3.2'
+    zeppelinInterpreterVersion = '0.7.3.4'
     buildFlags = ''
     createdBy = System.getProperty('user.name')
     osArch = System.getProperty('os.arch')
@@ -139,7 +139,7 @@ allprojects {
     buildNumber = new Date().format('MMddyy')
     jdkVersion = System.getProperty('java.version')
     sparkJobServerVersion = '0.6.2.8'
-    kolobokeVersion = '1.0.0'
+    eclipseCollectionsVersion = '9.2.0'
 
     gitCmd = "git --git-dir=${rootDir}/.git --work-tree=${rootDir}"
     gitBranch = "${gitCmd} rev-parse --abbrev-ref HEAD".execute().text.trim()
@@ -217,7 +217,7 @@ configure(subprojects.findAll {!(it.name ==~ /snappy-spark.*/ ||
     inputEncoding = 'UTF-8'
     outputEncoding = 'UTF-8'
     outputFile = "${buildDir}/scalastyle-output.xml"
-    includeTestSourceDirectory = false
+    includeTestSourceDirectory = true
     source = 'src/main/scala'
     testSource = 'src/test/scala'
     failOnViolation = true
@@ -974,12 +974,6 @@ task product(type: Zip) {
       from("${clusterProject.projectDir}/benchmark")
       into "${snappyProductDir}/benchmark"
     }
-    def jdbcProject = project(":snappy-jdbc_${scalaBinaryVersion}")
-    copy {
-      from jdbcProject.shadowJar.destinationDir
-      into "${snappyProductDir}/connectors"
-      include jdbcProject.shadowJar.archiveName
-    }
   }
 }
 

cluster/bin/load-snappy-env.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/bin/snappy-job.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/bin/snappy-sql

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/build.gradle

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+ * Copyright (c) 2018 SnappyData, Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You
@@ -34,6 +34,31 @@ dependencies {
   compile 'org.slf4j:jcl-over-slf4j:' + slf4jVersion
   compile 'org.slf4j:jul-to-slf4j:' + slf4jVersion
 
+  compile (project(':snappy-core_' + scalaBinaryVersion)) {
+    exclude(group: 'org.apache.spark', module: 'spark-unsafe_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-core_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-catalyst_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-sql_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-hive_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-streaming_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-streaming-kafka-0-10_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-sql-kafka-0-10_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-mllib_' + scalaBinaryVersion)
+    exclude(group: 'org.eclipse.jetty', module: 'jetty-servlet')
+  }
+  testCompile (project(path: ':snappy-core_' + scalaBinaryVersion, configuration: 'testOutput')) {
+    exclude(group: 'org.apache.spark', module: 'spark-unsafe_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-core_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-catalyst_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-sql_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-hive_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-streaming_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-streaming-kafka-0-10_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-sql-kafka-0-10_' + scalaBinaryVersion)
+    exclude(group: 'org.apache.spark', module: 'spark-mllib_' + scalaBinaryVersion)
+    exclude(group: 'org.eclipse.jetty', module: 'jetty-servlet')
+  }
+
   if (new File(rootDir, 'spark/build.gradle').exists()) {
     compile project(':snappy-spark:snappy-spark-core_' + scalaBinaryVersion)
     compile project(':snappy-spark:snappy-spark-catalyst_' + scalaBinaryVersion)
@@ -73,18 +98,6 @@ dependencies {
     testCompile group: 'io.snappydata', name: 'snappy-spark-sql_' + scalaBinaryVersion,
                 version: snappySparkVersion, classifier: 'tests'
   }
-  compile (project(':snappy-core_' + scalaBinaryVersion)) {
-    exclude(group: 'org.apache.spark', module: 'spark-unsafe_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-core_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-catalyst_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-sql_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-hive_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-streaming_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-streaming-kafka-0-10_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-sql-kafka-0-10_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-mllib_' + scalaBinaryVersion)
-    exclude(group: 'org.eclipse.jetty', module: 'jetty-servlet')
-  }
 
   if (new File(rootDir, 'store/build.gradle').exists()) {
     testCompile project(path: ':snappy-store:snappydata-store-tools', configuration: 'testOutput')
@@ -118,19 +131,7 @@ dependencies {
   }
 
   testCompile project(':dunit')
-  testCompile (project(path: ':snappy-core_' + scalaBinaryVersion, configuration: 'testOutput')) {
-    exclude(group: 'org.apache.spark', module: 'spark-unsafe_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-core_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-catalyst_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-sql_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-hive_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-streaming_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-streaming-kafka-0-10_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-sql-kafka-0-10_' + scalaBinaryVersion)
-    exclude(group: 'org.apache.spark', module: 'spark-mllib_' + scalaBinaryVersion)
-    exclude(group: 'org.eclipse.jetty', module: 'jetty-servlet')
-  }
-  testCompile 'it.unimi.dsi:fastutil:8.1.1'
+  testCompile 'it.unimi.dsi:fastutil:8.2.2'
   testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}"
 
   if (new File(rootDir, 'aqp/build.gradle').exists() && rootProject.hasProperty('snappydata.enterprise')) {

cluster/conf/leads.template

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/conf/locators.template

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/conf/log4j.properties.template

@@ -1,4 +1,21 @@
 #
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+#
+# Some parts taken from Spark's log4j.properties having license below.
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -26,6 +43,15 @@ log4j.appender.file.MaxBackupIndex=10000
 log4j.appender.file.layout=io.snappydata.log4j.PatternLayout
 log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS zzz} %t %p %c{1}: %m%n
 
+# Appender for code dumps of WholeStageCodegenExec, CodeGenerator etc
+log4j.appender.code=org.apache.log4j.RollingFileAppender
+log4j.appender.code.append=true
+log4j.appender.code.file=generatedcode.log
+log4j.appender.code.MaxFileSize=1GB
+log4j.appender.code.MaxBackupIndex=10000
+log4j.appender.code.layout=io.snappydata.log4j.PatternLayout
+log4j.appender.code.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS zzz} %t %p %c{1}: %m%n
+
 # Console appender
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.out
@@ -75,8 +101,6 @@ log4j.logger.org.apache.spark.scheduler.FairSchedulableBuilder=WARN
 log4j.logger.org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint=WARN
 log4j.logger.org.apache.spark.storage.BlockManagerInfo=WARN
 log4j.logger.org.apache.hadoop.hive=WARN
-# for all Spark generated code (including ad-hoc UnsafeProjection calls etc)
-log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=WARN
 log4j.logger.org.apache.spark.sql.execution.datasources=WARN
 log4j.logger.org.apache.spark.scheduler.SnappyTaskSchedulerImpl=WARN
 log4j.logger.org.apache.spark.MapOutputTrackerMasterEndpoint=WARN
@@ -88,7 +112,6 @@ log4j.logger.parquet=ERROR
 log4j.logger.org.apache.hadoop.io.compress=WARN
 log4j.logger.spark.jobserver.LocalContextSupervisorActor=WARN
 log4j.logger.spark.jobserver.JarManager=WARN
-log4j.logger.org.apache.spark.sql.hive.HiveClientUtil=WARN
 log4j.logger.org.datanucleus=ERROR
 # Task logger created in SparkEnv
 log4j.logger.org.apache.spark.Task=WARN
@@ -101,6 +124,13 @@ log4j.logger.io.snappydata.impl.LocatorImpl=INFO
 log4j.logger.spray.can.server.HttpListener=INFO
 
 # for generated code of plans
-# log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=DEBUG
+log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=INFO, code
+log4j.additivity.org.apache.spark.sql.execution.WholeStageCodegenExec=false
+log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenRDD=INFO, code
+log4j.additivity.org.apache.spark.sql.execution.WholeStageCodegenRDD=false
+# for all Spark generated code (including ad-hoc UnsafeProjection calls etc)
+log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=WARN, code
+log4j.additivity.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=false
 # for SnappyData generated code used on store (ComplexTypeSerializer, JDBC inserts ...)
-# log4j.logger.org.apache.spark.sql.store.CodeGeneration=DEBUG
+log4j.logger.org.apache.spark.sql.store.CodeGeneration=INFO, code
+log4j.additivity.org.apache.spark.sql.store.CodeGeneration=false

cluster/conf/servers.template

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/sbin/collect-debug-artifacts.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You
@@ -88,6 +88,9 @@ while [ "$1" != "" ]; do
     -d|--dump)
     DUMP_STACK=1
     ;;
+    -m|--hprofdump)
+    HPROF_DUMP=1
+    ;;
     *)
     usage
     exit 1 
@@ -223,10 +226,14 @@ function collect_data {
     INTERVAL_BETWEEN_DUMPS=0
   fi
 
+  if [ "${HPROF_DUMP}" != "1" ]; then
+    HPROF_DUMP=0
+  fi
+
   # Create the outdir with the same name on each remote and collect everything there.
   typeset -f | ssh $host "$(cat);collect_on_remote \"${wd}\" \"${NO_OF_STACK_DUMPS}\" \\
       \"${INTERVAL_BETWEEN_DUMPS}\" \"${GET_EVERYTHING}\" \"${collector_host}\" \\
-      \"${VERBOSE}\" \"${START_EPOCH}\" \"${END_EPOCH}\""
+      \"${VERBOSE}\" \"${START_EPOCH}\" \"${END_EPOCH}\" \"${HPROF_DUMP}\""
 }
 
 function collect_on_remote {
@@ -238,6 +245,7 @@ function collect_on_remote {
   verbose="$6"
   start_epoch="$7"
   end_epoch="$8"
+  get_hprof="$9"
 
   # Create a .tmpcda dir if not exists else empty it
   tmp_dir="$data_dir/.tmpcda"
@@ -278,10 +286,6 @@ function collect_on_remote {
     do
       files+=($l)
     done
-    for l in $( ls *.jfr* 2> /dev/null )
-    do
-      files+=($l)
-    done
     for l in $( ls *.conf* 2> /dev/null )
     do
       files+=($l)
@@ -424,6 +428,31 @@ function collect_on_remote {
     done
   fi
 
+  # Add other files
+  for l in $( ls *.jfr* 2> /dev/null )
+  do
+    files+=($l)
+  done
+  for l in $( ls jvmkill*.log 2> /dev/null )
+  do
+    files+=($l)
+  done
+  for l in $( ls *.jmap 2> /dev/null )
+  do
+    files+=($l)
+  done
+
+  # Add hprof files too if asked for
+  if [ "${get_hprof}" = "1" ]; then
+    if [ "${verbose}" = "1" ]; then
+      echo "collecting hprof files too"
+    fi
+    for l in $( ls *.hprof 2> /dev/null )
+    do
+      files+=($l)
+    done
+  fi
+
   for f in "${files[@]}"
   do
     if [ "${verbose}" = "1" ]; then

cluster/sbin/snappy-config.sh

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/sbin/snappy-lead.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/sbin/snappy-leads.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/sbin/snappy-locator.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You

cluster/sbin/snappy-locators.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+# Copyright (c) 2018 SnappyData, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License. You