Merge remote-tracking branch 'origin/master' into file-handle-optimizations

Author: JoshRosen

.rat-excludes

@@ -15,20 +15,8 @@ TAGS
 RELEASE
 control
 docs
-docker.properties.template
-fairscheduler.xml.template
-spark-defaults.conf.template
-log4j.properties
-log4j.properties.template
-metrics.properties
-metrics.properties.template
 slaves
-slaves.template
-spark-env.sh
 spark-env.cmd
-spark-env.sh.template
-log4j-defaults.properties
-log4j-defaults-repl.properties
 bootstrap-tooltip.js
 jquery-1.11.1.min.js
 d3.min.js
@@ -86,4 +74,12 @@ local-1430917381535_2
 DESCRIPTION
 NAMESPACE
 test_support/*
+.*Rd
+help/*
+html/*
+INDEX
 .lintr
+gen-java.*
+.*avpr
+org.apache.spark.sql.sources.DataSourceRegister
+.*parquet

LICENSE

@@ -572,3 +572,38 @@ Copyright 2009-2013 The Apache Software Foundation
 
 Apache Avro IPC
 Copyright 2009-2013 The Apache Software Foundation
+
+
+Vis.js
+Copyright 2010-2015 Almende B.V.
+
+Vis.js is dual licensed under both
+
+  * The Apache 2.0 License
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    and
+
+  * The MIT License
+    http://opensource.org/licenses/MIT
+
+Vis.js may be distributed under either license.
+
+
+Vis.js uses and redistributes the following third-party libraries:
+
+- component-emitter
+  https://github.com/component/emitter
+  The MIT License
+
+- hammer.js
+  http://hammerjs.github.io/
+  The MIT License
+
+- moment.js
+  http://momentjs.com/
+  The MIT License
+
+- keycharm
+  https://github.com/AlexDM0/keycharm
+  The MIT License

NOTICE

@@ -6,7 +6,7 @@ SparkR is an R package that provides a light-weight frontend to use Spark from R
 
 #### Build Spark
 
-Build Spark with [Maven](http://spark.apache.org/docs/latest/building-spark.html#building-with-buildmvn) and include the `-PsparkR` profile to build the R package. For example to use the default Hadoop versions you can run
+Build Spark with [Maven](http://spark.apache.org/docs/latest/building-spark.html#building-with-buildmvn) and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
 ```
   build/mvn -DskipTests -Psparkr package
 ```

R/README.md

@@ -39,7 +39,7 @@ pushd $FWDIR
 mkdir -p pkg/html
 pushd pkg/html
 
-Rscript -e 'library(SparkR, lib.loc="../../lib"); library(knitr); knit_rd("SparkR")'
+Rscript -e 'libDir <- "../../lib"; library(SparkR, lib.loc=libDir); library(knitr); knit_rd("SparkR", links = tools::findHTMLlinks(paste(libDir, "SparkR", sep="/")))'
 
 popd
 

R/create-docs.sh

@@ -34,12 +34,12 @@ LIB_DIR="$FWDIR/lib"
 
 mkdir -p $LIB_DIR
 
-pushd $FWDIR
+pushd $FWDIR > /dev/null
 
 # Generate Rd files if devtools is installed
 Rscript -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'
 
 # Install SparkR to $LIB_DIR
 R CMD INSTALL --library=$LIB_DIR $FWDIR/pkg/
 
-popd
+popd > /dev/null

R/install-dev.sh

@@ -1,7 +1,7 @@
 Package: SparkR
 Type: Package
 Title: R frontend for Spark
-Version: 1.4.0
+Version: 1.6.0
 Date: 2013-09-09
 Author: The Apache Software Foundation
 Maintainer: Shivaram Venkataraman <shivaram@cs.berkeley.edu>
@@ -29,7 +29,10 @@ Collate:
     'client.R'
     'context.R'
     'deserialize.R'
+    'functions.R'
+    'mllib.R'
     'serialize.R'
     'sparkR.R'
+    'stats.R'
+    'types.R'
     'utils.R'
-    'zzz.R'

R/pkg/DESCRIPTION

@@ -10,6 +10,11 @@ export("sparkR.init")
 export("sparkR.stop")
 export("print.jobj")
 
+# MLlib integration
+exportMethods("glm",
+              "predict",
+              "summary")
+
 # Job group lifecycle management methods
 export("setJobGroup",
        "clearJobGroup",
@@ -18,11 +23,18 @@ export("setJobGroup",
 exportClasses("DataFrame")
 
 exportMethods("arrange",
+              "as.data.frame",
+              "attach",
               "cache",
               "collect",
+              "coltypes",
               "columns",
               "count",
+              "cov",
+              "corr",
+              "crosstab",
               "describe",
+              "dim",
               "distinct",
               "dropna",
               "dtypes",
@@ -31,6 +43,7 @@ exportMethods("arrange",
               "fillna",
               "filter",
               "first",
+              "freqItems",
               "group_by",
               "groupBy",
               "head",
@@ -39,16 +52,22 @@ exportMethods("arrange",
               "isLocal",
               "join",
               "limit",
-              "orderBy",
+              "merge",
               "mutate",
+              "na.omit",
               "names",
+              "ncol",
+              "nrow",
+              "orderBy",
               "persist",
               "printSchema",
+              "rbind",
               "registerTempTable",
               "rename",
               "repartition",
               "sample",
               "sample_frac",
+              "sampleBy",
               "saveAsParquetFile",
               "saveAsTable",
               "saveDF",
@@ -57,77 +76,173 @@ exportMethods("arrange",
               "selectExpr",
               "show",
               "showDF",
+              "subset",
               "summarize",
+              "summary",
               "take",
+              "transform",
               "unionAll",
+              "unique",
               "unpersist",
               "where",
+              "with",
               "withColumn",
               "withColumnRenamed",
               "write.df")
 
 exportClasses("Column")
 
-exportMethods("abs",
+exportMethods("%in%",
+              "abs",
               "acos",
+              "add_months",
               "alias",
               "approxCountDistinct",
               "asc",
+              "ascii",
               "asin",
               "atan",
               "atan2",
               "avg",
+              "base64",
+              "between",
+              "bin",
+              "bitwiseNOT",
               "cast",
               "cbrt",
+              "ceil",
               "ceiling",
+              "column",
+              "concat",
+              "concat_ws",
               "contains",
+              "conv",
               "cos",
               "cosh",
+              "count",
               "countDistinct",
+              "crc32",
+              "cumeDist",
+              "date_add",
+              "date_format",
+              "date_sub",
+              "datediff",
+              "dayofmonth",
+              "dayofyear",
+              "denseRank",
               "desc",
               "endsWith",
               "exp",
+              "explode",
               "expm1",
+              "expr",
+              "factorial",
+              "first",
               "floor",
+              "format_number",
+              "format_string",
+              "from_unixtime",
+              "from_utc_timestamp",
               "getField",
               "getItem",
+              "greatest",
+              "hex",
+              "hour",
               "hypot",
+              "ifelse",
+              "initcap",
+              "instr",
+              "isNaN",
               "isNotNull",
               "isNull",
+              "lag",
               "last",
+              "last_day",
+              "lead",
+              "least",
+              "length",
+              "levenshtein",
               "like",
+              "lit",
+              "locate",
               "log",
               "log10",
               "log1p",
+              "log2",
               "lower",
+              "lpad",
+              "ltrim",
               "max",
+              "md5",
               "mean",
               "min",
+              "minute",
+              "month",
+              "months_between",
               "n",
               "n_distinct",
+              "nanvl",
+              "negate",
+              "next_day",
+              "ntile",
+              "otherwise",
+              "percentRank",
+              "pmod",
+              "quarter",
+              "rand",
+              "randn",
+              "rank",
+              "regexp_extract",
+              "regexp_replace",
+              "reverse",
               "rint",
               "rlike",
+              "round",
+              "rowNumber",
+              "rpad",
+              "rtrim",
+              "second",
+              "sha1",
+              "sha2",
+              "shiftLeft",
+              "shiftRight",
+              "shiftRightUnsigned",
               "sign",
+              "signum",
               "sin",
               "sinh",
+              "size",
+              "soundex",
               "sqrt",
               "startsWith",
               "substr",
+              "substring_index",
               "sum",
               "sumDistinct",
               "tan",
               "tanh",
               "toDegrees",
               "toRadians",
-              "upper")
+              "to_date",
+              "to_utc_timestamp",
+              "translate",
+              "trim",
+              "unbase64",
+              "unhex",
+              "unix_timestamp",
+              "upper",
+              "weekofyear",
+              "when",
+              "year")
 
 exportClasses("GroupedData")
 exportMethods("agg")
 
 export("sparkRSQL.init",
        "sparkRHive.init")
 
-export("cacheTable",
+export("as.DataFrame",
+       "cacheTable",
        "clearCache",
        "createDataFrame",
        "createExternalTable",
@@ -149,4 +264,4 @@ export("structField",
        "structType",
        "structType.jobj",
        "structType.structField",
-       "print.structType")
+       "print.structType")