first commit

.Rbuildignore

@@ -0,0 +1,2 @@
+^.*\.Rproj$
+^\.Rproj\.user$

.gitignore

@@ -0,0 +1,4 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata

DESCRIPTION

@@ -0,0 +1,16 @@
+Package: sparkhail
+Type: Package
+Title: What the Package Does (Title Case)
+Version: 0.1.0
+Author: Who wrote it
+Maintainer: The package maintainer <yourself@somewhere.net>
+Description: More about what it does (maybe more than one line)
+  Use four spaces when indenting paragraphs within the Description.
+License: What license is it under?
+Encoding: UTF-8
+LazyData: true
+Depends:
+  R (>= 3.1.2)
+Imports:
+  sparklyr
+RoxygenNote: 6.1.1

NAMESPACE

@@ -0,0 +1,7 @@
+# Generated by roxygen2: do not edit by hand
+
+export(compile_jars)
+export(create_option_int)
+export(create_option_string)
+export(hail_context)
+import(sparklyr)

R/compile_jars.R

@@ -0,0 +1,23 @@
+#' @export
+compile_jars<- function(){
+  spark_2.2 <- spark_compilation_spec(
+    spark_version = "2.2.0",
+    scalac_path = sparklyr::find_scalac("2.11"),
+    jar_name = sprintf("sparkhail-2.2-2.11.jar"),
+    jar_path = NULL,
+    scala_filter = NULL
+  ) 
+
+  spark_2.4 <- spark_compilation_spec(
+    spark_version = "2.4.0",
+    scalac_path = sparklyr::find_scalac("2.11"),
+    jar_name = sprintf("sparkhail-2.4-2.11.jar"),
+    jar_path = NULL,
+    scala_filter = NULL
+  ) 
+
+  sparklyr::compile_package_jars(spec = spark_2.2)
+  sparklyr::compile_package_jars(spec = spark_2.4)
+
+  sparklyr::compile_package_jars()
+}

R/create_option.R

@@ -0,0 +1,11 @@
+#' @export
+create_option_string<- function(sc, value){
+  if (value == "none") {
+    sparklyr::invoke_static(sc, "sparkhail.CreateOption", "stringNone")
+  }
+}
+
+#' @export
+create_option_int <- function(sc, value){
+  sparklyr::invoke_static(sc, "sparkhail.CreateOption", "int", as.integer(value))
+}

R/dependencies.R

@@ -0,0 +1,21 @@
+spark_dependencies <- function(spark_version, scala_version, ...) {
+  sparklyr::spark_dependency(
+    jars = c(
+      system.file(
+        sprintf("java/sparkhail-%s-%s.jar", spark_version, scala_version),
+        package = "sparkhail"
+      ),
+      system.file(
+        sprintf("java/hail-all-spark.jar"),
+        package = "sparkhail"
+      )
+    ),
+    packages = c(
+    )
+  )
+}
+
+#' @import sparklyr
+.onLoad <- function(libname, pkgname) {
+  sparklyr::register_extension(pkgname)
+}

R/hail_context.R

@@ -0,0 +1,13 @@
+#' @export
+hail_context <- function(spark_version = "2.4.0"){
+  conf <- sparklyr::spark_config()
+  conf$`spark.serializer` = "org.apache.spark.serializer.KryoSerializer"
+  conf$`spark.kryo.registrator` = "is.hail.kryo.HailKryoRegistrator"
+  sc <- sparklyr::spark_connect(master = "local", config = conf, version = spark_version)
+
+  sparklyr::invoke_static(sc, "is.hail.HailContext", "getOrCreate",
+                          spark_context(sc), "Hail",
+                          invoke_static(sc, "sparkhail.CreateOption", "stringNone"),
+                          sc$master, "hail.log", FALSE, FALSE, 1, 50L, "/tmp", 3L )
+
+}

R/tests.R

@@ -0,0 +1,106 @@
+# library(sparklyr)
+# conf <- spark_config()
+# conf$`spark.serializer` = "org.apache.spark.serializer.KryoSerializer"
+# conf$`spark.kryo.registrator` = "is.hail.kryo.HailKryoRegistrator"
+# sc <- spark_connect(master = "local", config = conf)
+# conf$`spark.driver.allowMultipleContexts` <- TRUE
+
+# 
+# # 
+# # # #
+# hl <- sparklyr::invoke_static(sc, "is.hail.HailContext", "createSparkConf",
+#  sc$app_name, invoke_static(sc, "sparkhail.CreateOption", "none"), sc$master, 1L)
+# # 
+# # 
+# # sparklyr::invoke_static(sc, "is.hail.HailContext", "createSparkConf",
+# #                         sc$app_name, invoke_static(sc, "hailr.test.CreateOption", "none"), 8L, 1L)
+# # 
+# # 
+# # hl2 <- sparklyr::invoke_static(sc, "is.hail.HailContext", "theContext")
+# # # #
+# # #
+# # # sparklyr::invoke(hl, "is.hail.HailContext", "checkSparkConfiguration", hl)
+# # #
+# # # sparklyr::invoke_new(sc, "is.hail.HailContext", "HailContext")
+# # #
+# # #
+# # #                         sc$app_name, invoke_static(sc, "hailr.test.CreateOption", "none"),
+# # #                         sc$master, 1L)
+# # #
+# # 
+# # # invoke_stat
+# # 
+# hl2 <- sparklyr::invoke_static(sc, "is.hail.HailContext", "getOrCreate",
+# spark_context(sc), "Hail",invoke_static(sc, "sparkhail.CreateOption", "stringNone"),
+#  sc$master, "hail.log", FALSE, FALSE, 1, 50L, "/tmp", 3L )
+# 
+# mt <- invoke_static(sc, "is.hail.variant.MatrixTable", "read",
+#               hl2, "data/1kg.mt", FALSE, FALSE)
+# 
+# invoke_static(sc, "is.hail.TestUtils", "interceptFatal",
+#   hl2, "sdfsdf", FALSE, FALSE,
+#               invoke_static(sc, "hailr.test.CreateOption", "none"))
+# 
+# def importVCF(hc: HailContext, file: String, force: Boolean = false,
+#               forceBGZ: Boolean = false,
+#               headerFile: Option[String] = None,
+#               nPartitions: Option[Int] = None,
+#               dropSamples: Boolean = false,
+#               callFields: Set[String] = Set.empty[String],
+#               rg: Option[ReferenceGenome] = Some(ReferenceGenome.GRCh37),
+#               contigRecoding: Option[Map[String, String]] = None,
+#               arrayElementsRequired: Boolean = true,
+#               skipInvalidLoci: Boolean = false,
+#               partitionsJSON: String = null): MatrixTable
+# 
+# 
+# # 
+# # sc1 <- sparklyr::invoke_static(sc, "is.hail.HailContext", "configureAndCreateSparkContext",
+# #                               spark_context(sc))
+# # hl <- invoke_static(sc, "is.hail.HailContext", "backend")
+# # 
+# # sparklyr::invoke_static(sc, "is.hail.HailContext", "configureAndCreateSparkContext",
+# #                         sc$app_name, invoke_static(sc, "hailr.test.CreateOption", "none"), sc$master, 1L)
+# # 
+# # sparklyr::invoke_static(sc, "is.hail.HailContext", "checkSparkConfiguration",
+# #                         spark_context(sc))
+# # 
+# # 
+# # # hl2 <- sparklyr::invoke_static(sc, "is.hail.HailContext", "apply",
+# # #                                NULL, "Hail",invoke_static(sc, "hailr.test.CreateOption", "none"),
+# # #                                sc$master, "hail.log", FALSE, FALSE, 1L, 50, "/tmp", 3 )
+# # #
+# # #
+# # # sqlContext = invoke_new(sc, "org.apache.spark.sql.SQLContext", spark_context(sc))
+# # #
+# # # invoke_new(sc, "is.hail.HailContext", spark_context(sc), sqlContext, "hail.log", "/tmp", 50, 3)
+# # #
+# # #
+# # # scon <- invoke_static(sc, "is.hail.HailContext", "configureAndCreateSparkContext",
+# # #                       sc$app_name, invoke_static(sc, "hailr.test.CreateOption", "none"), sc$master, 1L)
+# # #
+# # # def getOrCreate(sc: SparkContext = null,
+# # #                 appName: String = "Hail",
+# # #                 master: Option[String] = None,
+# # #                 local: String = "local[*]",
+# # #                 logFile: String = "hail.log",
+# # #                 quiet: Boolean = false,
+# # #                 append: Boolean = false,
+# # #                 minBlockSize: Long = 1L,
+# # #                 branchingFactor: Int = 50,
+# # #                 tmpDir: String = "/tmp",
+# # #                 optimizerIterations: Int = 3)
+# # #
+# # #
+# # # spark_2.2 <- spark_compilation_spec(
+# # #   spark_version = "2.2.0",
+# # #   scalac_path = sparklyr::find_scalac("2.11"),
+# # #   jar_name = sprintf("hailr.test-2.2-2.11.jar"),
+# # #   jar_path = NULL,
+# # #   scala_filter = NULL
+# # # )
+# # #
+# # #
+# # # hl2 <- sparklyr::invoke_static(sc, "is.hail", "HailContext")
+# # #
+# # #

README.Rmd

@@ -0,0 +1,33 @@
+---
+title: "What the Package Does (Title Case)"
+output:
+  github_document:
+    fig_width: 9
+    fig_height: 5
+---
+
+## Building
+
+First build this package, then build its jars by running:
+
+```{r eval=FALSE}
+sparklyr::compile_package_jars()
+```
+
+then build the R package as usual.
+
+## Getting Started
+
+Connect and test this package as follows:
+
+```{r}
+library(hailr.test)
+library(sparklyr)
+sc <- spark_connect(master = "local")
+
+hailr.test_hello(sc)
+```
+
+```{r}
+spark_disconnect_all()
+```