build.gradle

buildscript {
    repositories {
        mavenCentral()
        jcenter()
    }
}

plugins {
  id "com.gradle.build-scan" version "1.0"
  id 'java'
  id 'scala'
  id 'idea'
  id 'maven'
  id 'jacoco'
  id 'com.github.johnrengelman.shadow' version '1.2.3'
}

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

repositories {
    // Get kudu-spark2_2.11 with custom pom from our repo since we can't
    // figure out how to enable spark2_2.11 profile in standard pom.
    maven {
        url 'https://storage.googleapis.com/hail-common/maven-repo'
    }
    mavenCentral()
    jcenter()
    maven {
        url "https://repository.cloudera.com/artifactory/cloudera-repos/" // kudu
    }
}

buildScan {
    licenseAgreementUrl = 'https://gradle.com/terms-of-service'
    licenseAgree = 'yes'
}

String sparkVersion = System.getProperty("spark.version","2.0.2")

if (!(sparkVersion ==~ /^2\..*/))
    ant.fail('Hail does not support Spark version ' + sparkVersion + '. Hail team recommends version 2.0.2.')

String scalaVersion = '2.11.8'
String scalaMajorVersion = '2.11'

String py4jVersion
String breezeVersion
if (sparkVersion ==~ /^2\.1.*/) {
    py4jVersion = '0.10.4'
    breezeVersion = '0.12'
} else {
    py4jVersion = '0.10.3'
    breezeVersion = '0.11.2'
}

String kuduSpark = 'spark2'
String mongoVersion = '2.0.0'

String sparkHome = System.getProperty("spark.home", System.env.SPARK_HOME)
String tutorialFileLoc = System.getProperty("tutorial.home", "")

sourceSets.main.scala.srcDir "src/main/java"
sourceSets.main.java.srcDirs = []
sourceSets.test.runtimeClasspath += files("src/main/c/lib")

task nativeLib(type: Exec) {
    workingDir 'src/main/c'
    executable 'make'
    outputs.upToDateWhen { false }
}

task nativeLibTest(type: Exec) {
    workingDir 'src/main/c'
    args('test')
    executable 'make'
    outputs.upToDateWhen { false }
}

task nativeLibClean(type: Exec) {
    workingDir 'src/main/c'
    args('clean')
    executable 'make'
    outputs.upToDateWhen { false }
}

sourceSets {
    main {
        resources {
            srcDirs "src/main/c/lib"
        }
    }
}
compileScala.dependsOn(nativeLib)
test.dependsOn(nativeLibTest)
clean.dependsOn(nativeLibClean)

compileJava {
    options.compilerArgs << "-Xlint:all" << "-Werror"
}

task generateBuildInfo(type: Exec) {
    commandLine 'sh', 'generate-build-info.sh', sparkVersion
    outputs.upToDateWhen { false }
}

compileScala {
    dependsOn generateBuildInfo

    scalaCompileOptions.additionalParameters = ["-feature"] // , "-Xfatal-warnings"]
}

dependencies {
    compile 'org.scala-lang:scala-library:' + scalaVersion
    compile 'org.scala-lang:scala-reflect:' + scalaVersion
    compile('org.apache.spark:spark-core_' + scalaMajorVersion + ':' + sparkVersion) {
        exclude module: 'hadoop-client'
    }
    compile('org.apache.hadoop:hadoop-client:2.7.1') {
        exclude module: 'servlet-api'
    }
    compile 'org.apache.spark:spark-sql_' + scalaMajorVersion + ':' + sparkVersion
    compile 'org.apache.spark:spark-mllib_' + scalaMajorVersion + ':' + sparkVersion
    compile 'net.jpountz.lz4:lz4:1.3.0'
    compile 'org.scalanlp:breeze-natives_' + scalaMajorVersion + ':' + breezeVersion
    compile 'com.github.samtools:htsjdk:2.5.0'
    compile 'org.apache.kudu:kudu-client:1.1.0'
    compile 'org.apache.kudu:kudu-' + kuduSpark + '_' + scalaMajorVersion + ':1.1.0'

    compile 'org.http4s:http4s-core_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-server_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-argonaut_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-dsl_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-scala-xml_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-client_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-websocket_' + scalaMajorVersion + ':0.1.3'
    compile 'org.http4s:http4s-blaze-core_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-blaze-client_' + scalaMajorVersion + ':0.12.3'
    compile 'org.http4s:http4s-blaze-server_' + scalaMajorVersion + ':0.12.3'
    compile 'org.json4s:json4s-core_' + scalaMajorVersion + ':3.2.10'
    // compile 'org.json4s:json4s-native_' + scalaMajorVersion + ':3.2.10'
    compile 'org.json4s:json4s-jackson_' + scalaMajorVersion + ':3.2.10'
    compile 'org.json4s:json4s-ast_' + scalaMajorVersion + ':3.2.10'
    //compile 'org.json4s:json4s-native_' + scalaMajorVersion + ':3.3.0'
    //compile 'org.json4s:json4s-jackson_' + scalaMajorVersion + ':3.3.0'
    //compile 'org.json4s:json4s-ast_' + scalaMajorVersion + ':3.3.0'

    compile 'org.mongodb.spark:mongo-spark-connector_' + scalaMajorVersion + ':' + mongoVersion

    compile 'org.apache.solr:solr-solrj:6.2.0'
    compile 'com.datastax.cassandra:cassandra-driver-core:3.0.0'

    compile 'com.jayway.restassured:rest-assured:2.8.0'

    compile group: 'org.ow2.asm', name: 'asm', version: '5.1'
    compile group: 'org.ow2.asm', name: 'asm-util', version: '5.1'
    compile group: 'org.ow2.asm', name: 'asm-analysis', version: '5.1'

    compile 'net.java.dev.jna:jna:4.2.2'
    compile 'net.sourceforge.jdistlib:jdistlib:0.4.5'

    testCompile 'org.testng:testng:6.8.21'
    testCompile 'org.scalatest:scalatest_' + scalaMajorVersion + ':2.2.4'
}

task(checkSettings) << {
    def checkSeed = System.getProperty("check.seed", "1")
    if (checkSeed == "random")
        checkSeed = new Random().nextInt().toString()
    def checkSize = System.getProperty("check.size", "1000")
    def checkCount = System.getProperty("check.count", "10")

    println "check: seed = $checkSeed, size = $checkSize, count = $checkCount"

    // override with these defaults, random seed
    System.setProperty("check.seed", checkSeed)
    System.setProperty("check.size", checkSize)
    System.setProperty("check.count", checkCount)
}

test {
    useTestNG {}

    systemProperties System.getProperties()

    testLogging {
        events "passed", "skipped", "failed"
    }

    // listen to events in the test execution lifecycle
    beforeTest { descriptor ->
        logger.lifecycle("Running test: " + descriptor)
    }
}

test.dependsOn(checkSettings)

task testPython(type: Exec, dependsOn: shadowJar) {
     commandLine 'python', '-m', 'unittest', 'hail.tests'

     environment SPARK_HOME: sparkHome
     environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
     environment SPARK_CLASSPATH: '' + projectDir + '/build/libs/hail-all-spark.jar'
}

task testAll(dependsOn: ['testPython', 'test'])

tasks.withType(ShadowJar) {
    manifest {
        attributes 'Implementation-Title': 'Hail',
                'Implementation-Version': '0.0.1-SNAPSHOT'
    }
    baseName = project.name + '-all'
    mergeServiceFiles()
    zip64 true
    // conflict with version in default Hadoop/Spark install
    relocate 'org.apache.http', 'is.hail.relocated.org.apache.http'
    relocate 'com.google.common', 'is.hail.relocated.com.google.common'
    relocate 'org.objectweb', 'is.hail.relocated.org.objectweb'
}

shadowJar {
    classifier = 'spark'
    from(project.sourceSets.main.output)
    configurations = [project.configurations.runtime]
    dependencies {
        include(dependency('net.jpountz.lz4:lz4:.*'))
        include(dependency('org.scalanlp:breeze-natives_' + scalaMajorVersion + ':.*'))
        include(dependency('com.github.samtools:htsjdk:.*'))

        include(dependency('org.json4s:json4s-core_' + scalaMajorVersion + ':.*'))
        // include(dependency('org.json4s:json4s-native_' + scalaMajorVersion + ':.*'))
        include(dependency('org.json4s:json4s-jackson_' + scalaMajorVersion + ':.*'))
        include(dependency('org.json4s:json4s-ast_' + scalaMajorVersion + ':.*'))

        include(dependency('org.http4s:http4s-core_' + scalaMajorVersion + ':.*'))

        include(dependency('org.apache.kudu:kudu-client:.*'))
        include(dependency('org.apache.kudu:kudu-' + kuduSpark + '_' + scalaMajorVersion + ':.*'))

        include(dependency('org.mongodb.spark:mongo-spark-connector_' + scalaMajorVersion + ':.*'))
        include(dependency('org.mongodb:mongo-java-driver:.*'))

        // solr dependencies
        include(dependency('org.apache.solr:solr-solrj:.*'))
        include(dependency('org.apache.httpcomponents:httpclient:.*'))
        include(dependency('org.apache.httpcomponents:httpcore:.*'))
        include(dependency('org.apache.httpcomponents:httpmime:.*'))
        include(dependency('org.apache.zookeeper:zookeeper:.*'))
        include(dependency('org.codehaus.woodstox:stax2-api:.*'))
        include(dependency('org.codehaus.woodstox:woodstox-core-asl:.*'))
        include(dependency('org.noggit:noggit:.*'))

        include(dependency('com.datastax.cassandra:cassandra-driver-core:.*'))
        include(dependency('com.google.guava:guava:.*'))
        include(dependency('org.apache.spark:kudu-client:.*'))
        include(dependency('org.apache.spark:kudu-spark_' + scalaMajorVersion + ':.*'))

        include(dependency('org.ow2.asm:asm:.*'))
        include(dependency('org.ow2.asm:asm-util:.*'))
        include(dependency('org.ow2.asm:asm-tree:.*'))
        include(dependency('org.ow2.asm:asm-analysis:.*'))

        include(dependency('net.java.dev.jna:jna:.*'))
        include(dependency('net.sourceforge.jdistlib:jdistlib:.*'))
    }
}

task shadowTestJar(type: ShadowJar) {
    classifier = 'spark-test'
    from(project.sourceSets.main.output, project.sourceSets.test.output)
    configurations = [project.configurations.testRuntime]
    dependencies {
        include(dependency('net.jpountz.lz4:lz4:.*'))
        include(dependency('org.scalanlp:breeze-natives_' + scalaMajorVersion + ':.*'))
        include(dependency('com.github.samtools:htsjdk:.*'))

        include(dependency('org.json4s:json4s-core_' + scalaMajorVersion + ':.*'))
        // include(dependency('org.json4s:json4s-native_' + scalaMajorVersion + ':.*'))
        include(dependency('org.json4s:json4s-jackson_' + scalaMajorVersion + ':.*'))
        include(dependency('org.json4s:json4s-ast_' + scalaMajorVersion + ':.*'))

        include(dependency('org.http4s:http4s-core_' + scalaMajorVersion + ':.*'))
        include(dependency('org.scalaz:scalaz-core_' + scalaMajorVersion + ':.*'))
        include(dependency('org.scalaz:scalaz-concurrent_' + scalaMajorVersion + ':.*'))
        include(dependency('org.scalaz:scalaz-stream_' + scalaMajorVersion + ':.*'))

        include(dependency('org.apache.kudu:kudu-client:.*'))
        include(dependency('org.apache.kudu:kudu-' + kuduSpark + '_' + scalaMajorVersion + ':.*'))

        include(dependency('org.mongodb.spark:mongo-spark-connector_' + scalaMajorVersion + ':.*'))
        include(dependency('org.mongodb:mongo-java-driver:.*'))

        // solr dependencies
        include(dependency('org.apache.solr:solr-solrj:.*'))
        include(dependency('org.apache.httpcomponents:httpclient:.*'))
        include(dependency('org.apache.httpcomponents:httpcore:.*'))
        include(dependency('org.apache.httpcomponents:httpmime:.*'))
        include(dependency('org.apache.zookeeper:zookeeper:.*'))
        include(dependency('org.codehaus.woodstox:stax2-api:.*'))
        include(dependency('org.codehaus.woodstox:woodstox-core-asl:.*'))
        include(dependency('org.noggit:noggit:.*'))

        include(dependency('com.datastax.cassandra:cassandra-driver-core:.*'))
        include(dependency('com.google.guava:guava:.*'))
        include(dependency('org.apache.spark:kudu-client:.*'))
        include(dependency('org.apache.spark:kudu-spark_' + scalaMajorVersion + ':.*'))

        include(dependency('org.ow2.asm:asm:.*'))
        include(dependency('org.ow2.asm:asm-util:.*'))
        include(dependency('org.ow2.asm:asm-tree:.*'))
        include(dependency('org.ow2.asm:asm-analysis:.*'))

        include(dependency('org.testng:testng:.*'))
        include(dependency('com.beust:jcommander:.*'))
        include(dependency('org.scalatest:scalatest_' + scalaMajorVersion + ':.*'))

        include(dependency('net.java.dev.jna:jna:.*'))
        include(dependency('net.sourceforge.jdistlib:jdistlib:.*'))
    }
}

jacocoTestReport {
    dependsOn test
    reports {
        xml.enabled false
        csv.enabled false
        html.destination "${buildDir}/reports/coverage"
    }
}

task coverage(dependsOn: jacocoTestReport)

task testJar(type: Jar) {
    classifier = 'tests'
    from sourceSets.test.output
}

task archiveZip(type: Zip) {
    from fileTree('python')
    classifier = 'python'
}

task wrapper(type: Wrapper) {
    gradleVersion = '2.14.1'
}

task cleanHailDocs(type: Exec) {
    commandLine 'rm', '-rf', 'build/www/hail', 'build/tmp/python', 'build/tmp/docs'
}

task setupDocsDirs(type: Exec, dependsOn: cleanHailDocs) {
    commandLine 'mkdir', '-p', 'build/tmp/docs/', 'build/www/', 'build/tmp/python/'
}

task copyHtml(type: Copy, dependsOn: setupDocsDirs) {
    from 'docs/html'
    into 'build/tmp/docs'
}

task copyJavascript(type: Copy, dependsOn: setupDocsDirs) {
    from 'docs/javascript'
    into 'build/tmp/docs'
}

task copyPDF(type: Copy, dependsOn: setupDocsDirs) {
    from 'docs/LeveneHaldane.pdf'
    into 'build/www'
}

task copyWebsiteContent(type: Copy, dependsOn: setupDocsDirs) {
    from 'www/'
    into 'build/www'
}

task runPandoc(type: Exec, dependsOn: setupDocsDirs) {
    commandLine 'pandoc', 'README.md', '-f', 'markdown', '-t', 'html', '--mathjax',
            '--highlight-style=pygments', '--columns', '10000', '-o', 'build/tmp/docs/README.html'
}

task prepareJavascript(type: Exec, dependsOn: 'copyJavascript') {
    workingDir 'build/tmp/docs'
    commandLine 'npm', 'install'


    standardOutput = new ByteArrayOutputStream()
    errorOutput = standardOutput
    ignoreExitValue = true
    doLast {
        if (execResult.exitValue != 0) {
            println(standardOutput.toString())
            throw new GradleException("exec failed; see output above")
        }
    }
}

task buildIndex(type: Exec, dependsOn: ['runPandoc', 'prepareJavascript', 'copyHtml']) {
    args('template.html', 'README.html')
    executable 'build/tmp/docs/buildIndex.js'
}

task copyPythonDirToTmp(type: Copy, dependsOn: setupDocsDirs) {
    from 'python'
    into 'build/tmp/python'
}

task setupTutorial(type: Exec, dependsOn: copyPythonDirToTmp) {
    workingDir 'build/tmp/python/hail/docs'
    args tutorialFileLoc
    executable 'scripts/setupTutorial.sh'
}

task makeFunctionsRst(type: Exec, dependsOn: ['shadowJar', 'copyPythonDirToTmp']) {
    workingDir 'build/tmp/'
    commandLine 'python', '-c', 'from hail import *; from hail.utils import *; hc = HailContext(); fd = FunctionDocumentation(); fd.types_rst("python/hail/docs/types.rst"); fd.functions_rst("python/hail/docs/functions.rst")'
    environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
    environment SPARK_HOME: sparkHome
    environment SPARK_CLASSPATH: '' + projectDir + '/build/libs/hail-all-spark.jar'
}

task makeHailDocs(type: Exec, dependsOn: ['shadowJar', 'setupTutorial', 'makeFunctionsRst', 'copyPythonDirToTmp']) {
    workingDir 'build/tmp/python/hail/docs'
    commandLine 'make', 'SPHINXOPTS="-tchecktutorial"', 'clean', 'html', 'doctest'
    environment SPARK_HOME: sparkHome
    environment SPARK_CLASSPATH: '' + projectDir + '/build/libs/hail-all-spark.jar'
    environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
}

task makeHailDocsNoTest(type: Exec, dependsOn: ['shadowJar', 'setupTutorial', 'makeFunctionsRst', 'copyPythonDirToTmp']) {
    workingDir 'build/tmp/python/hail/docs'
    commandLine 'make', 'clean', 'html'
    environment SPARK_HOME: sparkHome
    environment SPARK_CLASSPATH: '' + projectDir + '/build/libs/hail-all-spark.jar'
    environment PYTHONPATH: '' + projectDir + '/python:' + sparkHome + '/python:' + sparkHome + '/python/lib/py4j-' + py4jVersion + '-src.zip'
}

task copyHailDocs(type: Exec, dependsOn: ['makeHailDocs', 'setupDocsDirs']) {
    commandLine 'mv', 'build/tmp/python/hail/docs/_build/html', 'build/www/hail'
}

task copyHailDocsNoTest(type: Exec, dependsOn: ['makeHailDocsNoTest', 'setupDocsDirs']) {
    commandLine 'mv', 'build/tmp/python/hail/docs/_build/html', 'build/www/hail'
}

task copyCompiledDocs(type: Copy, dependsOn: ['buildIndex']) {
    from 'build/tmp/docs/index.html'
    into 'build/www'
}

task createWebsite(dependsOn: ['copyPDF', 'copyWebsiteContent', 'copyCompiledDocs', 'copyHailDocs'])

task createDocs(dependsOn: createWebsite)

task createDocsNoTest(dependsOn: ['copyPDF', 'copyWebsiteContent', 'copyCompiledDocs', 'copyHailDocsNoTest', 'copyPythonDirToTmp'])