Add examples to demonstrate how to construct and execute CoRB jobs

.gitignore

@@ -17,3 +17,5 @@ nb-configuration.xml
 /dependency-reduced-pom.xml
 /cov-int/
 src/test/resources/loader.zip
+examples/corb.bat
+examples/corb.sh

examples/PostBatchUpdateFileTask-dedup/job.properties

@@ -0,0 +1,14 @@
+## A job to generate a report of all the distinct element QNames, in Clark-notation, from documents in the database.
+#
+# Inline module to select all URIs
+URIS-MODULE=INLINE-XQUERY|xdmp:estimate(fn:doc()), cts:uris("",(),cts:true-query())
+# Inline module to return a distinct list of all elements in the document on a separate line
+PROCESS-MODULE=INLINE-XQUERY|declare variable $URI external; string-join(fn:distinct-values(fn:doc($URI)//*/xdmp:key-from-QName(fn:node-name(.))),"
")
+# Write the results of each process module to a single file
+PROCESS-TASK=com.marklogic.developer.corb.ExportBatchToFileTask
+EXPORT-FILE-NAME=element-names.txt
+EXPORT-DIR=build
+# After the batch processing is completed, sort and dedup the element names
+POST-BATCH-TASK=com.marklogic.developer.corb.PostBatchUpdateFileTask
+EXPORT-FILE-SORT=ascending|distinct
+THREAD-COUNT=10

examples/README.md

@@ -0,0 +1,27 @@
+# Examples
+
+##Setup
+In order to run any of the example jobs locally, run:
+
+    ./gradlew setup
+
+This will invoke ml-gradle task `mlDeploy` to setup a content and modules database, as well as an application server.
+
+For convenience, the `setup` task will also execute the `createCorbShell` and `createCorbBatch` tasks to generate a `corb.sh` shell script and `corb.bat` Windows batch file. It sets the classpath and the XCC connection string dusing values specified in the gradle.properties.
+
+##Running Example Jobs
+
+Specify the **OPTIONS-FILE** for one of the example jobs 
+and any other CoRB options that you would like to apply or override from the example jobs.
+
+When executing the `corb` task, set the options file as a project property with the `-P` switch:
+
+    ./gradlew corb -PcorbOptionsFile=inline-adhoc/job.properties
+
+You could also choose to use the generated corb.sh and set **OPTIONS-FILE** as a system property with the `-D` switch:
+
+    ./corb.sh -DOPTIONS-FILE=inline-adhoc/job.properties 
+
+or the corb.bat file:
+
+    corb -DOPTIONS-FILE=inline-adhoc/job.properties 

examples/build.gradle

@@ -0,0 +1,125 @@
+/*
+ * This build file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a commented-out sample Java project to get you started.
+ * For more details take a look at the Java Quickstart chapter in the Gradle
+ * user guide available at https://docs.gradle.org/3.5/userguide/tutorial_java_projects.html
+ */
+
+buildscript {
+    repositories {
+        mavenLocal()
+        jcenter()
+    }
+    dependencies {
+        //Needed for CorbTask to dynamically generate properties from CoRB Options class
+        classpath "com.marklogic:marklogic-corb:${corbVersion}"
+        classpath "com.marklogic:ml-gradle:${mlGradleVersion}"
+    }
+}
+
+plugins {id 'com.marklogic.ml-gradle' version "${mlGradleVersion}"}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    // This configuration captures the dependencies for running corb (Content Reprocessing in Bulk).
+    // This is only needed if you want to run corb via Gradle tasks.
+    // If you do, using com.marklogic.gradle.task.CorbTask is a useful starting point, as shown below.
+    corb
+}
+
+dependencies {
+    // required to run CoRB2
+    corb "com.marklogic:marklogic-corb:${corbVersion}"
+    // optional
+    corb 'org.jasypt:jasypt:1.9.2' // necessary to leverage JasyptDecrypter
+}
+
+ext {
+    // XCC URL for running corb task below
+    contentXccUrl = "${xccProtocol}://${mlUsername}:${mlPassword}@${mlHost}:${mlRestPort}"
+}
+
+/*
+ * This is an example of both a custom Gradle task and an easy way of invoking Corb.
+ *
+ * Since a custom configuration named "corb" is defined above, the classpath is automatically set.
+ *
+ * Any of the CoRB2 options can be set for the task several ways:
+ * 1.) Properties of the task with a lowerCamelCased naming convention
+ *     (i.e. the CoRB2 option THREAD-COUNT is threadCount)
+ * 2.) Project properties with the naming convention of a "corb" prefix and the UpperCamelCased
+ *     option name (i.e. the CoRB2 option THREAD-COUNT is corbThreadCount and can
+ *     be specified on the commandline as -PcorbThreadCount=12)
+ * 3.) System properties with the CoRB2 option name. (i.e. THREAD-COUNT can be set on the commandline as
+ *       -DTHREAD-COUNT=16)
+ *
+ * A complete list of CoRB2 options can be found at: https://github.com/marklogic/corb2#options
+ *
+ * Examples of how to invoke corb and set options:
+ * 1.) specify options from a properties file using the CoRB2 OPTIONS-FILE option
+ *     gradle corb -PcorbOptionsFile=src/main/ml-modules/ext/corb2-project/corb/options.properties
+ * 2.) Set corb project properties on the commandline
+ *     gradle corb -PcorbUrisModule="src/main/ml-modules/ext/corb2-project/uris.xqy|ADHOC" -PcorbProcessModule="src/main/ml-modules/ext/corb2-project/corb/transform.xqy|ADHOC"
+ * 3.) Set CoRB2 options as System properties on the commandline
+ *     gradle corb -DURIS-MODULE="src/main/ml-modules/ext/corb2-project/uris.xqy|ADHOC" -DPROCESS-MODULE="src/main/ml-modules/ext/corb2-project/corb/transform.xqy|ADHOC"
+ */
+task corb(type: com.marklogic.gradle.task.CorbTask) {
+    /* Either uncomment and set the xccConnectionUri below and set an appropriate XCC connection string,
+     * or specify on the commandline as a project property: -PcorbXccConnectionUri=xcc://user:pass@host:port/content-database
+     * or specify as a System property: -DXCC-CONNECTION-URI=xcc://user:pass@host:port/content-database
+     * or instread of setting the XCC-CONNECTION-URI, set the individual XCC options
+     * (XCC-HOSTNAME, XCC-PORT, XCC-USERNAME, XCC-PASSWORD, XCC-DBNAME)
+     */
+    xccConnectionUri = contentXccUrl
+}
+
+task copyLibs(type: Copy) {
+    into 'build/lib'
+    from configurations.corb
+}
+
+task createCorbShell() {
+    dependsOn copyLibs
+    doLast {
+        File shell = new File('corb.sh')
+        shell.withWriter('UTF-8') {
+            it.writeLine("#! /bin/sh")
+            it.writeLine('java -cp "build/lib/*" $@ com.marklogic.developer.corb.Manager ' + "${contentXccUrl}")
+        }
+        ant.chmod(file: shell.path, perm: "+x")
+    }
+}
+
+task createCorbBatch() {
+    dependsOn copyLibs
+    doLast {
+        new File('corb.bat').withWriter('UTF-8') {
+            it.writeLine('java -cp "build/lib/*" %* com.marklogic.developer.corb.Manager ' + "${contentXccUrl}")
+            it.writeLine('pause')
+        }
+    }
+}
+
+task setup() {
+    dependsOn 'mlDeploy'
+    dependsOn 'createCorbShell'
+    dependsOn 'createCorbBatch'
+}
+
+task clean() {
+    doLast() {
+        delete 'build'
+        delete 'corb.bat'
+        delete 'corb.sh'
+    }
+}
+
+task teardown() {
+    dependsOn 'clean'
+    dependsOn 'mlUndeploy'
+}

examples/feature-demo/job.properties

@@ -0,0 +1,31 @@
+THREAD-COUNT=8
+# you can include the code inline in the properties file, rather that creating a module file or loading into the modules DB
+URIS-MODULE=INLINE-XQUERY|let $uris := (1 to 500000) return (count($uris), $uris)
+# or read a module file from the filesystem and run adhoc without installing in the modules DB
+PROCESS-MODULE=../src/test/resources/transformSlow.xqy|ADHOC
+# use a command file
+COMMAND-FILE=feature-demo/job.properties
+#Uncomment the line below and save while the job is running, the job will pause. Change back to resume and save and the job will continue.
+#COMMAND=pause
+# Default location to store temporary files required for the disk queue, sorting and dedup, XML splitting, etc
+TEMP-DIR=build
+DISK-QUEUE=true
+
+# Override the TEMP-DIR location with a different path to be used for the DISK-QUEUE
+#DISK-QUEUE-TEMP-DIR=
+
+ERROR-FILE-NAME=build/errors/feature-demo.txt
+FAIL-ON-ERROR=false
+
+# Enable the UI and webserver to view job stats, and ability to pause/resume and change thread counts.
+#Provide a range of ports to try to obtain and use for the Job UI and metrics service
+JOB-SERVER-PORT=8000-9000
+JOB-NAME=feature demo
+
+# Save job metrics for trends and analysis
+METRICS-COLLECTIONS=corb-example
+METRICS-LOG-LEVEL=info
+METRICS-NUM-FAILED-TRANSACTIONS=10
+METRICS-NUM-SLOW-TRANSACTIONS=20
+METRICS-DATABASE=corb2-examples-content
+METRICS-SYNC-FREQUENCY=60

examples/gradle.properties

@@ -0,0 +1,10 @@
+mlHost=localhost
+mlAppName=corb2-examples
+mlRestPort=8227
+mlUsername=admin
+mlPassword=admin
+xccProtocol=xcc
+# once ml-gradle applies a fix to handle non-public properties, we can upgrade to the latest CoRB version
+mlGradleVersion=4.1.1
+corbVersion=2.4.6
+#corbVersion=2.5.1

examples/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed Oct 21 23:54:49 EDT 2020
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip