PIP-93 xAPI 2.0.0 Support

.github/workflows/ci.yml

@@ -31,17 +31,9 @@ jobs:
       - name: Test Xapipe
         run: make test
 
-  nvd_scan:
-    uses: yetanalytics/actions/.github/workflows/nvd-scan.yml@v0.0.4
-    with:
-      nvd-clojure-version: '3.6.0'
-      classpath-command: 'clojure -Spath -A:cli'
-      nvd-config-filename: '.nvd/config.json'
-
   docker:
     needs:
       - test
-      - nvd_scan
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

README.md

@@ -49,6 +49,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti
 - [Docker Container](doc/docker.md)
 - [Demo](doc/demo.md)
 - [Sample AWS Deployment](doc/aws.md)
+- [xAPI Version Support](doc/xapi_version.md)
 
 ## Contribution
 

deps.edn

@@ -6,7 +6,7 @@
         commons-fileupload/commons-fileupload {:mvn/version "1.6.0"}
         commons-io/commons-io {:mvn/version "2.14.0"}
         com.yetanalytics/xapi-schema
-        {:mvn/version "1.2.0"
+        {:mvn/version "1.4.1"
          :exclusions [org.clojure/clojure
                       org.clojure/clojurescript
                       org.clojure/data.json]}
@@ -39,10 +39,10 @@
           io.github.cognitect-labs/test-runner
           {:git/url "https://github.com/cognitect-labs/test-runner"
            :sha "dd6da11611eeb87f08780a30ac8ea6012d4c05ce"}
-          com.yetanalytics/lrs {:mvn/version "1.2.11"}
+          com.yetanalytics/lrs {:mvn/version "1.4.1"}
           io.pedestal/pedestal.jetty {:mvn/version "0.5.9"}
           ;; Some integration tests use logback
-          ch.qos.logback/logback-classic {:mvn/version "1.5.15"
+          ch.qos.logback/logback-classic {:mvn/version "1.5.19"
                                           :exclusions [org.slf4j/slf4j-api]}
           org.slf4j/slf4j-api {:mvn/version "2.0.12"}
           org.slf4j/jul-to-slf4j {:mvn/version "2.0.12"}

dev-resources/jobs/simple-v0.json

@@ -0,0 +1,48 @@
+{
+  "id": "foo",
+  "config": {
+    "get-buffer-size": 10,
+    "statement-buffer-size": 500,
+    "batch-buffer-size": 10,
+    "batch-timeout": 200,
+    "source": {
+      "request-config": {
+        "url-base": "http://0.0.0.0:8080",
+        "xapi-prefix": "/xapi"
+      },
+      "batch-size": 50,
+      "backoff-opts": {
+        "budget": 10000,
+        "max-attempt": 10
+      },
+      "poll-interval": 1000,
+      "get-params": {
+        "limit": 50
+      }
+    },
+    "target": {
+      "request-config": {
+        "url-base": "http://0.0.0.0:8081",
+        "xapi-prefix": "/xapi"
+      },
+      "batch-size": 50,
+      "backoff-opts": {
+        "budget": 10000,
+        "max-attempt": 10
+      }
+    },
+    "filter": {}
+  },
+  "state": {
+    "status": "init",
+    "cursor": "1970-01-01T00:00:00.000000000Z",
+    "source": {
+      "errors": []
+    },
+    "target": {
+      "errors": []
+    },
+    "errors": [],
+    "filter": {}
+  }
+}

dev-resources/jobs/simple.json

@@ -1 +1,51 @@
-{"id":"foo","config":{"get-buffer-size":10,"statement-buffer-size":500,"batch-buffer-size":10,"batch-timeout":200,"source":{"request-config":{"url-base":"http://0.0.0.0:8080","xapi-prefix":"/xapi"},"batch-size":50,"backoff-opts":{"budget":10000,"max-attempt":10},"poll-interval":1000,"get-params":{"limit":50}},"target":{"request-config":{"url-base":"http://0.0.0.0:8081","xapi-prefix":"/xapi"},"batch-size":50,"backoff-opts":{"budget":10000,"max-attempt":10}},"filter":{}},"state":{"status":"init","cursor":"1970-01-01T00:00:00.000000000Z","source":{"errors":[]},"target":{"errors":[]},"errors":[],"filter":{}}}
+{
+  "id": "foo",
+  "version": 1,
+  "config": {
+    "get-buffer-size": 10,
+    "statement-buffer-size": 500,
+    "batch-buffer-size": 10,
+    "batch-timeout": 200,
+    "source": {
+      "request-config": {
+        "url-base": "http://0.0.0.0:8080",
+        "xapi-prefix": "/xapi",
+        "xapi-version": "1.0.3"
+      },
+      "batch-size": 50,
+      "backoff-opts": {
+        "budget": 10000,
+        "max-attempt": 10
+      },
+      "poll-interval": 1000,
+      "get-params": {
+        "limit": 50
+      }
+    },
+    "target": {
+      "request-config": {
+        "url-base": "http://0.0.0.0:8081",
+        "xapi-prefix": "/xapi",
+        "xapi-version": "1.0.3"
+      },
+      "batch-size": 50,
+      "backoff-opts": {
+        "budget": 10000,
+        "max-attempt": 10
+      }
+    },
+    "filter": {}
+  },
+  "state": {
+    "status": "init",
+    "cursor": "1970-01-01T00:00:00.000000000Z",
+    "source": {
+      "errors": []
+    },
+    "target": {
+      "errors": []
+    },
+    "errors": [],
+    "filter": {}
+  }
+}

doc/index.md

@@ -16,3 +16,4 @@ This process is one-way and any statements in the target LRS will not be replica
 - [Docker Container](docker.md)
 - [Demo](demo.md)
 - [Sample AWS Deployment](aws.md)
+- [xAPI Version Support](xapi_version.md)

doc/options.md

@@ -36,6 +36,7 @@ All options:
       --json JSON                                             Take a job specification as a JSON string
       --json-file FILE                                        Take a job specification from a JSON file
       --json-out FILE                                         Write JOB to a JSON file
+      --[no-]upgrade                                          Automatically upgrade jobs to the latest version (defaults to true)
   -s, --storage STORAGE                 :file                 Select storage backend, file (default), redis or noop, mem is for testing only
       --redis-uri URI                   redis://0.0.0.0:6379  Redis Connection URI
       --redis-prefix PREFIX             xapipe                Redis key prefix
@@ -48,6 +49,7 @@ All options:
   -p, --xapi-get-param KEY=VALUE        {}                    xAPI GET Parameters
       --source-username USERNAME                              Source LRS BASIC Auth username
       --source-password PASSWORD                              Source LRS BASIC Auth password
+      --source-xapi-version VERSION     1.0.3                 Source LRS xAPI Version
       --json-only                                             Only operate in JSON statement mode for data transfer, ignoring Attachments/multipart (for compatibility issues)
       --source-auth-uri URI                                   Source LRS OAuth autentication URI
       --source-client-id ID                                   Source LRS OAuth client ID
@@ -62,6 +64,7 @@ All options:
       --target-batch-size SIZE          50                    Target LRS POST desired batch size
       --target-username USERNAME                              Target LRS BASIC Auth username
       --target-password PASSWORD                              Target LRS BASIC Auth password
+      --target-xapi-version VERSION     1.0.3                 Target LRS xAPI Version
       --target-auth-uri URI                                   Target LRS OAuth autentication URI
       --target-client-id ID                                   Target LRS OAuth client ID
       --target-client-secret SECRET                           Target LRS OAuth client secret

doc/xapi_version.md

@@ -0,0 +1,14 @@
+[<- Back to Index](index.md)
+
+# xAPI Version Support
+
+As of v0.0.34 LRSPipe supports xAPI versions 1.0.3 and 2.0.0 using the `--source-xapi-version` and `--target-xapi-version` arguments. To pass validation checks LRSPipe must sometimes downgrade statement data to move it between LRS instances. Refer to the table below for more information:
+
+| version      | 1.0.3 target                  | 2.0.0 target |
+|--------------|-------------------------------|--------------|
+| 1.0.3 source | 2.0.0 data will be downgraded | no change    |
+| 2.0.0 source | 2.0.0 data will be downgraded | no change    |
+
+Note that even when using version 1.0.3 for both source and target (the default) downgrading may occur if 2.0.0 data is present on the 1.0.3 LRS. To ensure 2.0.0 data like `$.context.contextAgents` is always preserved use the `--target-xapi-version 2.0.0` argument.
+
+[<- Back to Index](index.md)

src/cli/com/yetanalytics/xapipe/cli.clj

@@ -211,6 +211,7 @@
    :source-backoff-max-attempt [:source :backoff-opts :max-attempt]
    :source-backoff-j-range     [:source :backoff-opts :j-range]
    :source-backoff-initial     [:source :backoff-opts :initial]
+   :source-xapi-version        [:source :request-config :xapi-version]
 
    ;; Target LRS
    :target-batch-size          [:target :batch-size]
@@ -225,6 +226,7 @@
    :target-backoff-max-attempt [:target :backoff-opts :max-attempt]
    :target-backoff-j-range     [:target :backoff-opts :j-range]
    :target-backoff-initial     [:target :backoff-opts :initial]
+   :target-xapi-version        [:target :request-config :xapi-version]
 
    ;; Filters
    :filter-template-profile-urls  [:filter :template :profile-urls]

src/cli/com/yetanalytics/xapipe/cli/options.clj

@@ -156,7 +156,10 @@
                  (-> filepath
                      slurp
                      job-json/json->job))]
-    [nil "--json-out FILE" "Write JOB to a JSON file"]]
+    [nil "--json-out FILE" "Write JOB to a JSON file"]
+    [nil "--[no-]upgrade" "Automatically upgrade jobs to the latest version (defaults to true)"
+     :id :upgrade-jobs
+     :default true]]
    (concat storage-options
            metrics-options)))
 
@@ -250,6 +253,9 @@
            m)))]
     [nil "--source-username USERNAME" "Source LRS BASIC Auth username"]
     [nil "--source-password PASSWORD" "Source LRS BASIC Auth password"]
+    [nil "--source-xapi-version VERSION" "Source LRS xAPI Version"
+     :validate [#{"1.0.3" "2.0.0"} "Must be 1.0.3 or 2.0.0"]
+     :default "1.0.3"]
     [nil "--json-only" "Only operate in JSON statement mode for data transfer, ignoring Attachments/multipart (for compatibility issues)"
      :default false]]
    (concat
@@ -266,7 +272,10 @@
           :validate [pos-int? "Must be a positive integer"]
           :default 50]
          [nil "--target-username USERNAME" "Target LRS BASIC Auth username"]
-         [nil "--target-password PASSWORD" "Target LRS BASIC Auth password"]]
+         [nil "--target-password PASSWORD" "Target LRS BASIC Auth password"]
+         [nil "--target-xapi-version VERSION" "Target LRS xAPI Version"
+          :validate [#{"1.0.3" "2.0.0"} "Must be 1.0.3 or 2.0.0"]
+          :default "1.0.3"]]
         (concat
          (oauth-opts "target")
          (backoff-opts "target"))))

src/cli/com/yetanalytics/xapipe/main.clj

@@ -40,6 +40,7 @@ Delete a Job:
             ?json          :json
             ?json-file     :json-file
             ?json-out      :json-out
+            upgrade-jobs?  :upgrade-jobs
             :as            options} :options
            :keys         [summary]} (opts/args->options args)]
       (if help?
@@ -83,10 +84,11 @@ Delete a Job:
                     ;; Found in storage
                     ?from-storage
                     (-> ?from-storage
-                        job/upgrade-job
                         (cond->
+                          ;; upgrade retrieved job if desired
+                          upgrade-jobs? job/upgrade-job
                           ;; If the user has requested force resume we clear
-                         force-resume?
+                          force-resume?
                           (-> (update :state state/clear-errors)
                               (update :state state/set-status :paused)))
                         (job/reconfigure-job
@@ -99,7 +101,8 @@ Delete a Job:
                     ;; Json is provided
                     ?from-json
                     (-> ?from-json
-                        job/upgrade-job
+                        (cond->
+                          upgrade-jobs? job/upgrade-job)
                         (update :config cli/reconfigure-with-options
                                 ;; reparse args w/o defaults
                                 (:options