Collection Summary Configuration and Ingestion Spec

ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json

@@ -0,0 +1,200 @@
+{
+  "type": "index",
+  "spec": {
+    "dataSchema": {
+      "dataSource": "collection-summary",
+      "parser": {
+        "type": "string",
+        "parseSpec": {
+          "format": "json",
+          "flattenSpec": {
+            "useFieldDiscovery": false,
+            "fields": [
+              {
+                "type": "root",
+                "name": "content_org",
+                "expr": "contentorg"
+              },
+              {
+                "type": "root",
+                "name": "user_org",
+                "expr": "orgname"
+              },
+              {
+                "type": "root",
+                "name": "batch_start_date",
+                "expr": "startdate"
+              },
+              {
+                "type": "root",
+                "name": "batch_end_date",
+                "expr": "enddate"
+              },
+              {
+                "type": "root",
+                "name": "has_certificate",
+                "expr": "hascertified"
+              },
+              {
+                "type": "root",
+                "name": "collection_id",
+                "expr": "courseid"
+              },
+              {
+                "type": "root",
+                "name": "batch_id",
+                "expr": "batchid"
+              },
+              {
+                "type": "root",
+                "name": "collection_name",
+                "expr": "collectionname"
+              },
+              {
+                "type": "root",
+                "name": "total_enrolment",
+                "expr": "enrolleduserscount"
+              },
+              {
+                "type": "root",
+                "name": "total_completion",
+                "expr": "completionuserscount"
+              },
+              {
+                "type": "root",
+                "name": "total_certificates_issued",
+                "expr": "certificateissuedcount"
+              },
+              {
+                "type": "root",
+                "name": "content_status",
+                "expr": "contentstatus"
+              },
+              {
+                "type": "root",
+                "name": "user_state",
+                "expr": "state"
+              },
+              {
+                "type": "root",
+                "name": "user_district",
+                "expr": "district"
+              },
+              {
+                "type": "root",
+                "name": "content_channel",
+                "expr": "channel"
+              },
+              {
+                "type": "root",
+                "name": "keywords",
+                "expr": "keywords"
+              },
+              {
+                "type": "root",
+                "name": "timestamp",
+                "expr": "timestamp"
+              }
+            ]
+          },
+          "dimensionsSpec": {
+            "dimensions": [
+              {
+                "name": "content_org"
+              },
+              {
+                "name": "user_org"
+              },
+              {
+                "type": "string",
+                "name": "batch_id"
+              },
+              {
+                "type": "string",
+                "name": "batch_start_date"
+              },
+              {
+                "type": "string",
+                "name": "batch_end_date"
+              },
+              {
+                "type": "string",
+                "name": "collection_id"
+              },
+              {
+                "type": "string",
+                "name": "collection_name"
+              },
+              {
+                "type": "long",
+                "name": "total_enrolment"
+              },
+              {
+                "type": "long",
+                "name": "total_completion"
+              },
+              {
+                "type": "long",
+                "name": "total_certificates_issued"
+              },
+              {
+                "type": "string",
+                "name": "content_status"
+              },
+              {
+                "type": "string",
+                "name": "user_state"
+              },
+              {
+                "type": "string",
+                "name": "user_district"
+              },
+              {
+                "name": "keywords"
+              },
+              {
+                "name": "has_certificate"
+              },
+              {
+                "type": "string",
+                "name": "content_channel"
+              }
+            ],
+            "dimensionsExclusions": []
+          },
+          "timestampSpec": {
+            "column": "timestamp",
+            "format": "auto"
+          }
+        }
+      },
+      "metricsSpec": [],
+      "granularitySpec": {
+        "type": "uniform",
+        "segmentGranularity": "day",
+        "queryGranularity": "none",
+        "rollup": true
+      }
+    },
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "reports",
+            "path": "/collection-summary-reports-v2/collection-summary-report-latest.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "targetPartitionSize": 5000000,
+      "maxRowsInMemory": 25000,
+      "forceExtendableShardSpecs": false,
+      "logParseExceptions": true
+    }
+  }
+}

ansible/roles/data-products-deploy/tasks/main.yml

@@ -212,7 +212,6 @@
     - cronjobs 
     - default-jobs
     - spark-jobs
-
 - name: Create admin-geo-reports cron job
   cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh admin-geo-reports"
   with_dict: "{{ run_admin_geo_reports_job }}"
@@ -285,7 +284,10 @@
     - default-jobs
     - spark-jobs
 
-
+- name: Copy collection-summary ingestion spec
+  copy: src="collection-summary-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - ed-dataproducts
 
 - name: Update start jobmanager
   template: src=start-jobmanager.j2 dest={{ analytics.home }}/scripts/start-jobmanager.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}

ansible/roles/data-products-deploy/templates/model-config.j2

@@ -28,6 +28,7 @@ config() {
 	producerEnv="{{ producer_env }}"
 	baseScriptPath="{{ spark_output_temp_dir }}"
 	reportPostContainer="{{ reports_container }}"
+	druidIngestionURL="http://{{groups['raw-overlord'][0]}}:8081/druid/indexer/v1/task"
 
 	if [ -z "$2" ]; then endDate=$(date --date yesterday "+%Y-%m-%d"); else endDate=$2; fi
 	if [ ! -z "$3" ]; then inputBucket=$3; fi
@@ -41,6 +42,9 @@ config() {
         "collection-summary-report")
 		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"], "contentType": "Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
 		;;
+		"collection-summary-report-v2")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live","Unlisted","Retired"],"contentType":"Course"},"fields":["identifier","name","organisation","channel","status","keywords"],"limit":10000}},"store":"azure","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}'
+		;;
         "userinfo-exhaust")
 		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}'
 		;;

ansible/roles/data-products-deploy/templates/run-job.j2

@@ -42,6 +42,8 @@ get_report_job_model_name(){
 		;;
 		"program-collection-summary-report") echo 'org.sunbird.analytics.job.report.CollectionSummaryJob'
 		;;
+		"collection-summary-report-v2") echo 'org.sunbird.analytics.job.report.CollectionSummaryJobV2'
+		;;
 		*) echo $1
 		;;
 	esac