Merge pull request #1097 from RevathiKotla/release-3.3.1

Fix Cluster partition submit script

Author: SanthoshVasabhaktula

ansible/roles/spark-cluster-data-products-submit/defaults/main.yml

@@ -26,9 +26,9 @@ spark_cluster:
   driver_memory: 7g
   memory_fraction: 0.3
   storage_fraction: 0.5
-  executor_core: 3
-  executor_memory: 12G
-  num_executors: 5
+  executor_core: 5
+  executor_memory: 19G
+  num_executors: 23
 
 analytics:
   home: "/tmp"
@@ -39,4 +39,4 @@ sink_topic: "{{ env }}.telemetry.sink"
 druid_broker_host: "{{groups['raw-broker'][0]}}"
 producer_env: "dev.sunbird"
 spark_output_temp_dir: /mount/data/analytics/tmp/
-reports_container: "reports"
+reports_container: "reports"

ansible/roles/spark-cluster-data-products-submit/templates/cluster-config.json.j2

@@ -18,7 +18,8 @@
         "spark.sql.autoBroadcastJoinThreshold" : "-1",
         "spark.dynamicAllocation.enabled" :"true", 
         "spark.shuffle.service.enabled" :"true",
+        "spark.scheduler.mode" : "FAIR",
         "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{dp_azure_account_name}} -Dreports_storage_secret={{dp_vault_azure_account_key}}",
         "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{dp_azure_account_name}} -Dreports_storage_secret={{dp_vault_azure_account_key}}"
     }
-}
+}

ansible/roles/spark-cluster-data-products-submit/templates/submit-script.j2

@@ -67,17 +67,17 @@ if [ "$mode" = "via-partition" ]; then
     for i in $(seq 0 $parallelisation $endPartitions)
         do 
             # add partitions to config
-            partitionString="\"delta\":0,\"partitions\":[$(seq -s , $i `expr $i + $parallelisation - 1`)]"
+            partitionString="\\\"delta\\\":0,\\\"partitions\\\":[$(seq -s , $i `expr $i + $parallelisation - 1`)]"
             if [ -z "$start_date" ]; then
                 job_config=$(config $job)
-                finalConfig=${job_config/'"delta":0'/$partitionString}
+                finalConfig=${job_config/'\"delta\":0'/$partitionString}
                 echo $finalConfig
                 echo "Running $job by partitions."
                 classVariable="org.ekstep.analytics.job.JobExecutor"
                 argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]"
             else 
                 job_config=$(config $job '__endDate__')
-                finalConfig=${job_config/'"delta":0'/$partitionString}
+                finalConfig=${job_config/'\"delta\":0'/$partitionString}
                 echo $finalConfig
                 echo "Running $job by partitions via Replay-Supervisor."
                 classVariable="org.ekstep.analytics.job.ReplaySupervisor"
@@ -109,4 +109,4 @@ else
     finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
     echo $finalRequestBody
     curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"    
-fi
+fi