Added cloud storage services AWS and GCP

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml

@@ -21,6 +21,7 @@ spec:
         workingDir: /opt/flink
         command: ["/opt/flink/bin/standalone-job.sh"]
 {{- $job-config-key := .Release.Name }}
+
         args: ["start-foreground",
                "--job-classname={{ index .Values $job-config-key.job_classname }}", 
                "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
@@ -29,7 +30,20 @@ spec:
                "-Dblob.server.port=6124", 
                "-Dqueryable-state.server.ports=6125",
                "-Djobmanager.heap.size={{ index .Values $job-config-key.job_manager_heap_size }}",
-               "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+               "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+               "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+               "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+               "-Ds3.endpoint={{ .Values.cloud_public_endpoint }}",
+               "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+               "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+               "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+               "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}"
+{{- end }}
                "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6123

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml

@@ -29,7 +29,20 @@ spec:
           "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
           "-Dtaskmanager.rpc.port=6122",
           "-Dtaskmanager.heap.size={{ index .Values $job-config-key.task_manager_heap_size }}",
-          "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+          "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+          "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+          "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+          "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
+          "-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+          "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+          "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+          "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
+{{- end }}
           "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6122

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2

@@ -6,8 +6,12 @@ dockerhub: {{ dockerhub }}
 repository: {{ datapipeline_repository|default('data-pipeline') }}
 image_tag: {{ image_tag }}
 
-azure_storage_account={{ sunbird_private_storage_account_name }}
-azure_storage_secret={{ sunbird_private_storage_account_key }}
+checkpoint_store_type: {{ cloud_service_provider }} 
+cloud_storage_key: {{ cloud_public_storage_accountname }}
+cloud_storage_secret: {{ cloud_public_storage_secret }}
+cloud_storage_endpoint: {{ cloud_public_storage_endpoint }}
+cloud_storage_path_style_access: {{ cloud_storage_pathstyle_access }}
+cloud_storage_project_id: {{ cloud_public_storage_project }}
 
 telemetry-extractor:
   job_name=telemetry-extractor

ansible/roles/analytics-druid/defaults/main.yml

@@ -31,7 +31,7 @@ druid_request_logging_type: "file"
 
 #Druid Extensions
 
-druid_storage_type: "azure"
+druid_storage_type: {{ cloud_storage_telemetry_type }}
 
 druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"'
 
@@ -40,7 +40,7 @@ druid_community_extensions:
 
 # End of druid_extensions
 
-druid_indexing_logs_type: azure
+druid_indexing_logs_type:  {{ cloud_storage_telemetry_type }}
 druid_indexing_log_dir: /var/druid/indexing-logs
 druid_indexing_storage_type : metadata
 druid_indexing_task_basedir : "/var/task"
@@ -126,23 +126,23 @@ default_druid_configs:
    druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
    #Druid Azure Details 
    druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
-   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
-   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
-   azure_container: "{{ druid_azure_container_name }}"
+   azure_account_name: "{{ cloud_public_storage_accountname }}"
+   azure_storage_secret: "{{ cloud_public_storage_secret }}"
+   azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    #Logging the indexing logs to azure
-   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    druid_log_azure_folder: "druidlogs"
    #Druid S3 Details
-   druid_storage_type: "{{ druid_storage_type }}"
-   s3_access_key: "{{ s3_storage_key }}"
-   s3_secret_key: "{{ s3_storage_secret }}"
-   s3_bucket: "{{ s3_storage_container }}"
-   s3_endpoint: "{{ s3_storage_endpoint }}"
+   druid_storage_type: "{{ cloud_storage_telemetry_type }}"
+   s3_access_key: "{{ cloud_public_storage_accountname }}"
+   s3_secret_key: "{{ cloud_public_storage_secret }}"
+   s3_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   s3_endpoint: "{{ cloud_public_storage_endpoint }}"
    s3_segment_dir: "druid/raw/segments"
-   s3_path_like_access: "{{ s3_path_style_access }}"
-   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   s3_path_like_access: "{{ cloud_storage_path_style_access }}"
+   s3_v4_sign_region: "{{ cloud_public_storage_region }}"
    #Logging the indexing logs to s3
-   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
    s3_indexer_logs_dir: "druid/raw/stage/indexing_logs"
    #Druid coordinator node configuration
    druid_coordinator_heap_size: 128m
@@ -200,23 +200,23 @@ default_druid_configs:
    druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
    #Druid Azure Details
    druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
-   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
-   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
-   azure_container: "{{ druid_azure_container_name }}"
+   azure_account_name: "{{ cloud_public_storage_accountname }}"
+   azure_storage_secret: "{{ cloud_public_storage_secret }}"
+   azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    #Logging the indexing logs to azure
-   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    druid_log_azure_folder: "druidlogs"
    #Druid S3 Details
-   druid_storage_type: "{{ druid_storage_type }}"
+   druid_storage_type: "{{ cloud_storage_telemetry_type }}"
    s3_access_key: "{{ s3_storage_key }}"
    s3_secret_key: "{{ s3_storage_secret }}"
    s3_bucket: "{{ s3_storage_container }}"
    s3_endpoint: "{{ s3_storage_endpoint }}"
    s3_segment_dir: "druid/rollup/segments"
-   s3_path_like_access: "{{ s3_path_style_access }}"
-   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   s3_path_like_access: "{{ cloud_storage_path_style_access }}"
+   s3_v4_sign_region: "{{ cloud_public_storage_region }}"
    #Logging the indexing logs to s3
-   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
    s3_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
    #Druid coordinator node configuration
    druid_coordinator_heap_size: 128m

ansible/roles/analytics-spark-provision/templates/spark-env.j2

@@ -70,12 +70,12 @@ export SPARK_WORKER_MEMORY={{ spark.worker.memory }}
 export SPARK_WORKER_INSTANCES={{ spark.worker.instances }}
 export SPARK_EXECUTOR_MEMORY={{ spark.executor.memory }}
 export SPARK_PUBLIC_DNS="{{ spark.public_dns }}"
-export reports_storage_key={{sunbird_private_storage_account_name}}
-export reports_storage_secret={{sunbird_private_storage_account_key}}
-export azure_storage_key={{sunbird_private_storage_account_name}}
-export azure_storage_secret={{sunbird_private_storage_account_key}}
-export druid_storage_account_key={{sunbird_public_storage_account_name}}
-export druid_storage_account_secret={{sunbird_public_storage_account_key}}
-export aws_storage_key={{ s3_storage_key }}
-export aws_storage_secret={{ s3_storage_secret }}
+export reports_storage_key={{cloud_public_storage_accountname}}
+export reports_storage_secret={{cloud_public_storage_secret}}
+export azure_storage_key={{cloud_private_storage_accountname}}
+export azure_storage_secret={{cloud_private_storage_secret}}
+export druid_storage_account_key={{cloud_public_storage_accountname}}
+export druid_storage_account_secret={{cloud_public_storage_secret}}
+export aws_storage_key={{ cloud_public_storage_accountname }}
+export aws_storage_secret={{ cloud_public_storage_secret }}
 

ansible/roles/data-products-deploy/templates/cluster-config.json.j2

@@ -1,12 +1,35 @@
 {
     "jars": [
-        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}",
-        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}",
-        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+    {% if cloud_storage_telemetry_type == "azure" %}
+        "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}",
+        "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}",
+        "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+    {% elif cloud_storage_telemetry_type == "s3" %}
+        "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_core_artifact }}",
+        "s3://{{ bucket }}/models-{{ model_version }}/{{ scruid_artifact }}",
+        "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+    {% elif cloud_storage_telemetry_type == "gcloud" %}
+        "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_core_artifact }}",
+        "gs://{{ bucket }}/models-{{ model_version }}/{{ scruid_artifact }}",
+        "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+    {% endif %}
     ],
-    "file": "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+    "file": 
+    {% if cloud_storage_telemetry_type == "azure" %}
+    "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+    {% elif cloud_storage_telemetry_type == "s3" %}
+    "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+    {% elif cloud_storage_telemetry_type == "gcloud" %}
+    "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+    {% endif %}
     "files": [
-        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/application.conf"
+    {% if cloud_storage_telemetry_type == "azure" %}
+        "wasbs://{{ bucket }}@{{cloud_private_storage_accountname}}.blob.core.windows.net/models-{{ model_version }}/application.conf"
+    {% elif cloud_storage_telemetry_type == "s3" %}
+        "s3://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}/application.conf",
+    {% elif cloud_storage_telemetry_type == "gcloud" %}
+        "gs://{{ bucket }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}/application.conf",
+    {% endif %}
     ],
     "className": "org.ekstep.analytics.job.JobExecutor",
     "executorCores": {{ spark_cluster.executor_core }},
@@ -21,7 +44,7 @@
         "spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}",
         "spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}",
         "spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}",
-        "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}",
-        "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}"
+        "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}",
+        "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}"
     }
 }

ansible/roles/data-products-deploy/templates/common.conf.j2

@@ -15,10 +15,10 @@ storage.secret.config="{{ dp_storage_secret_config }}"
 reports.storage.key.config="{{ dp_reports_storage_key_config }}"
 reports.storage.secret.config="{{ dp_reports_storage_secret_config }}"
 {% if dp_object_store_type == "azure" %}
-cloud_storage_type="azure"
+cloud_storage_telemetry_type="azure"
 {% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %}
-cloud_storage_type="s3"
-cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
+cloud_storage_telemetry_type="s3"
+cloud_public_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
 cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}"
 aws_storage_key="{{ s3_storage_key }}"
 aws_storage_secret="{{ s3_storage_secret }}"
@@ -305,4 +305,4 @@ uci.fushionauth.postgres.pass="{{ uci_postgres.fushionauth_db_psss }}"
 uci.exhaust.store.prefix=""
 uci.encryption.secret="{{ uci_encryption_secret_key }}"
 
-// END OF UCI Related Job Configs
+// END OF UCI Related Job Configs

ansible/roles/lpa-telemetry-backup-deploy/defaults/main.yml

@@ -2,8 +2,8 @@ analytics_user: analytics
 analytics_group: analytics
 analytics_user_home: /home/{{analytics_user}}
 sbin_path: "{{ analytics_user_home }}/sbin"
-azure_container_name: "{{secor_azure_container_name}}"
-azure_account_key: "{{sunbird_private_storage_account_key}}"
+azure_container_name: "{{cloud_storage_telemetry_bucketname}}"
+azure_account_key: "{{cloud_private_storage_secret}}"
 
 telemetry_ingestion_topic: "{{ env }}.telemetry.ingest"