From 5117061ccf24eb696b65ec31485a0b8ad2f6cde9 Mon Sep 17 00:00:00 2001
From: Manjunath Suryanarayana <manjunath15@gmail.com>
Date: Tue, 30 May 2023 19:36:28 +0530
Subject: [PATCH] Adding pipeline: snowflake_datacopy Adding dataflow: 1_
 CALL_CENTER Adding database: Sales

---
 database/Sales.json              | 22 +++++++++++++
 dataflow/1_ CALL_CENTER.json     | 53 ++++++++++++++++++++++++++++++++
 pipeline/snowflake_datacopy.json | 32 +++++++++++++++++++
 3 files changed, 107 insertions(+)
 create mode 100644 database/Sales.json
 create mode 100644 dataflow/1_ CALL_CENTER.json
 create mode 100644 pipeline/snowflake_datacopy.json

diff --git a/database/Sales.json b/database/Sales.json
new file mode 100644
index 0000000..ce95e8e
--- /dev/null
+++ b/database/Sales.json
@@ -0,0 +1,22 @@
+{
+	"name": "Sales",
+	"properties": {
+		"Name": "Sales",
+		"EntityType": "DATABASE",
+		"Origin": {
+			"Type": "SPARK"
+		},
+		"Properties": {
+			"IsSyMSCDMDatabase": true
+		},
+		"Source": {
+			"Provider": "ADLS",
+			"Location": "abfss://lakeshouse@mjlakeshousestroage.dfs.core.windows.net/Sales",
+			"Properties": {
+				"FormatType": "parquet",
+				"LinkedServiceName": "mj-synpase-lakehouse-WorkspaceDefaultStorage"
+			}
+		}
+	},
+	"type": "DATABASE"
+}
\ No newline at end of file
diff --git a/dataflow/1_ CALL_CENTER.json b/dataflow/1_ CALL_CENTER.json
new file mode 100644
index 0000000..3a1c3ca
--- /dev/null
+++ b/dataflow/1_ CALL_CENTER.json	
@@ -0,0 +1,53 @@
+{
+	"name": "1_ CALL_CENTER",
+	"properties": {
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "SnowflakeDataSetCallCenter",
+						"type": "DatasetReference"
+					},
+					"name": "snowflakecallcenter",
+					"description": "\nCALL_CENTER table from snowflake"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "mj-synpase-lakehouse-WorkspaceDefaultStorage",
+						"type": "LinkedServiceReference"
+					},
+					"name": "callcentertable"
+				}
+			],
+			"transformations": [],
+			"scriptLines": [
+				"source(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     limit: 100,",
+				"     query: 'select * from SNOWFLAKE_SAMPLE_DATA.TPCDS_SF100TCL.CALL_CENTER',",
+				"     format: 'query') ~> snowflakecallcenter",
+				"snowflakecallcenter sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'delta',",
+				"     fileSystem: 'lakeshouse',",
+				"     folderPath: 'synapse/workspaces/mj-synpase-lakehouse/Bronze/CallCenter',",
+				"     mergeSchema: false,",
+				"     autoCompact: false,",
+				"     optimizedWrite: false,",
+				"     vacuum: 0,",
+				"     deletable:false,",
+				"     insertable:true,",
+				"     updateable:false,",
+				"     upsertable:false,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> callcentertable"
+			]
+		}
+	}
+}
\ No newline at end of file
diff --git a/pipeline/snowflake_datacopy.json b/pipeline/snowflake_datacopy.json
new file mode 100644
index 0000000..7f52fb7
--- /dev/null
+++ b/pipeline/snowflake_datacopy.json
@@ -0,0 +1,32 @@
+{
+	"name": "snowflake_datacopy",
+	"properties": {
+		"activities": [
+			{
+				"name": "copy_ CALL_CENTER",
+				"type": "ExecuteDataFlow",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"dataflow": {
+						"referenceName": "1_ CALL_CENTER",
+						"type": "DataFlowReference"
+					},
+					"integrationRuntime": {
+						"referenceName": "AutoResolveIntegrationRuntime",
+						"type": "IntegrationRuntimeReference"
+					},
+					"traceLevel": "Fine"
+				}
+			}
+		],
+		"annotations": []
+	}
+}
\ No newline at end of file