Merge pull request #14 from sfc-gh-vraudszus/main

Cover new CoAs, PythonAPI and CLI features

Author: sfc-gh-vraudszus

.devcontainer/devcontainer.json

@@ -1,12 +1,24 @@
-// For format details, see https://aka.ms/devcontainer.json.
 {
-    "name": "Snowflake Demo Codespace",
-    // Configure tool-specific properties.
-    "customizations": {
-      "vscode": {
-        "extensions": [
-          "snowflake.snowflake-vsc"
-        ]
+  "name": "Snowflake Demo Codespace",
+  "features": {
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.11"
+    }
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "snowflake.snowflake-vsc"
+      ],
+      "settings": {
+        "snowflake.snowsqlConfigPath": "${containerWorkspaceFolder}/.snowflake/config.toml"
       }
     }
-  }
+  },
+  "updateContentCommand": "bash .devcontainer/install-dependencies.sh",
+  "postCreateCommand": "chmod 0600 \"${containerWorkspaceFolder}/.snowflake/config.toml\"",
+  "containerEnv": {
+    "SNOWFLAKE_HOME": "${containerWorkspaceFolder}/.snowflake"
+  }
+}

.devcontainer/install-dependencies.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+pip install snowflake-snowpark-python
+pip install snowflake.core
+pipx install snowflake-cli --python 3.11

.github/workflows/deploy_pipeline.yml

@@ -27,10 +27,11 @@ jobs:
         uses: actions/checkout@v4
 
       # Install Snowflake CLI GitHub Action and point to config file
-      - uses: Snowflake-Labs/snowflake-cli-action@v1
+      - name: Install snowflake-cli
+        uses: Snowflake-Labs/snowflake-cli-action@v1.5
         with:
           cli-version: "latest"
-          default-config-file-path: "config.toml"
+          default-config-file-path: ".snowflake/config.toml"
 
       # Update Snowflake's copy of the repository
       - name: Fetch repository changes
@@ -42,10 +43,10 @@ jobs:
           BRANCH_NAME=${{ github.ref_name }}
           if [ "${BRANCH_NAME}" == "main" ]; then
             RETENTION_TIME=1
-          elif [ "${BRANCH_NAME}" == "dev" ]; then
+          else
             RETENTION_TIME=0
           fi
           snow git execute \
-            "@${REPO_NAME}/branches/${BRANCH_NAME}/deploy_parametrized_pipeline.sql" \
+            "@${REPO_NAME}/branches/${BRANCH_NAME}/steps/0[134]_*" \
             -D "environment='${BRANCH_NAME}'" \
             -D "retention_time=${RETENTION_TIME}"

.gitignore

@@ -0,0 +1 @@
+.snowflake/logs

.snowflake/config.toml

@@ -0,0 +1,8 @@
+[connections.default]
+warehouse = "QUICKSTART_WH"
+role = "ACCOUNTADMIN"
+database = "QUICKSTART_COMMON"
+schema = "PUBLIC"
+account = "<INSERT YOUR ACCOUNT IDENTIFIER>"
+user = "<INSERT YOUR USERNAME>"
+password = "<INSERT YOUR PASSWORD>"

config.toml

@@ -1,10 +1,13 @@
 USE ROLE ACCOUNTADMIN;
 
-CREATE WAREHOUSE IF NOT EXISTS QUICKSTART_WH WAREHOUSE_SIZE = XSMALL, AUTO_SUSPEND = 300, AUTO_RESUME= TRUE;
+CREATE OR ALTER WAREHOUSE QUICKSTART_WH 
+  WAREHOUSE_SIZE = XSMALL 
+  AUTO_SUSPEND = 300 
+  AUTO_RESUME= TRUE;
 
 
 -- Separate database for git repository
-CREATE DATABASE IF NOT EXISTS QUICKSTART_COMMON;
+CREATE OR ALTER DATABASE QUICKSTART_COMMON;
 
 
 -- API integration is needed for GitHub integration
@@ -20,7 +23,7 @@ CREATE OR REPLACE GIT REPOSITORY quickstart_common.public.quickstart_repo
   ORIGIN = '<insert URL of forked GitHub repo>'; -- INSERT URL OF FORKED REPO HERE
 
 
-CREATE OR REPLACE DATABASE QUICKSTART_PROD;
+CREATE OR ALTER DATABASE QUICKSTART_PROD;
 
 
 -- To monitor data pipeline's completion
@@ -30,14 +33,14 @@ CREATE OR REPLACE NOTIFICATION INTEGRATION email_integration
 
 
 -- Database level objects
-CREATE SCHEMA IF NOT EXISTS bronze;
-CREATE SCHEMA IF NOT EXISTS silver;
-CREATE SCHEMA IF NOT EXISTS gold;
+CREATE OR ALTER SCHEMA bronze;
+CREATE OR ALTER SCHEMA silver;
+CREATE OR ALTER SCHEMA gold;
 
 
 -- Schema level objects
 CREATE OR REPLACE FILE FORMAT bronze.json_format TYPE = 'json';
-CREATE OR REPLACE STAGE bronze.raw;
+CREATE OR ALTER STAGE bronze.raw;
 
 
 -- Copy file from GitHub to internal stage

deploy_parametrized_pipeline.sql

@@ -4,5 +4,5 @@ USE ROLE ACCOUNTADMIN;
 SELECT * FROM oag_flight_emissions_data_sample.public.estimated_emissions_schedules_sample LIMIT 100;
 SELECT * FROM oag_flight_status_data_sample.public.flight_status_latest_sample LIMIT 100;
 SELECT * FROM global_weather__climate_data_for_bi.standard_tile.forecast_day LIMIT 100;
-SELECT * FROM government_essentials.cybersyn.datacommons_timeseries LIMIT 100;
-SELECT * FROM us_points_of_interest__addresses.cybersyn.point_of_interest_index LIMIT 100;
+SELECT * FROM global_government.cybersyn.datacommons_timeseries LIMIT 100;
+SELECT * FROM us_addresses__poi.cybersyn.point_of_interest_index LIMIT 100;