snowflakedb · sfc-gh-anavalos · May 12, 2025 · May 8, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,30 @@
 # Release History
 
+## 1.8.4
+
+### Bug Fixes
+
+- Registry: Default `enable_explainability` to True when the model can be deployed to Warehouse.
+- Registry: Add `custom_model.partitioned_api` decorator and deprecate `partitioned_inference_api`.
+- Registry: Fixed a bug when logging pytroch and tensorflow models that caused
+  `UnboundLocalError: local variable 'multiple_inputs' referenced before assignment`.
+
+### Breaking change
+
+- ML Job: Updated property `id` to be fully qualified name; Introduced new property `name` to represent the ML Job name
+- ML Job: Modified `list_jobs()` to return ML Job `name` instead of `id`
+- Registry: Error in `log_model` if `enable_explainability` is True and model is only deployed to
+   Snowpark Container Services, instead of just user warning.
+
+### New Features
+
+- ML Job: Extend `@remote` function decorator, `submit_file()` and `submit_directory()` to accept `database` and
+  `schema` parameters
+- ML Job: Support querying by fully qualified name in `get_job()`
+- Explainability: Added visualization functions to `snowflake.ml.monitoring` to plot explanations in notebooks.
+- Explainability: Support explain for categorical transforms for sklearn pipeline
+- Support categorical type for `xgboost.DMatrix` inputs.
+
 ## 1.8.3
 
 ### Bug Fixes
@@ -13,6 +38,7 @@
   as a list of strings
 - Registry: Support `ModelVersion.run_job` to run inference with a single-node Snowpark Container Services job.
 - DataConnector: Removed PrPr decorators
+- Registry: Default the target platform to warehouse when logging a partitioned model.
 
 ## 1.8.2
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -337,10 +337,11 @@ corresponding optional dependencies installed to run the tests or binaries. Here
 1. Tags in the same group should not have conflicting packages.
 1. Ideally, avoid too many groups and keep test numbers balanced across groups.
 
-If you add a new group, please run the, following command to update the bazelrc file.
+If you add a new group, please run the, following commands to update the bazelrc file.
 
 ```sh
 bazel run --config=pre_build //:sync_bazelrc
+bazel run --config=pre_build //ci/internal:sync_internal_requirements && bazel run --config=pre_build //bazel/requirements:sync_requirements
 ```
 
 ### Checking Package Latest Versions
@@ -353,7 +354,7 @@ Python packages from both PyPI and Snowflake Conda repositories against your cur
 Run the script in dry-run mode to safely check latest versions without modifying files:
 
 ```bash
-bazel run //bazel/requirements:update_version_requirements -- requirements.yml --dry-run
+bazel run --config=core //bazel/requirements:update_version_requirements -- requirements.yml --dry-run
 ```
 
 This will scan `requirements.yml` and show the latest version of packages and the suggest changes:

diff --git a/bazel/environments/conda-env-all.yml b/bazel/environments/conda-env-all.yml
@@ -8,6 +8,7 @@ channels:
 dependencies:
   - absl-py==1.4.0
   - aiohttp==3.8.3
+  - altair==5.5.0
   - anyio==3.5.0
   - boto3==1.34.69
   - cachetools==5.3.3
@@ -54,12 +55,13 @@ dependencies:
   - sentence-transformers==2.7.0
   - sentencepiece==0.1.99
   - shap==0.46.0
-  - snowflake-connector-python==3.12.0
+  - snowflake-connector-python==3.14.0
   - snowflake-snowpark-python==1.28.0
   - snowflake.core==1.0.2
   - sphinx==5.0.2
   - sqlparse==0.4.4
   - starlette==0.27.0
+  - streamlit==1.44.1
   - tensorflow==2.17.0
   - tokenizers==0.15.1
   - toml==0.10.2

diff --git a/bazel/environments/conda-env-core.yml b/bazel/environments/conda-env-core.yml
@@ -46,7 +46,8 @@ dependencies:
   - s3fs==2024.6.1
   - scikit-learn==1.5.1
   - scipy==1.10.1
-  - snowflake-connector-python==3.12.0
+  - shap==0.46.0
+  - snowflake-connector-python==3.14.0
   - snowflake-snowpark-python==1.28.0
   - snowflake.core==1.0.2
   - sphinx==5.0.2
@@ -59,7 +60,6 @@ dependencies:
   - types-toml==0.10.8.6
   - typing-extensions==4.11.0
   - werkzeug==2.2.2
-  - xgboost==2.1.1
   - pip
   - pip:
       - --extra-index-url https://pypi.org/simple

diff --git a/bazel/environments/conda-env-keras.yml b/bazel/environments/conda-env-keras.yml
@@ -48,7 +48,8 @@ dependencies:
   - s3fs==2024.6.1
   - scikit-learn==1.5.1
   - scipy==1.10.1
-  - snowflake-connector-python==3.12.0
+  - shap==0.46.0
+  - snowflake-connector-python==3.14.0
   - snowflake-snowpark-python==1.28.0
   - snowflake.core==1.0.2
   - sphinx==5.0.2
@@ -63,7 +64,6 @@ dependencies:
   - types-toml==0.10.8.6
   - typing-extensions==4.11.0
   - werkzeug==2.2.2
-  - xgboost==2.1.1
   - pip
   - pip:
       - --extra-index-url https://pypi.org/simple

diff --git a/bazel/environments/conda-env-ml.yml b/bazel/environments/conda-env-ml.yml
@@ -8,6 +8,7 @@ channels:
 dependencies:
   - absl-py==1.4.0
   - aiohttp==3.8.3
+  - altair==5.5.0
   - anyio==3.5.0
   - boto3==1.34.69
   - cachetools==5.3.3
@@ -50,12 +51,13 @@ dependencies:
   - scikit-learn==1.5.1
   - scipy==1.10.1
   - shap==0.46.0
-  - snowflake-connector-python==3.12.0
+  - snowflake-connector-python==3.14.0
   - snowflake-snowpark-python==1.28.0
   - snowflake.core==1.0.2
   - sphinx==5.0.2
   - sqlparse==0.4.4
   - starlette==0.27.0
+  - streamlit==1.44.1
   - toml==0.10.2
   - types-PyYAML==6.0.12.12
   - types-protobuf==4.23.0.1

diff --git a/bazel/environments/conda-env-torch.yml b/bazel/environments/conda-env-torch.yml
@@ -50,7 +50,8 @@ dependencies:
   - scipy==1.10.1
   - sentence-transformers==2.7.0
   - sentencepiece==0.1.99
-  - snowflake-connector-python==3.12.0
+  - shap==0.46.0
+  - snowflake-connector-python==3.14.0
   - snowflake-snowpark-python==1.28.0
   - snowflake.core==1.0.2
   - sphinx==5.0.2
@@ -66,7 +67,6 @@ dependencies:
   - types-toml==0.10.8.6
   - typing-extensions==4.11.0
   - werkzeug==2.2.2
-  - xgboost==2.1.1
   - pip
   - pip:
       - --extra-index-url https://pypi.org/simple

diff --git a/bazel/environments/conda-optional-dependency-ml.yml b/bazel/environments/conda-optional-dependency-ml.yml
@@ -6,7 +6,9 @@ channels:
   - https://repo.anaconda.com/pkgs/snowflake
   - nodefaults
 dependencies:
+  - altair>=5,<6
   - catboost>=1.2.0, <2
   - lightgbm>=4.1.0, <5
   - mlflow>=2.16.0, <3
-  - shap>=0.46.0,<1
+  - streamlit>=1.44.0,<2
+  - xgboost>=1.7.3,<3
diff --git a/bazel/environments/requirements_core.txt b/bazel/environments/requirements_core.txt
@@ -40,7 +40,8 @@ ruamel.yaml==0.17.21
 s3fs==2024.6.1
 scikit-learn==1.5.1
 scipy==1.10.1
-snowflake-connector-python[pandas]==3.12.0
+shap==0.46.0
+snowflake-connector-python[pandas]==3.14.0
 snowflake-snowpark-python==1.28.0
 snowflake.core==1.0.2
 sphinx==5.0.2
@@ -54,4 +55,3 @@ types-requests==2.30.0.0
 types-toml==0.10.8.6
 typing-extensions==4.11.0
 werkzeug==2.2.2
-xgboost==2.1.1
diff --git a/bazel/environments/requirements_keras.txt b/bazel/environments/requirements_keras.txt
@@ -41,7 +41,8 @@ ruamel.yaml==0.17.21
 s3fs==2024.6.1
 scikit-learn==1.5.1
 scipy==1.10.1
-snowflake-connector-python[pandas]==3.12.0
+shap==0.46.0
+snowflake-connector-python[pandas]==3.14.0
 snowflake-snowpark-python==1.28.0
 snowflake.core==1.0.2
 sphinx==5.0.2
@@ -59,4 +60,3 @@ types-requests==2.30.0.0
 types-toml==0.10.8.6
 typing-extensions==4.11.0
 werkzeug==2.2.2
-xgboost==2.1.1
diff --git a/bazel/environments/requirements_ml.txt b/bazel/environments/requirements_ml.txt
@@ -3,6 +3,7 @@
 
 --extra-index-url https://pypi.org/simple
 absl-py==1.4.0
+altair==5.5.0
 anyio==3.5.0
 boto3==1.34.69
 build==0.10.0
@@ -44,12 +45,13 @@ s3fs==2024.6.1
 scikit-learn==1.5.1
 scipy==1.10.1
 shap==0.46.0
-snowflake-connector-python[pandas]==3.12.0
+snowflake-connector-python[pandas]==3.14.0
 snowflake-snowpark-python==1.28.0
 snowflake.core==1.0.2
 sphinx==5.0.2
 sqlparse==0.4.4
 starlette==0.27.0
+streamlit==1.44.1
 toml==0.10.2
 types-PyYAML==6.0.12.12
 types-cachetools==4.2.2

diff --git a/bazel/environments/requirements_torch.txt b/bazel/environments/requirements_torch.txt
@@ -44,7 +44,8 @@ scikit-learn==1.5.1
 scipy==1.10.1
 sentence-transformers==2.7.0
 sentencepiece==0.1.99
-snowflake-connector-python[pandas]==3.12.0
+shap==0.46.0
+snowflake-connector-python[pandas]==3.14.0
 snowflake-snowpark-python==1.28.0
 snowflake.core==1.0.2
 sphinx==5.0.2
@@ -62,4 +63,3 @@ types-requests==2.30.0.0
 types-toml==0.10.8.6
 typing-extensions==4.11.0
 werkzeug==2.2.2
-xgboost==2.1.1
diff --git a/bazel/platforms/optional_dependency_groups.bzl b/bazel/platforms/optional_dependency_groups.bzl
@@ -1,5 +1,5 @@
 OPTIONAL_DEPENDENCY_GROUPS = {
     "keras": ["torch", "tensorflow", "keras"],
-    "ml": ["lightgbm", "catboost", "shap", "mlflow"],
+    "ml": ["lightgbm", "catboost", "mlflow", "altair", "xgboost", "streamlit"],
     "torch": ["torch", "transformers", "mlflow"],
 }
diff --git a/ci/RunBazelAction.sh b/ci/RunBazelAction.sh
@@ -112,7 +112,7 @@ merge_gate)
     affected_targets_file="${working_dir}/affected_targets"
     ./bazel/get_affected_targets.sh -b "${bazel}" -f "${affected_targets_file}"
 
-    query_expr='kind(".*_test rule", rdeps(//... - set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/slow.txt")"') - set('"$(<"ci/targets/local_only.txt")"'), set('"$(<"${affected_targets_file}")"')))'
+    query_expr='kind(".*_test rule", rdeps(//... - set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/exclude_from_merge_gate.txt")"') - set('"$(<"ci/targets/local_only.txt")"'), set('"$(<"${affected_targets_file}")"')))'
     ;;
 continuous_run)
     query_expr='kind(".*_test rule", //... - set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/local_only.txt")"'))'

diff --git a/ci/build_and_run_tests.sh b/ci/build_and_run_tests.sh
@@ -39,6 +39,7 @@ BAZEL="bazel"
 ENV="pip"
 WITH_SNOWPARK=false
 WITH_SPCS_IMAGE=false
+INCLUDE_AUTOGEN_TESTS=false
 MODE="continuous_run"
 PYTHON_VERSION=3.9
 PYTHON_ENABLE_SCRIPT="bin/activate"
@@ -65,6 +66,9 @@ while (($#)); do
     --with-snowpark)
         WITH_SNOWPARK=true
         ;;
+    --include-autogen-tests)
+        INCLUDE_AUTOGEN_TESTS=true
+        ;;
     --mode)
         shift
         if [[ $1 = "merge_gate" || $1 = "continuous_run" || $1 = "quarantined" || $1 = "release" ]]; then
@@ -196,13 +200,19 @@ pushd ${SNOWML_DIR}
 VERSION=$(grep -oE "VERSION = \"[0-9]+\\.[0-9]+\\.[0-9]+.*\"" snowflake/ml/version.py| cut -d'"' -f2)
 echo "Extracted Package Version from code: ${VERSION}"
 
-# Generate and copy auto-gen tests.
-"${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" build --config=build "${BAZEL_ADDITIONAL_BUILD_FLAGS[@]+"${BAZEL_ADDITIONAL_BUILD_FLAGS[@]}"}" //tests/integ/...
+if [ "${INCLUDE_AUTOGEN_TESTS}" = true ]; then
+    echo "Generating and copying autogen tests."
+    #
+    # Generate and copy auto-gen tests.
+    "${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" build --config=build "${BAZEL_ADDITIONAL_BUILD_FLAGS[@]+"${BAZEL_ADDITIONAL_BUILD_FLAGS[@]}"}" //tests/integ/...
 
-# Rsync cannot work well with path that has drive letter in Windows,
-# Thus, rsync has to use relative path instead of absolute ones.
+    # Rsync cannot work well with path that has drive letter in Windows,
+    # Thus, rsync has to use relative path instead of absolute ones.
 
-rsync -av --exclude '*.runfiles_manifest' --exclude '*.runfiles/**' "bazel-bin/tests" .
+    rsync -av --exclude '*.runfiles_manifest' --exclude '*.runfiles/**' "bazel-bin/tests" .
+else
+    echo "Excluding autogen tests."
+fi
 
 # Read environments from optional_dependency_groups.bzl
 groups=()

diff --git a/ci/conda_recipe/meta.yaml b/ci/conda_recipe/meta.yaml
@@ -17,7 +17,7 @@ build:
   noarch: python
 package:
   name: snowflake-ml-python
-  version: 1.8.3
+  version: 1.8.4
 requirements:
   build:
     - python
@@ -27,7 +27,7 @@ requirements:
     - aiohttp!=4.0.0a0, !=4.0.0a1
     - anyio>=3.5.0,<5
     - cachetools>=3.1.1,<6
-    - cloudpickle>=2.0.0,<3
+    - cloudpickle>=2.0.0
     - cryptography
     - fsspec>=2024.6.1,<2026
     - importlib_resources>=6.1.1, <7
@@ -42,28 +42,30 @@ requirements:
     - requests
     - retrying>=1.3.3,<2
     - s3fs>=2024.6.1,<2026
-    - scikit-learn>=1.4,<1.6
+    - scikit-learn<1.6
     - scipy>=1.9,<2
-    - snowflake-connector-python>=3.12.0,<4
+    - shap>=0.46.0,<1
+    - snowflake-connector-python>=3.14.0,<4
     - snowflake-snowpark-python>=1.17.0,<2,!=1.26.0
     - snowflake.core>=1.0.2,<2
     - sqlparse>=0.4,<1
     - typing-extensions>=4.1.0,<5
-    - xgboost>=1.7.3,<3
     - python>=3.9,<3.13
   run_constrained:
+    - altair>=5,<6
     - catboost>=1.2.0, <2
     - keras>=2.0.0,<4
     - lightgbm>=4.1.0, <5
     - mlflow>=2.16.0, <3
     - pytorch>=2.0.1,<3
     - sentence-transformers>=2.7.0,<4
     - sentencepiece>=0.1.95,<0.2.0
-    - shap>=0.46.0,<1
+    - streamlit>=1.44.0,<2
     - tensorflow>=2.17.0,<3
     - tokenizers>=0.15.1,<1
     - torchdata>=0.4,<1
     - transformers>=4.39.3,<5
+    - xgboost>=1.7.3,<3
 source:
   path: ../../
 test:

diff --git a/ci/get_excluded_tests.sh b/ci/get_excluded_tests.sh
@@ -99,7 +99,7 @@ if [[ $mode = "merge_gate" ]]; then
 
     # -- Begin of Query Rules Heredoc --
     cat >"${unaffected_test_rule_file}" <<EndOfMessage
-    let affected_targets = set($(<"${affected_targets_file}")) - set($(<"ci/targets/slow.txt")) - set($(<"ci/targets/quarantine/${SF_ENV}.txt")) in
+    let affected_targets = set($(<"${affected_targets_file}")) - set($(<"ci/targets/exclude_from_merge_gate.txt")) - set($(<"ci/targets/quarantine/${SF_ENV}.txt")) in
         let unaffected_targets = //tests/... - deps(\$affected_targets) - rdeps(//tests/..., \$affected_targets) in
             labels(srcs, \$unaffected_targets)
 EndOfMessage

diff --git a/ci/targets/slow.txt → ci/targets/exclude_from_merge_gate.txt b/ci/targets/slow.txt → ci/targets/exclude_from_merge_gate.txt
@@ -4,3 +4,5 @@
 //tests/integ/snowflake/ml/registry/services:registry_sentence_transformers_model_deployment_test
 //tests/integ/snowflake/ml/registry/services:registry_huggingface_pipeline_model_deployment_test
 //tests/integ/snowflake/ml/registry/services:registry_sklearn_model_deployment_test
+//tests/integ/snowflake/ml/registry/model:registry_artifact_repository_test
+//tests/integ/snowflake/ml/registry/model:registry_target_platforms_test
diff --git a/ci/targets/quarantine/prod3.txt b/ci/targets/quarantine/prod3.txt
@@ -1,2 +1,4 @@
 //tests/integ/snowflake/ml/extra_tests:xgboost_external_memory_training_test
 //tests/integ/snowflake/ml/extra_tests:pipeline_with_ohe_and_xgbr_test
+//tests/integ/snowflake/ml/registry/model:registry_artifact_repository_test
+//tests/integ/snowflake/ml/registry/model:registry_target_platforms_test