App Config Provider - Load Order Change (#42905)

* Changed to load all then process

* Moving async tests to aio folder

* updated with refresh

* fixing type hint

* Update sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_azureappconfigurationproviderbase.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Refresh re-work

* Updating kv processing

* reducing code reuse

* small updates

* Update assets.json

* Adding more tests and fixing validation

* Copilot review comments

* Update _azureappconfigurationproviderbase.py

* fixing backoff test

* review comments

* Review change + rename vars

* Simplifying Change

* simplifying change

* formatting fix

* Apply suggestions from code review

Co-authored-by: Ross Grambo <rossgrambo@microsoft.com>

* review comments

* rename _update_feature_filter_telemetry

* review comments

* split checking watch keys from refresh pull

* Moving sentinel key processing to it's own method

* Updated to a single return

* updated method name

* review comments

* fixed formatting

* fixed filter usage reset

* updating imports

* Updated Watch name

* comments

* more renaming

* Update _async_client_manager.py

* more renaming

* copilot suggestions

* Update _azureappconfigurationproviderbase.py

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Ross Grambo <rossgrambo@microsoft.com>

Author: 3 people

sdk/appconfiguration/azure-appconfiguration-provider/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/appconfiguration/azure-appconfiguration-provider",
-  "Tag": "python/appconfiguration/azure-appconfiguration-provider_b91db477af"
+  "Tag": "python/appconfiguration/azure-appconfiguration-provider_8a72ac47e0"
 }

sdk/appconfiguration/azure-appconfiguration-provider/azure/appconfiguration/provider/_azureappconfigurationprovider.py

@@ -3,6 +3,9 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # -------------------------------------------------------------------------
+import base64
+import hashlib
+import json
 import os
 import random
 import time
@@ -25,6 +28,10 @@
     ValuesView,
     TypeVar,
 )
+from azure.appconfiguration import (  # type:ignore # pylint:disable=no-name-in-module
+    ConfigurationSetting,
+    FeatureFlagConfigurationSetting,
+)
 from ._models import SettingSelector
 from ._constants import (
     REQUEST_TRACING_DISABLED_ENVIRONMENT_VARIABLE,
@@ -38,6 +45,16 @@
     PERCENTAGE_FILTER_KEY,
     TIME_WINDOW_FILTER_KEY,
     TARGETING_FILTER_KEY,
+    PERCENTAGE_FILTER_NAMES,
+    TIME_WINDOW_FILTER_NAMES,
+    TARGETING_FILTER_NAMES,
+    TELEMETRY_KEY,
+    METADATA_KEY,
+    ETAG_KEY,
+    FEATURE_FLAG_REFERENCE_KEY,
+    ALLOCATION_ID_KEY,
+    APP_CONFIG_AI_MIME_PROFILE,
+    APP_CONFIG_AICC_MIME_PROFILE,
 )
 
 JSON = Mapping[str, Any]
@@ -48,8 +65,13 @@
 
 
 def delay_failure(start_time: datetime.datetime) -> None:
-    # We want to make sure we are up a minimum amount of time before we kill the process. Otherwise, we could get stuck
-    # in a quick restart loop.
+    """
+    We want to make sure we are up a minimum amount of time before we kill the process.
+    Otherwise, we could get stuck in a quick restart loop.
+
+    :param start_time: The time when the process started.
+    :type start_time: datetime.datetime
+    """
     min_time = datetime.timedelta(seconds=min_uptime)
     current_time = datetime.datetime.now()
     if current_time - start_time < min_time:
@@ -161,11 +183,11 @@ def is_json_content_type(content_type: str) -> bool:
     return False
 
 
-def _build_sentinel(setting: Union[str, Tuple[str, str]]) -> Tuple[str, str]:
+def _build_watched_setting(setting: Union[str, Tuple[str, str]]) -> Tuple[str, str]:
     try:
         key, label = setting  # type:ignore
-    except IndexError:
-        key = setting
+    except (IndexError, ValueError):
+        key = str(setting)  # Ensure key is a string
         label = NULL_CHAR
     if "*" in key or "*" in label:
         raise ValueError("Wildcard key or label filters are not supported for refresh.")
@@ -260,7 +282,7 @@ class AzureAppConfigurationProviderBase(Mapping[str, Union[str, JSON]]):  # pyli
     """
 
     def __init__(self, **kwargs: Any) -> None:
-        self._origin_endpoint = kwargs.get("endpoint", None)
+        self._origin_endpoint: str = kwargs.get("endpoint", "")
         self._dict: Dict[str, Any] = {}
         self._selects: List[SettingSelector] = kwargs.pop(
             "selects", [SettingSelector(key_filter="*", label_filter=NULL_CHAR)]
@@ -270,7 +292,9 @@ def __init__(self, **kwargs: Any) -> None:
         self._trim_prefixes: List[str] = sorted(trim_prefixes, key=len, reverse=True)
 
         refresh_on: List[Tuple[str, str]] = kwargs.pop("refresh_on", None) or []
-        self._refresh_on: Mapping[Tuple[str, str], Optional[str]] = {_build_sentinel(s): None for s in refresh_on}
+        self._watched_settings: Dict[Tuple[str, str], Optional[str]] = {
+            _build_watched_setting(s): None for s in refresh_on
+        }
         self._refresh_timer: _RefreshTimer = _RefreshTimer(**kwargs)
         self._keyvault_credential = kwargs.pop("keyvault_credential", None)
         self._secret_resolver = kwargs.pop("secret_resolver", None)
@@ -282,10 +306,10 @@ def __init__(self, **kwargs: Any) -> None:
         )
         self._feature_flag_enabled = kwargs.pop("feature_flag_enabled", False)
         self._feature_flag_selectors = kwargs.pop("feature_flag_selectors", [SettingSelector(key_filter="*")])
-        self._refresh_on_feature_flags: Mapping[Tuple[str, str], Optional[str]] = {}
+        self._watched_feature_flags: Dict[Tuple[str, str], Optional[str]] = {}
         self._feature_flag_refresh_timer: _RefreshTimer = _RefreshTimer(**kwargs)
         self._feature_flag_refresh_enabled = kwargs.pop("feature_flag_refresh_enabled", False)
-        self._feature_filter_usage: Mapping[str, bool] = {}
+        self._feature_filter_usage: Dict[str, bool] = {}
         self._uses_load_balancing = kwargs.pop("load_balancing_enabled", False)
         self._uses_ai_configuration = False
         self._uses_aicc_configuration = False  # AI Chat Completion
@@ -301,6 +325,136 @@ def _process_key_name(self, config):
                 break
         return trimmed_key
 
+    def _update_ff_telemetry_metadata(
+        self, endpoint: str, feature_flag: FeatureFlagConfigurationSetting, feature_flag_value: Dict
+    ):
+        """
+        Add telemetry metadata to feature flag values.
+
+        :param endpoint: The App Configuration endpoint URL.
+        :type endpoint: str
+        :param feature_flag: The feature flag configuration setting.
+        :type feature_flag: FeatureFlagConfigurationSetting
+        :param feature_flag_value: The feature flag value dictionary to update.
+        :type feature_flag_value: Dict[str, Any]
+        """
+        if TELEMETRY_KEY in feature_flag_value:
+            if METADATA_KEY not in feature_flag_value[TELEMETRY_KEY]:
+                feature_flag_value[TELEMETRY_KEY][METADATA_KEY] = {}
+            feature_flag_value[TELEMETRY_KEY][METADATA_KEY][ETAG_KEY] = feature_flag.etag
+
+            if not endpoint.endswith("/"):
+                endpoint += "/"
+            feature_flag_reference = f"{endpoint}kv/{feature_flag.key}"
+            if feature_flag.label and not feature_flag.label.isspace():
+                feature_flag_reference += f"?label={feature_flag.label}"
+            if feature_flag_value[TELEMETRY_KEY].get("enabled"):
+                feature_flag_value[TELEMETRY_KEY][METADATA_KEY][FEATURE_FLAG_REFERENCE_KEY] = feature_flag_reference
+                allocation_id = self._generate_allocation_id(feature_flag_value)
+                if allocation_id:
+                    feature_flag_value[TELEMETRY_KEY][METADATA_KEY][ALLOCATION_ID_KEY] = allocation_id
+
+    def _update_feature_filter_telemetry(self, feature_flag: FeatureFlagConfigurationSetting):
+        """
+        Track feature filter usage for App Configuration telemetry.
+
+        :param feature_flag: The feature flag to analyze for filter usage.
+        :type feature_flag: FeatureFlagConfigurationSetting
+        """
+        if feature_flag.filters:
+            for filter in feature_flag.filters:
+                if filter.get("name") in PERCENTAGE_FILTER_NAMES:
+                    self._feature_filter_usage[PERCENTAGE_FILTER_KEY] = True
+                elif filter.get("name") in TIME_WINDOW_FILTER_NAMES:
+                    self._feature_filter_usage[TIME_WINDOW_FILTER_KEY] = True
+                elif filter.get("name") in TARGETING_FILTER_NAMES:
+                    self._feature_filter_usage[TARGETING_FILTER_KEY] = True
+                else:
+                    self._feature_filter_usage[CUSTOM_FILTER_KEY] = True
+
+    @staticmethod
+    def _generate_allocation_id(feature_flag_value: Dict[str, JSON]) -> Optional[str]:
+        """
+        Generates an allocation ID for the specified feature.
+        seed=123abc\ndefault_when_enabled=Control\npercentiles=0,Control,20;20,Test,100\nvariants=Control,standard;Test,special # pylint:disable=line-too-long
+
+        :param  Dict[str, JSON] feature_flag_value: The feature to generate an allocation ID for.
+        :rtype: str
+        :return: The allocation ID.
+        """
+
+        allocation_id = ""
+        allocated_variants = []
+
+        allocation: Optional[JSON] = feature_flag_value.get("allocation")
+
+        if not allocation:
+            return None
+
+        # Seed
+        allocation_id = f"seed={allocation.get('seed', '')}"
+
+        # DefaultWhenEnabled
+        if "default_when_enabled" in allocation:
+            allocated_variants.append(allocation.get("default_when_enabled"))
+
+        allocation_id += f"\ndefault_when_enabled={allocation.get('default_when_enabled', '')}"
+
+        # Percentile
+        allocation_id += "\npercentiles="
+
+        percentile = allocation.get("percentile")
+
+        if percentile:
+            percentile_allocations = sorted(
+                (x for x in percentile if x.get("from") != x.get("to")),
+                key=lambda x: x.get("from"),
+            )
+
+            for percentile_allocation in percentile_allocations:
+                if "variant" in percentile_allocation:
+                    allocated_variants.append(percentile_allocation.get("variant"))
+
+            allocation_id += ";".join(
+                f"{pa.get('from')}," f"{base64.b64encode(pa.get('variant').encode()).decode()}," f"{pa.get('to')}"
+                for pa in percentile_allocations
+            )
+
+        if not allocated_variants and not allocation.get("seed"):
+            return None
+
+        # Variants
+        allocation_id += "\nvariants="
+
+        variants_value = feature_flag_value.get("variants")
+        if variants_value and (isinstance(variants_value, list) or all(isinstance(v, dict) for v in variants_value)):
+            if (
+                allocated_variants
+                and isinstance(variants_value, list)
+                and all(isinstance(v, dict) for v in variants_value)
+            ):
+                sorted_variants: List[Dict[str, Any]] = sorted(
+                    (v for v in variants_value if v.get("name") in allocated_variants),
+                    key=lambda v: v.get("name"),
+                )
+
+                for v in sorted_variants:
+                    allocation_id += f"{base64.b64encode(v.get('name', '').encode()).decode()},"
+                    if "configuration_value" in v:
+                        allocation_id += (
+                            f"{json.dumps(v.get('configuration_value', ''), separators=(',', ':'), sort_keys=True)}"
+                        )
+                    allocation_id += ";"
+                if sorted_variants:
+                    allocation_id = allocation_id[:-1]
+
+        # Create a sha256 hash of the allocation_id
+        hash_object = hashlib.sha256(allocation_id.encode())
+        hash_digest = hash_object.digest()
+
+        # Encode the first 15 bytes in base64 url
+        return base64.urlsafe_b64encode(hash_digest[:15]).decode()
+
     def __getitem__(self, key: str) -> Any:
         # pylint:disable=docstring-missing-param,docstring-missing-return,docstring-missing-rtype
         """
@@ -366,14 +520,81 @@ def get(self, key: str, default: Optional[Union[str, JSON, _T]] = None) -> Union
         """
         Returns the value of the specified key. If the key does not exist, returns the default value.
 
-        :param str key: The key of the value to get.
+        :param key: The key of the value to get.
+        :type key: str
         :param default: The default value to return.
-        :type: str or None
+        :type default: Optional[Union[str, JSON, _T]]
         :return: The value of the specified key.
-        :rtype: Union[str, JSON]
+        :rtype: Union[str, JSON, _T, None]
         """
         with self._update_lock:
             return self._dict.get(key, default)
 
     def __ne__(self, other: Any) -> bool:
         return not self == other
+
+    def _process_key_value_base(self, config: ConfigurationSetting) -> Union[str, Dict[str, Any]]:
+        """
+        Process configuration values that are not KeyVault references. If the content type is None, the value is
+        returned as-is.
+
+        :param config: The configuration setting to process.
+        :type config: ConfigurationSetting
+        :return: The processed configuration value (JSON object if JSON content type, string otherwise).
+        :rtype: Union[str, Dict[str, Any]]
+        """
+        if config.content_type is None:
+            return config.value
+        if is_json_content_type(config.content_type) and not isinstance(config, FeatureFlagConfigurationSetting):
+            # Feature flags are of type json, but don't treat them as such
+            try:
+                if APP_CONFIG_AI_MIME_PROFILE in config.content_type:
+                    self._uses_ai_configuration = True
+                if APP_CONFIG_AICC_MIME_PROFILE in config.content_type:
+                    self._uses_aicc_configuration = True
+                return json.loads(config.value)
+            except json.JSONDecodeError:
+                try:
+                    # If the value is not a valid JSON, check if it has comments and remove them
+                    from ._json import remove_json_comments
+
+                    return json.loads(remove_json_comments(config.value))
+                except (json.JSONDecodeError, ValueError):
+                    # If the value is not a valid JSON, treat it like regular string value
+                    return config.value
+        return config.value
+
+    def _process_feature_flag(self, feature_flag: FeatureFlagConfigurationSetting) -> Dict[str, Any]:
+        feature_flag_value = json.loads(feature_flag.value)
+        self._update_ff_telemetry_metadata(self._origin_endpoint, feature_flag, feature_flag_value)
+        self._update_feature_filter_telemetry(feature_flag)
+        return feature_flag_value
+
+    def _update_watched_settings(
+        self, configuration_settings: List[ConfigurationSetting]
+    ) -> Dict[Tuple[str, str], Optional[str]]:
+        """
+        Updates the etags of watched settings that are part of the configuration
+        :param List[ConfigurationSetting] configuration_settings: The list of configuration settings to update
+        :return: A dictionary mapping (key, label) tuples to their updated etags
+        :rtype: Dict[Tuple[str, str], Optional[str]]
+        """
+        watched_settings: Dict[Tuple[str, str], Optional[str]] = {}
+        for config in configuration_settings:
+            if (config.key, config.label) in self._watched_settings:
+                watched_settings[(config.key, config.label)] = config.etag
+        return watched_settings
+
+    def _update_watched_feature_flags(
+        self, feature_flags: List[FeatureFlagConfigurationSetting]
+    ) -> Dict[Tuple[str, str], Optional[str]]:
+        """
+        Updates the etags of watched feature flags that are part of the configuration
+        :param List[FeatureFlagConfigurationSetting] feature_flags: The list of feature flags to update
+        :return: A dictionary mapping (key, label) tuples to their updated etags
+        :rtype: Dict[Tuple[str, str], Optional[str]]
+        """
+        watched_feature_flags: Dict[Tuple[str, str], Optional[str]] = {}
+        for feature_flag in feature_flags:
+            watched_feature_flags[(feature_flag.key, feature_flag.label)] = feature_flag.etag
+        return watched_feature_flags