Simplify code, remove LoggingFilter and move obfuscation functionalit…

…y to Formatter

airbyte-cdk/python/airbyte_cdk/entrypoint.py

@@ -5,13 +5,12 @@
 
 import argparse
 import importlib
-import logging
 import os.path
 import sys
 import tempfile
-from functools import partial
 from typing import Iterable, List
-from airbyte_cdk.logger import init_logger, AirbyteLogFormatter
+
+from airbyte_cdk.logger import AirbyteLogFormatter, init_logger
 from airbyte_cdk.models import AirbyteMessage, Status, Type
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
@@ -25,7 +24,8 @@ def __init__(self, source: Source):
         self.source = source
         self.logger = init_logger(f"airbyte.{getattr(source, 'name', '')}")
 
-    def parse_args(self, args: List[str]) -> argparse.Namespace:
+    @staticmethod
+    def parse_args(args: List[str]) -> argparse.Namespace:
         # set up parent parsers
         parent_parser = argparse.ArgumentParser(add_help=False)
         main_parser = argparse.ArgumentParser()
@@ -76,7 +76,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
 
                 # Now that we have the config, we can use it to get a list of ai airbyte_secrets
                 # that we should filter in logging to avoid leaking secrets
-                config_secrets = get_secrets(self.source, config)
+                config_secrets = get_secrets(self.source, config, self.logger)
                 AirbyteLogFormatter.update_secrets(config_secrets)
 
                 # Remove internal flags from config before validating so

airbyte-cdk/python/airbyte_cdk/logger.py

@@ -6,8 +6,8 @@
 import logging.config
 import sys
 import traceback
-from functools import reduce, partial
-from typing import List, Optional
+from functools import partial
+from typing import List
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage
 
@@ -43,6 +43,7 @@ def hook_fn(_logger, exception_type, exception_value, traceback_):
 
     sys.excepthook = partial(hook_fn, logger)
 
+
 def init_logger(name: str = None):
     """Initial set up of logger"""
     logging.setLoggerClass(AirbyteNativeLogger)
@@ -57,12 +58,12 @@ def init_logger(name: str = None):
 class AirbyteLogFormatter(logging.Formatter):
     """Output log records using AirbyteMessage"""
 
-    __secrets = []
+    _secrets = []
 
     @classmethod
     def update_secrets(cls, secrets: List[str]):
         """Update the list of secrets to be replaced in the log message"""
-        cls.__secrets = secrets
+        cls._secrets = secrets
 
     # Transforming Python log levels to Airbyte protocol log levels
     level_mapping = {
@@ -78,11 +79,8 @@ def format(self, record: logging.LogRecord) -> str:
         """Return a JSON representation of the log message"""
         message = super().format(record)
         airbyte_level = self.level_mapping.get(record.levelno, "INFO")
-        message = reduce(
-            lambda log_msg, secret: message.replace(str(secret), "****"),
-            self.__secrets,
-            record.msg,
-        )
+        for secret in AirbyteLogFormatter._secrets:
+            message = message.replace(secret, "****")
         log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=airbyte_level, message=message))
         return log_message.json(exclude_unset=True)
 

airbyte-cdk/python/airbyte_cdk/utils/airbyte_secrets_utils.py

@@ -2,19 +2,20 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import logging
 from typing import Any, List, Mapping
 
 from airbyte_cdk.sources import Source
 from airbyte_cdk.utils.mapping_utils import all_key_pairs_dot_notation, get_value_by_dot_notation
 
 
-def get_secrets(source: Source, config: Mapping[str, Any]) -> List[Any]:
+def get_secrets(source: Source, config: Mapping[str, Any], logger: logging.Logger) -> List[Any]:
     """
     Get a list of secrets from the source config based on the source specification
     """
-    flattened_key_values = all_key_pairs_dot_notation(source.spec().connectionSpecification.get("properties", {}))
+    flattened_key_values = all_key_pairs_dot_notation(source.spec(logger).connectionSpecification.get("properties", {}))
     secret_key_names = [
-        ".".join(key.split(".")[:1]) for key, value in flattened_key_values.items() if value and key.endswith(f"airbyte_secret")
+        ".".join(key.split(".")[:1]) for key, value in flattened_key_values.items() if value and key.endswith("airbyte_secret")
     ]
     result = [str(get_value_by_dot_notation(config, key)) for key in secret_key_names if config.get(key)]
     return result

airbyte-cdk/python/unit_tests/test_secure_logger.py

@@ -224,7 +224,7 @@ def read(
 
     try:
         list(entrypoint.run(parsed_args))
-    except Exception as e:
+    except Exception:
         sys.excepthook(*sys.exc_info())
         log_result = capsys.readouterr().out + capsys.readouterr().err
         assert NOT_A_SECRET_VALUE in log_result, "Should not have filtered non-secret value from exception"