refactor prints to log & ensure local|test mode don't send logs to se…

src/_main_/celery/app.py

@@ -3,7 +3,7 @@
 from celery import Celery
 from celery import shared_task
 from _main_.celery.config import CeleryConfig
-from celery.schedules import crontab
+from _main_.utils.massenergize_logger import log 
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "_main_.settings")
 app = Celery('massenergize_celeryapp')
@@ -23,4 +23,4 @@
 
 @shared_task(bind=True)
 def debug_task(self):
-    print('Request: {0!r}'.format(self.request))
+    log.info('Request: {0!r}'.format(self.request))

src/_main_/settings.py

@@ -234,17 +234,18 @@
 ### End Logger settings ###
 
 # Sentry Logging Initialization
-sentry_dsn = os.environ.get('SENTRY_DSN')
-if sentry_dsn:
-  sentry_sdk.init(
-    dsn=sentry_dsn,
-    integrations=[DjangoIntegration(
-            transaction_style='url',
-            middleware_spans=True,
-        ),
-           CeleryIntegration(),
-    ],
-    traces_sample_rate=1.0,
+if EnvConfig.can_send_logs_to_external_watchers():
+    sentry_dsn = os.environ.get('SENTRY_DSN')
+    if sentry_dsn:
+        sentry_sdk.init(
+            dsn=sentry_dsn,
+            integrations=[DjangoIntegration(
+                    transaction_style='url',
+                    middleware_spans=True,
+                ),
+                CeleryIntegration(),
+            ],
+            traces_sample_rate=1.0,
 
 
     # If you wish to associate users to errors (assuming you are using

src/_main_/utils/GeoIP.py

@@ -35,7 +35,6 @@ def getBrowser(self, request):
     # iPhone's user agent string
     #ua_string = 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B179 Safari/7534.48.3'
     user_agent = parse(ua_string)
-    #print(user_agent)
 
     client = {}
     # Accessing user agent's browser attributes
@@ -70,7 +69,7 @@ def getIP(self,request):
         ip = request.META.get('REMOTE_ADDR')
 
     if not ip_valid(ip):
-      print("GeoIP: IP address is NOT valid")
+      log.error("GeoIP: IP address is NOT valid")
       return None
 
     return ip
@@ -99,5 +98,5 @@ def getGeo(self, ip):
 
     #self.reader.close()
     except Exception as e:
-      # print(e)
+      log.exception(e)
       return e

src/_main_/utils/common.py

@@ -273,11 +273,9 @@ def get_cookie(request, key):
 
 
 def set_cookie(response, key, value):  # TODO
-    print(f"----- set_cookie: {response}")
     # set cookie on response before sending
     # cookie expiration set to 1yr
     MAX_AGE = 31536000
-
     response.set_cookie(key, value, MAX_AGE, samesite="Strict")
 
 

src/_main_/utils/massenergize_logger/__init__.py

@@ -11,53 +11,47 @@ def __init__(self):
         self.logger = logging.getLogger(EnvConfig.get_logger_identifier())
 
 
-    @run_in_background
     def _log(self, level, message, exception=None, extra={}):
 
-        if not EnvConfig.can_send_logs_to_cloudwatch():
-            return
-
         if exception:
             extra['exception'] = exception
 
-        # log through the python django logger which goes to Cloudwatch
-        if level >= logging.ERROR:
-            # send to cw
-            self.logger.log(
-                level,
-                message, 
-                exc_info=True,
-                stack_info=True, 
-                stacklevel=2,
-                extra=extra
-            )
-
-            # send to sentry
+        def _send_to_cw_and_log_files():
+            # log through the python django logger which logs to Cloudwatch (for dev, canary, prod)
+            # and to log files (for other modes like test, local)
+            if level >= logging.ERROR:
+                self.logger.log(
+                    level,
+                    message, 
+                    exc_info=True,
+                    stack_info=True, 
+                    extra=extra
+                )
+            else:
+                self.logger.log(level, message, extra=extra)
+
+        def _send_to_sentry():
             if extra:
                 with sentry_sdk.push_scope() as sentry_scope:
                     for k,v in extra.items():
                         sentry_scope.set_extra(k, v)
             sentry_sdk.capture_message(message, level)
-        else:
-            # send to cw
-            self.logger.log(level, message, extra=extra)
 
-            # send log to sentry
-            if extra:
-                with sentry_sdk.push_scope() as sentry_scope:
-                    for k,v in extra.items():
-                        sentry_scope.set_extra(k, v)
-            sentry_sdk.capture_message(message, level)
-
+        # write to log files and send to cw
+        _send_to_cw_and_log_files()
+
+        # don't send to sentry
+        if EnvConfig.can_send_logs_to_external_watchers():
+            _send_to_sentry()
 
     def info(self, message, extra={}):
         self._log(logging.INFO, message, extra=extra)
 
     def error(self, message=None, exception=None, extra={}):
         self._log(logging.ERROR, message or str(exception), exception, extra)
 
-    def exception(self, exception):
-        self.error(str(exception), exception)
+    def exception(self, exception, message=None, extra={}):
+        self.error(message, exception, extra)
         return exception
 
 

src/_main_/utils/metrics/__init__.py

@@ -57,7 +57,7 @@ def send_metric(func, execution_time, name_space=FUNCTION_LATENCY_NAMESPACE, ext
 
 
 def put_metric_data(name_space, metric_data):
-    if not EnvConfig.can_send_logs_to_cloudwatch():
+    if not EnvConfig.can_send_logs_to_external_watchers():
         return 
 
     try:

src/_main_/utils/stage/__init__.py

@@ -95,13 +95,15 @@ def is_local(self):
     def is_test(self):
         return self.name == "test"
 
-    def can_send_logs_to_cloudwatch(self):
+    def can_send_logs_to_external_watchers(self):
         return not (self.is_local() or self.is_test())
 
 
     def get_logging_settings(self):
-        if self.is_local() or self.is_test():
+        if self.is_local():
             return get_local_logging_settings()
+        elif self.is_test():
+            return get_testing_logging_settings()
         return get_default_logging_settings(self.name)
 
 
@@ -151,13 +153,13 @@ def _set_api_run_info(self):
         is_docker_mode = False
 
         current_run_file_path = Path('.') / '.massenergize'/ 'current_run_info.json'
-        if current_run_file_path.exists():
+        if not name and current_run_file_path.exists():
             _current_run_info = load_json(current_run_file_path)
             name = _current_run_info.get('django_env', name)
             is_docker_mode = _current_run_info.get('is_docker_mode', is_docker_mode)
         else:
             load_dotenv()
-            name = os.getenv("DJANGO_ENV", "dev")
+            name = name or "dev" # use this value if name is None
             is_docker_mode = "DOCKER_CONTAINER" in os.environ
 
         name = name.lower()

src/_main_/utils/stage/logging.py

@@ -7,6 +7,7 @@
 BASE_DIR = os.getcwd()
 LOG_DIR = os.path.join(BASE_DIR, '.logs')
 LOG_FILE_NAME = "api.log"
+TESTING_LOG_FILE_NAME = "api.testing.log"
 
 # Ensure the log directory exists
 if not os.path.exists(LOG_DIR):
@@ -31,7 +32,7 @@ def get_default_logging_settings(stage):
         },
         'handlers': {
             'external': {
-                'level': 'DEBUG',  
+                'level': 'INFO',  
                 'class': 'watchtower.CloudWatchLogHandler',
                 'log_group': f"/api/{stage}",
                 'stream_name': get_host_identifier(),
@@ -42,7 +43,7 @@ def get_default_logging_settings(stage):
                 'filters': ['cid'], 
             },
             'file': {
-                'level': 'DEBUG',
+                'level': 'INFO',
                 'class': 'logging.handlers.TimedRotatingFileHandler',
                 'filename': os.path.join(LOG_DIR, LOG_FILE_NAME),
                 'when': 'midnight',  # Rotate logs at midnight
@@ -107,18 +108,59 @@ def get_local_logging_settings():
         },
         "root": {
             "handlers": ["console", "file"],
-            "level": "WARNING",
+            "level": "INFO",
         },
         'loggers': {
             get_logger_name(): {
                 'handlers': ['console', 'file'],
-                'level': 'DEBUG',
+                'level': 'INFO',
                 'propagate': False,
             }
         }
     }
 
+def get_testing_logging_settings():
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,
+        'filters': {
+            'cid': {
+                '()': 'cid.log.CidContextFilter'
+            },
+        },
+        'formatters': {
+            'detailed': {
+                'format': '[request_id=%(cid)s] %(asctime)s %(levelname)s %(name)s %(message)s'
+            },
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+            },
+            'file': {
+                'level': 'WARNING',
+                'class': 'logging.handlers.TimedRotatingFileHandler',
+                'filename': os.path.join(LOG_DIR, TESTING_LOG_FILE_NAME),
+                'when': 'midnight',  # Rotate logs at midnight
+                'interval': 1,
+                'backupCount': 7,  # Keep last 7 days of logs
+                'filters': ['cid'], 
+            },
+        },
+        "root": {
+            "handlers": ["file"],
+            "level": "WARNING",
+        },
+        'loggers': {
+            get_logger_name(): {
+                'handlers': ['file'],
+                'level': 'DEBUG',
+                'propagate': False,
+            }
+        }
+    }
 
+
 def get_host_identifier():
     try:
         ec2 = boto3.resource('ec2')

src/api/handlers/misc.py

@@ -89,7 +89,6 @@ def backfill(self, request):
     def actions_report(self, request):
         context: Context = request.context
         args: dict = context.args
-        print("Got here")
         goal_info, err = self.service.actions_report(context, args)
         if err:
             return err

src/api/services/auth.py

@@ -81,11 +81,10 @@ def login(self, context: Context):
       else:
         return None, None, CustomMassenergizeError("invalid_auth")
     except PermissionError:
-      log.error("not_an_admin", level="error")
+      log.error("not_an_admin", exception=PermissionError)
       return None, None, CustomMassenergizeError('not_an_admin')
     except Exception as e:
-      print(e)
-      log.error("Authentication Error", level="error")
+      log.exception(e)
       return None, None, CustomMassenergizeError(e)
 
 

src/api/services/event.py

@@ -134,7 +134,6 @@ def copy_event(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
   def list_recurring_event_exceptions(self, context, args) -> Tuple[list, MassEnergizeAPIError]:
     exceptions, err = self.store.list_recurring_event_exceptions(context, args)
     if err:
-      print(err)
       return None, err
     return exceptions, None
 

src/api/services/userprofile.py

@@ -332,7 +332,7 @@ def import_from_csv(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
               invalid_emails.append({"line":line, "email":email}) 
 
         except Exception as e:
-          print("Error string: " + str(e))
+          log.exception(e)
           return None, CustomMassenergizeError(e)
       if err:
         return None, err
@@ -377,7 +377,6 @@ def import_from_list(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
             invalid_emails.append({"line":ix, "first_name":first_name, "last_name": last_name, "email":email}) 
 
         except Exception as e:
-          print(str(e))
           return None, CustomMassenergizeError(e)
       return {'invalidEmails': invalid_emails}, None
     except Exception as e: