Handle syslog writing from the command line (RasaHQ#9513)

* Handle syslog writing from the command line

* add an arg in the command line to use syslog

* respect standards

* line was too long

* new arg for command line

* add sanic logs to syslog

* del useless statement

* quick fix

* too many blank line

* Use default communication with syslog

* possibility to use a syslog server

* fix indent

* Fix for constants access

* fix typo

Co-authored-by: Joe Juzl <joejuzl@gmail.com>

* fix typo

* fix for constant definition in bad file

* Black linter compliant

* Function add_server_arguments had 26 lines of code (exceeds 25 allowed).

* fix for bad merge

* for lint compliance

* add required param

* adapt test for log enhancement

Co-authored-by: Joe Juzl <joejuzl@gmail.com>

rasa/cli/arguments/run.py

@@ -55,6 +55,27 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None:
         default=None,
         help="Store logs in specified file.",
     )
+    parser.add_argument(
+        "--use-syslog", action="store_true", help="Add syslog as a log handler",
+    )
+    parser.add_argument(
+        "--syslog-address",
+        type=str,
+        default=constants.DEFAULT_SYSLOG_HOST,
+        help="Address of the syslog server. --use-sylog flag is required",
+    )
+    parser.add_argument(
+        "--syslog-port",
+        type=int,
+        default=constants.DEFAULT_SYSLOG_PORT,
+        help="Port of the syslog server. --use-sylog flag is required",
+    )
+    parser.add_argument(
+        "--syslog-protocol",
+        type=str,
+        default=constants.DEFAULT_PROTOCOL,
+        help="Protocol used with the syslog server. Can be UDP (default) or TCP ",
+    )
     add_endpoint_param(
         parser,
         help_text="Configuration file for the model server and the connectors as a "
@@ -113,7 +134,6 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None:
         help="If your ssl-keyfile is protected by a password, you can specify it "
         "using this paramer.",
     )
-
     channel_arguments = parser.add_argument_group("Channels")
     channel_arguments.add_argument(
         "--credentials",

rasa/cli/x.py

@@ -262,7 +262,7 @@ def _configure_logging(args: argparse.Namespace) -> None:
     configure_logging_and_warnings(
         log_level, warn_only_once=False, filter_repeated_logs=False
     )
-    configure_file_logging(logging.root, args.log_file)
+    configure_file_logging(logging.root, args.log_file, False)
 
     logging.getLogger("werkzeug").setLevel(logging.WARNING)
     logging.getLogger("engineio").setLevel(logging.WARNING)

rasa/core/constants.py

@@ -70,3 +70,7 @@
 POLICY_PRIORITY = "priority"
 POLICY_FEATURIZER = "featurizer"
 POLICY_MAX_HISTORY = "max_history"
+
+DEFAULT_PROTOCOL = "UDP"
+DEFAULT_SYSLOG_HOST = "localhost"
+DEFAULT_SYSLOG_PORT = 514

rasa/core/run.py

@@ -29,7 +29,6 @@ def create_http_input_channels(
     channel: Optional[Text], credentials_file: Optional[Text]
 ) -> List["InputChannel"]:
     """Instantiate the chosen input channel."""
-
     if credentials_file:
         all_credentials = rasa.shared.utils.io.read_config_file(credentials_file)
     else:
@@ -90,10 +89,15 @@ def configure_app(
     endpoints: Optional[AvailableEndpoints] = None,
     log_file: Optional[Text] = None,
     conversation_id: Optional[Text] = uuid.uuid4().hex,
+    use_syslog: bool = False,
+    syslog_address: Optional[Text] = None,
+    syslog_port: Optional[int] = None,
+    syslog_protocol: Optional[Text] = None,
 ) -> Sanic:
     """Run the agent."""
-
-    rasa.core.utils.configure_file_logging(logger, log_file)
+    rasa.core.utils.configure_file_logging(
+        logger, log_file, use_syslog, syslog_address, syslog_port, syslog_protocol,
+    )
 
     if enable_api:
         app = server.create_app(
@@ -160,9 +164,12 @@ def serve_application(
     ssl_ca_file: Optional[Text] = None,
     ssl_password: Optional[Text] = None,
     conversation_id: Optional[Text] = uuid.uuid4().hex,
+    use_syslog: Optional[bool] = False,
+    syslog_address: Optional[Text] = None,
+    syslog_port: Optional[int] = None,
+    syslog_protocol: Optional[Text] = None,
 ) -> None:
     """Run the API entrypoint."""
-
     if not channel and not credentials:
         channel = "cmdline"
 
@@ -180,6 +187,10 @@ def serve_application(
         endpoints=endpoints,
         log_file=log_file,
         conversation_id=conversation_id,
+        use_syslog=use_syslog,
+        syslog_address=syslog_address,
+        syslog_port=syslog_port,
+        syslog_protocol=syslog_protocol,
     )
 
     ssl_context = server.create_ssl_context(
@@ -203,7 +214,10 @@ def serve_application(
         input_channels, endpoints, model_path, number_of_workers, enable_api
     )
 
-    rasa.utils.common.update_sanic_log_level(log_file)
+    rasa.utils.common.update_sanic_log_level(
+        log_file, use_syslog, syslog_address, syslog_port, syslog_protocol,
+    )
+
     app.run(
         host=interface,
         port=port,

rasa/core/utils.py

@@ -17,37 +17,56 @@
 
 import rasa.shared.utils.io
 from rasa.constants import DEFAULT_SANIC_WORKERS, ENV_SANIC_WORKERS
-from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH
+from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, TCP_PROTOCOL
 
 from rasa.core.lock_store import LockStore, RedisLockStore, InMemoryLockStore
 from rasa.utils.endpoints import EndpointConfig, read_endpoint_config
 from sanic import Sanic
 from sanic.views import CompositionView
+from socket import SOCK_DGRAM, SOCK_STREAM
 import rasa.cli.utils as cli_utils
 
 
 logger = logging.getLogger(__name__)
 
 
 def configure_file_logging(
-    logger_obj: logging.Logger, log_file: Optional[Text]
+    logger_obj: logging.Logger,
+    log_file: Optional[Text],
+    use_syslog: Optional[bool],
+    syslog_address: Optional[Text] = None,
+    syslog_port: Optional[int] = None,
+    syslog_protocol: Optional[Text] = None,
 ) -> None:
     """Configure logging to a file.
 
     Args:
         logger_obj: Logger object to configure.
         log_file: Path of log file to write to.
+        use_syslog: Add syslog as a logger.
+        syslog_address: Adress of the syslog server.
+        syslog_port: Port of the syslog server.
+        syslog_protocol: Protocol with the syslog server
     """
-    if not log_file:
-        return
-
-    formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
-    file_handler = logging.FileHandler(
-        log_file, encoding=rasa.shared.utils.io.DEFAULT_ENCODING
-    )
-    file_handler.setLevel(logger_obj.level)
-    file_handler.setFormatter(formatter)
-    logger_obj.addHandler(file_handler)
+    if use_syslog:
+        formatter = logging.Formatter(
+            "%(asctime)s [%(levelname)-5.5s] [%(process)d]" " %(message)s"
+        )
+        socktype = SOCK_STREAM if syslog_protocol == TCP_PROTOCOL else SOCK_DGRAM
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(syslog_address, syslog_port), socktype=socktype,
+        )
+        syslog_handler.setLevel(logger_obj.level)
+        syslog_handler.setFormatter(formatter)
+        logger_obj.addHandler(syslog_handler)
+    if log_file:
+        formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
+        file_handler = logging.FileHandler(
+            log_file, encoding=rasa.shared.utils.io.DEFAULT_ENCODING
+        )
+        file_handler.setLevel(logger_obj.level)
+        file_handler.setFormatter(formatter)
+        logger_obj.addHandler(file_handler)
 
 
 def one_hot(hot_idx: int, length: int, dtype: Optional[Text] = None) -> np.ndarray:

rasa/shared/constants.py

@@ -52,6 +52,7 @@
 
 DEFAULT_LOG_LEVEL = "INFO"
 ENV_LOG_LEVEL = "LOG_LEVEL"
+TCP_PROTOCOL = "TCP"
 
 DEFAULT_SENDER_ID = "default"
 UTTER_PREFIX = "utter_"

rasa/utils/common.py

@@ -19,11 +19,11 @@
     Set,
 )
 
+from socket import SOCK_DGRAM, SOCK_STREAM
 import numpy as np
-
 import rasa.utils.io
 from rasa.constants import DEFAULT_LOG_LEVEL_LIBRARIES, ENV_LOG_LEVEL_LIBRARIES
-from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL
+from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL, TCP_PROTOCOL
 import rasa.shared.utils.io
 
 logger = logging.getLogger(__name__)
@@ -178,9 +178,14 @@ def update_tensorflow_log_level() -> None:
     logging.getLogger("tensorflow").propagate = False
 
 
-def update_sanic_log_level(log_file: Optional[Text] = None) -> None:
-    """Set the log level of sanic loggers to the log level specified in the environment
-    variable 'LOG_LEVEL_LIBRARIES'."""
+def update_sanic_log_level(
+    log_file: Optional[Text] = None,
+    use_syslog: Optional[bool] = False,
+    syslog_address: Optional[Text] = None,
+    syslog_port: Optional[int] = None,
+    syslog_protocol: Optional[Text] = None,
+) -> None:
+    """Set the log level to 'LOG_LEVEL_LIBRARIES' environment variable ."""
     from sanic.log import logger, error_logger, access_logger
 
     log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)
@@ -201,6 +206,18 @@ def update_sanic_log_level(log_file: Optional[Text] = None) -> None:
         logger.addHandler(file_handler)
         error_logger.addHandler(file_handler)
         access_logger.addHandler(file_handler)
+    if use_syslog:
+        formatter = logging.Formatter(
+            "%(asctime)s [%(levelname)-5.5s] [%(process)d]" " %(message)s"
+        )
+        socktype = SOCK_STREAM if syslog_protocol == TCP_PROTOCOL else SOCK_DGRAM
+        syslog_handler = logging.handlers.SysLogHandler(
+            address=(syslog_address, syslog_port), socktype=socktype,
+        )
+        syslog_handler.setFormatter(formatter)
+        logger.addHandler(syslog_handler)
+        error_logger.addHandler(syslog_handler)
+        access_logger.addHandler(syslog_handler)
 
 
 def update_asyncio_log_level() -> None:

tests/cli/test_rasa_run.py

@@ -18,8 +18,11 @@ def test_run_help(run: Callable[..., RunResult]):
     output = run("run", "--help")
 
     help_text = """usage: rasa run [-h] [-v] [-vv] [--quiet] [-m MODEL] [--log-file LOG_FILE]
-                [--endpoints ENDPOINTS] [-i INTERFACE] [-p PORT]
-                [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]] [--enable-api]
+                [--use-syslog] [--syslog-address SYSLOG_ADDRESS]
+                [--syslog-port SYSLOG_PORT]
+                [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS]
+                [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN]
+                [--cors [CORS [CORS ...]]] [--enable-api]
                 [--response-timeout RESPONSE_TIMEOUT]
                 [--remote-storage REMOTE_STORAGE]
                 [--ssl-certificate SSL_CERTIFICATE]

tests/cli/test_rasa_shell.py

@@ -7,9 +7,13 @@ def test_shell_help(run: Callable[..., RunResult]):
 
     help_text = """usage: rasa shell [-h] [-v] [-vv] [--quiet]
                   [--conversation-id CONVERSATION_ID] [-m MODEL]
-                  [--log-file LOG_FILE] [--endpoints ENDPOINTS] [-i INTERFACE]
-                  [-p PORT] [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]]
-                  [--enable-api] [--response-timeout RESPONSE_TIMEOUT]
+                  [--log-file LOG_FILE] [--use-syslog]
+                  [--syslog-address SYSLOG_ADDRESS]
+                  [--syslog-port SYSLOG_PORT]
+                  [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS]
+                  [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN]
+                  [--cors [CORS [CORS ...]]] [--enable-api]
+                  [--response-timeout RESPONSE_TIMEOUT]
                   [--remote-storage REMOTE_STORAGE]
                   [--ssl-certificate SSL_CERTIFICATE]
                   [--ssl-keyfile SSL_KEYFILE] [--ssl-ca-file SSL_CA_FILE]

tests/cli/test_rasa_x.py

@@ -19,9 +19,12 @@ def test_x_help(run: Callable[..., RunResult]):
     help_text = """usage: rasa x [-h] [-v] [-vv] [--quiet] [-m MODEL] [--data DATA [DATA ...]]
               [-c CONFIG] [-d DOMAIN] [--no-prompt] [--production]
               [--rasa-x-port RASA_X_PORT] [--config-endpoint CONFIG_ENDPOINT]
-              [--log-file LOG_FILE] [--endpoints ENDPOINTS] [-i INTERFACE]
-              [-p PORT] [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]]
-              [--enable-api] [--response-timeout RESPONSE_TIMEOUT]
+              [--log-file LOG_FILE] [--use-syslog]
+              [--syslog-address SYSLOG_ADDRESS] [--syslog-port SYSLOG_PORT]
+              [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS]
+              [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN]
+              [--cors [CORS [CORS ...]]] [--enable-api]
+              [--response-timeout RESPONSE_TIMEOUT]
               [--remote-storage REMOTE_STORAGE]
               [--ssl-certificate SSL_CERTIFICATE] [--ssl-keyfile SSL_KEYFILE]
               [--ssl-ca-file SSL_CA_FILE] [--ssl-password SSL_PASSWORD]