Merge pull request #718 from vlad-perevezentsev/use_glog

Implementation of GLog

Author: oleksandr-pavlyk

.flake8

@@ -13,6 +13,7 @@ exclude =
     .git
 
 per-file-ignores =
+    dpctl/_diagnostics.pyx: E999
     dpctl/_sycl_context.pyx: E999, E225, E227
     dpctl/_sycl_device.pyx: E999, E225
     dpctl/_sycl_device_factory.pyx: E999, E225

CONTRIBUTING.md

@@ -161,13 +161,12 @@ these steps:
 3. Build dpctl with code coverage support.
 
     ```bash
-    python setup.py develop --coverage=True
-    pytest -q -ra --disable-warnings --cov dpctl --cov-report term-missing --pyargs dpctl -vv
+    python scripts/gen_coverage.py --oneapi
     coverage html
     ```
 
     Note that code coverage builds the C sources with debug symbols. For this
-    reason, the coverage flag is only available with the `develop` mode of
+    reason, the coverage script builds the package in `develop` mode of
     `setup.py`.
 
     The coverage results for the C and Python sources will be printed to the
@@ -191,3 +190,52 @@ these steps:
 >    ```
 >    The error is related to the `tcl` package. You should uninstall the `tcl`
 > package to resolve the error.
+
+## Error Reporting and Logging
+
+The SyclInterface library responds to `DPCTL_VERBOSITY` environment variable that controls the severity level of errors printed to console.
+One can specify one of the following severity levels (in increasing order of severity): `warning` and `error`.
+
+```bash
+export DPCTL_VERBOSITY=warning
+```
+
+Messages of a given severity are shown not only in the console for that severity, but also for the higher severity. For example, the severity level `warning` will output severity errors for `error` and `warning` to the console.
+
+### Optional use of the Google logging library (glog)
+
+Dpctl's error handler for libsyclinterface can be optionally configured to use [glog](https://github.com/google/glog). To use glog, follow the following steps:
+
+1. Install glog package of the latest version (0.5.0)
+
+```bash
+conda install glog
+```
+2. Build dpctl with glog support
+
+```bash
+python scripts/build_locally.py --oneapi --glog
+```
+
+3. Use `dpctl._diagnostics.syclinterface_diagnostics(verbosity="warning", log_dir=None)` context manager to switch library diagnostics on for a block of Python code.
+Use `DPCTLService_InitLogger` and `DPCTLService_ShutdownLogger` library C functions during library development to initialize the Google's logging library and de-initialize accordingly
+
+```python
+from dpctl._diagnostics import syclinterface_diagnostics
+import dpctl
+
+with syclinterface_diagnostics():
+    code
+```
+
+```c
+DPCTLService_InitLogger(const char *app_name, const char *log_dir);
+DPCTLService_ShutdownLogger();
+```
+
+ - `*app_name` - name of the executable file (prefix for logs of various levels).
+ - `*log_dir` - directory path for writing log files. Specifying `NULL` results in logging to ``std::cerr``.
+
+> **_NOTE:_**
+>
+> If `InitGoogleLogging` is not called before first use of glog, the library will self-initialize to `logtostderr` mode and log files will not be generated.

dpctl/_diagnostics.pyx

@@ -0,0 +1,80 @@
+#                      Data Parallel Control (dpctl)
+#
+# Copyright 2020-2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# distutils: language = c++
+# cython: language_level=3
+# cython: linetrace=True
+
+""" Implements developer utilities.
+"""
+import contextlib
+import os
+
+
+cdef extern from "syclinterface/dpctl_service.h":
+   cdef void DPCTLService_InitLogger(const char *, const char *)
+   cdef void DPCTLService_ShutdownLogger()
+
+
+def _init_logger(log_dir=None):
+    """Initialize logger to use given directory to save logs.
+
+    The call has no effect if `dpctl` was not built to use logger.
+    """
+    cdef bytes p = b""
+    cdef const char *app_name = "dpctl"
+    cdef char *ld_cstr = NULL
+    if log_dir:
+        if not os.path.exists(log_dir):
+            raise ValueError(f"Path {log_dir} does not exist")
+        if isinstance(log_dir, str):
+            p = bytes(log_dir, "utf-8")
+        else:
+            p = bytes(log_dir)
+        ld_cstr = <char *>p
+    DPCTLService_InitLogger(app_name, ld_cstr)
+
+
+def _shutdown_logger():
+    """Finalize logger.
+
+    The call has no effect if `dpctl` was not built to use logger.
+    """
+    DPCTLService_ShutdownLogger()
+
+
+@contextlib.contextmanager
+def syclinterface_diagnostics(verbosity="warning", log_dir=None):
+    """Context manager that activate verbosity of DPCTLSyclInterface
+    function calls.
+    """
+    _allowed_verbosity = ["warning", "error"]
+    if not verbosity in _allowed_verbosity:
+        raise ValueError(
+            f"Verbosity argument not understood. "
+            f"Permitted values are {_allowed_verbosity}"
+        )
+    _init_logger(log_dir=log_dir)
+    _saved_verbosity = os.environ.get("DPCTL_VERBOSITY", None)
+    os.environ["DPCTL_VERBOSITY"] = verbosity
+    try:
+        yield
+    finally:
+        _shutdown_logger()
+        if _saved_verbosity:
+            os.environ["DPCTL_VERBOSITY"] = _saved_verbosity
+        else:
+            del os.environ["DPCTL_VERBOSITY"]

dpctl/tests/test_service.py

@@ -107,3 +107,25 @@ def test___version__():
         r"0|[1-9][0-9]*))?(\+.*)?$"
     )
     assert re.match(reg_expr, dpctl_ver) is not None
+
+
+def test_dev_utils():
+    import tempfile
+
+    import dpctl._diagnostics as dd
+
+    ctx_mngr = dd.syclinterface_diagnostics
+
+    with ctx_mngr():
+        dpctl.SyclDevice().parent_device
+    with ctx_mngr(verbosity="error"):
+        dpctl.SyclDevice().parent_device
+    with pytest.raises(ValueError):
+        with ctx_mngr(verbosity="blah"):
+            dpctl.SyclDevice().parent_device
+    with tempfile.TemporaryDirectory() as temp_dir:
+        with ctx_mngr(log_dir=temp_dir):
+            dpctl.SyclDevice().parent_device
+    with pytest.raises(ValueError):
+        with ctx_mngr(log_dir="/not_a_dir"):
+            dpctl.SyclDevice().parent_device

libsyclinterface/CMakeLists.txt

@@ -38,6 +38,11 @@ option(DPCTL_BUILD_CAPI_TESTS
     "Build dpctl C API google tests"
     OFF
 )
+# Option to turn on logging support for dpctl C API
+option(DPCTL_ENABLE_GLOG
+    "Enable the Google logging module"
+    OFF
+)
 
 # Minimum version requirement only when oneAPI dpcpp is used.
 if(DPCTL_DPCPP_FROM_ONEAPI)
@@ -166,12 +171,25 @@ target_include_directories(DPCTLSyclInterface
     ${CMAKE_CURRENT_SOURCE_DIR}/helper/include/
     ${IntelSycl_SYCL_INCLUDE_DIR}
 )
-
 target_link_libraries(DPCTLSyclInterface
-    PRIVATE ${IntelSycl_SYCL_LIBRARY}
-    PRIVATE ${IntelSycl_OPENCL_LIBRARY}
+  PRIVATE ${IntelSycl_SYCL_LIBRARY}
+  PRIVATE ${IntelSycl_OPENCL_LIBRARY}
 )
 
+if(DPCTL_ENABLE_GLOG)
+    find_package(glog REQUIRED)
+
+    target_include_directories(DPCTLSyclInterface
+        PRIVATE
+        glog::glog
+    )
+    target_compile_definitions(DPCTLSyclInterface PRIVATE ENABLE_GLOG)
+    target_link_libraries(DPCTLSyclInterface
+        PRIVATE glog::glog
+    )
+endif()
+
+
 include(GetProjectVersion)
 # the get_version function is defined in the GetProjectVersion module and
 # defines: VERSION, SEMVER, MAJOR, MINOR, PATCH. These variables are populated

libsyclinterface/helper/source/dpctl_error_handlers.cpp

@@ -24,7 +24,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "dpctl_error_handlers.h"
+#include "dpctl_service.h"
 #include <cstring>
+#include <sstream>
+#ifdef ENABLE_GLOG
+#include <glog/logging.h>
+#endif
 
 void DPCTL_AsyncErrorHandler::operator()(
     const cl::sycl::exception_list &exceptions)
@@ -59,6 +64,34 @@ int requested_verbosity_level(void)
 
     return requested_level;
 }
+
+void output_message(std::string ss_str, error_level error_type)
+{
+#ifdef ENABLE_GLOG
+    switch (error_type) {
+    case error_level::error:
+        LOG(ERROR) << "[ERR] " << ss_str;
+        break;
+    case error_level::warning:
+        LOG(WARNING) << "[WARN] " << ss_str;
+        break;
+    default:
+        LOG(FATAL) << "[FATAL] " << ss_str;
+    }
+#else
+    switch (error_type) {
+    case error_level::error:
+        std::cerr << "[ERR] " << ss_str;
+        break;
+    case error_level::warning:
+        std::cerr << "[WARN] " << ss_str;
+        break;
+    default:
+        std::cerr << "[FATAL] " << ss_str;
+    }
+#endif
+}
+
 } // namespace
 
 void error_handler(const std::exception &e,
@@ -70,9 +103,14 @@ void error_handler(const std::exception &e,
     int requested_level = requested_verbosity_level();
     int error_level = static_cast<int>(error_type);
 
-    if (requested_level >= error_level) {
-        std::cerr << e.what() << " in " << func_name << " at " << file_name
-                  << ":" << line_num << std::endl;
+    bool to_output = requested_level >= error_level;
+
+    if (to_output) {
+        std::stringstream ss;
+        ss << e.what() << " in " << func_name << " at " << file_name << ":"
+           << line_num << std::endl;
+
+        output_message(ss.str(), error_type);
     }
 }
 
@@ -85,8 +123,13 @@ void error_handler(const std::string &what,
     int requested_level = requested_verbosity_level();
     int error_level = static_cast<int>(error_type);
 
-    if (requested_level >= error_level) {
-        std::cerr << what << " In " << func_name << " at " << file_name << ":"
-                  << line_num << std::endl;
+    bool to_output = requested_level >= error_level;
+
+    if (to_output) {
+        std::stringstream ss;
+        ss << what << " in " << func_name << " at " << file_name << ":"
+           << line_num << std::endl;
+
+        output_message(ss.str(), error_type);
     }
 }

libsyclinterface/include/dpctl_service.h

@@ -43,4 +43,22 @@ DPCTL_C_EXTERN_C_BEGIN
 DPCTL_API
 __dpctl_give const char *DPCTLService_GetDPCPPVersion(void);
 
+/*!
+ * @brief Initialize logger if compiled to use logger, no-op otherwise.
+ *
+ * @param app_name  C-string for application name reflected in the log.
+ * @paral log_dir   C-string for directory where log files are placed.
+ * @ingroup Service
+ */
+DPCTL_API
+void DPCTLService_InitLogger(const char *app_name, const char *log_dir);
+
+/*!
+ * @brief Finilize logger if enabled, no-op otherwise.
+ *
+ * @ingroup Service
+ */
+DPCTL_API
+void DPCTLService_ShutdownLogger(void);
+
 DPCTL_C_EXTERN_C_END

libsyclinterface/source/dpctl_service.cpp

@@ -26,13 +26,54 @@
 #include "dpctl_service.h"
 #include "Config/dpctl_config.h"
 
-#include "../helper/include/dpctl_string_utils.hpp"
+#include "dpctl_string_utils.hpp"
 #include <algorithm>
 #include <cstring>
 #include <iostream>
+#ifdef ENABLE_GLOG
+#include <filesystem>
+#include <glog/logging.h>
+#endif
 
 __dpctl_give const char *DPCTLService_GetDPCPPVersion(void)
 {
     std::string version = DPCTL_DPCPP_VERSION;
     return dpctl::helper::cstring_from_string(version);
 }
+
+#ifdef ENABLE_GLOG
+
+void DPCTLService_InitLogger(const char *app_name, const char *log_dir)
+{
+    google::InitGoogleLogging(app_name);
+    google::InstallFailureSignalHandler();
+
+    if (log_dir) {
+        namespace fs = std::filesystem;
+        const fs::path path(log_dir);
+        std::error_code ec;
+
+        if (fs::is_directory(path, ec)) {
+            google::EnableLogCleaner(0);
+            FLAGS_log_dir = log_dir;
+        }
+    }
+    else {
+        FLAGS_colorlogtostderr = true;
+        FLAGS_stderrthreshold = google::FATAL;
+        FLAGS_logtostderr = 1;
+    }
+}
+
+void DPCTLService_ShutdownLogger(void)
+{
+    google::ShutdownGoogleLogging();
+}
+
+#else
+void DPCTLService_InitLogger([[maybe_unused]] const char *app_name,
+                             [[maybe_unused]] const char *log_dir){};
+
+void DPCTLService_ShutdownLogger(void){};
+
+#endif