File change monitoring on s3_auth

configure.ac

@@ -1808,6 +1808,10 @@ AC_LANG_POP([C++])
 # Right now, the healthcheck plugins requires inotify_init (and friends)
 AM_CONDITIONAL([BUILD_HEALTHCHECK_PLUGIN], [ test "$ac_cv_func_inotify_init" = "yes" ])
 
+use_inotify=0
+AS_IF([test "x$ac_cv_func_inotify_init" = "xyes"], [use_inotify=1])
+AC_SUBST(use_inotify)
+
 #
 # Check for tcmalloc, jemalloc and mimalloc
 TS_CHECK_TCMALLOC
@@ -2252,7 +2256,8 @@ iocore_include_dirs="\
 -I\$(abs_top_srcdir)/iocore/hostdb \
 -I\$(abs_top_srcdir)/iocore/cache \
 -I\$(abs_top_srcdir)/iocore/utils \
--I\$(abs_top_srcdir)/iocore/dns"
+-I\$(abs_top_srcdir)/iocore/dns \
+-I\$(abs_top_srcdir)/iocore/fs"
 
 AC_SUBST([AM_CPPFLAGS])
 AC_SUBST([AM_CFLAGS])
@@ -2300,6 +2305,7 @@ AC_CONFIG_FILES([
   iocore/cache/Makefile
   iocore/dns/Makefile
   iocore/eventsystem/Makefile
+  iocore/fs/Makefile
   iocore/hostdb/Makefile
   iocore/net/Makefile
   iocore/net/quic/Makefile

doc/admin-guide/plugins/s3_auth.en.rst

@@ -44,7 +44,7 @@ Alternatively, you can store the access key and secret in an external configurat
 
    # remap.config
 
-   ...  @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v2.config
+   ...  @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v2.config @pparam=--watch-config @pparam=--ttl=5
 
 
 Where ``s3.config`` could look like::
@@ -94,6 +94,8 @@ The ``s3_auth_v4.config`` config file could look like this::
     v4-include-headers=<comma-separated-list-of-headers-to-be-signed>
     v4-exclude-headers=<comma-separated-list-of-headers-not-to-be-signed>
     v4-region-map=region_map.config
+    watch-config
+    ttl=20
 
 Where the ``region_map.config`` defines the entry-point hostname to region mapping i.e.::
 
@@ -123,6 +125,10 @@ If ``--v4-include-headers`` is not specified all headers except those specified
 
 If ``--v4-include-headers`` is specified only the headers specified will be signed except those specified in ``--v4-exclude-headers``
 
+If ``--watch-config`` is specified, the plugin will reload the config file set in ``--config`` when it changes
+
+If ``--ttl`` is specified, the plugin will cache configs for the specified number of seconds.  During the ttl period, manual config reloads and ``--watch-config`` will not cause the config to be updated.  The default is 60 seconds.  Setting ttl to zero causes all reloads to read from the config file.  This option is useful if the config file is fetched from a service, and you wish to limit the fetch rate.
+
 
 AWS Authentication version 2
 ============================

doc/developer-guide/testing/blackbox-testing.en.rst

@@ -262,6 +262,7 @@ Condition Testing
         - TS_HAS_128BIT_CAS
         - TS_HAS_TESTS
         - TS_HAS_WCCP
+        - TS_USE_INOTIFY
 
 
 Examples:

include/ts/apidefs.h.in

@@ -554,6 +554,11 @@ typedef enum {
   TS_EVENT_SSL_CLIENT_HELLO  = 60207,
   TS_EVENT_SSL_SECRET        = 60208,
 
+  TS_EVENT_FILE_CREATED = 60300,
+  TS_EVENT_FILE_UPDATED = 60301,
+  TS_EVENT_FILE_DELETED = 60302,
+  TS_EVENT_FILE_IGNORED = 60303,
+
   TS_EVENT_MGMT_UPDATE = 60300
 } TSEvent;
 #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */
@@ -1454,6 +1459,13 @@ namespace ts
 }
 #endif
 
+typedef int TSWatchDescriptor;
+typedef enum { TS_WATCH_CREATE, TS_WATCH_DELETE, TS_WATCH_MODIFY } TSFileWatchKind;
+typedef struct {
+  TSWatchDescriptor wd;
+  const char *name;
+} TSFileWatchData;
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */

include/ts/ts.h

@@ -2720,6 +2720,23 @@ tsapi void TSHostStatusSet(const char *hostname, const size_t hostname_len, TSHo
 tsapi bool TSHttpTxnCntlGet(TSHttpTxn txnp, TSHttpCntlType ctrl);
 tsapi TSReturnCode TSHttpTxnCntlSet(TSHttpTxn txnp, TSHttpCntlType ctrl, bool data);
 
+/*
+ * Get notified for file system events
+ *
+ * Currently, this only works in Linux using inotify.
+ *
+ * TODO: Fix multiple plugins watching the same path.
+ *
+ * The edata (a.k.a. cookie) field of the continuation handler will contain information
+ * depending on the type of file event.  edata is always a pointer to a TSFileWatchData.
+ * If the event is TS_EVENT_FILE_CREATED, name is a pointer to a null-terminated string
+ * containing the file name.  Otherwise, name is a nullptr.  wd is the watch descriptor
+ * for the event.
+ *
+ */
+tsapi TSWatchDescriptor TSFileEventRegister(const char *filename, TSFileWatchKind kind, TSCont contp);
+tsapi void TSFileEventUnRegister(TSWatchDescriptor wd);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */

include/tscore/ink_config.h.in

@@ -83,6 +83,7 @@
 #define TS_HAS_TLS_EARLY_DATA @has_tls_early_data@
 #define TS_HAS_TLS_SESSION_TICKET @has_tls_session_ticket@
 #define TS_HAS_VERIFY_CERT_STORE @has_verify_cert_store@
+#define TS_USE_INOTIFY @use_inotify@
 
 #define TS_USE_HRW_GEOIP @use_hrw_geoip@
 #define TS_USE_HRW_MAXMINDDB @use_hrw_maxminddb@

iocore/Makefile.am

@@ -16,4 +16,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-SUBDIRS = eventsystem net aio dns hostdb utils cache
+SUBDIRS = eventsystem net aio dns hostdb utils cache fs

iocore/fs/FileChange.cc

@@ -0,0 +1,227 @@
+/** @file FileChange.cc
+
+  Watch for file system changes.
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "FileChange.h"
+#include "tscore/Diags.h"
+#include "P_EventSystem.h"
+
+#include <cassert>
+#include <functional>
+#include <mutex>
+#include <optional>
+
+// Globals
+FileChangeManager fileChangeManager;
+static constexpr auto TAG ATS_UNUSED = "FileChange";
+
+// Wrap a continuation
+class FileChangeCallback : public Continuation
+{
+public:
+  explicit FileChangeCallback(Continuation *contp, TSEvent event) : Continuation(contp->mutex.get()), m_cont(contp), m_event(event)
+  {
+    SET_HANDLER(&FileChangeCallback::event_handler);
+  }
+
+  int
+  event_handler(int, void *eventp)
+  {
+    Event *e = reinterpret_cast<Event *>(eventp);
+    if (m_cont->mutex) {
+      MUTEX_TRY_LOCK(trylock, m_cont->mutex, this_ethread());
+      if (!trylock.is_locked()) {
+        eventProcessor.schedule_in(this, HRTIME_MSECONDS(10), ET_TASK);
+      } else {
+        m_cont->handleEvent(m_event, e->cookie);
+        delete this;
+      }
+    } else {
+      m_cont->handleEvent(m_event, e->cookie);
+      delete this;
+    }
+
+    return 0;
+  }
+
+  std::string filename; // File name if the event is a file creation event.  This is used in the cookie for a create event.
+  TSFileWatchData data;
+
+private:
+  Continuation *m_cont;
+  TSEvent m_event;
+};
+
+#if TS_USE_INOTIFY
+static constexpr size_t INOTIFY_BUF_SIZE = 4096;
+
+static void
+invoke(FileChangeCallback *cb)
+{
+  void *cookie = static_cast<void *>(&cb->data);
+  eventProcessor.schedule_imm(cb, ET_TASK, 1, cookie);
+}
+
+void
+FileChangeManager::process_file_event(struct inotify_event *event)
+{
+  std::shared_lock file_watches_read_lock(file_watches_mutex);
+  auto finfo_it = file_watches.find(event->wd);
+  if (finfo_it != file_watches.end()) {
+    TSEvent event_type            = TS_EVENT_NONE;
+    const struct file_info &finfo = finfo_it->second;
+    Continuation *contp           = finfo.contp;
+
+    if (event->mask & (IN_DELETE_SELF | IN_MOVED_FROM)) {
+      Debug(TAG, "Delete file event (%d) on %s", event->mask, finfo.path.c_str());
+      int rc2 = inotify_rm_watch(inotify_fd, event->wd);
+      if (rc2 == -1) {
+        Error("Failed to remove inotify watch on %s: %s (%d)", finfo.path.c_str(), strerror(errno), errno);
+      }
+      event_type             = TS_EVENT_FILE_DELETED;
+      FileChangeCallback *cb = new FileChangeCallback(contp, event_type);
+      cb->data.wd            = event->wd;
+      cb->data.name          = nullptr;
+      invoke(cb);
+    }
+
+    if (event->mask & (IN_CREATE | IN_MOVED_TO)) {
+      // Name may be padded with nul characters.  Trim them.
+      auto len = strnlen(event->name, event->len);
+      std::string name{event->name, len};
+      Debug(TAG, "Create file event (%d) on %s (wd = %d): %s", event->mask, finfo.path.c_str(), event->wd, name.c_str());
+      event_type = TS_EVENT_FILE_CREATED;
+
+      FileChangeCallback *cb = new FileChangeCallback(contp, event_type);
+      cb->filename           = name;
+      cb->data.wd            = event->wd;
+      cb->data.name          = cb->filename.c_str();
+      invoke(cb);
+    }
+
+    if (event->mask & (IN_CLOSE_WRITE | IN_ATTRIB)) {
+      Debug(TAG, "Modify file event (%d) on %s (wd = %d)", event->mask, finfo.path.c_str(), event->wd);
+      event_type             = TS_EVENT_FILE_UPDATED;
+      FileChangeCallback *cb = new FileChangeCallback(contp, event_type);
+      cb->data.wd            = event->wd;
+      cb->data.name          = nullptr;
+      invoke(cb);
+    }
+
+    if (event->mask & (IN_IGNORED)) {
+      Debug(TAG, "Ignored file event (%d) on %s (wd = %d)", event->mask, finfo.path.c_str(), event->wd);
+      event_type             = TS_EVENT_FILE_IGNORED;
+      FileChangeCallback *cb = new FileChangeCallback(contp, event_type);
+      cb->data.wd            = event->wd;
+      cb->data.name          = nullptr;
+      invoke(cb);
+    }
+  }
+}
+#endif
+
+void
+FileChangeManager::init()
+{
+#if TS_USE_INOTIFY
+  // TODO: auto configure based on whether inotify is available
+  inotify_fd = inotify_init1(IN_CLOEXEC);
+  if (inotify_fd == -1) {
+    Error("Failed to init inotify: %s (%d)", strerror(errno), errno);
+    return;
+  }
+  auto inotify_thread = [manager = this]() mutable {
+    for (;;) {
+      char inotify_buf[INOTIFY_BUF_SIZE];
+
+      // blocking read
+      ssize_t rc = read(manager->inotify_fd, inotify_buf, sizeof inotify_buf);
+
+      if (rc == -1) {
+        Error("Failed to read inotify: %s (%d)", strerror(errno), errno);
+        if (errno == EINTR) {
+          continue;
+        } else {
+          break;
+        }
+      }
+
+      ssize_t offset = 0;
+      while (offset < rc) {
+        struct inotify_event *event = reinterpret_cast<struct inotify_event *>(inotify_buf + offset);
+
+        // Process file events
+        manager->process_file_event(event);
+        offset += sizeof(struct inotify_event) + event->len;
+      }
+    }
+  };
+  poll_thread = std::thread(inotify_thread);
+  poll_thread.detach();
+#else
+  // Implement this
+#endif
+}
+
+watch_handle_t
+FileChangeManager::add(const ts::file::path &path, TSFileWatchKind kind, Continuation *contp)
+{
+#if TS_USE_INOTIFY
+  Debug(TAG, "Adding a watch on %s", path.c_str());
+  watch_handle_t wd = 0;
+
+  // Let the OS handle multiple watches on one file.
+  uint32_t mask = 0;
+  if (kind == TS_WATCH_CREATE) {
+    mask = IN_CREATE | IN_MOVED_TO | IN_ONLYDIR;
+  } else if (kind == TS_WATCH_DELETE) {
+    mask = IN_DELETE_SELF | IN_MOVED_FROM;
+  } else if (kind == TS_WATCH_MODIFY) {
+    mask = IN_CLOSE_WRITE | IN_ATTRIB;
+  }
+  wd = inotify_add_watch(inotify_fd, path.c_str(), mask);
+  if (wd == -1) {
+    Error("Failed to add file watch on %s: %s (%d)", path.c_str(), strerror(errno), errno);
+    return -1;
+  } else {
+    std::unique_lock file_watches_write_lock(file_watches_mutex);
+    file_watches[wd] = {path, contp};
+  }
+
+  Debug(TAG, "Watch handle = %d", wd);
+  return wd;
+#else
+  Warning("File change notification is not supported on this OS.");
+  return 0;
+#endif
+}
+
+void
+FileChangeManager::remove(watch_handle_t watch_handle)
+{
+#if TS_USE_INOTIFY
+  Debug(TAG, "Deleting watch %d", watch_handle);
+  inotify_rm_watch(inotify_fd, watch_handle);
+  std::unique_lock file_watches_write_lock(file_watches_mutex);
+  file_watches.erase(watch_handle);
+#endif
+}