[Linux]: add libev support when CHIP_SYSTEM_CONFIG_USE_LIBEV=1 (#24232)

When CHIP_SYSTEM_CONFIG_USE_LIBEV is set, SystemLayerImplSelect expects a *libev* mainloop to be present to schedule timers and socket watches (similar to CHIP_SYSTEM_CONFIG_USE_DISPATCH for Darwin).

A libev mainloop must be passed to SystemLayer using `SetLibEvLoop()` before any timers or socket watches are used - otherwise, `chipDie()` is invoked.

Platform manager events are also dispatched via the libev mainloop, using
ScheduleWork(). This eliminates the need for a separate thread for
dispatching events, and makes sure event handlers cannot run in parallel
with timers or I/O handlers. In consequence PostEvent() may not be called
without holding the chip lock.

The reason for running *matter* under libev in the first place
is to allow creating single-threaded apps including all of *matter*.
So disallowing PostEvent() "from any thread" is a design choice for
the libev case.

# Usage
The entire project needs to be build with `CHIP_SYSTEM_CONFIG_USE_LIBEV=1` (this can be done via invoking a project-specific extra config via the `default_configs_extra` argument in args.gni)

Setting up the libev mainloop and handing it over to SystemLayer must be done in application specific code, outside the code provided by chip examples. Also adding libev as a dependency must be done in the application's BUILD.gn.

# Background
*libev* is a multi-platform event library often used in embedded linux context to handle events, and builds the basis for various libraries with non-blocking APIs. This changeset allows using the *matter* stack with libev based applications.

# Example
The opensource bridge project p44mbrd (https://github.com/plan44/p44mbrd) is based on libev and makes use of this changeset.

src/include/platform/internal/GenericPlatformManagerImpl_POSIX.h

@@ -65,11 +65,14 @@ class GenericPlatformManagerImpl_POSIX : public GenericPlatformManagerImpl<ImplC
     pthread_t mChipTask;
     bool mInternallyManagedChipTask = false;
     std::atomic<State> mState{ State::kStopped };
+
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
     pthread_cond_t mEventQueueStoppedCond;
     pthread_mutex_t mStateLock;
 
     pthread_attr_t mChipTaskAttr;
     struct sched_param mChipTaskSchedParam;
+#endif
 
 #if CHIP_STACK_LOCK_TRACKING_ENABLED
     bool mChipStackIsLocked = false;
@@ -101,16 +104,25 @@ class GenericPlatformManagerImpl_POSIX : public GenericPlatformManagerImpl<ImplC
 
     inline ImplClass * Impl() { return static_cast<ImplClass *>(this); }
 
-    void ProcessDeviceEvents();
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    static void _DispatchEventViaScheduleWork(System::Layer * aLayer, void * appState);
+#else
 
     DeviceSafeQueue mChipEventQueue;
     std::atomic<bool> mShouldRunEventLoop{ true };
     static void * EventLoopTaskMain(void * arg);
+#endif
+    void ProcessDeviceEvents();
 };
 
 // Instruct the compiler to instantiate the template only when explicitly told to do so.
 extern template class GenericPlatformManagerImpl_POSIX<PlatformManagerImpl>;
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+// with external libev mainloop, this should be implemented externally to terminate the mainloop cleanly
+// (Note that there is a weak default implementation that just calls chipDie() when the external implementation is missing)
+extern void ExitExternalMainLoop();
+#endif
 } // namespace Internal
 } // namespace DeviceLayer
 } // namespace chip

src/include/platform/internal/GenericPlatformManagerImpl_POSIX.ipp

@@ -46,26 +46,31 @@ namespace chip {
 namespace DeviceLayer {
 namespace Internal {
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
 namespace {
 System::LayerSocketsLoop & SystemLayerSocketsLoop()
 {
     return static_cast<System::LayerSocketsLoop &>(DeviceLayer::SystemLayer());
 }
 } // anonymous namespace
+#endif
 
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_InitChipStack()
 {
     // Call up to the base class _InitChipStack() to perform the bulk of the initialization.
     ReturnErrorOnFailure(GenericPlatformManagerImpl<ImplClass>::_InitChipStack());
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+
     mShouldRunEventLoop.store(true, std::memory_order_relaxed);
 
     int ret = pthread_cond_init(&mEventQueueStoppedCond, nullptr);
     VerifyOrReturnError(ret == 0, CHIP_ERROR_POSIX(ret));
 
     ret = pthread_mutex_init(&mStateLock, nullptr);
     VerifyOrReturnError(ret == 0, CHIP_ERROR_POSIX(ret));
+#endif
 
     return CHIP_NO_ERROR;
 }
@@ -130,15 +135,44 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartChipTimer(System::
     return CHIP_NO_ERROR;
 }
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+template <class ImplClass>
+void GenericPlatformManagerImpl_POSIX<ImplClass>::_DispatchEventViaScheduleWork(System::Layer * aLayer, void * appState)
+{
+    auto * event = static_cast<const ChipDeviceEvent *>(appState);
+    PlatformMgrImpl().DispatchEvent(event);
+    delete event;
+}
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_PostEvent(const ChipDeviceEvent * event)
 {
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // Note: PostEvent() is documented to allow being called "from any thread".
+    //   In the libev mainloop case however, calling from another thread is NOT supported.
+    //   Introducing this restriction is OK because the very goal of using libev is to avoid
+    //   multiple threads by running matter and all application code in the same thread on the
+    //   libev mainloop. So getting called from another thread here is very likely a
+    //   application design error.
+    VerifyOrDieWithMsg(_IsChipStackLockedByCurrentThread(), DeviceLayer, "PostEvent() not allowed from outside chip stack lock");
+
+    // Schedule dispatching this event via System Layer's ScheduleWork
+    ChipDeviceEvent * eventCopyP = new ChipDeviceEvent;
+    VerifyOrDie(eventCopyP != nullptr);
+    *eventCopyP = *event;
+    SystemLayer().ScheduleWork(&_DispatchEventViaScheduleWork, eventCopyP);
+    return CHIP_NO_ERROR;
+#else
     mChipEventQueue.Push(*event);
 
     SystemLayerSocketsLoop().Signal(); // Trigger wake select on CHIP thread
     return CHIP_NO_ERROR;
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 template <class ImplClass>
 void GenericPlatformManagerImpl_POSIX<ImplClass>::ProcessDeviceEvents()
 {
@@ -152,6 +186,12 @@ void GenericPlatformManagerImpl_POSIX<ImplClass>::ProcessDeviceEvents()
 template <class ImplClass>
 void GenericPlatformManagerImpl_POSIX<ImplClass>::_RunEventLoop()
 {
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+
+    VerifyOrDieWithMsg(false, DeviceLayer, "libev based app should never try to run a separate event loop");
+
+#else
+
     pthread_mutex_lock(&mStateLock);
 
     //
@@ -203,6 +243,7 @@ void GenericPlatformManagerImpl_POSIX<ImplClass>::_RunEventLoop()
     // Shutdown() method.
     //
     mState.store(State::kStopped, std::memory_order_relaxed);
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
 template <class ImplClass>
@@ -213,9 +254,22 @@ void * GenericPlatformManagerImpl_POSIX<ImplClass>::EventLoopTaskMain(void * arg
     return nullptr;
 }
 
+#endif // !CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartEventLoopTask()
 {
+
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // Note: With libev, we dont need our own mainloop.
+    //   Still, we set State::kRunning to activate lock checking, because
+    //   calls to ScheduleWork and some System Layer methods may not
+    //   occur from other threads (which usually don't exist in a
+    //   libev app)
+    mState.store(State::kRunning, std::memory_order_relaxed);
+    return CHIP_NO_ERROR;
+#else
+
     int err;
     err = pthread_attr_init(&mChipTaskAttr);
     VerifyOrReturnError(err == 0, CHIP_ERROR_POSIX(err));
@@ -247,11 +301,30 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartEventLoopTask()
     pthread_mutex_unlock(&mStateLock);
 
     return CHIP_ERROR_POSIX(err);
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
+}
+
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+// fallback implementation
+void __attribute__((weak)) ExitExternalMainLoop()
+{
+    // FIXME: implement better exit
+    VerifyOrDieWithMsg(false, DeviceLayer, "Missing custom ExitExternalMainLoop() implementation for clean shutdown -> just die");
 }
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StopEventLoopTask()
 {
+
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // with libev, the mainloop is set up and managed externally
+    mState.store(State::kStopping, std::memory_order_relaxed);
+    ExitExternalMainLoop(); // this callback needs to be implemented.
+    mState.store(State::kStopped, std::memory_order_relaxed);
+    return CHIP_NO_ERROR;
+#else
+
     int err = 0;
 
     //
@@ -304,6 +377,7 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StopEventLoopTask()
 
 exit:
     return CHIP_ERROR_POSIX(err);
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
 template <class ImplClass>
@@ -316,8 +390,10 @@ void GenericPlatformManagerImpl_POSIX<ImplClass>::_Shutdown()
     //
     VerifyOrDie(mState.load(std::memory_order_relaxed) == State::kStopped);
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
     pthread_mutex_destroy(&mStateLock);
     pthread_cond_destroy(&mEventQueueStoppedCond);
+#endif
 
     //
     // Call up to the base class _Shutdown() to perform the actual stack de-initialization

src/platform/Darwin/PlatformManagerImpl.cpp

@@ -52,8 +52,10 @@ CHIP_ERROR PlatformManagerImpl::_InitChipStack()
     SuccessOrExit(err);
 #endif // CHIP_DISABLE_PLATFORM_KVS
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
     // Ensure there is a dispatch queue available
     static_cast<System::LayerSocketsLoop &>(DeviceLayer::SystemLayer()).SetDispatchQueue(GetWorkQueue());
+#endif
 
     // Call _InitChipStack() on the generic implementation base class
     // to finish the initialization process.

src/platform/Darwin/SystemPlatformConfig.h

@@ -34,9 +34,14 @@ struct ChipDeviceEvent;
 
 // ==================== Platform Adaptations ====================
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+// FIXME: these should not be hardcoded here, it is set via build config
+// Need to exclude these for now in libev case
 #define CHIP_SYSTEM_CONFIG_POSIX_LOCKING 0
 #define CHIP_SYSTEM_CONFIG_FREERTOS_LOCKING 0
 #define CHIP_SYSTEM_CONFIG_NO_LOCKING 1
+#endif
+
 #define CHIP_SYSTEM_CONFIG_PLATFORM_PROVIDES_TIME 1
 #define CHIP_SYSTEM_CONFIG_USE_POSIX_TIME_FUNCTS 1
 #define CHIP_SYSTEM_CONFIG_POOL_USE_HEAP 1

src/system/BUILD.gn

@@ -85,6 +85,7 @@ buildconfig_header("system_buildconfig") {
     "CHIP_SYSTEM_CONFIG_TEST=${chip_build_tests}",
     "CHIP_WITH_NLFAULTINJECTION=${chip_with_nlfaultinjection}",
     "CHIP_SYSTEM_CONFIG_USE_DISPATCH=${chip_system_config_use_dispatch}",
+    "CHIP_SYSTEM_CONFIG_USE_LIBEV=${chip_system_config_use_libev}",
     "CHIP_SYSTEM_CONFIG_USE_LWIP=${chip_system_config_use_lwip}",
     "CHIP_SYSTEM_CONFIG_USE_OPEN_THREAD_ENDPOINT=${chip_system_config_use_open_thread_inet_endpoints}",
     "CHIP_SYSTEM_CONFIG_USE_SOCKETS=${chip_system_config_use_sockets}",

src/system/SystemLayer.h

@@ -43,7 +43,9 @@
 
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
 #include <dispatch/dispatch.h>
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+#include <ev.h>
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
 #include <utility>
 
@@ -250,7 +252,10 @@ class LayerSocketsLoop : public LayerSockets
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
     virtual void SetDispatchQueue(dispatch_queue_t dispatchQueue) = 0;
     virtual dispatch_queue_t GetDispatchQueue()                   = 0;
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    virtual void SetLibEvLoop(struct ev_loop * aLibEvLoopP) = 0;
+    virtual struct ev_loop * GetLibEvLoop()                 = 0;
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 };
 
 #endif // CHIP_SYSTEM_CONFIG_USE_SOCKETS