TakeoverClient: add simulated errors for testing

Summary:
# This diff

Adds a new config that will force the TakeoverClient to crash prior to receiving takeover data from the existing server.

# Context

Crashes during graceful restart are plaguing macOS. Although these happen in the background, these greatly affect users because they cause Eden daemons to exit uncleanly. Because of the unclean exit, mount points get left in the mount table and cause all sorts of issues. The biggest issues are:

1) Any attempts to perform IO into the mount point fail
2) Any subsequent attempts to remount the mount point fail. These attempts fail because of #1, and Eden's startup/mount codepaths haven't been taught how to deal with this.

The goal of this stack is to:

1) Introduce a mechanism to easily repro graceful restart crashes (both crashes that occur in the TakeoverClient and TakeoverServer)
2) Use the new mechanism to troubleshoot issue #2 above
3) Use the learnings from #2 to make EdenFS automatically recover from graceful restart crashes (unmount stale mounts and successfully remount any mounts that were mounted prior to the crash).

Reviewed By: jdelliot

Differential Revision: D72490700

fbshipit-source-id: f86b7d1317ce8b5accbca3c6102d83e277b213ce

Author: MichaelCuevas

eden/fs/config/EdenConfig.h

@@ -231,6 +231,18 @@ class EdenConfig : private ConfigSettingManager {
       12,
       this};
 
+  /**
+   * Whether graceful takeover client should purposefully return an exception
+   * during the takeover process. Intended to be used only in integration tests.
+   *
+   * NOTE: if you want the takeover server to throw an exception, you can use
+   * FaultInjector with the key/value pair: ("takeover", "error during send")
+   */
+  ConfigSetting<bool> clientThrowsDuringTakeover{
+      "core:client-throws-during-takeover",
+      false,
+      this};
+
   // [config]
 
   /**

eden/fs/service/EdenServer.cpp

@@ -1143,6 +1143,12 @@ Future<Unit> EdenServer::prepare(std::shared_ptr<StartupLogger> logger) {
           [this] { runningState_.wlock()->state = RunState::RUNNING; });
 }
 
+namespace {
+bool shouldThrowDuringTakeover(const EdenConfig& config) {
+  return config.clientThrowsDuringTakeover.getValue();
+}
+} // namespace
+
 Future<Unit> EdenServer::prepareImpl(std::shared_ptr<StartupLogger> logger) {
   bool doingTakeover = false;
   if (!edenDir_.acquireLock()) {
@@ -1177,7 +1183,9 @@ Future<Unit> EdenServer::prepareImpl(std::shared_ptr<StartupLogger> logger) {
     logger->log(
         "Requesting existing edenfs process to gracefully "
         "transfer its mount points...");
-    takeoverData = takeoverMounts(takeoverPath);
+    takeoverData = takeoverMounts(
+        takeoverPath,
+        shouldThrowDuringTakeover(*serverState_->getEdenConfig()));
     logger->log(
         "Received takeover information for ",
         takeoverData.mountPoints.size(),

eden/fs/takeover/TakeoverClient.cpp

@@ -13,6 +13,7 @@
 #include <folly/io/async/EventBase.h>
 #include <folly/logging/xlog.h>
 #include <thrift/lib/cpp2/protocol/Serializer.h>
+#include "eden/common/utils/FaultInjector.h"
 #include "eden/common/utils/FutureUnixSocket.h"
 #include "eden/fs/takeover/TakeoverData.h"
 #include "eden/fs/takeover/gen-cpp2/takeover_types.h"
@@ -33,6 +34,7 @@ namespace facebook::eden {
 
 TakeoverData takeoverMounts(
     AbsolutePathPiece socketPath,
+    bool shouldThrowDuringTakeover,
     bool shouldPing,
     const std::set<int32_t>& supportedVersions,
     const uint64_t supportedTakeoverCapabilities) {
@@ -61,17 +63,32 @@ TakeoverData takeoverMounts(
         auto timeout = std::chrono::seconds(FLAGS_takeoverReceiveTimeout);
         return socket.receive(timeout);
       })
-      .thenValue([&socket, shouldPing](UnixSocket::Message&& msg) {
+      .thenValue([&socket, shouldPing, shouldThrowDuringTakeover](
+                     UnixSocket::Message&& msg) mutable {
         if (TakeoverData::isPing(&msg.data)) {
           if (shouldPing) {
             // Just send an empty message back here, the server knows it sent a
             // ping so it does not need to parse the message.
             UnixSocket::Message ping;
-            return socket.send(std::move(ping)).thenValue([&socket](auto&&) {
-              // Wait for the takeover data response
-              auto timeout = std::chrono::seconds(FLAGS_takeoverReceiveTimeout);
-              return socket.receive(timeout);
-            });
+            return socket.send(std::move(ping))
+                .thenValue([&socket,
+                            shouldThrowDuringTakeover](auto&&) mutable {
+                  // Possibly simulate a takeover error during data transfer
+                  // for testing purposes. While we would prefer to use
+                  // fault injection here, it's not possible to inject an
+                  // error into the TakeoverClient because the thrift server
+                  // is not yet running.
+                  if (shouldThrowDuringTakeover) {
+                    // throw std::runtime_error("simulated takeover error");
+                    return folly::makeFuture<UnixSocket::Message>(
+                        folly::exception_wrapper(
+                            std::runtime_error("simulated takeover error")));
+                  }
+                  // Wait for the takeover data response
+                  auto timeout =
+                      std::chrono::seconds(FLAGS_takeoverReceiveTimeout);
+                  return socket.receive(timeout);
+                });
           } else {
             // This should only be hit during integration tests.
             return folly::makeFuture<UnixSocket::Message>(

eden/fs/takeover/TakeoverClient.h

@@ -21,6 +21,7 @@ TakeoverData takeoverMounts(
     AbsolutePathPiece socketPath,
     // the following parameters are present for testing purposes and should not
     // normally be used in the production build.
+    bool shouldThrowDuringTakeover = false,
     bool shouldPing = true,
     const std::set<int32_t>& supportedTakeoverVersions =
         kSupportedTakeoverVersions,

eden/fs/takeover/test/TakeoverTest.cpp

@@ -79,6 +79,7 @@ Future<TakeoverData> takeoverViaEventBase(
     promise.setWith([&] {
       return takeoverMounts(
           path,
+          /*shouldThrowDuringTakeover=*/false,
           /*shouldPing=*/true,
           supportedVersions,
           supportedCapabilities);

eden/integration/helpers/TakeoverTool.cpp

@@ -29,6 +29,11 @@ DEFINE_bool(
     true,
     "This is used by integration tests to avoid sending a ping");
 
+DEFINE_bool(
+    shouldThrowDuringTakeover,
+    false,
+    "This can be used by tests to initiate an error in the TakeoverClient");
+
 using namespace facebook::eden::path_literals;
 
 /*
@@ -55,11 +60,15 @@ int main(int argc, char* argv[]) {
 
   facebook::eden::TakeoverData data;
   if (FLAGS_takeoverVersion == 0) {
-    data = facebook::eden::takeoverMounts(takeoverSocketPath, FLAGS_shouldPing);
+    data = facebook::eden::takeoverMounts(
+        takeoverSocketPath, FLAGS_shouldThrowDuringTakeover, FLAGS_shouldPing);
   } else {
     auto takeoverVersion = std::set<int32_t>{FLAGS_takeoverVersion};
     data = facebook::eden::takeoverMounts(
-        takeoverSocketPath, FLAGS_shouldPing, takeoverVersion);
+        takeoverSocketPath,
+        FLAGS_shouldThrowDuringTakeover,
+        FLAGS_shouldPing,
+        takeoverVersion);
   }
   for (const auto& mount : data.mountPoints) {
     const folly::File* mountFD = nullptr;