feat: Support "cancelled" and "failed" Shutdown operations [MLG-468]

harness/determined/searcher/_search_method.py

@@ -146,11 +146,15 @@ class Shutdown(Operation):
     Operation shutting the experiment down
     """
 
-    def __init__(self) -> None:
+    def __init__(self, cancel: bool = False, failure: bool = False) -> None:
         super().__init__()
+        self.cancel = cancel
+        self.failure = failure
 
     def _to_searcher_operation(self) -> bindings.v1SearcherOperation:
-        return bindings.v1SearcherOperation(shutDown=bindings.v1ShutDownOperation())
+        return bindings.v1SearcherOperation(
+            shutDown=bindings.v1ShutDownOperation(cancel=self.cancel, failure=self.failure)
+        )
 
 
 class Create(Operation):

master/internal/experiment.go

@@ -454,7 +454,12 @@ func (e *experiment) Receive(ctx *actor.Context) error {
 					ops = append(ops, *op)
 				}
 			case *experimentv1.SearcherOperation_ShutDown:
-				ops = append(ops, searcher.NewShutdown())
+				op, err := searcher.ShutdownFromProto(concreteOperation)
+				if err != nil {
+					ctx.Log().Error(err)
+				} else {
+					ops = append(ops, *op)
+				}
 			case *experimentv1.SearcherOperation_TrialOperation:
 				switch sub := concreteOperation.TrialOperation.GetUnion().(type) {
 				case *experimentv1.TrialOperation_ValidateAfter:

master/pkg/searcher/operations.go

@@ -280,6 +280,16 @@ func NewShutdown() Shutdown {
 	return Shutdown{}
 }
 
+// ShutdownFromProto creates a Shutdown from its protobuf representation.
+func ShutdownFromProto(
+	op *experimentv1.SearcherOperation_ShutDown,
+) (*Shutdown, error) {
+	return &Shutdown{
+		Cancel:  op.ShutDown.Cancel,
+		Failure: op.ShutDown.Failure,
+	}, nil
+}
+
 func (shutdown Shutdown) String() string {
-	return "{Shutdown}"
+	return fmt.Sprintf("{Shutdown Cancel: %v Failure: %v}", shutdown.Cancel, shutdown.Failure)
 }

proto/src/determined/experiment/v1/searcher.proto

@@ -165,7 +165,12 @@ message CloseTrialOperation {
 // Shut down custom searcher method.
 message ShutDownOperation {
   // Cannot have an empty message type.
+  // Deprecated: But maintained here for backwards compatibility
   int32 placeholder = 1;
+  // Defines whether the Searcher was cancelled
+  bool cancel = 2;
+  // Defines whether the Searcher failed
+  bool failure = 3;
 }
 
 // SearcherOperation is an operation issued by the custom searcher.