Improve error reporting from scale-in protection modification

Author: tevoinea

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -40,6 +40,10 @@ public enum ErrorCode {
     UNABLE_TO_SEND = 483,
     NODE_DELETED = 484,
     TASK_CANCELLED = 485,
+    SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS = 486,
+    SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS = 487,
+    SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT = 488,
+    SCALE_IN_PROTECTION_UNEXPECTED_ERROR = 489,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -113,7 +113,14 @@ await TryGetNodeInfo(node) is NodeInfo nodeInfo) {
 
             var r = await _context.VmssOperations.UpdateScaleInProtection(nodeInfo.Scaleset, instanceId, protectFromScaleIn: true);
             if (!r.IsOk) {
-                _logTracer.LogOneFuzzError(r.ErrorV);
+                switch (r.ErrorV.Code) {
+                    case ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS:
+                        _logTracer.LogWarning("Transiently failed to modify scale-in protection: {}", r.ErrorV);
+                        break;
+                    default:
+                        _logTracer.LogOneFuzzError(r.ErrorV);
+                        break;
+                }
                 _logTracer.LogMetric("FailedAcquiringScaleInProtection", 1);
                 return r.ErrorV;
             }
@@ -148,7 +155,15 @@ await TryGetNodeInfo(node) is NodeInfo nodeInfo) {
 
             var r = await _context.VmssOperations.UpdateScaleInProtection(nodeInfo.Scaleset, instanceId, protectFromScaleIn: false);
             if (!r.IsOk) {
-                _logTracer.LogOneFuzzError(r.ErrorV);
+                switch (r.ErrorV.Code) {
+                    case ErrorCode.SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS:
+                    case ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS:
+                        _logTracer.LogWarning("Transiently failed to modify scale-in protection: {}", r.ErrorV);
+                        break;
+                    default:
+                        _logTracer.LogOneFuzzError(r.ErrorV);
+                        break;
+                }
                 _logTracer.LogMetric("FailedReleasingScaleInProtection", 1);
                 return r;
             }

src/ApiService/ApiService/onefuzzlib/VmssOperations.cs

@@ -253,14 +253,14 @@ public async Async.Task<OneFuzzResultVoid> UpdateScaleInProtection(Scaleset scal
             _ = await vmCollection.CreateOrUpdateAsync(WaitUntil.Started, instanceId, data);
             return OneFuzzResultVoid.Ok;
         } catch (RequestFailedException ex) when (ex.Status == 409 && ex.Message.StartsWith("The request failed due to conflict with a concurrent request")) {
-            return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"protection policy update is already in progress: {instanceId} in vmss {scaleset.ScalesetId}");
+            return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS, $"protection policy update is already in progress: {instanceId} in vmss {scaleset.ScalesetId}");
         } catch (RequestFailedException ex) when (ex.Status == 400 && ex.Message.Contains("The provided instanceId") && ex.Message.Contains("not an active Virtual Machine Scale Set VM instanceId.")) {
-            return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"The node with instanceId {instanceId} no longer exists in scaleset {scaleset.ScalesetId}");
+            return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS, $"The node with instanceId {instanceId} no longer exists in scaleset {scaleset.ScalesetId}");
         } catch (RequestFailedException ex) when (ex.Status == 400 && ex.Message.Contains("reached its limit") && ex.Message.Contains("Upgrade the VMs to the latest model")) {
-            return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"VMSS has reached model limit. Could not update scaling protection for scaleset {scaleset.ScalesetId}.");
+            return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT, $"VMSS has reached model limit. Could not update scaling protection for scaleset {scaleset.ScalesetId}.");
         } catch (Exception ex) {
             _log.LogError(ex, "unable to set protection policy on: {InstanceId} in vmss {ScalesetId}", instanceId, scaleset.ScalesetId);
-            return OneFuzzResultVoid.Error(ErrorCode.UNABLE_TO_UPDATE, $"unable to set protection policy on: {instanceId} in vmss {scaleset.ScalesetId}");
+            return OneFuzzResultVoid.Error(ErrorCode.SCALE_IN_PROTECTION_UNEXPECTED_ERROR, $"unable to set protection policy on: {instanceId} in vmss {scaleset.ScalesetId}");
         }
 
     }

src/pytypes/onefuzztypes/enums.py

@@ -292,6 +292,10 @@ class ErrorCode(Enum):
     UNABLE_TO_SEND = 483
     NODE_DELETED = 484
     TASK_CANCELLED = 485
+    SCALE_IN_PROTECTION_UPDATE_ALREADY_IN_PROGRESS = 486
+    SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS = 487
+    SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT = 488
+    SCALE_IN_PROTECTION_UNEXPECTED_ERROR = 489
     # NB: if you update this enum, also update Enums.cs