feat(volume): remount volume if it becomes read only

ref: longhorn/longhorn 7843 Signed-off-by: Jack Lin <jack.lin@suse.com>
longhorn · Feb 20, 2024 · 041ef52 · 041ef52
1 parent ed48cfd
commit 041ef52
Show file tree

Hide file tree

Showing 13 changed files with 288 additions and 166 deletions.
diff --git a/controller/volume_controller.go b/controller/volume_controller.go
@@ -1651,10 +1651,24 @@ func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume
 		}
 
 		if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {
-			v.Status.RemountRequestedAt = c.nowHandler()
-			log.Infof("Volume request remount at %v due to engine detected read-only filesystem", v.Status.RemountRequestedAt)
-			msg := fmt.Sprintf("Volume %s requested remount at %v due to engine detected read-only filesystem", v.Name, v.Status.RemountRequestedAt)
-			c.eventRecorder.Eventf(v, corev1.EventTypeNormal, constant.EventReasonRemount, msg)
+			log.Infof("Auto remount volume to read write at due to engine detected read-only filesystem")
+			engineCliClient, err := engineapi.GetEngineBinaryClient(c.ds, v.Name, c.controllerID)
+			if err != nil {
+				log.WithError(err).Warnf("Failed to get engineCliClient when remounting read only volume")
+				return
+			}
+
+			engineClientProxy, err := engineapi.GetCompatibleClient(e, engineCliClient, c.ds, c.logger, c.proxyConnCounter)
+			if err != nil {
+				log.WithError(err).Warnf("Failed to get engineClientProxy when remounting read only volume")
+				return
+			}
+			defer engineClientProxy.Close()
+
+			if err := engineClientProxy.RemountReadOnlyVolume(e); err != nil {
+				log.WithError(err).Warnf("Failed to remount read only volume")
+				return
+			}
 		}
 	}
 }

diff --git a/engineapi/engine.go b/engineapi/engine.go
@@ -331,6 +331,10 @@ func (e *EngineBinary) MetricsGet(*longhorn.Engine) (*Metrics, error) {
 	return nil, fmt.Errorf(ErrNotImplement)
 }
 
+func (e *EngineBinary) RemountReadOnlyVolume(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}
+
 // addFlags always adds required flags to args. In addition, if the engine version is high enough, it adds additional
 // engine identity validation flags.
 func (e *EngineBinary) addFlags(args []string) ([]string, error) {

diff --git a/engineapi/enginesim.go b/engineapi/enginesim.go
@@ -268,3 +268,7 @@ func (e *EngineSimulator) MetricsGet(*longhorn.Engine) (*Metrics, error) {
 func (e *EngineSimulator) CleanupBackupMountPoints() error {
 	return fmt.Errorf(ErrNotImplement)
 }
+
+func (e *EngineSimulator) RemountReadOnlyVolume(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}
diff --git a/engineapi/proxy_volume.go b/engineapi/proxy_volume.go
@@ -43,3 +43,7 @@ func (p *Proxy) VolumeUnmapMarkSnapChainRemovedSet(e *longhorn.Engine) error {
 	return p.grpcClient.VolumeUnmapMarkSnapChainRemovedSet(string(e.Spec.BackendStoreDriver), e.Name, e.Spec.VolumeName,
 		p.DirectToURL(e), e.Spec.UnmapMarkSnapChainRemovedEnabled)
 }
+
+func (p *Proxy) RemountReadOnlyVolume(e *longhorn.Engine) error {
+	return p.grpcClient.RemountReadOnlyVolume(e.Spec.VolumeName)
+}
diff --git a/engineapi/types.go b/engineapi/types.go
@@ -106,6 +106,7 @@ type EngineClient interface {
 	CleanupBackupMountPoints() error
 
 	MetricsGet(engine *longhorn.Engine) (*Metrics, error)
+	RemountReadOnlyVolume(engine *longhorn.Engine) error
 }
 
 type EngineClientRequest struct {

diff --git a/go.mod b/go.mod
@@ -49,7 +49,7 @@ require (
 	github.com/longhorn/go-iscsi-helper v0.0.0-20231113050545-9df1e6b605c7
 	github.com/longhorn/go-spdk-helper v0.0.0-20231113055029-9acddd184246
 	github.com/longhorn/longhorn-engine v1.5.2-0.20230906182804-9475221d9535
-	github.com/longhorn/longhorn-instance-manager v1.5.4-rc1.0.20240206081407-9d69ea205aec
+	github.com/longhorn/longhorn-instance-manager v1.5.4-rc2.0.20240220083528-fa10cc97578b
 	github.com/longhorn/longhorn-share-manager v1.5.3-rc1.0.20231115141215-5109d5939e17
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0

diff --git a/go.sum b/go.sum
@@ -1034,8 +1034,8 @@ github.com/longhorn/go-spdk-helper v0.0.0-20231113055029-9acddd184246 h1:Pry2QHJ
 github.com/longhorn/go-spdk-helper v0.0.0-20231113055029-9acddd184246/go.mod h1:PCawMRVllTJnjbIYIC8axRfEI7aEhhjbsoBS/Y/EUYQ=
 github.com/longhorn/longhorn-engine v1.5.2-0.20230906182804-9475221d9535 h1:mfkKctSnweL36Y2gUxQoXDyfJ72CsAhwCgG4I2vhHMw=
 github.com/longhorn/longhorn-engine v1.5.2-0.20230906182804-9475221d9535/go.mod h1:OOjKIyGpTxmZFWkn8N1G+H4pga0niuMgzrR/h5HUJNo=
-github.com/longhorn/longhorn-instance-manager v1.5.4-rc1.0.20240206081407-9d69ea205aec h1:1E4Zqsf44EnsQuLbCB8ldU1IR5SGAhErtTCazy7cuno=
-github.com/longhorn/longhorn-instance-manager v1.5.4-rc1.0.20240206081407-9d69ea205aec/go.mod h1:gkU0l28b5QDmUPiO2WL1IHwxkUyvW2dhpIN52FNlrww=
+github.com/longhorn/longhorn-instance-manager v1.5.4-rc2.0.20240220083528-fa10cc97578b h1:aqBNcYL9JVtkVeBKC90MB+aHbMx9pkT6BsO5iQlcYd0=
+github.com/longhorn/longhorn-instance-manager v1.5.4-rc2.0.20240220083528-fa10cc97578b/go.mod h1:gkU0l28b5QDmUPiO2WL1IHwxkUyvW2dhpIN52FNlrww=
 github.com/longhorn/longhorn-share-manager v1.5.3-rc1.0.20231115141215-5109d5939e17 h1:0aCMBoX4nTKpHBFD1lwcRaRc8oQcr6bqfFEIp9VpRYw=
 github.com/longhorn/longhorn-share-manager v1.5.3-rc1.0.20231115141215-5109d5939e17/go.mod h1:2Ti49HwYOBt5O9aU3Eminw0aiu1e3S0qdn7yByI+C9Q=
 github.com/longhorn/longhorn-spdk-engine v0.0.0-20231005170812-e9b634e07e47 h1:hadV15fSF+akBFZ8dMQFscTsJUVpEC1t6Go3B28ZNYw=

diff --git a/vendor/github.com/longhorn/longhorn-instance-manager/pkg/client/proxy_volume.go b/vendor/github.com/longhorn/longhorn-instance-manager/pkg/client/proxy_volume.go