feat(volume): remount volume if it becomes read only

ref: longhorn/longhorn 7843

Signed-off-by: Jack Lin <jack.lin@suse.com>

controller/volume_controller.go

@@ -1521,10 +1521,25 @@ func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume
 		}
 
 		if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {
-			v.Status.RemountRequestedAt = c.nowHandler()
-			log.Infof("Volume request remount at %v due to engine detected read-only filesystem", v.Status.RemountRequestedAt)
-			msg := fmt.Sprintf("Volume %s requested remount at %v due to engine detected read-only filesystem", v.Name, v.Status.RemountRequestedAt)
-			c.eventRecorder.Eventf(v, corev1.EventTypeNormal, constant.EventReasonRemount, msg)
+			log.Infof("Auto remount volume to read write at due to engine detected read-only filesystem")
+			engineCliClient, err := engineapi.GetEngineBinaryClient(c.ds, v.Name, c.controllerID)
+			if err != nil {
+				log.WithError(err).Warnf("Failed to get engineCliClient when remounting read only volume")
+				return
+			}
+
+			engineClientProxy, err := engineapi.GetCompatibleClient(e, engineCliClient, c.ds, c.logger, c.proxyConnCounter)
+			if err != nil {
+				log.WithError(err).Warnf("Failed to get engineClientProxy when remounting read only volume")
+				return
+			}
+			defer engineClientProxy.Close()
+
+			if err := engineClientProxy.RemountReadOnlyVolume(e); err != nil {
+				log.WithError(err).Warnf("Failed to remount read only volume")
+				return
+			}
+
 		}
 	}
 }

engineapi/engine.go

@@ -352,6 +352,10 @@ func (e *EngineBinary) MetricsGet(*longhorn.Engine) (*Metrics, error) {
 	return nil, fmt.Errorf(ErrNotImplement)
 }
 
+func (e *EngineBinary) RemountReadOnlyVolume(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}
+
 // addFlags always adds required flags to args. In addition, if the engine version is high enough, it adds additional
 // engine identity validation flags.
 func (e *EngineBinary) addFlags(args []string) ([]string, error) {

engineapi/enginesim.go

@@ -276,3 +276,7 @@ func (e *EngineSimulator) MetricsGet(*longhorn.Engine) (*Metrics, error) {
 func (e *EngineSimulator) CleanupBackupMountPoints() error {
 	return fmt.Errorf(ErrNotImplement)
 }
+
+func (e *EngineSimulator) RemountReadOnlyVolume(*longhorn.Engine) error {
+	return fmt.Errorf(ErrNotImplement)
+}

engineapi/proxy_volume.go

@@ -53,3 +53,7 @@ func (p *Proxy) VolumeSnapshotMaxSizeSet(e *longhorn.Engine) error {
 	return p.grpcClient.VolumeSnapshotMaxSizeSet(string(e.Spec.DataEngine), e.Name, e.Spec.VolumeName,
 		p.DirectToURL(e), e.Spec.SnapshotMaxSize)
 }
+
+func (p *Proxy) RemountReadOnlyVolume(e *longhorn.Engine) error {
+	return p.grpcClient.RemountReadOnlyVolume(e.Spec.VolumeName)
+}

engineapi/types.go

@@ -111,6 +111,7 @@ type EngineClient interface {
 	CleanupBackupMountPoints() error
 
 	MetricsGet(engine *longhorn.Engine) (*Metrics, error)
+	RemountReadOnlyVolume(engine *longhorn.Engine) error
 }
 
 type EngineClientRequest struct {

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/longhorn/go-iscsi-helper v0.0.0-20240207150711-0713d1c9f5d1
 	github.com/longhorn/go-spdk-helper v0.0.0-20240219034107-a6d23f96da54
 	github.com/longhorn/longhorn-engine v1.6.0
-	github.com/longhorn/longhorn-instance-manager v1.6.0
+	github.com/longhorn/longhorn-instance-manager v1.6.0-dev-20240105.0.20240219053144-677fbd99cc56
 	github.com/longhorn/longhorn-share-manager v1.6.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
@@ -68,7 +68,6 @@ require (
 	google.golang.org/grpc v1.60.1
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.6
 	k8s.io/apiextensions-apiserver v0.25.4
 	k8s.io/apimachinery v0.28.6
@@ -88,15 +87,18 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
+	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.etcd.io/bbolt v1.3.8 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
@@ -122,7 +124,6 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/gammazero/workerpool v1.1.3 // indirect
-	github.com/go-co-op/gocron/v2 v2.2.4
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect