Merge pull request #790 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

.github/workflows/codegen.yml

@@ -38,18 +38,8 @@ jobs:
         with:
           go-version: "1.22"
 
-      - name: post-process checkout - copy to github.com
-        run: |
-          # copy to the github.com path so codegen will work as expected
-          mkdir -p $(go env GOPATH)/src/github.com/rook/rook/
-          cp -R . $(go env GOPATH)/src/github.com/rook/rook/
-
       - name: run codegen
-        run: |
-          cd $(go env GOPATH)/src/github.com/rook/rook
-          GOPATH=$(go env GOPATH) make codegen
+        run: GOPATH=$(go env GOPATH) make codegen
 
       - name: validate codegen
-        run: |
-          cd $(go env GOPATH)/src/github.com/rook/rook
-          tests/scripts/validate_modified_files.sh codegen
+        run: tests/scripts/validate_modified_files.sh codegen

Documentation/CRDs/Cluster/ceph-cluster-crd.md

@@ -357,6 +357,7 @@ The following storage selection settings are specific to Ceph and do not apply t
 * `encryptedDevice`**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See [encryption](http://docs.ceph.com/docs/master/ceph-volume/lvm/encryption/) for more information on encryption in Ceph. (Resizing is not supported for host-based clusters.)
 * `crushRoot`: The value of the `root` CRUSH map label. The default is `default`. Generally, you should not need to change this. However, if any of your topology labels may have the value `default`, you need to change `crushRoot` to avoid conflicts, since CRUSH map values need to be unique.
 * `enableCrushUpdates`: Enables rook to update the pool crush rule using Pool Spec. Can cause data remapping if crush rule changes, Defaults to false.
+* `migration`: Existing PVC based OSDs can be migrated to enable or disable encryption. Refer to the [osd management](../../Storage-Configuration/Advanced/ceph-osd-mgmt.md/#osd-encryption-as-day-2-operation) topic for details.
 
 Allowed configurations are:
 

Documentation/CRDs/specification.md

@@ -8049,6 +8049,65 @@ bool
 </tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.Migration">Migration
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.StorageScopeSpec">StorageScopeSpec</a>)
+</p>
+<div>
+<p>Migration handles the OSD migration</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>confirmation</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>A user confirmation to migrate the OSDs. It destroys each OSD one at a time, cleans up the backing disk
+and prepares OSD with same ID on that disk</p>
+</td>
+</tr>
+</tbody>
+</table>
+<h3 id="ceph.rook.io/v1.MigrationStatus">MigrationStatus
+</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.OSDStatus">OSDStatus</a>)
+</p>
+<div>
+<p>MigrationStatus status represents the current status of any OSD migration.</p>
+</div>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>pending</code><br/>
+<em>
+int
+</em>
+</td>
+<td>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="ceph.rook.io/v1.MirrorHealthCheckSpec">MirrorHealthCheckSpec
 </h3>
 <p>
@@ -9471,6 +9530,18 @@ map[string]int
 <p>StoreType is a mapping between the OSD backend stores and number of OSDs using these stores</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>migrationStatus</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.MigrationStatus">
+MigrationStatus
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.OSDStore">OSDStore
@@ -12935,6 +13006,20 @@ Selection
 </tr>
 <tr>
 <td>
+<code>migration</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.Migration">
+Migration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Migration handles the OSD migration</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>store</code><br/>
 <em>
 <a href="#ceph.rook.io/v1.OSDStore">

Documentation/Storage-Configuration/Advanced/ceph-osd-mgmt.md

@@ -190,3 +190,29 @@ If you don't see a new OSD automatically created, restart the operator (by delet
 
 !!! note
     The OSD might have a different ID than the previous OSD that was replaced.
+
+
+## OSD Migration
+
+Ceph does not support changing certain settings on existing OSDs. To support changing these settings on an OSD, the OSD must be destroyed and re-created with the new settings. Rook will automate this by migrating only one OSD at a time. The operator waits for the data to rebalance (PGs to become `active+clean`) before migrating the next OSD. This ensures that there is no data loss. Refer to the [OSD migration](https://github.com/rook/rook/blob/master/design/ceph/osd-migration.md) design doc for more information. 
+
+The following scenarios are supported for OSD migration:
+
+- Enable or disable OSD encryption for existing PVC-based OSDs by changing the `encrypted` setting under the `storageClassDeviceSets`
+
+For example:
+
+```yaml
+storage:
+    migration:
+        confirmation: "yes-really-migrate-osds" 
+    storageClassDeviceSets:
+        - name: set1
+          count: 3
+          encrypted: true  # change to true or false based on whether encryption needs to enable or disabled.
+```
+
+Details about the migration status can be found under the cephCluster `status.storage.osd.migrationStatus.pending` field which shows the total number of OSDs that are pending migration.
+
+!!! note
+    Performance of the cluster might be impacted during data rebalancing while OSDs are being migrated.

PendingReleaseNotes.md

@@ -8,3 +8,4 @@
 
 - Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)).
 - Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)).
+- Allow migration of PVC based OSDs to enable or disable encryption (see [#14776](https://github.com/rook/rook/pull/14776)).

build/codegen/codegen.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S bash -e
+#!/usr/bin/env bash
 
 # Copyright 2018 The Rook Authors. All rights reserved.
 #
@@ -14,29 +14,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-GROUP_VERSIONS="ceph.rook.io:v1"
+set -o errexit
+set -o nounset
+set -o pipefail
 
-scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-# CODE GENERATION
-# we run deepcopy and client,lister,informer generations separately so we can use the flag "--plural-exceptions"
-# which is only known by client,lister,informer binary and not the deepcopy binary
+base_dir="$(cd "${script_dir}/../.."  && pwd)"
 
-# run code deepcopy generation
-bash ${CODE_GENERATOR}/generate-groups.sh \
-    deepcopy \
-    github.com/rook/rook/pkg/client \
-    github.com/rook/rook/pkg/apis \
-    "${GROUP_VERSIONS}" \
-    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../../.." \
-    --go-header-file "${scriptdir}/boilerplate.go.txt"
+boilerplate="${base_dir}"/build/codegen/header.txt
 
-# run code client,lister,informer generation
-bash ${CODE_GENERATOR}/generate-groups.sh \
-    client,lister,informer \
-    github.com/rook/rook/pkg/client \
-    github.com/rook/rook/pkg/apis \
-    "${GROUP_VERSIONS}" \
-    --output-base "$(dirname "${BASH_SOURCE[0]}")/../../../../.." \
-    --go-header-file "${scriptdir}/boilerplate.go.txt" \
-    --plural-exceptions "CephNFS:CephNFSes" \
+source "${CODE_GENERATOR}/kube_codegen.sh"
+(
+  # because apis is a separate go module, code generation
+  # must be run from within the corresponding directory.
+  cd  ${base_dir}/pkg/apis
+
+  # run code deepcopy generation
+  kube::codegen::gen_helpers \
+      --boilerplate "${boilerplate}" \
+      "${base_dir}/pkg/apis" \
+
+  # run code client,lister,informer generation
+  kube::codegen::gen_client \
+      --output-dir "${base_dir}/pkg/client" \
+      --output-pkg "github.com/rook/rook/pkg/client" \
+      --boilerplate "${boilerplate}" \
+      --plural-exceptions "CephNFS:CephNFSes" \
+      --with-watch \
+      "${base_dir}/pkg/apis"
+)

build/csv/ceph/ceph.rook.io_cephclusters.yaml

@@ -1610,6 +1610,12 @@ spec:
                     minimum: 0
                     nullable: true
                     type: number
+                  migration:
+                    properties:
+                      confirmation:
+                        pattern: ^$|^yes-really-migrate-osds$
+                        type: string
+                    type: object
                   nearFullRatio:
                     maximum: 1
                     minimum: 0
@@ -3260,6 +3266,11 @@ spec:
                     type: array
                   osd:
                     properties:
+                      migrationStatus:
+                        properties:
+                          pending:
+                            type: integer
+                        type: object
                       storeType:
                         additionalProperties:
                           type: integer

build/makelib/golang.mk

@@ -231,7 +231,7 @@ $(YQ):
 	@curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ)
 	@chmod +x $(YQ)
 
-export CODE_GENERATOR_VERSION=0.20.0
+export CODE_GENERATOR_VERSION=0.31.3
 export CODE_GENERATOR=$(TOOLS_HOST_DIR)/code-generator-$(CODE_GENERATOR_VERSION)
 $(CODE_GENERATOR):
 	mkdir -p $(TOOLS_HOST_DIR)

cmd/rook/ceph/osd.go

@@ -261,10 +261,10 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 	}
 
 	// destroy the OSD using the OSD ID
-	var replaceOSD *oposd.OSDReplaceInfo
+	var replaceOSD *oposd.OSDInfo
 	if replaceOSDID != -1 {
 		logger.Infof("destroying osd.%d and cleaning its backing device", replaceOSDID)
-		replaceOSD, err = osddaemon.DestroyOSD(context, &clusterInfo, replaceOSDID, cfg.pvcBacked, cfg.storeConfig.EncryptedDevice)
+		replaceOSD, err = osddaemon.DestroyOSD(context, &clusterInfo, replaceOSDID, cfg.pvcBacked)
 		if err != nil {
 			rook.TerminateFatal(errors.Wrapf(err, "failed to destroy OSD %d.", replaceOSDID))
 		}

deploy/charts/rook-ceph/templates/resources.yaml

@@ -3445,6 +3445,16 @@ spec:
                       minimum: 0
                       nullable: true
                       type: number
+                    migration:
+                      description: Migration handles the OSD migration
+                      properties:
+                        confirmation:
+                          description: |-
+                            A user confirmation to migrate the OSDs. It destroys each OSD one at a time, cleans up the backing disk
+                            and prepares OSD with same ID on that disk
+                          pattern: ^$|^yes-really-migrate-osds$
+                          type: string
+                      type: object
                     nearFullRatio:
                       description: NearFullRatio is the ratio at which the cluster is considered nearly full and will raise a ceph health warning. Default is 0.85.
                       maximum: 1
@@ -5538,6 +5548,12 @@ spec:
                     osd:
                       description: OSDStatus represents OSD status of the ceph Cluster
                       properties:
+                        migrationStatus:
+                          description: MigrationStatus status represents the current status of any OSD migration.
+                          properties:
+                            pending:
+                              type: integer
+                          type: object
                         storeType:
                           additionalProperties:
                             type: integer

deploy/examples/crds.yaml

@@ -3443,6 +3443,16 @@ spec:
                       minimum: 0
                       nullable: true
                       type: number
+                    migration:
+                      description: Migration handles the OSD migration
+                      properties:
+                        confirmation:
+                          description: |-
+                            A user confirmation to migrate the OSDs. It destroys each OSD one at a time, cleans up the backing disk
+                            and prepares OSD with same ID on that disk
+                          pattern: ^$|^yes-really-migrate-osds$
+                          type: string
+                      type: object
                     nearFullRatio:
                       description: NearFullRatio is the ratio at which the cluster is considered nearly full and will raise a ceph health warning. Default is 0.85.
                       maximum: 1
@@ -5536,6 +5546,12 @@ spec:
                     osd:
                       description: OSDStatus represents OSD status of the ceph Cluster
                       properties:
+                        migrationStatus:
+                          description: MigrationStatus status represents the current status of any OSD migration.
+                          properties:
+                            pending:
+                              type: integer
+                          type: object
                         storeType:
                           additionalProperties:
                             type: integer

design/ceph/osd-migration.md

@@ -25,6 +25,7 @@ considered in the future:
   and application data on slower media.
 - Setups with multiple OSDs per drive, though with recent Ceph releases the
   motivation for deploying this way is mostly obviated.
+- OSDs where Persistent Volumes are using partitioned disks due to a [ceph issue](https://tracker.ceph.com/issues/68977). 
 
 ## Proposal
 - Since migration requires destroying of the OSD and cleaning data from the disk,

pkg/apis/ceph.rook.io/v1/types.go

@@ -500,7 +500,13 @@ type DeviceClasses struct {
 // OSDStatus represents OSD status of the ceph Cluster
 type OSDStatus struct {
 	// StoreType is a mapping between the OSD backend stores and number of OSDs using these stores
-	StoreType map[string]int `json:"storeType,omitempty"`
+	StoreType       map[string]int  `json:"storeType,omitempty"`
+	MigrationStatus MigrationStatus `json:"migrationStatus,omitempty"`
+}
+
+// MigrationStatus status represents the current status of any OSD migration.
+type MigrationStatus struct {
+	Pending int `json:"pending,omitempty"`
 }
 
 // ClusterVersion represents the version of a Ceph Cluster
@@ -3051,6 +3057,9 @@ type StorageScopeSpec struct {
 	// +nullable
 	// +optional
 	StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets,omitempty"`
+	// Migration handles the OSD migration
+	// +optional
+	Migration Migration `json:"migration,omitempty"`
 	// +optional
 	Store OSDStore `json:"store,omitempty"`
 	// +optional
@@ -3089,6 +3098,15 @@ type StorageScopeSpec struct {
 	AllowOsdCrushWeightUpdate bool `json:"allowOsdCrushWeightUpdate,omitempty"`
 }
 
+// Migration handles the OSD migration
+type Migration struct {
+	// A user confirmation to migrate the OSDs. It destroys each OSD one at a time, cleans up the backing disk
+	// and prepares OSD with same ID on that disk
+	// +optional
+	// +kubebuilder:validation:Pattern=`^$|^yes-really-migrate-osds$`
+	Confirmation string `json:"confirmation,omitempty"`
+}
+
 // OSDStore is the backend storage type used for creating the OSDs
 type OSDStore struct {
 	// Type of backend storage to be used while creating OSDs. If empty, then bluestore will be used