Set OSD pool size when creating ceph and cephfs storage pools

.github/actions/setup-microceph/action.yml

@@ -51,17 +51,33 @@ runs:
           }
           trap cleanup ERR HUP INT TERM
 
+          ephemeral_disk="${{ steps.free_ephemeral_disk.outputs.ephemeral_disk }}"
+          sudo parted "${ephemeral_disk}" --script mklabel gpt
+          sudo parted "${ephemeral_disk}" --script mkpart primary 0% 33%
+          sudo parted "${ephemeral_disk}" --script mkpart primary 33% 66%
+          sudo parted "${ephemeral_disk}" --script mkpart primary 66% 100%
+
+          disk1="$(losetup -f)"
+          sudo losetup "${disk1}" "${ephemeral_disk}1"
+          disk2="$(losetup -f)"
+          sudo losetup "${disk2}" "${ephemeral_disk}2"
+          disk3="$(losetup -f)"
+          sudo losetup "${disk3}" "${ephemeral_disk}3"
+
           sudo snap install microceph --channel "${{ inputs.microceph-channel }}"
           sudo microceph cluster bootstrap
           sudo microceph.ceph config set global osd_pool_default_size 1
+          sudo microceph.ceph config set global mon_allow_pool_size_one true
           sudo microceph.ceph config set global mon_allow_pool_delete true
           sudo microceph.ceph config set global osd_memory_target 939524096
           sudo microceph.ceph osd crush rule rm replicated_rule
           sudo microceph.ceph osd crush rule create-replicated replicated default osd
           for flag in nosnaptrim nobackfill norebalance norecover noscrub nodeep-scrub; do
               sudo microceph.ceph osd set $flag
           done
-          sudo microceph disk add --wipe "${{ steps.free_ephemeral_disk.outputs.ephemeral_disk }}"
+          sudo microceph disk add --wipe "${disk1}"
+          sudo microceph disk add --wipe "${disk2}"
+          sudo microceph disk add --wipe "${disk3}"
           sudo rm -rf /etc/ceph
           sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
           sudo microceph enable rgw

doc/api-extensions.md

@@ -2517,6 +2517,10 @@ Adds support for using a bridge network with a specified VLAN ID as an OVN uplin
 Adds `logical_cpus` field to `GET /1.0/cluster/members/{name}/state` which
 contains the total available logical CPUs available when LXD started.
 
+<<<<<<< HEAD
 ## `vm_limits_cpu_pin_strategy`
 
 Adds a new {config:option}`instance-resource-limits:limits.cpu.pin_strategy` configuration option for virtual machines. This option controls the CPU pinning strategy. When set to `none`, CPU auto pinning is disabled. When set to `auto`, CPU auto pinning is enabled.
+
+## `storage_ceph_osd_pool_size`
+This introduces the configuration keys {config:option}`storage-ceph-pool-conf:ceph.osd.pool_size`, and {config:option}`storage-cephfs-pool-conf:cephfs.osd_pool_size` to be used when adding or updating a `ceph` or `cephfs` storage pool to instruct LXD to create set the replication size for the underlying OSD pools.

doc/metadata.txt

@@ -4894,6 +4894,13 @@ Specify either a cron expression (`<minute> <hour> <dom> <month> <dow>`), a comm
 
 ```
 
+```{config:option} ceph.osd.pool_size storage-ceph-pool-conf
+:defaultdesc: "`3`"
+:shortdesc: "Number of RADOS object replicas. Set to 1 for no replication."
+:type: "string"
+
+```
+
 ```{config:option} ceph.rbd.clone_copy storage-ceph-pool-conf
 :defaultdesc: "`true`"
 :shortdesc: "Whether to use RBD lightweight clones"
@@ -5074,6 +5081,14 @@ This option specifies the number of OSD pool placement groups (`pg_num`) to use
 when creating a missing OSD pool.
 ```
 
+```{config:option} cephfs.osd_pool_size storage-cephfs-pool-conf
+:defaultdesc: "`3`"
+:shortdesc: "Number of RADOS object replicas. Set to 1 for no replication."
+:type: "string"
+This option specifies the number of OSD pool replicas to use
+when creating an OSD pool.
+```
+
 ```{config:option} cephfs.path storage-cephfs-pool-conf
 :defaultdesc: "`/`"
 :shortdesc: "The base path for the CephFS mount"

lxd/metadata/configuration.json

@@ -5517,6 +5517,14 @@
 							"type": "string"
 						}
 					},
+					{
+						"ceph.osd.pool_size": {
+							"defaultdesc": "`3`",
+							"longdesc": "",
+							"shortdesc": "Number of RADOS object replicas. Set to 1 for no replication.",
+							"type": "string"
+						}
+					},
 					{
 						"ceph.rbd.clone_copy": {
 							"defaultdesc": "`true`",
@@ -5708,6 +5716,14 @@
 							"type": "string"
 						}
 					},
+					{
+						"cephfs.osd_pool_size": {
+							"defaultdesc": "`3`",
+							"longdesc": "This option specifies the number of OSD pool replicas to use\nwhen creating an OSD pool.",
+							"shortdesc": "Number of RADOS object replicas. Set to 1 for no replication.",
+							"type": "string"
+						}
+					},
 					{
 						"cephfs.path": {
 							"defaultdesc": "`/`",

lxd/storage/drivers/driver_ceph.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"github.com/canonical/lxd/lxd/migration"
@@ -112,6 +114,29 @@ func (d *ceph) FillConfig() error {
 		d.config["ceph.osd.pg_num"] = "32"
 	}
 
+	if d.config["ceph.osd.pool_size"] == "" {
+		size, err := shared.TryRunCommand("ceph",
+			"--name", "client.%s"+d.config["ceph.user.name"],
+			"--cluster", d.config["ceph.cluster_name"],
+			"config",
+			"get",
+			"mon",
+			"osd_pool_default_size",
+			"--format",
+			"json")
+		if err != nil {
+			return err
+		}
+
+		var sizeInt int
+		err = json.Unmarshal([]byte(size), &sizeInt)
+		if err != nil {
+			return err
+		}
+
+		d.config["ceph.osd.pool_size"] = strconv.Itoa(sizeInt)
+	}
+
 	return nil
 }
 
@@ -158,7 +183,7 @@ func (d *ceph) Create() error {
 	if !poolExists {
 		// Create new osd pool.
 		_, err := shared.TryRunCommand("ceph",
-			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
+			"--name", "client.%s"+d.config["ceph.user.name"],
 			"--cluster", d.config["ceph.cluster_name"],
 			"osd",
 			"pool",
@@ -171,6 +196,20 @@ func (d *ceph) Create() error {
 
 		revert.Add(func() { _ = d.osdDeletePool() })
 
+		_, err = shared.TryRunCommand("ceph",
+			"--name", "client.%s"+d.config["ceph.user.name"],
+			"--cluster", d.config["ceph.cluster_name"],
+			"osd",
+			"pool",
+			"set",
+			d.config["ceph.osd.pool_name"],
+			"size",
+			d.config["ceph.osd.pool_size"],
+			"--yes-i-really-mean-it")
+		if err != nil {
+			return err
+		}
+
 		// Initialize the pool. This is not necessary but allows the pool to be monitored.
 		_, err = shared.TryRunCommand("rbd",
 			"--id", d.config["ceph.user.name"],
@@ -217,7 +256,7 @@ func (d *ceph) Create() error {
 
 		// Use existing OSD pool.
 		msg, err := shared.RunCommand("ceph",
-			"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
+			"--name", "client.%s"+d.config["ceph.user.name"],
 			"--cluster", d.config["ceph.cluster_name"],
 			"osd",
 			"pool",
@@ -303,6 +342,13 @@ func (d *ceph) Validate(config map[string]string) error {
 		//  defaultdesc: `32`
 		//  shortdesc: Number of placement groups for the OSD storage pool
 		"ceph.osd.pg_num": validate.IsAny,
+		// lxdmeta:generate(entities=storage-ceph; group=pool-conf; key=ceph.osd.pool_size)
+		//
+		// ---
+		//  type: string
+		//  defaultdesc: `3`
+		//  shortdesc: Number of RADOS object replicas. Set to 1 for no replication.
+		"ceph.osd.pool_size": validate.Optional(validate.IsInRange(1, math.MaxInt64)),
 		// lxdmeta:generate(entities=storage-ceph; group=pool-conf; key=ceph.osd.pool_name)
 		//
 		// ---
@@ -358,6 +404,23 @@ func (d *ceph) Validate(config map[string]string) error {
 
 // Update applies any driver changes required from a configuration change.
 func (d *ceph) Update(changedConfig map[string]string) error {
+	newSize := changedConfig["ceph.osd.pool_size"]
+	if newSize != d.config["ceph.osd.pool_size"] && newSize != "" {
+		_, err := shared.TryRunCommand("ceph",
+			"--name", "client.%s"+d.config["ceph.user.name"],
+			"--cluster", d.config["ceph.cluster_name"],
+			"osd",
+			"pool",
+			"set",
+			d.config["ceph.osd.pool_name"],
+			"size",
+			newSize,
+			"--yes-i-really-mean-it")
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -388,7 +451,7 @@ func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) {
 
 	err := shared.RunCommandWithFds(context.TODO(), nil, &stdout,
 		"ceph",
-		"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
+		"--name", "client.%s"+d.config["ceph.user.name"],
 		"--cluster", d.config["ceph.cluster_name"],
 		"df",
 		"-f", "json")

lxd/storage/drivers/driver_ceph_utils.go

@@ -49,7 +49,7 @@
 func (d *ceph) osdPoolExists() (bool, error) {
 	_, err := shared.RunCommand(
 		"ceph",
-		"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
+		"--name", "client.%s"+d.config["ceph.user.name"],
 		"--cluster", d.config["ceph.cluster_name"],
 		"osd",
 		"pool",
@@ -83,7 +83,7 @@
 func (d *ceph) osdDeletePool() error {
 	_, err := shared.RunCommand(
 		"ceph",
-		"--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]),
+		"--name", "client.%s"+d.config["ceph.user.name"],
 		"--cluster", d.config["ceph.cluster_name"],
 		"osd",
 		"pool",
@@ -950,7 +950,7 @@
 // <osd-pool-name>/<lxd-specific-prefix>_<rbd-storage-volume>
 // will be split into
 // <osd-pool-name>, <lxd-specific-prefix>, <rbd-storage-volume>.
 func (d *ceph) parseClone(clone string) (string, string, string, error) {
 	idx := strings.Index(clone, "/")
 	if idx == -1 {
 		return "", "", "", fmt.Errorf("Unexpected parsing error")