Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DPE-1720] Add backups to k8s charm and configure PBM #172

Merged
merged 13 commits into from
Jul 22, 2023
Prev Previous commit
Next Next commit
WIP: Add logging
  • Loading branch information
dmitry-ratushnyy authored and delgod committed Jul 22, 2023
commit 82894efac87ec492543fbefc7bf73c8b9f6b9322
47 changes: 37 additions & 10 deletions lib/charms/mongodb/v0/mongodb_backups.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
StatusBase,
WaitingStatus,
ModelError,
RuntimeError,
)
from tenacity import (
Retrying,
Expand Down Expand Up @@ -93,31 +92,30 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent):
# handling PBM configurations requires that MongoDB is running and the pbm snap is
# installed.
if not self.charm.db_initialised:
logger.debug("Cannot set PBM configurations, MongoDB has not yet started.")
logger.error("Cannot set PBM configurations, MongoDB has not yet started.")
dmitry-ratushnyy marked this conversation as resolved.
Show resolved Hide resolved
event.defer()
return

try:
self.charm.get_backup_service()
except ModelError as e:
logger.debug(f"Cannot set PBM configurations, pbm-agent service not found. {e}")
logger.error(f"Cannot set PBM configurations, pbm-agent service not found. {e}")
event.defer()
except RuntimeError as e:
logger.error(f"Cannot set PBM configurations. Failed to get pbm serivice. {e}")
self.charm.unit.status = BlockedStatus("couldn't configure s3 backup options.")
# TODO

try:
self._set_config_options()
self._resync_config_options()
except ResyncError as e:
self.charm.unit.status = WaitingStatus("waiting to sync s3 configurations.")
event.defer()
logger.debug(f"Sync-ing configurations needs more time: {e}")
logger.error(f"Sync-ing configurations needs more time: {e}")
return
except PBMBusyError as e:
self.charm.unit.status = WaitingStatus("waiting to sync s3 configurations.")
logger.debug(
logger.error(
f"Cannot update configs while PBM is running, must wait for PBM action to finish: {e}"
)
event.defer()
Expand All @@ -128,7 +126,6 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent):
self.charm.unit.status = self._get_pbm_status()



def _on_create_backup_action(self, event) -> None:
if self.model.get_relation(S3_RELATION) is None:
event.fail("Relation with s3-integrator charm missing, cannot create backup.")
Expand Down Expand Up @@ -163,9 +160,36 @@ def _on_create_backup_action(self, event) -> None:

def _get_pbm_status(self) -> StatusBase:
"""Retrieve pbm status."""
# TODO check pbm status
return ActiveStatus("")
try:
pmb_service = self.charm.get_backup_service()
except ModelError as e:
return BlockedStatus(f"pbm-agent service not found. {e}")

container = self.charm.get_container()
try:
pbm_status = container.exec("pbm status")
# pbm is running resync operation
if "Resync" in self._current_pbm_op(pbm_status.decode("utf-8")):
return WaitingStatus("waiting to sync s3 configurations.")

# no operations are currently running with pbm
if "(none)" in self._current_pbm_op(pbm_status.decode("utf-8")):
return ActiveStatus("")

if "Snapshot backup" in self._current_pbm_op(pbm_status.decode("utf-8")):
return MaintenanceStatus("backup started/running")

if "Snapshot restore" in self._current_pbm_op(pbm_status.decode("utf-8")):
return MaintenanceStatus("restore started/running")

except Exception as e:
# pbm pipes a return code of 1, but its output shows the true error code so it is
# necessary to parse the output
logger.error(f"Failed to get pbm status: {e}")
return BlockedStatus("PBM error")

return ActiveStatus("")

def _on_list_backups_action(self, event) -> None:
dmitry-ratushnyy marked this conversation as resolved.
Show resolved Hide resolved
if self.model.get_relation(S3_RELATION) is None:
event.fail("Relation with s3-integrator charm missing, cannot list backups.")
Expand Down Expand Up @@ -238,9 +262,11 @@ def _set_config_options(self):
)

# the pbm tool can only set one configuration at a time.
logger.error(f"_set_config_options.")
for pbm_key, pbm_value in self._get_pbm_configs():
config_cmd = f'pbm config --set {pbm_key}="{pbm_value}"'
try:
logger.error(f"Setting {pbm_key}={pbm_value} with pbm config command.")
container.exec(config_cmd)
except Exception as e:
logger.error(f"Failed to set {pbm_key}={pbm_value} with pbm config command. {e}")
Expand All @@ -256,6 +282,7 @@ def _get_pbm_configs(self) -> Dict:
continue

pbm_configs[S3_PBM_OPTION_MAP[s3_option]] = s3_value
logger.error(f"_get_pbm_configs. {pbm_configs}")
return pbm_configs

def _resync_config_options(self):
Expand Down Expand Up @@ -314,7 +341,7 @@ def _wait_pbm_status(self) -> None:
reraise=True,
):
with attempt:
pbm_status = container.exec("charmed-mongodb.pbm status", shell=True)
pbm_status = container.exec("pbm status")
if "Resync" in self._current_pbm_op(pbm_status.decode("utf-8")):
# since this process takes several minutes we should let the user know
# immediately.
Expand Down