diff --git a/lib/charms/mongodb/v0/mongodb_tls.py b/lib/charms/mongodb/v0/mongodb_tls.py index 0a9967fa6..2c31519be 100644 --- a/lib/charms/mongodb/v0/mongodb_tls.py +++ b/lib/charms/mongodb/v0/mongodb_tls.py @@ -24,6 +24,13 @@ from ops.framework import Object from ops.model import ActiveStatus, MaintenanceStatus, Unit +from config import Config + +APP_SCOPE = Config.Relations.APP_SCOPE +UNIT_SCOPE = Config.Relations.UNIT_SCOPE +Scopes = Config.Relations.Scopes + + # The unique Charmhub library identifier, never change it LIBID = "e02a50f0795e4dd292f58e93b4f493dd" @@ -36,7 +43,6 @@ logger = logging.getLogger(__name__) -TLS_RELATION = "certificates" class MongoDBTLS(Object): @@ -48,24 +54,30 @@ def __init__(self, charm, peer_relation, substrate): self.charm = charm self.substrate = substrate self.peer_relation = peer_relation - self.certs = TLSCertificatesRequiresV1(self.charm, TLS_RELATION) + self.certs = TLSCertificatesRequiresV1(self.charm, Config.TLS.TLS_PEER_RELATION) self.framework.observe( self.charm.on.set_tls_private_key_action, self._on_set_tls_private_key ) self.framework.observe( - self.charm.on[TLS_RELATION].relation_joined, self._on_tls_relation_joined + self.charm.on[Config.TLS.TLS_PEER_RELATION].relation_joined, + self._on_tls_relation_joined, ) self.framework.observe( - self.charm.on[TLS_RELATION].relation_broken, self._on_tls_relation_broken + self.charm.on[Config.TLS.TLS_PEER_RELATION].relation_broken, + self._on_tls_relation_broken, ) self.framework.observe(self.certs.on.certificate_available, self._on_certificate_available) self.framework.observe(self.certs.on.certificate_expiring, self._on_certificate_expiring) + def is_tls_enabled(self, scope: Scopes): + """Getting internal TLS flag (meaning).""" + return self.charm.get_secret(scope, Config.TLS.SECRET_CERT_LABEL) is not None + def _on_set_tls_private_key(self, event: ActionEvent) -> None: """Set the TLS private key, which will be used for requesting the certificate.""" logger.debug("Request to set TLS private key received.") try: - self._request_certificate("unit", event.params.get("external-key", None)) + self._request_certificate(UNIT_SCOPE, event.params.get("external-key", None)) if not self.charm.unit.is_leader(): event.log( @@ -73,12 +85,12 @@ def _on_set_tls_private_key(self, event: ActionEvent) -> None: ) return - self._request_certificate("app", event.params.get("internal-key", None)) + self._request_certificate(APP_SCOPE, event.params.get("internal-key", None)) logger.debug("Successfully set TLS private key.") except ValueError as e: event.fail(str(e)) - def _request_certificate(self, scope: str, param: Optional[str]): + def _request_certificate(self, scope: Scopes, param: Optional[str]): if param is None: key = generate_private_key() else: @@ -92,11 +104,11 @@ def _request_certificate(self, scope: str, param: Optional[str]): sans_ip=[str(self.charm.model.get_binding(self.peer_relation).network.bind_address)], ) - self.charm.set_secret(scope, "key", key.decode("utf-8")) - self.charm.set_secret(scope, "csr", csr.decode("utf-8")) - self.charm.set_secret(scope, "cert", None) + self.charm.set_secret(scope, Config.TLS.SECRET_KEY_LABEL, key.decode("utf-8")) + self.charm.set_secret(scope, Config.TLS.SECRET_CSR_LABEL, csr.decode("utf-8")) + self.charm.set_secret(scope, Config.TLS.SECRET_CERT_LABEL, None) - if self.charm.model.get_relation(TLS_RELATION): + if self.charm.model.get_relation(Config.TLS.TLS_PEER_RELATION): self.certs.request_certificate_creation(certificate_signing_request=csr) @staticmethod @@ -117,22 +129,24 @@ def _parse_tls_file(raw_content: str) -> bytes: def _on_tls_relation_joined(self, _: RelationJoinedEvent) -> None: """Request certificate when TLS relation joined.""" if self.charm.unit.is_leader(): - self._request_certificate("app", None) + self._request_certificate(APP_SCOPE, None) - self._request_certificate("unit", None) + self._request_certificate(UNIT_SCOPE, None) def _on_tls_relation_broken(self, event: RelationBrokenEvent) -> None: """Disable TLS when TLS relation broken.""" logger.debug("Disabling external TLS for unit: %s", self.charm.unit.name) - self.charm.set_secret("unit", "ca", None) - self.charm.set_secret("unit", "cert", None) - self.charm.set_secret("unit", "chain", None) + self.charm.set_secret(UNIT_SCOPE, Config.TLS.SECRET_CA_LABEL, None) + self.charm.set_secret(UNIT_SCOPE, Config.TLS.SECRET_CERT_LABEL, None) + self.charm.set_secret(UNIT_SCOPE, Config.TLS.SECRET_CHAIN_LABEL, None) + if self.charm.unit.is_leader(): logger.debug("Disabling internal TLS") - self.charm.set_secret("app", "ca", None) - self.charm.set_secret("app", "cert", None) - self.charm.set_secret("app", "chain", None) - if self.charm.get_secret("app", "cert"): + self.charm.set_secret(APP_SCOPE, Config.TLS.SECRET_CA_LABEL, None) + self.charm.set_secret(APP_SCOPE, Config.TLS.SECRET_CERT_LABEL, None) + self.charm.set_secret(APP_SCOPE, Config.TLS.SECRET_CHAIN_LABEL, None) + + if self.charm.get_secret(APP_SCOPE, Config.TLS.SECRET_CERT_LABEL): logger.debug( "Defer until the leader deletes the internal TLS certificate to avoid second restart." ) @@ -147,28 +161,27 @@ def _on_tls_relation_broken(self, event: RelationBrokenEvent) -> None: def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: """Enable TLS when TLS certificate available.""" - if ( - event.certificate_signing_request.rstrip() - == self.charm.get_secret("unit", "csr").rstrip() - ): + unit_csr = self.charm.get_secret(UNIT_SCOPE, Config.TLS.SECRET_CSR_LABEL) + app_csr = self.charm.get_secret(APP_SCOPE, Config.TLS.SECRET_CSR_LABEL) + + if unit_csr and event.certificate_signing_request.rstrip() == unit_csr.rstrip(): logger.debug("The external TLS certificate available.") - scope = "unit" # external crs - elif ( - event.certificate_signing_request.rstrip() - == self.charm.get_secret("app", "csr").rstrip() - ): + scope = UNIT_SCOPE # external crs + elif app_csr and event.certificate_signing_request.rstrip() == app_csr.rstrip(): logger.debug("The internal TLS certificate available.") - scope = "app" # internal crs + scope = APP_SCOPE # internal crs else: - logger.error("An unknown certificate available.") + logger.error("An unknown certificate is available -- ignoring.") return - if scope == "unit" or (scope == "app" and self.charm.unit.is_leader()): + if scope == UNIT_SCOPE or (scope == APP_SCOPE and self.charm.unit.is_leader()): self.charm.set_secret( - scope, "chain", "\n".join(event.chain) if event.chain is not None else None + scope, + Config.TLS.SECRET_CHAIN_LABEL, + "\n".join(event.chain) if event.chain is not None else None, ) - self.charm.set_secret(scope, "cert", event.certificate) - self.charm.set_secret(scope, "ca", event.ca) + self.charm.set_secret(scope, Config.TLS.SECRET_CERT_LABEL, event.certificate) + self.charm.set_secret(scope, Config.TLS.SECRET_CA_LABEL, event.ca) if self._waiting_for_certs(): logger.debug( @@ -177,7 +190,7 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: event.defer() return - logger.debug("Restarting mongod with TLS enabled.") + logger.info("Restarting mongod with TLS enabled.") self.charm.push_tls_certificate_to_workload() self.charm.unit.status = MaintenanceStatus("enabling TLS") @@ -186,10 +199,10 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: def _waiting_for_certs(self): """Returns a boolean indicating whether additional certs are needed.""" - if not self.charm.get_secret("app", "cert"): + if not self.charm.get_secret(APP_SCOPE, Config.TLS.SECRET_CERT_LABEL): logger.debug("Waiting for application certificate.") return True - if not self.charm.get_secret("unit", "cert"): + if not self.charm.get_secret(UNIT_SCOPE, Config.TLS.SECRET_CERT_LABEL): logger.debug("Waiting for application certificate.") return True @@ -197,21 +210,27 @@ def _waiting_for_certs(self): def _on_certificate_expiring(self, event: CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" - if event.certificate.rstrip() == self.charm.get_secret("unit", "cert").rstrip(): + if ( + event.certificate.rstrip() + == self.charm.get_secret(UNIT_SCOPE, Config.TLS.SECRET_CERT_LABEL).rstrip() + ): logger.debug("The external TLS certificate expiring.") - scope = "unit" # external cert - elif event.certificate.rstrip() == self.charm.get_secret("app", "cert").rstrip(): + scope = UNIT_SCOPE # external cert + elif ( + event.certificate.rstrip() + == self.charm.get_secret(APP_SCOPE, Config.TLS.SECRET_CERT_LABEL).rstrip() + ): logger.debug("The internal TLS certificate expiring.") if not self.charm.unit.is_leader(): return - scope = "app" # internal cert + scope = APP_SCOPE # internal cert else: logger.error("An unknown certificate expiring.") return logger.debug("Generating a new Certificate Signing Request.") - key = self.charm.get_secret(scope, "key").encode("utf-8") - old_csr = self.charm.get_secret(scope, "csr").encode("utf-8") + key = self.charm.get_secret(scope, Config.TLS.SECRET_KEY_LABEL).encode("utf-8") + old_csr = self.charm.get_secret(scope, Config.TLS.SECRET_CSR_LABEL).encode("utf-8") new_csr = generate_csr( private_key=key, subject=self.get_host(self.charm.unit), @@ -226,7 +245,7 @@ def _on_certificate_expiring(self, event: CertificateExpiringEvent) -> None: new_certificate_signing_request=new_csr, ) - self.charm.set_secret(scope, "csr", new_csr.decode("utf-8")) + self.charm.set_secret(scope, Config.TLS.SECRET_CSR_LABEL, new_csr.decode("utf-8")) def _get_sans(self) -> List[str]: """Create a list of DNS names for a MongoDB unit. @@ -242,19 +261,24 @@ def _get_sans(self) -> List[str]: str(self.charm.model.get_binding(self.peer_relation).network.bind_address), ] - def get_tls_files(self, scope: str) -> Tuple[Optional[str], Optional[str]]: + def get_tls_files(self, scope: Scopes) -> Tuple[Optional[str], Optional[str]]: """Prepare TLS files in special MongoDB way. MongoDB needs two files: — CA file should have a full chain. — PEM file should have private key and certificate without certificate chain. """ - ca = self.charm.get_secret(scope, "ca") - chain = self.charm.get_secret(scope, "chain") + if not self.is_tls_enabled(scope): + logging.debug(f"TLS disabled for {scope}") + return None, None + logging.debug(f"TLS *enabled* for {scope}, fetching data for CA and PEM files ") + + ca = self.charm.get_secret(scope, Config.TLS.SECRET_CA_LABEL) + chain = self.charm.get_secret(scope, Config.TLS.SECRET_CHAIN_LABEL) ca_file = chain if chain else ca - key = self.charm.get_secret(scope, "key") - cert = self.charm.get_secret(scope, "cert") + key = self.charm.get_secret(scope, Config.TLS.SECRET_KEY_LABEL) + cert = self.charm.get_secret(scope, Config.TLS.SECRET_CERT_LABEL) pem_file = key if cert: pem_file = key + "\n" + cert if key else cert diff --git a/poetry.lock b/poetry.lock index 0bdc7f26f..488cb9af3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1119,6 +1119,20 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "parameterized" +version = "0.9.0" +description = "Parameterized testing with any Python test framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, + {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, +] + +[package.extras] +dev = ["jinja2"] + [[package]] name = "paramiko" version = "2.12.0" @@ -1657,6 +1671,23 @@ pytest = ">=7.0.0" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] +[[package]] +name = "pytest-mock" +version = "3.11.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, + {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "pytest-operator" version = "0.27.0" @@ -2162,4 +2193,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8.10" -content-hash = "29846a2ba2a9700423b1eabc49a7539a8a51e4e190eb5807438a0f7918eed156" +content-hash = "c37816984caccfe6330dde60cb87ceb45f17c41672b9ec0891fb0fe51673ab77" diff --git a/pyproject.toml b/pyproject.toml index 6259eb051..984fbe12c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,12 @@ codespell = "^2.2.4" [tool.poetry.group.unit.dependencies] coverage = {extras = ["toml"], version = "^7.2.7"} pytest = "^7.3.1" +parameterized = "^0.9.0" [tool.poetry.group.integration.dependencies] lightkube = "^0.13.0" pytest = "^7.3.1" +pytest-mock = "^3.11.1" pytest-operator = "^0.27.0" juju = "2.9.42.1 || 3.1.0.1" diff --git a/requirements.txt b/requirements.txt index 98342f730..495991cfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -207,47 +207,47 @@ pyrsistent==0.19.3 ; python_full_version >= "3.8.10" and python_full_version < " --hash=sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1 \ --hash=sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9 \ --hash=sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c -pyyaml==6.0 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" \ - --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ - --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ - --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ - --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ - --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ - --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ - --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ - --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ - --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ - --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ - --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ - --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ - --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ - --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ - --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ - --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ - --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ - --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ - --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ - --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ - --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ - --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ - --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ - --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ - --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ - --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ - --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ - --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ - --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ - --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ - --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ - --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ - --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ - --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ - --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ - --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ - --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ - --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ - --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ - --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 +PyYAML==6.0.1; python_full_version >= "3.8.10" and python_full_version < "4.0.0" \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f tenacity==8.2.2 ; python_full_version >= "3.8.10" and python_full_version < "4.0.0" \ --hash=sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0 \ --hash=sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0 diff --git a/src/charm.py b/src/charm.py index 007b7f29a..5614f4bfd 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,7 +5,7 @@ import logging import time -from typing import List, Optional, Set +from typing import Dict, List, Optional, Set from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer @@ -30,6 +30,7 @@ OperatorUser, ) from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider +from ops import JujuVersion from ops.charm import ActionEvent, CharmBase, RelationDepartedEvent, StartEvent from ops.main import main from ops.model import ( @@ -37,6 +38,7 @@ Container, Relation, RelationDataContent, + SecretNotFoundError, Unit, WaitingStatus, ) @@ -45,12 +47,16 @@ from tenacity import before_log, retry, stop_after_attempt, wait_fixed from config import Config -from exceptions import AdminUserCreationError +from exceptions import AdminUserCreationError, MissingSecretError, SecretNotAddedError logger = logging.getLogger(__name__) UNIT_REMOVAL_TIMEOUT = 1000 +APP_SCOPE = Config.Relations.APP_SCOPE +UNIT_SCOPE = Config.Relations.UNIT_SCOPE +Scopes = Config.Relations.Scopes + class MongoDBCharm(CharmBase): """A Juju Charm to deploy MongoDB on Kubernetes.""" @@ -80,6 +86,9 @@ def __init__(self, *args): self.framework.observe(self.on.set_password_action, self._on_set_password) self.framework.observe(self.on.stop, self._on_stop) + self.framework.observe(self.on.secret_remove, self._on_secret_remove) + self.framework.observe(self.on.secret_changed, self._on_secret_changed) + self.client_relations = MongoDBProvider(self) self.tls = MongoDBTLS(self, Config.Relations.PEERS, Config.SUBSTRATE) @@ -93,6 +102,7 @@ def __init__(self, *args): relation_name=Config.Relations.LOGGING, container_name=Config.CONTAINER_NAME, ) + self.secrets = {APP_SCOPE: {}, UNIT_SCOPE: {}} # BEGIN: properties @@ -183,20 +193,25 @@ def _mongod_layer(self) -> Layer: return Layer(layer_config) # type: ignore @property - def unit_peer_data(self) -> RelationDataContent: + def relation(self) -> Optional[Relation]: + """Peer relation data object.""" + return self.model.get_relation(Config.Relations.PEERS) + + @property + def unit_peer_data(self) -> Dict: """Peer relation data object.""" - relation = self.model.get_relation(Config.Relations.PEERS) + relation = self.relation if relation is None: - return {} # type: ignore + return {} return relation.data[self.unit] @property def app_peer_data(self) -> RelationDataContent: """Peer relation data object.""" - relation = self.model.get_relation(Config.Relations.PEERS) + relation = self.relation if relation is None: - return {} # type: ignore + return {} return relation.data[self.app] @@ -215,12 +230,24 @@ def _db_initialised(self, value): # END: properties + # BEGIN: generic helper methods + + def _scope_opj(self, scope: Scopes): + if scope == APP_SCOPE: + return self.app + if scope == UNIT_SCOPE: + return self.unit + + def _peer_data(self, scope: Scopes): + return self.relation.data[self._scope_opj(scope)] + + # END: generic helper methods + # BEGIN: charm events def _on_mongod_pebble_ready(self, event) -> None: """Configure MongoDB pebble layer specification.""" # Get a reference the container attribute container = self.unit.get_container(Config.CONTAINER_NAME) - if not container.can_connect(): logger.debug("mongod container is not ready yet.") event.defer() @@ -233,8 +260,8 @@ def _on_mongod_pebble_ready(self, event) -> None: self._pull_licenses(container) self._set_data_dir_permissions(container) - except (PathError, ProtocolError) as e: - logger.error("Cannot put keyFile: %r", e) + except (PathError, ProtocolError, MissingSecretError) as e: + logger.error("Cannot initialize workload: %r", e) event.defer() return @@ -244,7 +271,12 @@ def _on_mongod_pebble_ready(self, event) -> None: container.replan() # when a network cuts and the pod restarts - reconnect to the exporter - self._connect_mongodb_exporter() + try: + self._connect_mongodb_exporter() + except MissingSecretError as e: + logger.error("Cannot connect mongodb exporter: %r", e) + event.defer() + return def _on_start(self, event) -> None: """Initialise MongoDB. @@ -331,6 +363,7 @@ def _relation_changes_handler(self, event) -> None: # app relations should be made aware of the new set of hosts self._update_app_relation_data(mongo.get_users()) + except NotReadyError: logger.info("Deferring reconfigure: another member doing sync right now") event.defer() @@ -363,7 +396,9 @@ def _on_get_password(self, event: ActionEvent) -> None: if not username: return key_name = MongoDBUser.get_password_key_name_for_user(username) - event.set_results({Config.Actions.PASSWORD_PARAM_NAME: self.get_secret("app", key_name)}) + event.set_results( + {Config.Actions.PASSWORD_PARAM_NAME: self.get_secret(APP_SCOPE, key_name)} + ) def _on_set_password(self, event: ActionEvent) -> None: """Set the password for the specified user.""" @@ -381,7 +416,7 @@ def _on_set_password(self, event: ActionEvent) -> None: new_password = event.params.get(Config.Actions.PASSWORD_PARAM_NAME, generate_password()) if new_password == self.get_secret( - "app", MonitorUser.get_password_key_name_for_user(username) + APP_SCOPE, MonitorUser.get_password_key_name_for_user(username) ): event.log("The old and new passwords are equal.") event.set_results({Config.Actions.PASSWORD_PARAM_NAME: new_password}) @@ -399,12 +434,35 @@ def _on_set_password(self, event: ActionEvent) -> None: event.fail(f"Failed changing the password: {e}") return - self.set_secret("app", MongoDBUser.get_password_key_name_for_user(username), new_password) + secret_id = self.set_secret( + APP_SCOPE, MongoDBUser.get_password_key_name_for_user(username), new_password + ) if username == MonitorUser.get_username(): self._connect_mongodb_exporter() - event.set_results({Config.Actions.PASSWORD_PARAM_NAME: new_password}) + event.set_results( + {Config.Actions.PASSWORD_PARAM_NAME: new_password, "secret-id": secret_id} + ) + + def _on_secret_remove(self, event): + logging.debug(f"Secret {event._id} seems to have no observers, could be removed") + + def _on_secret_changed(self, event): + secret = event.secret + + if secret.id == self.app_peer_data.get(Config.Secrets.SECRET_INTERNAL_LABEL, None): + scope = APP_SCOPE + elif secret.id == self.unit_peer_data.get(Config.Secrets.SECRET_INTERNAL_LABEL, None): + scope = UNIT_SCOPE + else: + logging.debug( + "Secret {event._id}:{event.secret.id} changed, but it's irrelevant for us" + ) + return + logging.debug(f"Secret {event._id} for scope {scope} changed, refreshing") + self._juju_secrets_get(scope) + self._connect_mongodb_exporter() # END: actions @@ -504,20 +562,24 @@ def _set_user_created(self, user: MongoDBUser) -> None: def _get_mongodb_config_for_user( self, user: MongoDBUser, hosts: List[str] ) -> MongoDBConfiguration: - external_ca, _ = self.tls.get_tls_files("unit") - internal_ca, _ = self.tls.get_tls_files("app") - password = self.get_secret("app", user.get_password_key_name()) - - return MongoDBConfiguration( - replset=self.app.name, - database=user.get_database_name(), - username=user.get_username(), - password=password, # type: ignore - hosts=set(hosts), - roles=set(user.get_roles()), - tls_external=external_ca is not None, - tls_internal=internal_ca is not None, - ) + external_ca, _ = self.tls.get_tls_files(UNIT_SCOPE) + internal_ca, _ = self.tls.get_tls_files(APP_SCOPE) + password = self.get_secret(APP_SCOPE, user.get_password_key_name()) + if not password: + raise MissingSecretError( + "Password for {APP_SCOPE}, {user.get_username()} couldn't be retrieved" + ) + else: + return MongoDBConfiguration( + replset=self.app.name, + database=user.get_database_name(), + username=user.get_username(), + password=password, # type: ignore + hosts=set(hosts), + roles=set(user.get_roles()), + tls_external=external_ca is not None, + tls_internal=internal_ca is not None, + ) def _get_user_or_fail_event(self, event: ActionEvent, default_username: str) -> Optional[str]: """Returns MongoDBUser object or raises ActionFail if user doesn't exist.""" @@ -532,15 +594,15 @@ def _get_user_or_fail_event(self, event: ActionEvent, default_username: str) -> def _check_or_set_user_password(self, user: MongoDBUser) -> None: key = user.get_password_key_name() - if not self.get_secret("app", key): - self.set_secret("app", key, generate_password()) + if not self.get_secret(APP_SCOPE, key): + self.set_secret(APP_SCOPE, key, generate_password()) def _check_or_set_keyfile(self) -> None: - if not self.get_secret("app", "keyfile"): + if not self.get_secret(APP_SCOPE, "keyfile"): self._generate_keyfile() def _generate_keyfile(self) -> None: - self.set_secret("app", "keyfile", generate_keyfile()) + self.set_secret(APP_SCOPE, "keyfile", generate_keyfile()) def _generate_secrets(self) -> None: """Generate passwords and put them into peer relation. @@ -558,7 +620,7 @@ def _update_app_relation_data(self, database_users: Set[str]) -> None: """Helper function to update application relation data.""" for relation in self.model.relations[Config.Relations.NAME]: username = self.client_relations._get_username_from_relation_id(relation.id) - password = relation.data[self.app]["password"] + password = relation.data[self.app][Config.Actions.PASSWORD_PARAM_NAME] if username in database_users: config = self.client_relations._get_config(username, password) relation.data[self.app].update( @@ -631,29 +693,141 @@ def _set_leader_unit_active_if_needed(self): ): self.unit.status = ActiveStatus() - def get_secret(self, scope: str, key: str) -> Optional[str]: - """Get TLS secret from the secret storage.""" - if scope == "unit": - return self.unit_peer_data.get(key, None) - elif scope == "app": - return self.app_peer_data.get(key, None) - else: - raise RuntimeError("Unknown secret scope.") + def _juju_secrets_get(self, scope: Scopes) -> Optional[bool]: + """Helper function to get Juju secret.""" + peer_data = self._peer_data(scope) - def set_secret(self, scope: str, key: str, value: Optional[str]) -> None: - """Set TLS secret in the secret storage.""" - if scope == "unit": - if not value: - del self.unit_peer_data[key] - return - self.unit_peer_data.update({key: value}) - elif scope == "app": - if not value: - del self.app_peer_data[key] + if not peer_data.get(Config.Secrets.SECRET_INTERNAL_LABEL): + return + + if Config.Secrets.SECRET_CACHE_LABEL not in self.secrets[scope]: + try: + # NOTE: Secret contents are not yet available! + secret = self.model.get_secret(id=peer_data[Config.Secrets.SECRET_INTERNAL_LABEL]) + except SecretNotFoundError as e: + logging.debug( + f"No secret found for ID {peer_data[Config.Secrets.SECRET_INTERNAL_LABEL]}, {e}" + ) return - self.app_peer_data.update({key: value}) + + logging.debug(f"Secret {peer_data[Config.Secrets.SECRET_INTERNAL_LABEL]} downloaded") + + # We keep the secret object around -- needed when applying modifications + self.secrets[scope][Config.Secrets.SECRET_LABEL] = secret + + # We retrieve and cache actual secret data for the lifetime of the event scope + self.secrets[scope][Config.Secrets.SECRET_CACHE_LABEL] = secret.get_content() + + if self.secrets[scope].get(Config.Secrets.SECRET_CACHE_LABEL): + return True + return False + + def _juju_secret_get_key(self, scope: Scopes, key: str) -> Optional[str]: + if not key: + return + + if self._juju_secrets_get(scope): + secret_cache = self.secrets[scope].get(Config.Secrets.SECRET_CACHE_LABEL) + if secret_cache: + secret_data = secret_cache.get(key) + if secret_data and secret_data != Config.Secrets.SECRET_DELETED_LABEL: + logging.debug(f"Getting secret {scope}:{key}") + return secret_data + logging.debug(f"No value found for secret {scope}:{key}") + + def get_secret(self, scope: Scopes, key: str) -> Optional[str]: + """Getting a secret.""" + peer_data = self._peer_data(scope) + + juju_version = JujuVersion.from_environ() + + if juju_version.has_secrets: + return self._juju_secret_get_key(scope, key) + else: + return peer_data.get(key) + + def _juju_secret_set(self, scope: Scopes, key: str, value: str) -> str: + """Helper function setting Juju secret.""" + peer_data = self._peer_data(scope) + self._juju_secrets_get(scope) + + secret = self.secrets[scope].get(Config.Secrets.SECRET_LABEL) + + # It's not the first secret for the scope, we can re-use the existing one + # that was fetched in the previous call + if secret: + secret_cache = self.secrets[scope][Config.Secrets.SECRET_CACHE_LABEL] + + if secret_cache.get(key) == value: + logging.debug(f"Key {scope}:{key} has this value defined already") + else: + secret_cache[key] = value + try: + secret.set_content(secret_cache) + except OSError as error: + logging.error( + f"Error in attempt to set {scope}:{key}. " + f"Existing keys were: {list(secret_cache.keys())}. {error}" + ) + logging.debug(f"Secret {scope}:{key} was {key} set") + + # We need to create a brand-new secret for this scope + else: + scope_obj = self._scope_opj(scope) + + secret = scope_obj.add_secret({key: value}) + if not secret: + raise SecretNotAddedError(f"Couldn't set secret {scope}:{key}") + + self.secrets[scope][Config.Secrets.SECRET_LABEL] = secret + self.secrets[scope][Config.Secrets.SECRET_CACHE_LABEL] = {key: value} + logging.debug(f"Secret {scope}:{key} published (as first). ID: {secret.id}") + peer_data.update({Config.Secrets.SECRET_INTERNAL_LABEL: secret.id}) + + return self.secrets[scope][Config.Secrets.SECRET_LABEL].id + + def set_secret(self, scope: Scopes, key: str, value: Optional[str]) -> Optional[str]: + """(Re)defining a secret.""" + if not value: + return self.remove_secret(scope, key) + + juju_version = JujuVersion.from_environ() + + result = None + if juju_version.has_secrets: + result = self._juju_secret_set(scope, key, value) else: - raise RuntimeError("Unknown secret scope.") + peer_data = self._peer_data(scope) + peer_data.update({key: value}) + + return result + + def _juju_secret_remove(self, scope: Scopes, key: str) -> None: + """Remove a Juju 3.x secret.""" + self._juju_secrets_get(scope) + + secret = self.secrets[scope].get(Config.Secrets.SECRET_LABEL) + if not secret: + logging.error(f"Secret {scope}:{key} wasn't deleted: no secrets are available") + return + + secret_cache = self.secrets[scope].get(Config.Secrets.SECRET_CACHE_LABEL) + if not secret_cache or key not in secret_cache: + logging.error(f"No secret {scope}:{key}") + return + + secret_cache[key] = Config.Secrets.SECRET_DELETED_LABEL + secret.set_content(secret_cache) + logging.debug(f"Secret {scope}:{key}") + + def remove_secret(self, scope, key) -> None: + """Removing a secret.""" + juju_version = JujuVersion.from_environ() + if juju_version.has_secrets: + return self._juju_secret_remove(scope, key) + + peer_data = self._peer_data(scope) + del peer_data[key] def restart_mongod_service(self): """Restart mongod service.""" @@ -667,21 +841,24 @@ def restart_mongod_service(self): def _push_keyfile_to_workload(self, container: Container) -> None: """Upload the keyFile to a workload container.""" - keyfile = self.get_secret("app", "keyfile") - - container.push( - Config.CONF_DIR + "/" + Config.TLS.KEY_FILE_NAME, - keyfile, # type: ignore - make_dirs=True, - permissions=0o400, - user=Config.UNIX_USER, - group=Config.UNIX_GROUP, - ) + keyfile = self.get_secret(APP_SCOPE, Config.Secrets.SECRET_KEYFILE_NAME) + if not keyfile: + raise MissingSecretError(f"No secret defined for {APP_SCOPE}, keyfile") + else: + container.push( + Config.CONF_DIR + "/" + Config.TLS.KEY_FILE_NAME, + keyfile, # type: ignore + make_dirs=True, + permissions=0o400, + user=Config.UNIX_USER, + group=Config.UNIX_GROUP, + ) def push_tls_certificate_to_workload(self) -> None: """Uploads certificate to the workload container.""" container = self.unit.get_container(Config.CONTAINER_NAME) - external_ca, external_pem = self.tls.get_tls_files("unit") + external_ca, external_pem = self.tls.get_tls_files(UNIT_SCOPE) + if external_ca is not None: logger.debug("Uploading external ca to workload container") container.push( @@ -703,7 +880,7 @@ def push_tls_certificate_to_workload(self) -> None: group=Config.UNIX_GROUP, ) - internal_ca, internal_pem = self.tls.get_tls_files("app") + internal_ca, internal_pem = self.tls.get_tls_files(APP_SCOPE) if internal_ca is not None: logger.debug("Uploading internal ca to workload container") container.push( @@ -727,12 +904,15 @@ def push_tls_certificate_to_workload(self) -> None: def delete_tls_certificate_from_workload(self) -> None: """Deletes certificate from the workload container.""" - logger.error("Deleting TLS certificate from workload container") + logger.info("Deleting TLS certificate from workload container") container = self.unit.get_container(Config.CONTAINER_NAME) - container.remove_path(Config.CONF_DIR + "/" + Config.TLS.EXT_CA_FILE) - container.remove_path(Config.CONF_DIR + "/" + Config.TLS.EXT_PEM_FILE) - container.remove_path(Config.CONF_DIR + "/" + Config.TLS.INT_CA_FILE) - container.remove_path(Config.CONF_DIR + "/" + Config.TLS.INT_PEM_FILE) + for file in [ + Config.TLS.EXT_CA_FILE, + Config.TLS.EXT_PEM_FILE, + Config.TLS.INT_CA_FILE, + Config.TLS.INT_PEM_FILE, + ]: + container.remove_path(f"{Config.CONF_DIR}/{file}") def get_hostname_for_unit(self, unit: Unit) -> str: """Create a DNS name for a MongoDB unit. @@ -754,7 +934,7 @@ def _connect_mongodb_exporter(self) -> None: return # must wait for leader to set URI before connecting - if not self.get_secret("app", MonitorUser.get_password_key_name()): + if not self.get_secret(APP_SCOPE, MonitorUser.get_password_key_name()): return # Add initial Pebble config layer using the Pebble API # mongodb_exporter --mongodb.uri= diff --git a/src/config.py b/src/config.py index b72d7d35e..33320da1a 100644 --- a/src/config.py +++ b/src/config.py @@ -2,6 +2,8 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +from typing import Literal + class Config: """Configuration for MongoDB Charm.""" @@ -18,6 +20,8 @@ class Config: CONTAINER_NAME = "mongod" SERVICE_NAME = "mongod" SOCKET_PATH = "/tmp/mongodb-27017.sock" + APP_SCOPE = "app" + UNIT_SCOPE = "unit" class Actions: """Actions related config for MongoDB Charm.""" @@ -44,14 +48,20 @@ class Monitoring: URI_PARAM_NAME = "monitor-uri" SERVICE_NAME = "mongodb-exporter" JOBS = [{"static_configs": [{"targets": [f"*:{MONGODB_EXPORTER_PORT}"]}]}] + APP_SCOPE = "app" + UNIT_SCOPE = "unit" class Relations: """Relations related config for MongoDB Charm.""" + APP_SCOPE = "app" + UNIT_SCOPE = "unit" NAME = "database" PEERS = "database-peers" LOGGING = "logging" + Scopes = Literal[APP_SCOPE, UNIT_SCOPE] + class TLS: """TLS related config for MongoDB Charm.""" @@ -60,6 +70,23 @@ class TLS: INT_PEM_FILE = "internal-cert.pem" INT_CA_FILE = "internal-ca.crt" KEY_FILE_NAME = "keyFile" + TLS_PEER_RELATION = "certificates" + + SECRET_CA_LABEL = "ca-secret" + SECRET_KEY_LABEL = "key-secret" + SECRET_CERT_LABEL = "cert-secret" + SECRET_CSR_LABEL = "csr-secret" + SECRET_CHAIN_LABEL = "chain-secret" + + class Secrets: + """Secrets related constants.""" + + SECRET_LABEL = "secret" + SECRET_CACHE_LABEL = "cache" + SECRET_KEYFILE_NAME = "keyfile" + SECRET_INTERNAL_LABEL = "internal-secret" + SECRET_DELETED_LABEL = "None" + SECRET_KEYFILE_NAME = "keyfile" @staticmethod def get_license_path(license_name: str) -> str: diff --git a/src/exceptions.py b/src/exceptions.py index cb78b3dbf..022182a4b 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -4,13 +4,37 @@ # See LICENSE file for licensing details. -class AdminUserCreationError(Exception): +class MongoError(Exception): + """Common parent for Mongo errors, allowing to catch them all at once.""" + + pass + + +class AdminUserCreationError(MongoError): """Raised when a commands to create an admin user on MongoDB fail.""" pass -class ApplicationHostNotFoundError(Exception): +class ApplicationHostNotFoundError(MongoError): """Raised when a queried host is not in the application peers or the current host.""" pass + + +class MongoSecretError(MongoError): + """Common parent for all Mongo Secret Exceptions.""" + + pass + + +class SecretNotAddedError(MongoSecretError): + """Raised when a Juju 3 secret couldn't be set or re-set.""" + + pass + + +class MissingSecretError(MongoSecretError): + """Could be raised when a Juju 3 mandatory secret couldn't be found.""" + + pass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..d64b1a3f5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,44 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +from importlib.metadata import version +from unittest.mock import PropertyMock + +import pytest +from ops import JujuVersion +from pytest_mock import MockerFixture + + +@pytest.fixture(autouse=True) +def juju_has_secrets(mocker: MockerFixture): + """This fixture will force the usage of secrets whenever run on Juju 3.x. + + NOTE: This is needed, as normally JujuVersion is set to 0.0.0 in tests + (i.e. not the real juju version) + """ + if version("juju") < "3": + mocker.patch.object( + JujuVersion, "has_secrets", new_callable=PropertyMock + ).return_value = False + return False + else: + mocker.patch.object( + JujuVersion, "has_secrets", new_callable=PropertyMock + ).return_value = True + return True + + +@pytest.fixture +def only_with_juju_secrets(juju_has_secrets): + """Pretty way to skip Juju 3 tests.""" + if not juju_has_secrets: + pytest.skip("Secrets test only applies on Juju 3.x") + + +@pytest.fixture +def only_without_juju_secrets(juju_has_secrets): + """Pretty way to skip Juju 2-specific tests. + + Typically: to save CI time, when the same check were executed in a Juju 3-specific way already + """ + if juju_has_secrets: + pytest.skip("Skipping legacy secrets tests") diff --git a/tests/integration/ha_tests/application_charm/src/charm.py b/tests/integration/ha_tests/application_charm/src/charm.py index fcdc2b7e3..9dff408ca 100755 --- a/tests/integration/ha_tests/application_charm/src/charm.py +++ b/tests/integration/ha_tests/application_charm/src/charm.py @@ -25,7 +25,7 @@ DATABASE_NAME = "continuous_writes_database" PEER = "application-peers" -LAST_WRITTEN_FILE = "/tmp/last_written_value" +LAST_WRITTEN_FILE = "last_written_value" PROC_PID_KEY = "proc-pid" @@ -124,7 +124,10 @@ def _stop_continuous_writes(self) -> Optional[int]: return None # Send a SIGTERM to the process and wait for the process to exit - os.kill(int(self.app_peer_data[PROC_PID_KEY]), signal.SIGTERM) + try: + os.kill(int(self.app_peer_data[PROC_PID_KEY]), signal.SIGTERM) + except ProcessLookupError: + logger.info(f"Process {PROC_PID_KEY} was killed already (or never existed)") del self.app_peer_data[PROC_PID_KEY] diff --git a/tests/integration/ha_tests/application_charm/src/continuous_writes.py b/tests/integration/ha_tests/application_charm/src/continuous_writes.py index 83ad65478..76caf6c10 100644 --- a/tests/integration/ha_tests/application_charm/src/continuous_writes.py +++ b/tests/integration/ha_tests/application_charm/src/continuous_writes.py @@ -52,7 +52,7 @@ def continous_writes(connection_string: str, starting_number: int): write_value += 1 - with open("/tmp/last_written_value", "w") as fd: + with open("last_written_value", "w") as fd: fd.write(str(write_value - 1)) diff --git a/tests/integration/ha_tests/helpers.py b/tests/integration/ha_tests/helpers.py index bc0d591e0..0d46f1794 100644 --- a/tests/integration/ha_tests/helpers.py +++ b/tests/integration/ha_tests/helpers.py @@ -612,7 +612,7 @@ def destroy_chaos_mesh(namespace: str) -> None: def isolate_instance_from_cluster(ops_test: OpsTest, unit_name: str) -> None: """Apply a NetworkChaos file to use chaos-mesh to simulate a network cut.""" - with tempfile.NamedTemporaryFile() as temp_file: + with tempfile.NamedTemporaryFile(dir=".") as temp_file: # Generates a manifest for chaosmesh to simulate network failure for a pod with open( "tests/integration/ha_tests/manifests/chaos_network_loss.yml", "r" diff --git a/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh b/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh index 753333145..09e4deef2 100755 --- a/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh +++ b/tests/integration/ha_tests/scripts/deploy_chaos_mesh.sh @@ -22,4 +22,5 @@ deploy_chaos_mesh() { } echo "namespace=${chaos_mesh_ns}" +chmod 0700 ~/.kube/config deploy_chaos_mesh diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f64b67a91..159c5fde4 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -72,6 +72,21 @@ async def get_password(ops_test: OpsTest, unit_id: int, username="operator") -> return action.results["password"] +async def set_password( + ops_test: OpsTest, unit_id: int, username: str = "operator", password: str = "secret" +) -> str: + """Use the charm action to retrieve the password from provided unit. + + Returns: + String with the password stored on the peer relation databag. + """ + action = await ops_test.model.units.get(f"{APP_NAME}/{unit_id}").run_action( + "set-password", **{"username": username, "password": password} + ) + action = await action.wait() + return action.results + + async def get_mongo_cmd(ops_test: OpsTest, unit_name: str): ls_code, _, _ = await ops_test.juju(f"ssh --container {unit_name} ls /usr/bin/mongosh") diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 300daa3d2..694489070 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -2,8 +2,10 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import json import logging import time +from uuid import uuid4 import pytest from lightkube import AsyncClient @@ -25,6 +27,7 @@ primary_host, run_mongo_op, secondary_mongo_uris_with_sync_delay, + set_password, ) logger = logging.getLogger(__name__) @@ -113,6 +116,90 @@ async def test_monitor_user(ops_test: OpsTest) -> None: assert return_code == 0, f"command rs.conf() on monitor user does not work, error: {stderr}" +async def test_only_leader_can_set_while_all_can_read_password_secret(ops_test: OpsTest) -> None: + """Test verifies that only the leader can set a password, while all units can read it.""" + # Setting existing password + leader_id = await get_leader_id(ops_test) + non_leaders = list(UNIT_IDS) + non_leaders.remove(leader_id) + + password = "blablabla" + await set_password(ops_test, unit_id=non_leaders[0], username="monitor", password=password) + password1 = await get_password(ops_test, unit_id=leader_id, username="monitor") + assert password1 != password + + await set_password(ops_test, unit_id=leader_id, username="monitor", password=password) + for unit_id in UNIT_IDS: + password2 = await get_password(ops_test, unit_id=unit_id, username="monitor") + assert password2 == password + + +@pytest.mark.usefixtures("only_with_juju_secrets") +async def test_reset_and_get_password_secret_same_as_cli(ops_test: OpsTest) -> None: + """Test verifies that we can set and retrieve the correct password using Juju 3.x secrets.""" + new_password = str(uuid4()) + + # Re=setting existing password + leader_id = await get_leader_id(ops_test) + result = await set_password( + ops_test, unit_id=leader_id, username="monitor", password=new_password + ) + + secret_id = result["secret-id"].split("/")[-1] + + # Getting back the pw programmatically + password = await get_password(ops_test, unit_id=leader_id, username="monitor") + + # Getting back the pw from juju CLI + complete_command = f"show-secret {secret_id} --reveal --format=json" + _, stdout, _ = await ops_test.juju(*complete_command.split()) + data = json.loads(stdout) + + assert password == new_password + assert data[secret_id]["content"]["Data"]["monitor-password"] == password + + +@pytest.mark.usefixtures("only_without_juju_secrets") +async def test_reset_and_get_password_no_secret(ops_test: OpsTest, mocker) -> None: + """Test verifies that we can set and retrieve the correct password using Juju 2.x.""" + new_password = str(uuid4()) + + # Re=setting existing password + leader_id = await get_leader_id(ops_test) + await set_password(ops_test, unit_id=leader_id, username="monitor", password=new_password) + + # Getting back the pw programmatically + password = await get_password(ops_test, unit_id=leader_id, username="monitor") + assert password == new_password + + +@pytest.mark.usefixtures("only_with_juju_secrets") +async def test_empty_password(ops_test: OpsTest) -> None: + """Test that the password can't be set to an empty string.""" + leader_id = await get_leader_id(ops_test) + + password1 = await get_password(ops_test, unit_id=leader_id, username="monitor") + await set_password(ops_test, unit_id=leader_id, username="monitor", password="") + password2 = await get_password(ops_test, unit_id=leader_id, username="monitor") + + # The password remained unchanged + assert password1 == password2 + + +@pytest.mark.usefixtures("only_with_juju_secrets") +async def test_no_password_change_on_invalid_password(ops_test: OpsTest) -> None: + """Test that in general, there is no change when password validation fails.""" + leader_id = await get_leader_id(ops_test) + password1 = await get_password(ops_test, unit_id=leader_id, username="monitor") + + # The password has to be minimum 3 characters + await set_password(ops_test, unit_id=leader_id, username="monitor", password="ca" * 1000000) + password2 = await get_password(ops_test, unit_id=leader_id, username="monitor") + + # The password didn't change + assert password1 == password2 + + async def test_scale_up(ops_test: OpsTest): """Tests juju add-unit functionality. diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 63e536e37..54d80861a 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -1,14 +1,17 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. import logging +import re import unittest from unittest import mock -from unittest.mock import patch +from unittest.mock import MagicMock, patch +import pytest from charms.mongodb.v0.helpers import CONF_DIR, DATA_DIR, KEY_FILE from ops.model import ModelError from ops.pebble import APIError, ExecError, PathError, ProtocolError from ops.testing import Harness +from parameterized import parameterized from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -33,6 +36,7 @@ class TestCharm(unittest.TestCase): @patch_network_get(private_address="1.1.1.1") def setUp(self): + self.maxDiff = None self.harness = Harness(MongoDBCharm) mongo_resource = { "registrypath": "mongo:4.4", @@ -44,6 +48,15 @@ def setUp(self): self.charm = self.harness.charm self.addCleanup(self.harness.cleanup) + @pytest.fixture + def use_caplog(self, caplog): + self._caplog = caplog + + def _setup_secrets(self): + self.harness.set_leader(True) + self.harness.charm._generate_secrets() + self.harness.set_leader(False) + @patch("charm.MongoDBCharm._pull_licenses") @patch("ops.framework.EventBase.defer") @patch("charm.MongoDBCharm._set_data_dir_permissions") @@ -618,6 +631,75 @@ def test_start_init_operator_user_after_second_call(self, connection, oversee_us defer.assert_not_called() + def test_get_password(self): + self._setup_secrets() + assert isinstance(self.harness.charm.get_secret("app", "monitor-password"), str) + self.harness.charm.get_secret("app", "non-existing-secret") is None + + self.harness.charm.set_secret("unit", "somekey", "bla") + assert isinstance(self.harness.charm.get_secret("unit", "somekey"), str) + self.harness.charm.get_secret("unit", "non-existing-secret") is None + + def test_set_reset_existing_password_app(self): + """NOTE: currently ops.testing seems to allow for non-leader to set secrets too!""" + self._setup_secrets() + + # Getting current password + self.harness.charm.set_secret("app", "monitor-password", "bla") + assert self.harness.charm.get_secret("app", "monitor-password") == "bla" + + self.harness.charm.set_secret("app", "monitor-password", "blablabla") + assert self.harness.charm.get_secret("app", "monitor-password") == "blablabla" + + @pytest.mark.usefixtures("only_with_juju_secrets") + @parameterized.expand([("app"), ("unit")]) + def test_set_secret_returning_secret_id(self, scope): + secret_id = self.harness.charm.set_secret("app", "somekey", "bla") + assert re.match("secret:([a-z0-9-]){36}", secret_id) + + @parameterized.expand([("app"), ("unit")]) + def test_set_reset_new_secret(self, scope): + """NOTE: currently ops.testing seems to allow for non-leader to set secrets too!""" + # Getting current password + self.harness.charm.set_secret(scope, "new-secret", "bla") + assert self.harness.charm.get_secret(scope, "new-secret") == "bla" + + # Reset new secret + self.harness.charm.set_secret(scope, "new-secret", "blablabla") + assert self.harness.charm.get_secret(scope, "new-secret") == "blablabla" + + # Set another new secret + self.harness.charm.set_secret(scope, "new-secret2", "blablabla") + assert self.harness.charm.get_secret(scope, "new-secret2") == "blablabla" + + @parameterized.expand([("app"), ("unit")]) + def test_invalid_secret(self, scope): + with self.assertRaises(TypeError): + self.harness.charm.set_secret("unit", "somekey", 1) + + self.harness.charm.set_secret("unit", "somekey", "") + assert self.harness.charm.get_secret(scope, "somekey") is None + + @pytest.mark.usefixtures("use_caplog") + def test_delete_password(self): + """NOTE: currently ops.testing seems to allow for non-leader to remove secrets too!""" + self._setup_secrets() + + assert self.harness.charm.get_secret("app", "monitor-password") + self.harness.charm.remove_secret("app", "monitor-password") + assert self.harness.charm.get_secret("app", "monitor-password") is None + + assert self.harness.charm.set_secret("unit", "somekey", "somesecret") + self.harness.charm.remove_secret("unit", "somekey") + assert self.harness.charm.get_secret("unit", "somekey") is None + + with self._caplog.at_level(logging.ERROR): + self.harness.charm.remove_secret("app", "non-existing-secret") + assert "No secret app:non-existing-secret" in self._caplog.text + + self.harness.charm.remove_secret("unit", "non-existing-secret") + assert "No secret unit:non-existing-secret" in self._caplog.text + @patch("charm.MongoDBConnection") @patch("charm.MongoDBCharm._connect_mongodb_exporter") def test_connect_to_mongo_exporter_on_set_password(self, connect_exporter, connection): @@ -632,6 +714,89 @@ def test_connect_to_mongo_exporter_on_set_password(self, connect_exporter, conne self.harness.charm._on_set_password(action_event) connect_exporter.assert_called() + @patch("charm.MongoDBConnection") + @patch("charm.MongoDBCharm._connect_mongodb_exporter") + def test_event_set_password_secrets(self, connect_exporter, connection): + """Test _connect_mongodb_exporter is called when the password is set for 'montior' user. + + Furthermore: in Juju 3.x we want to use secrets + """ + pw = "bla" + + self.harness.set_leader(True) + + action_event = mock.Mock() + action_event.set_results = MagicMock() + action_event.params = {"username": "monitor", "password": pw} + self.harness.charm._on_set_password(action_event) + connect_exporter.assert_called() + + action_event.set_results.assert_called() + args_pw_set = action_event.set_results.call_args.args[0] + assert "secret-id" in args_pw_set + + action_event.params = {"username": "monitor"} + self.harness.charm._on_get_password(action_event) + args_pw = action_event.set_results.call_args.args[0] + assert "password" in args_pw + assert args_pw["password"] == pw + + @patch("charm.MongoDBConnection") + @patch("charm.MongoDBCharm._connect_mongodb_exporter") + def test_event_auto_reset_password_secrets_when_no_pw_value_shipped( + self, connect_exporter, connection + ): + """Test _connect_mongodb_exporter is called when the password is set for 'montior' user. + + Furthermore: in Juju 3.x we want to use secrets + """ + self._setup_secrets() + self.harness.set_leader(True) + + action_event = mock.Mock() + action_event.set_results = MagicMock() + + # Getting current password + action_event.params = {"username": "monitor"} + self.harness.charm._on_get_password(action_event) + args_pw = action_event.set_results.call_args.args[0] + assert "password" in args_pw + pw1 = args_pw["password"] + + # No password value was shipped + action_event.params = {"username": "monitor"} + self.harness.charm._on_set_password(action_event) + connect_exporter.assert_called() + + # New password was generated + action_event.params = {"username": "monitor"} + self.harness.charm._on_get_password(action_event) + args_pw = action_event.set_results.call_args.args[0] + assert "password" in args_pw + pw2 = args_pw["password"] + + # a new password was created + assert pw1 != pw2 + + @patch("charm.MongoDBConnection") + @patch("charm.MongoDBCharm._connect_mongodb_exporter") + def test_event_any_unit_can_get_password_secrets(self, connect_exporter, connection): + """Test _connect_mongodb_exporter is called when the password is set for 'montior' user. + + Furthermore: in Juju 3.x we want to use secrets + """ + self._setup_secrets() + + action_event = mock.Mock() + action_event.set_results = MagicMock() + + # Getting current password + action_event.params = {"username": "monitor"} + self.harness.charm._on_get_password(action_event) + args_pw = action_event.set_results.call_args.args[0] + assert "password" in args_pw + assert args_pw["password"] + @patch("charm.MongoDBCharm._pull_licenses") @patch("ops.framework.EventBase.defer") @patch("charm.MongoDBCharm._set_data_dir_permissions") @@ -643,7 +808,7 @@ def test__connect_mongodb_exporter_success( container = self.harness.model.unit.get_container("mongod") self.harness.set_can_connect(container, True) self.harness.charm.on.mongod_pebble_ready.emit(container) - password = self.harness.charm.app_peer_data["monitor-password"] + password = self.harness.charm.get_secret("app", "monitor-password") uri_template = "mongodb://monitor:{password}@mongodb-k8s-0.mongodb-k8s-endpoints/?replicaSet=mongodb-k8s&authSource=admin" @@ -667,7 +832,7 @@ def test__connect_mongodb_exporter_success( action_event = mock.Mock() action_event.params = {"username": "monitor", "password": "mongo123"} self.harness.charm._on_set_password(action_event) - password = self.harness.charm.app_peer_data["monitor-password"] + password = self.harness.charm.get_secret("app", "monitor-password") updated_plan = self.harness.get_container_pebble_plan("mongod").to_dict() new_uri = ( @@ -702,7 +867,7 @@ def test__backup_user_created( container = self.harness.model.unit.get_container("mongod") self.harness.set_can_connect(container, True) self.harness.charm.on.start.emit() - password = self.harness.charm.app_peer_data["backup-password"] + password = self.harness.charm.get_secret("app", "backup-password") self.assertIsNotNone(password) # verify the password is set @patch("charm.MongoDBConnection") @@ -715,8 +880,7 @@ def test_set_password_provided(self, connection): action_event = mock.Mock() action_event.params = {"password": "canonical123", "username": "backup"} self.harness.charm._on_set_password(action_event) - new_password = self.harness.charm.app_peer_data["backup-password"] + new_password = self.harness.charm.get_secret("app", "backup-password") # verify app data is updated and results are reported to user self.assertEqual("canonical123", new_password) - action_event.set_results.assert_called_with({"password": "canonical123"}) diff --git a/tox.ini b/tox.ini index d08df443b..813f98721 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,13 @@ description = Run unit tests commands_pre = poetry install --with unit commands = - poetry export -f requirements.txt -o requirements.txt + # As long as the PyYAML issue is not resolved, + # we should manually maintain requirements.txt + # + # https://github.com/juju/python-libjuju/issues/913 + # https://chat.canonical.com/canonical/pl/zkz4odmhupgkdkekh985ohp1fw + # + # poetry export -f requirements.txt -o requirements.txt poetry run coverage run --source={[vars]src_path} \ -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit poetry run coverage report