Skip to content

Commit

Permalink
feat: refactor certs generation and add tests (#104)
Browse files Browse the repository at this point in the history 
* refactor: lint method for generating certs
* refactor: method for generatig certs, extract sslconfig to a file
* feat: refactor and test cert generation

* feat: more refactoring of cert generation
- moved gen_certs to certs.py
- added gen_certs_if_missing
- test gen_certs_if_missing
- remove unneeded mocked_gen_certs fixture

* fix: pin microk8s to 1.24
* feat: add error handling to push certs
* fix: return after generating certs
* fix: add microk8s add-ons to CI
* fix: use Jinja to render template
* fix: remove set model name
* feat: raise GenericCharmRuntimeError
* fix: restart controller test fixtures

---------

Co-authored-by: Daniela Plascencia <daniela.plascencia@canonical.com>
  • Loading branch information
@NohaIhab @DnPlas
NohaIhab and DnPlas authored May 30, 2023
1 parent 820f945 commit ba760a9
Show file tree
Hide file tree
Showing 5 changed files with 274 additions and 149 deletions.
97 changes: 97 additions & 0 deletions charms/kserve-controller/src/certs.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Copyright 2023 Canonical Ltd.
# See LICENSE file for licensing details.


import tempfile
from pathlib import Path
from subprocess import check_call

from jinja2 import Template

SSL_CONFIG_FILE = "src/templates/ssl.conf.j2"


def gen_certs(service_name: str, namespace: str, webhook_service: str):
"""Generate certificates."""

template = Template(Path(SSL_CONFIG_FILE).read_text())
ssl_conf = template.render(
service_name=str(service_name),
namespace=str(namespace),
webhook_server_service=str(webhook_service),
)

with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
(tmp_path / "ssl.conf").write_text(ssl_conf)

# execute OpenSSL commands
check_call(["openssl", "genrsa", "-out", tmp_path / "ca.key", "2048"])
check_call(["openssl", "genrsa", "-out", tmp_path / "server.key", "2048"])
check_call(
[
"openssl",
"req",
"-x509",
"-new",
"-sha256",
"-nodes",
"-days",
"3650",
"-key",
tmp_path / "ca.key",
"-subj",
"/CN=127.0.0.1",
"-out",
tmp_path / "ca.crt",
]
)
check_call(
[
"openssl",
"req",
"-new",
"-sha256",
"-key",
tmp_path / "server.key",
"-out",
tmp_path / "server.csr",
"-config",
tmp_path / "ssl.conf",
]
)
check_call(
[
"openssl",
"x509",
"-req",
"-sha256",
"-in",
tmp_path / "server.csr",
"-CA",
tmp_path / "ca.crt",
"-CAkey",
tmp_path / "ca.key",
"-CAcreateserial",
"-out",
tmp_path / "cert.pem",
"-days",
"365",
"-extensions",
"v3_ext",
"-extfile",
tmp_path / "ssl.conf",
]
)

ret_certs = {
"cert": (tmp_path / "cert.pem").read_text(),
"key": (tmp_path / "server.key").read_text(),
"ca": (tmp_path / "ca.crt").read_text(),
}

# cleanup temporary files
for file in tmp_path.glob("cert-gen-*"):
file.unlink()

return ret_certs
202 changes: 80 additions & 122 deletions charms/kserve-controller/src/charm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@

import logging
from base64 import b64encode
from pathlib import Path
from subprocess import check_call

from charmed_kubeflow_chisme.exceptions import ErrorWithStatus, GenericCharmRuntimeError
from charmed_kubeflow_chisme.kubernetes import KubernetesResourceHandler
Expand All @@ -28,32 +26,39 @@
)
from lightkube import ApiError
from ops.charm import CharmBase
from ops.framework import StoredState
from ops.main import main
from ops.model import (
ActiveStatus,
BlockedStatus,
Container,
MaintenanceStatus,
ModelError,
WaitingStatus,
)
from ops.pebble import APIError, Layer
from ops.pebble import APIError, Layer, PathError, ProtocolError

from certs import gen_certs

# from lightkube_custom_resources.serving import ClusterServingRuntime_v1alpha1

log = logging.getLogger(__name__)

CONFIG_FILES = ["src/templates/configmap_manifests.yaml.j2"]
CONTAINER_CERTS_DEST = "/tmp/k8s-webhook-server/serving-certs/"
K8S_RESOURCE_FILES = [
"src/templates/crd_manifests.yaml.j2",
"src/templates/auth_manifests.yaml.j2",
"src/templates/serving_runtimes_manifests.yaml.j2",
"src/templates/webhook_manifests.yaml.j2",
]
CONFIG_FILES = ["src/templates/configmap_manifests.yaml.j2"]


class KServeControllerCharm(CharmBase):
"""Charm the service."""

_stored = StoredState()

def __init__(self, *args):
super().__init__(*args)
self._ingress_gateway_requirer = GatewayRequirer(self, relation_name="ingress-gateway")
Expand Down Expand Up @@ -88,15 +93,20 @@ def __init__(self, *args):
self._lightkube_field_manager = "lightkube"
self._controller_container_name = "kserve-controller"
self.controller_container = self.unit.get_container(self._controller_container_name)
self._controller_service_name = self.app.name
self._namespace = self.model.name
self._webhook_service_name = "kserve-webhook-server-service"

# Generate self-signed certificates and store them
self._gen_certs_if_missing()

self._rbac_proxy_container_name = "kube-rbac-proxy"
self.rbac_proxy_container = self.unit.get_container(self._rbac_proxy_container_name)

@property
def _context(self):
"""Returns a dictionary containing context to be used for rendering."""
self.gen_certs(self.model.name, self.app.name)
ca_context = b64encode(Path("/run/ca.crt").read_text().encode("ascii"))
ca_context = b64encode(self._stored.ca.encode("ascii"))
return {
"app_name": self.app.name,
"namespace": self.model.name,
Expand Down Expand Up @@ -180,7 +190,7 @@ def _rbac_proxy_pebble_layer(self):
self._rbac_proxy_container_name: {
"override": "replace",
"summary": "Kube Rbac Proxy",
"command": "/usr/local/bin/kube-rbac-proxy --secure-listen-address=0.0.0.0:8443 --upstream=http://127.0.0.1:8080 --logtostderr=true --v=10",
"command": "/usr/local/bin/kube-rbac-proxy --secure-listen-address=0.0.0.0:8443 --upstream=http://127.0.0.1:8080 --logtostderr=true --v=10", # noqa E501
"startup": "enabled",
}
}
Expand All @@ -203,9 +213,11 @@ def _on_kserve_controller_ready(self, event):
Learn more about Pebble layers at https://github.com/canonical/pebble
"""
try:
self.gen_certs(self.model.name, self.app.name)
self._push_controller_certificates()

self._upload_certs_to_container(
container=self.controller_container,
destination_path=CONTAINER_CERTS_DEST,
certs_store=self._stored,
)
update_layer(
self._controller_container_name,
self.controller_container,
Expand Down Expand Up @@ -256,8 +268,7 @@ def _on_install(self, event):
except ApiError as api_err:
log.error(api_err)
raise
else:
self.model.unit.status = ActiveStatus()
self.model.unit.status = ActiveStatus()

def _on_config_changed(self, event):
self._on_install(event)
Expand Down Expand Up @@ -308,6 +319,18 @@ def _on_local_gateway_relation_broken(self, _) -> None:
self.unit.status = BlockedStatus("Please relate to knative-serving:local-gateway")
return

def _check_container_connection(self, container: Container) -> None:
"""Check if connection can be made with container.
Args:
container: the named container in a unit to check.
Raises:
ErrorWithStatus if the connection cannot be made.
"""
if not container.can_connect():
raise ErrorWithStatus("Pod startup is not complete", MaintenanceStatus)

def _generate_gateways_context(self) -> dict:
"""Generates the ingress context based on certain rules.
Expand Down Expand Up @@ -363,118 +386,53 @@ def _generate_gateways_context(self) -> dict:

return gateways_context

def gen_certs(self, namespace, service_name):
"""Generate certificates."""
if Path("/run/cert.pem").exists():
log.info("Found existing cert.pem, not generating new cert.")
return

Path("/run/ssl.conf").write_text(
f"""[ req ]
default_bits = 2048
prompt = no
default_md = sha256
req_extensions = req_ext
distinguished_name = dn
[ dn ]
C = GB
ST = Canonical
L = Canonical
O = Canonical
OU = Canonical
CN = 127.0.0.1
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
DNS.1 = {service_name}
DNS.2 = {service_name}.{namespace}
DNS.3 = {service_name}.{namespace}.svc
DNS.4 = {service_name}.{namespace}.svc.cluster
DNS.5 = {service_name}.{namespace}.svc.cluster.local
DNS.6 = kserve-webhook-server-service
DNS.7 = kserve-webhook-server-service.{namespace}
DNS.8 = kserve-webhook-server-service.{namespace}.svc
DNS.9 = kserve-webhook-server-service.{namespace}.svc.cluster
DNS.10 = kserve-webhook-server-service.{namespace}.svc.cluster.local
IP.1 = 127.0.0.1
[ v3_ext ]
authorityKeyIdentifier=keyid,issuer:always
basicConstraints=CA:FALSE
keyUsage=keyEncipherment,dataEncipherment,digitalSignature
extendedKeyUsage=serverAuth,clientAuth
subjectAltName=@alt_names"""
)

check_call(["openssl", "genrsa", "-out", "/run/ca.key", "2048"])
check_call(["openssl", "genrsa", "-out", "/run/server.key", "2048"])
check_call(
[
"openssl",
"req",
"-x509",
"-new",
"-sha256",
"-nodes",
"-days",
"3650",
"-key",
"/run/ca.key",
"-subj",
"/CN=127.0.0.1",
"-out",
"/run/ca.crt",
]
)
check_call(
[
"openssl",
"req",
"-new",
"-sha256",
"-key",
"/run/server.key",
"-out",
"/run/server.csr",
"-config",
"/run/ssl.conf",
]
)
check_call(
[
"openssl",
"x509",
"-req",
"-sha256",
"-in",
"/run/server.csr",
"-CA",
"/run/ca.crt",
"-CAkey",
"/run/ca.key",
"-CAcreateserial",
"-out",
"/run/cert.pem",
"-days",
"365",
"-extensions",
"v3_ext",
"-extfile",
"/run/ssl.conf",
]
def _gen_certs_if_missing(self) -> None:
"""Generate certificates if they don't already exist in _stored."""
log.info("Generating certificates if missing.")
cert_attributes = ["cert", "ca", "key"]
# Generate new certs if any cert attribute is missing
for cert_attribute in cert_attributes:
try:
getattr(self._stored, cert_attribute)
log.info(f"Certificate {cert_attribute} already exists, skipping generation.")
except AttributeError:
self._gen_certs()
return

def _gen_certs(self):
"""Refresh the certificates, overwriting all attributes if any attribute is missing."""
log.info("Generating certificates..")
certs = gen_certs(
service_name=self._controller_service_name,
namespace=self._namespace,
webhook_service=self._webhook_service_name,
)
for k, v in certs.items():
setattr(self._stored, k, v)

def _upload_certs_to_container(
self, container: Container, destination_path: str, certs_store: StoredState
) -> None:
"""Upload generated certs to container.
Args:
container (Container): the container object to push certs to.
destination_path (str): path in str format where certificates will
be stored in the container.
certs_store (StoredState): an object where the certificate contents are stored.
"""
try:
self._check_container_connection(container)
except ErrorWithStatus as error:
self.model.unit.status = error.status
return

def _push_controller_certificates(self):
"""Push certificates to the kserve-controller workload container."""
self.controller_container.push(
"/tmp/k8s-webhook-server/serving-certs/tls.crt",
Path("/run/cert.pem").read_text(),
make_dirs=True,
)
self.controller_container.push(
"/tmp/k8s-webhook-server/serving-certs/tls.key",
Path("/run/server.key").read_text(),
make_dirs=True,
)
try:
container.push(f"{destination_path}/tls.key", certs_store.key, make_dirs=True)
container.push(f"{destination_path}/tls.crt", certs_store.cert, make_dirs=True)
container.push(f"{destination_path}/ca.crt", certs_store.ca, make_dirs=True)
except (ProtocolError, PathError) as e:
raise GenericCharmRuntimeError("Failed to push certs to container") from e

def _restart_controller_service(self) -> None:
"""Restart the kserve-controller service.
Expand Down
Loading

0 comments on commit ba760a9

Please sign in to comment.