fix: return empty deployment if not found (#2716)

## Description
Restarts on grafloki start over k8s were not working because of the way
the not found error was being handled. This change returns a nil object
if a deployment isn't found by k8s instead of returning an error.

Author: tedim52

cli/cli/commands/service/service_helpers/service_helpers.go

@@ -130,7 +130,7 @@ func GetAddServiceStarlarkScript(serviceName string, serviceConfigStarlark strin
 }
 
 func RunAddServiceStarlarkScript(ctx context.Context, serviceName, enclaveIdentifier, starlarkScript string, enclaveCtx *enclaves.EnclaveContext) (*enclaves.StarlarkRunResult, error) {
-	logrus.Infof("ADD SERVICE STARLARK:\n%v", starlarkScript)
+	logrus.Debugf("Add service starlark:\n%v", starlarkScript)
 	starlarkRunResult, err := enclaveCtx.RunStarlarkScriptBlocking(ctx, starlarkScript, starlark_run_config.NewRunStarlarkConfig())
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error has occurred when running Starlark to add service")

cli/cli/helpers/grafloki/docker_grafloki.go

@@ -50,29 +50,38 @@ func StartGrafLokiInDocker(ctx context.Context, graflokiConfig resolved_config.G
 	}
 
 	var lokiHost string
+	var removeGrafanaAndLokiFunc func()
+	shouldRemoveGrafanaAndLoki := false
 	doesGrafanaAndLokiExist, lokiHost, err := checkGrafanaAndLokiContainerExistence(ctx, dockerManager, lokiContainerLabels, grafanaContainerLabels)
 	if err != nil {
 		return "", "", stacktrace.Propagate(err, "An error occurred checking if Grafana and Loki exist.")
 	}
 	if !doesGrafanaAndLokiExist {
 		logrus.Infof("No running Grafana and Loki containers found. Creating them...")
-		lokiHost, err = createGrafanaAndLokiContainers(ctx, dockerManager, graflokiConfig)
+		lokiHost, removeGrafanaAndLokiFunc, err = createGrafanaAndLokiContainers(ctx, dockerManager, graflokiConfig)
 		if err != nil {
 			return "", "", stacktrace.Propagate(err, "An error occurred creating Grafana and Loki containers.")
 		}
+		shouldRemoveGrafanaAndLoki = true
+		defer func() {
+			if shouldRemoveGrafanaAndLoki {
+				removeGrafanaAndLokiFunc()
+			}
+		}()
 	}
 
 	grafanaUrl := fmt.Sprintf("http://%v:%v", localhostAddr, grafanaPort)
+	shouldRemoveGrafanaAndLoki = false
 	return lokiHost, grafanaUrl, nil
 }
 
-func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_manager.DockerManager, graflokConfig resolved_config.GrafanaLokiConfig) (string, error) {
+func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_manager.DockerManager, graflokConfig resolved_config.GrafanaLokiConfig) (string, func(), error) {
 	lokiNatPort := nat.Port(strconv.Itoa(lokiPort) + "/tcp")
 	grafanaNatPort := nat.Port(strconv.Itoa(grafanaPort) + "/tcp")
 
 	bridgeNetworkId, err := dockerManager.GetNetworkIdByName(ctx, bridgeNetworkName)
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred getting Docker network id by Name: %v", bridgeNetworkName)
+		return "", nil, stacktrace.Propagate(err, "An error occurred getting Docker network id by Name: %v", bridgeNetworkName)
 	}
 
 	lokiImage := defaultLokiImage
@@ -81,7 +90,7 @@ func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_m
 	}
 	lokiUuid, err := uuid_generator.GenerateUUIDString()
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred generating a uuid for Loki.")
+		return "", nil, stacktrace.Propagate(err, "An error occurred generating a uuid for Loki.")
 	}
 	lokiContainerName := fmt.Sprintf("%v%v", LokiContainerNamePrefix, lokiUuid)
 	lokiArgs := docker_manager.NewCreateAndStartContainerArgsBuilder(lokiImage, lokiContainerName, bridgeNetworkId).
@@ -95,29 +104,32 @@ func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_m
 		Build()
 	lokiContainerId, _, err := dockerManager.CreateAndStartContainer(ctx, lokiArgs)
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred creating '%v' container.", lokiContainerName)
+		return "", nil, stacktrace.Propagate(err, "An error occurred creating '%v' container.", lokiContainerName)
 	}
 	shouldDestroyLokiContainer := true
-	defer func() {
+	removeLokiContainerFunc := func() {
 		if shouldDestroyLokiContainer {
 			err := dockerManager.RemoveContainer(ctx, lokiContainerId)
 			if err != nil {
 				logrus.Warnf("Attempted to remove Loki container after an error occurred creating it but an error occurred removing it.")
 				logrus.Warnf("Manually remove Loki container with id: %v", lokiContainerId)
 			}
 		}
+	}
+	defer func() {
+		removeLokiContainerFunc()
 	}()
 	logrus.Infof("Loki container started.")
 
 	lokiBridgeNetworkIpAddr, err := dockerManager.GetContainerIPOnNetwork(ctx, lokiContainerId, bridgeNetworkName)
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred getting container '%v' ip address on network '%v'.", lokiContainerId, bridgeNetworkName)
+		return "", nil, stacktrace.Propagate(err, "An error occurred getting container '%v' ip address on network '%v'.", lokiContainerId, bridgeNetworkName)
 	}
 
 	lokiBridgeNetworkIpAddress := fmt.Sprintf("http://%v:%v", lokiBridgeNetworkIpAddr, lokiPort)
 	lokiHostNetworkIpAddress := fmt.Sprintf("http://%v:%v", localhostAddr, lokiPort)
 	if err := waitForLokiReadiness(lokiHostNetworkIpAddress, lokiReadinessPath); err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred waiting for Loki container to become ready.")
+		return "", nil, stacktrace.Propagate(err, "An error occurred waiting for Loki container to become ready.")
 	}
 
 	grafanaDatasource := &GrafanaDatasources{
@@ -134,17 +146,17 @@ func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_m
 		}}
 	grafanaDatasourceYaml, err := yaml.Marshal(grafanaDatasource)
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred serializing Grafana datasource to yaml: %v", grafanaDatasourceYaml)
+		return "", nil, stacktrace.Propagate(err, "An error occurred serializing Grafana datasource to yaml: %v", grafanaDatasourceYaml)
 	}
 	logrus.Debugf("Grafana data source yaml %v", string(grafanaDatasourceYaml))
 
 	tmpFile, err := os.CreateTemp("", "grafana-datasource-*.yaml")
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred creating temp datasource config.")
+		return "", nil, stacktrace.Propagate(err, "An error occurred creating temp datasource config.")
 	}
 	defer tmpFile.Close()
 	if _, err := tmpFile.WriteString(string(grafanaDatasourceYaml)); err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred writing config.")
+		return "", nil, stacktrace.Propagate(err, "An error occurred writing config.")
 	}
 
 	grafanaImage := defaultGrafanaImage
@@ -154,7 +166,7 @@ func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_m
 	root := service_user.NewServiceUser(rootUserUid)
 	grafanaUuid, err := uuid_generator.GenerateUUIDString()
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error occurred generating a uuid for Grafana.")
+		return "", nil, stacktrace.Propagate(err, "An error occurred generating a uuid for Grafana.")
 	}
 	grafanaContainerName := fmt.Sprintf("%v%v", GrafanaContainerNamePrefix, grafanaUuid)
 	grafanaArgs := docker_manager.NewCreateAndStartContainerArgsBuilder(grafanaImage, grafanaContainerName, bridgeNetworkId).
@@ -177,23 +189,31 @@ func createGrafanaAndLokiContainers(ctx context.Context, dockerManager *docker_m
 		Build()
 	grafanaContainerId, _, err := dockerManager.CreateAndStartContainer(ctx, grafanaArgs)
 	if err != nil {
-		return "", stacktrace.Propagate(err, "An error creating creating '%v' container.", grafanaContainerName)
+		return "", nil, stacktrace.Propagate(err, "An error creating creating '%v' container.", grafanaContainerName)
 	}
 	shouldDestroyGrafanaContainer := true
-	defer func() {
+	removeGrafanaContainerFunc := func() {
 		if shouldDestroyGrafanaContainer {
 			err := dockerManager.RemoveContainer(ctx, grafanaContainerId)
 			if err != nil {
 				logrus.Warnf("Attempted to remove Grafana container after an error occurred creating it but an error occurred removing it.")
 				logrus.Warnf("Manually remove Grafana container with id: %v", grafanaContainerId)
 			}
 		}
+	}
+	defer func() {
+		removeGrafanaContainerFunc()
 	}()
 	logrus.Infof("Grafana container started.")
 
+	removeGrafanaAndLokiContainersFunc := func() {
+		removeLokiContainerFunc()
+		removeGrafanaContainerFunc()
+	}
+
 	shouldDestroyLokiContainer = false
 	shouldDestroyGrafanaContainer = false
-	return lokiBridgeNetworkIpAddress, nil
+	return lokiBridgeNetworkIpAddress, removeGrafanaAndLokiContainersFunc, nil
 }
 
 func waitForLokiReadiness(lokiHost string, readyPath string) error {

cli/cli/helpers/grafloki/kubernetes_grafloki.go

@@ -30,7 +30,7 @@ const (
 	lokiProbeTimeoutSeconds              = 10
 
 	// takes around 30 seconds for loki pod to become ready
-	lokiDeploymentMaxRetries    = 40
+	lokiDeploymentMaxRetries    = 60
 	lokiDeploymentRetryInterval = 1 * time.Second
 	defaultStorageClass         = ""
 )
@@ -53,8 +53,12 @@ func StartGrafLokiInKubernetes(ctx context.Context, graflokiConfig resolved_conf
 	var lokiHost string
 	var removeGrafanaAndLokiFunc func()
 	shouldRemoveGrafanaAndLoki := false
-	doesGrafanaAndLokiExist, lokiHost := checkGrafanaAndLokiDeploymentExistence(ctx, k8sManager)
+	doesGrafanaAndLokiExist, lokiHost, err := checkGrafanaAndLokiDeploymentExistence(ctx, k8sManager)
+	if err != nil {
+		return "", "", stacktrace.Propagate(err, "An error occurred checking if Grafana and Loki exist.")
+	}
 	if !doesGrafanaAndLokiExist {
+		logrus.Infof("No running Grafana and Loki deployments found. Creating them...")
 		lokiHost, removeGrafanaAndLokiFunc, err = createGrafanaAndLokiDeployments(ctx, k8sManager, graflokiConfig)
 		if err != nil {
 			return "", "", stacktrace.Propagate(err, "An error occurred creating Grafana and Loki deployments.")
@@ -445,27 +449,29 @@ func createGrafanaAndLokiDeployments(ctx context.Context, k8sManager *kubernetes
 	return lokiHost, removeGrafanaAndLokiDeploymentsFunc, nil
 }
 
-func checkGrafanaAndLokiDeploymentExistence(ctx context.Context, k8sManager *kubernetes_manager.KubernetesManager) (bool, string) {
+func checkGrafanaAndLokiDeploymentExistence(ctx context.Context, k8sManager *kubernetes_manager.KubernetesManager) (bool, string, error) {
 	existsLoki := false
 	existsGrafana := false
 	var lokiHost string
 
 	lokiDeployment, err := k8sManager.GetDeployment(ctx, graflokiNamespace, lokiDeploymentName)
-	if err == nil && lokiDeployment != nil {
+	if err != nil {
+		return false, "", stacktrace.Propagate(err, "An error occurred getting Loki deployment '%v'", lokiDeploymentName)
+	}
+	if lokiDeployment != nil {
 		existsLoki = true
 		lokiHost = getLokiUrlInsideK8sCluster(lokiServiceName, graflokiNamespace, lokiNodePort)
-	} else {
-		return existsLoki, "" // loki doesn't in this case so eject early
 	}
 
 	grafanaDeployment, err := k8sManager.GetDeployment(ctx, graflokiNamespace, grafanaDeploymentName)
-	if err == nil && grafanaDeployment != nil {
-		existsGrafana = false
-	} else {
-		return existsGrafana, ""
+	if err != nil {
+		return false, "", stacktrace.Propagate(err, "An error occurred getting Grafana deployment '%v'", grafanaDeploymentName)
+	}
+	if grafanaDeployment != nil {
+		existsGrafana = true
 	}
 
-	return existsLoki && existsGrafana, lokiHost
+	return existsLoki && existsGrafana, lokiHost, nil
 }
 
 func StopGrafLokiInKubernetes(ctx context.Context) error {

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"net/http"
 	"net/url"
 	"os"
@@ -1444,20 +1445,23 @@ func (manager *KubernetesManager) RemoveDeployment(ctx context.Context, namespac
 }
 
 func (manager *KubernetesManager) GetDeployment(ctx context.Context, namespace string, name string) (*v1.Deployment, error) {
-	daemonSetClient := manager.kubernetesClientSet.AppsV1().Deployments(namespace)
+	deploymentClient := manager.kubernetesClientSet.AppsV1().Deployments(namespace)
 
-	daemonSet, err := daemonSetClient.Get(ctx, name, metav1.GetOptions{
+	deployment, err := deploymentClient.Get(ctx, name, metav1.GetOptions{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "",
 			APIVersion: "",
 		},
 		ResourceVersion: "",
 	})
+	if apierrors.IsNotFound(err) {
+		return nil, nil // in the case the deployment doesn't exist, simply return a nil object
+	}
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "Failed to get daemon set with name '%s'", name)
+		return nil, stacktrace.Propagate(err, "Failed to get deployment with name '%s'", name)
 	}
 
-	return daemonSet, nil
+	return deployment, nil
 }
 
 func (manager *KubernetesManager) CreateDeployment(

docs/docs/cli-reference/service-add.md

@@ -10,12 +10,13 @@ To add a service to an enclave, run:
 kurtosis service add $THE_ENCLAVE_IDENTIFIER $THE_SERVICE_IDENTIFIER $CONTAINER_IMAGE
 ```
 
-where `$THE_ENCLAVE_IDENTIFIER` and the `$THE_SERVICE_IDENTIFIER` are [resource identifiers](../advanced-concepts/resource-identifier.md) for the enclave and service, respectively. 
+where `$THE_ENCLAVE_IDENTIFIER` and the `$THE_SERVICE_IDENTIFIER` are [resource identifiers](../advanced-concepts/resource-identifier.md) for the enclave and service, respectively.
 Note, the service identifier needs to be formatted according to RFC 1035. Specifically, 1-63 lowercase alphanumeric characters with dashes and cannot start or end with dashes. Also service names
-have to start with a lowercase alphabet. 
+have to start with a lowercase alphabet.
 
 Much like `docker run`, this command has multiple options available to customize the service that's started:
 
+1. The `--cmd` flag can be used to override the default command that the container runs
 1. The `--entrypoint` flag can be passed in to override the binary the service runs
 1. The `--env` flag can be used to specify a set of environment variables that should be set when running the service
 1. The `--ports` flag can be used to set the ports that the service will listen on
@@ -25,3 +26,20 @@ To override the service's CMD, add a `--` after the image name and then pass in
 ```bash
 kurtosis service add --entrypoint sh my-enclave test-service alpine -- -c "echo 'Hello world'"
 ```
+
+Alternatively, if you have an existing service config in JSON format (for example, one that was output using `kurtosis service inspect`), you can use the `--json-service-config` flag to add a service using that config:
+
+```bash
+kurtosis service add my-enclave test-service --json-service-config ./my-service-config.json
+```
+
+To read the JSON config from stdin, use:
+
+```bash
+kurtosis service add my-enclave test-service --json-service-config - < ./my-service-config.json
+```
+
+:::note Override
+When using `--json-service-config`, the standard flags and args like `--image`, `--cmd`, `--entrypoint`, `--env`, and `$CONTAINER_IMAGE` will be ignored in favor of the provided config.
+:::
+

docs/docs/cli-reference/service-inspect.md

@@ -22,3 +22,8 @@ Running the above command will print detailed information about:
 
 By default, the service UUID is shortened. To view the full UUID of your service, add the following flag:
 * `--full-uuid`
+
+You can also control the output format using the `--output` (`-o`) flag:
+* `--output yaml` will print the service config in YAML format
+* `--output json` will print the service config in JSON format (this can be piped into `service add` via `--json-service-config`)
+* If `--output` is omitted, the result will be printed in a human-readable format

docs/docs/cli-reference/service-update.md

@@ -0,0 +1,39 @@
+---
+title: service update
+sidebar_label: service update
+slug: /service-update
+---
+
+To update an existing service in an enclave, run:
+
+```bash
+kurtosis service update $THE_ENCLAVE_IDENTIFIER $THE_SERVICE_IDENTIFIER [flags]
+```
+
+where `$THE_ENCLAVE_IDENTIFIER` and `$THE_SERVICE_IDENTIFIER` are [resource identifiers](../advanced-concepts/resource-identifier.md) for the enclave and service, respectively.
+
+This command updates a service in-place by modifying its configuration. Only the specified parameters will be changed — the rest of the service config will remain as-is.
+
+Much like `docker run`, this command has multiple options available to customize the updated service:
+
+1. The `--image` flag can be used to update the service’s container image
+1. The `--entrypoint` flag can override the binary the service runs
+1. The `--env` flag can be used to set or override environment variables. Env var overrides with the same key will override existing env vars.
+1. The `--ports` flag can be used to add or override private port definitions. Port overrides with the same port id will override existing port bindings.
+1. The `--files` flag can be used to mount new file artifacts. Files artifacts overrides with the same key will override existing files artifact mounts.
+1. The `--cmd` flag can be used to override the CMD that is run when the container starts
+
+Example:
+
+```bash
+kurtosis service update my-enclave test-service \
+  --image my-custom-image \
+  --entrypoint my-binary \
+  --env "FOO:bar,BAR:baz" \
+  --ports "port1:8080/tcp"
+```
+
+:::note Restarted Container
+This command replaces the existing service with a new container using the updated configuration. The service will be briefly stopped and restarted as part of this process.
+:::
+