From ba7cc7a488e260162a14b14e559dbaa4d0ab10fa Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Thu, 7 Nov 2024 13:18:12 +1100 Subject: [PATCH] chore: dump previous logs as well --- charts/ftl/values.yaml | 4 -- deployment/values.yaml | 6 ++- internal/integration/harness.go | 81 +++++++++++++++++++++++++-------- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/charts/ftl/values.yaml b/charts/ftl/values.yaml index 4c15842d6..47dcea75a 100644 --- a/charts/ftl/values.yaml +++ b/charts/ftl/values.yaml @@ -186,10 +186,6 @@ runner: valueFrom: fieldRef: fieldPath: status.hostIP -# - name: OTEL_EXPORTER_OTLP_ENDPOINT -# value: "http://$(HOST_IP):4317" -# - name: OTEL_RESOURCE_ATTRIBUTES -# value: "env=ftlDefault" ports: - name: http diff --git a/deployment/values.yaml b/deployment/values.yaml index 1b7e5dabd..b9624ca26 100644 --- a/deployment/values.yaml +++ b/deployment/values.yaml @@ -17,7 +17,11 @@ controller: ports: - name: "http-8891" port: 8891 - +runner: + podAnnotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true + sidecar.istio.io/logLevel: "debug" provisioner: enabled: true diff --git a/internal/integration/harness.go b/internal/integration/harness.go index bc3fb7130..285c44138 100644 --- a/internal/integration/harness.go +++ b/internal/integration/harness.go @@ -542,29 +542,70 @@ func dumpKubePods(ctx context.Context, kubeClient optional.Option[kubernetes.Cli continue } for _, container := range pod.Spec.Containers { - path := filepath.Join(dumpPath, pod.Name, container.Name+".log") - req := client.CoreV1().Pods(kubeNamespace).GetLogs(pod.Name, &kubecore.PodLogOptions{Container: container.Name}) - podLogs, err := req.Stream(context.Background()) - if err != nil { - Infof("Error getting logs for pod %s: %v", pod.Name, err) - continue - } - defer func() { - _ = podLogs.Close() - }() - buf := new(bytes.Buffer) - _, err = io.Copy(buf, podLogs) - if err != nil { - Infof("Error copying logs for pod %s: %v", pod.Name, err) - continue - } - str := buf.String() - err = os.WriteFile(path, []byte(str), 0644) // #nosec - if err != nil { - Infof("Error writing logs for pod %s: %v", pod.Name, err) + for _, prev := range []bool{false, true} { + var path string + if prev { + path = filepath.Join(dumpPath, pod.Name, container.Name+"-previous.log") + } else { + path = filepath.Join(dumpPath, pod.Name, container.Name+".log") + } + req := client.CoreV1().Pods(kubeNamespace).GetLogs(pod.Name, &kubecore.PodLogOptions{Container: container.Name, Previous: prev}) + podLogs, err := req.Stream(context.Background()) + if err != nil { + if prev { + // This is pretty normal not to have previous logs + continue + } + Infof("Error getting logs for pod %s: %v previous: %v", pod.Name, err, prev) + continue + } + defer func() { + _ = podLogs.Close() + }() + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + Infof("Error copying logs for pod %s: %v", pod.Name, err) + continue + } + str := buf.String() + err = os.WriteFile(path, []byte(str), 0644) // #nosec + if err != nil { + Infof("Error writing logs for pod %s: %v", pod.Name, err) + } + } + } } } + istio, err := k8sscaling.CreateIstioClientSet() + if err != nil { + Infof("Error creating istio clientset: %v", err) + return + } + auths, err := istio.SecurityV1().AuthorizationPolicies(kubeNamespace).List(ctx, kubemeta.ListOptions{}) + if err == nil { + for _, policy := range auths.Items { + Infof("Dumping yamp for auth policy %s", policy.Name) + policyPath := filepath.Join(dumpPath, policy.Name) + err := os.MkdirAll(policyPath, 0755) // #nosec + if err != nil { + Infof("Error creating directory %s: %v", policyPath, err) + continue + } + podYaml, err := yaml.Marshal(policy) + if err != nil { + Infof("Error marshalling pod %s: %v", policy.Name, err) + continue + } + err = os.WriteFile(filepath.Join(policyPath, "pod.yaml"), podYaml, 0644) // #nosec + if err != nil { + Infof("Error writing policy %s: %v", policy.Name, err) + continue + } + + } + } } }