From 9164ecdc1f6c5d766c972da417698e03473bba69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kie=C5=82kowicz?= Date: Thu, 15 Sep 2022 13:49:25 +0200 Subject: [PATCH] .NET - enable bytecode instrumentation (#1081) * .NET Auto Instrumentation - support for byte code instrumentation * extract method to set env variable * code review feedback --- autoinstrumentation/dotnet/Dockerfile | 4 + pkg/instrumentation/dotnet.go | 109 ++++++----- pkg/instrumentation/dotnet_test.go | 180 +++++++++++++++++- pkg/instrumentation/podmutator_test.go | 16 ++ pkg/instrumentation/sdk_test.go | 16 ++ .../01-assert.yaml | 24 ++- .../02-assert.yaml | 12 +- .../e2e/instrumentation-dotnet/01-assert.yaml | 12 +- 8 files changed, 318 insertions(+), 55 deletions(-) diff --git a/autoinstrumentation/dotnet/Dockerfile b/autoinstrumentation/dotnet/Dockerfile index af31050341..2d6d75c2b3 100644 --- a/autoinstrumentation/dotnet/Dockerfile +++ b/autoinstrumentation/dotnet/Dockerfile @@ -3,9 +3,13 @@ # one init container will be created to copy the files to your app's container. # - Grant the necessary access to the files in the `/autoinstrumentation` directory. # - Following environment variables are injected to the application container to enable the auto-instrumentation. +# CORECLR_ENABLE_PROFILING=1 +# CORECLR_PROFILER={918728DD-259F-4A6A-AC2B-B85E1B658318} +# CORECLR_PROFILER_PATH=%InstallationLocation%/OpenTelemetry.AutoInstrumentation.Native.so # DOTNET_ADDITIONAL_DEPS=%InstallationLocation%/AdditionalDeps # DOTNET_SHARED_STORE=%InstallationLocation%/store # DOTNET_STARTUP_HOOKS=%InstallationLocation%/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll +# OTEL_DOTNET_AUTO_HOME=%InstallationLocation% FROM busybox diff --git a/pkg/instrumentation/dotnet.go b/pkg/instrumentation/dotnet.go index ee933ef339..210db334f0 100644 --- a/pkg/instrumentation/dotnet.go +++ b/pkg/instrumentation/dotnet.go @@ -24,12 +24,20 @@ import ( ) const ( - envDotNetAdditionalDeps = "DOTNET_ADDITIONAL_DEPS" - envDotNetSharedStore = "DOTNET_SHARED_STORE" - envDotNetStartupHook = "DOTNET_STARTUP_HOOKS" - dotNetAdditionalDepsPath = "./otel-auto-instrumentation/AdditionalDeps" - dotNetSharedStorePath = "/otel-auto-instrumentation/store" - dotNetStartupHookPath = "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" + envDotNetCoreClrEnableProfiling = "CORECLR_ENABLE_PROFILING" + envDotNetCoreClrProfiler = "CORECLR_PROFILER" + envDotNetCoreClrProfilerPath = "CORECLR_PROFILER_PATH" + envDotNetAdditionalDeps = "DOTNET_ADDITIONAL_DEPS" + envDotNetSharedStore = "DOTNET_SHARED_STORE" + envDotNetStartupHook = "DOTNET_STARTUP_HOOKS" + envDotNetOTelAutoHome = "OTEL_DOTNET_AUTO_HOME" + dotNetCoreClrEnableProfilingEnabled = "1" + dotNetCoreClrProfilerId = "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + dotNetCoreClrProfilerPath = "/otel-auto-instrumentation/OpenTelemetry.AutoInstrumentation.Native.so" + dotNetAdditionalDepsPath = "/otel-auto-instrumentation/AdditionalDeps" + dotNetOTelAutoHomePath = "/otel-auto-instrumentation" + dotNetSharedStorePath = "/otel-auto-instrumentation/store" + dotNetStartupHookPath = "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" ) func injectDotNetSDK(logger logr.Logger, dotNetSpec v1alpha1.DotNet, pod corev1.Pod, index int) corev1.Pod { @@ -44,49 +52,37 @@ func injectDotNetSDK(logger logr.Logger, dotNetSpec v1alpha1.DotNet, pod corev1. } } - idx := getIndexOfEnv(container.Env, envDotNetStartupHook) - if idx == -1 { - container.Env = append(container.Env, corev1.EnvVar{ - Name: envDotNetStartupHook, - Value: dotNetStartupHookPath, - }) - } else if idx > -1 { - if container.Env[idx].ValueFrom != nil { - // TODO add to status object or submit it as an event - logger.Info("Skipping DotNet SDK injection, the container defines DOTNET_STARTUP_HOOKS env var value via ValueFrom", "container", container.Name) - return pod - } - container.Env[idx].Value = fmt.Sprintf("%s:%s", container.Env[idx].Value, dotNetStartupHookPath) + const ( + doNotConcatEnvValues = false + concatEnvValues = true + ) + + if !trySetEnvVar(logger, &container, envDotNetCoreClrEnableProfiling, dotNetCoreClrEnableProfilingEnabled, doNotConcatEnvValues) { + return pod } - idx = getIndexOfEnv(container.Env, envDotNetAdditionalDeps) - if idx == -1 { - container.Env = append(container.Env, corev1.EnvVar{ - Name: envDotNetAdditionalDeps, - Value: dotNetAdditionalDepsPath, - }) - } else if idx > -1 { - if container.Env[idx].ValueFrom != nil { - // TODO add to status object or submit it as an event - logger.Info("Skipping DotNet SDK injection, the container defines DOTNET_ADDITIONAL_DEPS env var value via ValueFrom", "container", container.Name) - return pod - } - container.Env[idx].Value = fmt.Sprintf("%s:%s", container.Env[idx].Value, dotNetAdditionalDepsPath) + if !trySetEnvVar(logger, &container, envDotNetCoreClrProfiler, dotNetCoreClrProfilerId, doNotConcatEnvValues) { + return pod } - idx = getIndexOfEnv(container.Env, envDotNetSharedStore) - if idx == -1 { - container.Env = append(container.Env, corev1.EnvVar{ - Name: envDotNetSharedStore, - Value: dotNetSharedStorePath, - }) - } else if idx > -1 { - if container.Env[idx].ValueFrom != nil { - // TODO add to status object or submit it as an event - logger.Info("Skipping DotNet SDK injection, the container defines DOTNET_SHARED_STORE env var value via ValueFrom", "container", container.Name) - return pod - } - container.Env[idx].Value = fmt.Sprintf("%s:%s", container.Env[idx].Value, dotNetSharedStorePath) + if !trySetEnvVar(logger, &container, envDotNetCoreClrProfilerPath, dotNetCoreClrProfilerPath, doNotConcatEnvValues) { + return pod + } + + if !trySetEnvVar(logger, &container, envDotNetStartupHook, dotNetStartupHookPath, concatEnvValues) { + return pod + } + + if !trySetEnvVar(logger, &container, envDotNetAdditionalDeps, dotNetAdditionalDepsPath, concatEnvValues) { + return pod + } + + if !trySetEnvVar(logger, &container, envDotNetOTelAutoHome, dotNetOTelAutoHomePath, doNotConcatEnvValues) { + return pod + } + + if !trySetEnvVar(logger, &container, envDotNetSharedStore, dotNetSharedStorePath, concatEnvValues) { + return pod } container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ @@ -116,3 +112,26 @@ func injectDotNetSDK(logger logr.Logger, dotNetSpec v1alpha1.DotNet, pod corev1. pod.Spec.Containers[index] = container return pod } + +func trySetEnvVar(logger logr.Logger, container *corev1.Container, envVarName string, envVarValue string, concatValues bool) bool { + idx := getIndexOfEnv(container.Env, envVarName) + if idx < 0 { + container.Env = append(container.Env, corev1.EnvVar{ + Name: envVarName, + Value: envVarValue, + }) + return true + } + + if container.Env[idx].ValueFrom != nil { + // TODO add to status object or submit it as an event + logger.Info("Skipping DotNet SDK injection, the container defines env var value via ValueFrom", "envVar", envVarName, "container", container.Name) + return false + } + + if concatValues { + container.Env[idx].Value = fmt.Sprintf("%s:%s", container.Env[idx].Value, envVarValue) + } + + return true +} diff --git a/pkg/instrumentation/dotnet_test.go b/pkg/instrumentation/dotnet_test.go index b2ec849e23..00177e15f0 100644 --- a/pkg/instrumentation/dotnet_test.go +++ b/pkg/instrumentation/dotnet_test.go @@ -33,7 +33,7 @@ func TestInjectDotNetSDK(t *testing.T) { expected corev1.Pod }{ { - name: "DOTNET_STARTUP_HOOKS, DOTNET_SHARED_STORE, DOTNET_ADDITIONAL_DEPS not defined", + name: "CORECLR_ENABLE_PROFILING, CORECLR_PROFILER, CORECLR_PROFILER_PATH, DOTNET_STARTUP_HOOKS, DOTNET_SHARED_STORE, DOTNET_ADDITIONAL_DEPS, OTEL_DOTNET_AUTO_HOME not defined", DotNet: v1alpha1.DotNet{Image: "foo/bar:1", Env: []corev1.EnvVar{}}, pod: corev1.Pod{ Spec: corev1.PodSpec{ @@ -72,6 +72,18 @@ func TestInjectDotNetSDK(t *testing.T) { }, }, Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + Value: dotNetCoreClrEnableProfilingEnabled, + }, + { + Name: envDotNetCoreClrProfiler, + Value: dotNetCoreClrProfilerId, + }, + { + Name: envDotNetCoreClrProfilerPath, + Value: dotNetCoreClrProfilerPath, + }, { Name: envDotNetStartupHook, Value: dotNetStartupHookPath, @@ -80,6 +92,10 @@ func TestInjectDotNetSDK(t *testing.T) { Name: envDotNetAdditionalDeps, Value: dotNetAdditionalDepsPath, }, + { + Name: envDotNetOTelAutoHome, + Value: dotNetOTelAutoHomePath, + }, { Name: envDotNetSharedStore, Value: dotNetSharedStorePath, @@ -91,13 +107,25 @@ func TestInjectDotNetSDK(t *testing.T) { }, }, { - name: "DOTNET_STARTUP_HOOKS, DOTNET_ADDITIONAL_DEPS, DOTNET_SHARED_STORE defined", + name: "CORECLR_ENABLE_PROFILING, CORECLR_PROFILER, CORECLR_PROFILER_PATH, DOTNET_STARTUP_HOOKS, DOTNET_ADDITIONAL_DEPS, DOTNET_SHARED_STORE, OTEL_DOTNET_AUTO_HOME defined", DotNet: v1alpha1.DotNet{Image: "foo/bar:1"}, pod: corev1.Pod{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + Value: "/foo:/bar", + }, + { + Name: envDotNetCoreClrProfiler, + Value: "/foo:/bar", + }, + { + Name: envDotNetCoreClrProfilerPath, + Value: "/foo:/bar", + }, { Name: envDotNetStartupHook, Value: "/foo:/bar", @@ -110,6 +138,10 @@ func TestInjectDotNetSDK(t *testing.T) { Name: envDotNetSharedStore, Value: "/foo:/bar", }, + { + Name: envDotNetOTelAutoHome, + Value: "/foo:/bar", + }, }, }, }, @@ -145,6 +177,18 @@ func TestInjectDotNetSDK(t *testing.T) { }, }, Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + Value: "/foo:/bar", + }, + { + Name: envDotNetCoreClrProfiler, + Value: "/foo:/bar", + }, + { + Name: envDotNetCoreClrProfilerPath, + Value: "/foo:/bar", + }, { Name: envDotNetStartupHook, Value: fmt.Sprintf("%s:%s", "/foo:/bar", dotNetStartupHookPath), @@ -157,6 +201,106 @@ func TestInjectDotNetSDK(t *testing.T) { Name: envDotNetSharedStore, Value: fmt.Sprintf("%s:%s", "/foo:/bar", dotNetSharedStorePath), }, + { + Name: envDotNetOTelAutoHome, + Value: "/foo:/bar", + }, + }, + }, + }, + }, + }, + }, + { + name: "CORECLR_ENABLE_PROFILING defined as ValueFrom", + DotNet: v1alpha1.DotNet{Image: "foo/bar:1"}, + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + expected: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + }, + { + name: "CORECLR_PROFILER defined as ValueFrom", + DotNet: v1alpha1.DotNet{Image: "foo/bar:1"}, + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrProfiler, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + expected: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrProfiler, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + }, + { + name: "CORECLR_PROFILER_PATH defined as ValueFrom", + DotNet: v1alpha1.DotNet{Image: "foo/bar:1"}, + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrProfilerPath, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + expected: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrProfilerPath, + ValueFrom: &corev1.EnvVarSource{}, + }, }, }, }, @@ -259,6 +403,38 @@ func TestInjectDotNetSDK(t *testing.T) { }, }, }, + { + name: "OTEL_DOTNET_AUTO_HOME defined as ValueFrom", + DotNet: v1alpha1.DotNet{Image: "foo/bar:1"}, + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetOTelAutoHome, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + expected: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Env: []corev1.EnvVar{ + { + Name: envDotNetOTelAutoHome, + ValueFrom: &corev1.EnvVarSource{}, + }, + }, + }, + }, + }, + }, + }, } for _, test := range tests { diff --git a/pkg/instrumentation/podmutator_test.go b/pkg/instrumentation/podmutator_test.go index f44861f2b9..101d24fe24 100644 --- a/pkg/instrumentation/podmutator_test.go +++ b/pkg/instrumentation/podmutator_test.go @@ -650,6 +650,18 @@ func TestMutatePod(t *testing.T) { Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: "http://localhost:4317", }, + { + Name: envDotNetCoreClrEnableProfiling, + Value: dotNetCoreClrEnableProfilingEnabled, + }, + { + Name: envDotNetCoreClrProfiler, + Value: dotNetCoreClrProfilerId, + }, + { + Name: envDotNetCoreClrProfilerPath, + Value: dotNetCoreClrProfilerPath, + }, { Name: envDotNetStartupHook, Value: dotNetStartupHookPath, @@ -658,6 +670,10 @@ func TestMutatePod(t *testing.T) { Name: envDotNetAdditionalDeps, Value: dotNetAdditionalDepsPath, }, + { + Name: envDotNetOTelAutoHome, + Value: dotNetOTelAutoHomePath, + }, { Name: envDotNetSharedStore, Value: dotNetSharedStorePath, diff --git a/pkg/instrumentation/sdk_test.go b/pkg/instrumentation/sdk_test.go index 3e976619e9..31c29282f6 100644 --- a/pkg/instrumentation/sdk_test.go +++ b/pkg/instrumentation/sdk_test.go @@ -730,6 +730,18 @@ func TestInjectDotNet(t *testing.T) { }, }, Env: []corev1.EnvVar{ + { + Name: envDotNetCoreClrEnableProfiling, + Value: dotNetCoreClrEnableProfilingEnabled, + }, + { + Name: envDotNetCoreClrProfiler, + Value: dotNetCoreClrProfilerId, + }, + { + Name: envDotNetCoreClrProfilerPath, + Value: dotNetCoreClrProfilerPath, + }, { Name: envDotNetStartupHook, Value: dotNetStartupHookPath, @@ -738,6 +750,10 @@ func TestInjectDotNet(t *testing.T) { Name: envDotNetAdditionalDeps, Value: dotNetAdditionalDepsPath, }, + { + Name: envDotNetOTelAutoHome, + Value: dotNetOTelAutoHomePath, + }, { Name: envDotNetSharedStore, Value: dotNetSharedStorePath, diff --git a/tests/e2e/instrumentation-dotnet-multicontainer/01-assert.yaml b/tests/e2e/instrumentation-dotnet-multicontainer/01-assert.yaml index 7923b8873e..c2ed9a8886 100644 --- a/tests/e2e/instrumentation-dotnet-multicontainer/01-assert.yaml +++ b/tests/e2e/instrumentation-dotnet-multicontainer/01-assert.yaml @@ -13,10 +13,18 @@ spec: env: - name: OTEL_LOG_LEVEL value: "debug" + - name: CORECLR_ENABLE_PROFILING + value: "1" + - name: CORECLR_PROFILER + value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + - name: CORECLR_PROFILER_PATH + value: /otel-auto-instrumentation/OpenTelemetry.AutoInstrumentation.Native.so - name: DOTNET_STARTUP_HOOKS - value: "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" + value: /otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll - name: DOTNET_ADDITIONAL_DEPS - value: ./otel-auto-instrumentation/AdditionalDeps + value: /otel-auto-instrumentation/AdditionalDeps + - name: OTEL_DOTNET_AUTO_HOME + value: /otel-auto-instrumentation - name: DOTNET_SHARED_STORE value: /otel-auto-instrumentation/store - name: OTEL_TRACES_EXPORTER @@ -46,10 +54,18 @@ spec: env: - name: OTEL_LOG_LEVEL value: "debug" + - name: CORECLR_ENABLE_PROFILING + value: "1" + - name: CORECLR_PROFILER + value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + - name: CORECLR_PROFILER_PATH + value: /otel-auto-instrumentation/OpenTelemetry.AutoInstrumentation.Native.so - name: DOTNET_STARTUP_HOOKS - value: "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" + value: /otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll - name: DOTNET_ADDITIONAL_DEPS - value: ./otel-auto-instrumentation/AdditionalDeps + value: /otel-auto-instrumentation/AdditionalDeps + - name: OTEL_DOTNET_AUTO_HOME + value: /otel-auto-instrumentation - name: DOTNET_SHARED_STORE value: /otel-auto-instrumentation/store - name: OTEL_TRACES_EXPORTER diff --git a/tests/e2e/instrumentation-dotnet-multicontainer/02-assert.yaml b/tests/e2e/instrumentation-dotnet-multicontainer/02-assert.yaml index 1fd49964a2..fdb39b7c1a 100644 --- a/tests/e2e/instrumentation-dotnet-multicontainer/02-assert.yaml +++ b/tests/e2e/instrumentation-dotnet-multicontainer/02-assert.yaml @@ -16,10 +16,18 @@ spec: env: - name: OTEL_LOG_LEVEL value: "debug" + - name: CORECLR_ENABLE_PROFILING + value: "1" + - name: CORECLR_PROFILER + value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + - name: CORECLR_PROFILER_PATH + value: /otel-auto-instrumentation/OpenTelemetry.AutoInstrumentation.Native.so - name: DOTNET_STARTUP_HOOKS - value: "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" + value: /otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll - name: DOTNET_ADDITIONAL_DEPS - value: ./otel-auto-instrumentation/AdditionalDeps + value: /otel-auto-instrumentation/AdditionalDeps + - name: OTEL_DOTNET_AUTO_HOME + value: /otel-auto-instrumentation - name: DOTNET_SHARED_STORE value: /otel-auto-instrumentation/store - name: OTEL_TRACES_EXPORTER diff --git a/tests/e2e/instrumentation-dotnet/01-assert.yaml b/tests/e2e/instrumentation-dotnet/01-assert.yaml index 6c38bc686d..28b3a289fd 100644 --- a/tests/e2e/instrumentation-dotnet/01-assert.yaml +++ b/tests/e2e/instrumentation-dotnet/01-assert.yaml @@ -10,10 +10,18 @@ spec: containers: - name: myapp env: + - name: CORECLR_ENABLE_PROFILING + value: "1" + - name: CORECLR_PROFILER + value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + - name: CORECLR_PROFILER_PATH + value: /otel-auto-instrumentation/OpenTelemetry.AutoInstrumentation.Native.so - name: DOTNET_STARTUP_HOOKS - value: "/otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll" + value: /otel-auto-instrumentation/netcoreapp3.1/OpenTelemetry.AutoInstrumentation.StartupHook.dll - name: DOTNET_ADDITIONAL_DEPS - value: ./otel-auto-instrumentation/AdditionalDeps + value: /otel-auto-instrumentation/AdditionalDeps + - name: OTEL_DOTNET_AUTO_HOME + value: /otel-auto-instrumentation - name: DOTNET_SHARED_STORE value: /otel-auto-instrumentation/store - name: OTEL_TRACES_EXPORTER