From 2ac3b6c6ae481ad6bf4f54db76d5a9ec1e25e04c Mon Sep 17 00:00:00 2001 From: Christos Markou Date: Tue, 30 Jul 2024 15:44:02 +0300 Subject: [PATCH] [chore] Add benchmark for container parser (#34281) **Description:** This PR adds benchmark for the [container parser](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/container.md). Results: ``` # Test PerformanceResults Started: Sun, 28 Jul 2024 20:31:06 +0300 Test |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Items|Received Items| ----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:| Log10kDPS/OTLP |PASS | 15s| 10.4| 11.0| 68| 94| 150100| 150100| Log10kDPS/OTLP-HTTP |PASS | 15s| 7.9| 9.7| 63| 90| 150100| 150100| Log10kDPS/filelog |PASS | 15s| 7.8| 8.7| 65| 93| 150100| 150100| Log10kDPS/filelog_checkpoints |PASS | 15s| 7.1| 8.0| 63| 91| 150100| 150100| Log10kDPS/kubernetes_containers |PASS | 15s| 18.3| 19.7| 67| 96| 150100| 150100| Log10kDPS/kubernetes_containers_parser |PASS | 15s| 18.2| 19.0| 66| 95| 150100| 150100| Log10kDPS/k8s_CRI-Containerd |PASS | 15s| 15.4| 16.3| 66| 95| 150100| 150100| Log10kDPS/k8s_CRI-Containerd_no_attr_ops|PASS | 15s| 15.1| 16.3| 66| 95| 150100| 150100| Log10kDPS/CRI-Containerd |PASS | 15s| 11.1| 13.0| 65| 93| 150100| 150100| Log10kDPS/syslog-tcp-batch-1 |PASS | 15s| 16.4| 18.0| 63| 91| 150100| 150100| Log10kDPS/syslog-tcp-batch-100 |PASS | 15s| 6.1| 6.7| 61| 90| 150100| 150100| Log10kDPS/FluentForward-SplunkHEC |PASS | 15s| 26.7| 28.0| 65| 94| 149800| 149800| Log10kDPS/tcp-batch-1 |PASS | 15s| 17.7| 18.7| 62| 91| 150100| 150100| Log10kDPS/tcp-batch-100 |PASS | 15s| 6.2| 7.0| 62| 90| 150100| 150100| Total duration: 213s ``` **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/31959 **Testing:** **Documentation:** FYI @djaglowski @TylerHelmuth Signed-off-by: ChrsMark --- examples/kubernetes/README.md | 13 +++++++------ testbed/datasenders/k8s.go | 15 +++++++++++++++ testbed/tests/log_test.go | 9 +++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index dec70d63c596..5ec5db489d94 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -47,12 +47,13 @@ Following table shows result of example run: Test |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Items|Received Items| ----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:| -Log10kDPS/OTLP |PASS | 15s| 15.2| 15.7| 69| 73| 149900| 149900| -Log10kDPS/filelog |PASS | 15s| 16.5| 18.0| 61| 74| 150000| 150000| -Log10kDPS/kubernetes_containers |PASS | 15s| 42.3| 44.0| 66| 80| 150000| 150000| -Log10kDPS/k8s_CRI-Containerd |PASS | 15s| 36.7| 38.0| 64| 78| 150000| 150000| -Log10kDPS/k8s_CRI-Containerd_no_attr_ops|PASS | 15s| 28.9| 29.7| 64| 77| 150000| 150000| -Log10kDPS/CRI-Containerd |PASS | 15s| 19.0| 21.0| 63| 77| 150000| 150000| +Log10kDPS/OTLP |PASS | 15s| 10.4| 11.0| 68| 94| 150100| 150100| +Log10kDPS/filelog |PASS | 15s| 7.8| 8.7| 65| 93| 150100| 150100| +Log10kDPS/kubernetes_containers |PASS | 15s| 18.3| 19.7| 67| 96| 150100| 150100| +Log10kDPS/kubernetes_containers_parser |PASS | 15s| 18.2| 19.0| 66| 95| 150100| 150100| +Log10kDPS/k8s_CRI-Containerd |PASS | 15s| 15.4| 16.3| 66| 95| 150100| 150100| +Log10kDPS/k8s_CRI-Containerd_no_attr_ops|PASS | 15s| 15.1| 16.3| 66| 95| 150100| 150100| +Log10kDPS/CRI-Containerd |PASS | 15s| 11.1| 13.0| 65| 93| 150100| 150100| ## ToDo diff --git a/testbed/datasenders/k8s.go b/testbed/datasenders/k8s.go index a5f9dd30a6ea..a894b2c366f3 100644 --- a/testbed/datasenders/k8s.go +++ b/testbed/datasenders/k8s.go @@ -223,6 +223,21 @@ func NewKubernetesContainerWriter() *FileLogK8sWriter { `) } +// NewKubernetesContainerParserWriter returns FileLogK8sWriter with configuration +// to recognize and parse kubernetes container logs using the container parser +func NewKubernetesContainerParserWriter() *FileLogK8sWriter { + return NewFileLogK8sWriter(` + filelog: + include: [ %s ] + start_at: beginning + include_file_path: true + include_file_name: false + operators: + - type: container + id: container-parser + `) +} + // NewKubernetesCRIContainerdWriter returns FileLogK8sWriter with configuration // to parse only CRI-Containerd kubernetes logs func NewKubernetesCRIContainerdWriter() *FileLogK8sWriter { diff --git a/testbed/tests/log_test.go b/testbed/tests/log_test.go index e34605af3216..e9e1437a467e 100644 --- a/testbed/tests/log_test.go +++ b/testbed/tests/log_test.go @@ -69,6 +69,15 @@ func TestLog10kDPS(t *testing.T) { ExpectedMaxRAM: 150, }, }, + { + name: "kubernetes containers parser", + sender: datasenders.NewKubernetesContainerParserWriter(), + receiver: testbed.NewOTLPDataReceiver(testutil.GetAvailablePort(t)), + resourceSpec: testbed.ResourceSpec{ + ExpectedMaxCPU: 110, + ExpectedMaxRAM: 150, + }, + }, { name: "k8s CRI-Containerd", sender: datasenders.NewKubernetesCRIContainerdWriter(),