clickhouse logs exporter added to deployment file (SigNoz#1500)

* clickhouse logs exporter added to deployment file * updated to latest otel collector
bmike78 · Aug 10, 2022 · b6afc93 · b6afc93
1 parent dda8247
commit b6afc93
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 6 deletions.
diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
@@ -81,14 +81,16 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-config.yaml"]
+    # user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
     ports:
       # - "1777:1777"     # pprof extension
       - "4317:4317"     # OTLP gRPC receiver
       - "4318:4318"     # OTLP HTTP receiver
+      - "4319:4319"     # OTLP logs receiver
       # - "8888:8888"     # OtelCollector internal metrics
       # - "8889:8889"     # signoz spanmetrics exposed by the agent
       # - "9411:9411"     # Zipkin port
@@ -111,7 +113,7 @@ services:
       - clickhouse
 
   otel-collector-metrics:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-metrics-config.yaml"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml

diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
@@ -30,6 +30,10 @@ receivers:
       disk: {}
       filesystem: {}
       network: {}
+  otlp/logs:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4319
 
 processors:
   batch:
@@ -76,6 +80,16 @@ exporters:
   prometheus:
     endpoint: 0.0.0.0:8889
   # logging: {}
+  clickhouselogsexporter:
+    dsn: tcp://clickhouse:9000/
+    timeout: 5s
+    sending_queue:
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
 
 extensions:
   health_check:
@@ -106,3 +120,7 @@ service:
     metrics/spanmetrics:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
+    logs:
+      receivers: [otlp/logs]
+      processors: [batch]
+      exporters: [clickhouselogsexporter]
diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml
@@ -78,8 +78,9 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz/signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-config.yaml"]
+    # user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
     environment:
@@ -88,6 +89,7 @@ services:
       # - "1777:1777"     # pprof extension
       - "4317:4317"     # OTLP gRPC receiver
       - "4318:4318"     # OTLP HTTP receiver
+      - "4319:4319"     # OTLP logs receiver
       # - "8888:8888"     # OtelCollector internal metrics
       # - "8889:8889"     # signoz spanmetrics exposed by the agent
       # - "9411:9411"     # Zipkin port
@@ -103,7 +105,7 @@ services:
         condition: service_healthy
 
   otel-collector-metrics:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz/signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-metrics-config.yaml"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml

diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
@@ -31,6 +31,11 @@ receivers:
       filesystem: {}
       network: {}
 
+  otlp/logs:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4319
+
 processors:
   batch:
     send_batch_size: 10000
@@ -85,6 +90,17 @@ exporters:
     endpoint: 0.0.0.0:8889
   # logging: {}
 
+  clickhouselogsexporter:
+    dsn: tcp://clickhouse:9000/
+    timeout: 5s
+    sending_queue:
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
 service:
   telemetry:
     metrics:
@@ -109,3 +125,7 @@ service:
     metrics/spanmetrics:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
+    logs:
+      receivers: [otlp/logs]
+      processors: [batch]
+      exporters: [clickhouselogsexporter]
diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml
@@ -59,7 +59,7 @@ services:
         condition: service_healthy
 
   otel-collector:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@@ -74,7 +74,7 @@ services:
         condition: service_healthy
 
   otel-collector-metrics:
-    image: signoz/otelcontribcol:0.45.1-1.3
+    image: signoz-otel-collector:0.55.0-rc.3
     command: ["--config=/etc/otel-collector-metrics-config.yaml"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml