[WIP]Integrate the observability (#726)

* Integrate the observability * Update code to clean up unused resources and optimize the configuration * Enrich labels for Metrics, Logs * Apply observability to all services * Fix newline ending * Cleanup and update dashboard for Metrics --------- Co-authored-by: Loi Nguyen Xuan <loinguyenx@nashtechglobal.com>
nashtech-garage · Oct 12, 2023 · 4da68a0 · 4da68a0
1 parent 58880ea
commit 4da68a0
Show file tree

Hide file tree

Showing 72 changed files with 16,709 additions and 1,043 deletions.
diff --git a/.env b/.env
@@ -1 +1,83 @@
-COMPOSE_FILE=docker-compose.yml:docker-compose.search.yml:docker-compose.o11y.yml
+
+# Demo Platform
+ENV_PLATFORM=local
+
+# OpenTelemetry Collector
+OTEL_COLLECTOR_HOST=collector
+OTEL_COLLECTOR_PORT_GRPC=5555
+OTEL_COLLECTOR_PORT_HTTP=6666
+OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_GRPC}
+OTEL_LOGS_EXPORTER=otlp
+OTEL_TRACES_EXPORTER=otlp
+OTEL_METRICS_EXPORTER=otlp
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_JAVAAGENT_LOGGING=application
+OTEL_JAVAAGENT_ENABLED=true
+OTEL_JAVAAGENT_DEBUG=false
+OTEL_INSTRUMENTATION_MESSAGING_EXPERIMENTAL_RECEIVE_TELEMETRY_ENABLED=true
+# OpenTelemetry Resource Definitions
+OTEL_RESOURCE_ATTRIBUTES="service.namespace=yas-msa"
+
+# Metrics Temporality
+OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=cumulative
+OTEL_INSTRUMENTATION_LOGBACK-MDC_ADD-BAGGAGE=true
+
+# ******************
+# Services
+# ******************
+# Kafka
+KAFKA_SERVICE_HOST=kafka
+KAFKA_SERVICE_PORT=9092
+KAFKA_SERVICE_ADDR=kafka:${KAFKA_SERVICE_PORT}
+KAFKA_BROKER_ID=1
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${KAFKA_SERVICE_ADDR},PLAINTEXT_HOST://kafka:29092
+KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+
+# Postgres
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=admin
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+
+# ********************
+# Telemetry Components
+# ********************
+# Grafana
+GRAFANA_SERVICE_PORT=3000
+GRAFANA_SERVICE_HOST=grafana
+
+# Jaeger
+JAEGER_SERVICE_PORT=16686
+JAEGER_SERVICE_HOST=jaeger
+
+# Prometheus
+PROMETHEUS_SERVICE_PORT=9090
+PROMETHEUS_SERVICE_HOST=prometheus
+PROMETHEUS_ADDR=${PROMETHEUS_SERVICE_HOST}:${PROMETHEUS_SERVICE_PORT}
+
+
+# *******************************************************
+# Environment Variables are injected to the microservices
+# *******************************************************
+JAVA_TOOL_OPTIONS=-javaagent:opentelemetry-javaagent.jar
+LOGGING_CONFIG=/app-config/logback-spring.xml
+
+# The API Endpoint
+YAS_PUBLIC_API_URL=http://api.yas.local
+YAS_SERVICES_CART=http//cart/cart
+YAS_SERVICES_CUSTOMER=http://customer/customer
+YAS_SERVICES_PRODUCT=http://product/product
+YAS_SERVICES_TAX=http://tax/tax
+YAS_SERVICES_ORDER=http://order/order
+YAS_SERVICES_PAYMENT=http://payment/payment
+YAS_SERVICES_LOCATION=http://location/location
+YAS_SERVICES_MEDIA=http://media/media
+YAS_SERVICES_PROMOTION=http://promotion/promotion
+YAS_SERVICES_INVENTORY=http://inventory/inventory
+YAS_SERVICES_RATING=http://rating/rating
+SERVER_PORT=80
+# Swagger UI
+URLS=[{ url: 'http://api.yas.local/product/v3/api-docs', name: 'Product' },{ url: 'http://api.yas.local/media/v3/api-docs', name: 'Media' },{ url: 'http://api.yas.local/customer/v3/api-docs', name: 'Customer' },{ url: 'http://api.yas.local/cart/v3/api-docs', name: 'Cart'},{ url: 'http://api.yas.local/rating/v3/api-docs', name: 'Rating' }, { url: 'http://api.yas.local/order/v3/api-docs', name: 'Order'},{ url: 'http://api.yas.local/payment/v3/api-docs', name: 'Payment'},{ url: 'http://api.yas.local/payment-paypal/v3/api-docs', name: 'Payment-paypal'},{ url: 'http://api.yas.local/location/v3/api-docs', name: 'Location'}, { url: 'http://api.yas.local/inventory/v3/api-docs', name: 'Inventory'},{ url: 'http://api.yas.local/tax/v3/api-docs', name: 'Tax' },{ url: 'http://api.yas.local/promotion/v3/api-docs', name: 'Promotion'},{ url: 'http://api.yas.local/search/v3/api-docs', name: 'Search'}]
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# fix line ending
+* text=auto
diff --git a/.gitignore b/.gitignore
@@ -25,4 +25,6 @@
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 .project
-.settings
+.settings*
+.project
+.vscode
diff --git a/backoffice-bff/pom.xml b/backoffice-bff/pom.xml
@@ -19,6 +19,8 @@
         <sonar.organization>nashtech-garage</sonar.organization>
 		<sonar.host.url>https://sonarcloud.io</sonar.host.url>
 		<sonar.projectKey>nashtech-garage_yas-backoffice-bff</sonar.projectKey>
+		<logstash-logback-encoder.version>7.4</logstash-logback-encoder.version>
+		<opentelemetry-logback.vesion>1.30.0-alpha</opentelemetry-logback.vesion>
 	</properties>
 	<dependencies>
 		<dependency>
@@ -60,6 +62,15 @@
 			<artifactId>spring-boot-starter-test</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>net.logstash.logback</groupId>
+			<artifactId>logstash-logback-encoder</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>io.opentelemetry.instrumentation</groupId>
+			<artifactId>opentelemetry-logback-mdc-1.0</artifactId>
+			<scope>runtime</scope>
+		</dependency>
 	</dependencies>
 	<dependencyManagement>
 		<dependencies>
@@ -70,6 +81,16 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
+			<dependency>
+				<groupId>net.logstash.logback</groupId>
+				<artifactId>logstash-logback-encoder</artifactId>
+				<version>${logstash-logback-encoder.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>io.opentelemetry.instrumentation</groupId>
+				<artifactId>opentelemetry-logback-mdc-1.0</artifactId>
+				<version>${opentelemetry-logback.vesion}</version>
+			</dependency>			
 		</dependencies>
 	</dependencyManagement>
 	<repositories>

diff --git a/backoffice-bff/src/main/resources/application.yaml b/backoffice-bff/src/main/resources/application.yaml
@@ -18,16 +18,9 @@ spring:
             client-secret: TVacLC0cQ8tiiEKiTVerTb2YvwQ1TRJF
             scope: openid, profile, email, roles
 management:
-  otlp:
-    tracing:
-      endpoint: http://tempo:4318/v1/traces
   tracing:
     sampling:
       probability: "1.0"
-  endpoints:
-    web:
-      exposure:
-        include: prometheus
   metrics:
     distribution:
       percentiles-histogram:

diff --git a/backoffice-bff/src/main/resources/logback-spring.xml b/backoffice-bff/src/main/resources/logback-spring.xml
@@ -1,24 +1,58 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
-    <include resource="org/springframework/boot/logging/logback/base.xml" />
-    <springProperty scope="context" name="appName" source="spring.application.name"/>
+	<property name="ENCODING" value="UTF-8" />
+	<property name="LOG_LEVEL" value="DEBUG" />
+	<include
+		resource="org/springframework/boot/logging/logback/base.xml" />
+	<springProperty scope="context" name="appName"
+		source="spring.application.name" />
+	<!-- Reuse the log from Spring -->
+	<include
+		resource="org/springframework/boot/logging/logback/defaults.xml" />
+	<include
+		resource="org/springframework/boot/logging/logback/console-appender.xml" />
+	<!-- Define own appender to be used in the logger to append the data into -->
+	<appender name="JSON_CONSOLE"
+		class="ch.qos.logback.core.ConsoleAppender">
+		<Target>${user-system}.out</Target>
+		<encoder
+			class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+			<providers>
+				<mdc />
+				<pattern>
+					<pattern>
+						{
+						"timestamp": "%date{yyyy-MM-dd'T'HH:mm:ss.SSS,UTC}",
+						"log.level": "%level",
+						"thread": "%thread",
+						"logger": "%logger",
+						"message": "%message",
+						"host": "${HOSTNAME}"
+						}
+					</pattern>
+				</pattern>
+				<stackTrace>
+					<throwableConverter
+						class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
+						<rootCauseFirst>true</rootCauseFirst>
+					</throwableConverter>
+				</stackTrace>
+			</providers>
+		</encoder>
+	</appender>
 
-    <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
-        <http>
-            <url>http://loki:3100/loki/api/v1/push</url>
-        </http>
-        <format>
-            <label>
-                <pattern>app=${appName},host=${HOSTNAME},traceID=%X{traceId:-NONE},level=%level</pattern>
-            </label>
-            <message>
-                <pattern>${FILE_LOG_PATTERN}</pattern>
-            </message>
-            <sortByTime>true</sortByTime>
-        </format>
-    </appender>
 
-    <root level="INFO">
-        <appender-ref ref="LOKI"/>
-    </root>
-</configuration>
+	<!-- This appender is to define the log to console that human friendly on 
+		dev machine -->
+	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> 
+		<Target>${user-system}.out</Target> <encoder> <pattern>${CONSOLE_LOG_PATTERN}</pattern> 
+		<charset>${ENCODING}</charset> </encoder> </appender>
+	<!-- Define the logger and the log level to use (TRACE, DEBUG, INFO, WARN, 
+		ERROR, FATAL) -->
+	<logger
+		name="org.springframework.web.filter.CommonsRequestLoggingFilter"
+		level="DEBUG" />
+	<root level="INFO">
+		<appender-ref ref="CONSOLE" />
+	</root>
+</configuration>
diff --git a/cart/pom.xml b/cart/pom.xml
@@ -1,19 +1,18 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<parent>
 		<groupId>org.springframework.boot</groupId>
 		<artifactId>spring-boot-starter-parent</artifactId>
 		<version>3.1.0</version>
-		<relativePath/> <!-- lookup parent from repository -->
+		<relativePath /> <!-- lookup parent from repository -->
 	</parent>
 	<groupId>com.yas</groupId>
 	<artifactId>cart</artifactId>
 	<version>0.0.1-SNAPSHOT</version>
 	<name>cart</name>
 	<description>YAS Cart service</description>
-	
+
 	<properties>
 		<java.version>17</java.version>
 		<maven.compiler.source>17</maven.compiler.source>
@@ -23,11 +22,23 @@
 		<org.mapstruct.version>1.4.1.Final</org.mapstruct.version>
 		<spring-cloud.version>2022.0.3</spring-cloud.version>
 		<sonar.organization>nashtech-garage</sonar.organization>
-        <sonar.host.url>https://sonarcloud.io</sonar.host.url>
+		<sonar.host.url>https://sonarcloud.io</sonar.host.url>
 		<sonar.projectKey>nashtech-garage_yas-cart</sonar.projectKey>
+		<logstash-logback-encoder.version>7.4</logstash-logback-encoder.version>
+		<opentelemetry-logback.vesion>1.30.0-alpha</opentelemetry-logback.vesion>
 	</properties>
 
 	<dependencies>
+		<!-- https://mvnrepository.com/artifact/net.logstash.logback/logstash-logback-encoder -->
+		<dependency>
+			<groupId>net.logstash.logback</groupId>
+			<artifactId>logstash-logback-encoder</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>io.opentelemetry.instrumentation</groupId>
+			<artifactId>opentelemetry-logback-mdc-1.0</artifactId>
+			<scope>runtime</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-web</artifactId>
@@ -78,11 +89,11 @@
 			<artifactId>lombok</artifactId>
 			<optional>true</optional>
 		</dependency>
-        <dependency>
-            <groupId>com.h2database</groupId>
-            <artifactId>h2</artifactId>
-            <scope>test</scope>
-        </dependency>
+		<dependency>
+			<groupId>com.h2database</groupId>
+			<artifactId>h2</artifactId>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>io.micrometer</groupId>
 			<artifactId>micrometer-registry-prometheus</artifactId>
@@ -118,6 +129,16 @@
 	</dependencies>
 	<dependencyManagement>
 		<dependencies>
+			<dependency>
+				<groupId>net.logstash.logback</groupId>
+				<artifactId>logstash-logback-encoder</artifactId>
+				<version>${logstash-logback-encoder.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>io.opentelemetry.instrumentation</groupId>
+				<artifactId>opentelemetry-logback-mdc-1.0</artifactId>
+				<version>${opentelemetry-logback.vesion}</version>
+			</dependency>
 			<dependency>
 				<groupId>org.springframework.cloud</groupId>
 				<artifactId>spring-cloud-dependencies</artifactId>
@@ -127,29 +148,6 @@
 			</dependency>
 		</dependencies>
 	</dependencyManagement>
-	<repositories>
-		<repository>
-			<id>spring-snapshots</id>
-			<url>https://repo.spring.io/snapshot</url>
-			<snapshots>
-				<enabled>true</enabled>
-			</snapshots>
-		</repository>
-		<repository>
-			<id>spring-milestones</id>
-			<url>https://repo.spring.io/milestone</url>
-		</repository>
-	</repositories>
-	<pluginRepositories>
-		<pluginRepository>
-			<id>spring-snapshots</id>
-			<url>https://repo.spring.io/snapshot</url>
-		</pluginRepository>
-		<pluginRepository>
-			<id>spring-milestones</id>
-			<url>https://repo.spring.io/milestone</url>
-		</pluginRepository>
-	</pluginRepositories>
 	<build>
 		<plugins>
 			<plugin>

diff --git a/cart/src/main/resources/application.properties b/cart/src/main/resources/application.properties
@@ -2,9 +2,8 @@ server.port=8084
 server.servlet.context-path=/cart
 
 spring.application.name=cart
-management.otlp.tracing.endpoint=http://tempo:4318/v1/traces
 management.tracing.sampling.probability=1.0
-management.endpoints.web.exposure.include=prometheus
+management.endpoints.web.exposure.include=*
 management.metrics.distribution.percentiles-histogram.http.server.requests=true
 management.metrics.tags.application=${spring.application.name}