[Python] Custom Export Pipeline Update (#489)

Author: Miqueasher

sample-apps/python/django_frontend_service/ec2-requirements.txt

@@ -4,4 +4,5 @@ pymysql==1.1.1
 python-dotenv~=1.0.1
 requests~=2.25.1
 schedule~=1.2.1
+opentelemetry-sdk>=1.33.1,<2.0.0
 opentelemetry-api>=1.33.1,<2.0.0

sample-apps/python/django_frontend_service/frontend_service_app/views.py

@@ -16,12 +16,52 @@
 
 logger = logging.getLogger(__name__)
 
+# Custom export pipeline - only create if specific env vars exist
+pipeline_meter = None
+if os.environ.get('SERVICE_NAME') and os.environ.get('DEPLOYMENT_ENVIRONMENT_NAME'):
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+    from opentelemetry.semconv.resource import ResourceAttributes
+    
+    service_name = os.environ.get('SERVICE_NAME')
+    deployment_env = os.environ.get('DEPLOYMENT_ENVIRONMENT_NAME')
+    pipeline_resource = Resource.create({
+        #ResourceAttributes.DEPLOYMENT_ENVIRONMENT_NAME maps to dimension 'deployment.name' so "deployment.environment.name" used 
+        #to assign value correctly.
+        ResourceAttributes.SERVICE_NAME: service_name,
+        "deployment.environment.name": deployment_env
+    })
+    
+    pipeline_metric_exporter = OTLPMetricExporter(endpoint="localhost:4317")
+    pipeline_metric_reader = PeriodicExportingMetricReader(
+        exporter=pipeline_metric_exporter,
+        export_interval_millis=1000
+    )
+    
+    pipeline_meter_provider = MeterProvider(
+        resource=pipeline_resource,
+        metric_readers=[pipeline_metric_reader]
+    )
+    
+    pipeline_meter = pipeline_meter_provider.get_meter("myMeter")
+
+
 #python equivalent of Meter meter = GlobalOpenTelemetry.getMeter("myMeter"); for custom metrics.
 meter = metrics.get_meter("myMeter")
 agent_based_counter = meter.create_counter("agent_based_counter", unit="1", description="agent export counter")
 agent_based_histogram = meter.create_histogram("agent_based_histogram", description="agent export histogram")
 agent_based_gauge = meter.create_up_down_counter("agent_based_gauge", unit="1", description="agent export gauge")
 
+# Create pipeline metrics only if pipeline exists
+custom_pipeline_counter = None
+custom_pipeline_histogram = None
+custom_pipeline_gauge = None
+if pipeline_meter:
+    custom_pipeline_counter = pipeline_meter.create_counter("custom_pipeline_counter", unit="1", description="pipeline export counter")
+    custom_pipeline_histogram = pipeline_meter.create_histogram("custom_pipeline_histogram", description="pipeline export histogram")
+    custom_pipeline_gauge = pipeline_meter.create_up_down_counter("custom_pipeline_gauge", unit="1", description="pipeline export gauge")
 
 should_send_local_root_client_call = False
 lock = threading.Lock()
@@ -63,6 +103,11 @@ def aws_sdk_call(request):
     agent_based_histogram.record(random.randint(100, 1000), {"Operation": "histogram"})
     agent_based_gauge.add(random.randint(-10, 10), {"Operation": "gauge"})
 
+    if pipeline_meter:
+        custom_pipeline_counter.add(1, {"Operation": "pipeline_counter"})
+        custom_pipeline_histogram.record(random.randint(100, 1000), {"Operation": "pipeline_histogram"})
+        custom_pipeline_gauge.add(random.randint(-10, 10), {"Operation": "pipeline_gauge"})
+
     bucket_name = "e2e-test-bucket-name"
 
     # Add a unique test ID to bucketname to associate buckets to specific test runs

terraform/python/ec2/default/main.tf

@@ -180,7 +180,9 @@ resource "null_resource" "main_service_setup" {
       export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=localhost:4317
       export OTEL_EXPORTER_OTLP_METRICS_INSECURE=true
       export OTEL_TRACES_SAMPLER=always_on
-      export OTEL_RESOURCE_ATTRIBUTES="service.name=python-sample-application-${var.test_id},deployment.environment.name=ec2:default"
+      export SERVICE_NAME='python-sample-application-${var.test_id}'
+      export DEPLOYMENT_ENVIRONMENT_NAME='ec2:default'
+      export OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment.name=$${DEPLOYMENT_ENVIRONMENT_NAME}"
       export AWS_REGION='${var.aws_region}'
       python${var.language_version} manage.py migrate
       nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &

validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache

@@ -156,4 +156,72 @@
       value: ANY_VALUE
     -
       name: cloud.platform
-      value: aws_ec2
+      value: aws_ec2
+      
+# Export pipeline metrics
+-
+  metricName: custom_pipeline_counter
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_counter
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE
+-
+  metricName: custom_pipeline_histogram
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_histogram
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE
+-
+  metricName: custom_pipeline_gauge
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_gauge
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE