[AIP-49] Completes the Breeze OTel integration and adds a banner to the UI

BREEZE.rst

@@ -621,22 +621,22 @@ change by the time Airflow fully supports emitting metrics via OpenTelemetry.
 
 ----
 
-You can launch an instance of Breeze pre-configured to emit OTel metrics using
-``breeze start-airflow --integration otel``.  This will launch an Airflow webserver
-within the Breeze environment as well as containers running OpenTelemetry-Collector,
-Prometheus, and Grafana.  The integration configures the "Targets" in Prometheus,
-the "Datasources" in Grafana, and includes a default dashboard in Grafana.
+You can launch an instance of Breeze pre-configured to emit OpenTelemetry metrics
+using ``breeze start-airflow --integration otel``.  This will launch Airflow within
+the Breeze environment as well as containers running OpenTelemetry-Collector,
+Prometheus, and Grafana.  The integration handles all configuration of the
+"Targets" in Prometheus and the "Datasources" in Grafana, so it is ready to use.
 
 When you run Airflow Breeze with this integration, in addition to the standard ports
 (See "Port Forwarding" below), the following are also automatically forwarded:
 
-* 28888 -> forwarded to OpenTelemetry Collector -> breeze-otel_collector:8888
+* 28889 -> forwarded to OpenTelemetry Collector -> breeze-otel-collector:8889
 * 29090 -> forwarded to Prometheus -> breeze-prometheus:9090
 * 23000 -> forwarded to Grafana -> breeze-grafana:3000
 
-You can connect to these ports/databases using:
+You can connect to these ports using:
 
-* OpenTelemetry Collector: http://127.0.0.1:28888/metrics
+* OpenTelemetry Collector: http://127.0.0.1:28889/metrics
 * Prometheus Targets: http://127.0.0.1:29090/targets
 * Grafana Dashboards: http://127.0.0.1:23000/dashboards
 

CONTRIBUTING.rst

@@ -619,10 +619,10 @@ devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exa
 gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http,
 imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
 microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
-opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
-qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,
-smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
-webhdfs, winrm, yandex, zendesk
+opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto,
+qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity,
+slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica,
+virtualenv, webhdfs, winrm, yandex, zendesk
   .. END EXTRAS HERE
 
 Provider packages

INSTALL

@@ -103,10 +103,10 @@ devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exa
 gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http,
 imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
 microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
-opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
-qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,
-smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
-webhdfs, winrm, yandex, zendesk
+opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto,
+qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity,
+slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica,
+virtualenv, webhdfs, winrm, yandex, zendesk
 # END EXTRAS HERE
 
 # For installing Airflow in development environments - see CONTRIBUTING.rst

airflow/config_templates/config.yml

@@ -977,6 +977,37 @@ metrics:
       type: boolean
       example: ~
       default: "False"
+    otel_on:
+      description: |
+        Enables sending metrics to OpenTelemetry.
+      version_added: 2.5.1
+      type: string
+      example: ~
+      default: "False"
+    otel_host:
+      description: ~
+      version_added: 2.5.1
+      type: string
+      example: ~
+      default: "localhost"
+    otel_port:
+      description: ~
+      version_added: 2.5.1
+      type: string
+      example: ~
+      default: "8889"
+    otel_prefix:
+      description: ~
+      version_added: 2.0.0
+      type: string
+      example: ~
+      default: "airflow"
+    otel_interval_milliseconds:
+      description: ~
+      version_added: 2.0.0
+      type: integer
+      example: ~
+      default: "60000"
 secrets:
   description: ~
   options:

airflow/config_templates/default_airflow.cfg

@@ -539,6 +539,13 @@ statsd_disabled_tags = job_id,run_id
 # To enable sending Airflow metrics with StatsD-Influxdb tagging convention.
 statsd_influxdb_enabled = False
 
+# Enables sending metrics to OpenTelemetry.
+otel_on = False
+otel_host = localhost
+otel_port = 8889
+otel_prefix = airflow
+otel_interval_milliseconds = 60000
+
 [secrets]
 # Full class name of secrets backend to enable (will precede env vars and metastore in search path)
 # Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend

airflow/www/templates/airflow/dags.html

@@ -119,6 +119,12 @@
       <a href={{ get_docs_url("executor/index.html") }}><b>Click here</b></a> for more information.
     {% endcall %}
   {% endif %}
+  {% if otel_on | default(false) %}
+    {% call show_message(category='warning', dismissible=false) %}
+      OpenTelemetry support is experimental. <a href="https://github.com/apache/airflow/blob/main/BREEZE.rst#running-breeze-with-an-opentelemetry-metrics-stack">
+        <b>Click here</b></a> for more information.
+    {% endcall %}
+  {% endif %}
 {% endblock %}
 
 {% block content %}

airflow/www/views.py

@@ -627,6 +627,7 @@ class AirflowBaseView(BaseView):
         extra_args["sqlite_warning"] = settings.engine.dialect.name == "sqlite"
         if not executor.is_production:
             extra_args["production_executor_warning"] = executor.__name__
+        extra_args["otel_on"] = conf.getboolean("metrics", "otel_on")
 
     line_chart_attr = {
         "legend.maxKeyLength": 200,

docs/apache-airflow/extra-packages-ref.rst

@@ -62,6 +62,8 @@ python dependencies for the provided package.
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | leveldb             | ``pip install 'apache-airflow[leveldb]'``           | Required for use leveldb extra in google provider                          |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
+| otel                | ``pip install 'apache-airflow[otel]'``              | Required for OpenTelemetry metrics                                         |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | pandas              | ``pip install 'apache-airflow[pandas]'``            | Install Pandas library compatible with Airflow                             |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | password            | ``pip install 'apache-airflow[password]'``          | Password authentication for users                                          |

docs/spelling_wordlist.txt

@@ -1031,6 +1031,7 @@ os
 OSS
 oss
 ot
+otel
 overridable
 oversubscription
 Pagerduty

scripts/ci/docker-compose/integration-otel.yml

@@ -18,16 +18,14 @@
 version: "3.7"
 services:
   otel-collector:
-    image: otel/opentelemetry-collector:0.70.0
-    container_name: "breeze-opentelemetry-collector"
+    image: otel/opentelemetry-collector-contrib:0.70.0
+    container_name: "breeze-otel-collector"
+    command: [--config=/etc/otel-collector-config.yml]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
     ports:
-      - "1888:1888"     # pprof extension
-      - "28888:8888"    # Prometheus metrics exposed by the collector
+      - "24318:4318"    # OTLP http receiver
       - "28889:8889"    # Prometheus exporter metrics
-      - "13133:13133"   # health_check extension
-      - "4317:4317"     # OTLP gRPC receiver
-      - "4318:4318"     # OTLP http receiver
-      - "55679:55679"   # zpages extension
 
   prometheus:
     image: prom/prometheus
@@ -37,7 +35,6 @@ services:
       - "29090:9090"
     volumes:
       - ./prometheus-config.yml:/etc/prometheus/prometheus.yml
-      - ./prometheus/volume:/prometheus
 
   grafana:
     image: grafana/grafana:8.2.4
@@ -56,8 +53,12 @@ services:
       - ./grafana/volume/provisioning:/grafana/provisioning
 
   airflow:
-    #  Environment Variables will need to be set in order to configure
-    #  Breeze to emit the metrics.  See: integration-statsd.yml
+    environment:
+      - AIRFLOW__METRICS__OTEL_ON=True
+      - AIRFLOW__METRICS__OTEL_HOST=breeze-otel-collector
+      - AIRFLOW__METRICS__OTEL_PORT=4318
+      - AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS=30000
+
     depends_on:
       - otel-collector
       - prometheus

scripts/ci/docker-compose/otel-collector-config.yml

@@ -15,40 +15,33 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-extensions:
-  memory_ballast:
-    size_mib: 512
-  zpages:
-    endpoint: 0.0.0.0:55679
+# Based on the default config found here:
+# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/configs/otelcol-contrib.yaml
 
 receivers:
   otlp:
     protocols:
-      grpc:
       http:
 
 processors:
   batch:
-  memory_limiter:
-    limit_mib: 1536
-    spike_limit_mib: 512
-    check_interval: 5s
 
 exporters:
   logging:
-    logLevel: debug
+    verbosity: detailed
   prometheus:
-    metrics:
-      endpoint: "http://localhost:29090"
+    endpoint: 0.0.0.0:8889
 
 service:
-  extensions: [memory_ballast, zpages]
+
   pipelines:
+
     traces:
       receivers: [otlp]
-      processors: [memory_limiter, batch]
+      processors: [batch]
       exporters: [logging]
+
     metrics:
       receivers: [otlp]
-      processors: [memory_limiter, batch]
+      processors: [batch]
       exporters: [logging, prometheus]

scripts/ci/docker-compose/prometheus-config.yml

@@ -33,6 +33,6 @@ scrape_configs:
       tls_config:
           insecure_skip_verify: true
 
-    - job_name: 'opentelemetry-collector'
+    - job_name: 'otel-collector'
       static_configs:
-          - targets: ['breeze-opentelemetry-collector:8888']
+          - targets: ['breeze-otel-collector:8889']

setup.py

@@ -300,6 +300,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
     "python-ldap",
 ]
 leveldb = ["plyvel"]
+otel = ["opentelemetry-api==1.15.0", "opentelemetry-exporter-otlp", "opentelemetry-exporter-prometheus"]
 pandas = [
     "pandas>=0.17.1",
 ]
@@ -327,7 +328,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
 
 # Mypy 0.900 and above ships only with stubs from stdlib so if we need other stubs, we need to install them
 # manually as `types-*`. See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-# for details. Wy want to install them explicitly because we want to eventually move to
+# for details. We want to install them explicitly because we want to eventually move to
 # mypyd which does not support installing the types dynamically with --install-types
 mypy_dependencies = [
     # TODO: upgrade to newer versions of MyPy continuously as they are released
@@ -469,6 +470,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]):
     "kerberos": kerberos,
     "ldap": ldap,
     "leveldb": leveldb,
+    "otel": otel,
     "pandas": pandas,
     "password": password,
     "rabbitmq": rabbitmq,