diff --git a/docs/opentelemetry-tracing.rst b/docs/opentelemetry-tracing.rst index 9b3dea276f..cb9a2b1350 100644 --- a/docs/opentelemetry-tracing.rst +++ b/docs/opentelemetry-tracing.rst @@ -8,10 +8,8 @@ To take advantage of these traces, we first need to install OpenTelemetry: .. code-block:: sh - pip install opentelemetry-api opentelemetry-sdk opentelemetry-instrumentation - - # [Optional] Installs the cloud monitoring exporter, however you can use any exporter of your choice - pip install opentelemetry-exporter-google-cloud + pip install opentelemetry-api opentelemetry-sdk + pip install opentelemetry-exporter-gcp-trace We also need to tell OpenTelemetry which exporter to use. To export Spanner traces to `Cloud Tracing `_, add the following lines to your application: @@ -19,21 +17,37 @@ We also need to tell OpenTelemetry which exporter to use. To export Spanner trac from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.trace.sampling import ProbabilitySampler + from opentelemetry.sdk.trace.sampling import TraceIdRatioBased from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter - # BatchExportSpanProcessor exports spans to Cloud Trace + # BatchSpanProcessor exports spans to Cloud Trace # in a seperate thread to not block on the main thread - from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + from opentelemetry.sdk.trace.export import BatchSpanProcessor # Create and export one trace every 1000 requests - sampler = ProbabilitySampler(1/1000) + sampler = TraceIdRatioBased(1/1000) # Use the default tracer provider trace.set_tracer_provider(TracerProvider(sampler=sampler)) trace.get_tracer_provider().add_span_processor( # Initialize the cloud tracing exporter - BatchExportSpanProcessor(CloudTraceSpanExporter()) + BatchSpanProcessor(CloudTraceSpanExporter()) ) + +To get more fine-grained traces from gRPC, you can enable the gRPC instrumentation by the following + +.. code-block:: sh + + pip install opentelemetry-instrumentation opentelemetry-instrumentation-grpc + +and then in your Python code, please add the following lines: + +.. code:: python + + from opentelemetry.instrumentation.grpc import GrpcInstrumentorClient + grpc_client_instrumentor = GrpcInstrumentorClient() + grpc_client_instrumentor.instrument() + + Generated spanner traces should now be available on `Cloud Trace `_. Tracing is most effective when many libraries are instrumented to provide insight over the entire lifespan of a request. diff --git a/examples/grpc_instrumentation_enabled.py b/examples/grpc_instrumentation_enabled.py new file mode 100644 index 0000000000..c8bccd0a9d --- /dev/null +++ b/examples/grpc_instrumentation_enabled.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import os +import time + +import google.cloud.spanner as spanner +from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry import trace + +# Enable the gRPC instrumentation if you'd like more introspection. +from opentelemetry.instrumentation.grpc import GrpcInstrumentorClient + +grpc_client_instrumentor = GrpcInstrumentorClient() +grpc_client_instrumentor.instrument() + + +def main(): + # Setup common variables that'll be used between Spanner and traces. + project_id = os.environ.get('SPANNER_PROJECT_ID', 'test-project') + + # Setup OpenTelemetry, trace and Cloud Trace exporter. + tracer_provider = TracerProvider(sampler=ALWAYS_ON) + trace_exporter = CloudTraceSpanExporter(project_id=project_id) + tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) + trace.set_tracer_provider(tracer_provider) + # Retrieve a tracer from the global tracer provider. + tracer = tracer_provider.get_tracer('MyApp') + + # Setup the Cloud Spanner Client. + spanner_client = spanner.Client(project_id) + + instance = spanner_client.instance('test-instance') + database = instance.database('test-db') + + # Now run our queries + with tracer.start_as_current_span('QueryInformationSchema'): + with database.snapshot() as snapshot: + with tracer.start_as_current_span('InformationSchema'): + info_schema = snapshot.execute_sql( + 'SELECT * FROM INFORMATION_SCHEMA.TABLES') + for row in info_schema: + print(row) + + with tracer.start_as_current_span('ServerTimeQuery'): + with database.snapshot() as snapshot: + # Purposefully issue a bad SQL statement to examine exceptions + # that get recorded and a ERROR span status. + try: + data = snapshot.execute_sql('SELECT CURRENT_TIMESTAMPx()') + for row in data: + print(row) + except Exception as e: + pass + + +if __name__ == '__main__': + main() diff --git a/examples/trace.py b/examples/trace.py new file mode 100644 index 0000000000..791b6cd20b --- /dev/null +++ b/examples/trace.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import os +import time + +import google.cloud.spanner as spanner +from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry import trace + + +def main(): + # Setup common variables that'll be used between Spanner and traces. + project_id = os.environ.get('SPANNER_PROJECT_ID', 'test-project') + + # Setup OpenTelemetry, trace and Cloud Trace exporter. + tracer_provider = TracerProvider(sampler=ALWAYS_ON) + trace_exporter = CloudTraceSpanExporter(project_id=project_id) + tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) + trace.set_tracer_provider(tracer_provider) + # Retrieve a tracer from the global tracer provider. + tracer = tracer_provider.get_tracer('MyApp') + + # Setup the Cloud Spanner Client. + spanner_client = spanner.Client(project_id) + instance = spanner_client.instance('test-instance') + database = instance.database('test-db') + + # Now run our queries + with tracer.start_as_current_span('QueryInformationSchema'): + with database.snapshot() as snapshot: + with tracer.start_as_current_span('InformationSchema'): + info_schema = snapshot.execute_sql( + 'SELECT * FROM INFORMATION_SCHEMA.TABLES') + for row in info_schema: + print(row) + + with tracer.start_as_current_span('ServerTimeQuery'): + with database.snapshot() as snapshot: + # Purposefully issue a bad SQL statement to examine exceptions + # that get recorded and a ERROR span status. + try: + data = snapshot.execute_sql('SELECT CURRENT_TIMESTAMPx()') + for row in data: + print(row) + except Exception as e: + print(e) + + +if __name__ == '__main__': + main() diff --git a/google/cloud/spanner_v1/_opentelemetry_tracing.py b/google/cloud/spanner_v1/_opentelemetry_tracing.py index 8f9f8559ef..51501a07a3 100644 --- a/google/cloud/spanner_v1/_opentelemetry_tracing.py +++ b/google/cloud/spanner_v1/_opentelemetry_tracing.py @@ -16,17 +16,39 @@ from contextlib import contextmanager -from google.api_core.exceptions import GoogleAPICallError from google.cloud.spanner_v1 import SpannerClient +from google.cloud.spanner_v1 import gapic_version try: from opentelemetry import trace from opentelemetry.trace.status import Status, StatusCode + from opentelemetry.semconv.attributes.otel_attributes import ( + OTEL_SCOPE_NAME, + OTEL_SCOPE_VERSION, + ) HAS_OPENTELEMETRY_INSTALLED = True except ImportError: HAS_OPENTELEMETRY_INSTALLED = False +TRACER_NAME = "cloud.google.com/python/spanner" +TRACER_VERSION = gapic_version.__version__ + + +def get_tracer(tracer_provider=None): + """ + get_tracer is a utility to unify and simplify retrieval of the tracer, without + leaking implementation details given that retrieving a tracer requires providing + the full qualified library name and version. + When the tracer_provider is set, it'll retrieve the tracer from it, otherwise + it'll fall back to the global tracer provider and use this library's specific semantics. + """ + if not tracer_provider: + # Acquire the global tracer provider. + tracer_provider = trace.get_tracer_provider() + + return tracer_provider.get_tracer(TRACER_NAME, TRACER_VERSION) + @contextmanager def trace_call(name, session, extra_attributes=None): @@ -35,7 +57,7 @@ def trace_call(name, session, extra_attributes=None): yield None return - tracer = trace.get_tracer(__name__) + tracer = get_tracer() # Set base attributes that we know for every trace created attributes = { @@ -43,6 +65,8 @@ def trace_call(name, session, extra_attributes=None): "db.url": SpannerClient.DEFAULT_ENDPOINT, "db.instance": session._database.name, "net.host.name": SpannerClient.DEFAULT_ENDPOINT, + OTEL_SCOPE_NAME: TRACER_NAME, + OTEL_SCOPE_VERSION: TRACER_VERSION, } if extra_attributes: @@ -52,9 +76,10 @@ def trace_call(name, session, extra_attributes=None): name, kind=trace.SpanKind.CLIENT, attributes=attributes ) as span: try: - span.set_status(Status(StatusCode.OK)) yield span - except GoogleAPICallError as error: - span.set_status(Status(StatusCode.ERROR)) + except Exception as error: + span.set_status(Status(StatusCode.ERROR, str(error))) span.record_exception(error) raise + else: + span.set_status(Status(StatusCode.OK)) diff --git a/setup.py b/setup.py index 98b1a61748..544d117fd7 100644 --- a/setup.py +++ b/setup.py @@ -47,9 +47,9 @@ ] extras = { "tracing": [ - "opentelemetry-api >= 1.1.0", - "opentelemetry-sdk >= 1.1.0", - "opentelemetry-instrumentation >= 0.20b0, < 0.23dev", + "opentelemetry-api >= 1.22.0", + "opentelemetry-sdk >= 1.22.0", + "opentelemetry-semantic-conventions >= 0.43b0", ], "libcst": "libcst >= 0.2.5", } diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 20170203f5..e468d57168 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -10,9 +10,9 @@ grpc-google-iam-v1==0.12.4 libcst==0.2.5 proto-plus==1.22.0 sqlparse==0.4.4 -opentelemetry-api==1.1.0 -opentelemetry-sdk==1.1.0 -opentelemetry-instrumentation==0.20b0 +opentelemetry-api==1.22.0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 protobuf==3.20.2 deprecated==1.2.14 grpc-interceptor==0.15.4 diff --git a/tests/_helpers.py b/tests/_helpers.py index 42178fd439..5e514f2586 100644 --- a/tests/_helpers.py +++ b/tests/_helpers.py @@ -1,6 +1,10 @@ import unittest import mock +from google.cloud.spanner_v1 import gapic_version + +LIB_VERSION = gapic_version.__version__ + try: from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -8,6 +12,11 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) + from opentelemetry.semconv.attributes.otel_attributes import ( + OTEL_SCOPE_NAME, + OTEL_SCOPE_VERSION, + ) + from opentelemetry.trace.status import StatusCode trace.set_tracer_provider(TracerProvider()) @@ -30,6 +39,18 @@ def get_test_ot_exporter(): return _TEST_OT_EXPORTER +def enrich_with_otel_scope(attrs): + """ + This helper enriches attrs with OTEL_SCOPE_NAME and OTEL_SCOPE_VERSION + for the purpose of avoiding cumbersome duplicated imports. + """ + if HAS_OPENTELEMETRY_INSTALLED: + attrs[OTEL_SCOPE_NAME] = "cloud.google.com/python/spanner" + attrs[OTEL_SCOPE_VERSION] = LIB_VERSION + + return attrs + + def use_test_ot_exporter(): global _TEST_OT_PROVIDER_INITIALIZED diff --git a/tests/system/test_session_api.py b/tests/system/test_session_api.py index d0421d3a70..5322527d12 100644 --- a/tests/system/test_session_api.py +++ b/tests/system/test_session_api.py @@ -346,6 +346,8 @@ def _make_attributes(db_instance, **kwargs): "net.host.name": "spanner.googleapis.com", "db.instance": db_instance, } + ot_helpers.enrich_with_otel_scope(attributes) + attributes.update(kwargs) return attributes diff --git a/tests/unit/test__opentelemetry_tracing.py b/tests/unit/test__opentelemetry_tracing.py index 25870227bf..20e31d9ea6 100644 --- a/tests/unit/test__opentelemetry_tracing.py +++ b/tests/unit/test__opentelemetry_tracing.py @@ -12,7 +12,11 @@ from google.api_core.exceptions import GoogleAPICallError from google.cloud.spanner_v1 import _opentelemetry_tracing -from tests._helpers import OpenTelemetryBase, HAS_OPENTELEMETRY_INSTALLED +from tests._helpers import ( + OpenTelemetryBase, + HAS_OPENTELEMETRY_INSTALLED, + enrich_with_otel_scope, +) def _make_rpc_error(error_cls, trailing_metadata=None): @@ -55,11 +59,13 @@ def test_trace_call(self): "db.instance": "database_name", } - expected_attributes = { - "db.type": "spanner", - "db.url": "spanner.googleapis.com", - "net.host.name": "spanner.googleapis.com", - } + expected_attributes = enrich_with_otel_scope( + { + "db.type": "spanner", + "db.url": "spanner.googleapis.com", + "net.host.name": "spanner.googleapis.com", + } + ) expected_attributes.update(extra_attributes) with _opentelemetry_tracing.trace_call( @@ -80,11 +86,13 @@ def test_trace_call(self): def test_trace_error(self): extra_attributes = {"db.instance": "database_name"} - expected_attributes = { - "db.type": "spanner", - "db.url": "spanner.googleapis.com", - "net.host.name": "spanner.googleapis.com", - } + expected_attributes = enrich_with_otel_scope( + { + "db.type": "spanner", + "db.url": "spanner.googleapis.com", + "net.host.name": "spanner.googleapis.com", + } + ) expected_attributes.update(extra_attributes) with self.assertRaises(GoogleAPICallError): @@ -106,11 +114,13 @@ def test_trace_error(self): def test_trace_grpc_error(self): extra_attributes = {"db.instance": "database_name"} - expected_attributes = { - "db.type": "spanner", - "db.url": "spanner.googleapis.com:443", - "net.host.name": "spanner.googleapis.com:443", - } + expected_attributes = enrich_with_otel_scope( + { + "db.type": "spanner", + "db.url": "spanner.googleapis.com:443", + "net.host.name": "spanner.googleapis.com:443", + } + ) expected_attributes.update(extra_attributes) with self.assertRaises(GoogleAPICallError): @@ -129,11 +139,13 @@ def test_trace_grpc_error(self): def test_trace_codeless_error(self): extra_attributes = {"db.instance": "database_name"} - expected_attributes = { - "db.type": "spanner", - "db.url": "spanner.googleapis.com:443", - "net.host.name": "spanner.googleapis.com:443", - } + expected_attributes = enrich_with_otel_scope( + { + "db.type": "spanner", + "db.url": "spanner.googleapis.com:443", + "net.host.name": "spanner.googleapis.com:443", + } + ) expected_attributes.update(extra_attributes) with self.assertRaises(GoogleAPICallError): diff --git a/tests/unit/test_batch.py b/tests/unit/test_batch.py index ee96decf5e..2f6b5e4ae9 100644 --- a/tests/unit/test_batch.py +++ b/tests/unit/test_batch.py @@ -14,7 +14,11 @@ import unittest -from tests._helpers import OpenTelemetryBase, StatusCode +from tests._helpers import ( + OpenTelemetryBase, + StatusCode, + enrich_with_otel_scope, +) from google.cloud.spanner_v1 import RequestOptions TABLE_NAME = "citizens" @@ -29,6 +33,7 @@ "db.instance": "testing", "net.host.name": "spanner.googleapis.com", } +enrich_with_otel_scope(BASE_ATTRIBUTES) class _BaseTest(unittest.TestCase): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index d4052f0ae3..2ae0cb94b8 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -19,7 +19,7 @@ from tests._helpers import ( OpenTelemetryBase, StatusCode, - HAS_OPENTELEMETRY_INSTALLED, + enrich_with_otel_scope, ) @@ -31,11 +31,6 @@ def _make_rpc_error(error_cls, trailing_metadata=None): return error_cls("error", errors=(grpc_error,)) -class _ConstantTime: - def time(self): - return 1 - - class TestSession(OpenTelemetryBase): PROJECT_ID = "project-id" INSTANCE_ID = "instance-id" @@ -51,6 +46,7 @@ class TestSession(OpenTelemetryBase): "db.instance": DATABASE_NAME, "net.host.name": "spanner.googleapis.com", } + enrich_with_otel_scope(BASE_ATTRIBUTES) def _getTargetClass(self): from google.cloud.spanner_v1.session import Session @@ -1337,17 +1333,9 @@ def _time(_results=[1, 1.5]): return _results.pop(0) with mock.patch("time.time", _time): - if HAS_OPENTELEMETRY_INSTALLED: - with mock.patch("opentelemetry.util._time", _ConstantTime()): - with mock.patch("time.sleep") as sleep_mock: - with self.assertRaises(Aborted): - session.run_in_transaction( - unit_of_work, "abc", timeout_secs=1 - ) - else: - with mock.patch("time.sleep") as sleep_mock: - with self.assertRaises(Aborted): - session.run_in_transaction(unit_of_work, "abc", timeout_secs=1) + with mock.patch("time.sleep") as sleep_mock: + with self.assertRaises(Aborted): + session.run_in_transaction(unit_of_work, "abc", timeout_secs=1) sleep_mock.assert_not_called() @@ -1418,15 +1406,9 @@ def _time(_results=[1, 2, 4, 8]): return _results.pop(0) with mock.patch("time.time", _time): - if HAS_OPENTELEMETRY_INSTALLED: - with mock.patch("opentelemetry.util._time", _ConstantTime()): - with mock.patch("time.sleep") as sleep_mock: - with self.assertRaises(Aborted): - session.run_in_transaction(unit_of_work, timeout_secs=8) - else: - with mock.patch("time.sleep") as sleep_mock: - with self.assertRaises(Aborted): - session.run_in_transaction(unit_of_work, timeout_secs=8) + with mock.patch("time.sleep") as sleep_mock: + with self.assertRaises(Aborted): + session.run_in_transaction(unit_of_work, timeout_secs=8) # unpacking call args into list call_args = [call_[0][0] for call_ in sleep_mock.call_args_list] diff --git a/tests/unit/test_snapshot.py b/tests/unit/test_snapshot.py index bf5563dcfd..bf7363fef2 100644 --- a/tests/unit/test_snapshot.py +++ b/tests/unit/test_snapshot.py @@ -21,6 +21,7 @@ OpenTelemetryBase, StatusCode, HAS_OPENTELEMETRY_INSTALLED, + enrich_with_otel_scope, ) from google.cloud.spanner_v1.param_types import INT64 from google.api_core.retry import Retry @@ -46,6 +47,8 @@ "db.instance": "testing", "net.host.name": "spanner.googleapis.com", } +enrich_with_otel_scope(BASE_ATTRIBUTES) + DIRECTED_READ_OPTIONS = { "include_replicas": { "replica_selections": [ @@ -530,12 +533,14 @@ def test_iteration_w_multiple_span_creation(self): self.assertEqual(span.name, name) self.assertEqual( dict(span.attributes), - { - "db.type": "spanner", - "db.url": "spanner.googleapis.com", - "db.instance": "testing", - "net.host.name": "spanner.googleapis.com", - }, + enrich_with_otel_scope( + { + "db.type": "spanner", + "db.url": "spanner.googleapis.com", + "db.instance": "testing", + "net.host.name": "spanner.googleapis.com", + } + ), ) diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index b40ae8843f..d52fb61db1 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -21,7 +21,11 @@ from google.api_core.retry import Retry from google.api_core import gapic_v1 -from tests._helpers import OpenTelemetryBase, StatusCode +from tests._helpers import ( + OpenTelemetryBase, + StatusCode, + enrich_with_otel_scope, +) TABLE_NAME = "citizens" COLUMNS = ["email", "first_name", "last_name", "age"] @@ -58,6 +62,7 @@ class TestTransaction(OpenTelemetryBase): "db.instance": "testing", "net.host.name": "spanner.googleapis.com", } + enrich_with_otel_scope(BASE_ATTRIBUTES) def _getTargetClass(self): from google.cloud.spanner_v1.transaction import Transaction