Protobuf system test for schema tracking

docs/edit/add-new-test.md

@@ -3,7 +3,7 @@ Whether it's adding a new test or modifying an existing test, a moderate amount
 Once the changes are complete, post them in a PR.
 
 #### Notes
-* Each test class tests only one feature
+* Each test class tests only one feature (see [the doc on features](https://github.com/DataDog/system-tests/blob/main/docs/edit/features.md))
 * A test class can have several tests
 * If an RFC for the feature exists, you must use the decorator `rfc` decorator:
 ```python

docs/weblog/README.md

@@ -949,3 +949,35 @@ stripped (accounting for a quirk in the boto3 library).
 Examples:
 - `GET`: `/mock_s3/multipart_upload?bucket=somebucket&key=somekey`
 
+
+### \[GET\] /mock_s3/copy_object
+
+This endpoint is used to test the s3 integration. It creates a bucket if
+necessary based on the `original_bucket` query parameter and puts an object at
+the `original_key` query parameter. The body of the object is just the bytes of
+the key, encoded with utf-8. The method then creates another `bucket` if
+necessary and copies the object into the `key` location. Returns a result
+object with an `object` JSON object field containing the `e_tag` field with the
+ETag of the copied object. The `e_tag` field has any extra double-quotes
+stripped (accounting for a quirk in the boto3 library).
+
+Examples:
+- `GET`: `/mock_s3/copy_object?original_bucket=somebucket&original_key=somekey&bucket=someotherbucket&key=someotherkey`
+
+
+### \[GET\] /protobuf/serialize
+
+This endpoint serializes a constant protobuf message. Returns the serialized message as a base64 encoded string. It is meant to be used to test the protobuf serialization integration.
+
+Examples:
+- `GET`: `/protobuf/serialize`
+
+### \[GET\] /protobuf/deserialize
+
+This endpoint deserializes a protobuf message from a base64 encoded string provided in the query parameters. Returns "ok" if deserialization is successful. The simplest way to use it is to pass the output of the `/protobuf/serialize` endpoint to it. It is meant to be used to test the protobuf deserialization integration.
+
+Expected query params:
+- `msg`: Base64 encoded protobuf message to deserialize
+
+Examples:
+- `GET`: `/protobuf/deserialize?msg=<base64_encoded_message>`

manifests/cpp_httpd.yml

@@ -76,6 +76,7 @@ tests/:
   test_library_conf.py: irrelevant
   test_miscs.py: irrelevant
   test_profiling.py: missing_feature (missing profiling module in weblog)
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: v1.0.0  # real version unknown
     Test_SamplingDecisionAdded: v1.0.0  # real version unknown

manifests/cpp_nginx.yml

@@ -158,6 +158,7 @@ tests/:
   test_ipv6.py: missing_feature (APMAPI-869)
   test_library_conf.py: irrelevant
   test_miscs.py: irrelevant
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: v1.0.0  # real version unknown
     Test_SamplingDecisionAdded: v1.0.0  # real version unknown

manifests/dotnet.yml

@@ -612,6 +612,8 @@ tests/:
     Test_Profile:
       "*": v1.0  # real version not known
       uds: missing_feature (to be confirmed)
+  test_protobuf.py:
+    Test_Protobuf: v3.15.0
   test_sampling_rates.py:
     Test_SampleRateFunction: v3.11.1  # real version unknown
     Test_SamplingDecisionAdded: v3.11.1  # real version unknown

manifests/golang.yml

@@ -752,6 +752,7 @@ tests/:
     Test_HeaderTags_Whitespace_Tag: v1.53.0
     Test_HeaderTags_Whitespace_Val_Long: v1.53.0
     Test_HeaderTags_Whitespace_Val_Short: v1.53.0
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: v1.72.1  # real version unknown
     Test_SamplingDecisionAdded: v1.72.1  # real version unknown

manifests/java.yml

@@ -2029,6 +2029,10 @@ tests/:
       spring-boot-3-native: v1.39.0
       vertx3: missing_feature (Endpoint not implemented)
       vertx4: missing_feature (Endpoint not implemented)
+  test_protobuf.py:
+    Test_Protobuf:
+      '*': missing_feature
+      spring-boot: v1.47.0
   test_sampling_rates.py:
     Test_SampleRateFunction:
       '*': missing_feature (Endpoint /sample_rate_route not implemented)

manifests/nodejs.yml

@@ -1099,6 +1099,7 @@ tests/:
     Test_HeaderTags_Whitespace_Val_Short: *ref_4_11_0
   test_profiling.py:
     Test_Profile: *ref_5_16_0  #actual version unknown
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: bug (APMAPI-736)
     Test_SamplingDecisionAdded: *ref_5_17_0  # real version unknown

manifests/php.yml

@@ -595,6 +595,7 @@ tests/:
     Test_HeaderTags_Whitespace_Val_Short: v0.74.0
   test_profiling.py:
     Test_Profile: missing_feature (profiling seems not to be activated)
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: bug (APMAPI-736)
     Test_SamplingDecisionAdded: v1.7.2  # real version unknown

manifests/python.yml

@@ -1056,6 +1056,7 @@ tests/:
       '*': v0.1  # actual version unknown
       django-py3.13: missing_feature (python 3.13 not supported yet)
       python3.12: v2.1.0
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: bug (APMAPI-736)
     Test_SamplingDecisionAdded: v2.8.0

manifests/ruby.yml

@@ -720,6 +720,7 @@ tests/:
     Test_HeaderTags_Whitespace_Val_Short: v1.13.0
   test_profiling.py:
     Test_Profile: missing_feature (temporary fix, scenario not working on dd-trace-rb CI)
+  test_protobuf.py: missing_feature
   test_sampling_rates.py:
     Test_SampleRateFunction: missing_feature  # very flakey
     Test_SamplingDecisionAdded: v2.12.1  # real version unknown

tests/test_protobuf.py

@@ -0,0 +1,47 @@
+import json
+
+from utils import weblog, interfaces, features
+
+
+# this test relies on the proto file at utils/build/docker/common/message.proto
+@features.datastreams_monitoring_protobuf_schema_tracking
+class Test_Protobuf:
+    def setup_protobuf(self):
+        self.serialization_response = weblog.get("/protobuf/serialize")
+        self.deserialization_response = weblog.get(f"/protobuf/deserialize?msg={self.serialization_response.text}")
+
+    def test_protobuf(self):
+        assert self.serialization_response.status_code == 200, self.serialization_response.text
+        assert self.deserialization_response.status_code == 200, self.deserialization_response.text
+
+        def validator(trace: list):
+            if len(trace) == 1:
+                span = trace[0]
+            else:
+                # find root span
+                span = next(s for s in trace if s["parent_id"] == 0)
+                # then find child-most span
+                while True:
+                    child = next((s for s in trace if s["parent_id"] == span["span_id"]), None)
+                    if child:
+                        span = child
+                    else:
+                        break
+
+            meta = span.get("meta", {})
+            assert "schema.id" in meta
+            assert "schema.type" in meta
+            assert "schema.definition" in meta
+            assert "schema.name" in meta
+            assert "schema.operation" in meta
+
+            assert meta["schema.id"] == "14603317962659197404", "hash should be the same across tracers"
+            assert meta["schema.type"] == "protobuf"
+            json.loads(meta["schema.definition"])  # will throw if malformed
+            assert "x-protobuf-number" in meta["schema.definition"]  # rough check that we register the protobuf numbers
+            assert meta["schema.name"] == "proto_message.AddressBook"
+
+            return True
+
+        interfaces.library.validate_traces(request=self.serialization_response, validator=validator)
+        interfaces.library.validate_traces(request=self.deserialization_response, validator=validator)

utils/_features.py

@@ -2,13 +2,9 @@
 
 
 class _Features:
-    """Data source is https://dd-feature-parity.azurewebsites.net/Import/Features
+    """See https://github.com/DataDog/system-tests/blob/main/docs/edit/features.md
 
-    run this command to get new features:
-
-    ```
-    PYTHONPATH=. python utils/scripts/update_features.py
-    ```
+    Data source is the feature parity dashboard https://feature-parity.us1.prod.dog/
     """
 
     @staticmethod
@@ -2469,6 +2465,15 @@ def debugger_inproduct_enablement(test_object):
         pytest.mark.features(feature_id=369)(test_object)
         return test_object
 
+    @staticmethod
+    def datastreams_monitoring_protobuf_schema_tracking(test_object):
+        """Dynamically enable debugger products
+
+        https://feature-parity.us1.prod.dog/#/?feature=371
+        """
+        pytest.mark.features(feature_id=371)(test_object)
+        return test_object
+
     @staticmethod
     def trace_enablement(test_object):
         """Enforces standardized behaviors for configurations across the tracing libraries.

utils/build/docker/common/message.proto

@@ -0,0 +1,109 @@
+// to regenerate the source files when modifying this, run:
+// protoc --csharp_out=../dotnet/weblog/Models/ --java_out=../java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/ message.proto
+
+// used in tests/test_protobuf.py
+
+syntax = "proto3";
+package proto_message;
+
+message AddressBook {
+  map<int32, Person> people = 1; // checks that the types generated by the usage of a map are correctly handled
+  PhoneNumber central = 2; // here to check that we don't re-extract the PhoneNumber message when we encounter it again
+
+  // now just put every type to be exhaustive
+  // to help debugging a bit, the number given is the enum value of the type + 10
+  double my_double = 11;
+  float my_float = 12;
+  int64 my_int64 = 13;
+  uint64 my_uint64 = 14;
+  int32 my_int32 = 15;
+  fixed64 my_fixed64 = 16;
+  fixed32 my_fixed32 = 17;
+  bool my_bool = 18;
+  string my_string = 19;
+  bytes my_bytes = 22;
+  uint32 my_uint32 = 23;
+  sfixed32 my_sfixed32 = 25;
+  sfixed64 my_sfixed64 = 26;
+  sint32 my_sint32 = 27;
+  sint64 my_sint64 = 28;
+
+  // here to test that we stop extracting after a certain depth
+  Deep d = 40;
+}
+
+message PhoneNumber {
+  string number = 1;
+  PhoneType type = 2; // check that enum are handled properly
+}
+
+enum PhoneType {
+  UNSPECIFIED = 0;
+  MOBILE = 1;
+  HOME = 2;
+  WORK = 3;
+}
+
+message Person {
+  string name = 1;
+  repeated PhoneNumber phones = 4; // check repeated fields
+  uint64 created = 5;
+  double last_updated = 6;
+
+  // this one is very artificial, but it's here to check that we stop at the same depth for all tracers
+  RecursiveField recurse = 7;
+
+  message RecursiveField {
+    sint32 value = 1;
+    RecursiveField deeper = 2;
+  }
+}
+
+message Deep {
+  A x = 1;
+
+  message A{
+    B x = 1;
+  }
+
+  message B{
+    C x = 1;
+  }
+
+  message C{
+    D x = 1;
+  }
+
+  message D{
+    E x = 1;
+  }
+
+  message E{
+    F x = 1;
+  }
+
+  message F{
+    G x = 1;
+  }
+
+  message G{
+    H x = 1;
+  }
+
+  message H{
+    I x = 1; // extraction should stop here (max depth = 10)
+  }
+
+  message I{
+    J x = 1;
+  }
+
+  message J{
+    int32 x = 1;
+  }
+}
+
+// just here to check that messages that are just floating around in the same file do not get extracted
+message Ignored {
+  int32 x = 1;
+}

utils/build/docker/dotnet/install_ddtrace.sh

@@ -10,7 +10,7 @@ get_latest_release() {
 mkdir -p /opt/datadog
 
 if [ $(ls /binaries/Datadog.Trace.ClrProfiler.Native.so | wc -l) = 1 ]; then
-    echo "Install from local folder"
+    echo "Install ddtrace from local folder"
     cp -r /binaries/* /opt/datadog/
 else
     if [ $(ls datadog-dotnet-apm*.tar.gz | wc -l) = 1 ]; then

utils/build/docker/dotnet/weblog/Endpoints/ProtobufEndpoint.cs

@@ -0,0 +1,34 @@
+using System;
+using System.IO;
+using Google.Protobuf;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using ProtoMessage;
+
+namespace weblog;
+
+public class ProtobufEndpoint : ISystemTestEndpoint
+{
+    public void Register(Microsoft.AspNetCore.Routing.IEndpointRouteBuilder routeBuilder)
+    {
+        routeBuilder.MapGet("/protobuf/serialize", async context =>
+        {
+            // the content of the message does not really matter since it's the schema that is observed.
+            var msg = new AddressBook { Central = new PhoneNumber { Number = "0123", Type = PhoneType.Work } };
+            using (MemoryStream stream = new MemoryStream())
+            {
+                msg.WriteTo(stream);
+                var b64Proto = Convert.ToBase64String(stream.ToArray());
+                await context.Response.WriteAsync(b64Proto);
+            }
+        });
+
+        routeBuilder.MapGet("/protobuf/deserialize", async context =>
+        {
+            var b64Msg = context.Request.Query["msg"];
+            var msg = Convert.FromBase64String(b64Msg);
+            new AddressBook().MergeFrom(msg);
+            await context.Response.WriteAsync("ok");
+        });
+    }
+}