Merge pull request #3424 from DataDog/anmarchenko/source_code_integra…

…tion

[CIVIS-8604] APM source code integration

lib/datadog/core/environment/git.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative '../git/ext'
+require_relative '../utils/url'
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves git repository information from environment variables
+      module Git
+        def self.git_repository_url
+          return @git_repository_url if defined?(@git_repository_url)
+
+          @git_repository_url = Utils::Url.filter_basic_auth(ENV[Datadog::Core::Git::Ext::ENV_REPOSITORY_URL])
+        end
+
+        def self.git_commit_sha
+          return @git_commit_sha if defined?(@git_commit_sha)
+
+          @git_commit_sha = ENV[Datadog::Core::Git::Ext::ENV_COMMIT_SHA]
+        end
+      end
+    end
+  end
+end

lib/datadog/core/git/ext.rb

@@ -5,32 +5,11 @@ module Core
     module Git
       # Defines constants for Git tags
       module Ext
-        GIT_SHA_LENGTH = 40
-
-        TAG_BRANCH = 'git.branch'
-        TAG_REPOSITORY_URL = 'git.repository_url'
-        TAG_TAG = 'git.tag'
-
-        TAG_COMMIT_AUTHOR_DATE = 'git.commit.author.date'
-        TAG_COMMIT_AUTHOR_EMAIL = 'git.commit.author.email'
-        TAG_COMMIT_AUTHOR_NAME = 'git.commit.author.name'
-        TAG_COMMIT_COMMITTER_DATE = 'git.commit.committer.date'
-        TAG_COMMIT_COMMITTER_EMAIL = 'git.commit.committer.email'
-        TAG_COMMIT_COMMITTER_NAME = 'git.commit.committer.name'
-        TAG_COMMIT_MESSAGE = 'git.commit.message'
-        TAG_COMMIT_SHA = 'git.commit.sha'
+        TAG_REPOSITORY_URL = '_dd.git.repository_url'
+        TAG_COMMIT_SHA = '_dd.git.commit.sha'
 
         ENV_REPOSITORY_URL = 'DD_GIT_REPOSITORY_URL'
         ENV_COMMIT_SHA = 'DD_GIT_COMMIT_SHA'
-        ENV_BRANCH = 'DD_GIT_BRANCH'
-        ENV_TAG = 'DD_GIT_TAG'
-        ENV_COMMIT_MESSAGE = 'DD_GIT_COMMIT_MESSAGE'
-        ENV_COMMIT_AUTHOR_NAME = 'DD_GIT_COMMIT_AUTHOR_NAME'
-        ENV_COMMIT_AUTHOR_EMAIL = 'DD_GIT_COMMIT_AUTHOR_EMAIL'
-        ENV_COMMIT_AUTHOR_DATE = 'DD_GIT_COMMIT_AUTHOR_DATE'
-        ENV_COMMIT_COMMITTER_NAME = 'DD_GIT_COMMIT_COMMITTER_NAME'
-        ENV_COMMIT_COMMITTER_EMAIL = 'DD_GIT_COMMIT_COMMITTER_EMAIL'
-        ENV_COMMIT_COMMITTER_DATE = 'DD_GIT_COMMIT_COMMITTER_DATE'
       end
     end
   end

lib/datadog/core/utils/url.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Datadog
+  module Core
+    module Utils
+      # Helpers class that provides methods to process URLs
+      # such as filtering sensitive information.
+      module Url
+        def self.filter_basic_auth(url)
+          return nil if url.nil?
+
+          URI(url).tap do |u|
+            u.user = nil
+            u.password = nil
+          end.to_s
+        # Git scheme: git@github.com:DataDog/dd-trace-rb.git
+        rescue URI::InvalidURIError
+          url
+        end
+      end
+    end
+  end
+end

lib/datadog/profiling/ext.rb

@@ -23,6 +23,8 @@ module HTTP
           FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'
           FORM_FIELD_TAG_SERVICE = 'service'
           FORM_FIELD_TAG_VERSION = 'version'
+          TAG_GIT_REPOSITORY_URL = 'git.repository_url'
+          TAG_GIT_COMMIT_SHA = 'git.commit.sha'
 
           PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof'
           CODE_PROVENANCE_FILENAME = 'code-provenance.json'

lib/datadog/profiling/tag_builder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../core/utils'
+require_relative '../core/environment/git'
 
 module Datadog
   module Profiling
@@ -23,6 +24,8 @@ def self.call(
         runtime_id: Core::Environment::Identity.id,
         runtime_platform: Core::Environment::Identity.lang_platform,
         runtime_version: Core::Environment::Identity.lang_version,
+        git_repository_url: Core::Environment::Git.git_repository_url,
+        git_commit_sha: Core::Environment::Git.git_commit_sha,
         # User-provided tags
         user_tags: settings.tags
       )
@@ -42,6 +45,8 @@ def self.call(
         tags[FORM_FIELD_TAG_ENV] = env if env
         tags[FORM_FIELD_TAG_SERVICE] = service if service
         tags[FORM_FIELD_TAG_VERSION] = version if version
+        tags[TAG_GIT_REPOSITORY_URL] = git_repository_url if git_repository_url
+        tags[TAG_GIT_COMMIT_SHA] = git_commit_sha if git_commit_sha
 
         # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
         user_tags.merge(tags).map do |key, value|

lib/datadog/tracing/transport/trace_formatter.rb

@@ -2,6 +2,8 @@
 
 require_relative '../../core/environment/identity'
 require_relative '../../core/environment/socket'
+require_relative '../../core/environment/git'
+require_relative '../../core/git/ext'
 require_relative '../../core/runtime/ext'
 require_relative '../metadata/ext'
 require_relative '../trace_segment'
@@ -13,6 +15,7 @@ module Transport
       class TraceFormatter
         attr_reader \
           :root_span,
+          :first_span,
           :trace
 
         def self.format!(trace)
@@ -22,6 +25,9 @@ def self.format!(trace)
         def initialize(trace)
           @trace = trace
           @root_span = find_root_span(trace)
+          # source code integration uses the "first span in trace chunk" concept instead of root span
+          # see: https://github.com/DataDog/dd-trace-rb/pull/3424
+          @first_span = trace.spans.first
         end
 
         # Modifies a trace so suitable for transport
@@ -54,6 +60,11 @@ def format!
           tag_sampling_priority!
           tag_profiling_enabled!
 
+          if first_span
+            tag_git_repository_url!
+            tag_git_commit_sha!
+          end
+
           trace
         end
 
@@ -185,6 +196,18 @@ def tag_profiling_enabled!
           )
         end
 
+        def tag_git_repository_url!
+          return if git_repository_url.nil?
+
+          first_span.set_tag(Core::Git::Ext::TAG_REPOSITORY_URL, git_repository_url)
+        end
+
+        def tag_git_commit_sha!
+          return if git_commit_sha.nil?
+
+          first_span.set_tag(Core::Git::Ext::TAG_COMMIT_SHA, git_commit_sha)
+        end
+
         private
 
         def partial?
@@ -203,6 +226,14 @@ def find_root_span(trace)
           # when root span is not found, fall back to last span (partial flush)
           root_span || trace.spans.last
         end
+
+        def git_repository_url
+          Core::Environment::Git.git_repository_url
+        end
+
+        def git_commit_sha
+          Core::Environment::Git.git_commit_sha
+        end
       end
     end
   end

sig/datadog/core/environment/git.rbs

@@ -0,0 +1,15 @@
+module Datadog
+  module Core
+    module Environment
+      module Git
+        @git_repository_url: String?
+
+        @git_commit_sha: String?
+
+        def self?.git_repository_url: () -> String?
+
+        def self?.git_commit_sha: () -> String?
+      end
+    end
+  end
+end

sig/datadog/core/git/ext.rbs

@@ -2,51 +2,13 @@ module Datadog
   module Core
     module Git
       module Ext
-        GIT_SHA_LENGTH: ::Regexp
+        TAG_REPOSITORY_URL: "_dd.git.repository_url"
 
-        TAG_BRANCH: "git.branch"
-
-        TAG_REPOSITORY_URL: "git.repository_url"
-
-        TAG_TAG: "git.tag"
-
-        TAG_COMMIT_AUTHOR_DATE: "git.commit.author.date"
-
-        TAG_COMMIT_AUTHOR_EMAIL: "git.commit.author.email"
-
-        TAG_COMMIT_AUTHOR_NAME: "git.commit.author.name"
-
-        TAG_COMMIT_COMMITTER_DATE: "git.commit.committer.date"
-
-        TAG_COMMIT_COMMITTER_EMAIL: "git.commit.committer.email"
-
-        TAG_COMMIT_COMMITTER_NAME: "git.commit.committer.name"
-
-        TAG_COMMIT_MESSAGE: "git.commit.message"
-
-        TAG_COMMIT_SHA: "git.commit.sha"
+        TAG_COMMIT_SHA: "_dd.git.commit.sha"
 
         ENV_REPOSITORY_URL: "DD_GIT_REPOSITORY_URL"
 
         ENV_COMMIT_SHA: "DD_GIT_COMMIT_SHA"
-
-        ENV_BRANCH: "DD_GIT_BRANCH"
-
-        ENV_TAG: "DD_GIT_TAG"
-
-        ENV_COMMIT_MESSAGE: "DD_GIT_COMMIT_MESSAGE"
-
-        ENV_COMMIT_AUTHOR_NAME: "DD_GIT_COMMIT_AUTHOR_NAME"
-
-        ENV_COMMIT_AUTHOR_EMAIL: "DD_GIT_COMMIT_AUTHOR_EMAIL"
-
-        ENV_COMMIT_AUTHOR_DATE: "DD_GIT_COMMIT_AUTHOR_DATE"
-
-        ENV_COMMIT_COMMITTER_NAME: "DD_GIT_COMMIT_COMMITTER_NAME"
-
-        ENV_COMMIT_COMMITTER_EMAIL: "DD_GIT_COMMIT_COMMITTER_EMAIL"
-
-        ENV_COMMIT_COMMITTER_DATE: "DD_GIT_COMMIT_COMMITTER_DATE"
       end
     end
   end

sig/datadog/core/utils/url.rbs

@@ -0,0 +1,9 @@
+module Datadog
+  module Core
+    module Utils
+      module Url
+        def self?.filter_basic_auth: (::String? url) -> ::String?
+      end
+    end
+  end
+end

sig/datadog/profiling/ext.rbs

@@ -23,6 +23,8 @@ module Datadog
           FORM_FIELD_TAG_RUNTIME_VERSION: "runtime_version"
           FORM_FIELD_TAG_SERVICE: "service"
           FORM_FIELD_TAG_VERSION: "version"
+          TAG_GIT_REPOSITORY_URL: "git.repository_url"
+          TAG_GIT_COMMIT_SHA: "git.commit.sha"
           PPROF_DEFAULT_FILENAME: "rubyprofile.pprof"
           CODE_PROVENANCE_FILENAME: "code-provenance.json"
         end

sig/datadog/tracing/transport/trace_formatter.rbs

@@ -4,6 +4,8 @@ module Datadog
       class TraceFormatter
         attr_reader root_span: untyped
 
+        attr_reader first_span: untyped
+
         attr_reader trace: untyped
 
         def self.format!: (untyped trace) -> untyped
@@ -40,11 +42,19 @@ module Datadog
 
         def tag_high_order_trace_id!: () -> (nil | untyped)
 
+        def tag_git_repository_url!: () -> (nil | untyped)
+
+        def tag_git_commit_sha!: () -> (nil | untyped)
+
         private
 
         def partial?: () -> untyped
 
         def find_root_span: (untyped trace) -> untyped
+
+        def git_repository_url: () -> (nil | untyped)
+
+        def git_commit_sha: () -> (nil | untyped)
       end
     end
   end

spec/datadog/core/environment/git_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+require 'datadog/core/environment/git'
+
+RSpec.describe Datadog::Core::Environment::Git do
+  def remove_cached_variables
+    if described_class.instance_variable_defined?(:@git_repository_url)
+      described_class.remove_instance_variable(:@git_repository_url)
+    end
+
+    if described_class.instance_variable_defined?(:@git_commit_sha)
+      described_class.remove_instance_variable(:@git_commit_sha)
+    end
+  end
+
+  around do |example|
+    ClimateControl.modify(env_key => env_value) do
+      example.run
+    end
+  end
+
+  before do
+    remove_cached_variables
+  end
+
+  after do
+    remove_cached_variables
+  end
+
+  describe '.git_repository_url' do
+    subject { described_class.git_repository_url }
+
+    let(:env_key) { Datadog::Core::Git::Ext::ENV_REPOSITORY_URL }
+    let(:env_value) { 'https://gitlab-ci-token:AAA_bbb@gitlab.com/DataDog/systems-test.git' }
+
+    it { is_expected.to eq('https://gitlab.com/DataDog/systems-test.git') }
+  end
+
+  describe '.git_commit_sha' do
+    subject { described_class.git_commit_sha }
+
+    let(:env_key) { Datadog::Core::Git::Ext::ENV_COMMIT_SHA }
+    let(:env_value) { '1234567890abcdef' }
+
+    it { is_expected.to eq('1234567890abcdef') }
+  end
+end