Merge branch 'master-merge-to-di' into base-merge

* master-merge-to-di: (37 commits)
  Revert "Revert "DEBUG-2334 Add Debugger component (DataDog#3640)""
  Update lockfiles for release 2.0.0
  Bump version 2.0.0.rc1 to 2.0.0
  Add 2.0.0 to CHANGELOG.md
  Add missing Apache 2.0 license from gemspec
  Use ffi 1.16.3
  Add platform
  Output gem version
  Output debug logs
  Fix spec to exclude .vscode directory
  Fix docker run command
  Add vscode settings for dockerfile file associations
  Update gem version
  Update gemfiles/*
  Update `ffi` in Gemfile
  Remove experimental warning from OpenTelemetry.md
  Remove latest tag on docker images
  add installer system tests
  Extract trace id format helper method (DataDog#3670)
  Remaps http status code tag for otel span for trace metrics (DataDog#3664)
  ...

.circleci/images/primary/Dockerfile-jruby-9.4.0.0 → .circleci/images/primary/Dockerfile-jruby-9.4.7.0

@@ -1,22 +1,11 @@
-# Note:
-#
-# There is an incompatibility between ethon and httprb on debian 11 bullseye.
-# This is why this image is based on debian 10 buster.
-#
-# See:
-# - https://github.com/jruby/jruby/issues/7033
-# - https://github.com/DataDog/dd-trace-rb/pull/2380#issuecomment-1320994823
-
 # Note: See the "Publishing updates to images" note in ./README.md for how to publish new builds of this container image
 
-# openjdk:11-jre image is from https://github.com/docker-library/openjdk/blob/8dfb0c5645098b8c330c4811c8228cae52f18388/11/jre/buster/Dockerfile
-# note: docker-library/openjdk is deprecated, there is a later move to https://hub.docker.com/_/eclipse-temurin
-FROM openjdk:11-jre-buster AS jruby-9.4.0.0-jre11
+FROM eclipse-temurin:11-jammy AS jruby-9.4.7.0-jre11
 
 RUN apt-get update && apt-get install -y libc6-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
-ENV JRUBY_VERSION 9.4.0.0
-ENV JRUBY_SHA256 897bb8a98ad43adcbf5fd3aa75ec85b3312838c949592ca3f623dc1f569d2870
+ENV JRUBY_VERSION 9.4.7.0
+ENV JRUBY_SHA256 f1c39f8257505300a528ff83fe4721fbe61a855abb25e3d27d52d43ac97a4d80
 RUN mkdir /opt/jruby \
   && curl -fSL https://repo1.maven.org/maven2/org/jruby/jruby-dist/${JRUBY_VERSION}/jruby-dist-${JRUBY_VERSION}-bin.tar.gz -o /tmp/jruby.tar.gz \
   && echo "$JRUBY_SHA256 /tmp/jruby.tar.gz" | sha256sum -c - \
@@ -46,7 +35,7 @@ RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" \
 
 CMD [ "irb" ]
 
-FROM jruby-9.4.0.0-jre11
+FROM jruby-9.4.7.0-jre11
 
 # Make apt non-interactive
 RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \

.github/labeler.yml

@@ -43,3 +43,15 @@ dev/ci:
 # Version bump pull request
 release:
   - all: [ '{CHANGELOG.md,lib/datadog/version.rb}' ]
+
+# Changes to OpenTelemetry
+otel:
+  - any: [ 'lib/datadog/opentelemetry/**' ]
+
+# Changes to Single Step Instrumentation
+single-step:
+  - any: [ 'lib-injection/**' ]
+
+# Changes to Debugging
+debugging:
+  - any: [ 'lib/datadog/debugging/**' ]

.github/workflows/build-ruby.yml

@@ -50,8 +50,8 @@ jobs:
             version: 9.3.9.0
             dockerfile: Dockerfile-jruby-9.3.9.0
           - engine: jruby
-            version: 9.4.0.0
-            dockerfile: Dockerfile-jruby-9.4.0.0
+            version: 9.4.7.0
+            dockerfile: Dockerfile-jruby-9.4.7.0
     runs-on: ubuntu-latest
     name: Build (${{ matrix.engine }} ${{ matrix.version }})
     steps:

.github/workflows/test-macos.yaml

@@ -38,7 +38,7 @@ jobs:
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-          bundler: latest # needed to fix issue with steep on Ruby 3.0/3.1
-          cache-version: v2 # bump this to invalidate cache
+          rubygems: 3.3.26
+          bundler: 2.3.26 # needed to fix issue with steep on Ruby 3.0/3.1
+      - run: bundle install
       - run: bundle exec rake spec:main

.gitlab-ci.yml

@@ -108,6 +108,9 @@ build-gem:
   script:
     - export RUBY_PACKAGE_VERSION=$(cat tmp/version.txt)
     - export DATADOG_GEM_LOCATION=$(readlink -f pkg/datadog-*.gem)
+    - ruby -v
+    - gem -v
+    - bundler -v
     - ruby .gitlab/install_datadog_deps.rb
   artifacts:
     paths:
@@ -149,10 +152,59 @@ package-arm64:
     - ../.gitlab/build-deb-rpm.sh
     - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint
 
+package-oci-amd64:
+  extends: .package-oci
+  stage: package
+  needs:
+    - build-gem
+    - install-dependencies-amd64
+  variables:
+    ARCH: amd64
+  script:
+    - cp ../lib-injection/host_inject.rb ../tmp
+    - export RUBY_PACKAGE_VERSION=$(cat ../tmp/version.txt)
+    - ../.gitlab/build-oci-package.sh
+
+package-oci-arm64:
+  extends: .package-oci
+  stage: package
+  needs:
+    - build-gem
+    - install-dependencies-arm64
+  variables:
+    ARCH: arm64
+  script:
+    - cp ../lib-injection/host_inject.rb ../tmp
+    - export RUBY_PACKAGE_VERSION=$(cat ../tmp/version.txt)
+    - ../.gitlab/build-oci-package.sh
+
+oci-internal-publish:
+  extends: .oci-internal-publish
+  stage: package
+  needs: [ package-oci-arm64, package-oci-amd64 ]
+  rules:
+    - when: on_success
+  variables:
+    FLAVOR: datadog-apm-library-ruby
+
+oci-internal-test-ecr-publish:
+  stage: package
+  needs: [ oci-internal-publish ]
+  rules:
+    - when: on_success
+  trigger:
+    project: DataDog/public-images
+    branch: main
+    strategy: depend
+  variables:
+    IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-apm-library-ruby:pipeline-${CI_PIPELINE_ID}-1
+    IMG_DESTINATIONS: apm-library-ruby-package:pipeline-${CI_PIPELINE_ID}
+    IMG_REGISTRIES: agent-qa
+
 onboarding_tests:
   extends: .base_job_onboarding_tests
   stage: integration-tests
-  needs: [ package-amd64, package-arm64]
+  needs: [ package-amd64, package-arm64, oci-internal-test-ecr-publish]
   allow_failure: false
   variables:
     TEST_LIBRARY: ruby
@@ -163,10 +215,13 @@ onboarding_tests:
           SCENARIO: [SIMPLE_HOST_AUTO_INJECTION]
         - ONBOARDING_FILTER_WEBLOG: [test-app-ruby-container]
           SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION]
+        - ONBOARDING_FILTER_WEBLOG: [test-app-ruby,test-app-ruby-container]
+          SCENARIO: [INSTALLER_AUTO_INJECTION]
   script:
     - git clone https://git@github.com/DataDog/system-tests.git system-tests
     - cp packaging/*.rpm system-tests/binaries
     - cp packaging/*.deb system-tests/binaries
+    - export DD_INSTALLER_LIBRARY_VERSION="pipeline-${CI_PIPELINE_ID}"
     - ls system-tests/binaries
     - cd system-tests
     - ./build.sh -i runner
@@ -194,40 +249,42 @@ deploy_to_reliability_env:
     UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA
 
-deploy_to_docker_registries:
+prepare_image_destinations:
   stage: deploy
+  tags: ["arch:amd64"]
+  image: $DOCKER_REGISTRY/images/mirror/ruby:3.2.2
   rules:
-    - if: "$POPULATE_CACHE"
+    - if: '$POPULATE_CACHE'
       when: never
-    - if: $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ # Exclude prerelease
-      when: delayed
-      start_in: 1 day
+    - if:  $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ # Exclude prerelease
+      when: always
     - when: manual
       allow_failure: true
-  trigger:
-    project: DataDog/public-images
-    branch: main
-    strategy: depend
-  variables:
-    IMG_SOURCES: ghcr.io/datadog/dd-trace-rb/dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_DESTINATIONS: dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_SIGNING: "false"
+  script:
+    - ruby -v
+    - gem -v
+    - bundle -v
+    - ./.gitlab/check_gem_presence.rb $CI_COMMIT_TAG
+    - |
+      IMG_DESTINATIONS=$(./.gitlab/prepare_image_destinations.rb dd-lib-ruby-init $CI_COMMIT_TAG) || {
+        echo "Failed to prepare image destinations: $CI_COMMIT_TAG"
+        exit 1
+      }
+      echo "IMG_DESTINATIONS=$IMG_DESTINATIONS" > build.env
+  artifacts:
+    reports:
+      dotenv: build.env
 
-deploy_latest_tag_to_docker_registries:
+deploy_to_docker_registries:
   stage: deploy
-  rules:
-    - if: "$POPULATE_CACHE"
-      when: never
-    - if: $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ && $CI_COMMIT_BRANCH == 'master' # Exclude prerelease and only for master branch
-      when: delayed
-      start_in: 1 day
-    - when: manual
-      allow_failure: true
+  needs:
+    - job: prepare_image_destinations
+      artifacts: true
   trigger:
     project: DataDog/public-images
     branch: main
     strategy: depend
   variables:
     IMG_SOURCES: ghcr.io/datadog/dd-trace-rb/dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_DESTINATIONS: dd-lib-ruby-init:latest
+    IMG_DESTINATIONS: $IMG_DESTINATIONS
     IMG_SIGNING: "false"

.gitlab/Dockerfile-2.7.8

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system 3.3.27;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.0.6

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.1.4

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.2.2

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image, but
+# compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -99,6 +100,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/build-oci-package.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+mkdir sources
+
+cp -r ../tmp/* sources
+sed -i "s#/opt/datadog/apm/library/ruby/#/opt/datadog-packages/datadog-apm-library-ruby/$RUBY_PACKAGE_VERSION/#g" sources/host_inject.rb
+
+echo -n "$RUBY_PACKAGE_VERSION" > sources/version
+
+datadog-package create \
+  --version="$RUBY_PACKAGE_VERSION" \
+  --package="datadog-apm-library-ruby" \
+  --archive=true \
+  --archive-path="datadog-apm-library-ruby-$RUBY_PACKAGE_VERSION-$ARCH.tar" \
+  --arch "$ARCH" \
+  --os "linux" \
+  ./sources

.gitlab/check_gem_presence.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'bundler/inline'
+
+gemfile { gem 'gems', source: 'https://rubygems.org' }
+
+require 'rubygems'
+require 'gems'
+
+version = ARGV[0].chomp
+version = version.delete_prefix('v') if version.start_with?('v')
+
+candidate = Gem::Version.new(version)
+
+retry_count = 0
+max_retries = 60
+interval = 60
+
+loop do
+  versions = Gems.versions('datadog').map { |h| Gem::Version.new(h['number']) }
+
+  if versions.include?(candidate)
+    puts "Gem version #{candidate} found!"
+    exit 0
+  else
+    retry_count += 1
+    puts "Attempt(#{retry_count}):  Gem 'datadog' version '#{candidate}' not found."
+
+    if retry_count >= max_retries
+      puts "Max retries(#{max_retries}) reached, stopping..."
+      exit 1
+    else
+      puts "Retrying in #{interval} seconds..."
+      sleep interval
+    end
+  end
+end

.gitlab/install_datadog_deps.rb

@@ -4,8 +4,22 @@
 require 'fileutils'
 require 'pathname'
 
+puts '=== RUBY_VERSION ==='
+puts RUBY_VERSION
+puts '=== RUBY_ENGINE ==='
+puts RUBY_ENGINE
+puts '=== RUBY_ENGINE_VERSION ==='
+puts RUBY_ENGINE_VERSION
+puts '=== RUBY_PLATFORM ==='
+puts RUBY_PLATFORM
+puts '=== GEM PLATFORM ==='
+puts Gem::Platform.local
+
 ruby_api_version = Gem.ruby_api_version
 
+puts '=== RUBY API VERISON ==='
+puts ruby_api_version
+
 current_path = Pathname.new(FileUtils.pwd)
 
 tmp_path = current_path.join('tmp')
@@ -19,6 +33,7 @@
 File.open(gemfile_file_path, 'w') do |file|
   file.write("source 'https://rubygems.org'\n")
   file.write("gem 'datadog', '#{ENV.fetch('RUBY_PACKAGE_VERSION')}', path: '#{current_path}'\n")
+  file.write("gem 'ffi', '1.16.3'\n")
   # Mimick outdated `msgpack` version, uncomment below line to test
   # file.write("gem 'msgpack', '1.6.0'\n")
 end

.gitlab/onboarding_tests.yml

@@ -10,6 +10,7 @@
     - export DD_APP_KEY_ONBOARDING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.dd-app-key-onboarding --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SUBNET_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.aws-infra-subnet-id --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SECURITY_GROUPS_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.aws-infra-securiy-groups-id --with-decryption --query "Parameter.Value" --out text)
+    - export ONBOARDING_AWS_INFRA_IAM_INSTANCE_PROFILE=ec2InstanceRole
     - export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.pulumi-config-passphrase --with-decryption --query "Parameter.Value" --out text) 
     #Install plugins for PULUMI you need connect to gh. Sometimes this problem arises: GitHub rate limit exceeded
     - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.gh-token --with-decryption --query "Parameter.Value" --out text) 
@@ -30,4 +31,4 @@
   artifacts:
       when: always
       paths:
-        - system-tests/reports/
+        - system-tests/reports/