Merge branch 'master' into pr-3661

* master: (34 commits)
  Add hanami app to integration CI configurations (DataDog#3639)
  Update UpgradeGuide in README.md
  Update lockfiles for release 2.0.0
  Bump version 2.0.0.rc1 to 2.0.0
  Add 2.0.0 to CHANGELOG.md
  Add missing Apache 2.0 license from gemspec
  Use ffi 1.16.3
  Add platform
  Output gem version
  Output debug logs
  Fix spec to exclude .vscode directory
  Fix docker run command
  Add vscode settings for dockerfile file associations
  Update gem version
  Update gemfiles/*
  Update `ffi` in Gemfile
  Remove experimental warning from OpenTelemetry.md
  Remove latest tag on docker images
  add installer system tests
  Extract trace id format helper method (DataDog#3670)
  ...

.circleci/config.yml

@@ -531,12 +531,12 @@ workflows:
           <<: *filters_all_branches_and_tags
       - orb/build_and_test_integration:
           name: build_and_test_integration-2.6
-          integration_apps: 'rack rails-five rails-six sinatra2-classic sinatra2-modular'
+          integration_apps: 'rack rails-five rails-six sinatra2-classic sinatra2-modular hanami'
           ruby_version: '2.6'
           <<: *filters_all_branches_and_tags
       - orb/build_and_test_integration:
           name: build_and_test_integration-2.7
-          integration_apps: 'rack rails-five rails-six rails-seven sinatra2-classic sinatra2-modular'
+          integration_apps: 'rack rails-five rails-six rails-seven sinatra2-classic sinatra2-modular hanami'
           ruby_version: '2.7'
           <<: *filters_all_branches_and_tags
       - orb/build_and_test_integration:

.dockerignore

@@ -0,0 +1,8 @@
+# Integration tests that run in docker should not receive the gemfile lock
+# files that aren't checked into git, because these lock files are dependent
+# on the system set of installed software which is different between the
+# host and the docker container.
+#
+# Gemfile lock files that *are* checked into git should be in docker
+# containers also.
+Gemfile.lock

.github/labeler.yml

@@ -43,3 +43,15 @@ dev/ci:
 # Version bump pull request
 release:
   - all: [ '{CHANGELOG.md,lib/datadog/version.rb}' ]
+
+# Changes to OpenTelemetry
+otel:
+  - any: [ 'lib/datadog/opentelemetry/**' ]
+
+# Changes to Single Step Instrumentation
+single-step:
+  - any: [ 'lib-injection/**' ]
+
+# Changes to Debugging
+debugging:
+  - any: [ 'lib/datadog/debugging/**' ]

.gitlab-ci.yml

@@ -108,6 +108,9 @@ build-gem:
   script:
     - export RUBY_PACKAGE_VERSION=$(cat tmp/version.txt)
     - export DATADOG_GEM_LOCATION=$(readlink -f pkg/datadog-*.gem)
+    - ruby -v
+    - gem -v
+    - bundler -v
     - ruby .gitlab/install_datadog_deps.rb
   artifacts:
     paths:
@@ -149,10 +152,59 @@ package-arm64:
     - ../.gitlab/build-deb-rpm.sh
     - find . -iregex '.*\.\(deb\|rpm\)' -printf '%f\0' | xargs -0 dd-pkg lint
 
+package-oci-amd64:
+  extends: .package-oci
+  stage: package
+  needs:
+    - build-gem
+    - install-dependencies-amd64
+  variables:
+    ARCH: amd64
+  script:
+    - cp ../lib-injection/host_inject.rb ../tmp
+    - export RUBY_PACKAGE_VERSION=$(cat ../tmp/version.txt)
+    - ../.gitlab/build-oci-package.sh
+
+package-oci-arm64:
+  extends: .package-oci
+  stage: package
+  needs:
+    - build-gem
+    - install-dependencies-arm64
+  variables:
+    ARCH: arm64
+  script:
+    - cp ../lib-injection/host_inject.rb ../tmp
+    - export RUBY_PACKAGE_VERSION=$(cat ../tmp/version.txt)
+    - ../.gitlab/build-oci-package.sh
+
+oci-internal-publish:
+  extends: .oci-internal-publish
+  stage: package
+  needs: [ package-oci-arm64, package-oci-amd64 ]
+  rules:
+    - when: on_success
+  variables:
+    FLAVOR: datadog-apm-library-ruby
+
+oci-internal-test-ecr-publish:
+  stage: package
+  needs: [ oci-internal-publish ]
+  rules:
+    - when: on_success
+  trigger:
+    project: DataDog/public-images
+    branch: main
+    strategy: depend
+  variables:
+    IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-apm-library-ruby:pipeline-${CI_PIPELINE_ID}-1
+    IMG_DESTINATIONS: apm-library-ruby-package:pipeline-${CI_PIPELINE_ID}
+    IMG_REGISTRIES: agent-qa
+
 onboarding_tests:
   extends: .base_job_onboarding_tests
   stage: integration-tests
-  needs: [ package-amd64, package-arm64]
+  needs: [ package-amd64, package-arm64, oci-internal-test-ecr-publish]
   allow_failure: false
   variables:
     TEST_LIBRARY: ruby
@@ -163,10 +215,13 @@ onboarding_tests:
           SCENARIO: [SIMPLE_HOST_AUTO_INJECTION]
         - ONBOARDING_FILTER_WEBLOG: [test-app-ruby-container]
           SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION]
+        - ONBOARDING_FILTER_WEBLOG: [test-app-ruby,test-app-ruby-container]
+          SCENARIO: [INSTALLER_AUTO_INJECTION]
   script:
     - git clone https://git@github.com/DataDog/system-tests.git system-tests
     - cp packaging/*.rpm system-tests/binaries
     - cp packaging/*.deb system-tests/binaries
+    - export DD_INSTALLER_LIBRARY_VERSION="pipeline-${CI_PIPELINE_ID}"
     - ls system-tests/binaries
     - cd system-tests
     - ./build.sh -i runner
@@ -194,40 +249,42 @@ deploy_to_reliability_env:
     UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA
 
-deploy_to_docker_registries:
+prepare_image_destinations:
   stage: deploy
+  tags: ["arch:amd64"]
+  image: $DOCKER_REGISTRY/images/mirror/ruby:3.2.2
   rules:
-    - if: "$POPULATE_CACHE"
+    - if: '$POPULATE_CACHE'
       when: never
-    - if: $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ # Exclude prerelease
-      when: delayed
-      start_in: 1 day
+    - if:  $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ # Exclude prerelease
+      when: always
     - when: manual
       allow_failure: true
-  trigger:
-    project: DataDog/public-images
-    branch: main
-    strategy: depend
-  variables:
-    IMG_SOURCES: ghcr.io/datadog/dd-trace-rb/dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_DESTINATIONS: dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_SIGNING: "false"
+  script:
+    - ruby -v
+    - gem -v
+    - bundle -v
+    - ./.gitlab/check_gem_presence.rb $CI_COMMIT_TAG
+    - |
+      IMG_DESTINATIONS=$(./.gitlab/prepare_image_destinations.rb dd-lib-ruby-init $CI_COMMIT_TAG) || {
+        echo "Failed to prepare image destinations: $CI_COMMIT_TAG"
+        exit 1
+      }
+      echo "IMG_DESTINATIONS=$IMG_DESTINATIONS" > build.env
+  artifacts:
+    reports:
+      dotenv: build.env
 
-deploy_latest_tag_to_docker_registries:
+deploy_to_docker_registries:
   stage: deploy
-  rules:
-    - if: "$POPULATE_CACHE"
-      when: never
-    - if: $CI_COMMIT_TAG =~ /^v(\d+)\.(\d+)\.(\d+)$/ && $CI_COMMIT_BRANCH == 'master' # Exclude prerelease and only for master branch
-      when: delayed
-      start_in: 1 day
-    - when: manual
-      allow_failure: true
+  needs:
+    - job: prepare_image_destinations
+      artifacts: true
   trigger:
     project: DataDog/public-images
     branch: main
     strategy: depend
   variables:
     IMG_SOURCES: ghcr.io/datadog/dd-trace-rb/dd-lib-ruby-init:$CI_COMMIT_TAG
-    IMG_DESTINATIONS: dd-lib-ruby-init:latest
+    IMG_DESTINATIONS: $IMG_DESTINATIONS
     IMG_SIGNING: "false"

.gitlab/Dockerfile-2.7.8

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system 3.3.27;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.0.6

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.1.4

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image
+# but compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -75,6 +76,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/Dockerfile-3.2.2

@@ -1,4 +1,5 @@
-# This is copied from official Docker image, but compile Ruby with `--disable-shared` instead.
+# This is copied from official Docker image, but
+# compile Ruby with `--disable-shared` and update gem version
 
 FROM registry.ddbuild.io/images/mirror/buildpack-deps:buster
 
@@ -99,6 +100,8 @@ RUN set -eux; \
 # verify we have no "ruby" packages installed
 	if dpkg -l | grep -i ruby; then exit 1; fi; \
 	[ "$(command -v ruby)" = '/usr/local/bin/ruby' ]; \
+# update gem version
+	gem update --system;\
 # rough smoke test
 	ruby --version; \
 	gem --version; \

.gitlab/build-oci-package.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+mkdir sources
+
+cp -r ../tmp/* sources
+sed -i "s#/opt/datadog/apm/library/ruby/#/opt/datadog-packages/datadog-apm-library-ruby/$RUBY_PACKAGE_VERSION/#g" sources/host_inject.rb
+
+echo -n "$RUBY_PACKAGE_VERSION" > sources/version
+
+datadog-package create \
+  --version="$RUBY_PACKAGE_VERSION" \
+  --package="datadog-apm-library-ruby" \
+  --archive=true \
+  --archive-path="datadog-apm-library-ruby-$RUBY_PACKAGE_VERSION-$ARCH.tar" \
+  --arch "$ARCH" \
+  --os "linux" \
+  ./sources

.gitlab/check_gem_presence.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'bundler/inline'
+
+gemfile { gem 'gems', source: 'https://rubygems.org' }
+
+require 'rubygems'
+require 'gems'
+
+version = ARGV[0].chomp
+version = version.delete_prefix('v') if version.start_with?('v')
+
+candidate = Gem::Version.new(version)
+
+retry_count = 0
+max_retries = 60
+interval = 60
+
+loop do
+  versions = Gems.versions('datadog').map { |h| Gem::Version.new(h['number']) }
+
+  if versions.include?(candidate)
+    puts "Gem version #{candidate} found!"
+    exit 0
+  else
+    retry_count += 1
+    puts "Attempt(#{retry_count}):  Gem 'datadog' version '#{candidate}' not found."
+
+    if retry_count >= max_retries
+      puts "Max retries(#{max_retries}) reached, stopping..."
+      exit 1
+    else
+      puts "Retrying in #{interval} seconds..."
+      sleep interval
+    end
+  end
+end

.gitlab/install_datadog_deps.rb

@@ -4,8 +4,22 @@
 require 'fileutils'
 require 'pathname'
 
+puts '=== RUBY_VERSION ==='
+puts RUBY_VERSION
+puts '=== RUBY_ENGINE ==='
+puts RUBY_ENGINE
+puts '=== RUBY_ENGINE_VERSION ==='
+puts RUBY_ENGINE_VERSION
+puts '=== RUBY_PLATFORM ==='
+puts RUBY_PLATFORM
+puts '=== GEM PLATFORM ==='
+puts Gem::Platform.local
+
 ruby_api_version = Gem.ruby_api_version
 
+puts '=== RUBY API VERISON ==='
+puts ruby_api_version
+
 current_path = Pathname.new(FileUtils.pwd)
 
 tmp_path = current_path.join('tmp')
@@ -19,6 +33,7 @@
 File.open(gemfile_file_path, 'w') do |file|
   file.write("source 'https://rubygems.org'\n")
   file.write("gem 'datadog', '#{ENV.fetch('RUBY_PACKAGE_VERSION')}', path: '#{current_path}'\n")
+  file.write("gem 'ffi', '1.16.3'\n")
   # Mimick outdated `msgpack` version, uncomment below line to test
   # file.write("gem 'msgpack', '1.6.0'\n")
 end

.gitlab/onboarding_tests.yml

@@ -10,6 +10,7 @@
     - export DD_APP_KEY_ONBOARDING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.dd-app-key-onboarding --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SUBNET_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.aws-infra-subnet-id --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SECURITY_GROUPS_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.aws-infra-securiy-groups-id --with-decryption --query "Parameter.Value" --out text)
+    - export ONBOARDING_AWS_INFRA_IAM_INSTANCE_PROFILE=ec2InstanceRole
     - export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.pulumi-config-passphrase --with-decryption --query "Parameter.Value" --out text) 
     #Install plugins for PULUMI you need connect to gh. Sometimes this problem arises: GitHub rate limit exceeded
     - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-rb.gh-token --with-decryption --query "Parameter.Value" --out text) 
@@ -30,4 +31,4 @@
   artifacts:
       when: always
       paths:
-        - system-tests/reports/
+        - system-tests/reports/