Merge branch 'master' into master

.github/workflows/appsec.yml

@@ -122,7 +122,7 @@ jobs:
   express:
     runs-on: ubuntu-latest
     env:
-      PLUGINS: express|body-parser|cookie-parser
+      PLUGINS: express|body-parser|cookie-parser|multer
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/node/setup

.github/workflows/llmobs.yml

@@ -0,0 +1,49 @@
+name: LLMObs
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  schedule:
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - uses: ./.github/actions/install
+      - uses: ./.github/actions/node/18
+      - run: yarn test:llmobs:sdk:ci
+      - uses: ./.github/actions/node/20
+      - run: yarn test:llmobs:sdk:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:llmobs:sdk:ci
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+      - uses: codecov/codecov-action@v3
+
+  openai:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: openai
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - uses: ./.github/actions/install
+      - uses: ./.github/actions/node/oldest
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: codecov/codecov-action@v3
+      - if: always()
+        uses: ./.github/actions/testagent/logs

.github/workflows/plugins.yml

@@ -221,6 +221,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  body-parser:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: body-parser
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   bunyan:
     runs-on: ubuntu-latest
     env:
@@ -259,6 +267,14 @@ jobs:
       - run: yarn test:plugins:ci
       - uses: codecov/codecov-action@v2
 
+  cookie-parser:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: cookie-parser
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   couchbase:
     strategy:
       matrix:
@@ -366,7 +382,22 @@ jobs:
   express:
     runs-on: ubuntu-latest
     env:
-      PLUGINS: express|body-parser|cookie-parser
+      PLUGINS: express
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
+  express-mongo-sanitize:
+    runs-on: ubuntu-latest
+    services:
+      mongodb:
+        image: circleci/mongo
+        ports:
+          - 27017:27017
+    env:
+      PLUGINS: express-mongo-sanitize
+      PACKAGE_NAMES: express-mongo-sanitize
+      SERVICES: mongo
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
@@ -548,6 +579,23 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  mariadb:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mariadb:10.4
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: 'db'
+        ports:
+          - 3306:3306
+    env:
+      PLUGINS: mariadb
+      SERVICES: mariadb
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   memcached:
     runs-on: ubuntu-latest
     services:
@@ -641,12 +689,29 @@ jobs:
         ports:
           - 3306:3306
     env:
-      PLUGINS: mysql|mysql2|mariadb # TODO: move mysql2 to its own job
+      PLUGINS: mysql
       SERVICES: mysql
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  mysql2:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mariadb:10.4
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_DATABASE: 'db'
+        ports:
+          - 3306:3306
+    env:
+      PLUGINS: mysql2
+      SERVICES: mysql2
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   net:
     runs-on: ubuntu-latest
     env:
@@ -934,6 +999,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/plugins/test
 
+  url:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: url
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/plugins/test
+
   when:
     runs-on: ubuntu-latest
     env:

.github/workflows/project.yml

@@ -44,6 +44,20 @@ jobs:
       - uses: ./.github/actions/install
       - run: node node_modules/.bin/mocha --colors --timeout 30000 integration-tests/init.spec.js
 
+  integration-guardrails-unsupported:
+    strategy:
+      matrix:
+        version: ['0.8', '0.10', '0.12', '4', '6', '8', '10']
+    runs-on: ubuntu-latest
+    env:
+      DD_INJECTION_ENABLED: 'true'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.version }}
+      - run: node ./init
+
   integration-ci:
     strategy:
       matrix:

.gitignore

@@ -106,6 +106,7 @@ typings/
 
 # End of https://www.gitignore.io/api/node,macos,visualstudiocode
 
+.github/notes
 .next
 package-lock.json
 out

.gitlab-ci.yml

@@ -19,8 +19,16 @@ onboarding_tests_installer:
   parallel:
     matrix:
       - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs,test-app-nodejs-container]
-        SCENARIO: [ INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ]
+        SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ]
 
 onboarding_tests_k8s_injection:
   variables:
     WEBLOG_VARIANT: sample-app
+
+requirements_json_test:
+  rules:
+    - when: on_success
+  variables:
+    REQUIREMENTS_BLOCK_JSON_PATH: ".gitlab/requirements_block.json"
+    REQUIREMENTS_ALLOW_JSON_PATH: ".gitlab/requirements_allow.json"
+

.gitlab/prepare-oci-package.sh

@@ -21,3 +21,5 @@ fi
 echo -n $JS_PACKAGE_VERSION > packaging/sources/version
 
 cd packaging
+
+cp ../requirements.json sources/requirements.json

.gitlab/requirements_allow.json

@@ -0,0 +1,19 @@
+[
+  {"name": "min glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.17"}},
+  {"name": "ok glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.23"}},
+  {"name": "high glibc x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:3.0"}},
+  {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}},
+  {"name": "min glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.17"}},
+  {"name": "ok glibc arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.27"}},
+  {"name": "glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.19"}},
+  {"name": "musl arm","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm", "libc": "musl:1.2.2"}},
+  {"name": "musl arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "musl:1.2.2"}},
+  {"name": "musl x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "musl:1.2.2"}},
+  {"name": "musl x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "musl:1.2.2"}},
+  {"name": "windows x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x64"}},
+  {"name": "windows x86", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "windows", "arch": "x86"}},
+  {"name": "macos x64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "x64"}},
+  {"name": "macos arm64", "filepath": "/some/path", "args": [], "envars": [], "host": {"os": "darwin", "arch": "arm64"}},
+  {"name": "node app", "filepath": "/pathto/node", "args": ["/pathto/node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "ts-node app", "filepath": "/pathto/ts-node", "args": ["/pathto/ts-node", "./app.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}
+]

.gitlab/requirements_block.json

@@ -0,0 +1,11 @@
+[
+  {"name": "unsupported 2.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16"}},
+  {"name": "unsupported 1.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:1.22"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.16.9"}},
+  {"name": "unsupported 2.x glibc arm64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16"}},
+  {"name": "unsupported 2.x.x glibc x64","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "arm64", "libc": "glibc:2.16.9"}},
+  {"name": "unsupported 2.x.x glibc x86","filepath": "/some/path", "args": [], "envars": [], "host": {"os": "linux", "arch": "x86", "libc": "glibc:2.17"}},
+  {"name": "npm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/npm-cli.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "yarn","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/yarn.js"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}},
+  {"name": "pnpm","filepath": "/pathto/node", "args": ["/pathto/node", "/pathto/pnpm.cjs"], "envars": [], "host": {"os": "linux", "arch": "x64", "libc": "glibc:2.40"}}
+]

CODEOWNERS

@@ -53,6 +53,11 @@
 /packages/dd-trace/src/service-naming/ @Datadog/apm-idm-js
 /packages/dd-trace/test/service-naming/ @Datadog/apm-idm-js
 
+/packages/dd-trace/src/llmobs/ @DataDog/ml-observability
+/packages/dd-trace/test/llmobs/ @DataDog/ml-observability
+/packages/datadog-plugin-openai/ @DataDog/ml-observability
+/packages/datadog-instrumentations/src/openai.js @DataDog/ml-observability
+
 # CI
 /.github/workflows/appsec.yml @DataDog/asm-js
 /.github/workflows/ci-visibility-performance.yml @DataDog/ci-app-libraries

CONTRIBUTING.md

@@ -72,12 +72,18 @@ Eventually we plan to look into putting these permission-required tests behind a
 
 ## Development Requirements
 
-Since this project supports multiple Node versions, using a version
-manager such as [nvm](https://github.com/creationix/nvm) is recommended.
+Since this project supports multiple Node.js versions, using a version manager
+such as [nvm](https://github.com/creationix/nvm) is recommended. If you're
+unsure which version of Node.js to use, just use the latest version, which
+should always work.
 
-We use [yarn](https://yarnpkg.com/) for its workspace functionality, so make sure to install that as well.
+We use [yarn](https://yarnpkg.com/) 1.x for its workspace functionality, so make sure to install that as well. The easist way to install yarn 1.x with with npm:
 
-To install dependencies once you have Node and yarn installed, run:
+```sh  
+$ npm install -g yarn
+```
+
+To install dependencies once you have Node and yarn installed, run this in the project directory:
 
 ```sh
 $ yarn
@@ -91,23 +97,42 @@ $ yarn
 The `pg-native` package requires `pg_config` to be in your `$PATH` to be able to install.
 Please refer to [the "Install" section](https://github.com/brianc/node-postgres/tree/master/packages/pg-native#install) of the `pg-native` documentation for how to ensure your environment is configured correctly.
 
-### Setup
-
-Before running _plugin_ tests, the data stores need to be running.
-The easiest way to start all of them is to use the provided
-docker-compose configuration:
+### Plugin Tests
 
-```sh
-$ docker-compose up -d -V --remove-orphans --force-recreate
-$ yarn services
-```
+Before running _plugin_ tests, the supporting docker containers need to be running. You _can_ attempt to start all of them using docker-compose, but that's a drain on your system, and not all the images will even run at all on AMD64 devices.
 
 > **Note**
 > The `aerospike`, `couchbase`, `grpc` and `oracledb` instrumentations rely on
 > native modules that do not compile on ARM64 devices (for example M1/M2 Mac)
 > - their tests cannot be run locally on these devices.
 
-### Unit Tests
+Instead, you can follow this procedure for the plugin you want to run tests for:
+
+1. Check the CI config in `.github/workflows/plugins.yml` to see what the appropriate values for the `SERVICES` and `PLUGINS` environment variables are for the plugin you're trying to test (noting that not all plugins require `SERVICES`). For example, for the `amqplib` plugin, the `SERVICES` value is `rabbitmq`, and the `PLUGINS` value is `amqplib`.
+2. Run the appropriate docker-compose command to start the required services. For example, for the `amqplib` plugin, you would run: `docker compose up -d rabbitmq`.
+3. Run `yarn services`, with the environment variables set above. This will install any versions of the library to be tested against into the `versions` directory, and check that the appropriate services are running prior to running the test.
+4. Now, you can run `yarn test:plugins` with the environment variables set above to run the tests for the plugin you're interested in.
+
+To wrap that all up into a simple few lines of shell commands, here is all of the above, for the `amqplib` plugin:
+
+```sh
+# These are exported for simplicity, but you can also just set them inline.
+export SERVICES="rabbitmq" # retrieved from .github/workflows/plugins.yml
+export PLUGINS="amqplib" # retrieved from .github/workflows/plugins.yml
+
+docker compose up -d $SERVICES
+yarn services
+
+yarn test:plugins # This one actually runs the tests. Can be run many times.
+```
+
+You can also run the tests for multiple plugins at once by separating them with a pipe (`|`) delimiter. For example, to run the tests for the `amqplib` and `bluebird` plugins:
+
+```sh
+PLUGINS="amqplib|bluebird" yarn test:plugins
+```
+
+### Other Unit Tests
 
 There are several types of unit tests, for various types of components. The
 following commands may be useful:
@@ -124,17 +149,6 @@ $ yarn test:instrumentations
 Several other components have test commands as well. See `package.json` for
 details.
 
-To test _plugins_ (i.e. components in `packages/datadog-plugin-XXXX`
-directories, set the `PLUGINS` environment variable to the plugin you're
-interested in, and use `yarn test:plugins`. If you need to test multiple
-plugins you may separate then with a pipe (`|`) delimiter. Here's an
-example testing the `express` and `bluebird` plugins:
-
-```sh
-PLUGINS="express|bluebird" yarn test:plugins
-```
-
-
 ### Linting
 
 We use [ESLint](https://eslint.org) to make sure that new code
@@ -146,6 +160,9 @@ To run the linter, use:
 $ yarn lint
 ```
 
+This also checks that the `LICENSE-3rdparty.csv` file is up-to-date, and checks
+dependencies for vulnerabilities.
+
 
 ### Benchmarks
 

docker-compose.yml

@@ -129,7 +129,7 @@ services:
       - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
       - KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
       - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
-      - KAFKA_CLUSTER_ID=r4zt_wrqTRuT7W2NJsB_GA
+      - CLUSTER_ID=5L6g3nShT-eMCtK--X86sw
       - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
       - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
       - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT