Merge remote-tracking branch 'origin/master' into coverity_fixes

Author: rcsanchez97

.clang-format

@@ -53,6 +53,9 @@ AlwaysBreakBeforeMultilineStrings: false
 AlwaysBreakTemplateDeclarations: MultiLine
 AttributeMacros:
   - __capability
+  - BSON_GNUC_WARN_UNUSED_RESULT
+  - BSON_DEPRECATED
+  - BSON_DEPRECATED_FOR
 BinPackArguments: false
 BinPackParameters: false
 BitFieldColonSpacing: Both
@@ -102,7 +105,12 @@ ForEachMacros:
   - foreach
   - Q_FOREACH
   - BOOST_FOREACH
+  - mlib_foreach_irange
+  - mlib_foreach_urange
+  - mlib_foreach
+  - mlib_foreach_arr
 IfMacros:
+  - mlib_assert_aborts
   - KJ_IF_MAYBE
 IncludeBlocks:   Preserve
 IncludeCategories:

.evergreen/benchmark.yml

@@ -35,7 +35,7 @@ variables:
           set -o errexit
           set -o verbose
 
-          cmake -DENABLE_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=mongoc -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF . && make -j8 && make install
+          cmake -DENABLE_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=mongoc . && make -j8 && make install
           git clone https://github.com/mongodb/mongo-c-driver-performance.git
           cd mongo-c-driver-performance
           cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=../mongoc . && make -j8
@@ -170,10 +170,44 @@ functions:
       file_location: mongo-c-driver/report.json
 
   "send dashboard data" :
-    command: perf.send
-    params:
-      name: perf
-      file: mongo-c-driver/results.json
+    # Here we begin to generate the request to send the data to Signal Processing Service (SPS)
+    - command: shell.exec
+      params:
+        script: |
+          # We use the requester expansion to determine whether the data is from a mainline evergreen run or not
+          if [ "${requester}" == "commit" ]; then
+            echo "is_mainline: true" >> expansion.yml
+          else
+            echo "is_mainline: false" >> expansion.yml
+          fi
+
+          # We parse the username out of the order_id as patches append that in and SPS does not need that information
+          echo "parsed_order_id: $(echo "${revision_order_id}" | awk -F'_' '{print $NF}')"  >> expansion.yml
+    - command: expansions.update
+      params:
+        file: expansion.yml
+    - command: shell.exec
+      params:
+        script: |
+          # Submit the performance data to the SPS endpoint
+          response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
+            "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=${parsed_order_id}&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=${is_mainline}" \
+            -H 'accept: application/json' \
+            -H 'Content-Type: application/json' \
+            -d @mongo-c-driver/results.json)
+
+          http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
+          response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
+
+          # We want to throw an error if the data was not successfully submitted
+          if [ "$http_status" -ne 200 ]; then
+            echo "Error: Received HTTP status $http_status"
+            echo "Response Body: $response_body"
+            exit 1
+          fi
+
+          echo "Response Body: $response_body"
+          echo "HTTP Status: $http_status"
 
   "backtrace":
     - command: shell.exec

.evergreen/config_generator/components/abi_compliance_check.py

@@ -2,45 +2,49 @@
 from shrub.v3.evg_command import s3_put
 from shrub.v3.evg_task import EvgTask
 
+from config_generator.components.funcs.set_cache_dir import SetCacheDir
+
 from config_generator.etc.function import Function
 from config_generator.etc.utils import bash_exec
 
 
 class CheckABICompliance(Function):
     name = 'abi-compliance-check'
-    commands = [
+    commands = SetCacheDir.commands + [
         bash_exec(
             command_type=EvgCommandType.SETUP,
             working_dir='mongoc',
+            include_expansions_in_env=['MONGO_C_DRIVER_CACHE_DIR'],
             script='.evergreen/scripts/abi-compliance-check-setup.sh'
         ),
         bash_exec(
-            command_type=EvgCommandType.SETUP,
+            command_type=EvgCommandType.TEST,
             add_expansions_to_env=True,
             working_dir='mongoc',
+            include_expansions_in_env=['MONGO_C_DRIVER_CACHE_DIR'],
             script='.evergreen/scripts/abi-compliance-check.sh'
         ),
-        bash_exec(
-            command_type=EvgCommandType.TEST,
-            working_dir='mongoc',
-            env={
-                'AWS_ACCESS_KEY_ID': '${aws_key}',
-                'AWS_SECRET_ACCESS_KEY': '${aws_secret}',
-            },
-            script='''\
-                aws s3 cp abi-compliance/compat_reports s3://mciuploads/${project}/${build_variant}/${revision}/${version_id}/${build_id}/abi-compliance/compat_reports --recursive --acl public-read --region us-east-1
-                [[ ! -f ./abi-compliance/abi-error.txt ]]
-            '''
-        ),
         s3_put(
+            command_type=EvgCommandType.SYSTEM,
             aws_key='${aws_key}',
             aws_secret='${aws_secret}',
             bucket='mciuploads',
             content_type='text/html',
-            display_name='ABI Report:',
-            local_files_include_filter='mongoc/abi-compliance/compat_reports/**/*.html',
+            display_name='ABI Compliance Check: ',
+            local_files_include_filter='abi-compliance/compat_reports/**/compat_report.html',
+            permissions='public-read',
+            remote_file='${project}/${branch_name}/${revision}/${version_id}/${build_id}/${task_id}/${execution}/abi-compliance-check/',
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/plain',
+            display_name='ABI Compliance Check: ',
+            local_files_include_filter='abi-compliance/logs/**/log.txt',
             permissions='public-read',
-            remote_file='${project}/${build_variant}/${revision}/${version_id}/${build_id}/abi-compliance/compat_report.html',
+            remote_file='${project}/${branch_name}/${revision}/${version_id}/${build_id}/${task_id}/${execution}/abi-compliance-check/',
         ),
     ]
 

.evergreen/config_generator/components/c_std_compile.py

@@ -6,7 +6,7 @@
 
 from config_generator.etc.distros import find_large_distro
 from config_generator.etc.distros import make_distro_str
-from config_generator.etc.distros import to_cc
+from config_generator.etc.distros import compiler_to_vars
 from config_generator.etc.function import Function
 from config_generator.etc.utils import bash_exec
 
@@ -17,14 +17,32 @@
 # pylint: disable=line-too-long
 # fmt: off
 MATRIX = [
-    ('debian92',          'clang',     None,   [11,   ]),
-    ('debian10',          'clang',     None,   [11,   ]),
-    ('debian10',          'gcc',       None,   [11, 17]),
-    ('debian11',          'clang',     None,   [11,   ]),
-    ('debian11',          'gcc',       None,   [11, 17]),
-    ('ubuntu2004',        'clang',     None,   [11,   ]),
-    ('ubuntu2004',        'gcc',       None,   [11,   ]),
-    ('windows-vsCurrent', 'vs2017x64', None,   [11, 17]),
+    ('rhel80',     'clang',    None, [99, 11, 17,   ]), # Clang 7.0
+    ('ubuntu2004', 'clang-10', None, [99, 11, 17, 23]), # Clang 10.0 (max: C2x)
+    ('rhel84',     'clang',    None, [99, 11, 17, 23]), # Clang 11.0 (max: C2x)
+    ('ubuntu2204', 'clang-12', None, [99, 11, 17, 23]), # Clang 12.0 (max: C2x)
+    ('rhel90',     'clang',    None, [99, 11, 17, 23]), # Clang 13.0 (max: C2x)
+    ('rhel91',     'clang',    None, [99, 11, 17, 23]), # Clang 14.0 (max: C2x)
+    ('rhel92',     'clang',    None, [99, 11, 17, 23]), # Clang 15.0 (max: C2x)
+    ('rhel93',     'clang',    None, [99, 11, 17, 23]), # Clang 16.0 (max: C2x)
+    ('rhel94',     'clang',    None, [99, 11, 17, 23]), # Clang 17.0 (max: C2x)
+    ('rhel95',     'clang',    None, [99, 11, 17, 23]), # Clang 18.0 (max: C23)
+
+    ('rhel76',     'gcc',    None, [99, 11,       ]), # GCC 4.8 (max: C11)
+    ('rhel80',     'gcc',    None, [99, 11, 17,   ]), # GCC 8.2 (max: C17)
+    ('debian10',   'gcc-8',  None, [99, 11, 17,   ]), # GCC 8.3 (max: C17)
+    ('rhel84',     'gcc',    None, [99, 11, 17,   ]), # GCC 8.4 (max: C17)
+    ('ubuntu2004', 'gcc-9',  None, [99, 11, 17, 23]), # GCC 9.4 (max: C2x)
+    ('debian11',   'gcc-10', None, [99, 11, 17, 23]), # GCC 10.2 (max: C2x)
+    ('rhel90',     'gcc',    None, [99, 11, 17, 23]), # GCC 11.2 (max: C2x)
+    ('rhel92',     'gcc',    None, [99, 11, 17, 23]), # GCC 11.3 (max: C2x)
+    ('rhel94',     'gcc',    None, [99, 11, 17, 23]), # GCC 11.4 (max: C2x)
+    ('rhel95',     'gcc',    None, [99, 11, 17, 23]), # GCC 11.5 (max: C2x)
+
+    ('windows-vsCurrent', 'vs2015x64', None, [99, 11,   ]), # Max: C11
+    ('windows-vsCurrent', 'vs2017x64', None, [99, 11,   ]), # Max: C11
+    ('windows-vsCurrent', 'vs2019x64', None, [99, 11, 17]), # Max: C17
+    ('windows-vsCurrent', 'vs2022x64', None, [99, 11, 17]), # Max: C17
 ]
 # fmt: on
 # pylint: enable=line-too-long
@@ -54,18 +72,20 @@ def tasks():
     res = []
 
     for distro_name, compiler, arch, stds in MATRIX:
-        tags = [TAG, distro_name, compiler, 'compile']
+        compiler_type = compiler.split('-')[0]
+
+        tags = [TAG, distro_name, compiler_type, 'compile']
 
         distro = find_large_distro(distro_name)
 
         compile_vars = None
-        compile_vars = {'CC': to_cc(compiler)}
+        compile_vars = compiler_to_vars(compiler)
 
         if arch:
             tags.append(arch)
             compile_vars.update({'MARCH': arch})
 
-        distro_str = make_distro_str(distro_name, compiler, arch)
+        distro_str = make_distro_str(distro_name, compiler_type, arch)
 
         for std in stds:
             with_std = {'C_STD_VERSION': std}

.evergreen/config_generator/components/cse/winssl.py

@@ -14,7 +14,6 @@
 # pylint: disable=line-too-long
 # fmt: off
 COMPILE_MATRIX = [
-    ('windows-64-vs2015', 'vs2015x64', None, ['cyrus']),
     ('windows-vsCurrent', 'vs2017x64', None, ['cyrus']),
 ]
 

.evergreen/config_generator/components/earthly.py

@@ -27,7 +27,7 @@
     "alpine3.18",
     "alpine3.19",
     "archlinux",
-    "centos7"
+    "centos7",
 ]
 "Identifiers for environments. These correspond to special 'env.*' targets in the Earthfile."
 CompilerName = Literal["gcc", "clang"]
@@ -36,13 +36,13 @@
 # Other options: SSPI (Windows only), AUTO (not reliably test-able without more environments)
 SASLOption = Literal["Cyrus", "off"]
 "Valid options for the SASL configuration parameter"
-TLSOption = Literal["LibreSSL", "OpenSSL", "off"]
+TLSOption = Literal["OpenSSL", "off"]
 "Options for the TLS backend configuration parameter (AKA 'ENABLE_SSL')"
-CxxVersion = Literal["r3.8.0", "r3.9.0", "none"]
+CxxVersion = Literal["none"] # TODO: Once CXX-3103 is released, add latest C++ release tag.
 "C++ driver refs that are under CI test"
 
 # A separator character, since we cannot use whitespace
-_SEPARATOR = "\N{no-break space}\N{bullet}\N{no-break space}"
+_SEPARATOR = "\N{NO-BREAK SPACE}\N{BULLET}\N{NO-BREAK SPACE}"
 
 
 def os_split(env: EnvKey) -> tuple[str, None | str]:
@@ -59,9 +59,7 @@ def os_split(env: EnvKey) -> tuple[str, None | str]:
         case "centos7":
             return "CentOS", "7.0"
         case _:
-            raise ValueError(
-                f"Failed to split OS env key {env=} into a name+version pair (unrecognized)"
-            )
+            raise ValueError(f"Failed to split OS env key {env=} into a name+version pair (unrecognized)")
 
 
 class EarthlyVariant(NamedTuple):
@@ -135,7 +133,7 @@ class Configuration(NamedTuple):
 
     @property
     def suffix(self) -> str:
-        return f"{_SEPARATOR}".join(f"{k}={v}" for k, v in self._asdict().items())
+        return _SEPARATOR.join(f"{k}={v}" for k, v in self._asdict().items())
 
 
 def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
@@ -144,27 +142,10 @@ def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     configuration values.
     """
     match env, conf:
-        # We only need one task with "sasl=off"
-        case ["u22", "gcc"], ("off", "OpenSSL", "r3.8.0"):
-            return True
-        # The Ubuntu 18.04 GCC has a bug that fails to build the 3.8.0 C++ driver
-        case ["u18", "gcc"], [_, _, "r3.8.0"]:
-            return False
-        # Other sasl=off tasks we'll just ignore:
-        case _, ("off", _tls, _cxx):
-            return False
-        # Ubuntu and CentOS do not ship with a LibreSSL package:
-        case e, (_sasl, "LibreSSL", _cxx) if e.display_name.startswith("Ubuntu") or e.display_name.startswith("CentOS"):
-            return False
-        # u16 is not capable of building mongocxx
-        case e, (_, _, "none") if e.display_name.startswith("Ubuntu 16") or e.display_name.startswith("CentOS 7"):
-            return True
-         # Exclude u16 for all other configurations
-        case e, _ if e.display_name.startswith("Ubuntu 16") or e.display_name.startswith("CentOS 7"):
-            return False
-        # Exclude all other envs
-        case _, (_, _, "none"):
-            return False
+        # u16/centos7 are not capable of building mongocxx
+        case e, (_sasl, _tls, cxx) if re.match(r"^Ubuntu 16|^CentOS 7", e.display_name):
+            # Only build if C++ driver is test is disabled
+            return cxx == "none"
         # Anything else: Allow it to run:
         case _:
             return True
@@ -230,11 +211,11 @@ def earthly_task(
         commands=[
             # Ensure subsequent Docker commands are authenticated.
             subprocess_exec(
-                binary='bash',
-                command_type="setup",
+                binary="bash",
+                command_type=EvgCommandType.SETUP,
                 args=[
                     "-c",
-                    'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"'
+                    r'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"',
                 ],
             ),
             # First, just build the "env-warmup" which will prepare the build environment.
@@ -253,8 +234,8 @@ def earthly_task(
                 # The "targets" arg is for +run to specify which targets to run
                 args={"targets": " ".join(targets)} | earthly_args,
             ),
-        ],
-        tags=[f"earthly", "pr-merge-gate", *env_tags],
+        ],  # type: ignore (The type annots on `commands` is wrong)
+        tags=["earthly", "pr-merge-gate", *env_tags],
         run_on=CONTAINER_RUN_DISTROS,
     )
 

.evergreen/config_generator/components/funcs/fetch_source.py

@@ -18,14 +18,7 @@ class FetchSource(Function):
                 set -o errexit
                 set -o pipefail
                 if [ -n "${github_pr_number}" -o "${is_patch}" = "true" ]; then
-                    # This is a GitHub PR or patch build, probably branched from master
-                    if command -v python3 &>/dev/null; then
-                        # Prefer python3 if it is available
-                        echo $(python3 ./build/calc_release_version.py --next-minor) > VERSION_CURRENT
-                    else
-                        echo $(python ./build/calc_release_version.py --next-minor) > VERSION_CURRENT
-                    fi
-                    VERSION=$VERSION_CURRENT-${version_id}
+                    VERSION=patch-${version_id}
                 else
                     VERSION=latest
                 fi