[PROF-10241] Extract libdatadog crashtracker telemetry into separate extension

Rakefile

@@ -383,6 +383,10 @@ NATIVE_EXTS = [
 
   Rake::ExtensionTask.new("datadog_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
     ext.ext_dir = 'ext/datadog_profiling_loader'
+  end,
+
+  Rake::ExtensionTask.new("libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}") do |ext|
+    ext.ext_dir = 'ext/libdatadog_api'
   end
 ].freeze
 

datadog.gemspec

@@ -69,10 +69,13 @@ Gem::Specification.new do |spec|
   # Used by appsec
   spec.add_dependency 'libddwaf', '~> 1.14.0.0.0'
 
-  # Used by profiling (and possibly others in the future)
-  # When updating the version here, please also update the version in `native_extension_helpers.rb`
+  # When updating the version here, please also update the version in `libdatadog_extconf_helpers.rb`
   # (and yes we have a test for it)
   spec.add_dependency 'libdatadog', '~> 11.0.0.1.0'
 
-  spec.extensions = ['ext/datadog_profiling_native_extension/extconf.rb', 'ext/datadog_profiling_loader/extconf.rb']
+  spec.extensions = [
+    'ext/datadog_profiling_native_extension/extconf.rb',
+    'ext/datadog_profiling_loader/extconf.rb',
+    'ext/libdatadog_api/extconf.rb'
+  ]
 end

ext/datadog_profiling_native_extension/datadog_ruby_common.c

@@ -0,0 +1,110 @@
+#include "datadog_ruby_common.h"
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name) {
+  rb_exc_raise(
+    rb_exc_new_str(
+      rb_eTypeError,
+      rb_sprintf("wrong argument %"PRIsVALUE" for '%s' (expected a %s) at %s:%d:in `%s'",
+        rb_inspect(value),
+        value_name,
+        type_name,
+        file,
+        line,
+        function_name
+      )
+    )
+  );
+}
+
+VALUE datadog_gem_version(void) {
+  VALUE ddtrace_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  ENFORCE_TYPE(ddtrace_module, T_MODULE);
+  VALUE version_module = rb_const_get(ddtrace_module, rb_intern("VERSION"));
+  ENFORCE_TYPE(version_module, T_MODULE);
+  VALUE version_string = rb_const_get(version_module, rb_intern("STRING"));
+  ENFORCE_TYPE(version_string, T_STRING);
+  return version_string;
+}
+
+__attribute__((warn_unused_result))
+ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
+  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
+
+  VALUE exporter_working_mode = rb_ary_entry(exporter_configuration, 0);
+  ENFORCE_TYPE(exporter_working_mode, T_SYMBOL);
+  ID working_mode = SYM2ID(exporter_working_mode);
+
+  ID agentless_id = rb_intern("agentless");
+  ID agent_id = rb_intern("agent");
+
+  if (working_mode != agentless_id && working_mode != agent_id) {
+    rb_raise(rb_eArgError, "Failed to initialize transport: Unexpected working mode, expected :agentless or :agent");
+  }
+
+  if (working_mode == agentless_id) {
+    VALUE site = rb_ary_entry(exporter_configuration, 1);
+    VALUE api_key = rb_ary_entry(exporter_configuration, 2);
+
+    return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
+  } else { // agent_id
+    VALUE base_url = rb_ary_entry(exporter_configuration, 1);
+
+    return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
+  }
+}
+
+static VALUE log_failure_to_process_tag(VALUE err_details) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE logger = rb_funcall(datadog_module, rb_intern("logger"), 0);
+
+  return rb_funcall(logger, rb_intern("warn"), 1, rb_sprintf("Failed to convert tag: %"PRIsVALUE, err_details));
+}
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
+  ENFORCE_TYPE(tags_as_array, T_ARRAY);
+
+  long tags_count = RARRAY_LEN(tags_as_array);
+  ddog_Vec_Tag tags = ddog_Vec_Tag_new();
+
+  for (long i = 0; i < tags_count; i++) {
+    VALUE name_value_pair = rb_ary_entry(tags_as_array, i);
+
+    if (!RB_TYPE_P(name_value_pair, T_ARRAY)) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(name_value_pair, T_ARRAY);
+    }
+
+    // Note: We can index the array without checking its size first because rb_ary_entry returns Qnil if out of bounds
+    VALUE tag_name = rb_ary_entry(name_value_pair, 0);
+    VALUE tag_value = rb_ary_entry(name_value_pair, 1);
+
+    if (!(RB_TYPE_P(tag_name, T_STRING) && RB_TYPE_P(tag_value, T_STRING))) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(tag_name, T_STRING);
+      ENFORCE_TYPE(tag_value, T_STRING);
+    }
+
+    ddog_Vec_Tag_PushResult push_result =
+      ddog_Vec_Tag_push(&tags, char_slice_from_ruby_string(tag_name), char_slice_from_ruby_string(tag_value));
+
+    if (push_result.tag == DDOG_VEC_TAG_PUSH_RESULT_ERR) {
+      // libdatadog validates tags and may catch invalid tags that ddtrace didn't actually catch.
+      // We warn users about such tags, and then just ignore them.
+
+      int exception_state;
+      rb_protect(log_failure_to_process_tag, get_error_details_and_drop(&push_result.err), &exception_state);
+
+      // Since we are calling into Ruby code, it may raise an exception. Ensure that dynamically-allocated tags
+      // get cleaned before propagating the exception.
+      if (exception_state) {
+        ddog_Vec_Tag_drop(tags);
+        rb_jump_tag(exception_state);  // "Re-raise" exception
+      }
+    }
+  }
+
+  return tags;
+}

ext/datadog_profiling_native_extension/datadog_ruby_common.h

@@ -0,0 +1,57 @@
+#pragma once
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+#include <ruby.h>
+#include <datadog/profiling.h>
+
+// Used to mark symbols to be exported to the outside of the extension.
+// Consider very carefully before tagging a function with this.
+#define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
+
+// Used to mark function arguments that are deliberately left unused
+#ifdef __GNUC__
+  #define DDTRACE_UNUSED  __attribute__((unused))
+#else
+  #define DDTRACE_UNUSED
+#endif
+
+#define ADD_QUOTES_HELPER(x) #x
+#define ADD_QUOTES(x) ADD_QUOTES_HELPER(x)
+
+// Ruby has a Check_Type(value, type) that is roughly equivalent to this BUT Ruby's version is rather cryptic when it fails
+// e.g. "wrong argument type nil (expected String)". This is a replacement that prints more information to help debugging.
+#define ENFORCE_TYPE(value, type) \
+  { if (RB_UNLIKELY(!RB_TYPE_P(value, type))) raise_unexpected_type(value, ADD_QUOTES(value), ADD_QUOTES(type), __FILE__, __LINE__, __func__); }
+
+#define ENFORCE_BOOLEAN(value) \
+  { if (RB_UNLIKELY(value != Qtrue && value != Qfalse)) raise_unexpected_type(value, ADD_QUOTES(value), "true or false", __FILE__, __LINE__, __func__); }
+
+// Called by ENFORCE_TYPE; should not be used directly
+NORETURN(void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name));
+
+// Helper to retrieve Datadog::VERSION::STRING
+VALUE datadog_gem_version(void);
+
+inline static ddog_CharSlice char_slice_from_ruby_string(VALUE string) {
+  ENFORCE_TYPE(string, T_STRING);
+  ddog_CharSlice char_slice = {.ptr = StringValuePtr(string), .len = RSTRING_LEN(string)};
+  return char_slice;
+}
+
+__attribute__((warn_unused_result))
+ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration);
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array);
+
+inline static VALUE ruby_string_from_error(const ddog_Error *error) {
+  ddog_CharSlice char_slice = ddog_Error_message(error);
+  return rb_str_new(char_slice.ptr, char_slice.len);
+}
+
+inline static VALUE get_error_details_and_drop(ddog_Error *error) {
+  VALUE result = ruby_string_from_error(error);
+  ddog_Error_drop(error);
+  return result;
+}

ext/datadog_profiling_native_extension/extconf.rb

@@ -2,6 +2,7 @@
 # rubocop:disable Style/GlobalVars
 
 require_relative 'native_extension_helpers'
+require_relative '../libdatadog_extconf_helpers'
 
 SKIPPED_REASON_FILE = "#{__dir__}/skipped_reason.txt".freeze
 # Not a problem if the file doesn't exist or we can't delete it
@@ -114,7 +115,7 @@ def add_compiler_flag(flag)
 add_compiler_flag '-Wall'
 add_compiler_flag '-Wextra'
 
-if ENV['DDTRACE_DEBUG']
+if ENV['DDTRACE_DEBUG'] == 'true'
   $defs << '-DDD_DEBUG'
   CONFIG['optflags'] = '-O0'
   CONFIG['debugflags'] = '-ggdb3'
@@ -201,7 +202,7 @@ def add_compiler_flag(flag)
   Logging.message("[datadog] Ruby detected the pkg-config command is #{$PKGCONFIG.inspect}\n")
 
   skip_building_extension!(
-    if Datadog::Profiling::NativeExtensionHelpers::Supported.pkg_config_missing?
+    if Datadog::LibdatadogExtconfHelpers.pkg_config_missing?
       Datadog::Profiling::NativeExtensionHelpers::Supported::PKG_CONFIG_IS_MISSING
     else
       # Less specific error message
@@ -218,8 +219,8 @@ def add_compiler_flag(flag)
 # The extremely excessive escaping around ORIGIN below seems to be correct and was determined after a lot of
 # experimentation. We need to get these special characters across a lot of tools untouched...
 extra_relative_rpaths = [
-  Datadog::Profiling::NativeExtensionHelpers.libdatadog_folder_relative_to_native_lib_folder,
-  *Datadog::Profiling::NativeExtensionHelpers.libdatadog_folder_relative_to_ruby_extensions_folders,
+  Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_native_lib_folder(current_folder: __dir__),
+  *Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_ruby_extensions_folders,
 ]
 extra_relative_rpaths.each { |folder| $LDFLAGS += " -Wl,-rpath,$$$\\\\{ORIGIN\\}/#{folder.to_str}" }
 Logging.message("[datadog] After pkg-config $LDFLAGS were set to: #{$LDFLAGS.inspect}\n")

ext/datadog_profiling_native_extension/helpers.h

@@ -2,17 +2,6 @@
 
 #include <stdint.h>
 
-// Used to mark symbols to be exported to the outside of the extension.
-// Consider very carefully before tagging a function with this.
-#define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
-
-// Used to mark function arguments that are deliberately left unused
-#ifdef __GNUC__
-  #define DDTRACE_UNUSED  __attribute__((unused))
-#else
-  #define DDTRACE_UNUSED
-#endif
-
 // @ivoanjo: After trying to read through https://stackoverflow.com/questions/3437404/min-and-max-in-c I decided I
 // don't like C and I just implemented this as a function.
 inline static uint64_t uint64_max_of(uint64_t a, uint64_t b) { return a > b ? a : b; }

ext/datadog_profiling_native_extension/http_transport.c

@@ -53,7 +53,7 @@ void http_transport_init(VALUE profiling_module) {
   ok_symbol = ID2SYM(rb_intern_const("ok"));
   error_symbol = ID2SYM(rb_intern_const("error"));
 
-  library_version_string = ddtrace_version();
+  library_version_string = datadog_gem_version();
   rb_global_variable(&library_version_string);
 }
 

ext/datadog_profiling_native_extension/libdatadog_helpers.c

@@ -2,8 +2,6 @@
 
 #include <ruby.h>
 
-static VALUE log_failure_to_process_tag(VALUE err_details);
-
 const char *ruby_value_type_to_string(enum ruby_value_type type) {
   return ruby_value_type_to_char_slice(type).ptr;
 }
@@ -62,87 +60,3 @@ size_t read_ddogerr_string_and_drop(ddog_Error *error, char *string, size_t capa
   ddog_Error_drop(error);
   return error_msg_size;
 }
-
-__attribute__((warn_unused_result))
-ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
-  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
-
-  VALUE exporter_working_mode = rb_ary_entry(exporter_configuration, 0);
-  ENFORCE_TYPE(exporter_working_mode, T_SYMBOL);
-  ID working_mode = SYM2ID(exporter_working_mode);
-
-  ID agentless_id = rb_intern("agentless");
-  ID agent_id = rb_intern("agent");
-
-  if (working_mode != agentless_id && working_mode != agent_id) {
-    rb_raise(rb_eArgError, "Failed to initialize transport: Unexpected working mode, expected :agentless or :agent");
-  }
-
-  if (working_mode == agentless_id) {
-    VALUE site = rb_ary_entry(exporter_configuration, 1);
-    VALUE api_key = rb_ary_entry(exporter_configuration, 2);
-    ENFORCE_TYPE(site, T_STRING);
-    ENFORCE_TYPE(api_key, T_STRING);
-
-    return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
-  } else { // agent_id
-    VALUE base_url = rb_ary_entry(exporter_configuration, 1);
-    ENFORCE_TYPE(base_url, T_STRING);
-
-    return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
-  }
-}
-
-__attribute__((warn_unused_result))
-ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
-  ENFORCE_TYPE(tags_as_array, T_ARRAY);
-
-  long tags_count = RARRAY_LEN(tags_as_array);
-  ddog_Vec_Tag tags = ddog_Vec_Tag_new();
-
-  for (long i = 0; i < tags_count; i++) {
-    VALUE name_value_pair = rb_ary_entry(tags_as_array, i);
-
-    if (!RB_TYPE_P(name_value_pair, T_ARRAY)) {
-      ddog_Vec_Tag_drop(tags);
-      ENFORCE_TYPE(name_value_pair, T_ARRAY);
-    }
-
-    // Note: We can index the array without checking its size first because rb_ary_entry returns Qnil if out of bounds
-    VALUE tag_name = rb_ary_entry(name_value_pair, 0);
-    VALUE tag_value = rb_ary_entry(name_value_pair, 1);
-
-    if (!(RB_TYPE_P(tag_name, T_STRING) && RB_TYPE_P(tag_value, T_STRING))) {
-      ddog_Vec_Tag_drop(tags);
-      ENFORCE_TYPE(tag_name, T_STRING);
-      ENFORCE_TYPE(tag_value, T_STRING);
-    }
-
-    ddog_Vec_Tag_PushResult push_result =
-      ddog_Vec_Tag_push(&tags, char_slice_from_ruby_string(tag_name), char_slice_from_ruby_string(tag_value));
-
-    if (push_result.tag == DDOG_VEC_TAG_PUSH_RESULT_ERR) {
-      // libdatadog validates tags and may catch invalid tags that ddtrace didn't actually catch.
-      // We warn users about such tags, and then just ignore them.
-
-      int exception_state;
-      rb_protect(log_failure_to_process_tag, get_error_details_and_drop(&push_result.err), &exception_state);
-
-      // Since we are calling into Ruby code, it may raise an exception. Ensure that dynamically-allocated tags
-      // get cleaned before propagating the exception.
-      if (exception_state) {
-        ddog_Vec_Tag_drop(tags);
-        rb_jump_tag(exception_state);  // "Re-raise" exception
-      }
-    }
-  }
-
-  return tags;
-}
-
-static VALUE log_failure_to_process_tag(VALUE err_details) {
-  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
-  VALUE logger = rb_funcall(datadog_module, rb_intern("logger"), 0);
-
-  return rb_funcall(logger, rb_intern("warn"), 1, rb_sprintf("Failed to add tag to profiling request: %"PRIsVALUE, err_details));
-}

ext/datadog_profiling_native_extension/libdatadog_helpers.h

@@ -3,27 +3,10 @@
 #include <datadog/profiling.h>
 #include "ruby_helpers.h"
 
-inline static ddog_CharSlice char_slice_from_ruby_string(VALUE string) {
-  ENFORCE_TYPE(string, T_STRING);
-  ddog_CharSlice char_slice = {.ptr = StringValuePtr(string), .len = RSTRING_LEN(string)};
-  return char_slice;
-}
-
 inline static VALUE ruby_string_from_vec_u8(ddog_Vec_U8 string) {
   return rb_str_new((char *) string.ptr, string.len);
 }
 
-inline static VALUE ruby_string_from_error(const ddog_Error *error) {
-  ddog_CharSlice char_slice = ddog_Error_message(error);
-  return rb_str_new(char_slice.ptr, char_slice.len);
-}
-
-inline static VALUE get_error_details_and_drop(ddog_Error *error) {
-  VALUE result = ruby_string_from_error(error);
-  ddog_Error_drop(error);
-  return result;
-}
-
 // Utility function to be able to extract an error cstring from a ddog_Error.
 // Returns the amount of characters written to string (which are necessarily
 // bounded by capacity - 1 since the string will be null-terminated).
@@ -40,7 +23,3 @@ ddog_CharSlice ruby_value_type_to_char_slice(enum ruby_value_type type);
 inline static char* string_from_char_slice(ddog_CharSlice slice) {
   return ruby_strndup(slice.ptr, slice.len);
 }
-
-ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration);
-
-ddog_Vec_Tag convert_tags(VALUE tags_as_array);