Add 'symbol_source' config

Let the user decide whether to use DebugInfo to determine uprobe
location, or use the Symbol Table.

The new symbol_source config takes the following two values:

- dwarf: uses the DebugInfo, which yield more accurate stack traces.
         However, if the DebugInfo was rewritten by a post-linkage
         optimisation tool (like BOLT or AutoFDO), we might get
         wrong addresses for the probe location.
- symbol_table: uses the target's Symbol Table, which won't suffer
                from improperly rewritten DebugInfo.

This will close bpftrace#3493.

Author: ttreyer

.github/include/aot_skip.txt

@@ -146,6 +146,7 @@ aot.variable.variable nested tuple string resize
 aot.variable.map nested tuple string resize
 aot.variable.map key nested tuple string resize
 aot.variable.map key tuple with casted ints
+aot.dwarf.uprobe without dwarf
 aot.dwarf.uprobe inlined function - func
 aot.dwarf.uprobe inlined function - probe
 aot.dwarf.uprobe inlined function - ustack

CHANGELOG.md

@@ -46,6 +46,8 @@ and this project adheres to
   - [#3515](https://github.com/bpftrace/bpftrace/pull/3515)
 - Configuration option to suppress printing maps by default at program exit
   - [#3547](https://github.com/bpftrace/bpftrace/pull/3547)
+- Add `symbol_source` config to source uprobe locations from either DWARF or the Symbol Table
+  - [#3504](https://github.com/bpftrace/bpftrace/pull/3504/)
 #### Changed
 - Merge output into `stdout` when `-lv`
   - [#3383](https://github.com/bpftrace/bpftrace/pull/3383)

man/adoc/bpftrace.adoc

@@ -3724,6 +3724,20 @@ Default: 1
 
 Controls whether maps are printed on exit. Set to `0` in order to change the default behavior and not automatically print maps at program exit.
 
+=== symbol_source
+
+Default: `dwarf` if `bpftrace` is compiled with LLDB, `symbol_table` otherwise
+
+Choose how bpftrace will resolve all `uprobe` symbol locations.
+
+Available options:
+
+- `dwarf` - locate uprobes using DebugInfo, which yields more accurate stack traces (`ustack`). Fall back to the Symbol Table if it can't locate the probe using DebugInfo.
+- `symbol_table` - don't use DebugInfo and rely on the ELF Symbol Table instead.
+
+If the DebugInfo was rewritten by a post-linkage optimisation tool (like BOLT or AutoFDO), it might yield an incorrect address for a probe location.
+This config can force using the Symbol Table, for when the DebugInfo returns invalid addresses.
+
 == Environment Variables
 
 These are not available as part of the standard set of <<Config Variables>> and can only be set as environment variables.

src/ast/passes/config_analyser.cpp

@@ -96,6 +96,20 @@ void ConfigAnalyser::set_user_symbol_cache_type_config(
     LOG(ERROR, assignment.expr->loc, err_);
 }
 
+void ConfigAnalyser::set_symbol_source_config(
+    AssignConfigVarStatement &assignment)
+{
+  auto &assignTy = assignment.expr->type;
+  if (!assignTy.IsStringTy()) {
+    log_type_error(assignTy, Type::string, assignment);
+    return;
+  }
+
+  auto val = dynamic_cast<String *>(assignment.expr)->str;
+  if (!config_setter_.set_symbol_source_config(val))
+    LOG(ERROR, assignment.expr->loc, err_);
+}
+
 void ConfigAnalyser::set_missing_probes_config(
     AssignConfigVarStatement &assignment)
 {
@@ -165,6 +179,9 @@ void ConfigAnalyser::visit(AssignConfigVarStatement &assignment)
           [&, this](ConfigKeyUserSymbolCacheType) {
             set_user_symbol_cache_type_config(assignment);
           },
+          [&, this](ConfigKeySymbolSource) {
+            set_symbol_source_config(assignment);
+          },
           [&, this](ConfigKeyMissingProbes) {
             set_missing_probes_config(assignment);
           } },

src/ast/passes/config_analyser.h

@@ -47,6 +47,7 @@ class ConfigAnalyser : public Visitor {
                          ConfigKeyString key);
   void set_stack_mode_config(AssignConfigVarStatement &assignment);
   void set_user_symbol_cache_type_config(AssignConfigVarStatement &assignment);
+  void set_symbol_source_config(AssignConfigVarStatement &assignment);
   void set_missing_probes_config(AssignConfigVarStatement &assignment);
 
   void log_type_error(SizedType &type,

src/attached_probe.cpp

@@ -341,7 +341,11 @@ static constexpr std::string_view hint_unsafe =
     "\nUse --unsafe to force attachment. WARNING: This option could lead to "
     "data corruption in the target process.";
 
-static void check_alignment(std::string &path,
+static constexpr std::string_view hint_symbol_source =
+    "\nUse config 'symbol_source = \"symbol_table\"' in case of bad DebugInfo.";
+
+static void check_alignment(std::string &orig_name,
+                            std::string &path,
                             std::string &symbol,
                             uint64_t sym_offset,
                             uint64_t func_offset,
@@ -357,13 +361,20 @@ static void check_alignment(std::string &path,
     case AlignState::Ok:
       return;
     case AlignState::NotAlign:
-      if (safe_mode)
-        throw FatalUserException("Could not add " + probetypeName(type) +
-                                 " into middle of instruction: " + tmp +
-                                 std::string{ hint_unsafe });
-      else
+      if (safe_mode) {
+        auto msg = "Could not add " + probetypeName(type) +
+                   " into middle of instruction: " + tmp +
+                   std::string{ hint_unsafe };
+        if (orig_name.find('*') != std::string::npos)
+          msg += hint_symbol_source;
+        throw FatalUserException(std::move(msg));
+      } else {
+        std::string_view hint;
+        if (orig_name.find('*') != std::string::npos)
+          hint = hint_symbol_source;
         LOG(WARNING) << "Unsafe " << type
-                     << " in the middle of the instruction: " << tmp;
+                     << " in the middle of the instruction: " << tmp << hint;
+      }
       break;
 
     case AlignState::Fail:
@@ -473,8 +484,13 @@ bool AttachedProbe::resolve_offset_uprobe(bool safe_mode, bool has_multiple_aps)
   if (func_offset == 0)
     return true;
 
-  check_alignment(
-      probe_.path, symbol, sym_offset, func_offset, safe_mode, probe_.type);
+  check_alignment(probe_.orig_name,
+                  probe_.path,
+                  symbol,
+                  sym_offset,
+                  func_offset,
+                  safe_mode,
+                  probe_.type);
   return true;
 }
 

src/bpftrace.cpp

@@ -164,15 +164,19 @@ int BPFtrace::add_probe(const ast::AttachPoint &ap,
              probetype(ap.provider) == ProbeType::kprobe) {
     bool locations_from_dwarf = false;
 
+    // Don't set the DWARF target when the user wants to use the symbol table.
     std::optional<std::string> target;
-    if (probetype(ap.provider) == ProbeType::uprobe) {
-      target = probe.path;
-    } else {
-      // Only use the DWARF information of the Kernel,
-      // if the user wants to to probe inlined kprobes.
-      // Otherwise, fall back to using the symbol table.
-      if (config_.get(ConfigKeyBool::probe_inline))
-        target = find_vmlinux();
+    if (config_.get(ConfigKeySymbolSource::default_) ==
+        ConfigSymbolSource::dwarf) {
+      if (probetype(ap.provider) == ProbeType::uprobe) {
+        target = probe.path;
+      } else {
+        // Only use the DWARF information of the Kernel,
+        // if the user wants to to probe inlined kprobes.
+        // Otherwise, fall back to using the symbol table.
+        if (config_.get(ConfigKeyBool::probe_inline))
+          target = find_vmlinux();
+      }
     }
 
     // If the user specified an address/offset, do not overwrite

src/config.cpp

@@ -27,6 +27,14 @@ Config::Config(bool has_cmd)
     { ConfigKeyInt::perf_rb_pages, { .value = static_cast<uint64_t>(64) } },
     { ConfigKeyStackMode::default_, { .value = StackMode::bpftrace } },
     { ConfigKeyString::str_trunc_trailer, { .value = std::string("..") } },
+    { ConfigKeySymbolSource::default_,
+      { .value =
+#ifdef HAVE_LIBLLDB
+            ConfigSymbolSource::dwarf
+#else
+            ConfigSymbolSource::symbol_table
+#endif
+      } },
     { ConfigKeyMissingProbes::default_,
       { .value = ConfigMissingProbes::warn } },
     // by default, cache user symbols per program if ASLR is disabled on system
@@ -148,6 +156,21 @@ bool ConfigSetter::set_user_symbol_cache_type(const std::string &s)
   return config_.set(ConfigKeyUserSymbolCacheType::default_, usct, source_);
 }
 
+bool ConfigSetter::set_symbol_source_config(const std::string &s)
+{
+  ConfigSymbolSource source;
+  if (s == "dwarf") {
+    source = ConfigSymbolSource::dwarf;
+  } else if (s == "symbol_table") {
+    source = ConfigSymbolSource::symbol_table;
+  } else {
+    LOG(ERROR) << "Invalid value for symbol_source: valid values are "
+                  "\"dwarf\" and \"symbol_table\".";
+    return false;
+  }
+  return config_.set(ConfigKeySymbolSource::default_, source, source_);
+}
+
 bool ConfigSetter::set_missing_probes_config(const std::string &s)
 {
   ConfigMissingProbes mp;

src/config.h

@@ -53,6 +53,15 @@ enum class ConfigKeyUserSymbolCacheType {
   default_,
 };
 
+enum class ConfigSymbolSource {
+  symbol_table,
+  dwarf,
+};
+
+enum class ConfigKeySymbolSource {
+  default_,
+};
+
 enum class ConfigMissingProbes {
   ignore,
   warn,
@@ -68,6 +77,7 @@ typedef std::variant<ConfigKeyBool,
                      ConfigKeyString,
                      ConfigKeyStackMode,
                      ConfigKeyUserSymbolCacheType,
+                     ConfigKeySymbolSource,
                      ConfigKeyMissingProbes>
     ConfigKey;
 
@@ -89,6 +99,7 @@ const std::map<std::string, ConfigKey> CONFIG_KEY_MAP = {
   { "probe_inline", ConfigKeyBool::probe_inline },
   { "stack_mode", ConfigKeyStackMode::default_ },
   { "str_trunc_trailer", ConfigKeyString::str_trunc_trailer },
+  { "symbol_source", ConfigKeySymbolSource::default_ },
   { "missing_probes", ConfigKeyMissingProbes::default_ },
   { "print_maps_on_exit", ConfigKeyBool::print_maps_on_exit },
 };
@@ -106,6 +117,7 @@ struct ConfigValue {
                std::string,
                StackMode,
                UserSymbolCacheType,
+               ConfigSymbolSource,
                ConfigMissingProbes>
       value;
 };
@@ -139,6 +151,11 @@ class Config {
     return get<UserSymbolCacheType>(key);
   }
 
+  ConfigSymbolSource get(ConfigKeySymbolSource key) const
+  {
+    return get<ConfigSymbolSource>(key);
+  }
+
   ConfigMissingProbes get(ConfigKeyMissingProbes key) const
   {
     return get<ConfigMissingProbes>(key);
@@ -226,6 +243,7 @@ class ConfigSetter {
 
   bool set_stack_mode(const std::string &s);
   bool set_user_symbol_cache_type(const std::string &s);
+  bool set_symbol_source_config(const std::string &s);
   bool set_missing_probes_config(const std::string &s);
 
   Config &config_;

tests/bpftrace.cpp

@@ -707,6 +707,46 @@ TEST(bpftrace, add_probes_uprobe_no_demangling)
                "uprobe:/bin/sh:cpp_mangled");
 }
 
+#ifdef HAVE_LIBLLDB
+#include "dwarf_common.h"
+
+class bpftrace_dwarf : public test_dwarf {};
+
+TEST_F(bpftrace_dwarf, add_probes_uprobe_symbol_source)
+{
+  auto uprobe = "uprobe:" + std::string(bin_) + ":func_1 {}";
+
+  {
+    BPFtrace bpftrace;
+    ConfigSetter configs{ bpftrace.config_, ConfigSource::script };
+    configs.set_symbol_source_config("dwarf");
+    parse_probe(uprobe, bpftrace);
+
+    ASSERT_EQ(bpftrace.resources.probes.size(), 1);
+    ASSERT_EQ(bpftrace.resources.special_probes.size(), 0);
+
+    auto &probe = bpftrace.resources.probes.at(0);
+    ASSERT_TRUE(probe.attach_point.empty());
+    ASSERT_NE(probe.address, 0);
+  }
+
+  {
+    BPFtrace bpftrace;
+    ConfigSetter configs{ bpftrace.config_, ConfigSource::script };
+    configs.set_symbol_source_config("symbol_table");
+    parse_probe(uprobe, bpftrace);
+
+    ASSERT_EQ(bpftrace.resources.probes.size(), 1);
+    ASSERT_EQ(bpftrace.resources.special_probes.size(), 0);
+
+    auto &probe = bpftrace.resources.probes.at(0);
+    ASSERT_FALSE(probe.attach_point.empty());
+    ASSERT_EQ(probe.address, 0);
+  }
+}
+
+#endif
+
 TEST(bpftrace, add_probes_usdt)
 {
   StrictMock<MockBPFtrace> bpftrace;