Skip to content

Commit

Permalink
Remove legacy TLS tracing feature toggle and transitional code (#1530)
Browse files Browse the repository at this point in the history 
Summary: Remove legacy TLS tracing feature toggle and transitional code

The new style of TLS tracing has been rolled out since April 11th
(vizier release v0.12.19). We believe that it is performing well and it
has already allowed Pixie's TLS tracing to cover more libraries (OpenSSL
v3) with BoringSSL coming soon.

This includes changes from #1518 and must be rebased once that is
merged.

Relevant Issues: #692

Type of change: /kind cleanup

Test Plan: Existing test coverage

---------

Signed-off-by: Dom Del Nano <ddelnano@pixielabs.ai>
  • Loading branch information
@ddelnano
ddelnano authored Jun 22, 2023
1 parent d7a1f67 commit c0ff63a
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 94 deletions.
5 changes: 0 additions & 5 deletions src/cloud/config_manager/controllers/vizier_feature_flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,6 @@ var availableFeatureFlags = []*featureFlag{
VizierFlagName: "PL_PROFILER_JAVA_SYMBOLS",
DefaultValue: true,
},
{
FeatureFlagName: "access-tls-socket-fd-via-syscall",
VizierFlagName: "PL_ACCESS_TLS_SOCKET_FD_VIA_SYSCALL",
DefaultValue: false,
},
}

// NewVizierFeatureFlagClient creates a LaunchDarkly feature flag client if the SDK key is provided,
Expand Down
48 changes: 12 additions & 36 deletions src/stirling/source_connectors/socket_tracer/bcc_bpf/socket_trace.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1058,9 +1058,7 @@ int syscall__probe_ret_write(struct pt_regs* ctx) {
// Syscalls that aren't exclusively used for networking must be
// validated to be a sock_event before propagating a socket fd to the
// tls tracing probes
if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, write_args->fd);
}
propagate_fd_to_user_space_call(id, write_args->fd);
process_syscall_data(ctx, id, kEgress, write_args, bytes_count);
}

Expand All @@ -1072,9 +1070,7 @@ int syscall__probe_ret_write(struct pt_regs* ctx) {
int syscall__probe_entry_send(struct pt_regs* ctx, int sockfd, char* buf, size_t len) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// Stash arguments.
struct data_args_t write_args = {};
Expand Down Expand Up @@ -1124,9 +1120,7 @@ int syscall__probe_ret_read(struct pt_regs* ctx) {
// Syscalls that aren't exclusively used for networking must be
// validated to be a sock_event before propagating a socket fd to the
// tls tracing probes
if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, read_args->fd);
}
propagate_fd_to_user_space_call(id, read_args->fd);
process_syscall_data(ctx, id, kIngress, read_args, bytes_count);
}

Expand All @@ -1138,9 +1132,7 @@ int syscall__probe_ret_read(struct pt_regs* ctx) {
int syscall__probe_entry_recv(struct pt_regs* ctx, int sockfd, char* buf, size_t len) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// Stash arguments.
struct data_args_t read_args = {};
Expand Down Expand Up @@ -1172,9 +1164,7 @@ int syscall__probe_entry_sendto(struct pt_regs* ctx, int sockfd, char* buf, size
const struct sockaddr* dest_addr, socklen_t addrlen) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// Stash arguments.
if (dest_addr != NULL) {
Expand Down Expand Up @@ -1235,9 +1225,7 @@ int syscall__probe_entry_recvfrom(struct pt_regs* ctx, int sockfd, char* buf, si
struct sockaddr* src_addr, socklen_t* addrlen) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// Stash arguments.
if (src_addr != NULL) {
Expand Down Expand Up @@ -1283,9 +1271,7 @@ int syscall__probe_entry_sendmsg(struct pt_regs* ctx, int sockfd,
const struct user_msghdr* msghdr) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

if (msghdr != NULL) {
// Stash arguments.
Expand Down Expand Up @@ -1333,9 +1319,7 @@ int syscall__probe_entry_sendmmsg(struct pt_regs* ctx, int sockfd, struct mmsghd
unsigned int vlen) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// TODO(oazizi): Right now, we only trace the first message in a sendmmsg() call.
if (msgvec != NULL && vlen >= 1) {
Expand Down Expand Up @@ -1389,9 +1373,7 @@ int syscall__probe_ret_sendmmsg(struct pt_regs* ctx) {
int syscall__probe_entry_recvmsg(struct pt_regs* ctx, int sockfd, struct user_msghdr* msghdr) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

if (msghdr != NULL) {
// Stash arguments.
Expand Down Expand Up @@ -1441,9 +1423,7 @@ int syscall__probe_entry_recvmmsg(struct pt_regs* ctx, int sockfd, struct mmsghd
unsigned int vlen) {
uint64_t id = bpf_get_current_pid_tgid();

if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, sockfd);
}
propagate_fd_to_user_space_call(id, sockfd);

// TODO(oazizi): Right now, we only trace the first message in a recvmmsg() call.
if (msgvec != NULL && vlen >= 1) {
Expand Down Expand Up @@ -1518,9 +1498,7 @@ int syscall__probe_ret_writev(struct pt_regs* ctx) {
// Syscalls that aren't exclusively used for networking must be
// validated to be a sock_event before propagating a socket fd to the
// tls tracing probes
if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, write_args->fd);
}
propagate_fd_to_user_space_call(id, write_args->fd);
process_syscall_data_vecs(ctx, id, kEgress, write_args, bytes_count);
}

Expand Down Expand Up @@ -1553,9 +1531,7 @@ int syscall__probe_ret_readv(struct pt_regs* ctx) {
// Syscalls that aren't exclusively used for networking must be
// validated to be a sock_event before propagating a socket fd to the
// tls tracing probes
if (ACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL) {
propagate_fd_to_user_space_call(id, read_args->fd);
}
propagate_fd_to_user_space_call(id, read_args->fd);
process_syscall_data_vecs(ctx, id, kIngress, read_args, bytes_count);
}

Expand Down
40 changes: 10 additions & 30 deletions src/stirling/source_connectors/socket_tracer/openssl_trace_bpf_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,10 @@ struct TraceRecords {
std::vector<std::string> remote_address;
};

template <typename TServerContainer, bool UseNestedSyscallFD>
class BaseOpenSSLTraceTest : public SocketTraceBPFTestFixture</* TClientSideTracing */ false> {
template <typename TServerContainer>
class OpenSSLTraceTest : public SocketTraceBPFTestFixture</* TClientSideTracing */ false> {
protected:
BaseOpenSSLTraceTest() {
OpenSSLTraceTest() {
// Run the nginx HTTPS server.
// The container runner will make sure it is in the ready state before unblocking.
// Stirling will run after this unblocks, as part of SocketTraceBPFTest SetUp().
Expand All @@ -111,12 +111,6 @@ class BaseOpenSSLTraceTest : public SocketTraceBPFTestFixture</* TClientSideTrac
sleep(1);
}

void SetUp() override {
FLAGS_access_tls_socket_fd_via_syscall = UseNestedSyscallFD;

SocketTraceBPFTestFixture::SetUp();
}

// Returns the trace records of the process specified by the input pid.
TraceRecords GetTraceRecords(int pid) {
std::vector<TaggedRecordBatch> tablets =
Expand Down Expand Up @@ -178,26 +172,12 @@ http::Record GetExpectedHTTPRecord() {

using OpenSSLServerImplementations =
Types<NginxOpenSSL_1_1_0_ContainerWrapper, NginxOpenSSL_1_1_1_ContainerWrapper,
NginxOpenSSL_3_0_8_ContainerWrapper, Python310ContainerWrapper,
Node12_3_1ContainerWrapper, Node14_18_1AlpineContainerWrapper>;
using OpenSSLServerNestedSyscallFDImplementations =
Types<Python310ContainerWrapper, NginxOpenSSL_1_1_1_ContainerWrapper,
NginxOpenSSL_3_0_8_ContainerWrapper>;

template <typename T>
using OpenSSLTraceTest = BaseOpenSSLTraceTest<T, false>;

template <typename T>
using OpenSSLTraceNestedSyscallFD = BaseOpenSSLTraceTest<T, true>;

#define OPENSSL_TYPED_TEST(TestCase, CodeBlock) \
TYPED_TEST(OpenSSLTraceTest, TestCase) \
CodeBlock TYPED_TEST(OpenSSLTraceNestedSyscallFD, TestCase) \
CodeBlock

TYPED_TEST_SUITE(OpenSSLTraceTest, OpenSSLServerImplementations);
TYPED_TEST_SUITE(OpenSSLTraceNestedSyscallFD, OpenSSLServerNestedSyscallFDImplementations);

OPENSSL_TYPED_TEST(ssl_capture_curl_client, {
TYPED_TEST(OpenSSLTraceTest, ssl_capture_curl_client) {
this->StartTransferDataThread();

// Make an SSL request with curl.
Expand All @@ -218,9 +198,9 @@ OPENSSL_TYPED_TEST(ssl_capture_curl_client, {

EXPECT_THAT(records.http_records, UnorderedElementsAre(EqHTTPRecord(expected_record)));
EXPECT_THAT(records.remote_address, UnorderedElementsAre(StrEq("127.0.0.1")));
})
}

OPENSSL_TYPED_TEST(ssl_capture_ruby_client, {
TYPED_TEST(OpenSSLTraceTest, ssl_capture_ruby_client) {
this->StartTransferDataThread();

// Make multiple requests and make sure we capture all of them.
Expand Down Expand Up @@ -261,9 +241,9 @@ OPENSSL_TYPED_TEST(ssl_capture_ruby_client, {
EqHTTPRecord(expected_record)));
EXPECT_THAT(records.remote_address,
UnorderedElementsAre(StrEq("127.0.0.1"), StrEq("127.0.0.1"), StrEq("127.0.0.1")));
})
}

OPENSSL_TYPED_TEST(ssl_capture_node_client, {
TYPED_TEST(OpenSSLTraceTest, ssl_capture_node_client) {
this->StartTransferDataThread();

// Make an SSL request with the client.
Expand All @@ -280,7 +260,7 @@ OPENSSL_TYPED_TEST(ssl_capture_node_client, {

EXPECT_THAT(records.http_records, UnorderedElementsAre(EqHTTPRecord(expected_record)));
EXPECT_THAT(records.remote_address, UnorderedElementsAre(StrEq("127.0.0.1")));
})
}

} // namespace stirling
} // namespace px
9 changes: 0 additions & 9 deletions src/stirling/source_connectors/socket_tracer/socket_trace_connector.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,6 @@ DEFINE_uint32(datastream_buffer_retention_size,
DEFINE_uint64(max_body_bytes, gflags::Uint64FromEnv("PL_STIRLING_MAX_BODY_BYTES", 512),
"The maximum number of bytes in the body of protocols like HTTP");

DEFINE_bool(
access_tls_socket_fd_via_syscall,
gflags::BoolFromEnv("PL_ACCESS_TLS_SOCKET_FD_VIA_SYSCALL", true),
"If true, stirling will identify a socket's fd based on the underlying syscall (read, write, "
"etc) while a user space tls function call occurs. When false, stirling attempts to access the "
"socket fd by walking user space data structures which may be brittle.");

OBJ_STRVIEW(socket_trace_bcc_script, socket_trace);

namespace px {
Expand Down Expand Up @@ -418,8 +411,6 @@ Status SocketTraceConnector::InitBPF() {
absl::StrCat("-DENABLE_NATS_TRACING=", protocol_transfer_specs_[kProtocolNATS].enabled),
absl::StrCat("-DENABLE_AMQP_TRACING=", protocol_transfer_specs_[kProtocolAMQP].enabled),
absl::StrCat("-DENABLE_MONGO_TRACING=", "true"),
absl::StrCat("-DACCESS_TLS_SK_FD_VIA_ACTIVE_SYSCALL=",
FLAGS_access_tls_socket_fd_via_syscall),
};
PX_RETURN_IF_ERROR(InitBPFProgram(socket_trace_bcc_script, defines));

Expand Down
45 changes: 32 additions & 13 deletions src/stirling/source_connectors/socket_tracer/uprobe_manager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,15 @@ StatusOr<std::vector<std::filesystem::path>> FindHostPathForPIDLibs(
HostPathForPIDPathSearchType::kSearchTypeEndsWith);
}

enum class SSLSocketFDAccess {
// Specifies that a connection's socket fd will be identified by accessing struct members
// of the SSL struct exposed by OpenSSL's API when the SSL_write/SSL_read functions are called.
kUserSpaceOffsets,
// Specifies that a connection's socket fd will be identified based on the underlying syscall
// (read, write, etc) while a user space tls function is on the stack.
kNestedSyscall,
};

// SSLLibMatcher allows customizing the search of shared object files
// that need to be traced with the SSL_write and SSL_read uprobes.
// In dynamically linked cases, it's likely that there are two
Expand All @@ -285,6 +294,7 @@ struct SSLLibMatcher {
std::string_view libssl;
std::string_view libcrypto;
HostPathForPIDPathSearchType search_type;
SSLSocketFDAccess socket_fd_access;
};

constexpr char kLibSSL_1_1[] = "libssl.so.1.1";
Expand All @@ -296,23 +306,29 @@ static constexpr const auto kLibSSLMatchers = MakeArray<SSLLibMatcher>({
.libssl = kLibSSL_1_1,
.libcrypto = "libcrypto.so.1.1",
.search_type = HostPathForPIDPathSearchType::kSearchTypeEndsWith,
.socket_fd_access = SSLSocketFDAccess::kNestedSyscall,
},
SSLLibMatcher{
.libssl = kLibSSL_3,
.libcrypto = "libcrypto.so.3",
.search_type = HostPathForPIDPathSearchType::kSearchTypeEndsWith,
.socket_fd_access = SSLSocketFDAccess::kNestedSyscall,
},
SSLLibMatcher{
// This must match independent of python version and INSTSONAME suffix
// (e.g. libpython3.10.so.0.1).
.libssl = kLibPython,
.libcrypto = kLibPython,
.search_type = HostPathForPIDPathSearchType::kSearchTypeContains,
.socket_fd_access = SSLSocketFDAccess::kNestedSyscall,
},
// non BIO native TLS applications cannot be probed by accessing the socket fd
// within the underlying syscall.
SSLLibMatcher{
.libssl = kLibNettyTcnativePrefix,
.libcrypto = kLibNettyTcnativePrefix,
.search_type = HostPathForPIDPathSearchType::kSearchTypeContains,
.socket_fd_access = SSLSocketFDAccess::kUserSpaceOffsets,
},
});

Expand All @@ -332,19 +348,26 @@ ssl_source_t SSLSourceFromLib(std::string_view libssl) {
return kSSLUnspecified;
}

std::string ProbeFuncForSocketAccessMethod(std::string_view probe_fn,
SSLSocketFDAccess socket_fd_access) {
std::string probe_suffix = "";
switch (socket_fd_access) {
case SSLSocketFDAccess::kUserSpaceOffsets:
break;
case SSLSocketFDAccess::kNestedSyscall:
probe_suffix = "_syscall_fd_access";
}

return absl::StrCat(probe_fn, probe_suffix);
}

// Return error if something unexpected occurs.
// Return 0 if nothing unexpected, but there is nothing to deploy (e.g. no OpenSSL detected).
StatusOr<int> UProbeManager::AttachOpenSSLUProbesOnDynamicLib(uint32_t pid) {
for (auto ssl_library_match : kLibSSLMatchers) {
const auto libssl = ssl_library_match.libssl;
const auto libcrypto = ssl_library_match.libcrypto;

// TODO(ddelnano): The legacy tls tracing implementation does not support OpenSSL v3.
// Remove this once that implementation is removed in addition to the feature toggle.
if (!FLAGS_access_tls_socket_fd_via_syscall && absl::EndsWith(libssl, "so.3")) {
continue;
}

const std::vector<std::string_view> lib_names = {libssl, libcrypto};
const auto search_type = ssl_library_match.search_type;

Expand Down Expand Up @@ -373,7 +396,7 @@ StatusOr<int> UProbeManager::AttachOpenSSLUProbesOnDynamicLib(uint32_t pid) {
return error::Internal("libcrypto not found [path = $0]", container_libcrypto.string());
}

if (!FLAGS_access_tls_socket_fd_via_syscall || libssl == kLibNettyTcnativePrefix) {
if (libssl == kLibNettyTcnativePrefix) {
auto fptr_manager = std::make_unique<obj_tools::RawFptrManager>(container_libcrypto);

PX_RETURN_IF_ERROR(UpdateOpenSSLSymAddrs(fptr_manager.get(), container_libcrypto, pid));
Expand All @@ -393,12 +416,8 @@ StatusOr<int> UProbeManager::AttachOpenSSLUProbesOnDynamicLib(uint32_t pid) {
openssl_source_map_->UpdateValue(pid, ssl_source);
for (auto spec : kOpenSSLUProbes) {
spec.binary_path = container_libssl.string();

// TODO(ddelnano): Remove this conditional logic once the new tls tracing
// implementation is the default.
if (FLAGS_access_tls_socket_fd_via_syscall && libssl != kLibNettyTcnativePrefix) {
spec.probe_fn = absl::Substitute("$0_syscall_fd_access", spec.probe_fn);
}
spec.probe_fn =
ProbeFuncForSocketAccessMethod(spec.probe_fn, ssl_library_match.socket_fd_access);

PX_RETURN_IF_ERROR(LogAndAttachUProbe(spec));
}
Expand Down
1 change: 0 additions & 1 deletion src/stirling/source_connectors/socket_tracer/uprobe_manager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
DECLARE_bool(stirling_rescan_for_dlopen);
DECLARE_bool(stirling_enable_grpc_c_tracing);
DECLARE_double(stirling_rescan_exp_backoff_factor);
DECLARE_bool(access_tls_socket_fd_via_syscall);

namespace px {
namespace stirling {
Expand Down

0 comments on commit c0ff63a

Please sign in to comment.