From e617bf40ea9e235fa837998f041384db4112a52c Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:06:17 +0200 Subject: [PATCH 01/10] bpflist: linter cleanup --- tools/bpflist.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/bpflist.py b/tools/bpflist.py index b2d75d06030b..5fd109ce4b31 100755 --- a/tools/bpflist.py +++ b/tools/bpflist.py @@ -40,7 +40,8 @@ def comm_for_pid(pid): counts = {} def parse_probes(typ): - if args.verbosity > 1: print("open %ss:" % typ) + if args.verbosity > 1: + print("open %ss:" % typ) for probe in open("/sys/kernel/debug/tracing/%s_events" % typ): # Probes opened by bcc have a specific pattern that includes the pid # of the requesting process. @@ -48,8 +49,10 @@ def parse_probes(typ): if match: pid = int(match.group(1)) counts[(pid, typ)] = counts.get((pid, typ), 0) + 1 - if args.verbosity > 1: print(probe.strip()) - if args.verbosity > 1: print("") + if args.verbosity > 1: + print(probe.strip()) + if args.verbosity > 1: + print("") if args.verbosity > 0: parse_probes("kprobe") From 46017920441ad134817db7186331da1f53d761ec Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:06:28 +0200 Subject: [PATCH 02/10] dbslower: linter cleanup --- tools/dbslower.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tools/dbslower.py b/tools/dbslower.py index b32f6c9a99c7..d6f36cccb784 100755 --- a/tools/dbslower.py +++ b/tools/dbslower.py @@ -2,17 +2,18 @@ # # dbslower Trace MySQL and PostgreSQL queries slower than a threshold. # -# USAGE: dbslower [-v] [-p PID [PID ...]] [-b PATH_TO_BINARY] [-m THRESHOLD] {mysql,postgres} +# USAGE: dbslower [-v] [-p PID [PID ...]] [-b PATH_TO_BINARY] [-m THRESHOLD] +# {mysql,postgres} # # By default, a threshold of 1ms is used. Set the threshold to 0 to trace all -# queries (verbose). -# -# Script works in two different modes: -# 1) USDT probes, which means it needs MySQL and PostgreSQL built with +# queries (verbose). +# +# Script works in two different modes: +# 1) USDT probes, which means it needs MySQL and PostgreSQL built with # USDT (DTrace) support. -# 2) uprobe and uretprobe on exported function of binary specified by +# 2) uprobe and uretprobe on exported function of binary specified by # PATH_TO_BINARY parameter. (At the moment only MySQL support) -# +# # If no PID or PATH_TO_BINARY is provided, the script attempts to discover # all MySQL or PostgreSQL database processes and uses USDT probes. # @@ -57,12 +58,13 @@ mode = "USDT" if args.path and not args.pids: if args.db == "mysql": - symbols = BPF.get_user_functions_and_addresses(args.path, "\\w+dispatch_command\\w+") + regex = "\\w+dispatch_command\\w+" + symbols = BPF.get_user_functions_and_addresses(args.path, regex) if len(symbols) == 0: print("Can't find function 'dispatch_command' in %s" % (args.path)) exit(1) - + (mysql_func_name, addr) = symbols[0] if mysql_func_name.find("COM_DATA") >= 0: @@ -71,7 +73,8 @@ mode = "MYSQL56" else: # Placeholder for PostrgeSQL - # Look on functions initStringInfo, pgstat_report_activity, EndCommand, NullCommand + # Look on functions initStringInfo, pgstat_report_activity, EndCommand, + # NullCommand print("Sorry at the moment PostgreSQL supports only USDT") exit(1) @@ -109,7 +112,7 @@ #if defined(MYSQL56) || defined(MYSQL57) /* -Trace only packets with enum_server_command == COM_QUERY +Trace only packets with enum_server_command == COM_QUERY */ #ifdef MYSQL56 u64 command = (u64) PT_REGS_PARM1(ctx); @@ -148,7 +151,7 @@ u64 delta = bpf_ktime_get_ns() - tempp->timestamp; #ifdef THRESHOLD if (delta >= THRESHOLD) { -#endif //THRESHOLD +#endif //THRESHOLD struct data_t data = {}; data.pid = pid >> 32; // only process id data.timestamp = tempp->timestamp; @@ -164,13 +167,16 @@ """.replace("DEFINE_USDT", "#define USDT" if mode == "USDT" else "") \ .replace("DEFINE_MYSQL56", "#define MYSQL56" if mode == "MYSQL56" else "") \ .replace("DEFINE_MYSQL57", "#define MYSQL57" if mode == "MYSQL57" else "") \ - .replace("DEFINE_THRESHOLD", ("#define THRESHOLD " + str(threshold_ns)) if threshold_ns > 0 else "") + .replace("DEFINE_THRESHOLD", + "#define THRESHOLD %d" % threshold_ns if threshold_ns > 0 else "") if mode.startswith("MYSQL"): # Uprobes mode bpf = BPF(text=program) - bpf.attach_uprobe(name=args.path, sym=mysql_func_name, fn_name="query_start") - bpf.attach_uretprobe(name=args.path, sym=mysql_func_name, fn_name="query_end") + bpf.attach_uprobe(name=args.path, sym=mysql_func_name, + fn_name="query_start") + bpf.attach_uretprobe(name=args.path, sym=mysql_func_name, + fn_name="query_end") else: # USDT mode if not args.pids or len(args.pids) == 0: From 6d9b1b22c45081304cc0f375fc4daaaa035c54eb Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:06:41 +0200 Subject: [PATCH 03/10] tcptop: linter cleanup --- tools/tcptop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tcptop.py b/tools/tcptop.py index 9ca10f62210c..4b448e850623 100755 --- a/tools/tcptop.py +++ b/tools/tcptop.py @@ -149,7 +149,7 @@ def range_check(string): u64 *val, zero = 0; if (copied <= 0) - return 0; + return 0; if (family == AF_INET) { struct ipv4_key_t ipv4_key = {.pid = pid}; From 7297af05c343bdeb5f3f1af3555e31ef8b23647b Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:06:58 +0200 Subject: [PATCH 04/10] tcptracer: linter cleanup --- tools/tcptracer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/tcptracer.py b/tools/tcptracer.py index 88225466ecdc..bc70598bb8bb 100755 --- a/tools/tcptracer.py +++ b/tools/tcptracer.py @@ -567,7 +567,7 @@ def print_ipv4_event(cpu, data, size): if args.verbose: print("%-14d" % (event.ts_ns - start_ts), end="") else: - print("%-9.3f" % ((float(event.ts_ns) - start_ts) / 1000000000), end="") + print("%-9.3f" % ((event.ts_ns - start_ts) / 1000000000.0), end="") if event.type == 1: type_str = "C" elif event.type == 2: @@ -604,7 +604,7 @@ def print_ipv6_event(cpu, data, size): if args.verbose: print("%-14d" % (event.ts_ns - start_ts), end="") else: - print("%-9.3f" % ((float(event.ts_ns) - start_ts) / 1000000000), end="") + print("%-9.3f" % ((event.ts_ns - start_ts) / 1000000000.0), end="") if event.type == 1: type_str = "C" elif event.type == 2: @@ -622,8 +622,8 @@ def print_ipv6_event(cpu, data, size): print("%-6d %-16s %-2d %-16s %-16s %-6d %-6d" % (event.pid, event.comm.decode('utf-8'), event.ip, - "["+inet_ntop(AF_INET6, event.saddr)+"]", - "["+inet_ntop(AF_INET6, event.daddr)+"]", + "[" + inet_ntop(AF_INET6, event.saddr) + "]", + "[" + inet_ntop(AF_INET6, event.daddr) + "]", event.sport, event.dport), end="") if args.verbose and not args.netns: From 2e07ddc55a6ddac20b0cdc3b92ee024323ae39b7 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:07:10 +0200 Subject: [PATCH 05/10] memleak: linter cleanup --- tools/memleak.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/memleak.py b/tools/memleak.py index 484c386488d6..75541b4dcf0d 100755 --- a/tools/memleak.py +++ b/tools/memleak.py @@ -281,7 +281,7 @@ def run_command_get_pid(command): memptrs.delete(&pid); - if (bpf_probe_read(&addr, sizeof(void*), (void*)(size_t)*memptr64) != 0) + if (bpf_probe_read(&addr, sizeof(void*), (void*)(size_t)*memptr64)) return 0; u64 addr64 = (u64)(size_t)addr; @@ -382,7 +382,7 @@ def run_command_get_pid(command): stack_flags += "|BPF_F_USER_STACK" bpf_source = bpf_source.replace("STACK_FLAGS", stack_flags) -bpf_program = BPF(text=bpf_source) +bpf = BPF(text=bpf_source) if not kernel_trace: print("Attaching to pid %d, Ctrl+C to quit." % pid) @@ -392,12 +392,12 @@ def attach_probes(sym, fn_prefix=None, can_fail=False): fn_prefix = sym try: - bpf_program.attach_uprobe(name=obj, sym=sym, - fn_name=fn_prefix+"_enter", - pid=pid) - bpf_program.attach_uretprobe(name=obj, sym=sym, - fn_name=fn_prefix+"_exit", - pid=pid) + bpf.attach_uprobe(name=obj, sym=sym, + fn_name=fn_prefix + "_enter", + pid=pid) + bpf.attach_uretprobe(name=obj, sym=sym, + fn_name=fn_prefix + "_exit", + pid=pid) except Exception: if can_fail: return @@ -411,8 +411,8 @@ def attach_probes(sym, fn_prefix=None, can_fail=False): attach_probes("valloc") attach_probes("memalign") attach_probes("pvalloc") - attach_probes("aligned_alloc", can_fail=True) # added in C11 - bpf_program.attach_uprobe(name=obj, sym="free", fn_name="free_enter", + attach_probes("aligned_alloc", can_fail=True) # added in C11 + bpf.attach_uprobe(name=obj, sym="free", fn_name="free_enter", pid=pid) else: @@ -423,11 +423,12 @@ def attach_probes(sym, fn_prefix=None, can_fail=False): # # Memory allocations in Linux kernel are not limited to malloc/free # equivalents. It's also common to allocate a memory page or multiple - # pages. Page allocator have two interfaces, one working with page frame - # numbers (PFN), while other working with page addresses. It's possible - # to allocate pages with one kind of functions, and free them with - # another. Code in kernel can easy convert PFNs to addresses and back, - # but it's hard to do the same in eBPF kprobe without fragile hacks. + # pages. Page allocator have two interfaces, one working with page + # frame numbers (PFN), while other working with page addresses. It's + # possible to allocate pages with one kind of functions, and free them + # with another. Code in kernel can easy convert PFNs to addresses and + # back, but it's hard to do the same in eBPF kprobe without fragile + # hacks. # # Fortunately, Linux exposes tracepoints for memory allocations, which # can be instrumented by eBPF programs. Tracepoint for page allocations @@ -438,8 +439,8 @@ def print_outstanding(): print("[%s] Top %d stacks with outstanding allocations:" % (datetime.now().strftime("%H:%M:%S"), top_stacks)) alloc_info = {} - allocs = bpf_program["allocs"] - stack_traces = bpf_program["stack_traces"] + allocs = bpf["allocs"] + stack_traces = bpf["stack_traces"] for address, info in sorted(allocs.items(), key=lambda a: a[1].size): if BPF.monotonic_time() - min_age_ns < info.timestamp_ns: continue @@ -451,7 +452,7 @@ def print_outstanding(): stack = list(stack_traces.walk(info.stack_id)) combined = [] for addr in stack: - combined.append(bpf_program.sym(addr, pid, + combined.append(bpf.sym(addr, pid, show_module=True, show_offset=True)) alloc_info[info.stack_id] = Allocation(combined, info.size) @@ -465,8 +466,8 @@ def print_outstanding(): (alloc.size, alloc.count, "\n\t\t".join(alloc.stack))) def print_outstanding_combined(): - stack_traces = bpf_program["stack_traces"] - stacks = sorted(bpf_program["combined_allocs"].items(), + stack_traces = bpf["stack_traces"] + stacks = sorted(bpf["combined_allocs"].items(), key=lambda a: -a[1].total_size) cnt = 1 entries = [] @@ -474,7 +475,7 @@ def print_outstanding_combined(): try: trace = [] for addr in stack_traces.walk(stack_id.value): - sym = bpf_program.sym(addr, pid, + sym = bpf.sym(addr, pid, show_module=True, show_offset=True) trace.append(sym) @@ -498,7 +499,7 @@ def print_outstanding_combined(): count_so_far = 0 while True: if trace_all: - print(bpf_program.trace_fields()) + print(bpf.trace_fields()) else: try: sleep(interval) From 044cceaae747ba5523e2b92850bc990a30d6adf8 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:07:29 +0200 Subject: [PATCH 06/10] nfsslower: linter cleanup --- tools/nfsslower.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nfsslower.py b/tools/nfsslower.py index 167c37ef49da..cacd473ddc32 100755 --- a/tools/nfsslower.py +++ b/tools/nfsslower.py @@ -318,7 +318,7 @@ def print_event(cpu, data, size): "OFF_KB", "LAT(ms)", "FILENAME")) - + b["events"].open_perf_buffer(print_event, page_cnt=64) while 1: b.kprobe_poll() From c8b4f679338378682cfd9861869272ff6545fc71 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:51:45 +0200 Subject: [PATCH 07/10] ucalls: linter cleanup --- tools/lib/ucalls.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/lib/ucalls.py b/tools/lib/ucalls.py index 2798793759b4..d1f404350f43 100755 --- a/tools/lib/ucalls.py +++ b/tools/lib/ucalls.py @@ -67,7 +67,8 @@ return_probe = "method__return" read_class = "bpf_usdt_readarg(2, ctx, &clazz);" read_method = "bpf_usdt_readarg(4, ctx, &method);" - extra_message = "If you do not see any results, make sure you ran java with option -XX:+ExtendedDTraceProbes" + extra_message = ("If you do not see any results, make sure you ran java" + " with option -XX:+ExtendedDTraceProbes") elif language == "python": entry_probe = "function__entry" return_probe = "function__return" @@ -84,7 +85,8 @@ return_probe = "function__return" read_class = "bpf_usdt_readarg(4, ctx, &clazz);" read_method = "bpf_usdt_readarg(1, ctx, &method);" - extra_message = "If you do not see any results, make sure the environment variable USE_ZEND_DTRACE is set to 1" + extra_message = ("If you do not see any results, make sure the environment" + " variable USE_ZEND_DTRACE is set to 1") elif not language or language == "none": if not args.syscalls: print("Nothing to do; use -S to trace syscalls.") From 897c6867967b16c0c0032100088adea3ffd67624 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:52:05 +0200 Subject: [PATCH 08/10] trace: linter cleanup --- tools/trace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/trace.py b/tools/trace.py index 793cd9f79892..439032899f70 100755 --- a/tools/trace.py +++ b/tools/trace.py @@ -492,8 +492,8 @@ def print_event(self, bpf, cpu, data, size): time = strftime("%H:%M:%S") if Probe.use_localtime else \ Probe._time_off_str(event.timestamp_ns) print("%-8s %-6d %-6d %-12s %-16s %s" % - (time[:8], event.tgid, event.pid, event.comm.decode(), - self._display_function(), msg)) + (time[:8], event.tgid, event.pid, + event.comm.decode(), self._display_function(), msg)) if self.kernel_stack: self.print_stack(bpf, event.kernel_stack_id, -1) From 7b911b51d8596e7b29fc51f94256281d7a4298e2 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:52:17 +0200 Subject: [PATCH 09/10] sslsniff: linter cleanup --- tools/sslsniff.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/sslsniff.py b/tools/sslsniff.py index 1a4df29249c1..a81b4a49bc88 100755 --- a/tools/sslsniff.py +++ b/tools/sslsniff.py @@ -199,13 +199,9 @@ def print_event(cpu, data, size, rw): e_mark = "-" * 5 + " END DATA (TRUNCATED, " + str(truncated_bytes) + \ " bytes lost) " + "-" * 5 - print("%-12s %-18.9f %-16s %-6d %-6d\n%s\n%s\n%s\n\n" % (rw, time_s, - event.comm.decode(), - event.pid, - event.len, - s_mark, - event.v0.decode(), - e_mark)) + fmt = "%-12s %-18.9f %-16s %-6d %-6d\n%s\n%s\n%s\n\n" + print(fmt % (rw, time_s, event.comm.decode(), event.pid, event.len, s_mark, + event.v0.decode(), e_mark)) b["perf_SSL_write"].open_perf_buffer(print_event_write) b["perf_SSL_read"].open_perf_buffer(print_event_read) From ae0e0257f9645ac6d7ce61906a5db3de53b75eda Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Sat, 7 Oct 2017 11:52:30 +0200 Subject: [PATCH 10/10] bashreadline: linter cleanup --- tools/bashreadline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bashreadline.py b/tools/bashreadline.py index db48e8392c6d..0be8b26d56cb 100755 --- a/tools/bashreadline.py +++ b/tools/bashreadline.py @@ -56,7 +56,8 @@ class Data(ct.Structure): def print_event(cpu, data, size): event = ct.cast(data, ct.POINTER(Data)).contents - print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str.decode())) + print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, + event.str.decode())) b["events"].open_perf_buffer(print_event) while 1: