[trace][intelpt] Support system-wide tracing [4] - Support per core tracing on lldb-server

This diffs implements per-core tracing on lldb-server. It also includes tests that ensure that tracing can be initiated from the client and that the jLLDBGetState ppacket returns the list of trace buffers per core.

This doesn't include any decoder changes.

Finally, this makes some little changes here and there improving the existing code.

A specific piece of code that can't reliably be tested is when tracing
per core fails due to permissions. In this case we add a
troubleshooting message and this is the manual test:

```
/proc/sys/kernel/perf_event_paranoid set to 1

(lldb) process trace start --per-core-tracing                                         error: perf event syscall failed: Permission denied
 You might need that /proc/sys/kernel/perf_event_paranoid has a value of 0 or -1.
``

Differential Revision: https://reviews.llvm.org/D124858

Author: Walter Erquinigo

lldb/docs/lldb-gdb-remote.txt

@@ -245,7 +245,7 @@ read packet: OK
 //  OUTPUT SCHEMA
 //   {
 //     "name": <string>,
-//         Tracing technology name, e.g. intel-pt, arm-coresight.
+//         Tracing technology name, e.g. intel-pt, arm-etm.
 //     "description": <string>,
 //         Description for this technology.
 //   }
@@ -280,7 +280,7 @@ read packet: {"name":<name>, "description":<description>}/E<error code>;AAAAAAAA
 // INPUT SCHEMA
 //  {
 //    "type": <string>,
-//        Tracing technology name, e.g. intel-pt, arm-coresight.
+//        Tracing technology name, e.g. intel-pt, arm-etm.
 //
 //    /* thread tracing only */
 //    "tids"?: [<decimal integer>],
@@ -369,8 +369,9 @@ read packet: {"name":<name>, "description":<description>}/E<error code>;AAAAAAAA
 //     /* process tracing only */
 //     "processBufferSizeLimit": <decimal integer>,
 //         Maximum total buffer size per process in bytes.
-//         This limit applies to the sum of the sizes of all trace buffers for
-//         the current process, excluding the ones started with "thread tracing".
+//         This limit applies to the sum of the sizes of all thread or core
+//         buffers for the current process, excluding the ones started with
+//         "thread tracing".
 //
 //         If "perCoreTracing" is false, whenever a thread is attempted to be
 //         traced due to "process tracing" and the limit would be reached, the
@@ -421,7 +422,7 @@ read packet: OK/E<error code>;AAAAAAAAA
 //
 //  {
 //    "type": <string>
-//       Tracing technology name, e.g. intel-pt, arm-coresight.
+//       Tracing technology name, e.g. intel-pt, arm-etm.
 //
 //    /* thread trace stopping only */
 //    "tids":  [<decimal integer>]
@@ -455,7 +456,7 @@ read packet: OK/E<error code>;AAAAAAAAA
 // INPUT SCHEMA
 //  {
 //     "type": <string>
-//        Tracing technology name, e.g. intel-pt, arm-coresight.
+//        Tracing technology name, e.g. intel-pt, arm-etm.
 //  }
 //
 // OUTPUT SCHEMA
@@ -481,10 +482,23 @@ read packet: OK/E<error code>;AAAAAAAAA
 //            Size in bytes of this thread data.
 //      },
 //    ],
+//    "cores"?: [
+//      "id": <decimal integer>,
+//          Identifier for this CPU logical core.
+//      "binaryData": [
+//        {
+//          "kind": <string>,
+//              Identifier for some binary data related to this thread to
+//              fetch with the jLLDBTraceGetBinaryData packet.
+//          "size": <decimal integer>,
+//              Size in bytes of this cpu core data.
+//        },
+//      ]
+//    ],
 //    "counters"?: {
 //      "info_kind": {...parameters specific to the provided counter info kind},
-//          Each entry includes information related to counters associated with the trace.
-//          They are described below.
+//          Each entry includes information related to counters associated with
+//          the trace. They are described below.
 //    }
 //  }
 //
@@ -494,6 +508,12 @@ read packet: OK/E<error code>;AAAAAAAAA
 //
 // INTEL PT
 //
+//  If per-core process tracing is enabled, "tracedThreads" will contain all
+//  the threads of the process without any trace buffers. Besides that, the
+//  "cores" field will also be returned with per core trace buffers.
+//  A side effect of per-core tracing is that all the threads of unrelated
+//  processes will also be traced, thus polluting the tracing data.
+//
 //  Binary data kinds:
 //    - traceBuffer: trace buffer for a thread or a core.
 //    - procfsCpuInfo: contents of the /proc/cpuinfo file.
@@ -536,9 +556,11 @@ read packet: {...object}/E<error code>;AAAAAAAAA
 //
 //  {
 //   "type": <string>,
-//       Tracing technology name, e.g. intel-pt, arm-coresight.
+//       Tracing technology name, e.g. intel-pt, arm-etm.
 //   "kind": <string>,
 //       Identifier for the data.
+//   "coreId": <Optional decimal>,
+//       Core id in decimal if the data belongs to a CPU core.
 //   "tid"?: <Optional decimal>,
 //       Tid in decimal if the data belongs to a thread.
 //   "offset": <decimal>,

lldb/include/lldb/Utility/TraceGDBRemotePackets.h

@@ -109,14 +109,25 @@ llvm::json::Value toJSON(const TraceBinaryData &packet);
 struct TraceThreadState {
   lldb::tid_t tid;
   /// List of binary data objects for this thread.
-  std::vector<TraceBinaryData> binaryData;
+  std::vector<TraceBinaryData> binary_data;
 };
 
 bool fromJSON(const llvm::json::Value &value, TraceThreadState &packet,
               llvm::json::Path path);
 
 llvm::json::Value toJSON(const TraceThreadState &packet);
 
+struct TraceCoreState {
+  lldb::core_id_t core_id;
+  /// List of binary data objects for this core.
+  std::vector<TraceBinaryData> binary_data;
+};
+
+bool fromJSON(const llvm::json::Value &value, TraceCoreState &packet,
+              llvm::json::Path path);
+
+llvm::json::Value toJSON(const TraceCoreState &packet);
+
 /// Interface for different algorithms used to convert trace
 /// counters into different units.
 template <typename ToType> class TraceCounterConversion {
@@ -143,8 +154,9 @@ using TraceTscConversionUP =
     std::unique_ptr<TraceCounterConversion<std::chrono::nanoseconds>>;
 
 struct TraceGetStateResponse {
-  std::vector<TraceThreadState> tracedThreads;
-  std::vector<TraceBinaryData> processBinaryData;
+  std::vector<TraceThreadState> traced_threads;
+  std::vector<TraceBinaryData> process_binary_data;
+  llvm::Optional<std::vector<TraceCoreState>> cores;
 };
 
 bool fromJSON(const llvm::json::Value &value, TraceGetStateResponse &packet,

lldb/include/lldb/Utility/TraceIntelPTGDBRemotePackets.h

@@ -48,6 +48,8 @@ struct TraceIntelPTStartRequest : TraceStartRequest {
 
   /// Whether to have a trace buffer per thread or per cpu core.
   llvm::Optional<bool> per_core_tracing;
+
+  bool IsPerCoreTracing() const;
 };
 
 bool fromJSON(const llvm::json::Value &value, TraceIntelPTStartRequest &packet,

lldb/include/lldb/lldb-types.h

@@ -89,6 +89,7 @@ typedef int32_t break_id_t;
 typedef int32_t watch_id_t;
 typedef void *opaque_compiler_type_t;
 typedef uint64_t queue_id_t;
+typedef uint32_t core_id_t; // CPU core id
 } // namespace lldb
 
 #endif // LLDB_LLDB_TYPES_H

lldb/packages/Python/lldbsuite/test/tools/intelpt/intelpt_testcase.py

@@ -33,6 +33,19 @@ def setUp(self):
         if 'intel-pt' not in configuration.enabled_plugins:
             self.skipTest("The intel-pt test plugin is not enabled")
 
+    def skipIfPerCoreTracingIsNotSupported(self):
+        def is_supported():
+            try:
+                with open("/proc/sys/kernel/perf_event_paranoid", "r") as permissions:
+                    value = int(permissions.readlines()[0])
+                    if value <= 0:
+                        return True
+            except:
+                return False
+        if not is_supported():
+            self.skipTest("Per core tracing is not supported. You need "
+                "/proc/sys/kernel/perf_event_paranoid to be 0 or -1.")
+
     def getTraceOrCreate(self):
         if not self.target().GetTrace().IsValid():
             error = lldb.SBError()
@@ -110,7 +123,7 @@ def traceStopProcess(self):
         else:
             self.expect("process trace stop")
 
-    def traceStopThread(self, thread=None, error=False):
+    def traceStopThread(self, thread=None, error=False, substrs=None):
         if self.USE_SB_API:
             thread = thread if thread is not None else self.thread()
             self.assertSBError(self.target().GetTrace().Stop(thread), error)
@@ -119,4 +132,4 @@ def traceStopThread(self, thread=None, error=False):
             command = "thread trace stop"
             if thread is not None:
                 command += " " + str(thread.GetIndexID())
-            self.expect(command, error=error)
+            self.expect(command, error=error, substrs=substrs)

lldb/source/Plugins/Process/Linux/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_lldb_library(lldbPluginProcessLinux
   IntelPTCollector.cpp
   IntelPTSingleBufferTrace.cpp
+  IntelPTMultiCoreTrace.cpp
   NativeProcessLinux.cpp
   NativeRegisterContextLinux.cpp
   NativeRegisterContextLinux_arm.cpp