Use llvm-symbolizer's JSON output to provide function start lines (#891)

* Use llvm-symbolizer's JSON output to provide function start lines

When analyzing a perf.data profile converted automatically via perf_to_profile via pprof -raw perf.data, no function start lines (s=0) are present in any of the locations. With 813a5fb, this can be easily solved by using the same JSON frame data from llvm-symbolizer to provide StartLine for Function.start_line. This solves #823.

* Fixed TestPEFile windows test

* Fixed TestMachoFiles mac tests

* Fix making sure llvm-symbolizer is available in the CI env for Linux tests

Modify the `Fetch dependencies` step in `test-linux` job to include the installation of llvm. Add `Verify llvm-symbolizer installation` step to ensure that llvm-symbolizer is available in the CI environment for the Linux tests. This should resolve the test failures related to the missing llvm-symbolizer tool.

* ci: Improve LLVM installation for Ubuntu 20.04 and 22.04

Enhances the CI workflow to ensure proper installation of LLVM and the `llvm-symbolizer` across both Ubuntu 20.04 and 22.04 environments. These changes address the issue of an outdated llvm-symbolizer on Ubuntu 20.04, while maintaining compatibility with Ubuntu 22.04.

- Add LLVM official repository for Ubuntu 20.04
- Install LLVM 14 on Ubuntu 20.04 to ensure a recent version
- Use update-alternatives to manage llvm-symbolizer versions
- Maintain existing LLVM 14 for Ubuntu 22.04

* ci: Make it clear that our goal is to install llvm-symbolizer

---------

Co-authored-by: Alexey Alexandrov <aalexand@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -144,6 +144,23 @@ jobs:
         run: |
           google-chrome --version
 
+      - name: Add LLVM 14.0 repository to ensure llvm-symbolizer 14.0.0+ on Ubuntu 20.04
+        if: matrix.os == 'ubuntu-20.04'
+        run: |
+          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main"
+          sudo apt-get update
+
+      - name: Install llvm-symbolizer
+        run: |
+          if [ "${{ matrix.os }}" = "ubuntu-20.04" ]; then
+            sudo apt-get install -y llvm-14 clang-14
+            sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-14 100
+          else
+            sudo apt-get update
+            sudo apt-get install -y llvm clang
+          fi
+
       - name: Fetch dependencies
         run: |
           sudo apt-get install graphviz
@@ -154,6 +171,10 @@ jobs:
           # Add PATH for installed tools.
           echo "$GOPATH/bin:$PATH" >> $GITHUB_PATH
 
+      - name: Check llvm-symbolizer installation
+        run: |
+          llvm-symbolizer --version
+
       - name: Run the script
         run: |
           go version

driver/driver.go

@@ -186,10 +186,11 @@ type ObjFile interface {
 
 // A Frame describes a single line in a source file.
 type Frame struct {
-	Func   string // name of function
-	File   string // source file name
-	Line   int    // line in file
-	Column int    // column in file
+	Func      string // name of function
+	File      string // source file name
+	Line      int    // line in file
+	Column    int    // column in file
+	StartLine int    // start line of function (if available)
 }
 
 // A Sym describes a single symbol in an object file.

internal/binutils/addr2liner_llvm.go

@@ -151,18 +151,19 @@ func (d *llvmSymbolizer) readCodeFrames() ([]plugin.Frame, error) {
 		Address    string `json:"Address"`
 		ModuleName string `json:"ModuleName"`
 		Symbol     []struct {
-			Line         int    `json:"Line"`
-			Column       int    `json:"Column"`
-			FunctionName string `json:"FunctionName"`
-			FileName     string `json:"FileName"`
+			Line          int    `json:"Line"`
+			Column        int    `json:"Column"`
+			FunctionName  string `json:"FunctionName"`
+			FileName      string `json:"FileName"`
+			StartLine     int    `json:"StartLine"`
 		} `json:"Symbol"`
 	}
 	if err := json.Unmarshal([]byte(line), &frame); err != nil {
 		return nil, err
 	}
 	var stack []plugin.Frame
 	for _, s := range frame.Symbol {
-		stack = append(stack, plugin.Frame{Func: s.FunctionName, File: s.FileName, Line: s.Line, Column: s.Column})
+		stack = append(stack, plugin.Frame{Func: s.FunctionName, File: s.FileName, Line: s.Line, Column: s.Column, StartLine: s.StartLine})
 	}
 	return stack, nil
 }

internal/binutils/binutils_test.go

@@ -373,7 +373,7 @@ func TestObjFile(t *testing.T) {
 				t.Fatalf("SourceLine: unexpected error %v", err)
 			}
 			wantFrames := []plugin.Frame{
-				{Func: "main", File: "/tmp/hello.c", Line: 3},
+				{Func: "main", File: "/tmp/hello.c", Line: 3, StartLine: 3},
 			}
 			if !reflect.DeepEqual(gotFrames, wantFrames) {
 				t.Fatalf("SourceLine for main: got %v; want %v\n", gotFrames, wantFrames)
@@ -402,17 +402,17 @@ func TestMachoFiles(t *testing.T) {
 		{"normal mapping", "exe_mac_64", 0x100000000, math.MaxUint64, 0,
 			0x100000f50, "_main",
 			[]plugin.Frame{
-				{Func: "main", File: "/tmp/hello.c", Line: 3},
+				{Func: "main", File: "/tmp/hello.c", Line: 3, StartLine: 3},
 			}},
 		{"other mapping", "exe_mac_64", 0x200000000, math.MaxUint64, 0,
 			0x200000f50, "_main",
 			[]plugin.Frame{
-				{Func: "main", File: "/tmp/hello.c", Line: 3},
+				{Func: "main", File: "/tmp/hello.c", Line: 3, StartLine: 3},
 			}},
 		{"lib normal mapping", "lib_mac_64", 0, math.MaxUint64, 0,
 			0xfa0, "_bar",
 			[]plugin.Frame{
-				{Func: "bar", File: "/tmp/lib.c", Line: 5},
+				{Func: "bar", File: "/tmp/lib.c", Line: 5, StartLine: 5},
 			}},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -461,8 +461,8 @@ func TestLLVMSymbolizer(t *testing.T) {
 		frames []plugin.Frame
 	}{
 		{0x10, false, []plugin.Frame{
-			{Func: "Inlined_0x10", File: "foo.h", Line: 0, Column: 0},
-			{Func: "Func_0x10", File: "foo.c", Line: 2, Column: 1},
+			{Func: "Inlined_0x10", File: "foo.h", Line: 0, Column: 0, StartLine: 0},
+			{Func: "Func_0x10", File: "foo.c", Line: 2, Column: 1, StartLine: 2},
 		}},
 		{0x20, true, []plugin.Frame{
 			{Func: "foo_0x20", File: "0x20 8"},
@@ -532,7 +532,7 @@ func TestPEFile(t *testing.T) {
 				t.Fatalf("SourceLine: unexpected error %v", err)
 			}
 			wantFrames := []plugin.Frame{
-				{Func: "main", File: "hello.c", Line: 3, Column: 12},
+				{Func: "main", File: "hello.c", Line: 3, Column: 12, StartLine: 3},
 			}
 			if !reflect.DeepEqual(gotFrames, wantFrames) {
 				t.Fatalf("SourceLine for main: got %v; want %v\n", gotFrames, wantFrames)

internal/binutils/testdata/fake-llvm-symbolizer

@@ -29,7 +29,7 @@ while read line; do
   addr=$3
   case ${kind} in
   CODE)
-    echo "{\"Address\":\"${addr}\",\"ModuleName\":\"${fname}\",\"Symbol\":[{\"Column\":0,\"FileName\":\"${fname}.h\",\"FunctionName\":\"Inlined_${addr}\",\"Line\":0},{\"Column\":1,\"FileName\":\"${fname}.c\",\"FunctionName\":\"Func_${addr}\",\"Line\":2}]}"
+    echo "{\"Address\":\"${addr}\",\"ModuleName\":\"${fname}\",\"Symbol\":[{\"Column\":0,\"FileName\":\"${fname}.h\",\"FunctionName\":\"Inlined_${addr}\",\"Line\":0,\"StartLine\":0},{\"Column\":1,\"FileName\":\"${fname}.c\",\"FunctionName\":\"Func_${addr}\",\"Line\":2,\"StartLine\":2}]}"
     ;;
   DATA)
     echo "{\"Address\":\"${addr}\",\"ModuleName\":\"${fname}\",\"Data\":{\"Name\":\"${fname}_${addr}\",\"Size\":\"0x8\",\"Start\":\"${addr}\"}}"

internal/plugin/plugin.go

@@ -159,10 +159,11 @@ type ObjFile interface {
 
 // A Frame describes a location in a single line in a source file.
 type Frame struct {
-	Func   string // name of function
-	File   string // source file name
-	Line   int    // line in file
-	Column int    // column in line (if available)
+	Func          string // name of function
+	File          string // source file name
+	Line          int    // line in file
+	Column        int    // column in line (if available)
+	StartLine     int    // start line of function (if available)
 }
 
 // A Sym describes a single symbol in an object file.

internal/symbolizer/symbolizer.go

@@ -228,6 +228,7 @@ func symbolizeOneMapping(m *profile.Mapping, locs []*profile.Location, obj plugi
 				Name:       frame.Func,
 				SystemName: frame.Func,
 				Filename:   frame.File,
+				StartLine:  int64(frame.StartLine),
 			})
 			l.Line[i] = profile.Line{
 				Function: f,