[DX-3763] dump inuse and memory profile

framework/.changeset/v0.15.16.md

@@ -0,0 +1,2 @@
+- Dump two memory profiles: inuse and alloc
+- Dump LOOPs profiles via admin command

framework/docker.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -186,6 +187,47 @@ func (dc *DockerClient) CopyFile(containerName, sourceFile, targetPath string) e
 	return dc.copyToContainer(containerID, sourceFile, targetPath)
 }
 
+// CopyFromContainer copies files from a container path and returns a tar archive stream.
+func (dc *DockerClient) CopyFromContainer(containerName, sourcePath string) (io.ReadCloser, container.PathStat, error) {
+	return dc.CopyFromContainerWithContext(context.Background(), containerName, sourcePath)
+}
+
+// CopyFromContainerWithContext copies files from a container path and returns a tar archive stream.
+func (dc *DockerClient) CopyFromContainerWithContext(ctx context.Context, containerName, sourcePath string) (io.ReadCloser, container.PathStat, error) {
+	containerID, err := dc.findContainerIDByName(ctx, containerName)
+	if err != nil {
+		return nil, container.PathStat{}, fmt.Errorf("failed to find container ID by name: %s", containerName)
+	}
+	reader, stat, err := dc.cli.CopyFromContainer(ctx, containerID, sourcePath)
+	if err != nil {
+		return nil, container.PathStat{}, fmt.Errorf("could not copy from container %s path %s: %w", containerName, sourcePath, err)
+	}
+	return reader, stat, nil
+}
+
+// CopyFromContainerToHost copies files from a container path and extracts them into hostDir.
+// If sourcePath points to a directory, only its contents are placed inside hostDir.
+func (dc *DockerClient) CopyFromContainerToHost(containerName, sourcePath, hostDir string) error {
+	return dc.CopyFromContainerToHostWithContext(context.Background(), containerName, sourcePath, hostDir)
+}
+
+// CopyFromContainerToHostWithContext copies files from a container path and extracts them into hostDir.
+// If sourcePath points to a directory, only its contents are placed inside hostDir.
+func (dc *DockerClient) CopyFromContainerToHostWithContext(ctx context.Context, containerName, sourcePath, hostDir string) error {
+	reader, _, err := dc.CopyFromContainerWithContext(ctx, containerName, sourcePath)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	if err := os.MkdirAll(hostDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create host destination directory %s: %w", hostDir, err)
+	}
+
+	stripTopDir := path.Base(path.Clean(sourcePath))
+	return extractTarArchiveToHostDir(reader, hostDir, stripTopDir)
+}
+
 // findContainerIDByName finds a container ID by its name
 func (dc *DockerClient) findContainerIDByName(ctx context.Context, containerName string) (string, error) {
 	containers, err := dc.cli.ContainerList(ctx, container.ListOptions{
@@ -245,6 +287,89 @@ func (dc *DockerClient) copyToContainer(containerID, sourceFile, targetPath stri
 	return nil
 }
 
+func extractTarArchiveToHostDir(reader io.Reader, hostDir, stripTopDir string) error {
+	tarReader := tar.NewReader(reader)
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("failed reading tar stream: %w", err)
+		}
+
+		relativePath, ok := normalizeTarEntryPath(header.Name, stripTopDir)
+		if !ok {
+			continue
+		}
+		targetPath := filepath.Join(hostDir, filepath.FromSlash(relativePath))
+		if err := ensureSubpath(hostDir, targetPath); err != nil {
+			return err
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(targetPath, 0o755); err != nil {
+				return fmt.Errorf("failed to create dir %s: %w", targetPath, err)
+			}
+		case tar.TypeReg, tar.TypeRegA:
+			if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
+				return fmt.Errorf("failed to create parent dir for %s: %w", targetPath, err)
+			}
+			file, createErr := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode))
+			if createErr != nil {
+				return fmt.Errorf("failed to create file %s: %w", targetPath, createErr)
+			}
+			if _, copyErr := io.Copy(file, tarReader); copyErr != nil {
+				_ = file.Close()
+				return fmt.Errorf("failed to write file %s: %w", targetPath, copyErr)
+			}
+			if closeErr := file.Close(); closeErr != nil {
+				return fmt.Errorf("failed to close file %s: %w", targetPath, closeErr)
+			}
+		}
+	}
+}
+
+func normalizeTarEntryPath(entryName, stripTopDir string) (string, bool) {
+	normalized := strings.TrimPrefix(entryName, "./")
+	normalized = path.Clean(normalized)
+	if normalized == "." || normalized == "/" {
+		return "", false
+	}
+
+	if stripTopDir != "" {
+		stripTopDir = path.Clean(stripTopDir)
+		if normalized == stripTopDir {
+			return "", false
+		}
+		prefix := stripTopDir + "/"
+		normalized = strings.TrimPrefix(normalized, prefix)
+	}
+
+	normalized = strings.TrimPrefix(normalized, "/")
+	if normalized == "" {
+		return "", false
+	}
+	return normalized, true
+}
+
+func ensureSubpath(baseDir, targetPath string) error {
+	baseAbs, err := filepath.Abs(baseDir)
+	if err != nil {
+		return fmt.Errorf("failed to resolve absolute path for %s: %w", baseDir, err)
+	}
+	targetAbs, err := filepath.Abs(targetPath)
+	if err != nil {
+		return fmt.Errorf("failed to resolve absolute path for %s: %w", targetPath, err)
+	}
+	prefix := baseAbs + string(filepath.Separator)
+	if targetAbs != baseAbs && !strings.HasPrefix(targetAbs, prefix) {
+		return fmt.Errorf("unsafe path detected outside destination: %s", targetPath)
+	}
+	return nil
+}
+
 // SearchLogFile searches logfile using regex and return matches or error
 func SearchLogFile(fp string, regex string) ([]string, error) {
 	file, err := os.Open(fp)

framework/leak/detector_cl_node.go

@@ -1,6 +1,7 @@
 package leak
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"strconv"
@@ -243,15 +244,36 @@ func (cd *CLNodesLeakDetector) Check(t *CLNodesCheck) error {
 		Str("TestDuration", t.End.Sub(t.Start).String()).
 		Float64("TestDurationSec", t.End.Sub(t.Start).Seconds()).
 		Msg("Leaks info")
-	framework.L.Info().Msg("Downloading pprof profile..")
+
+	profilesToDump := []string{DefaultProfileType, "memory:inuse_space:bytes:space:bytes"}
+	framework.L.Info().Msgf("Downloading %d pprof profiles..", len(profilesToDump))
 	dumper := NewProfileDumper(framework.LocalPyroscopeBaseURL)
-	profilePath, err := dumper.MemoryProfile(&ProfileDumperConfig{
-		ServiceName: "chainlink-node",
-	})
-	if err != nil {
-		errs = append(errs, fmt.Errorf("failed to download Pyroscopt profile: %w", err))
-		return errors.Join(errs...)
+
+	for _, profileType := range profilesToDump {
+		profileSplit := strings.Split(profileType, ":")
+		outputPath := DefaultOutputPath
+		if len(profileSplit) > 1 {
+			// e.g. for "memory:inuse_space:bytes:space:bytes" we want to have output file "memory-inuse_space.pprof"
+			outputPath = fmt.Sprintf("%s-%s.pprof", profileSplit[0], profileSplit[1])
+		}
+		profilePath, err := dumper.MemoryProfile(&ProfileDumperConfig{
+			ServiceName: "chainlink-node",
+			ProfileType: profileType,
+			OutputPath:  outputPath,
+		})
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to download Pyroscope profile %s: %w", profileType, err))
+			return errors.Join(errs...)
+		}
+		framework.L.Info().Str("Path", profilePath).Str("ProfileType", profileType).Msg("Saved pprof profile")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), DefaultNodeProfileDumpTimeout)
+	defer cancel()
+	if err := DumpNodeProfiles(ctx, cd.nodesetName, DefaultAdminProfilesDir); err != nil {
+		framework.L.Error().Err(err).Msg("Failed to dump node profiles")
+		errs = append(errs, fmt.Errorf("failed to dump node profiles: %w", err))
 	}
-	framework.L.Info().Str("Path", profilePath).Msg("Saved pprof profile")
+
 	return errors.Join(errs...)
 }

framework/leak/node_dumper.go

@@ -0,0 +1,140 @@
+package leak
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	f "github.com/smartcontractkit/chainlink-testing-framework/framework"
+)
+
+var containerNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]`)
+
+const (
+	DefaultAdminProfilesDir       = "admin-profiles"
+	DefaultNodeProfileDumpTimeout = 5 * time.Minute
+)
+
+// DumpNodeProfiles runs chainlink profile collection in each running container
+// with a name containing namePattern and copies ./profiles content to dst/profile-<container-name>.
+func DumpNodeProfiles(ctx context.Context, namePattern, dst string) error {
+	f.L.Info().
+		Str("NamePattern", namePattern).
+		Str("DestinationDir", dst).
+		Msg("Dumping node profiles by container name pattern")
+
+	if strings.TrimSpace(namePattern) == "" {
+		return fmt.Errorf("container name pattern must not be empty")
+	}
+	if strings.TrimSpace(dst) == "" {
+		return fmt.Errorf("destination path must not be empty")
+	}
+
+	if err := os.MkdirAll(dst, 0o755); err != nil {
+		return fmt.Errorf("failed to create destination directory %q: %w", dst, err)
+	}
+
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return fmt.Errorf("failed to create Docker client: %w", err)
+	}
+	defer cli.Close()
+	dc, err := f.NewDockerClient()
+	if err != nil {
+		return fmt.Errorf("failed to create framework docker client: %w", err)
+	}
+
+	containers, err := runningContainers(ctx, cli)
+	if err != nil {
+		return err
+	}
+
+	var errs []error
+	for _, c := range containers {
+		if !strings.Contains(c.name, namePattern) {
+			continue
+		}
+
+		// Keep destination names safe and filesystem-friendly.
+		safeName := containerNameSanitizer.ReplaceAllString(c.name, "_")
+		targetDir := filepath.Join(dst, fmt.Sprintf("profile-%s", safeName))
+		if err := os.MkdirAll(targetDir, 0o755); err != nil {
+			errs = append(errs, fmt.Errorf("failed to create destination directory for %s: %w", c.name, err))
+			continue
+		}
+
+		f.L.Info().Str("ContainerName", c.name).Msg("Collecting node profile")
+
+		out, execErr := dc.ExecContainerWithContext(
+			ctx,
+			c.name,
+			[]string{"chainlink", "admin", "profile", "-seconds", "1", "-output_dir", "./profiles"},
+		)
+		if execErr != nil {
+			errs = append(errs, fmt.Errorf("failed to execute profile command in container %s: %w, output: %s", c.name, execErr, strings.TrimSpace(out)))
+			continue
+		}
+
+		profilesPath := path.Clean(path.Join(c.workingDir, "profiles"))
+		if copyErr := dc.CopyFromContainerToHostWithContext(ctx, c.name, profilesPath, targetDir); copyErr != nil {
+			errs = append(errs, fmt.Errorf("failed to copy profiles from container %s to %s: %w", c.name, targetDir, copyErr))
+			continue
+		}
+
+		f.L.Info().Str("ContainerName", c.name).Str("Destination", targetDir).Msg("Profiles copied")
+	}
+
+	return errors.Join(errs...)
+}
+
+type runningContainer struct {
+	name       string
+	workingDir string
+}
+
+func runningContainers(ctx context.Context, cli *client.Client) ([]runningContainer, error) {
+	containers, err := cli.ContainerList(ctx, container.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list running Docker containers: %w", err)
+	}
+
+	res := make([]runningContainer, 0, len(containers))
+	for _, c := range containers {
+		name := firstContainerName(c.Names)
+		if name == "" {
+			continue
+		}
+
+		inspect, inspectErr := cli.ContainerInspect(ctx, c.ID)
+		if inspectErr != nil {
+			return nil, fmt.Errorf("failed to inspect container %s: %w", name, inspectErr)
+		}
+		workingDir := "/"
+		if inspect.Config != nil && inspect.Config.WorkingDir != "" {
+			workingDir = inspect.Config.WorkingDir
+		}
+		res = append(res, runningContainer{
+			name:       name,
+			workingDir: workingDir,
+		})
+	}
+	return res, nil
+}
+
+func firstContainerName(names []string) string {
+	for _, n := range names {
+		if n == "" {
+			continue
+		}
+		return strings.TrimPrefix(n, "/")
+	}
+	return ""
+}