Support health monitoring mode for NPD

launcher/container_runner.go

@@ -30,7 +30,6 @@ import (
 	"github.com/google/go-tpm-tools/client"
 	"github.com/google/go-tpm-tools/launcher/agent"
 	"github.com/google/go-tpm-tools/launcher/internal/signaturediscovery"
-	"github.com/google/go-tpm-tools/launcher/internal/systemctl"
 	"github.com/google/go-tpm-tools/launcher/launcherfile"
 	"github.com/google/go-tpm-tools/launcher/registryauth"
 	"github.com/google/go-tpm-tools/launcher/spec"
@@ -115,7 +114,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 	}
 
 	logger.Printf("Image Labels               : %v\n", imageConfig.Labels)
-	launchPolicy, err := spec.GetLaunchPolicy(imageConfig.Labels)
+	launchPolicy, err := spec.GetLaunchPolicy(imageConfig.Labels, logger)
 	if err != nil {
 		return nil, err
 	}
@@ -341,7 +340,7 @@ func (r *ContainerRunner) measureContainerClaims(ctx context.Context) error {
 // eventlog in the AttestationAgent.
 func (r *ContainerRunner) measureMemoryMonitor() error {
 	var enabled uint8
-	if r.launchSpec.MemoryMonitoringEnabled {
+	if r.launchSpec.MonitoringEnabled == spec.MemoryOnly {
 		enabled = 1
 	}
 	if err := r.attestAgent.MeasureEvent(cel.CosTlv{EventType: cel.MemoryMonitorType, EventContent: []byte{enabled}}); err != nil {
@@ -521,22 +520,9 @@ func (r *ContainerRunner) Run(ctx context.Context) error {
 	go teeServer.Serve()
 	defer teeServer.Shutdown(ctx)
 
-	// start node-problem-detector.service to collect memory related metrics.
-	if r.launchSpec.MemoryMonitoringEnabled {
-		r.logger.Println("MemoryMonitoring is enabled by the VM operator")
-		s, err := systemctl.New()
-		if err != nil {
-			return fmt.Errorf("failed to create systemctl client: %v", err)
-		}
-		defer s.Close()
-
-		r.logger.Println("Starting a systemctl operation: systemctl start node-problem-detector.service")
-		if err := s.Start("node-problem-detector.service"); err != nil {
-			return fmt.Errorf("failed to start node-problem-detector.service: %v", err)
-		}
-		r.logger.Println("node-problem-detector.service successfully started.")
-	} else {
-		r.logger.Println("MemoryMonitoring is disabled by the VM operator")
+	// Avoids breaking existing memory monitoring tests that depend on this log.
+	if r.launchSpec.MonitoringEnabled == spec.None {
+		r.logger.Printf("MemoryMonitoring is disabled by the VM operator")
 	}
 
 	var streamOpt cio.Opt

launcher/image/test/test_memory_monitoring.yaml

@@ -23,7 +23,7 @@ steps:
   id: CheckMemoryMonitoringEnabled
   entrypoint: 'bash'
   # Search a regex pattern that ensures memory monitoring is enabled and measured into COS event logs.
-  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-enable-${BUILD_ID}', '${_ZONE}', 'Successfully measured memory monitoring event.*node-problem-detector.service successfully started']
+  args: ['scripts/test_memory_monitoring.sh', '${_VM_NAME_PREFIX}-enable-${BUILD_ID}', '${_ZONE}', 'node-problem-detector.service successfully started.*Successfully measured memory monitoring event']
   waitFor: ['CreateVMMemoryMemonitorEnabled']
 - name: 'gcr.io/cloud-builders/gcloud'
   id: CleanUpVMMemoryMonitorEnabled

launcher/image/testworkloads/memorymonitoring/Dockerfile

@@ -7,7 +7,8 @@ COPY main /
 
 ENV env_bar="val_bar"
 
-LABEL "tee.launch_policy.monitoring_memory_allow"="always"
+LABEL "tee.launch_policy.hardened_monitoring"="MEMORY_ONLY"
+LABEL "tee.launch_policy.debug_monitoring"="NONE"
 
 ENTRYPOINT ["/main"]
 

launcher/image/testworkloads/memorymonitoringdebug/Dockerfile

@@ -7,7 +7,8 @@ COPY main /
 
 ENV env_bar="val_bar"
 
-LABEL "tee.launch_policy.monitoring_memory_allow"="debugonly"
+LABEL "tee.launch_policy.hardened_monitoring"="NONE"
+LABEL "tee.launch_policy.debug_monitoring"="MEMORY_ONLY"
 
 ENTRYPOINT ["/main"]
 

launcher/image/testworkloads/memorymonitoringnever/Dockerfile

@@ -7,8 +7,8 @@ COPY main /
 
 ENV env_bar="val_bar"
 
-LABEL "tee.launch_policy.monitoring_memory_allow"="never"
-
+LABEL "tee.launch_policy.hardened_monitoring"="NONE"
+LABEL "tee.launch_policy.debug_monitoring"="NONE"
 ENTRYPOINT ["/main"]
 
 CMD ["arg_foo"]

launcher/internal/healthmonitoring/nodeproblemdetector/systemstats_config.go

@@ -4,40 +4,101 @@ package nodeproblemdetector
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"time"
+
+	"github.com/google/go-tpm-tools/launcher/internal/systemctl"
 )
 
+var systemStatsFilePath = "/etc/node_problem_detector/system-stats-monitor.json"
+
 var defaultInvokeIntervalString = (60 * time.Second).String()
 
 type metricConfig struct {
 	DisplayName string `json:"displayName"`
 }
 
-type memoryStatsConfig struct {
+type statsConfig struct {
 	MetricsConfigs map[string]metricConfig `json:"metricsConfigs"`
 }
 
+type diskConfig struct {
+	IncludeAllAttachedBlk bool         `json:"includeAllAttachedBlk"`
+	IncludeRootBlk        bool         `json:"includeRootBlk"`
+	LsblkTimeout          string       `json:"lsblkTimeout"`
+	MetricsConfigs        *statsConfig `json:"metricsConfigs"`
+}
+
 // SystemStatsConfig contains configurations for `System Stats Monitor`,
 // a problem daemon in node-problem-detector that collects pre-defined health-related metrics from different system components.
 // For now we only consider collecting memory related metrics.
 // View the comprehensive configuration details on https://github.com/kubernetes/node-problem-detector/tree/master/pkg/systemstatsmonitor#detailed-configuration-options
 type SystemStatsConfig struct {
-	MemoryStatsConfig memoryStatsConfig `json:"memory"`
-	InvokeInterval    string            `json:"invokeInterval"`
+	CPU            *statsConfig `json:"cpu,omitempty"`
+	Disk           *diskConfig  `json:"disk,omitempty"`
+	Host           *statsConfig `json:"host,omitempty"`
+	Memory         *statsConfig `json:"memory,omitempty"`
+	InvokeInterval string       `json:"invokeInterval,omitempty"`
 }
 
 // NewSystemStatsConfig returns a new SystemStatsConfig struct with default configurations.
 func NewSystemStatsConfig() SystemStatsConfig {
 	return SystemStatsConfig{
-		MemoryStatsConfig: memoryStatsConfig{MetricsConfigs: map[string]metricConfig{}},
-		InvokeInterval:    defaultInvokeIntervalString,
+		Memory:         &statsConfig{MetricsConfigs: map[string]metricConfig{}},
+		InvokeInterval: defaultInvokeIntervalString,
 	}
 }
 
+var allConfig = &SystemStatsConfig{
+	CPU: &statsConfig{map[string]metricConfig{
+		"cpu/runnable_task_count": {"cpu/runnable_task_count"},
+		"cpu/usage_time":          {"cpu/usage_time"},
+		"cpu/load_1m":             {"cpu/load_1m"},
+		"cpu/load_5m":             {"cpu/load_5m"},
+		"cpu/load_15m":            {"cpu/load_15m"},
+		"system/cpu_stat":         {"system/cpu_stat"},
+		"system/interrupts_total": {"system/interrupts_total"},
+		"system/processes_total":  {"system/processes_total"},
+		"system/procs_blocked":    {"system/procs_blocked"},
+		"system/procs_running":    {"system/procs_running"},
+	}},
+	Disk: &diskConfig{
+		true, true, "5s",
+		&statsConfig{map[string]metricConfig{
+			"disk/avg_queue_len":          {"disk/avg_queue_len"},
+			"disk/bytes_used":             {"disk/bytes_used"},
+			"disk/percent_used":           {"disk/percent_used"},
+			"disk/io_time":                {"disk/io_time"},
+			"disk/merged_operation_count": {"disk/merged_operation_count"},
+			"disk/operation_bytes_count":  {"disk/operation_bytes_count"},
+			"disk/operation_count":        {"disk/operation_count"},
+			"disk/operation_time":         {"disk/operation_time"},
+			"disk/weighted_io":            {"disk/weighted_io"},
+		}},
+	},
+	Host: &statsConfig{map[string]metricConfig{
+		"host/uptime": {"host/uptime"},
+	}},
+	Memory: &statsConfig{map[string]metricConfig{
+		"memory/anonymous_used":   {"memory/anonymous_used"},
+		"memory/bytes_used":       {"memory/bytes_used"},
+		"memory/dirty_used":       {"memory/dirty_used"},
+		"memory/page_cache_used":  {"memory/page_cache_used"},
+		"memory/unevictable_used": {"memory/unevictable_used"},
+		"memory/percent_used":     {"memory/percent_used"},
+	}},
+	InvokeInterval: defaultInvokeIntervalString,
+}
+
+// EnableAllConfig overwrites system stats config with health monitoring config.
+func EnableAllConfig() error {
+	return allConfig.WriteFile(systemStatsFilePath)
+}
+
 // EnableMemoryBytesUsed enables "memory/bytes_used" for memory monitoring.
 func (ssc *SystemStatsConfig) EnableMemoryBytesUsed() {
-	ssc.MemoryStatsConfig.MetricsConfigs["memory/bytes_used"] = metricConfig{DisplayName: "memory/bytes_used"}
+	ssc.Memory.MetricsConfigs["memory/bytes_used"] = metricConfig{DisplayName: "memory/bytes_used"}
 }
 
 // WithInvokeInterval overrides the default invokeInterval.
@@ -53,3 +114,20 @@ func (ssc *SystemStatsConfig) WriteFile(path string) error {
 	}
 	return os.WriteFile(path, bytes, 0644)
 }
+
+// StartService starts Node Problem Detector.
+func StartService(logger *log.Logger) error {
+	s, err := systemctl.New()
+	if err != nil {
+		return fmt.Errorf("failed to create systemctl client: %v", err)
+	}
+	defer s.Close()
+
+	logger.Printf("Starting node-problem-detector.service")
+	if err := s.Start("node-problem-detector.service"); err != nil {
+		return fmt.Errorf("failed to start node-problem-detector.service")
+	}
+
+	logger.Printf("node-problem-detector.service successfully started")
+	return nil
+}

launcher/internal/healthmonitoring/nodeproblemdetector/systemstats_config_test.go

@@ -2,6 +2,7 @@ package nodeproblemdetector
 
 import (
 	"bytes"
+	"encoding/json"
 	"io"
 	"os"
 	"path"
@@ -11,12 +12,38 @@ import (
 	"github.com/google/go-cmp/cmp"
 )
 
+func TestEnableHealthMonitoringConfig(t *testing.T) {
+	tmpDir := t.TempDir()
+	systemStatsFilePath = path.Join(tmpDir, "system-stats-monitor.json")
+
+	wantBytes, err := json.Marshal(allConfig)
+	if err != nil {
+		t.Fatalf("Error marshaling expected config: %v", err)
+	}
+
+	EnableAllConfig()
+
+	file, err := os.OpenFile(systemStatsFilePath, os.O_RDONLY, 0)
+	if err != nil {
+		t.Fatalf("failed to open file %s: %v", systemStatsFilePath, err)
+	}
+
+	gotBytes, err := io.ReadAll(file)
+	if err != nil {
+		t.Fatalf("failed to read from file %s: %v", systemStatsFilePath, err)
+	}
+
+	if !bytes.Equal(gotBytes, wantBytes) {
+		t.Errorf("WriteFile() did not write expected contents, got %s, want %s", gotBytes, wantBytes)
+	}
+}
+
 func TestEnableMemoryBytesUsed(t *testing.T) {
 	got := NewSystemStatsConfig()
 	got.EnableMemoryBytesUsed()
 
 	want := SystemStatsConfig{
-		MemoryStatsConfig: memoryStatsConfig{
+		Memory: &statsConfig{
 			MetricsConfigs: map[string]metricConfig{
 				"memory/bytes_used": {DisplayName: "memory/bytes_used"},
 			},

launcher/launcher/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/google/go-tpm-tools/client"
 	"github.com/google/go-tpm-tools/launcher"
+	"github.com/google/go-tpm-tools/launcher/internal/healthmonitoring/nodeproblemdetector"
 	"github.com/google/go-tpm-tools/launcher/launcherfile"
 	"github.com/google/go-tpm-tools/launcher/registryauth"
 	"github.com/google/go-tpm-tools/launcher/spec"
@@ -95,6 +96,26 @@ func main() {
 		return
 	}
 
+	if launchSpec.MonitoringEnabled != spec.None {
+		logger.Printf("Health Monitoring is enabled by the VM operator")
+
+		if launchSpec.MonitoringEnabled == spec.All {
+			logger.Printf("All health monitoring metrics enabled")
+			if err := nodeproblemdetector.EnableAllConfig(); err != nil {
+				logger.Printf("failed to enable full monitoring config: %v", err)
+				return
+			}
+		} else if launchSpec.MonitoringEnabled == spec.MemoryOnly {
+			logger.Printf("memory/bytes_used enabled")
+		}
+
+		if err := nodeproblemdetector.StartService(logger); err != nil {
+			logger.Print(err)
+		}
+	} else {
+		logger.Printf("Health Monitoring is disabled")
+	}
+
 	defer func() {
 		// Catch panic to attempt to output to Cloud Logging.
 		if r := recover(); r != nil {