-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable memory monitoring in CS (#391)
* Implement memory monitoring * Adding image tests for memory monitoring
- Loading branch information
1 parent
8519135
commit 38bab91
Showing
24 changed files
with
619 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
launcher/image/nodeproblemdetector/boot-disk-size-consistency-monitor-cs.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "plugin": "custom", | ||
| "pluginConfig": { | ||
| "invoke_interval": "30m", | ||
| "timeout": "7s", | ||
| "max_output_length": 80, | ||
| "enable_message_change_based_condition_update": false | ||
| }, | ||
| "source": "boot-disk-size-consistency-monitor", | ||
| "metricsReporting": false, | ||
| "rules": [] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "plugin": "journald", | ||
| "pluginConfig": { | ||
| "source": "dockerd" | ||
| }, | ||
| "logPath": "/var/log/journal", | ||
| "lookback": "5m", | ||
| "bufferSize": 10, | ||
| "source": "docker-monitor", | ||
| "metricsReporting": false, | ||
| "conditions": [] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "plugin": "kmsg", | ||
| "logPath": "/dev/kmsg", | ||
| "lookback": "5m", | ||
| "bufferSize": 10, | ||
| "source": "kernel-monitor", | ||
| "metricsReporting": false, | ||
| "conditions": [], | ||
| "rules": [] | ||
| } |
10 changes: 10 additions & 0 deletions
10
launcher/image/nodeproblemdetector/system-stats-monitor-cs.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "memory": { | ||
| "metricsConfigs": { | ||
| "memory/bytes_used": { | ||
| "displayName": "memory/bytes_used" | ||
| } | ||
| } | ||
| }, | ||
| "invokeInterval": "60s" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
launcher/image/test/scripts/test_memory_monitoring_enabled.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| #!/bin/bash | ||
| set -euxo pipefail | ||
| source util/read_serial.sh | ||
|
|
||
| # This test requires the workload to run and printing | ||
| # corresponding messages to the serial console. | ||
| SERIAL_OUTPUT=$(read_serial $1 $2) | ||
| print_serial=false | ||
|
|
||
| if echo $SERIAL_OUTPUT | grep -q 'node-problem-detector.service successfully started' | ||
| then | ||
| echo "- memory monitoring enabled" | ||
| else | ||
| echo "FAILED: memory monitoring disabled" | ||
| echo 'TEST FAILED.' > /workspace/status.txt | ||
| print_serial=true | ||
| fi | ||
|
|
||
| if $print_serial; then | ||
| echo $SERIAL_OUTPUT | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| substitutions: | ||
| '_IMAGE_NAME': '' | ||
| '_IMAGE_PROJECT': '' | ||
| '_CLEANUP': 'true' | ||
| '_VM_NAME_PREFIX': 'memory-monitoring' | ||
| '_ZONE': 'us-east1-b' | ||
| '_WORKLOAD_IMAGE': 'us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/memorymonitoring:latest' | ||
|
|
||
| steps: | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CreateVM | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['create_vm.sh','-i', '${_IMAGE_NAME}', | ||
| '-p', '${_IMAGE_PROJECT}', | ||
| '-m', 'tee-image-reference=${_WORKLOAD_IMAGE},tee-monitoring-memory-enable=true', | ||
| '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}', | ||
| '-z', '${_ZONE}', | ||
| ] | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CheckMemoryMonitoringEnabled | ||
| entrypoint: 'bash' | ||
| args: ['scripts/test_memory_monitoring_enabled.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}', '${_ZONE}'] | ||
|
|
||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CleanUp | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'CLEANUP=$_CLEANUP' | ||
| args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}', '${_ZONE}'] | ||
| # Must come after cleanup. | ||
| - name: 'gcr.io/cloud-builders/gcloud' | ||
| id: CheckFailure | ||
| entrypoint: 'bash' | ||
| env: | ||
| - 'BUILD_ID=$BUILD_ID' | ||
| args: ['check_failure.sh'] | ||
|
|
||
| options: | ||
| pool: | ||
| name: 'projects/confidential-space-images-dev/locations/us-west1/workerPools/cs-image-build-vpc' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| # From current directory: | ||
| # GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o main ../basic | ||
| # gcloud builds submit --tag us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/memorymonitoring:latest --project confidential-space-images-dev | ||
| FROM alpine | ||
|
|
||
| COPY main / | ||
|
|
||
| ENV env_bar="val_bar" | ||
|
|
||
| LABEL "tee.launch_policy.monitoring_memory_allow"="always" | ||
|
|
||
| ENTRYPOINT ["/main"] | ||
|
|
||
| CMD ["arg_foo"] |
55 changes: 55 additions & 0 deletions
55
launcher/internal/healthmonitoring/nodeproblemdetector/systemstats_config.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| // Package nodeproblemdetector provides configurations for node-problem-detector.service. | ||
| package nodeproblemdetector | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "time" | ||
| ) | ||
|
|
||
| var defaultInvokeIntervalString = (60 * time.Second).String() | ||
|
|
||
| type metricConfig struct { | ||
| DisplayName string `json:"displayName"` | ||
| } | ||
|
|
||
| type memoryStatsConfig struct { | ||
| MetricsConfigs map[string]metricConfig `json:"metricsConfigs"` | ||
| } | ||
|
|
||
| // SystemStatsConfig contains configurations for `System Stats Monitor`, | ||
| // a problem daemon in node-problem-detector that collects pre-defined health-related metrics from different system components. | ||
| // For now we only consider collecting memory related metrics. | ||
| // View the comprehensive configuration details on https://github.com/kubernetes/node-problem-detector/tree/master/pkg/systemstatsmonitor#detailed-configuration-options | ||
| type SystemStatsConfig struct { | ||
| MemoryStatsConfig memoryStatsConfig `json:"memory"` | ||
| InvokeInterval string `json:"invokeInterval"` | ||
| } | ||
|
|
||
| // NewSystemStatsConfig returns a new SystemStatsConfig struct with default configurations. | ||
| func NewSystemStatsConfig() SystemStatsConfig { | ||
| return SystemStatsConfig{ | ||
| MemoryStatsConfig: memoryStatsConfig{MetricsConfigs: map[string]metricConfig{}}, | ||
| InvokeInterval: defaultInvokeIntervalString, | ||
| } | ||
| } | ||
|
|
||
| // EnableMemoryBytesUsed enables "memory/bytes_used" for memory monitoring. | ||
| func (ssc *SystemStatsConfig) EnableMemoryBytesUsed() { | ||
| ssc.MemoryStatsConfig.MetricsConfigs["memory/bytes_used"] = metricConfig{DisplayName: "memory/bytes_used"} | ||
| } | ||
|
|
||
| // WithInvokeInterval overrides the default invokeInterval. | ||
| func (ssc *SystemStatsConfig) WithInvokeInterval(interval time.Duration) { | ||
| ssc.InvokeInterval = interval.String() | ||
| } | ||
|
|
||
| // WriteFile writes systemStatsConfig data to the named file, creating it if necessary. | ||
| func (ssc *SystemStatsConfig) WriteFile(path string) error { | ||
| bytes, err := json.Marshal(ssc) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to marshal struct [%v]: %w", ssc, err) | ||
| } | ||
| return os.WriteFile(path, bytes, 0644) | ||
| } |
Oops, something went wrong.