add field bytes_added_packed in status file etc. (#519)

Author: jeinwag

monitor/prom/backup.go

@@ -13,18 +13,19 @@ const (
 )
 
 type BackupMetrics struct {
-	duration        *prometheus.GaugeVec
-	filesNew        *prometheus.GaugeVec
-	filesChanged    *prometheus.GaugeVec
-	filesUnmodified *prometheus.GaugeVec
-	dirNew          *prometheus.GaugeVec
-	dirChanged      *prometheus.GaugeVec
-	dirUnmodified   *prometheus.GaugeVec
-	filesTotal      *prometheus.GaugeVec
-	bytesAdded      *prometheus.GaugeVec
-	bytesTotal      *prometheus.GaugeVec
-	status          *prometheus.GaugeVec
-	time            *prometheus.GaugeVec
+	duration         *prometheus.GaugeVec
+	filesNew         *prometheus.GaugeVec
+	filesChanged     *prometheus.GaugeVec
+	filesUnmodified  *prometheus.GaugeVec
+	dirNew           *prometheus.GaugeVec
+	dirChanged       *prometheus.GaugeVec
+	dirUnmodified    *prometheus.GaugeVec
+	filesTotal       *prometheus.GaugeVec
+	bytesAdded       *prometheus.GaugeVec
+	bytesAddedPacked *prometheus.GaugeVec
+	bytesTotal       *prometheus.GaugeVec
+	status           *prometheus.GaugeVec
+	time             *prometheus.GaugeVec
 }
 
 func newBackupMetrics(labels []string) BackupMetrics {
@@ -83,6 +84,12 @@ func newBackupMetrics(labels []string) BackupMetrics {
 			Name:      "added_bytes",
 			Help:      "Total number of bytes added to the repository.",
 		}, labels),
+		bytesAddedPacked: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: backup,
+			Name:      "added_bytes_packed",
+			Help:      "Total number of bytes added to the repository after compression.",
+		}, labels),
 		bytesTotal: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: backup,

monitor/prom/metrics.go

@@ -52,8 +52,8 @@ func NewMetrics(profile, group, version string, resticversion string, configLabe
 	p.info.With(mergeLabels(cloneLabels(labels), map[string]string{goVersionLabel: runtime.Version(), versionLabel: version})).Set(1)
 
 	p.resticInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Name:      "restic_build_info",
-		Help:      "restic build information.",
+		Name: "restic_build_info",
+		Help: "restic build information.",
 	}, append(keys, versionLabel))
 	// send the information about the build right away
 	p.resticInfo.With(mergeLabels(cloneLabels(labels), map[string]string{versionLabel: resticversion})).Set(1)
@@ -72,6 +72,7 @@ func NewMetrics(profile, group, version string, resticversion string, configLabe
 		p.backup.dirUnmodified,
 		p.backup.filesTotal,
 		p.backup.bytesAdded,
+		p.backup.bytesAddedPacked,
 		p.backup.bytesTotal,
 		p.backup.status,
 		p.backup.time,
@@ -92,6 +93,7 @@ func (p *Metrics) BackupResults(status Status, summary monitor.Summary) {
 
 	p.backup.filesTotal.With(p.labels).Set(float64(summary.FilesTotal))
 	p.backup.bytesAdded.With(p.labels).Set(float64(summary.BytesAdded))
+	p.backup.bytesAddedPacked.With(p.labels).Set(float64(summary.BytesAddedPacked))
 	p.backup.bytesTotal.With(p.labels).Set(float64(summary.BytesTotal))
 	p.backup.status.With(p.labels).Set(float64(status))
 	p.backup.time.With(p.labels).Set(float64(time.Now().Unix()))

monitor/status/profile.go

@@ -30,15 +30,16 @@ type CommandStatus struct {
 // BackupStatus contains the last backup status
 type BackupStatus struct {
 	CommandStatus
-	FilesNew        int    `json:"files_new"`
-	FilesChanged    int    `json:"files_changed"`
-	FilesUnmodified int    `json:"files_unmodified"`
-	DirsNew         int    `json:"dirs_new"`
-	DirsChanged     int    `json:"dirs_changed"`
-	DirsUnmodified  int    `json:"dirs_unmodified"`
-	FilesTotal      int    `json:"files_total"`
-	BytesAdded      uint64 `json:"bytes_added"`
-	BytesTotal      uint64 `json:"bytes_total"`
+	FilesNew         int    `json:"files_new"`
+	FilesChanged     int    `json:"files_changed"`
+	FilesUnmodified  int    `json:"files_unmodified"`
+	DirsNew          int    `json:"dirs_new"`
+	DirsChanged      int    `json:"dirs_changed"`
+	DirsUnmodified   int    `json:"dirs_unmodified"`
+	FilesTotal       int    `json:"files_total"`
+	BytesAdded       uint64 `json:"bytes_added"`
+	BytesAddedPacked uint64 `json:"bytes_added_packed"`
+	BytesTotal       uint64 `json:"bytes_total"`
 }
 
 // BackupSuccess indicates the last backup was successful
@@ -50,15 +51,16 @@ func (p *Profile) BackupSuccess(summary monitor.Summary, stderr string) *Profile
 			Duration: int64(math.Ceil(summary.Duration.Seconds())),
 			Stderr:   stderr,
 		},
-		FilesNew:        summary.FilesNew,
-		FilesChanged:    summary.FilesChanged,
-		FilesUnmodified: summary.FilesUnmodified,
-		DirsNew:         summary.DirsNew,
-		DirsChanged:     summary.DirsChanged,
-		DirsUnmodified:  summary.DirsUnmodified,
-		FilesTotal:      summary.FilesTotal,
-		BytesAdded:      summary.BytesAdded,
-		BytesTotal:      summary.BytesTotal,
+		FilesNew:         summary.FilesNew,
+		FilesChanged:     summary.FilesChanged,
+		FilesUnmodified:  summary.FilesUnmodified,
+		DirsNew:          summary.DirsNew,
+		DirsChanged:      summary.DirsChanged,
+		DirsUnmodified:   summary.DirsUnmodified,
+		FilesTotal:       summary.FilesTotal,
+		BytesAdded:       summary.BytesAdded,
+		BytesAddedPacked: summary.BytesAddedPacked,
+		BytesTotal:       summary.BytesTotal,
 	}
 	return p
 }
@@ -73,15 +75,16 @@ func (p *Profile) BackupError(err error, summary monitor.Summary, stderr string)
 			Duration: int64(math.Ceil(summary.Duration.Seconds())),
 			Stderr:   stderr,
 		},
-		FilesNew:        0,
-		FilesChanged:    0,
-		FilesUnmodified: 0,
-		DirsNew:         0,
-		DirsChanged:     0,
-		DirsUnmodified:  0,
-		FilesTotal:      0,
-		BytesAdded:      0,
-		BytesTotal:      0,
+		FilesNew:         0,
+		FilesChanged:     0,
+		FilesUnmodified:  0,
+		DirsNew:          0,
+		DirsChanged:      0,
+		DirsUnmodified:   0,
+		FilesTotal:       0,
+		BytesAdded:       0,
+		BytesAddedPacked: 0,
+		BytesTotal:       0,
 	}
 	return p
 }

monitor/summary.go

@@ -4,17 +4,18 @@ import "time"
 
 // Summary of the profile run
 type Summary struct {
-	Duration        time.Duration
-	FilesNew        int
-	FilesChanged    int
-	FilesUnmodified int
-	DirsNew         int
-	DirsChanged     int
-	DirsUnmodified  int
-	FilesTotal      int
-	BytesAdded      uint64
-	BytesTotal      uint64
-	OutputAnalysis  OutputAnalysis
+	Duration         time.Duration
+	FilesNew         int
+	FilesChanged     int
+	FilesUnmodified  int
+	DirsNew          int
+	DirsChanged      int
+	DirsUnmodified   int
+	FilesTotal       int
+	BytesAdded       uint64
+	BytesAddedPacked uint64
+	BytesTotal       uint64
+	OutputAnalysis   OutputAnalysis
 }
 
 // OutputAnalysis of the profile run

shell/json_summary.go

@@ -21,6 +21,7 @@ type ResticJsonSummary struct {
 	DataBlobs           int     `json:"data_blobs"`
 	TreeBlobs           int     `json:"tree_blobs"`
 	DataAdded           uint64  `json:"data_added"`
+	DataAddedPacked     uint64  `json:"data_added_packed"`
 	TotalFilesProcessed int     `json:"total_files_processed"`
 	TotalBytesProcessed uint64  `json:"total_bytes_processed"`
 	TotalDuration       float64 `json:"total_duration"`
@@ -56,6 +57,7 @@ var ScanBackupJson ScanOutput = func(r io.Reader, summary *monitor.Summary, w io
 				summary.DirsUnmodified = jsonSummary.DirsUnmodified
 				summary.FilesTotal = jsonSummary.TotalFilesProcessed
 				summary.BytesAdded = jsonSummary.DataAdded
+				summary.BytesAddedPacked = jsonSummary.DataAddedPacked
 				summary.BytesTotal = jsonSummary.TotalBytesProcessed
 			}
 			continue

shell/json_summary_test.go

@@ -19,7 +19,7 @@ func TestScanJsonSummary(t *testing.T) {
 {"message_type":"status","percent_done":0.028711419769115988,"total_files":213,"files_done":13,"total_bytes":362948126,"bytes_done":10420756,"current_files":["/go/src/github.com/creativeprojects/resticprofile/build/restic","/go/src/github.com/creativeprojects/resticprofile/build/resticprofile"]}
 {"message_type":"status","percent_done":0.9763572825280271,"total_files":213,"files_done":163,"total_bytes":362948126,"bytes_done":354367046,"current_files":["/go/src/github.com/creativeprojects/resticprofile/resticprofile_darwin","/go/src/github.com/creativeprojects/resticprofile/resticprofile_linux"]}
 {"message_type":"status","seconds_elapsed":1,"percent_done":1,"total_files":213,"files_done":212,"total_bytes":362948126,"bytes_done":362948126,"current_files":["/go/src/github.com/creativeprojects/resticprofile/resticprofile_linux"]}
-{"message_type":"summary","files_new":213,"files_changed":11,"files_unmodified":12,"dirs_new":58,"dirs_changed":2,"dirs_unmodified":3,"data_blobs":402,"tree_blobs":59,"data_added":296530781,"total_files_processed":236,"total_bytes_processed":362948126,"total_duration":1.009156009,"snapshot_id":"6daa8ef6"}
+{"message_type":"summary","files_new":213,"files_changed":11,"files_unmodified":12,"dirs_new":58,"dirs_changed":2,"dirs_unmodified":3,"data_blobs":402,"tree_blobs":59,"data_added":296530781,"data_added_packed":74132695,"total_files_processed":236,"total_bytes_processed":362948126,"total_duration":1.009156009,"snapshot_id":"6daa8ef6"}
 `
 
 	if platform.IsWindows() {
@@ -64,6 +64,7 @@ func TestScanJsonSummary(t *testing.T) {
 	assert.Equal(t, 2, summary.DirsChanged)
 	assert.Equal(t, 3, summary.DirsUnmodified)
 	assert.Equal(t, uint64(296530781), summary.BytesAdded)
+	assert.Equal(t, uint64(74132695), summary.BytesAddedPacked)
 	assert.Equal(t, uint64(362948126), summary.BytesTotal)
 	assert.Equal(t, 236, summary.FilesTotal)
 }

shell/plain_summary.go

@@ -17,7 +17,7 @@ var ScanBackupPlain ScanOutput = func(r io.Reader, summary *monitor.Summary, w i
 	if runtime.GOOS == "windows" {
 		eol = "\r\n"
 	}
-	rawBytes, unit, duration := 0.0, "", ""
+	rawBytes, rawBytesStored, unit, unitStored, duration := 0.0, 0.0, "", "", ""
 	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
 		_, err := w.Write([]byte(scanner.Text() + eol))
@@ -28,9 +28,10 @@ var ScanBackupPlain ScanOutput = func(r io.Reader, summary *monitor.Summary, w i
 		_, _ = fmt.Sscanf(scanner.Text(), "Files: %d new, %d changed, %d unmodified", &summary.FilesNew, &summary.FilesChanged, &summary.FilesUnmodified)
 		_, _ = fmt.Sscanf(scanner.Text(), "Dirs: %d new, %d changed, %d unmodified", &summary.DirsNew, &summary.DirsChanged, &summary.DirsUnmodified)
 
-		n, err := fmt.Sscanf(scanner.Text(), "Added to the repo: %f %3s", &rawBytes, &unit)
-		if n == 2 && err == nil {
+		n, err := fmt.Sscanf(scanner.Text(), "Added to the repository: %f %3s (%f %3s stored)", &rawBytes, &unit, &rawBytesStored, &unitStored)
+		if n == 4 && err == nil {
 			summary.BytesAdded = unformatBytes(rawBytes, unit)
+			summary.BytesAddedPacked = unformatBytes(rawBytesStored, unitStored)
 		}
 
 		n, err = fmt.Sscanf(scanner.Text(), "processed %d files, %f %3s in %s", &summary.FilesTotal, &rawBytes, &unit, &duration)

shell/plain_summary_test.go

@@ -20,7 +20,7 @@ created new cache in /Users/home/Library/Caches/restic
 
 Files:         209 new,     2 changed,    12 unmodified
 Dirs:           58 new,     1 changed,    11 unmodified
-Added to the repo: 282.768 MiB
+Added to the repository: 282.768 MiB (70.691 MiB stored)
 
 processed 223 files, 346.107 MiB in 0:00
 snapshot 07ab30a5 saved
@@ -61,6 +61,7 @@ snapshot 07ab30a5 saved
 	assert.Equal(t, 1, summary.DirsChanged)
 	assert.Equal(t, 11, summary.DirsUnmodified)
 	assert.Equal(t, uint64(296503738), summary.BytesAdded)
+	assert.Equal(t, uint64(74124886), summary.BytesAddedPacked)
 	assert.Equal(t, uint64(362919494), summary.BytesTotal)
 	assert.Equal(t, 223, summary.FilesTotal)
 }