Skip to content

Commit bc8836c

Browse files
joaopapereirajoaopapereira
committed
Add new reporting interface that can be used to create progress tracker
Signed-off-by: João Pereira <joao.pereira@broadcom.com>
1 parent 89b7bf1 commit bc8836c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+6150
-21
lines changed

‎go.mod‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ require (
88
github.com/chrismellard/docker-credential-acr-env v0.0.0-20220327082430-c57b701bfc08
99
github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65
1010
github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14
11+
github.com/docker/go-units v0.5.0
1112
github.com/fatih/color v1.18.0 // indirect
1213
github.com/google/go-containerregistry v0.20.3
1314
github.com/mattn/go-isatty v0.0.20
1415
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
16+
github.com/moby/term v0.0.0-20221205130635-1aeaba878587
17+
github.com/morikuni/aec v1.0.0
1518
github.com/spf13/cobra v1.9.1
1619
github.com/stretchr/testify v1.10.0
1720
golang.org/x/sync v0.12.0
@@ -22,6 +25,7 @@ require (
2225
require (
2326
cloud.google.com/go v0.99.0 // indirect
2427
github.com/Azure/azure-sdk-for-go v55.0.0+incompatible // indirect
28+
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
2529
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
2630
github.com/Azure/go-autorest/autorest v0.11.18 // indirect
2731
github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
@@ -48,7 +52,6 @@ require (
4852
github.com/aws/smithy-go v1.22.2 // indirect
4953
github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
5054
github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835 // indirect
51-
github.com/creack/pty v1.1.11 // indirect
5255
github.com/davecgh/go-spew v1.1.1 // indirect
5356
github.com/dimchansky/utfbom v1.1.0 // indirect
5457
github.com/docker/cli v27.5.0+incompatible // indirect

‎go.sum‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
4747
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
4848
github.com/Azure/azure-sdk-for-go v55.0.0+incompatible h1:L4/vUGbg1Xkw5L20LZD+hJI5I+ibWSytqQ68lTCfLwY=
4949
github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
50+
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
51+
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
5052
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
5153
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
5254
github.com/Azure/go-autorest/autorest v0.11.6/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs=
@@ -130,8 +132,8 @@ github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 h1:MjRdR01xh0
130132
github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14/go.mod h1:AlgTssDlstr4mf92TR4DPITLfl5+7wEY4cKStCmeeto=
131133
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
132134
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
133-
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
134-
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
135+
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
136+
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
135137
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
136138
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
137139
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -144,6 +146,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi
144146
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
145147
github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
146148
github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
149+
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
150+
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
147151
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
148152
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
149153
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -271,6 +275,10 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlz
271275
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ=
272276
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
273277
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
278+
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
279+
github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
280+
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
281+
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
274282
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
275283
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
276284
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

‎pkg/imgpkg/cmd/copy.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func (c *CopyOptions) Run() error {
119119
}
120120

121121
imageSet := ctlimgset.NewImageSet(c.Concurrency, levelLogger, tagGen)
122-
tarImageSet := ctlimgset.NewTarImageSet(imageSet, c.Concurrency, levelLogger)
122+
tarImageSet := ctlimgset.NewTarImageSet(imageSet, c.Concurrency, levelLogger, &v1.NoopProgressReporter{})
123123

124124
var signatureRetriever v1.SignatureFetcher
125125
if c.SignatureFlags.CopyCosignSignatures {

‎pkg/imgpkg/imageset/tar_image_set.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,15 @@ import (
1818
)
1919

2020
type TarImageSet struct {
21-
imageSet ImageSet
22-
concurrency int
23-
logger Logger
21+
imageSet ImageSet
22+
concurrency int
23+
logger Logger
24+
progressReporter imagetar.ProgressReporter
2425
}
2526

2627
// NewTarImageSet provides export/import operations on a tarball for a set of images
27-
func NewTarImageSet(imageSet ImageSet, concurrency int, logger Logger) TarImageSet {
28-
return TarImageSet{imageSet, concurrency, logger}
28+
func NewTarImageSet(imageSet ImageSet, concurrency int, logger Logger, report imagetar.ProgressReporter) TarImageSet {
29+
return TarImageSet{imageSet: imageSet, concurrency: concurrency, logger: logger, progressReporter: report}
2930
}
3031

3132
// Export Creates a Tar with the provided Images
@@ -105,7 +106,7 @@ func (i *TarImageSet) Export(foundImages *UnprocessedImageRefs, outputPath strin
105106

106107
opts := imagetar.TarWriterOpts{Concurrency: i.concurrency}
107108

108-
err = imagetar.NewTarWriter(ids, outputFileOpener, opts, i.logger, imageLayerWriterCheck, alreadyDownloadedLayers).Write()
109+
err = imagetar.NewTarWriter(ids, outputFileOpener, opts, i.logger, imageLayerWriterCheck, alreadyDownloadedLayers, i.progressReporter).Write()
109110
return ids, err
110111
}
111112

‎pkg/imgpkg/imagetar/tar_writer.go‎

Lines changed: 73 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@ type Logger interface {
2323
Logf(str string, args ...interface{})
2424
}
2525

26+
// ProgressReporter used to report the current status of the read/write
27+
type ProgressReporter interface {
28+
StartReporting(id string, total int64) error
29+
Report(id string, completed int64, total int64, err error) error
30+
Finish(id string, total int64) error
31+
ActiveReporter() bool
32+
}
33+
2634
type TarWriterOpts struct {
2735
Concurrency int
2836
}
@@ -31,9 +39,10 @@ type TarWriter struct {
3139
ids *imagedesc.ImageRefDescriptors
3240
dstOpener func() (io.WriteCloser, error)
3341

34-
dst io.WriteCloser
35-
tf *tar.Writer
36-
layersToWrite []imagedesc.ImageLayerDescriptor
42+
dst io.WriteCloser
43+
tf *tar.Writer
44+
layersToWrite []imagedesc.ImageLayerDescriptor
45+
progressReporter ProgressReporter
3746

3847
opts TarWriterOpts
3948
logger Logger
@@ -44,7 +53,7 @@ type TarWriter struct {
4453
// NewTarWriter constructor returning a mechanism to write image refs / layers to a tarball on disk.
4554
func NewTarWriter(ids *imagedesc.ImageRefDescriptors, dstOpener func() (io.WriteCloser, error),
4655
opts TarWriterOpts, logger Logger, imageLayerWriterCheck ImageLayerWriterFilter,
47-
layersFromOtherSource []regv1.Layer) *TarWriter {
56+
layersFromOtherSource []regv1.Layer, reporter ProgressReporter) *TarWriter {
4857
knownlayers := map[string]regv1.Layer{}
4958
for _, layer := range layersFromOtherSource {
5059
d, err := layer.Digest()
@@ -53,13 +62,17 @@ func NewTarWriter(ids *imagedesc.ImageRefDescriptors, dstOpener func() (io.Write
5362
}
5463
knownlayers[d.String()] = layer
5564
}
65+
if reporter == nil {
66+
panic(fmt.Sprintf("Internal inconsistency: A Progress Reported need to be provided"))
67+
}
5668
return &TarWriter{
5769
ids: ids,
5870
dstOpener: dstOpener,
5971
opts: opts,
6072
logger: logger,
6173
imageLayerWriterCheck: imageLayerWriterCheck,
6274
layersFromOtherSource: knownlayers,
75+
progressReporter: reporter,
6376
}
6477
}
6578

@@ -138,6 +151,29 @@ func (w *TarWriter) writeImage(td imagedesc.ImageDescriptor) error {
138151
return nil
139152
}
140153

154+
type readerWithProgressLogger struct {
155+
reader io.Reader
156+
progressReporter ProgressReporter
157+
layerID string
158+
gotSize int64
159+
totalSize int64
160+
}
161+
162+
func (r *readerWithProgressLogger) Read(p []byte) (n int, err error) {
163+
readSize, err := r.reader.Read(p)
164+
r.gotSize += int64(readSize)
165+
if err != nil {
166+
if err == io.EOF {
167+
r.progressReporter.Finish(r.layerID, r.totalSize)
168+
return readSize, err
169+
}
170+
r.progressReporter.Report(r.layerID, r.gotSize, r.totalSize, err)
171+
return 0, err
172+
}
173+
r.progressReporter.Report(r.layerID, r.gotSize, r.totalSize, err)
174+
return readSize, nil
175+
}
176+
141177
type writtenLayer struct {
142178
Name string
143179
Offset int64
@@ -188,10 +224,19 @@ func (w *TarWriter) writeLayers() error {
188224
stream = nil
189225
} else {
190226
if sourceLayer, ok := w.layersFromOtherSource[digest.String()]; ok {
191-
stream, err = sourceLayer.Compressed()
227+
rLayer, err := sourceLayer.Compressed()
192228
if err != nil {
193229
return fmt.Errorf("failed to get compressed stuff: %s", err)
194230
}
231+
w.progressReporter.StartReporting(digest.Hex, imgLayer.Size)
232+
progressReader := &readerWithProgressLogger{
233+
reader: rLayer,
234+
progressReporter: w.progressReporter,
235+
layerID: digest.Hex,
236+
gotSize: 0,
237+
totalSize: imgLayer.Size,
238+
}
239+
stream = progressReader
195240
}
196241

197242
if stream == nil {
@@ -201,10 +246,19 @@ func (w *TarWriter) writeLayers() error {
201246
return err
202247
}
203248

204-
stream, err = foundLayer.Open()
249+
rLayer, err := foundLayer.Open()
205250
if err != nil {
206251
return err
207252
}
253+
w.progressReporter.StartReporting(digest.Hex, imgLayer.Size)
254+
progressReader := &readerWithProgressLogger{
255+
reader: rLayer,
256+
progressReporter: w.progressReporter,
257+
layerID: digest.Hex,
258+
gotSize: 0,
259+
totalSize: imgLayer.Size,
260+
}
261+
stream = progressReader
208262
} else {
209263
w.logger.Debugf("reusing layer: %s", digest.String())
210264
}
@@ -317,7 +371,16 @@ func (w *TarWriter) fillInLayer(wl writtenLayer) error {
317371
}
318372
w.logger.Tracef("took %s to prepare layer %s to be written", time.Since(startFillingLayer), wl.Layer.Digest)
319373

320-
err = w.writeTarEntry(tw, wl.Name, stream, wl.Layer.Size)
374+
w.progressReporter.StartReporting(wl.Layer.Digest, wl.Layer.Size)
375+
progressReader := &readerWithProgressLogger{
376+
reader: stream,
377+
progressReporter: w.progressReporter,
378+
layerID: wl.Layer.Digest,
379+
gotSize: 0,
380+
totalSize: wl.Layer.Size,
381+
}
382+
383+
err = w.writeTarEntry(tw, wl.Name, progressReader, wl.Layer.Size)
321384
if err != nil {
322385
return fmt.Errorf("Rewriting tar entry (%s): %s", wl.Name, err)
323386
}
@@ -353,7 +416,9 @@ func (w *TarWriter) writeTarEntry(tw *tar.Writer, path string, r io.Reader, size
353416
}
354417

355418
if !zerosFill {
356-
w.logger.Logf("done: file '%s' (%s)\n", path, time.Since(t1))
419+
if !w.progressReporter.ActiveReporter() {
420+
w.logger.Logf("done: file '%s' (%s)\n", path, time.Since(t1))
421+
}
357422
}
358423

359424
return nil

‎pkg/imgpkg/v1/copy.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type CopyOpts struct {
2828
SignatureRetriever SignatureFetcher
2929
IncludeNonDistributable bool
3030
Resume bool
31+
ProgressTracker ProgressTracker
3132
}
3233

3334
// CopyOrigin abstracts the original location to copy from
@@ -47,6 +48,9 @@ func CopyToTar(origin CopyOrigin, outputTarPath string, opts CopyOpts, reg regis
4748
return nil, err
4849
}
4950

51+
if opts.ProgressTracker != nil {
52+
opts.ProgressTracker.StartDisplay()
53+
}
5054
opts.Logger.Tracef("Exporting images to tar\n")
5155
ids, err := opts.TarImageSet.Export(unprocessedImageRefs, outputTarPath, reg, imagetar.NewImageLayerWriterCheck(opts.IncludeNonDistributable), opts.Resume)
5256
if err != nil {

‎pkg/imgpkg/v1/copy_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,12 @@ func testSetup(registryBuild *helpers.FakeTestRegistryBuilder, imageName string,
4040
uiLogger := util.NewUILevelLogger(util.LogWarn, util.NewBufferLogger(stdOut))
4141
tagGen := ctlimg.DefaultTagGenerator{}
4242
imageSet := imageset.NewImageSet(1, uiLogger, tagGen)
43+
noopProgress := v1.NoopProgressReporter{}
4344

4445
opts := v1.CopyOpts{
4546
Logger: uiLogger,
4647
ImageSet: imageSet,
47-
TarImageSet: imageset.NewTarImageSet(imageSet, 1, uiLogger),
48+
TarImageSet: imageset.NewTarImageSet(imageSet, 1, uiLogger, &noopProgress),
4849
Concurrency: 1,
4950
SignatureRetriever: &fakeSignatureRetriever{},
5051
Resume: false,

0 commit comments

Comments
 (0)