From 255f55389140ae7cfd34146a6e6ca26b240e2bd0 Mon Sep 17 00:00:00 2001 From: Kyle Quest Date: Tue, 24 Sep 2024 14:28:43 -0700 Subject: [PATCH] docker and podman engines for imagebuild Signed-off-by: Kyle Quest --- pkg/app/master/command/build/flags.go | 1 - pkg/app/master/command/build/image.go | 4 +- pkg/app/master/command/imagebuild/cli.go | 38 ++-- pkg/app/master/command/imagebuild/flags.go | 25 ++- .../imagebuild/handle_engine_buildkit.go | 27 ++- .../command/imagebuild/handle_engine_depot.go | 8 +- .../imagebuild/handle_engine_docker.go | 51 +++++- .../imagebuild/handle_engine_podman.go | 50 +++++- pkg/app/master/command/imagebuild/handler.go | 15 +- pkg/crt/clients.go | 8 + .../docker/dockercrtclient/dockercrtclient.go | 100 ++++++++++- .../podman/podmancrtclient/podmancrtclient.go | 153 +++++++++++++++- pkg/imagebuilder/imagebuilder.go | 1 + pkg/imagebuilder/standardbuilder/engine.go | 163 +++++++++--------- 14 files changed, 530 insertions(+), 114 deletions(-) diff --git a/pkg/app/master/command/build/flags.go b/pkg/app/master/command/build/flags.go index 53433ead..1902cbde 100644 --- a/pkg/app/master/command/build/flags.go +++ b/pkg/app/master/command/build/flags.go @@ -701,7 +701,6 @@ func GetContainerBuildOptions(ctx *cli.Context) (*config.ContainerBuildOptions, cbo.ExtraHosts = strings.Join(hosts, ",") cbo.BuildArgs = command.ParseKVParams(ctx.StringSlice(FlagCBOBuildArg)) - kvLabels := command.ParseKVParams(ctx.StringSlice(FlagCBOLabel)) for _, kv := range kvLabels { cbo.Labels[kv.Name] = kv.Value diff --git a/pkg/app/master/command/build/image.go b/pkg/app/master/command/build/image.go index 089516a6..5b759f35 100644 --- a/pkg/app/master/command/build/image.go +++ b/pkg/app/master/command/build/image.go @@ -255,7 +255,9 @@ func buildFatImage( "context": cbOpts.DockerfileContext, }) - fatBuilder, err := standardbuilder.New(client, doShowBuildLogs) + crtClient := dockercrtclient.NewBuilder(client, doShowBuildLogs) + fatBuilder, err := standardbuilder.New(crtClient) + options := imagebuilder.DockerfileBuildOptions{ Dockerfile: cbOpts.Dockerfile, BuildContext: cbOpts.DockerfileContext, diff --git a/pkg/app/master/command/imagebuild/cli.go b/pkg/app/master/command/imagebuild/cli.go index acf02b95..f8888cae 100644 --- a/pkg/app/master/command/imagebuild/cli.go +++ b/pkg/app/master/command/imagebuild/cli.go @@ -10,6 +10,7 @@ import ( "github.com/mintoolkit/mint/pkg/app" "github.com/mintoolkit/mint/pkg/app/master/command" cmd "github.com/mintoolkit/mint/pkg/command" + "github.com/mintoolkit/mint/pkg/imagebuilder" "github.com/mintoolkit/mint/pkg/util/fsutil" ) @@ -20,17 +21,18 @@ const ( ) type CommandParams struct { - Engine string `json:"engine,omitempty"` - EngineEndpoint string `json:"engine_endpoint,omitempty"` - EngineToken string `json:"engine_token,omitempty"` - EngineNamespace string `json:"engine_namespace,omitempty"` - ImageName string `json:"image_name,omitempty"` - ImageArchiveFile string `json:"image_archive_file,omitempty"` - Runtime string `json:"runtime,omitempty"` //runtime where to load the created image - Dockerfile string `json:"dockerfile,omitempty"` - ContextDir string `json:"context_dir,omitempty"` - BuildArgs []string `json:"build_args,omitempty"` - Architecture string `json:"architecture,omitempty"` + Engine string `json:"engine,omitempty"` + EngineEndpoint string `json:"engine_endpoint,omitempty"` + EngineToken string `json:"engine_token,omitempty"` + EngineNamespace string `json:"engine_namespace,omitempty"` + ImageName string `json:"image_name,omitempty"` + ImageArchiveFile string `json:"image_archive_file,omitempty"` + Runtime string `json:"runtime,omitempty"` //runtime where to load the created image + Dockerfile string `json:"dockerfile,omitempty"` + ContextDir string `json:"context_dir,omitempty"` + BuildArgs []imagebuilder.NVParam `json:"build_args,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Architecture string `json:"architecture,omitempty"` } var ImageBuildFlags = useAllFlags() @@ -60,11 +62,21 @@ var CLI = &cli.Command{ ImageArchiveFile: ctx.String(FlagImageArchiveFile), Dockerfile: ctx.String(FlagDockerfile), ContextDir: ctx.String(FlagContextDir), - BuildArgs: ctx.StringSlice(FlagBuildArg), Runtime: ctx.String(FlagRuntimeLoad), Architecture: ctx.String(FlagArchitecture), } + cboBuildArgs := command.ParseKVParams(ctx.StringSlice(FlagBuildArg)) + for _, val := range cboBuildArgs { + cparams.BuildArgs = append(cparams.BuildArgs, + imagebuilder.NVParam{Name: val.Name, Value: val.Value}) + } + + kvLabels := command.ParseKVParams(ctx.StringSlice(FlagLabel)) + for _, kv := range kvLabels { + cparams.Labels[kv.Name] = kv.Value + } + engineProps, found := BuildEngines[cparams.Engine] if !found { return command.ErrBadParamValue @@ -83,7 +95,7 @@ var CLI = &cli.Command{ } if cparams.Architecture == "" { - cparams.Architecture = DefaultBuildArch + cparams.Architecture = GetDefaultBuildArch() } if !IsArchValue(cparams.Architecture) { diff --git a/pkg/app/master/command/imagebuild/flags.go b/pkg/app/master/command/imagebuild/flags.go index f64adc8b..1dc20361 100644 --- a/pkg/app/master/command/imagebuild/flags.go +++ b/pkg/app/master/command/imagebuild/flags.go @@ -1,6 +1,8 @@ package imagebuild import ( + "runtime" + log "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -37,6 +39,9 @@ const ( FlagBuildArg = "build-arg" FlagBuildArgUsage = "build time variable (ARG)" + FlagLabel = "label" + FlagLabelUsage = "image label to add" + FlagArchitecture = "architecture" FlagArchitectureUsage = "build architecture" ) @@ -65,11 +70,21 @@ const ( Amd64Arch = "amd64" Arm64Arch = "arm64" - DefaultBuildArch = Amd64Arch DefaultRuntimeLoad = NoneRuntimeLoad DefaultEngineName = DockerBuildEngine ) +func GetDefaultBuildArch() string { + switch runtime.GOARCH { + case Amd64Arch: + return Amd64Arch + case Arm64Arch: + return Arm64Arch + default: + return Amd64Arch + } +} + type BuildEngineProps struct { Info string TokenRequired bool @@ -167,6 +182,12 @@ var Flags = map[string]cli.Flag{ Usage: FlagBuildArgUsage, EnvVars: []string{"DSLIM_IMAGEBUILD_BUILD_ARGS"}, }, + FlagLabel: &cli.StringSliceFlag{ + Name: FlagLabel, + Value: cli.NewStringSlice(""), + Usage: FlagLabelUsage, + EnvVars: []string{"DSLIM_IMAGEBUILD_LABELS"}, + }, FlagRuntimeLoad: &cli.StringFlag{ Name: FlagRuntimeLoad, Value: DefaultRuntimeLoad, @@ -175,7 +196,7 @@ var Flags = map[string]cli.Flag{ }, FlagArchitecture: &cli.StringFlag{ Name: FlagArchitecture, - Value: DefaultBuildArch, + Value: GetDefaultBuildArch(), Usage: FlagArchitectureUsage, EnvVars: []string{"DSLIM_IMAGEBUILD_ARCH"}, }, diff --git a/pkg/app/master/command/imagebuild/handle_engine_buildkit.go b/pkg/app/master/command/imagebuild/handle_engine_buildkit.go index 2fee3f0a..227286be 100644 --- a/pkg/app/master/command/imagebuild/handle_engine_buildkit.go +++ b/pkg/app/master/command/imagebuild/handle_engine_buildkit.go @@ -8,7 +8,6 @@ import ( "io" "os" "path/filepath" - "strings" log "github.com/sirupsen/logrus" @@ -28,6 +27,7 @@ func HandleBuildkitEngine( cparams *CommandParams) { logger.Trace("HandleBuildkitEngine.call") defer logger.Trace("HandleBuildkitEngine.exit") + xc.Out.State("buildkit.engine.image.build.started") ctx := context.Background() logger.Trace("buildkit.client.New") @@ -36,6 +36,8 @@ func HandleBuildkitEngine( berr = buildkitBuildImage(logger, xc, cparams, ctx, bclient) xc.FailOn(berr) + + xc.Out.State("buildkit.engine.image.build.completed") } func buildkitBuildImage( @@ -92,19 +94,26 @@ func buildkitBuildImage( }, } - for _, kvStr := range cparams.BuildArgs { - kv := strings.SplitN(kvStr, "=", 2) - if len(kv) != 2 { - logger.Debugf("malformed build arg: %s", kvStr) - continue - } + for _, kv := range cparams.BuildArgs { + /* + kv := strings.SplitN(kvStr, "=", 2) + if len(kv) != 2 { + logger.Debugf("malformed build arg: %s", kvStr) + continue + } + + opts.FrontendAttrs["build-arg:"+kv[0]] = kv[1] + */ + + opts.FrontendAttrs["build-arg:"+kv.Name] = kv.Value + } - opts.FrontendAttrs["build-arg:"+kv[0]] = kv[1] + for k, v := range cparams.Labels { + opts.FrontendAttrs["label:"+k] = v } var res *client.SolveResponse eg.Go(func() error { - //res, err = bclient.Build(ctx, opts, "", dockerfile.Build, ch) res, err = bclient.Solve(ctx, nil, opts, ch) return err }) diff --git a/pkg/app/master/command/imagebuild/handle_engine_depot.go b/pkg/app/master/command/imagebuild/handle_engine_depot.go index cbfeca62..4e1e89ab 100644 --- a/pkg/app/master/command/imagebuild/handle_engine_depot.go +++ b/pkg/app/master/command/imagebuild/handle_engine_depot.go @@ -22,6 +22,7 @@ func HandleDepotEngine( cparams *CommandParams) { logger.Trace("HandleDepotEngine.call") defer logger.Trace("HandleDepotEngine.exit") + xc.Out.State("depot.engine.image.build.started") ctx := context.Background() var doLoad bool @@ -51,9 +52,8 @@ func HandleDepotEngine( logger.Trace("depot.build.NewBuild") build, err := build.NewBuild(ctx, req, cparams.EngineToken) - if err != nil { - panic(err) - } + xc.FailOn(err) + logger.Tracef("depot.build.NewBuild -> id=%s buildURL='%s' useLocalRegistry=%v proxyImage=%s", build.ID, build.BuildURL, build.UseLocalRegistry, build.ProxyImage) @@ -85,4 +85,6 @@ func HandleDepotEngine( if berr != nil { return } + + xc.Out.State("depot.engine.image.build.completed") } diff --git a/pkg/app/master/command/imagebuild/handle_engine_docker.go b/pkg/app/master/command/imagebuild/handle_engine_docker.go index e04cc39f..a9656d86 100644 --- a/pkg/app/master/command/imagebuild/handle_engine_docker.go +++ b/pkg/app/master/command/imagebuild/handle_engine_docker.go @@ -1,12 +1,19 @@ package imagebuild import ( + "fmt" + "os" + dockerapi "github.com/fsouza/go-dockerclient" log "github.com/sirupsen/logrus" "github.com/mintoolkit/mint/pkg/app" "github.com/mintoolkit/mint/pkg/app/master/command" - //"github.com/mintoolkit/mint/pkg/util/jsonutil" + "github.com/mintoolkit/mint/pkg/crt/docker/dockercrtclient" + "github.com/mintoolkit/mint/pkg/imagebuilder" + "github.com/mintoolkit/mint/pkg/imagebuilder/standardbuilder" + "github.com/mintoolkit/mint/pkg/util/fsutil" + v "github.com/mintoolkit/mint/pkg/version" ) // HandleDockerEngine implements support for the Docker container build engine @@ -18,4 +25,46 @@ func HandleDockerEngine( client *dockerapi.Client) { logger.Trace("HandleDockerEngine.call") defer logger.Trace("HandleDockerEngine.exit") + xc.Out.State("docker.engine.image.build.started") + + doShowBuildLogs := true + crtClient := dockercrtclient.NewBuilder(client, doShowBuildLogs) + builder, err := standardbuilder.New(crtClient) + + //note: need to also "save" the created image if it needs to be loaded in a different runtime + options := imagebuilder.DockerfileBuildOptions{ + OutputStream: os.Stdout, //doShowBuildLogs + Dockerfile: cparams.Dockerfile, + BuildContext: cparams.ContextDir, + ImagePath: cparams.ImageName, + BuildArgs: cparams.BuildArgs, + Labels: cparams.Labels, + } + + if cparams.Architecture != "" { + options.Platforms = []string{ + fmt.Sprintf("linux/%s", cparams.Architecture), + } + } + + err = builder.Build(options) + if err != nil { + xc.Out.Info("build.error", + ovars{ + "status": "docker.engine.image.build.error", + "value": err, + }) + + exitCode := 721 + xc.Out.State("exited", + ovars{ + "exit.code": exitCode, + "version": v.Current(), + "location": fsutil.ExeDir(), + }) + + xc.Exit(exitCode) + } + + xc.Out.State("docker.engine.image.build.completed") } diff --git a/pkg/app/master/command/imagebuild/handle_engine_podman.go b/pkg/app/master/command/imagebuild/handle_engine_podman.go index 99c4f5ef..acc270ba 100644 --- a/pkg/app/master/command/imagebuild/handle_engine_podman.go +++ b/pkg/app/master/command/imagebuild/handle_engine_podman.go @@ -2,12 +2,18 @@ package imagebuild import ( "context" + "fmt" + "os" log "github.com/sirupsen/logrus" "github.com/mintoolkit/mint/pkg/app" "github.com/mintoolkit/mint/pkg/app/master/command" - //"github.com/mintoolkit/mint/pkg/util/jsonutil" + "github.com/mintoolkit/mint/pkg/crt/podman/podmancrtclient" + "github.com/mintoolkit/mint/pkg/imagebuilder" + "github.com/mintoolkit/mint/pkg/imagebuilder/standardbuilder" + "github.com/mintoolkit/mint/pkg/util/fsutil" + v "github.com/mintoolkit/mint/pkg/version" ) // HandlePodmanEngine implements support for the Podman container build engine @@ -19,4 +25,46 @@ func HandlePodmanEngine( client context.Context) { logger.Trace("HandlePodmanEngine.call") defer logger.Trace("HandlePodmanEngine.exit") + xc.Out.State("podman.engine.image.build.started") + + doShowBuildLogs := true + crtClient := podmancrtclient.NewBuilder(client, doShowBuildLogs) + builder, err := standardbuilder.New(crtClient) + + //note: need to also "save" the created image if it needs to be loaded in a different runtime + options := imagebuilder.DockerfileBuildOptions{ + OutputStream: os.Stdout, //doShowBuildLogs + Dockerfile: cparams.Dockerfile, + BuildContext: cparams.ContextDir, + ImagePath: cparams.ImageName, + BuildArgs: cparams.BuildArgs, + Labels: cparams.Labels, + } + + if cparams.Architecture != "" { + options.Platforms = []string{ + fmt.Sprintf("linux/%s", cparams.Architecture), + } + } + + err = builder.Build(options) + if err != nil { + xc.Out.Info("build.error", + ovars{ + "status": "podman.engine.image.build.error", + "value": err, + }) + + exitCode := 721 + xc.Out.State("exited", + ovars{ + "exit.code": exitCode, + "version": v.Current(), + "location": fsutil.ExeDir(), + }) + + xc.Exit(exitCode) + } + + xc.Out.State("podman.engine.image.build.completed") } diff --git a/pkg/app/master/command/imagebuild/handler.go b/pkg/app/master/command/imagebuild/handler.go index 6e1bd10d..664d434f 100644 --- a/pkg/app/master/command/imagebuild/handler.go +++ b/pkg/app/master/command/imagebuild/handler.go @@ -97,9 +97,13 @@ func OnCommand( } } + var isSame bool switch cparams.Engine { case DockerBuildEngine: initDockerClient() + if cparams.Runtime == DockerRuntimeLoad { + isSame = true + } if gparams.Debug { version.Print(xc, cmdName, logger, dclient, false, gparams.InContainer, gparams.IsDSImage) @@ -126,6 +130,9 @@ func OnCommand( HandleSimpleEngine(logger, xc, gparams, cparams) case PodmanBuildEngine: initPodmanClient() + if cparams.Runtime == PodmanRuntimeLoad { + isSame = true + } if gparams.Debug { version.Print(xc, Name, logger, nil, false, gparams.InContainer, gparams.IsDSImage) @@ -166,8 +173,12 @@ func OnCommand( "image.archive.file": cparams.ImageArchiveFile, }) - err = crtLoaderClient.LoadImage(cparams.ImageArchiveFile, os.Stdout) - xc.FailOn(err) + if !isSame { + err = crtLoaderClient.LoadImage(cparams.ImageArchiveFile, os.Stdout) + xc.FailOn(err) + } else { + xc.Out.Info("same.image.engine.runtime") + } } else { xc.Out.Info("runtime.load.image.none") } diff --git a/pkg/crt/clients.go b/pkg/crt/clients.go index 4cc4284c..a8326c49 100644 --- a/pkg/crt/clients.go +++ b/pkg/crt/clients.go @@ -6,6 +6,8 @@ import ( "regexp" "strings" "time" + + "github.com/mintoolkit/mint/pkg/imagebuilder" ) var ( @@ -131,10 +133,16 @@ type ImageLoaderAPIClient interface { LoadImage(localPath string, outputStream io.Writer) error } +type ImageBuilderAPIClient interface { + BuildImage(options imagebuilder.DockerfileBuildOptions) error + BuildOutputLog() string +} + type APIClient interface { InspectorAPIClient ImageSaverAPIClient ImageLoaderAPIClient + ImageBuilderAPIClient } func ImageToIdentity(info *ImageInfo) *ImageIdentity { diff --git a/pkg/crt/docker/dockercrtclient/dockercrtclient.go b/pkg/crt/docker/dockercrtclient/dockercrtclient.go index f6174308..426ac6db 100644 --- a/pkg/crt/docker/dockercrtclient/dockercrtclient.go +++ b/pkg/crt/docker/dockercrtclient/dockercrtclient.go @@ -1,18 +1,25 @@ package dockercrtclient import ( + "bytes" "fmt" "io" + "path/filepath" + "strings" docker "github.com/fsouza/go-dockerclient" log "github.com/sirupsen/logrus" "github.com/mintoolkit/mint/pkg/crt" "github.com/mintoolkit/mint/pkg/crt/docker/dockerutil" + "github.com/mintoolkit/mint/pkg/imagebuilder" + "github.com/mintoolkit/mint/pkg/util/fsutil" ) type Instance struct { - pclient *docker.Client + pclient *docker.Client + showBuildLogs bool + buildLog bytes.Buffer } func New(providerClient *docker.Client) *Instance { @@ -21,6 +28,97 @@ func New(providerClient *docker.Client) *Instance { } } +func NewBuilder(providerClient *docker.Client, showBuildLogs bool) *Instance { + ref := New(providerClient) + ref.showBuildLogs = showBuildLogs + return ref +} + +func (ref *Instance) BuildImage(options imagebuilder.DockerfileBuildOptions) error { + if len(options.Labels) > 0 { + //Docker has a limit on how long the labels can be + labels := map[string]string{} + for k, v := range options.Labels { + lineLen := len(k) + len(v) + 7 + if lineLen > 65535 { + //TODO: improve JSON data splitting + valueLen := len(v) + parts := valueLen / 50000 + parts++ + offset := 0 + for i := 0; i < parts && offset < valueLen; i++ { + chunkSize := 50000 + if (offset + chunkSize) > valueLen { + chunkSize = valueLen - offset + } + value := v[offset:(offset + chunkSize)] + offset += chunkSize + key := fmt.Sprintf("%s.%d", k, i) + labels[key] = value + } + } else { + labels[k] = v + } + } + options.Labels = labels + } + + //not using options.CacheTo in this image builder... + buildOptions := docker.BuildImageOptions{ + Dockerfile: options.Dockerfile, + Target: options.Target, + Name: options.ImagePath, + + NetworkMode: options.NetworkMode, + ExtraHosts: options.ExtraHosts, + CacheFrom: options.CacheFrom, + Labels: options.Labels, + RmTmpContainer: true, + } + + if len(options.Platforms) > 0 { + buildOptions.Platform = strings.Join(options.Platforms, ",") + } + + for _, nv := range options.BuildArgs { + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{Name: nv.Name, Value: nv.Value}) + } + + if strings.HasPrefix(options.BuildContext, "http://") || + strings.HasPrefix(options.BuildContext, "https://") { + buildOptions.Remote = options.BuildContext + } else { + if exists := fsutil.DirExists(options.BuildContext); exists { + buildOptions.ContextDir = options.BuildContext + } else { + return imagebuilder.ErrInvalidContextDir + } + } + + if !fsutil.Exists(buildOptions.Dockerfile) || !fsutil.IsRegularFile(buildOptions.Dockerfile) { + //a slightly hacky behavior using the build context directory if the dockerfile flag doesn't include a usable path + fullDockerfileName := filepath.Join(buildOptions.ContextDir, buildOptions.Dockerfile) + if !fsutil.Exists(fullDockerfileName) || !fsutil.IsRegularFile(fullDockerfileName) { + return fmt.Errorf("invalid dockerfile reference - %s", fullDockerfileName) + } + + buildOptions.Dockerfile = fullDockerfileName + } + + if options.OutputStream != nil { + buildOptions.OutputStream = options.OutputStream + } else if ref.showBuildLogs { + ref.buildLog.Reset() + buildOptions.OutputStream = &ref.buildLog + } + + return ref.pclient.BuildImage(buildOptions) +} + +func (ref *Instance) BuildOutputLog() string { + return ref.buildLog.String() +} + func (ref *Instance) HasImage(imageRef string) (*crt.ImageIdentity, error) { pii, err := dockerutil.HasImage(ref.pclient, imageRef) if err != nil { diff --git a/pkg/crt/podman/podmancrtclient/podmancrtclient.go b/pkg/crt/podman/podmancrtclient/podmancrtclient.go index c8b357b1..850a9a8d 100644 --- a/pkg/crt/podman/podmancrtclient/podmancrtclient.go +++ b/pkg/crt/podman/podmancrtclient/podmancrtclient.go @@ -3,21 +3,28 @@ package podmancrtclient import ( "io" //"fmt" + "bytes" "context" "errors" "fmt" + "path/filepath" "strings" - //log "github.com/sirupsen/logrus" + buildahDefine "github.com/containers/buildah/define" "github.com/containers/podman/v5/pkg/bindings/images" "github.com/containers/storage" + log "github.com/sirupsen/logrus" "github.com/mintoolkit/mint/pkg/crt" "github.com/mintoolkit/mint/pkg/crt/podman/podmanutil" + "github.com/mintoolkit/mint/pkg/imagebuilder" + "github.com/mintoolkit/mint/pkg/util/fsutil" ) type Instance struct { - pclient context.Context + pclient context.Context + showBuildLogs bool + buildLog bytes.Buffer } func New(providerClient context.Context) *Instance { @@ -26,6 +33,148 @@ func New(providerClient context.Context) *Instance { } } +func NewBuilder(providerClient context.Context, showBuildLogs bool) *Instance { + ref := New(providerClient) + ref.showBuildLogs = showBuildLogs + return ref +} + +func (ref *Instance) BuildImage(options imagebuilder.DockerfileBuildOptions) error { + if len(options.Labels) > 0 { + //todo: check if Podman has a limit on how long the labels can be + labels := map[string]string{} + for k, v := range options.Labels { + lineLen := len(k) + len(v) + 7 + if lineLen > 65535 { + //TODO: improve JSON data splitting + valueLen := len(v) + parts := valueLen / 50000 + parts++ + offset := 0 + for i := 0; i < parts && offset < valueLen; i++ { + chunkSize := 50000 + if (offset + chunkSize) > valueLen { + chunkSize = valueLen - offset + } + value := v[offset:(offset + chunkSize)] + offset += chunkSize + key := fmt.Sprintf("%s.%d", k, i) + labels[key] = value + } + } else { + labels[k] = v + } + } + options.Labels = labels + } + + var labelsList []string + for k, v := range options.Labels { + if v != "" { + labelsList = append(labelsList, fmt.Sprintf("%s=%s", k, v)) + } else { + labelsList = append(labelsList, k) + } + } + + var contextDir string + if strings.HasPrefix(options.BuildContext, "http://") || + strings.HasPrefix(options.BuildContext, "https://") { + log.Debugf("podmancrtclient.Instance.BuildImage: not using remote build context - '%s'", options.BuildContext) + //hacky... + contextDir = "." + } else { + if exists := fsutil.DirExists(options.BuildContext); exists { + contextDir = options.BuildContext + } else { + return imagebuilder.ErrInvalidContextDir + } + } + + fullDockerfileName := options.Dockerfile + if !fsutil.Exists(fullDockerfileName) || !fsutil.IsRegularFile(fullDockerfileName) { + //a slightly hacky behavior using the build context directory if the dockerfile flag doesn't include a usable path + fullDockerfileName = filepath.Join(contextDir, fullDockerfileName) + if !fsutil.Exists(fullDockerfileName) || !fsutil.IsRegularFile(fullDockerfileName) { + return fmt.Errorf("invalid dockerfile reference - %s", fullDockerfileName) + } + } + + buildOptions := images.BuildOptions{ + BuildOptions: buildahDefine.BuildOptions{ + Output: options.ImagePath, + ContextDirectory: contextDir, + Target: options.Target, + IgnoreFile: options.IgnoreFile, + Labels: labelsList, + RemoveIntermediateCtrs: true, + PullPolicy: buildahDefine.PullIfMissing, + OutputFormat: buildahDefine.Dockerv2ImageManifest, //buildah.OCIv1ImageManifest + CommonBuildOpts: &buildahDefine.CommonBuildOptions{ + AddHost: strings.Split(options.ExtraHosts, ","), + }, + + //ConfigureNetwork: tbd <- options.NetworkMode + //CacheFrom: tbd <- options.CacheFrom + //CacheTo: tbd <- options.CacheFrom + }, + ContainerFiles: []string{fullDockerfileName}, + } + + if len(options.Platforms) > 0 { + for _, val := range options.Platforms { + if strings.Contains(val, "/") { + parts := strings.Split(val, "/") + if len(parts) > 2 { + buildOptions.Platforms = append(buildOptions.Platforms, + struct{ OS, Arch, Variant string }{ + OS: parts[0], + Arch: parts[1], + Variant: parts[2], + }) + } else { + buildOptions.Platforms = append(buildOptions.Platforms, + struct{ OS, Arch, Variant string }{ + OS: parts[0], + Arch: parts[1], + }) + } + } else { + buildOptions.Platforms = append(buildOptions.Platforms, + struct{ OS, Arch, Variant string }{ + OS: "linux", + Arch: val, + }) + } + } + } + + for _, nv := range options.BuildArgs { + buildOptions.Args[nv.Name] = nv.Value + } + + if options.OutputStream != nil { + buildOptions.Out = options.OutputStream + buildOptions.Err = options.OutputStream + } else if ref.showBuildLogs { + ref.buildLog.Reset() + buildOptions.Out = &ref.buildLog + buildOptions.Err = &ref.buildLog + } + + report, err := images.Build(ref.pclient, buildOptions.ContainerFiles, buildOptions) + if err != nil { + return err + } + + log.Debugf("podmancrtclient.Instance.BuildImage: report{ID=%s,SaveFormat=%s}", report.ID, report.SaveFormat) + return nil +} + +func (ref *Instance) BuildOutputLog() string { + return ref.buildLog.String() +} + func (ref *Instance) HasImage(imageRef string) (*crt.ImageIdentity, error) { pii, err := podmanutil.HasImage(ref.pclient, imageRef) if err != nil { diff --git a/pkg/imagebuilder/imagebuilder.go b/pkg/imagebuilder/imagebuilder.go index 921a55b6..cb65b1b6 100644 --- a/pkg/imagebuilder/imagebuilder.go +++ b/pkg/imagebuilder/imagebuilder.go @@ -158,6 +158,7 @@ type NVParam struct { type DockerfileBuildOptions struct { Dockerfile string BuildContext string + IgnoreFile string ImagePath string BuildArgs []NVParam Labels map[string]string diff --git a/pkg/imagebuilder/standardbuilder/engine.go b/pkg/imagebuilder/standardbuilder/engine.go index 95823b85..9cd2f804 100644 --- a/pkg/imagebuilder/standardbuilder/engine.go +++ b/pkg/imagebuilder/standardbuilder/engine.go @@ -1,15 +1,16 @@ package standardbuilder import ( - "bytes" - "fmt" - "path/filepath" - "strings" + //"bytes" + //"fmt" + //"path/filepath" + //"strings" - docker "github.com/fsouza/go-dockerclient" + //docker "github.com/fsouza/go-dockerclient" + "github.com/mintoolkit/mint/pkg/crt" "github.com/mintoolkit/mint/pkg/imagebuilder" - "github.com/mintoolkit/mint/pkg/util/fsutil" + //"github.com/mintoolkit/mint/pkg/util/fsutil" ) const ( @@ -18,24 +19,27 @@ const ( // Engine is the standard container build engine type Engine struct { - pclient *docker.Client - showBuildLogs bool - buildLog bytes.Buffer + //pclient *docker.Client + pclient crt.ImageBuilderAPIClient + //showBuildLogs bool + //buildLog bytes.Buffer + //pushToDaemon bool //pushToRegistry bool } // New creates new Engine instances func New( - providerClient *docker.Client, - showBuildLogs bool, + client crt.ImageBuilderAPIClient, + //providerClient *docker.Client, + //showBuildLogs bool, //pushToDaemon bool, //pushToRegistry bool, ) (*Engine, error) { engine := &Engine{ - pclient: providerClient, - showBuildLogs: showBuildLogs, + pclient: client, + //showBuildLogs: showBuildLogs, //pushToDaemon: pushToDaemon, //pushToRegistry: pushToRegistry, } @@ -48,86 +52,89 @@ func (ref *Engine) Name() string { } func (ref *Engine) BuildLog() string { - return ref.buildLog.String() + return ref.pclient.BuildOutputLog() //ref.buildLog.String() } func (ref *Engine) Build(options imagebuilder.DockerfileBuildOptions) error { - if len(options.Labels) > 0 { - //Docker has a limit on how long the labels can be - labels := map[string]string{} - for k, v := range options.Labels { - lineLen := len(k) + len(v) + 7 - if lineLen > 65535 { - //TODO: improve JSON data splitting - valueLen := len(v) - parts := valueLen / 50000 - parts++ - offset := 0 - for i := 0; i < parts && offset < valueLen; i++ { - chunkSize := 50000 - if (offset + chunkSize) > valueLen { - chunkSize = valueLen - offset + return ref.pclient.BuildImage(options) + /* + if len(options.Labels) > 0 { + //Docker has a limit on how long the labels can be + labels := map[string]string{} + for k, v := range options.Labels { + lineLen := len(k) + len(v) + 7 + if lineLen > 65535 { + //TODO: improve JSON data splitting + valueLen := len(v) + parts := valueLen / 50000 + parts++ + offset := 0 + for i := 0; i < parts && offset < valueLen; i++ { + chunkSize := 50000 + if (offset + chunkSize) > valueLen { + chunkSize = valueLen - offset + } + value := v[offset:(offset + chunkSize)] + offset += chunkSize + key := fmt.Sprintf("%s.%d", k, i) + labels[key] = value } - value := v[offset:(offset + chunkSize)] - offset += chunkSize - key := fmt.Sprintf("%s.%d", k, i) - labels[key] = value + } else { + labels[k] = v } - } else { - labels[k] = v } + options.Labels = labels } - options.Labels = labels - } - //not using options.CacheTo in this image builder... - buildOptions := docker.BuildImageOptions{ - Dockerfile: options.Dockerfile, - Target: options.Target, - Name: options.ImagePath, - - NetworkMode: options.NetworkMode, - ExtraHosts: options.ExtraHosts, - CacheFrom: options.CacheFrom, - Labels: options.Labels, - RmTmpContainer: true, - } + //not using options.CacheTo in this image builder... + buildOptions := docker.BuildImageOptions{ + Dockerfile: options.Dockerfile, + Target: options.Target, + Name: options.ImagePath, + + NetworkMode: options.NetworkMode, + ExtraHosts: options.ExtraHosts, + CacheFrom: options.CacheFrom, + Labels: options.Labels, + RmTmpContainer: true, + } - if len(options.Platforms) > 0 { - buildOptions.Platform = strings.Join(options.Platforms, ",") - } + if len(options.Platforms) > 0 { + buildOptions.Platform = strings.Join(options.Platforms, ",") + } - for _, nv := range options.BuildArgs { - buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{Name: nv.Name, Value: nv.Value}) - } + for _, nv := range options.BuildArgs { + buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{Name: nv.Name, Value: nv.Value}) + } - if strings.HasPrefix(options.BuildContext, "http://") || - strings.HasPrefix(options.BuildContext, "https://") { - buildOptions.Remote = options.BuildContext - } else { - if exists := fsutil.DirExists(options.BuildContext); exists { - buildOptions.ContextDir = options.BuildContext + if strings.HasPrefix(options.BuildContext, "http://") || + strings.HasPrefix(options.BuildContext, "https://") { + buildOptions.Remote = options.BuildContext } else { - return imagebuilder.ErrInvalidContextDir + if exists := fsutil.DirExists(options.BuildContext); exists { + buildOptions.ContextDir = options.BuildContext + } else { + return imagebuilder.ErrInvalidContextDir + } } - } - if !fsutil.Exists(buildOptions.Dockerfile) || !fsutil.IsRegularFile(buildOptions.Dockerfile) { - //a slightly hacky behavior using the build context directory if the dockerfile flag doesn't include a usable path - fullDockerfileName := filepath.Join(buildOptions.ContextDir, buildOptions.Dockerfile) - if !fsutil.Exists(fullDockerfileName) || !fsutil.IsRegularFile(fullDockerfileName) { - return fmt.Errorf("invalid dockerfile reference - %s", fullDockerfileName) - } + if !fsutil.Exists(buildOptions.Dockerfile) || !fsutil.IsRegularFile(buildOptions.Dockerfile) { + //a slightly hacky behavior using the build context directory if the dockerfile flag doesn't include a usable path + fullDockerfileName := filepath.Join(buildOptions.ContextDir, buildOptions.Dockerfile) + if !fsutil.Exists(fullDockerfileName) || !fsutil.IsRegularFile(fullDockerfileName) { + return fmt.Errorf("invalid dockerfile reference - %s", fullDockerfileName) + } - buildOptions.Dockerfile = fullDockerfileName - } + buildOptions.Dockerfile = fullDockerfileName + } - if options.OutputStream != nil { - buildOptions.OutputStream = options.OutputStream - } else if ref.showBuildLogs { - ref.buildLog.Reset() - buildOptions.OutputStream = &ref.buildLog - } + if options.OutputStream != nil { + buildOptions.OutputStream = options.OutputStream + } else if ref.showBuildLogs { + ref.buildLog.Reset() + buildOptions.OutputStream = &ref.buildLog + } - return ref.pclient.BuildImage(buildOptions) + return ref.pclient.BuildImage(buildOptions) + */ }