From 4aa9e1faec8549509fa5435ff8c416338910d351 Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Thu, 19 May 2016 07:24:48 -0700 Subject: [PATCH] layout: unpacking initial commit Fixes partially #75 Signed-off-by: Sergiusz Urbaniak --- .gitignore | 4 +- cmd/oci-image-tool/autodetect.go | 103 ++++++++++++++ cmd/oci-image-tool/main.go | 2 + cmd/oci-image-tool/unpack.go | 101 ++++++++++++++ cmd/oci-image-tool/validate.go | 80 ----------- image/doc.go | 16 +++ image/unpack.go | 226 +++++++++++++++++++++++++++++++ image/walker.go | 111 +++++++++++++++ 8 files changed, 561 insertions(+), 82 deletions(-) create mode 100644 cmd/oci-image-tool/autodetect.go create mode 100644 cmd/oci-image-tool/unpack.go create mode 100644 image/doc.go create mode 100644 image/unpack.go create mode 100644 image/walker.go diff --git a/.gitignore b/.gitignore index 0248ce324..50f67b535 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ code-of-conduct.md -oci-image-tool -oci-validate-examples +/oci-image-tool +/oci-validate-examples output diff --git a/cmd/oci-image-tool/autodetect.go b/cmd/oci-image-tool/autodetect.go new file mode 100644 index 000000000..32dad3c18 --- /dev/null +++ b/cmd/oci-image-tool/autodetect.go @@ -0,0 +1,103 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "io" + "io/ioutil" + "net/http" + "os" + + "github.com/opencontainers/image-spec/schema" + "github.com/pkg/errors" +) + +// supported autodetection types +const ( + typeImageLayout = "imageLayout" + typeImage = "image" + typeManifest = "manifest" + typeManifestList = "manifestList" + typeConfig = "config" +) + +// autodetect detects the validation type for the given path +// or an error if the validation type could not be resolved. +func autodetect(path string) (string, error) { + fi, err := os.Stat(path) + if err != nil { + return "", errors.Wrapf(err, "unable to access path") // err from os.Stat includes path name + } + + if fi.IsDir() { + return typeImageLayout, nil + } + + f, err := os.Open(path) + if err != nil { + return "", errors.Wrap(err, "unable to open file") // os.Open includes the filename + } + defer f.Close() + + buf, err := ioutil.ReadAll(io.LimitReader(f, 512)) // read some initial bytes to detect content + if err != nil { + return "", errors.Wrap(err, "unable to read") + } + + mimeType := http.DetectContentType(buf) + + switch mimeType { + case "application/x-gzip": + return typeImage, nil + + case "application/octet-stream": + return typeImage, nil + + case "text/plain; charset=utf-8": + // might be a JSON file, will be handled below + + default: + return "", errors.New("unknown file type") + } + + if _, err := f.Seek(0, os.SEEK_SET); err != nil { + return "", errors.Wrap(err, "unable to seek") + } + + header := struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config interface{} `json:"config"` + }{} + + if err := json.NewDecoder(f).Decode(&header); err != nil { + return "", errors.Wrap(err, "unable to parse JSON") + } + + switch { + case header.MediaType == string(schema.MediaTypeManifest): + return typeManifest, nil + + case header.MediaType == string(schema.MediaTypeManifestList): + return typeManifestList, nil + + case header.MediaType == "" && header.SchemaVersion == 0 && header.Config != nil: + // config files don't have mediaType/schemaVersion header + return typeConfig, nil + } + + return "", errors.New("unknown media type") +} diff --git a/cmd/oci-image-tool/main.go b/cmd/oci-image-tool/main.go index 9206f28bd..ab911bb39 100644 --- a/cmd/oci-image-tool/main.go +++ b/cmd/oci-image-tool/main.go @@ -31,6 +31,8 @@ func main() { stderr := log.New(os.Stderr, "", 0) cmd.AddCommand(newValidateCmd(stdout, stderr)) + cmd.AddCommand(newUnpackCmd(stdout, stderr)) + if err := cmd.Execute(); err != nil { stderr.Println(err) os.Exit(1) diff --git a/cmd/oci-image-tool/unpack.go b/cmd/oci-image-tool/unpack.go new file mode 100644 index 000000000..9549c4eda --- /dev/null +++ b/cmd/oci-image-tool/unpack.go @@ -0,0 +1,101 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/opencontainers/image-spec/image" + "github.com/spf13/cobra" +) + +// supported unpack types +var unpackTypes = []string{ + typeImageLayout, + typeImage, +} + +type unpackCmd struct { + stdout *log.Logger + stderr *log.Logger + typ string // the type to validate, can be empty string + manifest string +} + +func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command { + v := &unpackCmd{ + stdout: stdout, + stderr: stderr, + } + + cmd := &cobra.Command{ + Use: "unpack [src] [dest]", + Short: "Unpack an image or image source layout", + Long: `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`, + Run: v.Run, + } + + cmd.Flags().StringVar( + &v.typ, "type", "", + fmt.Sprintf( + `Type of the file to unpack. If unset, oci-image-tool will try to auto-detect the type. One of "%s"`, + strings.Join(unpackTypes, ","), + ), + ) + + cmd.Flags().StringVar( + &v.manifest, "manifest", "v1.0", + `The manifest to unpack. This must be present in the "manifests" subdirectory of the image.`, + ) + + return cmd +} + +func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + v.stderr.Print("both src and dest must be provided") + if err := cmd.Usage(); err != nil { + v.stderr.Println(err) + } + os.Exit(1) + } + + if v.typ == "" { + typ, err := autodetect(args[0]) + if err != nil { + v.stderr.Printf("%q: autodetection failed: %v", args[0], err) + os.Exit(1) + } + v.typ = typ + } + + var err error + switch v.typ { + case typeImageLayout: + err = image.UnpackLayout(args[0], args[1], v.manifest) + case typeImage: + err = image.Unpack(args[0], args[1], v.manifest) + } + + if err != nil { + v.stderr.Printf("unpacking failed: %v", err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/oci-image-tool/validate.go b/cmd/oci-image-tool/validate.go index 76891bba8..0a4fc16b0 100644 --- a/cmd/oci-image-tool/validate.go +++ b/cmd/oci-image-tool/validate.go @@ -15,12 +15,8 @@ package main import ( - "encoding/json" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" "strings" @@ -30,14 +26,6 @@ import ( ) // supported validation types -const ( - typeImageLayout = "imageLayout" - typeImage = "image" - typeManifest = "manifest" - typeManifestList = "manifestList" - typeConfig = "config" -) - var validateTypes = []string{ typeImageLayout, typeImage, @@ -145,71 +133,3 @@ func (v *validateCmd) validatePath(name string) error { return fmt.Errorf("type %q unimplemented", typ) } - -// autodetect detects the validation type for the given path -// or an error if the validation type could not be resolved. -func autodetect(path string) (string, error) { - fi, err := os.Stat(path) - if err != nil { - return "", errors.Wrapf(err, "unable to access path") // err from os.Stat includes path name - } - - if fi.IsDir() { - return typeImageLayout, nil - } - - f, err := os.Open(path) - if err != nil { - return "", errors.Wrap(err, "unable to open file") // os.Open includes the filename - } - defer f.Close() - - buf, err := ioutil.ReadAll(io.LimitReader(f, 512)) // read some initial bytes to detect content - if err != nil { - return "", errors.Wrap(err, "unable to read") - } - - mimeType := http.DetectContentType(buf) - - switch mimeType { - case "application/x-gzip": - return typeImage, nil - - case "application/octet-stream": - return typeImage, nil - - case "text/plain; charset=utf-8": - // might be a JSON file, will be handled below - - default: - return "", errors.New("unknown file type") - } - - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - return "", errors.Wrap(err, "unable to seek") - } - - header := struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Config interface{} `json:"config"` - }{} - - if err := json.NewDecoder(f).Decode(&header); err != nil { - return "", errors.Wrap(err, "unable to parse JSON") - } - - switch { - case header.MediaType == string(schema.MediaTypeManifest): - return typeManifest, nil - - case header.MediaType == string(schema.MediaTypeManifestList): - return typeManifestList, nil - - case header.MediaType == "" && header.SchemaVersion == 0 && header.Config != nil: - // config files don't have mediaType/schemaVersion header - return typeConfig, nil - } - - return "", errors.New("unknown media type") -} diff --git a/image/doc.go b/image/doc.go new file mode 100644 index 000000000..fc5de0fe6 --- /dev/null +++ b/image/doc.go @@ -0,0 +1,16 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package image defines methods for validating, and unpacking OCI images. +package image diff --git a/image/unpack.go b/image/unpack.go new file mode 100644 index 000000000..7feea811b --- /dev/null +++ b/image/unpack.go @@ -0,0 +1,226 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/opencontainers/image-spec/schema" + "github.com/pkg/errors" +) + +var ( + errEOW = fmt.Errorf("end of walk") // error to signal stop walking +) + +// UnpackLayout walks through the file tree given given by src and +// using the layers specified in the given manifest +// and unpacks all layers in the given destination directory +// or returns an error if the unpacking failed. +func UnpackLayout(src, dest, manifest string) error { + return unpack(newPathWalker(src), dest, manifest) +} + +// Unpack walks through the given .tar file and +// using the layers specified in the given manifest +// and unpacks all layers in the given destination directory +// or returns an error if the unpacking failed. +func Unpack(tarFile, dest, manifest string) error { + f, err := os.Open(tarFile) + if err != nil { + return errors.Wrap(err, "unable to open file") + } + defer f.Close() + + return unpack(newTarWalker(f), dest, manifest) +} + +func unpack(w walker, dest, manifest string) error { + digests, err := diffIDs(w, manifest) + if err != nil { + return errors.Wrap(err, "error reading diff IDs") + } + + return extractLayers(w, digests, dest) +} + +func diffIDs(w walker, manifest string) ([]string, error) { + manifest = filepath.Join("manifests", manifest) + res := []string{} + + f := func(path string, info os.FileInfo, r io.Reader) error { + if info.IsDir() { + return nil + } + + if filepath.Clean(path) != manifest { + return nil + } + + manifest := struct { + Layers []struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + } `json:"layers"` + }{} + + if err := json.NewDecoder(r).Decode(&manifest); err != nil { + return err + } + + for _, l := range manifest.Layers { + if l.MediaType != string(schema.MediaTypeImageSerialization) { + continue + } + + res = append(res, l.Digest) + } + + if len(res) == 0 { + return fmt.Errorf("%s: no layers found", path) + } + + return errEOW + } + + if err := w.walk(f); err != nil && err != errEOW { + return nil, err + } + + return res, nil +} + +func extractLayers(w walker, digests []string, dest string) error { + for _, d := range digests { + println(d) + f := func(path string, info os.FileInfo, r io.Reader) error { + if info.IsDir() { + return nil + } + + dd, err := filepath.Rel("blobs", filepath.Clean(path)) + if err != nil || d != dd { + return nil // ignore + } + + if err := extractLayer(dest, r); err != nil { + return errors.Wrap(err, "error extracting layer") + } + + return errEOW + } + + err := w.walk(f) + if err != nil && err != errEOW { + return err + } + } + + return nil +} + +func extractLayer(dest string, r io.Reader) error { + gz, err := gzip.NewReader(r) + if err != nil { + return errors.Wrap(err, "error creating gzip reader") + } + defer gz.Close() + + tr := tar.NewReader(gz) + +loop: + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + break loop + case nil: + // success, continue below + default: + return errors.Wrapf(err, "error advancing tar stream") + } + + path := filepath.Join(dest, filepath.Clean(hdr.Name)) + info := hdr.FileInfo() + + println(path) + + if strings.HasPrefix(info.Name(), `.wh.`) { + path = strings.Replace(path, `.wh.`, "", 1) + + if err := os.RemoveAll(path); err != nil { + return errors.Wrap(err, "unable to delete whiteout path") + } + + continue loop + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, info.Mode()); err != nil { + return errors.Wrap(err, "error creating directory") + } + + case tar.TypeReg, tar.TypeRegA: + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) + if err != nil { + return errors.Wrap(err, "unable to open file") + } + + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return errors.Wrap(err, "unable to copy") + } + f.Close() + + case tar.TypeLink: + target := filepath.Join(dest, hdr.Linkname) + + if !strings.HasPrefix(target, dest) { + return fmt.Errorf("invalid hardlink %q -> %q", target, hdr.Linkname) + } + + if err := os.Link(target, path); err != nil { + return err + } + + case tar.TypeSymlink: + target := filepath.Join(filepath.Dir(path), hdr.Linkname) + + if !strings.HasPrefix(target, dest) { + return fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname) + } + + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } + + } + + if err := os.Chtimes(path, time.Now().UTC(), info.ModTime()); err != nil { + return errors.Wrap(err, "error changing time") + } + } + + return nil +} diff --git a/image/walker.go b/image/walker.go new file mode 100644 index 000000000..cdfa661f8 --- /dev/null +++ b/image/walker.go @@ -0,0 +1,111 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "archive/tar" + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// walkFunc is a function type that gets called for each file or directory visited by the Walker. +type walkFunc func(path string, _ os.FileInfo, _ io.Reader) error + +// walker is the interface that walks through a file tree, +// calling walk for each file or directory in the tree. +type walker interface { + walk(walkFunc) error +} + +type tarWalker struct { + r io.ReadSeeker +} + +// newTarWalker returns a Walker that walks through .tar files. +func newTarWalker(r io.ReadSeeker) walker { + return &tarWalker{r} +} + +func (w *tarWalker) walk(f walkFunc) error { + if _, err := w.r.Seek(0, os.SEEK_SET); err != nil { + return errors.Wrapf(err, "unable to reset") + } + + tr := tar.NewReader(w.r) + +loop: + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + break loop + case nil: + // success, continue below + default: + return errors.Wrapf(err, "error advancing tar stream") + } + + info := hdr.FileInfo() + if err := f(hdr.Name, info, tr); err != nil { + return err + } + } + + return nil +} + +type eofReader struct{} + +func (eofReader) Read(_ []byte) (int, error) { + return 0, io.EOF +} + +type pathWalker struct { + root string +} + +// newPathWalker returns a Walker that walks through directories +// starting at the given root path. It does not follow symlinks. +func newPathWalker(root string) walker { + return &pathWalker{root} +} + +func (w *pathWalker) walk(f walkFunc) error { + return filepath.Walk(w.root, func(path string, info os.FileInfo, err error) error { + rel, err := filepath.Rel(w.root, path) + if err != nil { + return errors.Wrap(err, "error walking path") + } + + if err != nil { + return errors.Wrap(err, "error walking path") // err from filepath.Walk includes path name + } + + if info.IsDir() { // behave like a tar reader for directories + return f(rel, info, eofReader{}) + } + + file, err := os.Open(path) + if err != nil { + return errors.Wrap(err, "unable to open file") // os.Open includes the path + } + defer file.Close() + + return f(rel, info, file) + }) +}