Skip to content

Commit

Permalink
Fix Windows path compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
@jillguyonnet
jillguyonnet committed Feb 14, 2023
1 parent 8dc08d1 commit 2a99b9d
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 56 deletions.
4 changes: 2 additions & 2 deletions main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ func TestAllPackageIndex(t *testing.T) {
// find all manifests
var manifests []string
for _, path := range packagesBasePaths {
m, err := filepath.Glob(path + "/*/*/manifest.yml")
m, err := filepath.Glob(filepath.Join(path, "*", "*", "manifest.yml"))
require.NoError(t, err)
manifests = append(manifests, m...)
}
Expand Down Expand Up @@ -574,7 +574,7 @@ func listArchivedFiles(t *testing.T, body []byte) []byte {
var listing bytes.Buffer

for _, f := range zipReader.File {
listing.WriteString(fmt.Sprintf("%d %s\n", f.UncompressedSize64, f.Name))
listing.WriteString(fmt.Sprintf("%d %s\n", f.UncompressedSize64, filepath.ToSlash(f.Name)))

}
return listing.Bytes()
Expand Down
30 changes: 15 additions & 15 deletions packages/datastream.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"path"
"strings"

"github.com/pkg/errors"
Expand Down Expand Up @@ -117,15 +117,15 @@ func NewDataStream(basePath string, p *Package) (*DataStream, error) {
}
defer fs.Close()

manifestPath := filepath.Join(basePath, "manifest.yml")
manifestPath := path.Join(basePath, "manifest.yml")

// Check if manifest exists
_, err = fs.Stat(manifestPath)
if err != nil && os.IsNotExist(err) {
return nil, errors.Wrapf(err, "manifest does not exist for data stream: %s", p.BasePath)
}

dataStreamPath := filepath.Base(basePath)
dataStreamPath := path.Base(basePath)

b, err := ReadAll(fs, manifestPath)
if err != nil {
Expand Down Expand Up @@ -177,16 +177,16 @@ func NewDataStream(basePath string, p *Package) (*DataStream, error) {
return nil, fmt.Errorf("invalid release: %q", d.Release)
}

pipelineDir := filepath.Join(d.BasePath, "elasticsearch", DirIngestPipeline)
paths, err := fs.Glob(filepath.Join(pipelineDir, "*"))
pipelineDir := path.Join(d.BasePath, "elasticsearch", DirIngestPipeline)
paths, err := fs.Glob(path.Join(pipelineDir, "*"))
if err != nil {
return nil, err
}

if d.Elasticsearch != nil && d.Elasticsearch.IngestPipelineName == "" {
// Check that no ingest pipeline exists in the directory except default
for _, path := range paths {
if filepath.Base(path) == DefaultPipelineNameJSON || filepath.Base(path) == DefaultPipelineNameYAML {
for _, p := range paths {
if path.Base(p) == DefaultPipelineNameJSON || path.Base(p) == DefaultPipelineNameYAML {
d.Elasticsearch.IngestPipelineName = DefaultPipelineName
// TODO: remove because of legacy
d.IngestPipeline = DefaultPipelineName
Expand All @@ -196,8 +196,8 @@ func NewDataStream(basePath string, p *Package) (*DataStream, error) {
// TODO: Remove, only here for legacy
} else if d.IngestPipeline == "" {
// Check that no ingest pipeline exists in the directory except default
for _, path := range paths {
if filepath.Base(path) == DefaultPipelineNameJSON || filepath.Base(path) == DefaultPipelineNameYAML {
for _, p := range paths {
if path.Base(p) == DefaultPipelineNameJSON || path.Base(p) == DefaultPipelineNameYAML {
d.IngestPipeline = DefaultPipelineName
break
}
Expand Down Expand Up @@ -229,11 +229,11 @@ func (d *DataStream) Validate() error {
defer fs.Close()

// In case an ingest pipeline is set, check if it is around
pipelineDir := filepath.Join(d.BasePath, "elasticsearch", DirIngestPipeline)
pipelineDir := path.Join(d.BasePath, "elasticsearch", DirIngestPipeline)
if d.IngestPipeline != "" {
var validFound bool

jsonPipelinePath := filepath.Join(pipelineDir, d.IngestPipeline+".json")
jsonPipelinePath := path.Join(pipelineDir, d.IngestPipeline+".json")
_, errJSON := fs.Stat(jsonPipelinePath)
if errJSON != nil && !os.IsNotExist(errJSON) {
return errors.Wrapf(errJSON, "stat ingest pipeline JSON file failed (path: %s)", jsonPipelinePath)
Expand All @@ -246,7 +246,7 @@ func (d *DataStream) Validate() error {
validFound = true
}

yamlPipelinePath := filepath.Join(pipelineDir, d.IngestPipeline+".yml")
yamlPipelinePath := path.Join(pipelineDir, d.IngestPipeline+".yml")
_, errYAML := fs.Stat(yamlPipelinePath)
if errYAML != nil && !os.IsNotExist(errYAML) {
return errors.Wrapf(errYAML, "stat ingest pipeline YAML file failed (path: %s)", jsonPipelinePath)
Expand Down Expand Up @@ -282,7 +282,7 @@ func validateIngestPipelineFile(fs PackageFileSystem, pipelinePath string) error
return errors.Wrapf(err, "reading ingest pipeline file failed (path: %s)", pipelinePath)
}

ext := filepath.Ext(pipelinePath)
ext := path.Ext(pipelinePath)
var m map[string]interface{}
switch ext {
case ".json":
Expand All @@ -297,10 +297,10 @@ func validateIngestPipelineFile(fs PackageFileSystem, pipelinePath string) error

// validateRequiredFields method loads fields from all files and checks if required fields are present.
func (d *DataStream) validateRequiredFields(fs PackageFileSystem) error {
fieldsDirPath := filepath.Join(d.BasePath, "fields")
fieldsDirPath := path.Join(d.BasePath, "fields")

// Collect fields from all files
fieldsFiles, err := fs.Glob(filepath.Join(fieldsDirPath, "*"))
fieldsFiles, err := fs.Glob(path.Join(fieldsDirPath, "*"))
if err != nil {
return err
}
Expand Down
67 changes: 42 additions & 25 deletions packages/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"io"
"io/fs"
"os"
"path/filepath"
"path"
"strings"
)

Expand All @@ -28,32 +28,47 @@ type PackageFileSystem interface {
// ExtractedPackageFileSystem provides utils to access files in an extracted package.
type ExtractedPackageFileSystem struct {
path string
root fs.FS
}

func NewExtractedPackageFileSystem(p *Package) (*ExtractedPackageFileSystem, error) {
return &ExtractedPackageFileSystem{path: p.BasePath}, nil
return &ExtractedPackageFileSystem{path: p.BasePath, root: os.DirFS(p.BasePath)}, nil
}

func (fs *ExtractedPackageFileSystem) Stat(name string) (os.FileInfo, error) {
return os.Stat(filepath.Join(fs.path, name))
func (fsys *ExtractedPackageFileSystem) Stat(name string) (os.FileInfo, error) {
name = strings.TrimPrefix(name, "/")
return fs.Stat(fsys.root, name)
}

func (fs *ExtractedPackageFileSystem) Open(name string) (PackageFile, error) {
return os.Open(filepath.Join(fs.path, name))
}

func (fs *ExtractedPackageFileSystem) Glob(pattern string) (matches []string, err error) {
matches, err = filepath.Glob(filepath.Join(fs.path, pattern))
func (fsys *ExtractedPackageFileSystem) Open(name string) (PackageFile, error) {
name = strings.TrimPrefix(name, "/")
f, err := fsys.root.Open(name)
if err != nil {
return
return nil, err
}
for i := range matches {
match, err := filepath.Rel(fs.path, matches[i])
pf, ok := f.(PackageFile)
if !ok {
defer f.Close()
return nil, fmt.Errorf("file does not implement PackageFile interface: %q", name)
}
return pf, nil
}

func (fsys *ExtractedPackageFileSystem) Glob(pattern string) (matches []string, err error) {
fs.WalkDir(fsys.root, ".", func(p string, d fs.DirEntry, err error) error {
if err != nil {
return nil, fmt.Errorf("failed to obtain path under package root path (%s): %w", fs.path, err)
return fmt.Errorf("failed to walk path %q: %w", p, err)
}
matches[i] = match
}
match, err := path.Match(pattern, p)
if err != nil {
return fmt.Errorf("failed to obtain path under package root path (%s): %w", pattern, err)
}
if match {
name := p
matches = append(matches, name)
}
return nil
})
return
}

Expand All @@ -73,10 +88,12 @@ func NewZipPackageFileSystem(p *Package) (*ZipPackageFileSystem, error) {
var root string
found := false
for _, f := range reader.File {
name := filepath.Clean(f.Name)
parts := strings.Split(name, string(filepath.Separator))
if len(parts) == 2 && parts[1] == "manifest.yml" {
root = parts[0]
name := path.Clean(f.Name)
dir, base := path.Split(name)
dir = path.Clean(dir)
// Find manifest files at the root of the package
if dir != "." && path.Dir(dir) == "." && base == "manifest.yml" {
root = dir
found = true
break
}
Expand All @@ -91,7 +108,7 @@ func NewZipPackageFileSystem(p *Package) (*ZipPackageFileSystem, error) {
}

func (fs *ZipPackageFileSystem) Stat(name string) (os.FileInfo, error) {
path := filepath.Join(fs.root, name)
path := path.Join(fs.root, name)
f, err := fs.reader.Open(path)
if err != nil {
return nil, err
Expand All @@ -101,7 +118,7 @@ func (fs *ZipPackageFileSystem) Stat(name string) (os.FileInfo, error) {
}

func (fs *ZipPackageFileSystem) Open(name string) (PackageFile, error) {
path := filepath.Join(fs.root, name)
path := path.Join(fs.root, name)
f, err := fs.reader.Open(path)
if err != nil {
return nil, err
Expand All @@ -114,14 +131,14 @@ func (fs *ZipPackageFileSystem) Open(name string) (PackageFile, error) {
}

func (fs *ZipPackageFileSystem) Glob(pattern string) (matches []string, err error) {
pattern = filepath.Join(fs.root, pattern)
pattern = path.Join(fs.root, pattern)
for _, f := range fs.reader.File {
match, err := filepath.Match(pattern, filepath.Clean(f.Name))
match, err := path.Match(pattern, path.Clean(f.Name))
if err != nil {
return nil, err
}
if match {
name := strings.TrimPrefix(f.Name, fs.root+string(filepath.Separator))
name := strings.TrimPrefix(f.Name, fs.root+"/")
matches = append(matches, name)
}
}
Expand Down
23 changes: 11 additions & 12 deletions packages/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"strings"

"github.com/Masterminds/semver/v3"
Expand Down Expand Up @@ -221,7 +220,7 @@ func NewPackage(basePath string, fsBuilder FileSystemBuilder) (*Package, error)
}

// Store policy template specific README
readmePath := filepath.Join("docs", p.PolicyTemplates[i].Name+".md")
readmePath := path.Join("docs", p.PolicyTemplates[i].Name+".md")
readme, err := fs.Stat(readmePath)
if err != nil {
if _, ok := err.(*os.PathError); !ok {
Expand Down Expand Up @@ -277,7 +276,7 @@ func NewPackage(basePath string, fsBuilder FileSystemBuilder) (*Package, error)
return nil, fmt.Errorf("invalid release: %q", p.Release)
}

readmePath := filepath.Join("docs", "README.md")
readmePath := path.Join("docs", "README.md")
// Check if readme
readme, err := fs.Stat(readmePath)
if err != nil {
Expand Down Expand Up @@ -452,7 +451,7 @@ func collectAssets(fs PackageFileSystem, pattern string) ([]string, error) {
return nil, err
}
if len(assets) != 0 {
a, err := collectAssets(fs, filepath.Join(pattern, "*"))
a, err := collectAssets(fs, path.Join(pattern, "*"))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -542,7 +541,7 @@ func (p *Package) validateVersionConsistency() error {
return errors.Wrap(err, "invalid version defined in manifest")
}

baseDir := filepath.Base(p.BasePath)
baseDir := path.Base(p.BasePath)
versionDir, err := semver.NewVersion(baseDir)
if err != nil {
// TODO: There should be a flag passed to the registry to accept these kind of packages
Expand All @@ -568,17 +567,17 @@ func (p *Package) GetDataStreamPaths() ([]string, error) {

// Look for a file here that a data_stream must have, some file systems as Zip files
// may not have entries for directories.
paths, err := fs.Glob(filepath.Join(dataStreamBasePath, "*", "manifest.yml"))
paths, err := fs.Glob(path.Join(dataStreamBasePath, "*", "manifest.yml"))
if err != nil {
return nil, err
}

for i := range paths {
relPath, err := filepath.Rel(dataStreamBasePath, paths[i])
if err != nil {
return nil, fmt.Errorf("failed to get data stream path inside package (%s): %w", dataStreamBasePath, err)
if !strings.HasPrefix(paths[i], dataStreamBasePath) && !strings.HasPrefix(paths[i], "/data_stream") {
return nil, fmt.Errorf("failed to get data stream path inside package: cannot make %q relative to %q", paths[i], dataStreamBasePath)
}
paths[i] = filepath.Dir(relPath)
relPath := strings.TrimPrefix(paths[i], dataStreamBasePath)
paths[i] = path.Dir(relPath)
}

return paths, nil
Expand All @@ -592,7 +591,7 @@ func (p *Package) LoadDataSets() error {

dataStreamsBasePath := "data_stream"
for _, dataStreamPath := range dataStreamPaths {
dataStreamBasePath := filepath.Join(dataStreamsBasePath, dataStreamPath)
dataStreamBasePath := path.Join(dataStreamsBasePath, dataStreamPath)

d, err := NewDataStream(dataStreamBasePath, p)
if err != nil {
Expand All @@ -614,7 +613,7 @@ func (p *Package) ValidateDataStreams() error {

dataStreamsBasePath := "data_stream"
for _, dataStreamPath := range dataStreamPaths {
dataStreamBasePath := filepath.Join(dataStreamsBasePath, dataStreamPath)
dataStreamBasePath := path.Join(dataStreamsBasePath, dataStreamPath)

d, err := NewDataStream(dataStreamBasePath, p)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions storage/paths.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package storage

import (
"net/url"
"path/filepath"
"path"

"github.com/pkg/errors"
)
Expand Down Expand Up @@ -37,7 +37,7 @@ func extractBucketNameFromURL(anURL string) (string, string, error) {
}

func joinObjectPaths(paths ...string) string {
p := filepath.Join(paths...)
p := path.Join(paths...)
return normalizeObjectPath(p)
}

Expand Down

0 comments on commit 2a99b9d

Please sign in to comment.