Verify I/O, system errors while loading packages (#438)

* Verify I/O, system errors while loading packages

* Update CHANGELOG

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Bugfixes
 
 ### Added
+* Use filepath.Walk to find valid package content data. [#438](https://github.com/elastic/package-registry/pull/438)
 
 ### Deprecated
 

main.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -57,30 +58,31 @@ func main() {
 
 	config, err := getConfig()
 	if err != nil {
-		log.Print(err)
-		os.Exit(1)
+		log.Fatal(err)
 	}
 
 	log.Println("Cache time for /search: ", config.CacheTimeSearch)
 	log.Println("Cache time for /categories: ", config.CacheTimeCategories)
 	log.Println("Cache time for all others: ", config.CacheTimeCatchAll)
 
-	packagesBasePath := config.PublicDir + "/" + packageDir
-
-	// Prefill the package cache
+	packagesBasePath := filepath.Join(config.PublicDir, packageDir)
 	packages, err := util.GetPackages(packagesBasePath)
 	if err != nil {
-		log.Print(err)
-		os.Exit(1)
+		log.Fatal(err)
+	}
+
+	if len(packages) == 0 {
+		log.Fatal("No packages available")
 	}
+
 	log.Printf("%v package manifests loaded into memory.\n", len(packages))
 
 	server := &http.Server{Addr: address, Handler: getRouter(*config, packagesBasePath)}
 
 	go func() {
 		err := server.ListenAndServe()
-		if err != nil {
-			log.Printf("Error serving: %s", err)
+		if err != nil && err != http.ErrServerClosed {
+			log.Fatalf("Error occurred while serving: %s", err)
 		}
 	}()
 
@@ -90,7 +92,7 @@ func main() {
 
 	ctx := context.TODO()
 	if err := server.Shutdown(ctx); err != nil {
-		log.Print(err)
+		log.Fatal(err)
 	}
 }
 

util/packages.go

@@ -5,8 +5,10 @@
 package util
 
 import (
+	"log"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 var packageList []Package
@@ -38,24 +40,29 @@ func GetPackages(packagesBasePath string) ([]Package, error) {
 
 // getPackagePaths returns list of available packages, one for each version.
 func getPackagePaths(packagesPath string) ([]string, error) {
+	log.Printf("List packages in %s", packagesPath)
 
-	allPaths, err := filepath.Glob(packagesPath + "/*/*")
-	if err != nil {
-		return nil, err
-	}
-
-	var packagePaths []string
-	for _, path := range allPaths {
-		p, err := os.Stat(path)
+	var foundPaths []string
+	return foundPaths, filepath.Walk(packagesPath, func(path string, info os.FileInfo, err error) error {
+		relativePath, err := filepath.Rel(packagesPath, path)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		if !p.IsDir() {
-			continue
+
+		dirs := strings.Split(relativePath, string(filepath.Separator))
+		if len(dirs) < 2 {
+			return nil // need to go to the package version level
 		}
 
-		packagePaths = append(packagePaths, path)
-	}
+		p, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
 
-	return packagePaths, nil
+		if p.IsDir() {
+			log.Printf("%-20s\t%10s\t%s", dirs[0], dirs[1], path)
+			foundPaths = append(foundPaths, path)
+		}
+		return filepath.SkipDir // don't need to go deeper
+	})
 }