Generate package index.json (#479)

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Serve favicon as embedded resource. [#468](https://github.com/elastic/package-registry/pull/468)
 * Generate index.json file. [#470](https://github.com/elastic/package-registry/pull/470)
 * Stream archived package content. [#472](https://github.com/elastic/package-registry/pull/472)
+* Generate package index.json files. [#479](https://github.com/elastic/package-registry/pull/479)
 
 ### Deprecated
 

archiver/archive.go

@@ -7,15 +7,17 @@ package archiver
 import (
 	"archive/tar"
 	"compress/gzip"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/joeshaw/multierror"
-
 	"github.com/pkg/errors"
+
+	"github.com/elastic/package-registry/util"
 )
 
 // PackageProperties defines properties describing the package. The structure is used for archiving.
@@ -53,6 +55,13 @@ func ArchivePackage(w io.Writer, properties PackageProperties) (err error) {
 
 	rootDir := fmt.Sprintf("%s-%s", properties.Name, properties.Version)
 
+	// Kibana still relies on the "index.json" file to be present in the archive.
+	// This asset can be removed in the future, once not needed anymore.
+	err = writePackageIndexToArchive(properties.Path, rootDir, tarWriter)
+	if err != nil {
+		return errors.Wrapf(err, "writing package index failed")
+	}
+
 	err = filepath.Walk(properties.Path, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -114,6 +123,33 @@ func buildArchiveHeader(info os.FileInfo, relativePath string) (*tar.Header, err
 	return header, nil
 }
 
+func writePackageIndexToArchive(path, rootDir string, tarWriter *tar.Writer) error {
+	aPackage, err := util.NewPackageWithResources(path)
+	if err != nil {
+		return errors.Wrapf(err, "building package failed (path: %s)", path)
+	}
+
+	packageIndexBody, err := json.MarshalIndent(aPackage, "", "  ")
+	if err != nil {
+		return errors.Wrapf(err, "marshaling package 'index.json' failed (path: %s)", path)
+	}
+
+	err = tarWriter.WriteHeader(&tar.Header{
+		Name: filepath.Join(rootDir, "index.json"),
+		Size: int64(len(packageIndexBody)),
+		Mode: 0644,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "writing package 'index.json' header failed (path: %s)", path)
+	}
+
+	_, err = tarWriter.Write(packageIndexBody)
+	if err != nil {
+		return errors.Wrapf(err, "writing package 'index.json' body failed (path: %s)", path)
+	}
+	return nil
+}
+
 func writeFileContentToArchive(path string, writer io.Writer) (err error) {
 	var f *os.File
 	f, err = os.Open(path)

dev/generator/main.go

@@ -178,11 +178,6 @@ func buildPackage(packagesBasePath string, p util.Package) error {
 		return err
 	}
 
-	err = writeJsonFile(p, filepath.Join(packagesBasePath, p.GetPath(), "index.json"))
-	if err != nil {
-		return err
-	}
-
 	// Get all Kibana files
 	savedObjects1, err := filepath.Glob(filepath.Join(packagesBasePath, p.GetPath(), "dataset", "*", "kibana", "*", "*"))
 	if err != nil {
@@ -214,15 +209,6 @@ func buildPackage(packagesBasePath string, p util.Package) error {
 	return nil
 }
 
-func writeJsonFile(v interface{}, path string) error {
-	data, err := json.MarshalIndent(v, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(path, data, 0644)
-}
-
 var (
 	fieldsToEncode = []string{
 		"attributes.kibanaSavedObjectMeta.searchSourceJSON",

docs/api/artifact-package-invalid-version.txt

@@ -0,0 +1 @@
+invalid package version

docs/api/artifact-package-not-found.txt

@@ -0,0 +1 @@
+artifact not found

docs/api/artifact-package-version-not-found.txt

@@ -0,0 +1 @@
+artifact not found

docs/api/example-0.0.2.tar.gz-preview.txt

@@ -1,3 +1,4 @@
+1916 example-0.0.2/index.json
 0 example-0.0.2/docs/
 622 example-0.0.2/docs/README.md
 0 example-0.0.2/elasticsearch/
@@ -9,7 +10,6 @@
 900 example-0.0.2/elasticsearch/ingest-pipeline/pipeline-tcp.json
 0 example-0.0.2/img/
 482070 example-0.0.2/img/kibana-envoyproxy.jpg
-1916 example-0.0.2/index.json
 0 example-0.0.2/kibana/
 0 example-0.0.2/kibana/dashboard/
 2221 example-0.0.2/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json

docs/api/index-package-invalid-version.txt

@@ -0,0 +1 @@
+invalid package version

docs/api/index-package-not-found.txt

@@ -0,0 +1 @@
+package revision not found

docs/api/index-package-revision-not-found.txt

@@ -0,0 +1 @@
+package revision not found

main.go

@@ -130,6 +130,8 @@ func getRouter(config Config, packagesBasePath string) (*mux.Router, error) {
 		return nil, err
 	}
 
+	packageIndexHandler := packageIndexHandler(packagesBasePath, config.CacheTimeCatchAll)
+
 	router := mux.NewRouter().StrictSlash(true)
 	router.HandleFunc("/", indexHandlerFunc)
 	router.HandleFunc("/index.json", indexHandlerFunc)
@@ -138,6 +140,8 @@ func getRouter(config Config, packagesBasePath string) (*mux.Router, error) {
 	router.HandleFunc("/health", healthHandler)
 	router.HandleFunc("/favicon.ico", faviconHandleFunc)
 	router.HandleFunc(artifactsRouterPath, artifactsHandler)
+	router.HandleFunc(packageIndexRouterPath1, packageIndexHandler)
+	router.HandleFunc(packageIndexRouterPath2, packageIndexHandler)
 	router.PathPrefix("/package").HandlerFunc(catchAll(http.Dir(config.PublicDir), config.CacheTimeCatchAll))
 	router.Use(loggingMiddleware)
 	return router, nil

main_test.go

@@ -63,7 +63,6 @@ func TestEndpoints(t *testing.T) {
 		{"/search?internal=bar", "/search", "search-package-internal-error.json", searchHandler(packagesBasePath, testCacheTime)},
 		{"/search?experimental=true", "/search", "search-package-experimental.json", searchHandler(packagesBasePath, testCacheTime)},
 		{"/search?experimental=foo", "/search", "search-package-experimental-error.json", searchHandler(packagesBasePath, testCacheTime)},
-		{"/package/example/1.0.0", "", "package.json", catchAll(http.Dir(publicPath), testCacheTime)},
 		{"/favicon.ico", "", "favicon.ico", faviconHandleFunc},
 	}
 
@@ -87,9 +86,39 @@ func TestArtifacts(t *testing.T) {
 		handler  func(w http.ResponseWriter, r *http.Request)
 	}{
 		{"/epr/example/example-0.0.2.tar.gz", artifactsRouterPath, "example-0.0.2.tar.gz-preview.txt", artifactsHandler},
-		{"/epr/example/example-999.0.2.tar.gz", artifactsRouterPath, "package-version-not-found.txt", artifactsHandler},
-		{"/epr/example/missing-0.1.2.tar.gz", artifactsRouterPath, "package-missing.txt", artifactsHandler},
-		{"/epr/example/example-a.b.c.tar.gz", artifactsRouterPath, "package-invalid-version.txt", artifactsHandler},
+		{"/epr/example/example-999.0.2.tar.gz", artifactsRouterPath, "artifact-package-version-not-found.txt", artifactsHandler},
+		{"/epr/example/missing-0.1.2.tar.gz", artifactsRouterPath, "artifact-package-not-found.txt", artifactsHandler},
+		{"/epr/example/example-a.b.c.tar.gz", artifactsRouterPath, "artifact-package-invalid-version.txt", artifactsHandler},
+	}
+
+	for _, test := range tests {
+		t.Run(test.endpoint, func(t *testing.T) {
+			runEndpoint(t, test.endpoint, test.path, test.file, test.handler)
+		})
+	}
+}
+
+func TestPackageIndex(t *testing.T) {
+	publicPath := "./testdata/public"
+	packagesBasePath := publicPath + "/package"
+
+	packageIndexHandler := packageIndexHandler(packagesBasePath, testCacheTime)
+
+	tests := []struct {
+		endpoint string
+		path     string
+		file     string
+		handler  func(w http.ResponseWriter, r *http.Request)
+	}{
+		{"/package/example/1.0.0/index.json", packageIndexRouterPath1, "package.json", packageIndexHandler},
+		{"/package/missing/1.0.0/index.json", packageIndexRouterPath1, "index-package-not-found.txt", packageIndexHandler},
+		{"/package/example/999.0.0/index.json", packageIndexRouterPath1, "index-package-revision-not-found.txt", packageIndexHandler},
+		{"/package/example/a.b.c/index.json", packageIndexRouterPath1, "index-package-invalid-version.txt", packageIndexHandler},
+
+		{"/package/example/1.0.0/", packageIndexRouterPath2, "package.json", packageIndexHandler},
+		{"/package/missing/1.0.0/", packageIndexRouterPath2, "index-package-not-found.txt", packageIndexHandler},
+		{"/package/example/999.0.0/", packageIndexRouterPath2, "index-package-revision-not-found.txt", packageIndexHandler},
+		{"/package/example/a.b.c/", packageIndexRouterPath2, "index-package-invalid-version.txt", packageIndexHandler},
 	}
 
 	for _, test := range tests {

package_index.go

@@ -0,0 +1,84 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/blang/semver"
+	"github.com/gorilla/mux"
+
+	"github.com/elastic/package-registry/util"
+)
+
+const (
+	packageIndexRouterPath1 = "/package/{packageName:[a-z_]+}/{packageVersion}/index.json"
+	packageIndexRouterPath2 = "/package/{packageName:[a-z_]+}/{packageVersion}/"
+)
+
+var errPackageRevisionNotFound = errors.New("package revision not found")
+
+func packageIndexHandler(packagesBasePath string, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		packageName, ok := vars["packageName"]
+		if !ok {
+			badRequest(w, "missing package name")
+			return
+		}
+
+		packageVersion, ok := vars["packageVersion"]
+		if !ok {
+			badRequest(w, "missing package version")
+			return
+		}
+
+		_, err := semver.Parse(packageVersion)
+		if err != nil {
+			badRequest(w, "invalid package version")
+			return
+		}
+
+		packagePath := filepath.Join(packagesBasePath, packageName, packageVersion)
+		_, err = os.Stat(packagePath)
+		if os.IsNotExist(err) {
+			notFoundError(w, errPackageRevisionNotFound)
+			return
+		}
+		if err != nil {
+			log.Printf("stat package path '%s' failed: %v", packagePath, err)
+
+			http.Error(w, "internal server error", http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		cacheHeaders(w, cacheTime)
+
+		aPackage, err := util.NewPackageWithResources(packagePath)
+		if err != nil {
+			log.Printf("loading package from path '%s' failed: %v", packagePath, err)
+
+			http.Error(w, "internal server error", http.StatusInternalServerError)
+			return
+		}
+
+		body, err := json.MarshalIndent(aPackage, "", "  ")
+		if err != nil {
+			log.Printf("marshaling package index failed (path '%s'): %v", packagePath, err)
+
+			http.Error(w, "internal server error", http.StatusInternalServerError)
+			return
+		}
+		w.Write(body)
+	}
+}