From c7a896764cd619eff420f5776b266d71805e013f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 25 May 2020 15:42:49 +0200 Subject: [PATCH] Generate package index.json (#479) --- CHANGELOG.md | 1 + archiver/archive.go | 38 ++- dev/generator/main.go | 14 - ...n-not-found.txt => artifact-not-found.txt} | 0 docs/api/artifact-package-invalid-version.txt | 1 + docs/api/artifact-package-not-found.txt | 1 + .../artifact-package-version-not-found.txt | 1 + docs/api/example-0.0.2.tar.gz-preview.txt | 2 +- docs/api/index-package-invalid-version.txt | 1 + docs/api/index-package-not-found.txt | 1 + docs/api/index-package-revision-not-found.txt | 1 + main.go | 4 + main_test.go | 37 ++- package_index.go | 84 +++++ .../package/datasources/1.0.0/index.json | 314 ------------------ .../package/default-pipeline/0.0.2/index.json | 88 ----- .../public/package/example/0.0.2/index.json | 44 --- .../public/package/example/1.0.0/index.json | 120 ------- .../package/experimental/0.0.1/index.json | 25 -- testdata/public/package/foo/1.0.0/index.json | 29 -- .../public/package/internal/1.2.0/index.json | 24 -- .../package/multiple-false/0.0.1/index.json | 88 ----- util/package.go | 21 +- 23 files changed, 185 insertions(+), 754 deletions(-) rename docs/api/{package-version-not-found.txt => artifact-not-found.txt} (100%) create mode 100644 docs/api/artifact-package-invalid-version.txt create mode 100644 docs/api/artifact-package-not-found.txt create mode 100644 docs/api/artifact-package-version-not-found.txt create mode 100644 docs/api/index-package-invalid-version.txt create mode 100644 docs/api/index-package-not-found.txt create mode 100644 docs/api/index-package-revision-not-found.txt create mode 100644 package_index.go delete mode 100644 testdata/public/package/datasources/1.0.0/index.json delete mode 100644 testdata/public/package/default-pipeline/0.0.2/index.json delete mode 100644 testdata/public/package/example/0.0.2/index.json delete mode 100644 testdata/public/package/example/1.0.0/index.json delete mode 100644 testdata/public/package/experimental/0.0.1/index.json delete mode 100644 testdata/public/package/foo/1.0.0/index.json delete mode 100644 testdata/public/package/internal/1.2.0/index.json delete mode 100644 testdata/public/package/multiple-false/0.0.1/index.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 740aeec3b..2ac89159b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Serve favicon as embedded resource. [#468](https://github.com/elastic/package-registry/pull/468) * Generate index.json file. [#470](https://github.com/elastic/package-registry/pull/470) * Stream archived package content. [#472](https://github.com/elastic/package-registry/pull/472) +* Generate package index.json files. [#479](https://github.com/elastic/package-registry/pull/479) ### Deprecated diff --git a/archiver/archive.go b/archiver/archive.go index 45fd44a04..73a9d0ab9 100644 --- a/archiver/archive.go +++ b/archiver/archive.go @@ -7,6 +7,7 @@ package archiver import ( "archive/tar" "compress/gzip" + "encoding/json" "fmt" "io" "os" @@ -14,8 +15,9 @@ import ( "strings" "github.com/joeshaw/multierror" - "github.com/pkg/errors" + + "github.com/elastic/package-registry/util" ) // PackageProperties defines properties describing the package. The structure is used for archiving. @@ -53,6 +55,13 @@ func ArchivePackage(w io.Writer, properties PackageProperties) (err error) { rootDir := fmt.Sprintf("%s-%s", properties.Name, properties.Version) + // Kibana still relies on the "index.json" file to be present in the archive. + // This asset can be removed in the future, once not needed anymore. + err = writePackageIndexToArchive(properties.Path, rootDir, tarWriter) + if err != nil { + return errors.Wrapf(err, "writing package index failed") + } + err = filepath.Walk(properties.Path, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -114,6 +123,33 @@ func buildArchiveHeader(info os.FileInfo, relativePath string) (*tar.Header, err return header, nil } +func writePackageIndexToArchive(path, rootDir string, tarWriter *tar.Writer) error { + aPackage, err := util.NewPackageWithResources(path) + if err != nil { + return errors.Wrapf(err, "building package failed (path: %s)", path) + } + + packageIndexBody, err := json.MarshalIndent(aPackage, "", " ") + if err != nil { + return errors.Wrapf(err, "marshaling package 'index.json' failed (path: %s)", path) + } + + err = tarWriter.WriteHeader(&tar.Header{ + Name: filepath.Join(rootDir, "index.json"), + Size: int64(len(packageIndexBody)), + Mode: 0644, + }) + if err != nil { + return errors.Wrapf(err, "writing package 'index.json' header failed (path: %s)", path) + } + + _, err = tarWriter.Write(packageIndexBody) + if err != nil { + return errors.Wrapf(err, "writing package 'index.json' body failed (path: %s)", path) + } + return nil +} + func writeFileContentToArchive(path string, writer io.Writer) (err error) { var f *os.File f, err = os.Open(path) diff --git a/dev/generator/main.go b/dev/generator/main.go index 2879e5ded..c444fad26 100644 --- a/dev/generator/main.go +++ b/dev/generator/main.go @@ -178,11 +178,6 @@ func buildPackage(packagesBasePath string, p util.Package) error { return err } - err = writeJsonFile(p, filepath.Join(packagesBasePath, p.GetPath(), "index.json")) - if err != nil { - return err - } - // Get all Kibana files savedObjects1, err := filepath.Glob(filepath.Join(packagesBasePath, p.GetPath(), "dataset", "*", "kibana", "*", "*")) if err != nil { @@ -214,15 +209,6 @@ func buildPackage(packagesBasePath string, p util.Package) error { return nil } -func writeJsonFile(v interface{}, path string) error { - data, err := json.MarshalIndent(v, "", " ") - if err != nil { - return err - } - - return ioutil.WriteFile(path, data, 0644) -} - var ( fieldsToEncode = []string{ "attributes.kibanaSavedObjectMeta.searchSourceJSON", diff --git a/docs/api/package-version-not-found.txt b/docs/api/artifact-not-found.txt similarity index 100% rename from docs/api/package-version-not-found.txt rename to docs/api/artifact-not-found.txt diff --git a/docs/api/artifact-package-invalid-version.txt b/docs/api/artifact-package-invalid-version.txt new file mode 100644 index 000000000..ada053801 --- /dev/null +++ b/docs/api/artifact-package-invalid-version.txt @@ -0,0 +1 @@ +invalid package version diff --git a/docs/api/artifact-package-not-found.txt b/docs/api/artifact-package-not-found.txt new file mode 100644 index 000000000..b49b07fba --- /dev/null +++ b/docs/api/artifact-package-not-found.txt @@ -0,0 +1 @@ +artifact not found diff --git a/docs/api/artifact-package-version-not-found.txt b/docs/api/artifact-package-version-not-found.txt new file mode 100644 index 000000000..b49b07fba --- /dev/null +++ b/docs/api/artifact-package-version-not-found.txt @@ -0,0 +1 @@ +artifact not found diff --git a/docs/api/example-0.0.2.tar.gz-preview.txt b/docs/api/example-0.0.2.tar.gz-preview.txt index f3532105f..6de5ddf1b 100644 --- a/docs/api/example-0.0.2.tar.gz-preview.txt +++ b/docs/api/example-0.0.2.tar.gz-preview.txt @@ -1,3 +1,4 @@ +1916 example-0.0.2/index.json 0 example-0.0.2/docs/ 622 example-0.0.2/docs/README.md 0 example-0.0.2/elasticsearch/ @@ -9,7 +10,6 @@ 900 example-0.0.2/elasticsearch/ingest-pipeline/pipeline-tcp.json 0 example-0.0.2/img/ 482070 example-0.0.2/img/kibana-envoyproxy.jpg -1916 example-0.0.2/index.json 0 example-0.0.2/kibana/ 0 example-0.0.2/kibana/dashboard/ 2221 example-0.0.2/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json diff --git a/docs/api/index-package-invalid-version.txt b/docs/api/index-package-invalid-version.txt new file mode 100644 index 000000000..ada053801 --- /dev/null +++ b/docs/api/index-package-invalid-version.txt @@ -0,0 +1 @@ +invalid package version diff --git a/docs/api/index-package-not-found.txt b/docs/api/index-package-not-found.txt new file mode 100644 index 000000000..9ad9fb8cf --- /dev/null +++ b/docs/api/index-package-not-found.txt @@ -0,0 +1 @@ +package revision not found diff --git a/docs/api/index-package-revision-not-found.txt b/docs/api/index-package-revision-not-found.txt new file mode 100644 index 000000000..9ad9fb8cf --- /dev/null +++ b/docs/api/index-package-revision-not-found.txt @@ -0,0 +1 @@ +package revision not found diff --git a/main.go b/main.go index e3fcc6c16..e929593b0 100644 --- a/main.go +++ b/main.go @@ -130,6 +130,8 @@ func getRouter(config Config, packagesBasePath string) (*mux.Router, error) { return nil, err } + packageIndexHandler := packageIndexHandler(packagesBasePath, config.CacheTimeCatchAll) + router := mux.NewRouter().StrictSlash(true) router.HandleFunc("/", indexHandlerFunc) router.HandleFunc("/index.json", indexHandlerFunc) @@ -138,6 +140,8 @@ func getRouter(config Config, packagesBasePath string) (*mux.Router, error) { router.HandleFunc("/health", healthHandler) router.HandleFunc("/favicon.ico", faviconHandleFunc) router.HandleFunc(artifactsRouterPath, artifactsHandler) + router.HandleFunc(packageIndexRouterPath1, packageIndexHandler) + router.HandleFunc(packageIndexRouterPath2, packageIndexHandler) router.PathPrefix("/package").HandlerFunc(catchAll(http.Dir(config.PublicDir), config.CacheTimeCatchAll)) router.Use(loggingMiddleware) return router, nil diff --git a/main_test.go b/main_test.go index c204529db..0ad5b2e72 100644 --- a/main_test.go +++ b/main_test.go @@ -63,7 +63,6 @@ func TestEndpoints(t *testing.T) { {"/search?internal=bar", "/search", "search-package-internal-error.json", searchHandler(packagesBasePath, testCacheTime)}, {"/search?experimental=true", "/search", "search-package-experimental.json", searchHandler(packagesBasePath, testCacheTime)}, {"/search?experimental=foo", "/search", "search-package-experimental-error.json", searchHandler(packagesBasePath, testCacheTime)}, - {"/package/example/1.0.0", "", "package.json", catchAll(http.Dir(publicPath), testCacheTime)}, {"/favicon.ico", "", "favicon.ico", faviconHandleFunc}, } @@ -87,9 +86,39 @@ func TestArtifacts(t *testing.T) { handler func(w http.ResponseWriter, r *http.Request) }{ {"/epr/example/example-0.0.2.tar.gz", artifactsRouterPath, "example-0.0.2.tar.gz-preview.txt", artifactsHandler}, - {"/epr/example/example-999.0.2.tar.gz", artifactsRouterPath, "package-version-not-found.txt", artifactsHandler}, - {"/epr/example/missing-0.1.2.tar.gz", artifactsRouterPath, "package-missing.txt", artifactsHandler}, - {"/epr/example/example-a.b.c.tar.gz", artifactsRouterPath, "package-invalid-version.txt", artifactsHandler}, + {"/epr/example/example-999.0.2.tar.gz", artifactsRouterPath, "artifact-package-version-not-found.txt", artifactsHandler}, + {"/epr/example/missing-0.1.2.tar.gz", artifactsRouterPath, "artifact-package-not-found.txt", artifactsHandler}, + {"/epr/example/example-a.b.c.tar.gz", artifactsRouterPath, "artifact-package-invalid-version.txt", artifactsHandler}, + } + + for _, test := range tests { + t.Run(test.endpoint, func(t *testing.T) { + runEndpoint(t, test.endpoint, test.path, test.file, test.handler) + }) + } +} + +func TestPackageIndex(t *testing.T) { + publicPath := "./testdata/public" + packagesBasePath := publicPath + "/package" + + packageIndexHandler := packageIndexHandler(packagesBasePath, testCacheTime) + + tests := []struct { + endpoint string + path string + file string + handler func(w http.ResponseWriter, r *http.Request) + }{ + {"/package/example/1.0.0/index.json", packageIndexRouterPath1, "package.json", packageIndexHandler}, + {"/package/missing/1.0.0/index.json", packageIndexRouterPath1, "index-package-not-found.txt", packageIndexHandler}, + {"/package/example/999.0.0/index.json", packageIndexRouterPath1, "index-package-revision-not-found.txt", packageIndexHandler}, + {"/package/example/a.b.c/index.json", packageIndexRouterPath1, "index-package-invalid-version.txt", packageIndexHandler}, + + {"/package/example/1.0.0/", packageIndexRouterPath2, "package.json", packageIndexHandler}, + {"/package/missing/1.0.0/", packageIndexRouterPath2, "index-package-not-found.txt", packageIndexHandler}, + {"/package/example/999.0.0/", packageIndexRouterPath2, "index-package-revision-not-found.txt", packageIndexHandler}, + {"/package/example/a.b.c/", packageIndexRouterPath2, "index-package-invalid-version.txt", packageIndexHandler}, } for _, test := range tests { diff --git a/package_index.go b/package_index.go new file mode 100644 index 000000000..c6afe3cde --- /dev/null +++ b/package_index.go @@ -0,0 +1,84 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + + "github.com/blang/semver" + "github.com/gorilla/mux" + + "github.com/elastic/package-registry/util" +) + +const ( + packageIndexRouterPath1 = "/package/{packageName:[a-z_]+}/{packageVersion}/index.json" + packageIndexRouterPath2 = "/package/{packageName:[a-z_]+}/{packageVersion}/" +) + +var errPackageRevisionNotFound = errors.New("package revision not found") + +func packageIndexHandler(packagesBasePath string, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + packageName, ok := vars["packageName"] + if !ok { + badRequest(w, "missing package name") + return + } + + packageVersion, ok := vars["packageVersion"] + if !ok { + badRequest(w, "missing package version") + return + } + + _, err := semver.Parse(packageVersion) + if err != nil { + badRequest(w, "invalid package version") + return + } + + packagePath := filepath.Join(packagesBasePath, packageName, packageVersion) + _, err = os.Stat(packagePath) + if os.IsNotExist(err) { + notFoundError(w, errPackageRevisionNotFound) + return + } + if err != nil { + log.Printf("stat package path '%s' failed: %v", packagePath, err) + + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + cacheHeaders(w, cacheTime) + + aPackage, err := util.NewPackageWithResources(packagePath) + if err != nil { + log.Printf("loading package from path '%s' failed: %v", packagePath, err) + + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + body, err := json.MarshalIndent(aPackage, "", " ") + if err != nil { + log.Printf("marshaling package index failed (path '%s'): %v", packagePath, err) + + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + w.Write(body) + } +} diff --git a/testdata/public/package/datasources/1.0.0/index.json b/testdata/public/package/datasources/1.0.0/index.json deleted file mode 100644 index b56d2bf5d..000000000 --- a/testdata/public/package/datasources/1.0.0/index.json +++ /dev/null @@ -1,314 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "datasources", - "title": "Default datasource Integration", - "version": "1.0.0", - "readme": "/package/datasources/1.0.0/docs/README.md", - "license": "basic", - "description": "Package with data sources", - "type": "integration", - "categories": [ - "logs" - ], - "release": "beta", - "removable": true, - "requirement": { - "kibana": {}, - "elasticsearch": {} - }, - "assets": [ - "/package/datasources/1.0.0/manifest.yml", - "/package/datasources/1.0.0/docs/README.md", - "/package/datasources/1.0.0/dataset/examplelog1/manifest.yml", - "/package/datasources/1.0.0/dataset/examplelog2/manifest.yml", - "/package/datasources/1.0.0/dataset/examplemetric/manifest.yml", - "/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml", - "/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml", - "/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml", - "/package/datasources/1.0.0/dataset/examplelog1/agent/stream/logs.yml", - "/package/datasources/1.0.0/dataset/examplelog1/agent/stream/syslog.yml", - "/package/datasources/1.0.0/dataset/examplelog2/agent/stream/stream.yml.hbs", - "/package/datasources/1.0.0/dataset/examplemetric/agent/stream/stream.yml.hbs" - ], - "datasets": [ - { - "id": "datasources.examplelog1", - "title": "Example dataset with inputs", - "release": "experimental", - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx error log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/error.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/error.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*error.log*" - ] - } - } - } - ], - "template_path": "logs.yml", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - }, - { - "input": "syslog", - "template_path": "syslog.yml", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ], - "package": "datasources", - "path": "examplelog1" - }, - { - "id": "datasources.examplelog2", - "title": "Example dataset with inputs", - "release": "experimental", - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/access.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/access.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*access.log*" - ] - } - } - } - ], - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ], - "package": "datasources", - "path": "examplelog2" - }, - { - "id": "datasources.examplemetric", - "title": "Example dataset with inputs", - "release": "experimental", - "type": "metrics", - "streams": [ - { - "input": "nginx/metrics", - "vars": [ - { - "name": "url", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": false, - "required": true, - "show_user": false, - "default": "localhost" - } - ], - "title": "Title of the stream", - "description": "Not enabled data source.", - "enabled": false - } - ], - "package": "datasources", - "path": "examplemetric" - } - ], - "datasources": [ - { - "name": "nginx", - "title": "Datasource title", - "description": "Details about the data source.", - "inputs": [ - { - "type": "nginx/metrics", - "vars": [ - { - "name": "hosts", - "type": "text", - "description": "Nginx hosts", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "http://127.0.0.1" - ] - }, - { - "name": "period", - "type": "duration", - "description": "Collection period. Valid values: 10s, 5m, 2h", - "multi": false, - "required": false, - "show_user": false, - "default": "10s" - }, - { - "name": "username", - "type": "text", - "multi": false, - "required": false, - "show_user": false - }, - { - "name": "password", - "type": "password", - "multi": false, - "required": false, - "show_user": false - } - ], - "description": "Collecting metrics for nginx.", - "streams": [ - { - "input": "nginx/metrics", - "vars": [ - { - "name": "url", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": false, - "required": true, - "show_user": false, - "default": "localhost" - } - ], - "dataset": "datasources.examplemetric", - "template_path": "stream.yml.hbs", - "template": "metric: foo\n", - "title": "Title of the stream", - "description": "Not enabled data source.", - "enabled": false - } - ] - }, - { - "type": "logs", - "description": "Collect nginx logs.", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx error log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/error.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/error.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*error.log*" - ] - } - } - } - ], - "dataset": "datasources.examplelog1", - "template_path": "logs.yml", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - }, - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/access.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/access.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*access.log*" - ] - } - } - } - ], - "dataset": "datasources.examplelog2", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - }, - { - "type": "syslog", - "streams": [ - { - "input": "syslog", - "dataset": "datasources.examplelog1", - "template_path": "syslog.yml", - "template": "syslog: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], - "download": "/epr/datasources/datasources-1.0.0.tar.gz", - "path": "/package/datasources/1.0.0" -} \ No newline at end of file diff --git a/testdata/public/package/default-pipeline/0.0.2/index.json b/testdata/public/package/default-pipeline/0.0.2/index.json deleted file mode 100644 index 8b7da2f66..000000000 --- a/testdata/public/package/default-pipeline/0.0.2/index.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "default-pipeline", - "title": "Default pipeline Integration", - "version": "0.0.2", - "readme": "/package/default-pipeline/0.0.2/docs/README.md", - "license": "basic", - "description": "Tests if no pipeline is set, it defaults to the default one", - "type": "integration", - "categories": [ - "logs" - ], - "release": "beta", - "removable": true, - "requirement": { - "kibana": {}, - "elasticsearch": {} - }, - "assets": [ - "/package/default-pipeline/0.0.2/manifest.yml", - "/package/default-pipeline/0.0.2/docs/README.md", - "/package/default-pipeline/0.0.2/dataset/foo/manifest.yml", - "/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml", - "/package/default-pipeline/0.0.2/dataset/foo/agent/stream/stream.yml.hbs", - "/package/default-pipeline/0.0.2/dataset/foo/elasticsearch/ingest-pipeline/default.json" - ], - "datasets": [ - { - "id": "default-pipeline.foo", - "title": "Foo", - "release": "experimental", - "type": "logs", - "ingest_pipeline": "default", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "enabled": true - } - ], - "package": "default-pipeline", - "path": "foo" - } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "default-pipeline.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], - "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", - "path": "/package/default-pipeline/0.0.2" -} \ No newline at end of file diff --git a/testdata/public/package/example/0.0.2/index.json b/testdata/public/package/example/0.0.2/index.json deleted file mode 100644 index 2ca508bcd..000000000 --- a/testdata/public/package/example/0.0.2/index.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "example", - "title": "Example", - "version": "0.0.2", - "readme": "/package/example/0.0.2/docs/README.md", - "license": "basic", - "description": "This is the example integration.", - "type": "integration", - "categories": [ - "logs" - ], - "release": "beta", - "removable": true, - "requirement": { - "kibana": { - "versions": "\u003e=6.0.0" - }, - "elasticsearch": { - "versions": "\u003e=6.0.0 \u003c=6.8.0" - } - }, - "assets": [ - "/package/example/0.0.2/manifest.yml", - "/package/example/0.0.2/docs/README.md", - "/package/example/0.0.2/img/kibana-envoyproxy.jpg", - "/package/example/0.0.2/elasticsearch/ingest-pipeline/pipeline-entry.json", - "/package/example/0.0.2/elasticsearch/ingest-pipeline/pipeline-http.json", - "/package/example/0.0.2/elasticsearch/ingest-pipeline/pipeline-json.json", - "/package/example/0.0.2/elasticsearch/ingest-pipeline/pipeline-plaintext.json", - "/package/example/0.0.2/elasticsearch/ingest-pipeline/pipeline-tcp.json", - "/package/example/0.0.2/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json", - "/package/example/0.0.2/kibana/index-pattern/filebeat.json", - "/package/example/0.0.2/kibana/infrastructure-ui-source/default.json", - "/package/example/0.0.2/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json", - "/package/example/0.0.2/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json", - "/package/example/0.0.2/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json", - "/package/example/0.0.2/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json", - "/package/example/0.0.2/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json", - "/package/example/0.0.2/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json" - ], - "download": "/epr/example/example-0.0.2.tar.gz", - "path": "/package/example/0.0.2" -} \ No newline at end of file diff --git a/testdata/public/package/example/1.0.0/index.json b/testdata/public/package/example/1.0.0/index.json deleted file mode 100644 index 6ab742c8a..000000000 --- a/testdata/public/package/example/1.0.0/index.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "example", - "title": "Example Integration", - "version": "1.0.0", - "readme": "/package/example/1.0.0/docs/README.md", - "license": "basic", - "description": "This is the example integration", - "type": "integration", - "categories": [ - "logs", - "metrics" - ], - "release": "ga", - "removable": true, - "requirement": { - "kibana": { - "versions": "\u003e=7.0.0" - }, - "elasticsearch": { - "versions": "\u003e=7.0.0 \u003c=7.5.0" - } - }, - "screenshots": [ - { - "src": "/package/example/1.0.0/img/kibana-iptables.png", - "title": "IP Tables Overview dashboard", - "size": "1492x1382" - }, - { - "src": "/package/example/1.0.0/img/kibana-iptables-ubiquity.png", - "title": "IP Tables Ubiquity Dashboard", - "size": "1492x1464", - "type": "image/png" - } - ], - "assets": [ - "/package/example/1.0.0/manifest.yml", - "/package/example/1.0.0/docs/README.md", - "/package/example/1.0.0/img/icon.png", - "/package/example/1.0.0/img/kibana-envoyproxy.jpg", - "/package/example/1.0.0/dataset/foo/manifest.yml", - "/package/example/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/kibana/index-pattern/filebeat.json", - "/package/example/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json", - "/package/example/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json", - "/package/example/1.0.0/dataset/foo/fields/base-fields.yml", - "/package/example/1.0.0/dataset/foo/agent/stream/stream.yml.hbs", - "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json", - "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json", - "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json", - "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json", - "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json" - ], - "datasets": [ - { - "id": "bar", - "title": "Foo", - "release": "experimental", - "type": "logs", - "ingest_pipeline": "pipeline-entry", - "streams": [ - { - "input": "foo", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "enabled": true - } - ], - "package": "example", - "path": "foo" - } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "foo", - "streams": [ - { - "input": "foo", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "bar", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], - "download": "/epr/example/example-1.0.0.tar.gz", - "path": "/package/example/1.0.0" -} \ No newline at end of file diff --git a/testdata/public/package/experimental/0.0.1/index.json b/testdata/public/package/experimental/0.0.1/index.json deleted file mode 100644 index 0b3d27ee7..000000000 --- a/testdata/public/package/experimental/0.0.1/index.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "experimental", - "title": "Experimental", - "version": "0.0.1", - "readme": "/package/experimental/0.0.1/docs/README.md", - "license": "basic", - "description": "Experimental package, should be set by default", - "type": "solution", - "categories": [ - "metrics" - ], - "release": "experimental", - "removable": true, - "requirement": { - "kibana": {}, - "elasticsearch": {} - }, - "assets": [ - "/package/experimental/0.0.1/manifest.yml", - "/package/experimental/0.0.1/docs/README.md" - ], - "download": "/epr/experimental/experimental-0.0.1.tar.gz", - "path": "/package/experimental/0.0.1" -} \ No newline at end of file diff --git a/testdata/public/package/foo/1.0.0/index.json b/testdata/public/package/foo/1.0.0/index.json deleted file mode 100644 index 5403b15c0..000000000 --- a/testdata/public/package/foo/1.0.0/index.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "foo", - "title": "Foo", - "version": "1.0.0", - "readme": "/package/foo/1.0.0/docs/README.md", - "license": "basic", - "description": "This is the foo integration", - "type": "solution", - "categories": [ - "metrics" - ], - "release": "beta", - "removable": true, - "requirement": { - "kibana": { - "versions": "\u003e=7.0.0" - }, - "elasticsearch": { - "versions": "\u003e=7.0.0 \u003c=7.5.0" - } - }, - "assets": [ - "/package/foo/1.0.0/manifest.yml", - "/package/foo/1.0.0/docs/README.md" - ], - "download": "/epr/foo/foo-1.0.0.tar.gz", - "path": "/package/foo/1.0.0" -} \ No newline at end of file diff --git a/testdata/public/package/internal/1.2.0/index.json b/testdata/public/package/internal/1.2.0/index.json deleted file mode 100644 index 22b21e43e..000000000 --- a/testdata/public/package/internal/1.2.0/index.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "internal", - "title": "Internal", - "version": "1.2.0", - "readme": "/package/internal/1.2.0/docs/README.md", - "license": "basic", - "description": "Internal package", - "type": "integration", - "categories": [], - "release": "beta", - "removable": false, - "requirement": { - "kibana": {}, - "elasticsearch": {} - }, - "assets": [ - "/package/internal/1.2.0/manifest.yml", - "/package/internal/1.2.0/docs/README.md" - ], - "internal": true, - "download": "/epr/internal/internal-1.2.0.tar.gz", - "path": "/package/internal/1.2.0" -} \ No newline at end of file diff --git a/testdata/public/package/multiple-false/0.0.1/index.json b/testdata/public/package/multiple-false/0.0.1/index.json deleted file mode 100644 index 3cbab4c84..000000000 --- a/testdata/public/package/multiple-false/0.0.1/index.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "format_version": "1.0.0", - "name": "multiple-false", - "title": "Multiple false", - "version": "0.0.1", - "readme": "/package/multiple-false/0.0.1/docs/README.md", - "license": "basic", - "description": "Tests that multiple can be set to false", - "type": "integration", - "categories": [ - "logs" - ], - "release": "beta", - "removable": true, - "requirement": { - "kibana": {}, - "elasticsearch": {} - }, - "assets": [ - "/package/multiple-false/0.0.1/manifest.yml", - "/package/multiple-false/0.0.1/docs/README.md", - "/package/multiple-false/0.0.1/dataset/foo/manifest.yml", - "/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml", - "/package/multiple-false/0.0.1/dataset/foo/agent/stream/stream.yml.hbs", - "/package/multiple-false/0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json" - ], - "datasets": [ - { - "id": "multiple-false.foo", - "title": "Foo", - "release": "experimental", - "type": "logs", - "ingest_pipeline": "default", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "enabled": true - } - ], - "package": "multiple-false", - "path": "foo" - } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "multiple-false.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": false - } - ], - "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz", - "path": "/package/multiple-false/0.0.1" -} \ No newline at end of file diff --git a/util/package.go b/util/package.go index 590c64664..ad8057aa9 100644 --- a/util/package.go +++ b/util/package.go @@ -12,9 +12,8 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" - "github.com/blang/semver" + "github.com/pkg/errors" ucfg "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" @@ -179,6 +178,24 @@ func NewPackage(basePath string) (*Package, error) { return p, nil } +func NewPackageWithResources(path string) (*Package, error) { + aPackage, err := NewPackage(path) + if err != nil { + return nil, errors.Wrapf(err, "building package from path '%s' failed", path) + } + + err = aPackage.LoadAssets(aPackage.GetPath()) + if err != nil { + return nil, errors.Wrapf(err, "loading package assets failed (path '%s')", path) + } + + err = aPackage.LoadDataSets(aPackage.GetPath()) + if err != nil { + return nil, errors.Wrapf(err, "loading package datasets failed (path '%s')", path) + } + return aPackage, nil +} + func (p *Package) HasCategory(category string) bool { for _, c := range p.Categories { if c == category {