diff --git a/cmd/guacingest/cmd/ingest.go b/cmd/guacingest/cmd/ingest.go index c095214b1a..6b4ac02183 100644 --- a/cmd/guacingest/cmd/ingest.go +++ b/cmd/guacingest/cmd/ingest.go @@ -47,6 +47,7 @@ type options struct { headerFile string queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } func ingest(cmd *cobra.Command, args []string) { @@ -60,6 +61,7 @@ func ingest(cmd *cobra.Command, args []string) { viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -99,7 +101,16 @@ func ingest(cmd *cobra.Command, args []string) { defer csubClient.Close() emit := func(d *processor.Document) error { - if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil { + if _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ); err != nil { var urlErr *url.Error if errors.As(err, &urlErr) { return fmt.Errorf("unable to ingest document due to connection error with graphQL %q : %w", d.SourceInformation.Source, urlErr) @@ -130,7 +141,7 @@ func ingest(cmd *cobra.Command, args []string) { } func validateFlags(pubsubAddr, blobAddr, csubAddr, graphqlEndpoint, headerFile string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, args []string) (options, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (options, error) { var opts options opts.pubsubAddr = pubsubAddr opts.blobAddr = blobAddr @@ -143,6 +154,7 @@ func validateFlags(pubsubAddr, blobAddr, csubAddr, graphqlEndpoint, headerFile s opts.headerFile = headerFile opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion return opts, nil } diff --git a/cmd/guacingest/cmd/root.go b/cmd/guacingest/cmd/root.go index e8ea385491..fd1a2d21ae 100644 --- a/cmd/guacingest/cmd/root.go +++ b/cmd/guacingest/cmd/root.go @@ -31,7 +31,7 @@ func init() { cobra.OnInitialize(cli.InitConfig) set, err := cli.BuildFlags([]string{"pubsub-addr", "blob-addr", "csub-addr", "gql-addr", - "header-file", "add-vuln-on-ingest", "add-license-on-ingest"}) + "header-file", "add-vuln-on-ingest", "add-license-on-ingest", "add-eol-on-ingest"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err) os.Exit(1) diff --git a/cmd/guacone/cmd/deps_dev.go b/cmd/guacone/cmd/deps_dev.go index 1d6bed4986..60b7899fd1 100644 --- a/cmd/guacone/cmd/deps_dev.go +++ b/cmd/guacone/cmd/deps_dev.go @@ -52,6 +52,7 @@ type depsDevOptions struct { headerFile string queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool // sets artificial latency on the deps.dev collector (default to nil) addedLatency *time.Duration } @@ -87,7 +88,16 @@ var depsDevCmd = &cobra.Command{ emit := func(d *processor.Document) error { totalNum += 1 - if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csc, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil { + if _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csc, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ); err != nil { gotErr = true return fmt.Errorf("unable to ingest document: %w", err) } @@ -145,6 +155,7 @@ func validateDepsDevFlags(args []string) (*depsDevOptions, client.Client, error) headerFile: viper.GetString("header-file"), queryVulnOnIngestion: viper.GetBool("add-vuln-on-ingest"), queryLicenseOnIngestion: viper.GetBool("add-license-on-ingest"), + queryEOLOnIngestion: viper.GetBool("add-eol-on-ingest"), } addedLatencyStr := viper.GetString("deps-dev-latency") diff --git a/cmd/guacone/cmd/eol.go b/cmd/guacone/cmd/eol.go new file mode 100644 index 0000000000..5b782889be --- /dev/null +++ b/cmd/guacone/cmd/eol.go @@ -0,0 +1,281 @@ +// +// Copyright 2024 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/Khan/genqlient/graphql" + "github.com/guacsec/guac/pkg/assembler/clients/generated" + "github.com/guacsec/guac/pkg/certifier/certify" + "github.com/guacsec/guac/pkg/certifier/components/root_package" + "github.com/guacsec/guac/pkg/certifier/eol" + "github.com/guacsec/guac/pkg/cli" + csub_client "github.com/guacsec/guac/pkg/collectsub/client" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/guacsec/guac/pkg/ingestor" + "github.com/guacsec/guac/pkg/logging" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + eolQuerySize = 1000 +) + +type eolOptions struct { + graphqlEndpoint string + headerFile string + poll bool + csubClientOptions csub_client.CsubClientOptions + interval time.Duration + addedLatency *time.Duration + batchSize int + lastScan *int +} + +var eolCmd = &cobra.Command{ + Use: "eol [flags]", + Short: "runs the End of Life (EOL) certifier", + Run: func(cmd *cobra.Command, args []string) { + opts, err := validateEOLFlags( + viper.GetString("gql-addr"), + viper.GetString("header-file"), + viper.GetString("interval"), + viper.GetString("csub-addr"), + viper.GetBool("poll"), + viper.GetBool("csub-tls"), + viper.GetBool("csub-tls-skip-verify"), + viper.GetString("certifier-latency"), + viper.GetInt("certifier-batch-size"), + viper.GetInt("last-scan"), + ) + if err != nil { + fmt.Printf("unable to validate flags: %v\n", err) + _ = cmd.Help() + os.Exit(1) + } + + ctx := logging.WithLogger(context.Background()) + logger := logging.FromContext(ctx) + transport := cli.HTTPHeaderTransport(ctx, opts.headerFile, http.DefaultTransport) + + if err := certify.RegisterCertifier(eol.NewEOLCertifier, eol.EOLCollector); err != nil { + logger.Fatalf("unable to register certifier: %v", err) + } + + // initialize collectsub client + csubClient, err := csub_client.NewClient(opts.csubClientOptions) + if err != nil { + logger.Infof("collectsub client initialization failed, this ingestion will not pull in any additional data through the collectsub service: %v", err) + csubClient = nil + } else { + defer csubClient.Close() + } + + httpClient := http.Client{Transport: transport} + gqlclient := graphql.NewClient(opts.graphqlEndpoint, &httpClient) + packageQuery := root_package.NewPackageQuery(gqlclient, generated.QueryTypeEol, opts.batchSize, eolQuerySize, opts.addedLatency, opts.lastScan) + + totalNum := 0 + docChan := make(chan *processor.Document) + ingestionStop := make(chan bool, 1) + tickInterval := 30 * time.Second + ticker := time.NewTicker(tickInterval) + + var gotErr int32 + var wg sync.WaitGroup + ingestion := func() { + defer wg.Done() + var totalDocs []*processor.Document + const threshold = 1000 + stop := false + for !stop { + select { + case <-ticker.C: + if len(totalDocs) > 0 { + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + if err != nil { + stop = true + atomic.StoreInt32(&gotErr, 1) + logger.Errorf("unable to ingest documents: %v", err) + } + totalDocs = []*processor.Document{} + } + ticker.Reset(tickInterval) + case d := <-docChan: + totalNum += 1 + totalDocs = append(totalDocs, d) + if len(totalDocs) >= threshold { + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + if err != nil { + stop = true + atomic.StoreInt32(&gotErr, 1) + logger.Errorf("unable to ingest documents: %v", err) + } + totalDocs = []*processor.Document{} + ticker.Reset(tickInterval) + } + case <-ingestionStop: + stop = true + case <-ctx.Done(): + return + } + } + for len(docChan) > 0 { + totalNum += 1 + totalDocs = append(totalDocs, <-docChan) + if len(totalDocs) >= threshold { + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + if err != nil { + atomic.StoreInt32(&gotErr, 1) + logger.Errorf("unable to ingest documents: %v", err) + } + totalDocs = []*processor.Document{} + } + } + if len(totalDocs) > 0 { + err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, false, false, false) + if err != nil { + atomic.StoreInt32(&gotErr, 1) + logger.Errorf("unable to ingest documents: %v", err) + } + } + } + wg.Add(1) + go ingestion() + + // Set emit function to go through the entire pipeline + emit := func(d *processor.Document) error { + docChan <- d + return nil + } + + // Collect + errHandler := func(err error) bool { + if err != nil { + logger.Errorf("certifier ended with error: %v", err) + atomic.StoreInt32(&gotErr, 1) + } + // process documents already captures + return true + } + + ctx, cf := context.WithCancel(ctx) + done := make(chan bool, 1) + wg.Add(1) + go func() { + defer wg.Done() + if err := certify.Certify(ctx, packageQuery, emit, errHandler, opts.poll, opts.interval); err != nil { + logger.Errorf("Unhandled error in the certifier: %s", err) + } + done <- true + }() + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + select { + case s := <-sigs: + logger.Infof("Signal received: %s, shutting down gracefully\n", s.String()) + cf() + case <-done: + logger.Infof("All certifiers completed") + } + ingestionStop <- true + wg.Wait() + cf() + + if atomic.LoadInt32(&gotErr) == 1 { + logger.Errorf("completed ingestion with errors") + } else { + logger.Infof("completed ingesting %v documents", totalNum) + } + }, +} + +func validateEOLFlags( + graphqlEndpoint, + headerFile, + interval, + csubAddr string, + poll, + csubTls, + csubTlsSkipVerify bool, + certifierLatencyStr string, + batchSize int, lastScan int, +) (eolOptions, error) { + var opts eolOptions + opts.graphqlEndpoint = graphqlEndpoint + opts.headerFile = headerFile + opts.poll = poll + + if interval == "" { + // 14 days by default + opts.interval = 14 * 24 * time.Hour + } else { + i, err := time.ParseDuration(interval) + if err != nil { + return opts, err + } + opts.interval = i + } + + if certifierLatencyStr != "" { + addedLatency, err := time.ParseDuration(certifierLatencyStr) + if err != nil { + return opts, fmt.Errorf("failed to parse duration with error: %w", err) + } + opts.addedLatency = &addedLatency + } else { + opts.addedLatency = nil + } + + opts.batchSize = batchSize + + if lastScan != 0 { + opts.lastScan = &lastScan + } + + csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify) + if err != nil { + return opts, fmt.Errorf("unable to validate csub client flags: %w", err) + } + opts.csubClientOptions = csubOpts + + return opts, nil +} + +func init() { + set, err := cli.BuildFlags([]string{"certifier-latency", + "certifier-batch-size", "last-scan"}) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err) + os.Exit(1) + } + eolCmd.PersistentFlags().AddFlagSet(set) + if err := viper.BindPFlags(eolCmd.PersistentFlags()); err != nil { + fmt.Fprintf(os.Stderr, "failed to bind flags: %v", err) + os.Exit(1) + } + certifierCmd.AddCommand(eolCmd) +} diff --git a/cmd/guacone/cmd/files.go b/cmd/guacone/cmd/files.go index caecfb1cf0..5608261bdf 100644 --- a/cmd/guacone/cmd/files.go +++ b/cmd/guacone/cmd/files.go @@ -53,6 +53,7 @@ type fileOptions struct { csubClientOptions csub_client.CsubClientOptions queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } var filesCmd = &cobra.Command{ @@ -69,6 +70,7 @@ var filesCmd = &cobra.Command{ viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -129,7 +131,16 @@ var filesCmd = &cobra.Command{ emit := func(d *processor.Document) error { totalNum += 1 - if _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion); err != nil { + if _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ); err != nil { gotErr = true filesWithErrors = append(filesWithErrors, d.SourceInformation.Source) return fmt.Errorf("unable to ingest document: %w", err) @@ -162,7 +173,7 @@ var filesCmd = &cobra.Command{ } func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, args []string) (fileOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (fileOptions, error) { var opts fileOptions opts.graphqlEndpoint = graphqlEndpoint opts.headerFile = headerFile @@ -190,6 +201,7 @@ func validateFilesFlags(keyPath, keyID, graphqlEndpoint, headerFile, csubAddr st opts.path = args[0] opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion return opts, nil } diff --git a/cmd/guacone/cmd/gcs.go b/cmd/guacone/cmd/gcs.go index 96f35a6891..ae07390a69 100644 --- a/cmd/guacone/cmd/gcs.go +++ b/cmd/guacone/cmd/gcs.go @@ -42,6 +42,7 @@ type gcsOptions struct { bucket string queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } const gcsCredentialsPathFlag = "gcp-credentials-path" @@ -61,6 +62,7 @@ var gcsCmd = &cobra.Command{ viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -112,7 +114,16 @@ var gcsCmd = &cobra.Command{ emit := func(d *processor.Document) error { totalNum += 1 - _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { gotErr = true @@ -143,7 +154,7 @@ var gcsCmd = &cobra.Command{ } func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, args []string) (gcsOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (gcsOptions, error) { var opts gcsOptions opts.graphqlEndpoint = gqlEndpoint opts.headerFile = headerFile @@ -164,6 +175,7 @@ func validateGCSFlags(gqlEndpoint, headerFile, csubAddr, credentialsPath string, } opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion return opts, nil } diff --git a/cmd/guacone/cmd/gcs_test.go b/cmd/guacone/cmd/gcs_test.go index 7fd7640358..11cbb288f6 100644 --- a/cmd/guacone/cmd/gcs_test.go +++ b/cmd/guacone/cmd/gcs_test.go @@ -62,7 +62,7 @@ func TestValidateGCSFlags(t *testing.T) { t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/path/to/creds.json") } - o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, tc.args) + o, err := validateGCSFlags("", "", "", tc.credentialsPath, false, false, false, false, false, tc.args) if err != nil { if tc.errorMsg != err.Error() { t.Errorf("expected error message: %s, got: %s", tc.errorMsg, err.Error()) diff --git a/cmd/guacone/cmd/github.go b/cmd/guacone/cmd/github.go index 6f78bdeb37..9379ed4a85 100644 --- a/cmd/guacone/cmd/github.go +++ b/cmd/guacone/cmd/github.go @@ -69,6 +69,7 @@ type githubOptions struct { headerFile string queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } var githubCmd = &cobra.Command{ @@ -91,6 +92,7 @@ var githubCmd = &cobra.Command{ viper.GetBool("poll"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -155,7 +157,16 @@ var githubCmd = &cobra.Command{ var errFound bool emit := func(d *processor.Document) error { - _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { errFound = true @@ -209,7 +220,7 @@ var githubCmd = &cobra.Command{ } func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, workflowFileName, csubAddr string, csubTls, - csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, args []string) (githubOptions, error) { + csubTlsSkipVerify, useCsub, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (githubOptions, error) { var opts githubOptions opts.graphqlEndpoint = graphqlEndpoint opts.headerFile = headerFile @@ -219,6 +230,7 @@ func validateGithubFlags(graphqlEndpoint, headerFile, githubMode, sbomName, work opts.workflowFileName = workflowFileName opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion if useCsub { csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify) diff --git a/cmd/guacone/cmd/license.go b/cmd/guacone/cmd/license.go index fc8d606bf7..6472b3cabc 100644 --- a/cmd/guacone/cmd/license.go +++ b/cmd/guacone/cmd/license.go @@ -53,6 +53,7 @@ type cdOptions struct { interval time.Duration queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -76,6 +77,7 @@ var cdCmd = &cobra.Command{ viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), viper.GetInt("last-scan"), @@ -124,8 +126,15 @@ var cdCmd = &cobra.Command{ select { case <-ticker.C: if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, - opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest(ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -138,8 +147,15 @@ var cdCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, d) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, - opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest(ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -158,7 +174,16 @@ var cdCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, <-docChan) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) @@ -167,7 +192,16 @@ var cdCmd = &cobra.Command{ } } if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) @@ -234,6 +268,7 @@ func validateCDFlags( csubTlsSkipVerify bool, queryVulnIngestion bool, queryLicenseIngestion bool, + queryEOLIngestion bool, certifierLatencyStr string, batchSize int, lastScan int, ) (cdOptions, error) { @@ -270,6 +305,7 @@ func validateCDFlags( opts.csubClientOptions = csubOpts opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion return opts, nil } diff --git a/cmd/guacone/cmd/oci.go b/cmd/guacone/cmd/oci.go index 0da2276d27..5f090333ff 100644 --- a/cmd/guacone/cmd/oci.go +++ b/cmd/guacone/cmd/oci.go @@ -43,6 +43,7 @@ type ociOptions struct { csubClientOptions csub_client.CsubClientOptions queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } var ociCmd = &cobra.Command{ @@ -58,6 +59,7 @@ var ociCmd = &cobra.Command{ viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), args) if err != nil { fmt.Printf("unable to validate flags: %v\n", err) @@ -90,7 +92,16 @@ var ociCmd = &cobra.Command{ // Set emit function to go through the entire pipeline emit := func(d *processor.Document) error { totalNum += 1 - _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { gotErr = true @@ -121,12 +132,13 @@ var ociCmd = &cobra.Command{ } func validateOCIFlags(gqlEndpoint, headerFile, csubAddr string, csubTls, csubTlsSkipVerify bool, - queryVulnIngestion bool, queryLicenseIngestion bool, args []string) (ociOptions, error) { + queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool, args []string) (ociOptions, error) { var opts ociOptions opts.graphqlEndpoint = gqlEndpoint opts.headerFile = headerFile opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify) if err != nil { diff --git a/cmd/guacone/cmd/osv.go b/cmd/guacone/cmd/osv.go index 2aa0dd3117..d58d41add5 100644 --- a/cmd/guacone/cmd/osv.go +++ b/cmd/guacone/cmd/osv.go @@ -53,6 +53,7 @@ type osvOptions struct { interval time.Duration queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -76,6 +77,7 @@ var osvCmd = &cobra.Command{ viper.GetBool("csub-tls-skip-verify"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), viper.GetInt("last-scan"), @@ -124,8 +126,16 @@ var osvCmd = &cobra.Command{ select { case <-ticker.C: if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, - opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -138,8 +148,16 @@ var osvCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, d) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, - opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { stop = true atomic.StoreInt32(&gotErr, 1) @@ -158,7 +176,16 @@ var osvCmd = &cobra.Command{ totalNum += 1 totalDocs = append(totalDocs, <-docChan) if len(totalDocs) >= threshold { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) @@ -167,7 +194,16 @@ var osvCmd = &cobra.Command{ } } if len(totalDocs) > 0 { - err = ingestor.MergedIngest(ctx, totalDocs, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + err = ingestor.MergedIngest( + ctx, + totalDocs, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { atomic.StoreInt32(&gotErr, 1) logger.Errorf("unable to ingest documents: %v", err) @@ -235,6 +271,7 @@ func validateOSVFlags( csubTlsSkipVerify bool, queryVulnIngestion bool, queryLicenseIngestion bool, + queryEOLIngestion bool, certifierLatencyStr string, batchSize int, lastScan int, ) (osvOptions, error) { @@ -271,6 +308,7 @@ func validateOSVFlags( opts.csubClientOptions = csubOpts opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLIngestion return opts, nil } diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index e992deb7d8..19e078e2a7 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -51,6 +51,7 @@ type s3Options struct { csubClientOptions csub_client.CsubClientOptions // options for the collectsub client queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool } var s3Cmd = &cobra.Command{ @@ -96,6 +97,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll viper.GetBool("poll"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), ) if err != nil { fmt.Printf("failed to validate flags: %v\n", err) @@ -137,7 +139,16 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll errFound := false emit := func(d *processor.Document) error { - _, err := ingestor.Ingest(ctx, d, s3Opts.graphqlEndpoint, transport, csubClient, s3Opts.queryVulnOnIngestion, s3Opts.queryLicenseOnIngestion) + _, err := ingestor.Ingest( + ctx, + d, + s3Opts.graphqlEndpoint, + transport, + csubClient, + s3Opts.queryVulnOnIngestion, + s3Opts.queryLicenseOnIngestion, + s3Opts.queryEOLOnIngestion, + ) if err != nil { errFound = true @@ -184,7 +195,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll } func validateS3Opts(graphqlEndpoint, headerFile, csubAddr, s3url, s3bucket, s3path, region, s3item, mp, mpEndpoint, queues string, - csubTls, csubTlsSkipVerify, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool) (s3Options, error) { + csubTls, csubTlsSkipVerify, poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, queryEOLIngestion bool) (s3Options, error) { var opts s3Options if poll { @@ -208,7 +219,7 @@ func validateS3Opts(graphqlEndpoint, headerFile, csubAddr, s3url, s3bucket, s3pa } opts = s3Options{s3url, s3bucket, s3path, s3item, region, queues, mp, mpEndpoint, poll, graphqlEndpoint, headerFile, - csubClientOptions, queryVulnIngestion, queryLicenseIngestion} + csubClientOptions, queryVulnIngestion, queryLicenseIngestion, queryEOLIngestion} return opts, nil } diff --git a/cmd/guacone/cmd/scorecard.go b/cmd/guacone/cmd/scorecard.go index 678969d1f7..2c939363fc 100644 --- a/cmd/guacone/cmd/scorecard.go +++ b/cmd/guacone/cmd/scorecard.go @@ -49,6 +49,7 @@ type scorecardOptions struct { csubClientOptions csub_client.CsubClientOptions queryVulnOnIngestion bool queryLicenseOnIngestion bool + queryEOLOnIngestion bool // sets artificial latency on the certifier (default to nil) addedLatency *time.Duration // sets the batch size for pagination query for the certifier @@ -69,6 +70,7 @@ var scorecardCmd = &cobra.Command{ viper.GetBool("poll"), viper.GetBool("add-vuln-on-ingest"), viper.GetBool("add-license-on-ingest"), + viper.GetBool("add-eol-on-ingest"), viper.GetString("certifier-latency"), viper.GetInt("certifier-batch-size"), ) @@ -132,7 +134,16 @@ var scorecardCmd = &cobra.Command{ // Set emit function to go through the entire pipeline emit := func(d *processor.Document) error { totalNum += 1 - _, err := ingestor.Ingest(ctx, d, opts.graphqlEndpoint, transport, csubClient, opts.queryVulnOnIngestion, opts.queryLicenseOnIngestion) + _, err := ingestor.Ingest( + ctx, + d, + opts.graphqlEndpoint, + transport, + csubClient, + opts.queryVulnOnIngestion, + opts.queryLicenseOnIngestion, + opts.queryEOLOnIngestion, + ) if err != nil { return fmt.Errorf("unable to ingest document: %v", err) @@ -190,6 +201,7 @@ func validateScorecardFlags( poll bool, queryVulnIngestion bool, queryLicenseIngestion bool, + queryEOLOnIngestion bool, certifierLatencyStr string, batchSize int, ) (scorecardOptions, error) { @@ -223,6 +235,7 @@ func validateScorecardFlags( opts.interval = i opts.queryVulnOnIngestion = queryVulnIngestion opts.queryLicenseOnIngestion = queryLicenseIngestion + opts.queryEOLOnIngestion = queryEOLOnIngestion return opts, nil } diff --git a/guac.yaml b/guac.yaml index 3aab9de8bc..077c780369 100644 --- a/guac.yaml +++ b/guac.yaml @@ -45,6 +45,9 @@ add-vuln-on-ingest: false # query licenses during ingestion add-license-on-ingest: false +# query eol during ingestion +add-eol-on-ingest: false + # CSub setup csub-addr: localhost:2782 csub-listen-port: 2782 diff --git a/internal/testing/cmd/ingest/cmd/example.go b/internal/testing/cmd/ingest/cmd/example.go index 66c202d16a..f56b3d8382 100644 --- a/internal/testing/cmd/ingest/cmd/example.go +++ b/internal/testing/cmd/ingest/cmd/example.go @@ -59,7 +59,7 @@ func ingestExample(cmd *cobra.Command, args []string) { var inputs []assembler.IngestPredicates for _, doc := range docs { // This is a test example, so we will ignore calling out to a collectsub service - input, _, err := parser.ParseDocumentTree(ctx, doc, false, false) + input, _, err := parser.ParseDocumentTree(ctx, doc, false, false, false) if err != nil { logger.Fatalf("unable to parse document: %v", err) } diff --git a/internal/testing/testdata/exampledata/eol-all.json b/internal/testing/testdata/exampledata/eol-all.json new file mode 100644 index 0000000000..e7e9e9ccb9 --- /dev/null +++ b/internal/testing/testdata/exampledata/eol-all.json @@ -0,0 +1,344 @@ +[ + "akeneo-pim", + "alibaba-dragonwell", + "almalinux", + "alpine", + "amazon-cdk", + "amazon-corretto", + "amazon-eks", + "amazon-glue", + "amazon-linux", + "amazon-neptune", + "amazon-rds-mariadb", + "amazon-rds-mysql", + "amazon-rds-postgresql", + "android", + "angular", + "angularjs", + "ansible", + "ansible-core", + "antix", + "apache", + "apache-activemq", + "apache-airflow", + "apache-apisix", + "apache-camel", + "apache-cassandra", + "apache-couchdb", + "apache-flink", + "apache-groovy", + "apache-hadoop", + "apache-hop", + "apache-kafka", + "apache-lucene", + "apache-spark", + "apache-struts", + "api-platform", + "apple-watch", + "arangodb", + "argo-cd", + "artifactory", + "aws-lambda", + "azul-zulu", + "azure-devops-server", + "azure-kubernetes-service", + "bazel", + "beats", + "bellsoft-liberica", + "blender", + "bootstrap", + "bun", + "cakephp", + "calico", + "centos", + "centos-stream", + "centreon", + "cert-manager", + "cfengine", + "chef-infra-client", + "chef-infra-server", + "chef-inspec", + "citrix-vad", + "ckeditor", + "clamav", + "cockroachdb", + "coder", + "coldfusion", + "composer", + "confluence", + "consul", + "containerd", + "contao", + "contour", + "controlm", + "cortex-xdr", + "cos", + "couchbase-server", + "craft-cms", + "dbt-core", + "dce", + "debian", + "dependency-track", + "devuan", + "django", + "docker-engine", + "dotnet", + "dotnetfx", + "drupal", + "drush", + "eclipse-jetty", + "eclipse-temurin", + "elasticsearch", + "electron", + "elixir", + "emberjs", + "envoy", + "erlang", + "esxi", + "etcd", + "eurolinux", + "exim", + "fairphone", + "fedora", + "ffmpeg", + "filemaker", + "firefox", + "fluent-bit", + "flux", + "fortios", + "freebsd", + "gerrit", + "gitlab", + "go", + "goaccess", + "godot", + "google-kubernetes-engine", + "google-nexus", + "gorilla", + "graalvm", + "gradle", + "grafana", + "grafana-loki", + "grails", + "graylog", + "gstreamer", + "haproxy", + "harbor", + "hashicorp-vault", + "hbase", + "horizon", + "ibm-aix", + "ibm-i", + "ibm-semeru-runtime", + "icinga", + "icinga-web", + "intel-processors", + "internet-explorer", + "ionic", + "ios", + "ipad", + "ipados", + "iphone", + "isc-dhcp", + "istio", + "jekyll", + "jenkins", + "jhipster", + "jira-software", + "joomla", + "jquery", + "jquery-ui", + "jreleaser", + "kde-plasma", + "keda", + "keycloak", + "kibana", + "kindle", + "kirby", + "kong-gateway", + "kotlin", + "kubernetes", + "kubernetes-csi-node-driver-registrar", + "kubernetes-node-feature-discovery", + "kuma", + "kyverno", + "laravel", + "libreoffice", + "lineageos", + "linux", + "linuxmint", + "log4j", + "logstash", + "looker", + "lua", + "macos", + "mageia", + "magento", + "mandrel", + "mariadb", + "mastodon", + "matomo", + "mattermost", + "mautic", + "maven", + "mediawiki", + "meilisearch", + "memcached", + "micronaut", + "microsoft-build-of-openjdk", + "mongodb", + "moodle", + "motorola-mobility", + "msexchange", + "mssqlserver", + "mulesoft-runtime", + "mxlinux", + "mysql", + "neo4j", + "neos", + "netbsd", + "nextcloud", + "nextjs", + "nexus", + "nginx", + "nix", + "nixos", + "nodejs", + "nokia", + "nomad", + "numpy", + "nutanix-aos", + "nutanix-files", + "nutanix-prism", + "nuxt", + "nvidia", + "nvidia-gpu", + "office", + "oneplus", + "openbsd", + "openjdk-builds-from-oracle", + "opensearch", + "openssl", + "opensuse", + "opentofu", + "openvpn", + "openwrt", + "openzfs", + "opnsense", + "oracle-apex", + "oracle-database", + "oracle-jdk", + "oracle-linux", + "oracle-solaris", + "ovirt", + "pangp", + "panos", + "pci-dss", + "perl", + "photon", + "php", + "phpbb", + "phpmyadmin", + "pixel", + "plesk", + "pnpm", + "podman", + "pop-os", + "postfix", + "postgresql", + "postmarketos", + "powershell", + "privatebin", + "prometheus", + "protractor", + "proxmox-ve", + "puppet", + "python", + "qt", + "quarkus-framework", + "quasar", + "rabbitmq", + "rails", + "rancher", + "raspberry-pi", + "react", + "react-native", + "readynas", + "red-hat-openshift", + "redhat-build-of-openjdk", + "redhat-jboss-eap", + "redhat-satellite", + "redis", + "redmine", + "rhel", + "robo", + "rocket-chat", + "rocky-linux", + "ros", + "ros-2", + "roundcube", + "ruby", + "rust", + "salt", + "samsung-mobile", + "sapmachine", + "scala", + "sharepoint", + "shopware", + "silverstripe", + "slackware", + "sles", + "solr", + "sonar", + "sourcegraph", + "splunk", + "spring-boot", + "spring-framework", + "sqlite", + "squid", + "steamos", + "subversion", + "surface", + "suse-manager", + "symfony", + "tails", + "tarantool", + "telegraf", + "terraform", + "tomcat", + "traefik", + "twig", + "typo3", + "ubuntu", + "umbraco", + "unity", + "unrealircd", + "varnish", + "vcenter", + "veeam-backup-and-replication", + "visionos", + "visual-cobol", + "visual-studio", + "vmware-cloud-foundation", + "vmware-harbor-registry", + "vmware-srm", + "vue", + "vuetify", + "wagtail", + "watchos", + "weechat", + "windows", + "windows-embedded", + "windows-nano-server", + "windows-server", + "windows-server-core", + "wireshark", + "wordpress", + "xcp-ng", + "yarn", + "yocto", + "zabbix", + "zentyal", + "zerto", + "zookeeper" +] diff --git a/internal/testing/testdata/exampledata/eol-sapmachine.json b/internal/testing/testdata/exampledata/eol-sapmachine.json new file mode 100644 index 0000000000..bc9fb87a0a --- /dev/null +++ b/internal/testing/testdata/exampledata/eol-sapmachine.json @@ -0,0 +1,106 @@ +[ + { + "cycle": "23", + "releaseDate": "2024-09-18", + "eol": "2025-03-18", + "latest": "23.0.1", + "latestReleaseDate": "2024-10-15", + "lts": false + }, + { + "cycle": "22", + "releaseDate": "2024-03-18", + "eol": "2024-09-17", + "latest": "22.0.2", + "latestReleaseDate": "2024-07-17", + "lts": false + }, + { + "cycle": "21", + "lts": true, + "releaseDate": "2023-09-18", + "eol": "2028-09-01", + "latest": "21.0.5", + "latestReleaseDate": "2024-10-15" + }, + { + "cycle": "20", + "releaseDate": "2023-03-17", + "eol": "2023-09-19", + "latest": "20.0.2", + "latestReleaseDate": "2023-07-18", + "lts": false + }, + { + "cycle": "19", + "releaseDate": "2022-09-19", + "eol": "2023-03-17", + "latest": "19.0.2", + "latestReleaseDate": "2023-01-17", + "lts": false + }, + { + "cycle": "18", + "releaseDate": "2022-03-21", + "eol": "2022-09-19", + "latest": "18.0.2.1", + "latestReleaseDate": "2022-08-23", + "lts": false + }, + { + "cycle": "17", + "lts": true, + "releaseDate": "2021-09-14", + "eol": "2026-09-01", + "latest": "17.0.13", + "latestReleaseDate": "2024-10-15" + }, + { + "cycle": "16", + "releaseDate": "2021-03-15", + "eol": "2021-09-14", + "latest": "16.0.2", + "latestReleaseDate": "2021-07-22", + "lts": false + }, + { + "cycle": "15", + "releaseDate": "2020-09-16", + "eol": "2021-03-15", + "latest": "15.0.2", + "latestReleaseDate": "2021-01-22", + "lts": false + }, + { + "cycle": "14", + "releaseDate": "2020-03-18", + "eol": "2020-09-16", + "latest": "14.0.2", + "latestReleaseDate": "2020-07-16", + "lts": false + }, + { + "cycle": "13", + "releaseDate": "2019-09-18", + "eol": "2020-03-18", + "latest": "13.0.2", + "latestReleaseDate": "2020-01-16", + "lts": false + }, + { + "cycle": "12", + "releaseDate": "2019-03-21", + "eol": "2019-09-18", + "latest": "12.0.2", + "latestReleaseDate": "2019-07-17", + "lts": false + }, + { + "cycle": "11", + "lts": true, + "releaseDate": "2019-01-16", + "eol": "2024-12-01", + "latest": "11.0.25", + "latestReleaseDate": "2024-10-15" + } +] diff --git a/internal/testing/testdata/models.go b/internal/testing/testdata/models.go index 960856b7f6..706d861f24 100644 --- a/internal/testing/testdata/models.go +++ b/internal/testing/testdata/models.go @@ -560,3 +560,51 @@ var P7 = &model.PkgInputSpec{ Value: "https://alternative.report.url/", }}, } + +// ITE6EOLNodejs is a test document for the EOL ingestor +var ITE6EOLNodejs = []byte(`{ + "type": "https://in-toto.io/Statement/v1", + "subject": [ + { + "uri": "pkg:npm/nodejs@14.17.0" + } + ], + "predicateType": "https://in-toto.io/attestation/eol/v0.1", + "predicate": { + "product": "nodejs", + "cycle": "14", + "version": "14.17.0", + "isEOL": true, + "eolDate": "2023-04-30", + "lts": true, + "latest": "14.21.3", + "releaseDate": "2021-05-11", + "metadata": { + "scannedOn": "2024-03-15T12:00:00Z" + } + } +}`) + +// ITE6EOLPython is a test document for the EOL ingestor +var ITE6EOLPython = []byte(`{ + "type": "https://in-toto.io/Statement/v1", + "subject": [ + { + "uri": "pkg:pypi/python@3.9.5" + } + ], + "predicateType": "https://in-toto.io/attestation/eol/v0.1", + "predicate": { + "product": "python", + "cycle": "3.9", + "version": "3.9.5", + "isEOL": false, + "eolDate": "2025-10-05", + "lts": false, + "latest": "3.9.16", + "releaseDate": "2021-05-03", + "metadata": { + "scannedOn": "2024-03-15T12:00:00Z" + } + } +}`) diff --git a/internal/testing/testdata/testdata.go b/internal/testing/testdata/testdata.go index fd9a9f239f..edeb61e546 100644 --- a/internal/testing/testdata/testdata.go +++ b/internal/testing/testdata/testdata.go @@ -204,6 +204,12 @@ var ( //go:embed exampledata/cyclonedx-components-flat.json CycloneDXComponentsFlat []byte + //go:embed exampledata/eol-all.json + EOLAll []byte + + //go:embed exampledata/eol-sapmachine.json + EOLSapMachine []byte + // json format json = jsoniter.ConfigCompatibleWithStandardLibrary // CycloneDX VEX testdata unaffected diff --git a/pkg/assembler/backends/ent/backend/search.go b/pkg/assembler/backends/ent/backend/search.go index cc46e8e0d6..e647ea50af 100644 --- a/pkg/assembler/backends/ent/backend/search.go +++ b/pkg/assembler/backends/ent/backend/search.go @@ -30,6 +30,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/ent/certifylegal" "github.com/guacsec/guac/pkg/assembler/backends/ent/certifyvuln" "github.com/guacsec/guac/pkg/assembler/backends/ent/dependency" + "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" "github.com/guacsec/guac/pkg/assembler/backends/ent/predicate" @@ -168,7 +169,7 @@ func (b *EntBackend) FindPackagesThatNeedScanning(ctx context.Context, queryType if err != nil { return nil, fmt.Errorf("failed aggregate packages based on certifyVuln with error: %w", err) } - } else { + } else if queryType == model.QueryTypeLicense { err := b.client.PackageVersion.Query(). Where(notGUACTypePackagePredicates()). WithName(func(q *ent.PackageNameQuery) {}). @@ -183,6 +184,25 @@ func (b *EntBackend) FindPackagesThatNeedScanning(ctx context.Context, queryType if err != nil { return nil, fmt.Errorf("failed aggregate packages based on certifyLegal with error: %w", err) } + } else { // queryType == model.QueryTypeEol via hasMetadata + err := b.client.PackageVersion.Query(). + Where(notGUACTypePackagePredicates()). + WithName(func(q *ent.PackageNameQuery) {}). + GroupBy(packageversion.FieldID). // Group by Package ID + Aggregate(func(s *sql.Selector) string { + t := sql.Table(hasmetadata.Table) + s.LeftJoin(t).On(s.C(packageversion.FieldID), t.C(hasmetadata.FieldPackageVersionID)) + s.Where(sql.And( + sql.NotNull(t.C(hasmetadata.FieldTimestamp)), + sql.EQ(t.C(hasmetadata.FieldKey), "endoflife"), + )) + return sql.As(sql.Max(t.C(hasmetadata.FieldTimestamp)), "max") + }). + Scan(ctx, &pkgLatestScan) + + if err != nil { + return nil, fmt.Errorf("failed aggregate packages based on hasMetadata with error: %w", err) + } } lastScanTime := time.Now().Add(time.Duration(-*lastScan) * time.Hour).UTC() diff --git a/pkg/assembler/backends/keyvalue/search.go b/pkg/assembler/backends/keyvalue/search.go index af0469d78f..2de09bfbe6 100644 --- a/pkg/assembler/backends/keyvalue/search.go +++ b/pkg/assembler/backends/keyvalue/search.go @@ -415,7 +415,7 @@ func (c *demoClient) FindPackagesThatNeedScanning(ctx context.Context, queryType } else { pkgIDs = append(pkgIDs, pkgVer.ThisID) } - } else { + } else if queryType == model.QueryTypeLicense { if len(pkgVer.CertifyLegals) > 0 { var timeScanned []time.Time for _, certLegalID := range pkgVer.CertifyLegals { @@ -434,6 +434,28 @@ func (c *demoClient) FindPackagesThatNeedScanning(ctx context.Context, queryType } else { pkgIDs = append(pkgIDs, pkgVer.ThisID) } + } else { // queryType == model.QueryTypeEol via hasMetadataLink + if len(pkgVer.HasMetadataLinks) > 0 { + var timeScanned []time.Time + for _, hasMetadataLinkID := range pkgVer.HasMetadataLinks { + link, err := byIDkv[*hasMetadataLink](ctx, hasMetadataLinkID, c) + if err != nil { + continue + } + // only pick endoflife metadata + if link.MDKey != "endoflife" { + continue + } + timeScanned = append(timeScanned, link.Timestamp) + } + lastScanTime := latestTime(timeScanned) + lastIntervalTime := time.Now().Add(time.Duration(-*lastScan) * time.Hour).UTC() + if lastScanTime.Before(lastIntervalTime) { + pkgIDs = append(pkgIDs, pkgVer.ThisID) + } + } else { + pkgIDs = append(pkgIDs, pkgVer.ThisID) + } } } } diff --git a/pkg/assembler/clients/generated/operations.go b/pkg/assembler/clients/generated/operations.go index 8af6124104..20914a0a80 100644 --- a/pkg/assembler/clients/generated/operations.go +++ b/pkg/assembler/clients/generated/operations.go @@ -27471,6 +27471,8 @@ const ( QueryTypeVulnerability QueryType = "VULNERABILITY" // indirect dependency QueryTypeLicense QueryType = "LICENSE" + // indirect dependency + QueryTypeEol QueryType = "EOL" ) // SLSAInputSpec is the same as SLSA but for mutation input. diff --git a/pkg/assembler/graphql/generated/root_.generated.go b/pkg/assembler/graphql/generated/root_.generated.go index c1b72c6923..335dc859f1 100644 --- a/pkg/assembler/graphql/generated/root_.generated.go +++ b/pkg/assembler/graphql/generated/root_.generated.go @@ -7670,6 +7670,8 @@ enum QueryType { VULNERABILITY "indirect dependency" LICENSE + "indirect dependency" + EOL } extend type Query { diff --git a/pkg/assembler/graphql/model/nodes.go b/pkg/assembler/graphql/model/nodes.go index 732259581a..d42b74e705 100644 --- a/pkg/assembler/graphql/model/nodes.go +++ b/pkg/assembler/graphql/model/nodes.go @@ -2692,16 +2692,19 @@ const ( QueryTypeVulnerability QueryType = "VULNERABILITY" // indirect dependency QueryTypeLicense QueryType = "LICENSE" + // indirect dependency + QueryTypeEol QueryType = "EOL" ) var AllQueryType = []QueryType{ QueryTypeVulnerability, QueryTypeLicense, + QueryTypeEol, } func (e QueryType) IsValid() bool { switch e { - case QueryTypeVulnerability, QueryTypeLicense: + case QueryTypeVulnerability, QueryTypeLicense, QueryTypeEol: return true } return false diff --git a/pkg/assembler/graphql/schema/search.graphql b/pkg/assembler/graphql/schema/search.graphql index 465ed427cd..28da4275c0 100644 --- a/pkg/assembler/graphql/schema/search.graphql +++ b/pkg/assembler/graphql/schema/search.graphql @@ -51,6 +51,8 @@ enum QueryType { VULNERABILITY "indirect dependency" LICENSE + "indirect dependency" + EOL } extend type Query { diff --git a/pkg/certifier/attestation/attestation_eol.go b/pkg/certifier/attestation/attestation_eol.go new file mode 100644 index 0000000000..0d18fc74c6 --- /dev/null +++ b/pkg/certifier/attestation/attestation_eol.go @@ -0,0 +1,51 @@ +// +// Copyright 2022 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package attestation + +import ( + "time" + + attestationv1 "github.com/in-toto/attestation/go/v1" +) + +const ( + PredicateEOL = "https://in-toto.io/attestation/eol/v0.1" +) + +// EOLStatement defines the statement header and the EOL predicate +type EOLStatement struct { + attestationv1.Statement + // Predicate contains type specific metadata. + Predicate EOLPredicate `json:"predicate"` +} + +// EOLMetadata defines when the last scan was done +type EOLMetadata struct { + ScannedOn *time.Time `json:"scannedOn,omitempty"` +} + +// EOLPredicate defines predicate definition of the EOL attestation +type EOLPredicate struct { + Product string `json:"product"` + Cycle string `json:"cycle"` + Version string `json:"version"` + IsEOL bool `json:"isEOL"` + EOLDate string `json:"eolDate"` + LTS bool `json:"lts"` + Latest string `json:"latest"` + ReleaseDate string `json:"releaseDate"` + Metadata EOLMetadata `json:"metadata"` +} diff --git a/pkg/certifier/certifier.go b/pkg/certifier/certifier.go index a968a3bb23..35e47ea99e 100644 --- a/pkg/certifier/certifier.go +++ b/pkg/certifier/certifier.go @@ -50,4 +50,5 @@ const ( CertifierOSV CertifierType = "OSV" CertifierClearlyDefined CertifierType = "CD" CertifierScorecard CertifierType = "scorecard" + CertifierEOL CertifierType = "EOL" ) diff --git a/pkg/certifier/eol/eol.go b/pkg/certifier/eol/eol.go new file mode 100644 index 0000000000..8ee85201f7 --- /dev/null +++ b/pkg/certifier/eol/eol.go @@ -0,0 +1,427 @@ +// Copyright 2024 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eol + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/guacsec/guac/pkg/certifier" + "github.com/guacsec/guac/pkg/certifier/attestation" + "github.com/guacsec/guac/pkg/certifier/components/root_package" + "github.com/guacsec/guac/pkg/clients" + "github.com/guacsec/guac/pkg/events" + "github.com/guacsec/guac/pkg/handler/processor" + attestationv1 "github.com/in-toto/attestation/go/v1" + "golang.org/x/time/rate" +) + +var ( + eolAPIBase = "https://endoflife.date/api" + ErrEOLComponenetTypeMismatch = errors.New("rootComponent type is not []*root_package.PackageNode") +) + +const ( + EOLCollector = "endoflife.date" + rateLimit = 10 + rateLimitBurst = 1 +) + +type eolCertifier struct { + client *http.Client +} + +// EOLStringOrBool represents a value that can be either a string, boolean, or null +type EOLStringOrBool struct { + value interface{} +} + +// NewStringValue creates a EOLStringOrBool from a string +func NewStringValue(s string) EOLStringOrBool { + return EOLStringOrBool{value: s} +} + +// NewBoolValue creates a EOLStringOrBool from a bool +func NewBoolValue(b bool) EOLStringOrBool { + return EOLStringOrBool{value: b} +} + +// UnmarshalJSON implements json.Unmarshaler +func (f *EOLStringOrBool) UnmarshalJSON(data []byte) error { + // Try string first + var s string + if err := json.Unmarshal(data, &s); err == nil { + f.value = s + return nil + } + + // Try boolean + var b bool + if err := json.Unmarshal(data, &b); err == nil { + f.value = b + return nil + } + + // Try null + var n interface{} + if err := json.Unmarshal(data, &n); err != nil { + return err + } + // If it's null, leave value as nil + f.value = n + return nil +} + +// IsString returns true if the value is a string +func (f *EOLStringOrBool) IsString() bool { + _, ok := f.value.(string) + return ok +} + +// IsBool returns true if the value is a boolean +func (f *EOLStringOrBool) IsBool() bool { + _, ok := f.value.(bool) + return ok +} + +// String returns the string value or empty string if not a string +func (f *EOLStringOrBool) String() string { + s, ok := f.value.(string) + if !ok { + return "" + } + return s +} + +// Bool returns a boolean value: +// - If value is bool: returns that value +// - If value is string: tries to parse as date and compares with current time +// - Otherwise: returns false +func (f *EOLStringOrBool) Bool() bool { + // First try as boolean + if b, ok := f.value.(bool); ok { + return b + } + + // Then try as string + if s, ok := f.value.(string); ok { + // If string is empty or "false", return false + if s == "" || strings.EqualFold(s, "false") { + return false + } + + // If string is "true", return true + if strings.EqualFold(s, "true") { + return true + } + + // Try to parse as date + if t, err := time.Parse("2006-01-02", s); err == nil { + // Compare with current time to determine if EOL + return time.Now().After(t) + } + } + + return false +} + +// ToBool converts the value to a boolean based on type: +// - bool: returns the value +// - string: returns true if non-empty +// - nil: returns false +func (f *EOLStringOrBool) ToBool() bool { + switch v := f.value.(type) { + case bool: + return v + case string: + return v != "" && v != "false" + default: + return false + } +} + +// CycleData represents the endoflife.date API data type for a cycle +type CycleData struct { + Cycle string `json:"cycle"` + ReleaseDate string `json:"releaseDate"` + EOL EOLStringOrBool `json:"eol"` // Can be string or boolean + Latest string `json:"latest"` + Link *string `json:"link"` // Can be null + LTS EOLStringOrBool `json:"lts"` // Can be string or boolean + Support EOLStringOrBool `json:"support"` // Can be string or boolean + Discontinued EOLStringOrBool `json:"discontinued"` // Can be string or boolean +} + +// EOLData is a list of CycleData, the response from the endoflife.date API +type EOLData = []CycleData + +func NewEOLCertifier() certifier.Certifier { + limiter := rate.NewLimiter(rate.Every(time.Second/time.Duration(rateLimit)), rateLimitBurst) + client := &http.Client{ + Transport: clients.NewRateLimitedTransport(http.DefaultTransport, limiter), + } + return &eolCertifier{client: client} +} + +func (e *eolCertifier) CertifyComponent(ctx context.Context, rootComponent interface{}, docChannel chan<- *processor.Document) error { + packageNodes, ok := rootComponent.([]*root_package.PackageNode) + if !ok { + return ErrEOLComponenetTypeMismatch + } + + var purls []string + for _, node := range packageNodes { + purls = append(purls, node.Purl) + } + + if _, err := EvaluateEOLResponse(ctx, e.client, purls, docChannel); err != nil { + return fmt.Errorf("could not generate document from EOL results: %w", err) + } + return nil +} + +func EvaluateEOLResponse(ctx context.Context, client *http.Client, purls []string, docChannel chan<- *processor.Document) ([]*processor.Document, error) { + packMap := map[string]bool{} + var generatedEOLDocs []*processor.Document + + products, err := fetchAllProducts(ctx, client) + if err != nil { + return nil, fmt.Errorf("failed to fetch all products: %w", err) + } + + for _, purl := range purls { + if strings.Contains(purl, "pkg:guac") { + continue + } + if _, ok := packMap[purl]; ok { + continue + } + + product, found := findMatchingProduct(purl, products) + if !found { + continue + } + + eolData, err := fetchProductEOL(ctx, client, product) + if err != nil { + return nil, fmt.Errorf("failed to fetch EOL data for %s: %w", product, err) + } + + cycle, version := extractCycleAndVersion(purl) + var relevantCycle *CycleData + for i := range eolData { + if eolData[i].Cycle == cycle { + relevantCycle = &eolData[i] + break + } + } + + if relevantCycle == nil && len(eolData) > 0 { + // If no matching cycle is found, use the latest (first in the list) + relevantCycle = &eolData[0] + } + + if relevantCycle != nil { + currentTime := time.Now() + + // Get EOL status and date + isEOL := relevantCycle.EOL.Bool() + eolDateStr := relevantCycle.EOL.String() + + statement := &attestation.EOLStatement{ + Statement: attestationv1.Statement{ + Type: attestationv1.StatementTypeUri, + PredicateType: attestation.PredicateEOL, + Subject: []*attestationv1.ResourceDescriptor{{Uri: purl}}, + }, + Predicate: attestation.EOLPredicate{ + Product: product, + Cycle: relevantCycle.Cycle, + Version: version, + IsEOL: isEOL, + EOLDate: eolDateStr, + LTS: relevantCycle.LTS.Bool(), + Latest: relevantCycle.Latest, + ReleaseDate: relevantCycle.ReleaseDate, + Metadata: attestation.EOLMetadata{ + ScannedOn: ¤tTime, + }, + }, + } + + payload, err := json.Marshal(statement) + if err != nil { + return nil, fmt.Errorf("unable to marshal attestation: %w", err) + } + + doc := &processor.Document{ + Blob: payload, + Type: processor.DocumentITE6EOL, + Format: processor.FormatJSON, + SourceInformation: processor.SourceInformation{ + Collector: EOLCollector, + Source: EOLCollector, + DocumentRef: events.GetDocRef(payload), + }, + } + + if docChannel != nil { + docChannel <- doc + } + generatedEOLDocs = append(generatedEOLDocs, doc) + } + + packMap[purl] = true + } + + return generatedEOLDocs, nil +} + +func fetchAllProducts(ctx context.Context, client *http.Client) ([]string, error) { + resp, err := client.Get(fmt.Sprintf("%s/all.json", eolAPIBase)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var products []string + if err := json.NewDecoder(resp.Body).Decode(&products); err != nil { + return nil, err + } + + return products, nil +} + +func fetchProductEOL(ctx context.Context, client *http.Client, product string) (EOLData, error) { + resp, err := client.Get(fmt.Sprintf("%s/%s.json", eolAPIBase, product)) + if err != nil { + return nil, fmt.Errorf("failed to fetch EOL data for %s: %w", product, err) + } + defer resp.Body.Close() + + // Check for non-200 status codes + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received non-200 status code (%d) for product %s", resp.StatusCode, product) + } + + var rawData []map[string]json.RawMessage + if err := json.NewDecoder(resp.Body).Decode(&rawData); err != nil { + return nil, fmt.Errorf("failed to decode EOL data for %s: %w", product, err) + } + + eolData := make(EOLData, len(rawData)) + for i, item := range rawData { + var cycle CycleData + + // Decode the simple string fields directly + if v, ok := item["cycle"]; ok { + if err := json.Unmarshal(v, &cycle.Cycle); err != nil { + return nil, fmt.Errorf("failed to decode cycle for %s: %w", product, err) + } + } + if v, ok := item["releaseDate"]; ok { + if err := json.Unmarshal(v, &cycle.ReleaseDate); err != nil { + return nil, fmt.Errorf("failed to decode releaseDate for %s: %w", product, err) + } + } + if v, ok := item["latest"]; ok { + if err := json.Unmarshal(v, &cycle.Latest); err != nil { + return nil, fmt.Errorf("failed to decode latest for %s: %w", product, err) + } + } + + // Handle the flexible type fields + if v, ok := item["eol"]; ok { + var eol EOLStringOrBool + if err := json.Unmarshal(v, &eol); err != nil { + return nil, fmt.Errorf("failed to decode eol for %s: %w", product, err) + } + cycle.EOL = eol + } + if v, ok := item["lts"]; ok { + var lts EOLStringOrBool + if err := json.Unmarshal(v, <s); err != nil { + return nil, fmt.Errorf("failed to decode lts for %s: %w", product, err) + } + cycle.LTS = lts + } + if v, ok := item["support"]; ok { + var support EOLStringOrBool + if err := json.Unmarshal(v, &support); err != nil { + return nil, fmt.Errorf("failed to decode support for %s: %w", product, err) + } + cycle.Support = support + } + if v, ok := item["discontinued"]; ok { + var discontinued EOLStringOrBool + if err := json.Unmarshal(v, &discontinued); err != nil { + return nil, fmt.Errorf("failed to decode discontinued for %s: %w", product, err) + } + cycle.Discontinued = discontinued + } + + // Optional link field + if v, ok := item["link"]; ok { + var link string + if err := json.Unmarshal(v, &link); err != nil { + return nil, fmt.Errorf("failed to decode link for %s: %w", product, err) + } + cycle.Link = &link + } + + eolData[i] = cycle + } + + return eolData, nil +} + +func findMatchingProduct(purl string, products []string) (string, bool) { + parts := strings.Split(purl, "/") + if len(parts) < 2 { + return "", false + } + + packageName := strings.Split(parts[1], "@")[0] + packageName = strings.ToLower(packageName) + + for _, product := range products { + if strings.Contains(packageName, product) || strings.Contains(product, packageName) { + return product, true + } + } + + return "", false +} + +func extractCycleAndVersion(purl string) (string, string) { + parts := strings.Split(purl, "@") + if len(parts) < 2 { + return "", "" + } + + version := parts[1] + versionParts := strings.Split(version, ".") + + if len(versionParts) > 0 { + return versionParts[0], version + } + + return "", version +} diff --git a/pkg/certifier/eol/eol_test.go b/pkg/certifier/eol/eol_test.go new file mode 100644 index 0000000000..0ace388989 --- /dev/null +++ b/pkg/certifier/eol/eol_test.go @@ -0,0 +1,343 @@ +// +// Copyright 2024 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eol + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/guacsec/guac/internal/testing/testdata" + "github.com/guacsec/guac/pkg/certifier/attestation" + "github.com/guacsec/guac/pkg/certifier/components/root_package" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEOLStringOrBool_Bool(t *testing.T) { + now := time.Now() + pastDate := now.Add(-24 * time.Hour) + futureDate := now.Add(24 * time.Hour) + + tests := []struct { + name string + value EOLStringOrBool + want bool + }{ + { + name: "direct boolean true", + value: NewBoolValue(true), + want: true, + }, + { + name: "direct boolean false", + value: NewBoolValue(false), + want: false, + }, + { + name: "past date string (should be true)", + value: NewStringValue(pastDate.Format("2006-01-02")), + want: true, + }, + { + name: "future date string (should be false)", + value: NewStringValue(futureDate.Format("2006-01-02")), + want: false, + }, + { + name: "string 'true'", + value: NewStringValue("true"), + want: true, + }, + { + name: "string 'false'", + value: NewStringValue("false"), + want: false, + }, + { + name: "empty string", + value: NewStringValue(""), + want: false, + }, + { + name: "invalid date string", + value: NewStringValue("not-a-date"), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.value.Bool(); got != tt.want { + t.Errorf("EOLStringOrBool.Bool() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewEOLCertifier(t *testing.T) { + certifier := NewEOLCertifier() + assert.NotNil(t, certifier, "NewEOLCertifier should return a non-nil certifier") + _, ok := certifier.(*eolCertifier) + assert.True(t, ok, "NewEOLCertifier should return an instance of eolCertifier") +} + +func TestCertifyComponent(t *testing.T) { + // Mock HTTP server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + switch r.URL.Path { + case "/api/all.json": + // For raw JSON bytes, write directly to the response + _, err = w.Write(testdata.EOLAll) + case "/api/sapmachine.json": + // For raw JSON bytes, write directly to the response + _, err = w.Write(testdata.EOLSapMachine) + default: + http.NotFound(w, r) + } + if err != nil { + t.Errorf("Failed to write response: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + })) + defer server.Close() + eolAPIBase = server.URL + "/api" + + // Helper function to verify the generated document + verifyDoc := func(doc *processor.Document) { + var statement attestation.EOLStatement + err := json.Unmarshal(doc.Blob, &statement) + require.NoError(t, err) + + // For debugging + t.Logf("Statement EOLDate: %s", statement.Predicate.EOLDate) + t.Logf("Statement LTS: %v", statement.Predicate.LTS) + + // Check all fields + assert.Equal(t, "21", statement.Predicate.Cycle) + assert.Equal(t, "21.0.5", statement.Predicate.Latest) + assert.Equal(t, "2028-09-01", statement.Predicate.EOLDate) + assert.True(t, statement.Predicate.LTS) + assert.False(t, statement.Predicate.IsEOL) // Future date should not be EOL + + assert.Equal(t, processor.DocumentITE6EOL, doc.Type) + assert.Equal(t, processor.FormatJSON, doc.Format) + assert.Equal(t, EOLCollector, doc.SourceInformation.Collector) + + sha256sum := sha256.Sum256(doc.Blob) + expectDocumentRef := fmt.Sprintf("sha256_%s", hex.EncodeToString(sha256sum[:])) + assert.Contains(t, doc.SourceInformation.DocumentRef, expectDocumentRef) + } + + certifier := &eolCertifier{ + client: server.Client(), + } + + rootComponent := []*root_package.PackageNode{ + {Purl: "pkg:maven/com.sap.sapmachine/sapmachine@21.0.5"}, + {Purl: "pkg:npm/unknown@2.0.0"}, + } + + docChan := make(chan *processor.Document, 10) + + err := certifier.CertifyComponent(context.Background(), rootComponent, docChan) + require.NoError(t, err) + + close(docChan) + docs := make([]*processor.Document, 0) + for doc := range docChan { + docs = append(docs, doc) + } + + require.Len(t, docs, 1, "Expected exactly one document") + verifyDoc(docs[0]) +} + +func TestFetchAllProducts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err := json.NewEncoder(w).Encode([]string{"sapmachine", "nodejs"}) + require.NoError(t, err) + })) + defer server.Close() + eolAPIBase = server.URL + "/api" + + certifier := &eolCertifier{ + client: server.Client(), + } + + products, err := fetchAllProducts(context.Background(), certifier.client) + require.NoError(t, err) + assert.Equal(t, []string{"sapmachine", "nodejs"}, products) +} + +func TestFetchProductEOL(t *testing.T) { + tests := []struct { + name string + responseCode int + responseBody string + wantErr bool + validateData func(*testing.T, EOLData) + }{ + { + name: "successful response with mixed types", + responseCode: http.StatusOK, + responseBody: `[ + { + "cycle": "21", + "releaseDate": "2023-09-18", + "eol": "2028-09-01", + "latest": "21.0.5", + "latestReleaseDate": "2024-10-15", + "lts": true, + "support": "2024-12-31", + "discontinued": false + }, + { + "cycle": "20", + "releaseDate": "2023-03-17", + "eol": true, + "latest": "20.0.2", + "latestReleaseDate": "2023-07-18", + "lts": "2023-09-01", + "support": false + } + ]`, + wantErr: false, + validateData: func(t *testing.T, data EOLData) { + require.Len(t, data, 2) + + // Check first entry + assert.Equal(t, "21", data[0].Cycle) + assert.Equal(t, "2028-09-01", data[0].EOL.String()) + assert.True(t, data[0].LTS.Bool()) + + // Check second entry + assert.Equal(t, "20", data[1].Cycle) + assert.True(t, data[1].EOL.Bool()) + assert.True(t, data[1].LTS.Bool()) // Non-empty date string should be true + }, + }, + { + name: "non-200 response", + responseCode: http.StatusNotFound, + responseBody: `{"error": "Product not found"}`, + wantErr: true, + }, + { + name: "invalid JSON response", + responseCode: http.StatusOK, + responseBody: `{invalid json}`, + wantErr: true, + }, + { + name: "empty response", + responseCode: http.StatusOK, + responseBody: `[]`, + wantErr: false, + validateData: func(t *testing.T, data EOLData) { + assert.Len(t, data, 0) + }, + }, + { + name: "malformed field types", + responseCode: http.StatusOK, + responseBody: `[{"cycle": true, "eol": {}}]`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.responseCode) + _, err := w.Write([]byte(tt.responseBody)) + require.NoError(t, err) + })) + defer server.Close() + + eolAPIBase = server.URL + "/api" + client := &http.Client{} + + data, err := fetchProductEOL(context.Background(), client, "test-product") + if tt.wantErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + if tt.validateData != nil { + tt.validateData(t, data) + } + }) + } +} + +// Test mixed types for EOL and LTS values +func TestEOLDataMixedTypes(t *testing.T) { + data := []struct { + name string + eolValue EOLStringOrBool + ltsValue EOLStringOrBool + wantIsEOL bool + wantIsLTS bool + }{ + { + name: "string date EOL and boolean LTS", + eolValue: NewStringValue("2023-01-01"), + ltsValue: NewBoolValue(true), + wantIsEOL: true, // Past date + wantIsLTS: true, + }, + { + name: "boolean EOL and string date LTS", + eolValue: NewBoolValue(true), + ltsValue: NewStringValue("2023-01-01"), + wantIsEOL: true, + wantIsLTS: true, // Non-empty string + }, + { + name: "future date EOL", + eolValue: NewStringValue("2030-01-01"), + ltsValue: NewBoolValue(false), + wantIsEOL: false, // Future date + wantIsLTS: false, + }, + } + + for _, tt := range data { + t.Run(tt.name, func(t *testing.T) { + cycleData := CycleData{ + Cycle: "test", + EOL: tt.eolValue, + LTS: tt.ltsValue, + ReleaseDate: "2022-01-01", + Latest: "1.0.0", + } + + assert.Equal(t, tt.wantIsEOL, cycleData.EOL.Bool(), "EOL bool value mismatch") + assert.Equal(t, tt.wantIsLTS, cycleData.LTS.Bool(), "LTS bool value mismatch") + }) + } +} diff --git a/pkg/cli/store.go b/pkg/cli/store.go index cf62a3896f..4e97f3b1f9 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -71,6 +71,9 @@ func init() { // the ingestor will query and ingest clearly defined for licenses set.Bool("add-license-on-ingest", false, "if enabled, the ingestor will query and ingest clearly defined for licenses. Warning: This will increase ingestion times") + // the ingestor will query and ingest endoflife.date for EOL + set.Bool("add-eol-on-ingest", false, "if enabled, the ingestor will query and ingest endoflife.date for EOL data. Warning: This will increase ingestion times") + set.String("neptune-endpoint", "localhost", "address to neptune db") set.Int("neptune-port", 8182, "port used for neptune db connection") set.String("neptune-region", "us-east-1", "region to connect to neptune db") diff --git a/pkg/handler/processor/processor.go b/pkg/handler/processor/processor.go index 21c2def9f1..b9ffa36294 100644 --- a/pkg/handler/processor/processor.go +++ b/pkg/handler/processor/processor.go @@ -59,6 +59,7 @@ const ( DocumentITE6SLSA DocumentType = "SLSA" DocumentITE6Generic DocumentType = "ITE6" DocumentITE6Vul DocumentType = "ITE6VUL" + DocumentITE6EOL DocumentType = "ITE6EOL" // ClearlyDefined DocumentITE6ClearlyDefined DocumentType = "ITE6CD" DocumentDSSE DocumentType = "DSSE" @@ -71,6 +72,7 @@ const ( DocumentOpenVEX DocumentType = "OPEN_VEX" DocumentIngestPredicates DocumentType = "INGEST_PREDICATES" DocumentUnknown DocumentType = "UNKNOWN" + // End of life ) // FormatType describes the document format for malform checks diff --git a/pkg/ingestor/ingestor.go b/pkg/ingestor/ingestor.go index a678049f10..83d67b6f27 100644 --- a/pkg/ingestor/ingestor.go +++ b/pkg/ingestor/ingestor.go @@ -44,11 +44,12 @@ func Ingest( csubClient csub_client.Client, scanForVulns bool, scanForLicense bool, + scanForEOL bool, ) (*helpers.AssemblerIngestedIDs, error) { logger := d.ChildLogger // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense) + ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, d.ChildLogger, graphqlEndpoint, transport) @@ -87,11 +88,12 @@ func MergedIngest( csubClient csub_client.Client, scanForVulns bool, scanForLicense bool, + scanForEOL bool, ) error { logger := logging.FromContext(ctx) // Get pipeline of components processorFunc := GetProcessor(ctx) - ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense) + ingestorFunc := GetIngestor(ctx, scanForVulns, scanForLicense, scanForEOL) collectSubEmitFunc := GetCollectSubEmit(ctx, csubClient) assemblerFunc := GetAssembler(ctx, logger, graphqlEndpoint, transport) @@ -164,9 +166,9 @@ func GetProcessor(ctx context.Context) func(*processor.Document) (processor.Docu } } -func GetIngestor(ctx context.Context, scanForVulns bool, scanForLicense bool) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { +func GetIngestor(ctx context.Context, scanForVulns bool, scanForLicense bool, scanForEOL bool) func(processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { return func(doc processor.DocumentTree) ([]assembler.IngestPredicates, []*parser_common.IdentifierStrings, error) { - return parser.ParseDocumentTree(ctx, doc, scanForVulns, scanForLicense) + return parser.ParseDocumentTree(ctx, doc, scanForVulns, scanForLicense, scanForEOL) } } diff --git a/pkg/ingestor/parser/common/scanner/scanner.go b/pkg/ingestor/parser/common/scanner/scanner.go index 194ec7a976..1665981d93 100644 --- a/pkg/ingestor/parser/common/scanner/scanner.go +++ b/pkg/ingestor/parser/common/scanner/scanner.go @@ -23,8 +23,10 @@ import ( "github.com/guacsec/guac/pkg/assembler" cd_certifier "github.com/guacsec/guac/pkg/certifier/clearlydefined" osv_certifier "github.com/guacsec/guac/pkg/certifier/osv" + eol_certifier "github.com/guacsec/guac/pkg/certifier/eol" "github.com/guacsec/guac/pkg/ingestor/parser/clearlydefined" "github.com/guacsec/guac/pkg/ingestor/parser/common" + "github.com/guacsec/guac/pkg/ingestor/parser/eol" "github.com/guacsec/guac/pkg/ingestor/parser/vuln" "github.com/guacsec/guac/pkg/version" ) @@ -127,3 +129,26 @@ func runQueryOnBatchedPurls(ctx context.Context, cdParser common.DocumentParser, } return certLegalIngest, hasSourceAtIngest, nil } + +func PurlsEOLScan(ctx context.Context, purls []string) ([]assembler.HasMetadataIngest, error) { + // use the existing EOL parser to parse and obtain EOL metadata + eolParser := eol.NewEOLCertificationParser() + var eolIngest []assembler.HasMetadataIngest + + if eolProcessorDocs, err := eol_certifier.EvaluateEOLResponse(ctx, &http.Client{ + Transport: version.UATransport, + }, purls, nil); err != nil { + return nil, fmt.Errorf("failed to get response from endoflife.date with error: %w", err) + } else { + for _, doc := range eolProcessorDocs { + err := eolParser.Parse(ctx, doc) + if err != nil { + return nil, fmt.Errorf("EOL parser failed with error: %w", err) + } + preds := eolParser.GetPredicates(ctx) + common.AddMetadata(preds, nil, doc.SourceInformation) + eolIngest = append(eolIngest, preds.HasMetadata...) + } + } + return eolIngest, nil +} diff --git a/pkg/ingestor/parser/common/scanner/scanner_test.go b/pkg/ingestor/parser/common/scanner/scanner_test.go index 37779dacac..948c92de33 100644 --- a/pkg/ingestor/parser/common/scanner/scanner_test.go +++ b/pkg/ingestor/parser/common/scanner/scanner_test.go @@ -478,3 +478,200 @@ func TestPurlsLicenseScan(t *testing.T) { }) } } + +func TestPurlsEOLScan(t *testing.T) { + ctx := logging.WithLogger(context.Background()) + + tests := []struct { + name string + purls []string + wantHM func() []assembler.HasMetadataIngest + wantErr bool + }{ + { + name: "valid nodejs EOL data", + purls: []string{"pkg:npm/nodejs@14.17.0"}, + wantHM: func() []assembler.HasMetadataIngest { + return []assembler.HasMetadataIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "npm", + Name: "nodejs", + Version: ptrfrom.String("14.17.0"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:nodejs,cycle:14,version:14.17.0,isEOL:true,eolDate:2023-04-30,lts:true,latest:14.21.3,releaseDate:2020-04-21", + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + } + }, + wantErr: false, + }, + { + name: "valid python EOL data", + purls: []string{"pkg:pypi/python@3.9.5"}, + wantHM: func() []assembler.HasMetadataIngest { + return []assembler.HasMetadataIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "pypi", + Name: "python", + Version: ptrfrom.String("3.9.5"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:python,cycle:3.13,version:3.9.5,isEOL:false,eolDate:2029-10-31,lts:false,latest:3.13.0,releaseDate:2024-10-07", + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + } + }, + wantErr: false, + }, + { + name: "no purl", + purls: []string{""}, + wantHM: func() []assembler.HasMetadataIngest { return nil }, + wantErr: false, + }, + { + name: "skip guac purl", + purls: []string{"pkg:guac/python@3.9.5"}, + wantHM: func() []assembler.HasMetadataIngest { return nil }, + wantErr: false, + }, + { + name: "unsupported product", + purls: []string{"pkg:npm/unsupported-product@1.0.0"}, + wantHM: func() []assembler.HasMetadataIngest { return nil }, + wantErr: false, + }, + { + name: "multiple purls", + purls: []string{"pkg:npm/nodejs@14.17.0", "pkg:pypi/python@3.9.5"}, + wantHM: func() []assembler.HasMetadataIngest { + return []assembler.HasMetadataIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "npm", + Name: "nodejs", + Version: ptrfrom.String("14.17.0"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:nodejs,cycle:14,version:14.17.0,isEOL:true,eolDate:2023-04-30,lts:true,latest:14.21.3,releaseDate:2020-04-21", + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + { + Pkg: &generated.PkgInputSpec{ + Type: "pypi", + Name: "python", + Version: ptrfrom.String("3.9.5"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:python,cycle:3.13,version:3.9.5,isEOL:false,eolDate:2029-10-31,lts:false,latest:3.13.0,releaseDate:2024-10-07", + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + } + }, + wantErr: false, + }, + } + + var ignoreMetadataTimestamp = cmp.FilterPath(func(p cmp.Path) bool { + return strings.Compare(".Timestamp", p[len(p)-1].String()) == 0 + }, cmp.Ignore()) + + var ignoreMetadataValue = cmp.FilterPath(func(p cmp.Path) bool { + return p.String() == "HasMetadata.Value" + }, cmp.Comparer(func(x, y string) bool { + return verifyEOLValue(x, y) + })) + + hmSortOpt := cmp.Transformer("Sort", func(in []assembler.HasMetadataIngest) []assembler.HasMetadataIngest { + out := append([]assembler.HasMetadataIngest(nil), in...) + sort.Slice(out, func(i, j int) bool { + return strings.Compare(out[i].Pkg.Name, out[j].Pkg.Name) < 0 + }) + return out + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHM, err := PurlsEOLScan(ctx, tt.purls) + if (err != nil) != tt.wantErr { + t.Errorf("PurlsEOLScan() error = %v, wantErr %v", err, tt.wantErr) + return + } + + wantHM := tt.wantHM() + if diff := cmp.Diff(wantHM, gotHM, hmSortOpt, ignoreMetadataTimestamp, + ignoreMetadataValue, + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, "DocumentRef")); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +} + +// verifyEOLValue validates the EOL value string has correct format and expected fields +func verifyEOLValue(got, want string) bool { + gotMap := parseEOLValue(got) + wantMap := parseEOLValue(want) + + // Verify required fields exist + requiredFields := []string{"product", "cycle", "version", "eolDate", "releaseDate"} + for _, field := range requiredFields { + if gotMap[field] != wantMap[field] { + return false + } + } + + // Verify boolean fields have valid values + boolFields := []string{"isEOL", "lts"} + for _, field := range boolFields { + gotVal := gotMap[field] + if gotVal != "true" && gotVal != "false" { + return false + } + } + + return true +} + +// parseEOLValue parses an EOL value string into a map +func parseEOLValue(value string) map[string]string { + result := make(map[string]string) + for _, pair := range strings.Split(value, ",") { + parts := strings.Split(pair, ":") + if len(parts) == 2 { + result[parts[0]] = parts[1] + } + } + return result +} diff --git a/pkg/ingestor/parser/eol/eol.go b/pkg/ingestor/parser/eol/eol.go new file mode 100644 index 0000000000..e0fa6a0350 --- /dev/null +++ b/pkg/ingestor/parser/eol/eol.go @@ -0,0 +1,173 @@ +// +// Copyright 2024 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eol + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/guacsec/guac/pkg/assembler" + "github.com/guacsec/guac/pkg/assembler/clients/generated" + "github.com/guacsec/guac/pkg/assembler/helpers" + "github.com/guacsec/guac/pkg/certifier/attestation" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/guacsec/guac/pkg/ingestor/parser/common" + "github.com/guacsec/guac/pkg/logging" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary + +const ( + justification = "Retrieved from endoflife.date" +) + +type parser struct { + doc *processor.Document + pkg *generated.PkgInputSpec + collectedEOLInfo []assembler.HasMetadataIngest + identifierStrings *common.IdentifierStrings + timeScanned time.Time +} + +// NewEOLCertificationParser initializes the parser +func NewEOLCertificationParser() common.DocumentParser { + return &parser{ + identifierStrings: &common.IdentifierStrings{}, + } +} + +// initializeEOLParser clears out all values for the next iteration +func (e *parser) initializeEOLParser() { + e.doc = nil + e.pkg = nil + e.collectedEOLInfo = make([]assembler.HasMetadataIngest, 0) + e.identifierStrings = &common.IdentifierStrings{} + e.timeScanned = time.Now() +} + +// Parse breaks out the document into the graph components +func (e *parser) Parse(ctx context.Context, doc *processor.Document) error { + logger := logging.FromContext(ctx) + e.initializeEOLParser() + e.doc = doc + + statement, err := parseEOLCertifyPredicate(doc.Blob) + if err != nil { + return fmt.Errorf("failed to parse EOL predicate: %w", err) + } + + if statement.Predicate.Metadata.ScannedOn != nil { + e.timeScanned = *statement.Predicate.Metadata.ScannedOn + } else { + logger.Warn("no scan time found in EOL statement") + e.timeScanned = time.Now() + } + + if err := e.parseSubject(statement); err != nil { + logger.Warnf("unable to parse subject of statement: %v", err) + return fmt.Errorf("unable to parse subject of statement: %w", err) + } + + if err := e.parseEOL(ctx, statement); err != nil { + logger.Warnf("unable to parse EOL statement: %v", err) + return fmt.Errorf("unable to parse EOL statement: %w", err) + } + + return nil +} + +func parseEOLCertifyPredicate(p []byte) (*attestation.EOLStatement, error) { + predicate := attestation.EOLStatement{} + if err := json.Unmarshal(p, &predicate); err != nil { + return nil, fmt.Errorf("failed to unmarshal EOL predicate: %w", err) + } + return &predicate, nil +} + +func (e *parser) parseSubject(s *attestation.EOLStatement) error { + if len(s.Statement.Subject) == 0 { + return fmt.Errorf("no subject found in EOL statement") + } + + for _, sub := range s.Statement.Subject { + p, err := helpers.PurlToPkg(sub.Uri) + if err != nil { + return fmt.Errorf("failed to parse uri: %s to a package with error: %w", sub.Uri, err) + } + e.pkg = p + e.identifierStrings.PurlStrings = append(e.identifierStrings.PurlStrings, sub.Uri) + } + return nil +} + +// parseEOL parses the attestation to collect the EOL information +func (e *parser) parseEOL(_ context.Context, s *attestation.EOLStatement) error { + if e.pkg == nil { + return fmt.Errorf("package not specified for EOL information") + } + + // Create metadata for EOL status + eolInfo := assembler.HasMetadataIngest{ + Pkg: e.pkg, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: fmt.Sprintf("product:%s,cycle:%s,version:%s,isEOL:%v,eolDate:%s,lts:%v,latest:%s,releaseDate:%s", + s.Predicate.Product, + s.Predicate.Cycle, + s.Predicate.Version, + s.Predicate.IsEOL, + s.Predicate.EOLDate, + s.Predicate.LTS, + s.Predicate.Latest, + s.Predicate.ReleaseDate), + Timestamp: e.timeScanned, + Justification: justification, + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + } + + e.collectedEOLInfo = append(e.collectedEOLInfo, eolInfo) + + return nil +} + +func (e *parser) GetPredicates(ctx context.Context) *assembler.IngestPredicates { + logger := logging.FromContext(ctx) + preds := &assembler.IngestPredicates{} + + if e.pkg == nil { + logger.Error("error getting predicates: unable to find package element") + return preds + } + + preds.HasMetadata = e.collectedEOLInfo + return preds +} + +// GetIdentities gets the identity node from the document if they exist +func (e *parser) GetIdentities(ctx context.Context) []common.TrustInformation { + return nil +} + +func (e *parser) GetIdentifiers(ctx context.Context) (*common.IdentifierStrings, error) { + common.RemoveDuplicateIdentifiers(e.identifierStrings) + return e.identifierStrings, nil +} diff --git a/pkg/ingestor/parser/eol/eol_test.go b/pkg/ingestor/parser/eol/eol_test.go new file mode 100644 index 0000000000..d63b0f618b --- /dev/null +++ b/pkg/ingestor/parser/eol/eol_test.go @@ -0,0 +1,131 @@ +// +// Copyright 2024 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eol + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/testdata" + "github.com/guacsec/guac/pkg/assembler" + "github.com/guacsec/guac/pkg/assembler/clients/generated" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/guacsec/guac/pkg/logging" +) + +func TestParser(t *testing.T) { + ctx := logging.WithLogger(context.Background()) + tm, _ := time.Parse(time.RFC3339, "2024-03-15T12:00:00Z") + tests := []struct { + name string + doc *processor.Document + wantHM []assembler.HasMetadataIngest + wantErr bool + }{ + { + name: "valid EOL data for Node.js", + doc: &processor.Document{ + Blob: testdata.ITE6EOLNodejs, + Format: processor.FormatJSON, + Type: processor.DocumentITE6EOL, + SourceInformation: processor.SourceInformation{ + Collector: "TestCollector", + Source: "TestSource", + }, + }, + wantHM: []assembler.HasMetadataIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "npm", + Name: "nodejs", + Version: ptrfrom.String("14.17.0"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:nodejs,cycle:14,version:14.17.0,isEOL:true,eolDate:2023-04-30,lts:true,latest:14.21.3,releaseDate:2021-05-11", + Timestamp: tm, + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + }, + wantErr: false, + }, + { + name: "valid EOL data for Python", + doc: &processor.Document{ + Blob: testdata.ITE6EOLPython, + Format: processor.FormatJSON, + Type: processor.DocumentITE6EOL, + SourceInformation: processor.SourceInformation{ + Collector: "TestCollector", + Source: "TestSource", + }, + }, + wantHM: []assembler.HasMetadataIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "pypi", + Name: "python", + Version: ptrfrom.String("3.9.5"), + Namespace: ptrfrom.String(""), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "endoflife", + Value: "product:python,cycle:3.9,version:3.9.5,isEOL:false,eolDate:2025-10-05,lts:false,latest:3.9.16,releaseDate:2021-05-03", + Timestamp: tm, + Justification: "Retrieved from endoflife.date", + Origin: "GUAC EOL Certifier", + Collector: "GUAC", + }, + }, + }, + wantErr: false, + }, + } + + var ignoreHMTimestamp = cmp.FilterPath(func(p cmp.Path) bool { + return strings.Compare(".Timestamp", p[len(p)-1].String()) == 0 + }, cmp.Ignore()) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewEOLCertificationParser() + err := s.Parse(ctx, tt.doc) + if (err != nil) != tt.wantErr { + t.Fatalf("parser.Parse() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + return + } + ip := s.GetPredicates(ctx) + if diff := cmp.Diff(tt.wantHM, ip.HasMetadata, ignoreHMTimestamp); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/ingestor/parser/parser.go b/pkg/ingestor/parser/parser.go index cff72e8db3..013824ae0f 100644 --- a/pkg/ingestor/parser/parser.go +++ b/pkg/ingestor/parser/parser.go @@ -29,6 +29,7 @@ import ( "github.com/guacsec/guac/pkg/ingestor/parser/cyclonedx" "github.com/guacsec/guac/pkg/ingestor/parser/deps_dev" "github.com/guacsec/guac/pkg/ingestor/parser/dsse" + "github.com/guacsec/guac/pkg/ingestor/parser/eol" "github.com/guacsec/guac/pkg/ingestor/parser/open_vex" "github.com/guacsec/guac/pkg/ingestor/parser/scorecard" "github.com/guacsec/guac/pkg/ingestor/parser/slsa" @@ -47,6 +48,7 @@ func init() { _ = RegisterDocumentParser(deps_dev.NewDepsDevParser, processor.DocumentDepsDev) _ = RegisterDocumentParser(csaf.NewCsafParser, processor.DocumentCsaf) _ = RegisterDocumentParser(open_vex.NewOpenVEXParser, processor.DocumentOpenVEX) + _ = RegisterDocumentParser(eol.NewEOLCertificationParser, processor.DocumentITE6EOL) } var ( @@ -75,7 +77,7 @@ func RegisterDocumentParser(p func() common.DocumentParser, d processor.Document } // ParseDocumentTree takes the DocumentTree and create graph inputs (nodes and edges) per document node. -func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scanForVulns bool, scanForLicense bool) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { +func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scanForVulns bool, scanForLicense bool, scanForEOL bool) ([]assembler.IngestPredicates, []*common.IdentifierStrings, error) { var wg sync.WaitGroup assemblerInputs := []assembler.IngestPredicates{} @@ -143,6 +145,27 @@ func ParseDocumentTree(ctx context.Context, docTree processor.DocumentTree, scan } }() } + + if scanForEOL { + wg.Add(1) + go func() { + defer wg.Done() + // scrape EOL information from the EOL API + var purls []string + for _, idString := range identifierStrings { + purls = append(purls, idString.PurlStrings...) + } + + eolData, err := scanner.PurlsEOLScan(ctx, purls) + if err != nil { + logger.Errorf("error scraping purls for EOL information %v", err) + } else { + if len(assemblerInputs) > 0 { + assemblerInputs[0].HasMetadata = eolData + } + } + }() + } wg.Wait() return assemblerInputs, identifierStrings, nil diff --git a/pkg/ingestor/parser/parser_test.go b/pkg/ingestor/parser/parser_test.go index d6c405c8be..5aa639b552 100644 --- a/pkg/ingestor/parser/parser_test.go +++ b/pkg/ingestor/parser/parser_test.go @@ -293,7 +293,7 @@ func TestParseDocumentTree(t *testing.T) { _ = RegisterDocumentParser(f, test.registerDocType) // Ignoring error because it is mutating a global variable - got, got1, err := ParseDocumentTree(ctx, test.docTree, true, true) + got, got1, err := ParseDocumentTree(ctx, test.docTree, true, true, true) if (err != nil) != test.wantErr { t.Errorf("ParseDocumentTree() error = %v, wantErr %v", err, test.wantErr)