Merge pull request #208 from intelops/quay

quay-working

agent/container/cfg.yaml

@@ -3,4 +3,5 @@ generate:
   gin-server: true
   models: true
   embedded-spec: true
-output: agent/container/api/agent.gen.go
+output: api/agent.gen.go
+

agent/container/openapi.yaml

@@ -44,5 +44,13 @@ paths:
       responses:
         '200':
           description: OK
+  /event/quay/container:
+    post:
+      tags:
+        - public
+      summary: Post quay Container Registry webhook events
+      responses:
+        '200':
+          description: OK
 
 # oapi-codegen -config ./cfg.yaml ./openapi.yaml

agent/container/pkg/handler/api_handler.go

@@ -35,6 +35,7 @@ func (ah *APIHandler) BindRequest(r *gin.Engine) {
 		apiGroup.GET("/status", ah.GetStatus)
 		apiGroup.POST("/event/docker/hub", ah.PostEventDockerHub)
 		apiGroup.POST("/event/azure/container", ah.PostEventAzureContainer)
+		apiGroup.POST("/event/quay/container", ah.PostEventQuayContainer)
 	}
 }
 

agent/container/pkg/handler/quay_handler.go

@@ -0,0 +1,40 @@
+package handler
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/intelops/kubviz/model"
+)
+
+func (ah *APIHandler) PostEventQuayContainer(c *gin.Context) {
+	defer func() {
+		_, _ = io.Copy(io.Discard, c.Request.Body)
+		_ = c.Request.Body.Close()
+	}()
+	payload, err := io.ReadAll(c.Request.Body)
+	if err != nil || len(payload) == 0 {
+		log.Printf("%v: %v", ErrReadingBody, err)
+		c.Status(http.StatusBadRequest)
+		return
+	}
+	var pushEvent model.QuayImagePushPayload
+	err = json.Unmarshal(payload, &pushEvent)
+	if err != nil {
+		log.Printf("%v: %v", ErrInvalidPayload, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Bad Request"})
+		return
+	}
+	log.Printf("Received event from Quay Container Registry: %v", pushEvent)
+
+	err = ah.conn.Publish(payload, "Quay_Container_Registry")
+	if err != nil {
+		log.Printf("%v: %v", ErrPublishToNats, err)
+		c.Status(http.StatusInternalServerError)
+		return
+	}
+	c.Status(http.StatusOK)
+}

client/pkg/clickhouse/db_client.go

@@ -41,6 +41,7 @@ type DBInterface interface {
 	RetrieveKubvizEvent() ([]model.DbEvent, error)
 	InsertContainerEventDockerHub(model.DockerHubBuild)
 	InsertContainerEventAzure(model.AzureContainerPushEventPayload)
+	InsertContainerEventQuay(model.QuayImagePushPayload)
 	InsertContainerEventGithub(string)
 	InsertGitCommon(metrics model.GitCommonAttribute, statement dbstatement.DBStatement) error
 	Close()
@@ -72,7 +73,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
 		return nil, err
 	}
 
-	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
+	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable,quayContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
 	for _, table := range tables {
 		if err = splconn.Exec(context.Background(), string(table)); err != nil {
 			return nil, err
@@ -131,6 +132,48 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE
 		log.Fatal(err)
 	}
 }
+func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload) {
+	var (
+		tx, _   = c.conn.Begin()
+		stmt, _ = tx.Prepare(string(InsertQuayContainerPushEvent))
+	)
+	defer stmt.Close()
+	dockerURL := pushEvent.DockerURL
+	repository := pushEvent.Repository
+	//tag := pushEvent.UpdatedTags
+	name := pushEvent.Name
+	nameSpace := pushEvent.Namespace
+	homePage := pushEvent.Homepage
+
+	var tag string
+	if pushEvent.UpdatedTags != nil {
+		tag = strings.Join(pushEvent.UpdatedTags, ",")
+	} else {
+		tag = ""
+	}
+
+	// Marshaling the pushEvent into a JSON string
+	pushEventJSON, err := json.Marshal(pushEvent)
+	if err != nil {
+		log.Printf("Error while marshaling Quay Container Registry payload: %v", err)
+		return
+	}
+
+	if _, err := stmt.Exec(
+		name,
+		repository,
+		nameSpace,
+		dockerURL,
+		homePage,
+		tag,
+		string(pushEventJSON),
+	); err != nil {
+		log.Fatal(err)
+	}
+	if err := tx.Commit(); err != nil {
+		log.Fatal(err)
+	}
+}
 
 func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
 	var (

client/pkg/clickhouse/statements.go

@@ -157,6 +157,17 @@ const azureContainerPushEventTable DBStatement = `
 		SHAID String
 	) engine=File(TabSeparated)
 	`
+const quayContainerPushEventTable DBStatement = `
+	CREATE TABLE IF NOT EXISTS quaycontainerpush (
+		name String,
+		repository String,
+		nameSpace String,
+		dockerURL String,
+		homePage String,
+		tag String,
+		Event String
+	) engine=File(TabSeparated)
+	`
 const trivySbomTable DBStatement = `
 	CREATE TABLE IF NOT EXISTS trivysbom (
 		id UUID,
@@ -199,3 +210,4 @@ const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artif
 const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"
 const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number,  version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+const InsertQuayContainerPushEvent DBStatement = "INSERT INTO quaycontainerpush (name, repository, nameSpace, dockerURL, homePage,tag, Event) VALUES (?, ?, ?, ?, ?, ?, ?)"

client/pkg/clients/container_client.go

@@ -59,6 +59,16 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) {
 			// Extract the necessary information from pushEvent and insert into ClickHouse
 			conn.InsertContainerEventAzure(pushEvent)
 			log.Println("Inserted Azure Container Registry metrics:", string(msg.Data))
+		}else if repoName == "Quay_Container_Registry" {
+			var pushEvent model.QuayImagePushPayload
+			err := json.Unmarshal(msg.Data, &pushEvent)
+			if err != nil {
+				log.Printf("Error while unmarshaling Quay Container Registry payload: %v", err)
+				return
+			}
+			// Extract the necessary information from pushEvent and insert into ClickHouse
+			conn.InsertContainerEventQuay(pushEvent)
+			log.Println("Inserted Quay Container Registry metrics:", string(msg.Data))
 		}
 
 	}, nats.Durable(string(containerConsumer)), nats.ManualAck())

model/quay.go

@@ -0,0 +1,10 @@
+package model
+
+type QuayImagePushPayload struct {
+	Name        string   `json:"name"`
+	Repository  string   `json:"repository"`
+	Namespace   string   `json:"namespace"`
+	DockerURL   string   `json:"docker_url"`
+	Homepage    string   `json:"homepage"`
+	UpdatedTags []string `json:"updated_tags"`
+}