Merge pull request #210 from intelops/jfrogcontainer

Jfrogcontainer-registry

agent/container/openapi.yaml

@@ -52,5 +52,13 @@ paths:
       responses:
         '200':
           description: OK
+  /event/jfrog/container:
+    post:
+      tags:
+        - public
+      summary: Post Jfrog Container Registry webhook events
+      responses:
+        '200':
+          description: OK 
 
 # oapi-codegen -config ./cfg.yaml ./openapi.yaml

agent/container/pkg/handler/api_handler.go

@@ -36,6 +36,7 @@ func (ah *APIHandler) BindRequest(r *gin.Engine) {
 		apiGroup.POST("/event/docker/hub", ah.PostEventDockerHub)
 		apiGroup.POST("/event/azure/container", ah.PostEventAzureContainer)
 		apiGroup.POST("/event/quay/container", ah.PostEventQuayContainer)
+		apiGroup.POST("/event/jfrog/container", ah.PostEventJfrogContainer)
 	}
 }
 

agent/container/pkg/handler/jfrog_container.go

@@ -0,0 +1,45 @@
+package handler
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"log"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/intelops/kubviz/model"
+)
+
+var ErrInvalidPayloads = errors.New("invalid or malformed jfrog Container Registry webhook payload")
+
+func (ah *APIHandler) PostEventJfrogContainer(c *gin.Context) {
+	defer func() {
+		_, _ = io.Copy(io.Discard, c.Request.Body)
+		_ = c.Request.Body.Close()
+	}()
+	payload, err := io.ReadAll(c.Request.Body)
+	if err != nil || len(payload) == 0 {
+		log.Printf("%v: %v", ErrReadingBody, err)
+		c.Status(http.StatusBadRequest)
+		return
+	}
+
+	var pushEvent model.JfrogContainerPushEventPayload
+	err = json.Unmarshal(payload, &pushEvent)
+	if err != nil {
+		log.Printf("%v: %v", ErrInvalidPayloads, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Bad Request"})
+		return
+	}
+
+	log.Printf("Received event from jfrog Container Registry: %v", pushEvent)
+
+	err = ah.conn.Publish(payload, "Jfrog_Container_Registry")
+	if err != nil {
+		log.Printf("%v: %v", ErrPublishToNats, err)
+		c.Status(http.StatusInternalServerError)
+		return
+	}
+	c.Status(http.StatusOK)
+}

client/pkg/clickhouse/db_client.go

@@ -42,6 +42,7 @@ type DBInterface interface {
 	InsertContainerEventDockerHub(model.DockerHubBuild)
 	InsertContainerEventAzure(model.AzureContainerPushEventPayload)
 	InsertContainerEventQuay(model.QuayImagePushPayload)
+	InsertContainerEventJfrog(model.JfrogContainerPushEventPayload)
 	InsertContainerEventGithub(string)
 	InsertGitCommon(metrics model.GitCommonAttribute, statement dbstatement.DBStatement) error
 	Close()
@@ -73,7 +74,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
 		return nil, err
 	}
 
-	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable,quayContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
+	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, quayContainerPushEventTable, jfrogContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
 	for _, table := range tables {
 		if err = splconn.Exec(context.Background(), string(table)); err != nil {
 			return nil, err
@@ -174,7 +175,47 @@ func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload
 		log.Fatal(err)
 	}
 }
+func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushEventPayload) {
+	var (
+		tx, _   = c.conn.Begin()
+		stmt, _ = tx.Prepare(string(InsertJfrogContainerPushEvent))
+	)
+	defer stmt.Close()
+	registryURL := pushEvent.Data.Path
+	repositoryName := pushEvent.Data.Name
+	tag := pushEvent.Data.Tag
+
+	if tag == "" {
+		tag = "latest"
+	}
+	imageName := pushEvent.Data.ImageName
+	size := pushEvent.Data.Size
+	shaID := pushEvent.Data.SHA256
 
+	// Marshaling the pushEvent into a JSON string
+	pushEventJSON, err := json.Marshal(pushEvent)
+	if err != nil {
+		log.Printf("Error while marshaling Jfrog Container Registry payload: %v", err)
+		return
+	}
+
+	if _, err := stmt.Exec(
+		pushEvent.Domain,
+		pushEvent.EventType,
+		registryURL,
+		repositoryName,
+		shaID,
+		size,
+		imageName,
+		tag,
+		string(pushEventJSON),
+	); err != nil {
+		log.Fatal(err)
+	}
+	if err := tx.Commit(); err != nil {
+		log.Fatal(err)
+	}
+}
 func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
 	var (
 		tx, _   = c.conn.Begin()

client/pkg/clickhouse/statements.go

@@ -51,6 +51,19 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs (
 	Scope String
 ) engine=File(TabSeparated)
 `
+const jfrogContainerPushEventTable DBStatement = `
+	CREATE TABLE IF NOT EXISTS jfrogcontainerpush (
+		Domain String,
+		EventType String,
+		RegistryURL String,
+		RepositoryName String,
+		SHAID String,
+		Size Int32,
+		ImageName String,
+		Tag String,
+		Event String
+	) engine=File(TabSeparated)
+	`
 const ketallTable DBStatement = `
 CREATE TABLE IF NOT EXISTS getall_resources (
 	ClusterName String,
@@ -211,3 +224,4 @@ const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_na
 const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number,  version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 const InsertQuayContainerPushEvent DBStatement = "INSERT INTO quaycontainerpush (name, repository, nameSpace, dockerURL, homePage,tag, Event) VALUES (?, ?, ?, ?, ?, ?, ?)"
+const InsertJfrogContainerPushEvent DBStatement = "INSERT INTO jfrogcontainerpush (Domain, EventType,RegistryURL, RepositoryName,SHAID, Size, ImageName ,Tag, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?,?)"

client/pkg/clients/container_client.go

@@ -59,7 +59,7 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) {
 			// Extract the necessary information from pushEvent and insert into ClickHouse
 			conn.InsertContainerEventAzure(pushEvent)
 			log.Println("Inserted Azure Container Registry metrics:", string(msg.Data))
-		}else if repoName == "Quay_Container_Registry" {
+		} else if repoName == "Quay_Container_Registry" {
 			var pushEvent model.QuayImagePushPayload
 			err := json.Unmarshal(msg.Data, &pushEvent)
 			if err != nil {
@@ -69,6 +69,16 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) {
 			// Extract the necessary information from pushEvent and insert into ClickHouse
 			conn.InsertContainerEventQuay(pushEvent)
 			log.Println("Inserted Quay Container Registry metrics:", string(msg.Data))
+		} else if repoName == "Jfrog_Container_Registry" {
+			var pushEvent model.JfrogContainerPushEventPayload
+			err := json.Unmarshal(msg.Data, &pushEvent)
+			if err != nil {
+				log.Printf("Error while unmarshaling Jfrog Container Registry payload: %v", err)
+				return
+			}
+			// Extract the necessary information from pushEvent and insert into ClickHouse.
+			conn.InsertContainerEventJfrog(pushEvent)
+			log.Println("Inserted Jfrog Container Registry metrics:", string(msg.Data))
 		}
 
 	}, nats.Durable(string(containerConsumer)), nats.ManualAck())

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/nats-io/nats.go v1.27.1
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/vijeyash1/go-github-container v1.0.0
 	golang.org/x/term v0.10.0
 	k8s.io/api v0.27.3
 	k8s.io/apimachinery v0.27.3

go.sum

@@ -440,8 +440,6 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
-github.com/vijeyash1/go-github-container v1.0.0 h1:SWtzxwGFFSCn8UB27IMcCbQ9xg1l6sQgk3pW2aD0fsQ=
-github.com/vijeyash1/go-github-container v1.0.0/go.mod h1:yljHpWvbjXtjy48MXoBonmrTBUYNk8iA0cACfyU0Om4=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=

model/jfrogcontainer.go

@@ -0,0 +1,18 @@
+package model
+
+type JfrogContainerPushEventPayload struct {
+	Domain    string `json:"domain"`
+	EventType string `json:"event_type"`
+	Data      struct {
+		RepoKey   string `json:"repo_key"`
+		Path      string `json:"path"`
+		Name      string `json:"name"`
+		SHA256    string `json:"sha256"`
+		Size      int32  `json:"size"`
+		ImageName string `json:"image_name"`
+		Tag       string `json:"tag"`
+	} `json:"data"`
+	SubscriptionKey string `json:"subscription_key"`
+	JPDOrigin       string `json:"jpd_origin"`
+	Source          string `json:"source"`
+}