From d5958c7563f982577b447f8d984b755245fe5824 Mon Sep 17 00:00:00 2001 From: an1l4 <1995anila@gmail.com> Date: Tue, 5 Sep 2023 11:08:04 +0530 Subject: [PATCH] quay-working --- agent/container/api/agent.gen.go | 30 ++++++++++---- agent/container/cfg.yaml | 3 +- agent/container/openapi.yaml | 8 ++++ agent/container/pkg/handler/api_handler.go | 1 + agent/container/pkg/handler/quay_handler.go | 40 ++++++++++++++++++ client/pkg/clickhouse/db_client.go | 45 ++++++++++++++++++++- client/pkg/clickhouse/statements.go | 12 ++++++ client/pkg/clients/container_client.go | 10 +++++ model/quay.go | 10 +++++ 9 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 agent/container/pkg/handler/quay_handler.go create mode 100644 model/quay.go diff --git a/agent/container/api/agent.gen.go b/agent/container/api/agent.gen.go index 3b9beab7..81131417 100644 --- a/agent/container/api/agent.gen.go +++ b/agent/container/api/agent.gen.go @@ -27,6 +27,9 @@ type ServerInterface interface { // Post Dockerhub artifactory events // (POST /event/docker/hub) PostEventDockerHub(c *gin.Context) + // Post quay Container Registry webhook events + // (POST /event/quay/container) + PostEventQuayContainer(c *gin.Context) // Kubernetes readiness and liveness probe endpoint // (GET /status) GetStatus(c *gin.Context) @@ -71,6 +74,16 @@ func (siw *ServerInterfaceWrapper) PostEventDockerHub(c *gin.Context) { siw.Handler.PostEventDockerHub(c) } +// PostEventQuayContainer operation middleware +func (siw *ServerInterfaceWrapper) PostEventQuayContainer(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + } + + siw.Handler.PostEventQuayContainer(c) +} + // GetStatus operation middleware func (siw *ServerInterfaceWrapper) GetStatus(c *gin.Context) { @@ -116,6 +129,8 @@ func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options router.POST(options.BaseURL+"/event/docker/hub", wrapper.PostEventDockerHub) + router.POST(options.BaseURL+"/event/quay/container", wrapper.PostEventQuayContainer) + router.GET(options.BaseURL+"/status", wrapper.GetStatus) return router @@ -124,13 +139,14 @@ func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/6SSQWscMQyF/4rQebqzaW9zC01oQwoJ2d5CDh5buyMyaxtJnrJd5r8Xz9K0pCUJ7ckI", - "3nv6JOuIPu1zihRNsTvODXLcJuyOGEi9cDZOETv8mKI5jiTQC4cdwU2mCHeXm69wfnsFmsnzlr1b5A0a", - "20iv2zbPbBOJnvqdrdarNc4NpkzRZcYOP6zWqzNsMDsbKiu2LvO7kPxS7MjqkzLJknYVsMNPZOeZL6qk", - "QSHNKSot8vfr9Z9D3lzjPDeoZb93csAOv7AapG1lVciSJg4UoD+ADQRKMrGnOq3bKXb3mEs/sseHGtLS", - "RNFa970Itf7nGmrPnPQvqLdJ7bJazqvjaW//xl3DYAmCXz9wRztWkwN8o35I6REWQn2ZPyT/SNIOpX8D", - "+sUi/lz6/6A+ZQylByfGW+ctyeEVVDVn5cUr2JwUb8HS4j2pbssITzHPQK9LTxLJSEHIBY6kCi4GGHmi", - "pciSegKKISeO9ju38OSMKniNJKknj939EYuM2GGL88P8IwAA//90yrlhlQMAAA==", + "H4sIAAAAAAAC/6yTQW/TQBCF/8poziZO4eZbRCuoitTScKt6WK8nyajO7jIza2Qi/3e0jiiooBIKJ2uk", + "995882wf0Md9ioGCKTaHqUIOm4jNATtSL5yMY8AG38ZgjgMJtMLdluA6UYDbi/UnWN1cgibyvGHvZnmF", + "xtbTn23rJ7aBRI/7zhbLxRKnCmOi4BJjg28Wy8UZVpic7Qor1i7xqy76ediSlUdMJHPaZYcNviNbJT4v", + "kgqFNMWgNMtfL5e/Hnl9hdNUoeb93smIDX5gNYibwqqQJA7cUQftCLYjUJKBPZVr3VaxucOU25493peQ", + "mgYKVruvWaj232soO1PU36DeRLWLYlkVx2NvL+MuYTAHwY83cEtbVpMRvlC7i/EBZkJ9nr+L/oGk3uX2", + "BPTzWfw+t/9AfczY5RacGG+ctyjjSaifsxv/qumP2Y3/o+iy+OU9qznLz37C66PiFEDN3pPqJvfwGPME", + "+Sq3JIGMFIRcx4FUwYUOeh5oHpLEloBClyIH+5lbeHBGBbxEkpT/FZu7A2bpscEap/vpWwAAAP//KFB7", + "SlIEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/agent/container/cfg.yaml b/agent/container/cfg.yaml index 845d5fb1..59622b8b 100644 --- a/agent/container/cfg.yaml +++ b/agent/container/cfg.yaml @@ -3,4 +3,5 @@ generate: gin-server: true models: true embedded-spec: true -output: agent/container/api/agent.gen.go +output: api/agent.gen.go + diff --git a/agent/container/openapi.yaml b/agent/container/openapi.yaml index bcd76a59..3c0dba28 100755 --- a/agent/container/openapi.yaml +++ b/agent/container/openapi.yaml @@ -44,5 +44,13 @@ paths: responses: '200': description: OK + /event/quay/container: + post: + tags: + - public + summary: Post quay Container Registry webhook events + responses: + '200': + description: OK # oapi-codegen -config ./cfg.yaml ./openapi.yaml diff --git a/agent/container/pkg/handler/api_handler.go b/agent/container/pkg/handler/api_handler.go index 5e5e2834..0408926d 100755 --- a/agent/container/pkg/handler/api_handler.go +++ b/agent/container/pkg/handler/api_handler.go @@ -35,6 +35,7 @@ func (ah *APIHandler) BindRequest(r *gin.Engine) { apiGroup.GET("/status", ah.GetStatus) apiGroup.POST("/event/docker/hub", ah.PostEventDockerHub) apiGroup.POST("/event/azure/container", ah.PostEventAzureContainer) + apiGroup.POST("/event/quay/container", ah.PostEventQuayContainer) } } diff --git a/agent/container/pkg/handler/quay_handler.go b/agent/container/pkg/handler/quay_handler.go new file mode 100644 index 00000000..a002e05d --- /dev/null +++ b/agent/container/pkg/handler/quay_handler.go @@ -0,0 +1,40 @@ +package handler + +import ( + "encoding/json" + "io" + "log" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/intelops/kubviz/model" +) + +func (ah *APIHandler) PostEventQuayContainer(c *gin.Context) { + defer func() { + _, _ = io.Copy(io.Discard, c.Request.Body) + _ = c.Request.Body.Close() + }() + payload, err := io.ReadAll(c.Request.Body) + if err != nil || len(payload) == 0 { + log.Printf("%v: %v", ErrReadingBody, err) + c.Status(http.StatusBadRequest) + return + } + var pushEvent model.QuayImagePushPayload + err = json.Unmarshal(payload, &pushEvent) + if err != nil { + log.Printf("%v: %v", ErrInvalidPayload, err) + c.JSON(http.StatusBadRequest, gin.H{"error": "Bad Request"}) + return + } + log.Printf("Received event from Quay Container Registry: %v", pushEvent) + + err = ah.conn.Publish(payload, "Quay_Container_Registry") + if err != nil { + log.Printf("%v: %v", ErrPublishToNats, err) + c.Status(http.StatusInternalServerError) + return + } + c.Status(http.StatusOK) +} \ No newline at end of file diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index 7ef2c634..470f2514 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -41,6 +41,7 @@ type DBInterface interface { RetrieveKubvizEvent() ([]model.DbEvent, error) InsertContainerEventDockerHub(model.DockerHubBuild) InsertContainerEventAzure(model.AzureContainerPushEventPayload) + InsertContainerEventQuay(model.QuayImagePushPayload) InsertContainerEventGithub(string) InsertGitCommon(metrics model.GitCommonAttribute, statement dbstatement.DBStatement) error Close() @@ -72,7 +73,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { return nil, err } - tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable,quayContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} for _, table := range tables { if err = splconn.Exec(context.Background(), string(table)); err != nil { return nil, err @@ -131,6 +132,48 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE log.Fatal(err) } } +func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload) { + var ( + tx, _ = c.conn.Begin() + stmt, _ = tx.Prepare(string(InsertQuayContainerPushEvent)) + ) + defer stmt.Close() + dockerURL := pushEvent.DockerURL + repository := pushEvent.Repository + //tag := pushEvent.UpdatedTags + name := pushEvent.Name + nameSpace := pushEvent.Namespace + homePage := pushEvent.Homepage + + var tag string + if pushEvent.UpdatedTags != nil { + tag = strings.Join(pushEvent.UpdatedTags, ",") + } else { + tag = "" + } + + // Marshaling the pushEvent into a JSON string + pushEventJSON, err := json.Marshal(pushEvent) + if err != nil { + log.Printf("Error while marshaling Quay Container Registry payload: %v", err) + return + } + + if _, err := stmt.Exec( + name, + repository, + nameSpace, + dockerURL, + homePage, + tag, + string(pushEventJSON), + ); err != nil { + log.Fatal(err) + } + if err := tx.Commit(); err != nil { + log.Fatal(err) + } +} func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) { var ( diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index 7646af14..27e2c50d 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -157,6 +157,17 @@ const azureContainerPushEventTable DBStatement = ` SHAID String ) engine=File(TabSeparated) ` +const quayContainerPushEventTable DBStatement = ` + CREATE TABLE IF NOT EXISTS quaycontainerpush ( + name String, + repository String, + nameSpace String, + dockerURL String, + homePage String, + tag String, + Event String + ) engine=File(TabSeparated) + ` const trivySbomTable DBStatement = ` CREATE TABLE IF NOT EXISTS trivysbom ( id UUID, @@ -199,3 +210,4 @@ const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artif const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)" const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +const InsertQuayContainerPushEvent DBStatement = "INSERT INTO quaycontainerpush (name, repository, nameSpace, dockerURL, homePage,tag, Event) VALUES (?, ?, ?, ?, ?, ?, ?)" diff --git a/client/pkg/clients/container_client.go b/client/pkg/clients/container_client.go index 3688a102..e7f60f5e 100644 --- a/client/pkg/clients/container_client.go +++ b/client/pkg/clients/container_client.go @@ -59,6 +59,16 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) { // Extract the necessary information from pushEvent and insert into ClickHouse conn.InsertContainerEventAzure(pushEvent) log.Println("Inserted Azure Container Registry metrics:", string(msg.Data)) + }else if repoName == "Quay_Container_Registry" { + var pushEvent model.QuayImagePushPayload + err := json.Unmarshal(msg.Data, &pushEvent) + if err != nil { + log.Printf("Error while unmarshaling Quay Container Registry payload: %v", err) + return + } + // Extract the necessary information from pushEvent and insert into ClickHouse + conn.InsertContainerEventQuay(pushEvent) + log.Println("Inserted Quay Container Registry metrics:", string(msg.Data)) } }, nats.Durable(string(containerConsumer)), nats.ManualAck()) diff --git a/model/quay.go b/model/quay.go new file mode 100644 index 00000000..f8ae495b --- /dev/null +++ b/model/quay.go @@ -0,0 +1,10 @@ +package model + +type QuayImagePushPayload struct { + Name string `json:"name"` + Repository string `json:"repository"` + Namespace string `json:"namespace"` + DockerURL string `json:"docker_url"` + Homepage string `json:"homepage"` + UpdatedTags []string `json:"updated_tags"` +}