diff --git a/agent/container/api/agent.gen.go b/agent/container/api/agent.gen.go index d9da5f63..3b9beab7 100644 --- a/agent/container/api/agent.gen.go +++ b/agent/container/api/agent.gen.go @@ -8,218 +8,129 @@ import ( "compress/gzip" "encoding/base64" "fmt" - "net/http" "net/url" "path" "strings" "github.com/getkin/kin-openapi/openapi3" - "github.com/go-chi/chi/v5" + "github.com/gin-gonic/gin" ) // ServerInterface represents all server handlers. type ServerInterface interface { // List of APIs provided by the service // (GET /api-docs) - GetApiDocs(w http.ResponseWriter, r *http.Request) + GetApiDocs(c *gin.Context) + // Post Azure Container Registry webhook events + // (POST /event/azure/container) + PostEventAzureContainer(c *gin.Context) // Post Dockerhub artifactory events // (POST /event/docker/hub) - PostEventDockerHub(w http.ResponseWriter, r *http.Request) + PostEventDockerHub(c *gin.Context) // Kubernetes readiness and liveness probe endpoint // (GET /status) - GetStatus(w http.ResponseWriter, r *http.Request) + GetStatus(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. type ServerInterfaceWrapper struct { Handler ServerInterface HandlerMiddlewares []MiddlewareFunc - ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) + ErrorHandler func(*gin.Context, error, int) } -type MiddlewareFunc func(http.Handler) http.Handler +type MiddlewareFunc func(c *gin.Context) // GetApiDocs operation middleware -func (siw *ServerInterfaceWrapper) GetApiDocs(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.GetApiDocs(w, r) - }) +func (siw *ServerInterfaceWrapper) GetApiDocs(c *gin.Context) { for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) + middleware(c) } - handler.ServeHTTP(w, r.WithContext(ctx)) + siw.Handler.GetApiDocs(c) } -// PostEventDockerHub operation middleware -func (siw *ServerInterfaceWrapper) PostEventDockerHub(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.PostEventDockerHub(w, r) - }) +// PostEventAzureContainer operation middleware +func (siw *ServerInterfaceWrapper) PostEventAzureContainer(c *gin.Context) { for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) + middleware(c) } - handler.ServeHTTP(w, r.WithContext(ctx)) + siw.Handler.PostEventAzureContainer(c) } -// GetStatus operation middleware -func (siw *ServerInterfaceWrapper) GetStatus(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.GetStatus(w, r) - }) +// PostEventDockerHub operation middleware +func (siw *ServerInterfaceWrapper) PostEventDockerHub(c *gin.Context) { for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) + middleware(c) } - handler.ServeHTTP(w, r.WithContext(ctx)) -} - -type UnescapedCookieParamError struct { - ParamName string - Err error -} - -func (e *UnescapedCookieParamError) Error() string { - return fmt.Sprintf("error unescaping cookie parameter '%s'", e.ParamName) -} - -func (e *UnescapedCookieParamError) Unwrap() error { - return e.Err -} - -type UnmarshallingParamError struct { - ParamName string - Err error -} - -func (e *UnmarshallingParamError) Error() string { - return fmt.Sprintf("Error unmarshalling parameter %s as JSON: %s", e.ParamName, e.Err.Error()) -} - -func (e *UnmarshallingParamError) Unwrap() error { - return e.Err -} - -type RequiredParamError struct { - ParamName string -} - -func (e *RequiredParamError) Error() string { - return fmt.Sprintf("Query argument %s is required, but not found", e.ParamName) -} - -type RequiredHeaderError struct { - ParamName string - Err error -} - -func (e *RequiredHeaderError) Error() string { - return fmt.Sprintf("Header parameter %s is required, but not found", e.ParamName) -} - -func (e *RequiredHeaderError) Unwrap() error { - return e.Err -} - -type InvalidParamFormatError struct { - ParamName string - Err error -} - -func (e *InvalidParamFormatError) Error() string { - return fmt.Sprintf("Invalid format for parameter %s: %s", e.ParamName, e.Err.Error()) + siw.Handler.PostEventDockerHub(c) } -func (e *InvalidParamFormatError) Unwrap() error { - return e.Err -} - -type TooManyValuesForParamError struct { - ParamName string - Count int -} +// GetStatus operation middleware +func (siw *ServerInterfaceWrapper) GetStatus(c *gin.Context) { -func (e *TooManyValuesForParamError) Error() string { - return fmt.Sprintf("Expected one value for %s, got %d", e.ParamName, e.Count) -} + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + } -// Handler creates http.Handler with routing matching OpenAPI spec. -func Handler(si ServerInterface) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{}) + siw.Handler.GetStatus(c) } -type ChiServerOptions struct { - BaseURL string - BaseRouter chi.Router - Middlewares []MiddlewareFunc - ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) +// GinServerOptions provides options for the Gin server. +type GinServerOptions struct { + BaseURL string + Middlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) } -// HandlerFromMux creates http.Handler with routing matching OpenAPI spec based on the provided mux. -func HandlerFromMux(si ServerInterface, r chi.Router) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{ - BaseRouter: r, - }) +// RegisterHandlers creates http.Handler with routing matching OpenAPI spec. +func RegisterHandlers(router *gin.Engine, si ServerInterface) *gin.Engine { + return RegisterHandlersWithOptions(router, si, GinServerOptions{}) } -func HandlerFromMuxWithBaseURL(si ServerInterface, r chi.Router, baseURL string) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{ - BaseURL: baseURL, - BaseRouter: r, - }) -} +// RegisterHandlersWithOptions creates http.Handler with additional options +func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options GinServerOptions) *gin.Engine { -// HandlerWithOptions creates http.Handler with additional options -func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handler { - r := options.BaseRouter + errorHandler := options.ErrorHandler - if r == nil { - r = chi.NewRouter() - } - if options.ErrorHandlerFunc == nil { - options.ErrorHandlerFunc = func(w http.ResponseWriter, r *http.Request, err error) { - http.Error(w, err.Error(), http.StatusBadRequest) + if errorHandler == nil { + errorHandler = func(c *gin.Context, err error, statusCode int) { + c.JSON(statusCode, gin.H{"msg": err.Error()}) } } + wrapper := ServerInterfaceWrapper{ Handler: si, HandlerMiddlewares: options.Middlewares, - ErrorHandlerFunc: options.ErrorHandlerFunc, + ErrorHandler: errorHandler, } - r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/api-docs", wrapper.GetApiDocs) - }) - r.Group(func(r chi.Router) { - r.Post(options.BaseURL+"/event/docker/hub", wrapper.PostEventDockerHub) - }) - r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/status", wrapper.GetStatus) - }) - - return r + router.GET(options.BaseURL+"/api-docs", wrapper.GetApiDocs) + + router.POST(options.BaseURL+"/event/azure/container", wrapper.PostEventAzureContainer) + + router.POST(options.BaseURL+"/event/docker/hub", wrapper.PostEventDockerHub) + + router.GET(options.BaseURL+"/status", wrapper.GetStatus) + + return router } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/5ySz2ocMQyHX0XoPN2ZtDffQhPakEJC01vIwWNrdkVnbSPJA8sy7148C6Wkf+nJCH76", - "9MnojCEfS06UTNGd1w45TRndGSNpEC7GOaHD9zmZ50QCo3DcEzwUSvD59ukLXD/egRYKPHHwW7xDY5vp", - "721Pr9oWEr3Mu9oNuwHXDnOh5Aujw3e7YXeFHRZvh+aKvS/8JuawFXuy9uRCstHuIjr8QHZd+KZFOhTS", - "kpPSFn87DD8v+XCP69qh1uPRywkdfmI1yFNzVSiSF44UYTyBHQiUZOFAbVu/V3TPWOo4c8CXBulpoWR9", - "zOErSX+oYxtXsv7C8jGr3bb0zRb+WMf/s20cuDAOdQQvxpMPluUEm4z+TlXNW/3jLz5dEv+ipTUEUp3q", - "DN8xr0Tv60iSyEhByEdOpAo+RZh5oa0okkcCSrFkTvajt/DijZp4Q5K0k0H3fMYqMzrscX1ZvwUAAP//", - "Lv/PMNUCAAA=", + "H4sIAAAAAAAC/6SSQWscMQyF/4rQebqzaW9zC01oQwoJ2d5CDh5buyMyaxtJnrJd5r8Xz9K0pCUJ7ckI", + "3nv6JOuIPu1zihRNsTvODXLcJuyOGEi9cDZOETv8mKI5jiTQC4cdwU2mCHeXm69wfnsFmsnzlr1b5A0a", + "20iv2zbPbBOJnvqdrdarNc4NpkzRZcYOP6zWqzNsMDsbKiu2LvO7kPxS7MjqkzLJknYVsMNPZOeZL6qk", + "QSHNKSot8vfr9Z9D3lzjPDeoZb93csAOv7AapG1lVciSJg4UoD+ADQRKMrGnOq3bKXb3mEs/sseHGtLS", + "RNFa970Itf7nGmrPnPQvqLdJ7bJazqvjaW//xl3DYAmCXz9wRztWkwN8o35I6REWQn2ZPyT/SNIOpX8D", + "+sUi/lz6/6A+ZQylByfGW+ctyeEVVDVn5cUr2JwUb8HS4j2pbssITzHPQK9LTxLJSEHIBY6kCi4GGHmi", + "pciSegKKISeO9ju38OSMKniNJKknj939EYuM2GGL88P8IwAA//90yrlhlQMAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/agent/container/cfg.yaml b/agent/container/cfg.yaml index de91ea73..845d5fb1 100644 --- a/agent/container/cfg.yaml +++ b/agent/container/cfg.yaml @@ -1,6 +1,6 @@ package: api generate: - chi-server: true + gin-server: true models: true embedded-spec: true output: agent/container/api/agent.gen.go diff --git a/agent/container/openapi.yaml b/agent/container/openapi.yaml index 3f2e7d24..bcd76a59 100755 --- a/agent/container/openapi.yaml +++ b/agent/container/openapi.yaml @@ -36,6 +36,13 @@ paths: responses: '200': description: OK - + /event/azure/container: + post: + tags: + - public + summary: Post Azure Container Registry webhook events + responses: + '200': + description: OK # oapi-codegen -config ./cfg.yaml ./openapi.yaml diff --git a/agent/container/pkg/application/application.go b/agent/container/pkg/application/application.go index 5e19a0f2..136fa332 100755 --- a/agent/container/pkg/application/application.go +++ b/agent/container/pkg/application/application.go @@ -7,7 +7,7 @@ import ( "log" "net/http" - "github.com/go-chi/chi/v5" + "github.com/gin-gonic/gin" "github.com/intelops/kubviz/agent/container/pkg/clients" "github.com/intelops/kubviz/agent/container/pkg/config" "github.com/intelops/kubviz/agent/container/pkg/handler" @@ -42,17 +42,12 @@ func New() *Application { log.Fatalf("API Handler initialisation failed: %v", err) } - mux := chi.NewMux() - apiServer.BindRequest(mux) - chi.Walk(mux, func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { - fmt.Printf("[%s]: '%s' has %d middlewares.\n", method, route, len(middlewares)) - return nil - }) + r := gin.Default() + apiServer.BindRequest(r) + httpServer := &http.Server{ - // TODO: remove hardcoding - // Addr: fmt.Sprintf("0.0.0.0:%d", cfg.Port), Addr: fmt.Sprintf(":%d", 8082), - Handler: mux, + Handler: r, } return &Application{ @@ -65,8 +60,6 @@ func New() *Application { } func (app *Application) Start() { - // TODO: remove hard coding - // log.Printf("Starting server at %v", app.httpServer.Addr) log.Printf("Starting server at %v", 8082) var err error if err = app.httpServer.ListenAndServe(); err != nil && errors.Is(err, http.ErrServerClosed) { diff --git a/agent/container/pkg/application/github.go b/agent/container/pkg/application/github.go index 37a50844..595b06f6 100644 --- a/agent/container/pkg/application/github.go +++ b/agent/container/pkg/application/github.go @@ -6,7 +6,6 @@ import ( "log" "github.com/intelops/kubviz/model" - v1 "github.com/vijeyash1/go-github-container/v1" ) // GithubContainerWatch monitors and publishes container image details from a specified GitHub organization's repositories. @@ -24,7 +23,7 @@ func (app *Application) GithubContainerWatch() { } // Create a new GitHub client with the provided organization and token. - client := v1.NewGithubClient(app.GithubConfig.Org, app.GithubConfig.Token) + client := NewGithubClient(app.GithubConfig.Org, app.GithubConfig.Token) // Fetch the list of packages (repositories) from the GitHub organization. packages, err := client.FetchPackages() @@ -61,7 +60,7 @@ func (app *Application) GithubContainerWatch() { } // BuildImageDetails constructs a model.GithubImage from the given v1.Package and v1.Version. -func BuildImageDetails(pkg v1.Package, version v1.Version) model.GithubImage { +func BuildImageDetails(pkg Package, version Version) model.GithubImage { // Create and return a new GithubImage object using the provided package and version information. return model.GithubImage{ PackageId: fmt.Sprint(pkg.ID), // Convert the package ID to a string and set it as PackageId. diff --git a/agent/container/pkg/application/githubmodule.go b/agent/container/pkg/application/githubmodule.go new file mode 100644 index 00000000..b2b766b4 --- /dev/null +++ b/agent/container/pkg/application/githubmodule.go @@ -0,0 +1,130 @@ +package application + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +type Version struct { + ID int `json:"id"` + Name string `json:"name"` + HTMLURL string `json:"html_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type Package struct { + ID int `json:"id"` + Name string `json:"name"` + PackageType string `json:"package_type"` + Owner Owner `json:"owner"` + VersionCount int `json:"version_count"` + Visibility string `json:"visibility"` + URL string `json:"url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + HTMLURL string `json:"html_url"` +} +type Owner struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` +} + +type GithubApiClient struct { + Org string + Token string +} + +func NewGithubClient(org string, token string) *GithubApiClient { + return &GithubApiClient{Org: org, Token: token} +} + +func (c *GithubApiClient) FetchPackages() ([]Package, error) { + url := fmt.Sprintf("https://api.github.com/orgs/%s/packages?package_type=container", c.Org) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Authorization", "Bearer "+c.Token) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var packages []Package + err = json.Unmarshal(body, &packages) + if err != nil { + return nil, err + } + + return packages, nil +} + +func (c *GithubApiClient) FetchVersions(packageName string) ([]Version, error) { + url := fmt.Sprintf("https://api.github.com/orgs/%s/packages/container/%s/versions", c.Org, packageName) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Authorization", "Bearer "+c.Token) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + // Check if the version is not found before unmarshalling + if string(body) == `{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}` { + fmt.Println("Version not found for package:", packageName) + return nil, fmt.Errorf("Version not found for package: Please provide proper semantic version for %s image", packageName) + } + var versions []Version + err = json.Unmarshal(body, &versions) + if err != nil { + return nil, fmt.Errorf("error occured while unmarshalling the version %w", err) + } + + return versions, nil +} diff --git a/agent/container/pkg/clients/nats_client.go b/agent/container/pkg/clients/nats_client.go index ca0a07b2..f42185cc 100755 --- a/agent/container/pkg/clients/nats_client.go +++ b/agent/container/pkg/clients/nats_client.go @@ -18,12 +18,25 @@ const ( eventSubject = "CONTAINERMETRICS.git" ) +// NATSContext encapsulates the connection and context for interacting with a NATS server +// and its associated JetStream. It includes the following fields: +// - conf: The configuration used to establish the connection, including server address, tokens, etc. +// - conn: The active connection to the NATS server, allowing for basic NATS operations. +// - stream: The JetStream context, enabling more advanced stream-based operations such as publishing and subscribing to messages. +// +// NATSContext is used throughout the application to send and receive messages via NATS, and to manage streams within JetStream. type NATSContext struct { conf *config.Config conn *nats.Conn stream nats.JetStreamContext } +// NewNATSContext establishes a connection to a NATS server using the provided configuration +// and initializes a JetStream context. It checks for the existence of a specific stream +// and creates the stream if it is not found. The function returns a NATSContext object, +// which encapsulates the NATS connection and JetStream context, allowing for publishing +// and subscribing to messages within the application. An error is returned if the connection +// or stream initialization fails. func NewNATSContext(conf *config.Config) (*NATSContext, error) { fmt.Println("Waiting before connecting to NATS at:", conf.NatsAddress) time.Sleep(1 * time.Second) @@ -48,6 +61,11 @@ func NewNATSContext(conf *config.Config) (*NATSContext, error) { return ctx, nil } +// CreateStream initializes a new JetStream within the NATS server, using the configuration +// stored in the NATSContext. It returns the JetStream context, allowing for further interaction +// with the stream, such as publishing and subscribing to messages. If the stream creation fails, +// an error is returned. This method is typically called during initialization to ensure that +// the required stream is available for the application's messaging needs. func (n *NATSContext) CreateStream() (nats.JetStreamContext, error) { // Creates JetStreamContext stream, err := n.conn.JetStream() @@ -88,11 +106,15 @@ func (n *NATSContext) Close() { n.conn.Close() } +// Publish sends a given event to the JetStream within the NATS server, including the repository information in the header. +// The event is provided as a byte slice, and the target repository information is identified by the repo string. +// This method leverages the JetStream context within the NATSContext to publish the event, ensuring that it is sent with the correct headers and to the appropriate stream within the NATS server. +// The repository information in the header can be used by subscribers to filter or route the event based on its origin or destination. +// An error is returned if the publishing process fails, such as if the connection is lost or if there are issues with the JetStream. func (n *NATSContext) Publish(event []byte, repo string) error { msg := nats.NewMsg(eventSubject) msg.Data = event msg.Header.Set("REPO_NAME", repo) _, err := n.stream.PublishMsgAsync(msg) - return err } diff --git a/agent/container/pkg/handler/api_handler.go b/agent/container/pkg/handler/api_handler.go index 61ec919c..5e5e2834 100755 --- a/agent/container/pkg/handler/api_handler.go +++ b/agent/container/pkg/handler/api_handler.go @@ -1,10 +1,9 @@ package handler import ( - "encoding/json" "net/http" - "github.com/go-chi/chi/v5" + "github.com/gin-gonic/gin" "github.com/intelops/kubviz/agent/container/api" "github.com/intelops/kubviz/agent/container/pkg/clients" ) @@ -18,28 +17,46 @@ const ( contentType = "Content-Type" ) +// NewAPIHandler creates a new instance of APIHandler, which is responsible for handling +// various API endpoints related to container events. It takes a NATSContext connection +// as an argument, allowing the handler to interact with a NATS messaging system. +// The returned APIHandler can be used to bind and handle specific routes, such as +// receiving events from Docker Hub or Azure Container Registry. func NewAPIHandler(conn *clients.NATSContext) (*APIHandler, error) { return &APIHandler{ conn: conn, }, nil } -func (ah *APIHandler) BindRequest(mux *chi.Mux) { - mux.Route("/", func(r chi.Router) { - api.HandlerFromMux(ah, r) - }) +func (ah *APIHandler) BindRequest(r *gin.Engine) { + apiGroup := r.Group("/") + { + apiGroup.GET("/api-docs", ah.GetApiDocs) + apiGroup.GET("/status", ah.GetStatus) + apiGroup.POST("/event/docker/hub", ah.PostEventDockerHub) + apiGroup.POST("/event/azure/container", ah.PostEventAzureContainer) + } } -func (ah *APIHandler) GetApiDocs(w http.ResponseWriter, r *http.Request) { +// GetApiDocs serves the Swagger API documentation generated from the OpenAPI YAML file. +// It responds with a JSON representation of the API's endpoints, parameters, responses, and other details. +// This endpoint can be used by tools like Swagger UI to provide interactive documentation for the API. +func (ah *APIHandler) GetApiDocs(c *gin.Context) { swagger, err := api.GetSwagger() if err != nil { - w.WriteHeader(http.StatusInternalServerError) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return } - w.Header().Set(contentType, appJSONContentType) - _ = json.NewEncoder(w).Encode(swagger) + c.Header(contentType, appJSONContentType) + c.JSON(http.StatusOK, swagger) } -func (ah *APIHandler) GetStatus(w http.ResponseWriter, r *http.Request) { - w.Header().Set(contentType, appJSONContentType) - w.WriteHeader(http.StatusOK) +// GetStatus responds with the current status of the application. This endpoint can be used +// by monitoring tools to check the health and readiness of the application. It typically +// includes information about the application's state, dependencies, and any ongoing issues. +// In this basic implementation, it simply responds with an OK status, indicating that the +// application is running and ready to handle requests. +func (ah *APIHandler) GetStatus(c *gin.Context) { + c.Header(contentType, appJSONContentType) + c.Status(http.StatusOK) } diff --git a/agent/container/pkg/handler/azure_container.go b/agent/container/pkg/handler/azure_container.go new file mode 100644 index 00000000..106d92e0 --- /dev/null +++ b/agent/container/pkg/handler/azure_container.go @@ -0,0 +1,50 @@ +package handler + +import ( + "encoding/json" + "errors" + "io" + "log" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/intelops/kubviz/model" +) + +var ErrInvalidPayload = errors.New("invalid or malformed Azure Container Registry webhook payload") + +// PostEventAzureContainer listens for Azure Container Registry image push events. +// When a new image is pushed, this endpoint receives the event payload, validates it, +// and then publishes it to a NATS messaging system. This allows client of the +// application to subscribe to these events and respond to changes in the container registry. +// If the payload is invalid or the publishing process fails, an error response is returned. +func (ah *APIHandler) PostEventAzureContainer(c *gin.Context) { + defer func() { + _, _ = io.Copy(io.Discard, c.Request.Body) + _ = c.Request.Body.Close() + }() + payload, err := io.ReadAll(c.Request.Body) + if err != nil || len(payload) == 0 { + log.Printf("%v: %v", ErrReadingBody, err) + c.Status(http.StatusBadRequest) + return + } + + var pushEvent model.AzureContainerPushEventPayload + err = json.Unmarshal(payload, &pushEvent) + if err != nil { + log.Printf("%v: %v", ErrInvalidPayload, err) + c.JSON(http.StatusBadRequest, gin.H{"error": "Bad Request"}) + return + } + + log.Printf("Received event from Azure Container Registry: %v", pushEvent) + + err = ah.conn.Publish(payload, "Azure_Container_Registry") + if err != nil { + log.Printf("%v: %v", ErrPublishToNats, err) + c.Status(http.StatusInternalServerError) + return + } + c.Status(http.StatusOK) +} diff --git a/agent/container/pkg/handler/docker_event_dockerhub.go b/agent/container/pkg/handler/docker_event_dockerhub.go index 0d6d0e08..d022d9a7 100644 --- a/agent/container/pkg/handler/docker_event_dockerhub.go +++ b/agent/container/pkg/handler/docker_event_dockerhub.go @@ -5,6 +5,8 @@ import ( "io" "log" "net/http" + + "github.com/gin-gonic/gin" ) // parse errors @@ -13,20 +15,23 @@ var ( ErrPublishToNats = errors.New("error while publishing to nats") ) -func (ah *APIHandler) PostEventDockerHub(w http.ResponseWriter, r *http.Request) { +func (ah *APIHandler) PostEventDockerHub(c *gin.Context) { defer func() { - _, _ = io.Copy(io.Discard, r.Body) - _ = r.Body.Close() + _, _ = io.Copy(io.Discard, c.Request.Body) + _ = c.Request.Body.Close() }() - payload, err := io.ReadAll(r.Body) + payload, err := io.ReadAll(c.Request.Body) if err != nil || len(payload) == 0 { log.Printf("%v: %v", ErrReadingBody, err) + c.Status(http.StatusBadRequest) return } log.Printf("Received event from docker artifactory: %v", string(payload)) err = ah.conn.Publish(payload, "Dockerhub_Registry") if err != nil { log.Printf("%v: %v", ErrPublishToNats, err) + c.Status(http.StatusInternalServerError) return } + c.Status(http.StatusOK) } diff --git a/agent/git/info.txt b/agent/git/info.txt index 82c33c9c..52fb123e 100644 --- a/agent/git/info.txt +++ b/agent/git/info.txt @@ -1 +1,3 @@ for agent : oapi-codegen -config ./agent/container/cfg.yaml ./agent/container/openapi.yaml + +for agent : oapi-codegen -config ./cfg.yaml ./openapi.yaml diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index 26d6dc73..e6205c5f 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -39,6 +39,7 @@ type DBInterface interface { RetriveKubepugEvent() ([]model.Result, error) RetrieveKubvizEvent() ([]model.DbEvent, error) InsertContainerEventDockerHub(model.DockerHubBuild) + InsertContainerEventAzure(model.AzureContainerPushEventPayload) InsertContainerEventGithub(string) InsertGitCommon(metrics model.GitCommonAttribute, statement dbstatement.DBStatement) error Close() @@ -70,7 +71,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { return nil, err } - tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} for _, table := range tables { if err = splconn.Exec(context.Background(), string(table)); err != nil { return nil, err @@ -89,6 +90,46 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { } return &DBClient{splconn: splconn, conn: stdconn, conf: conf}, nil } +func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushEventPayload) { + var ( + tx, _ = c.conn.Begin() + stmt, _ = tx.Prepare(string(InsertAzureContainerPushEvent)) + ) + defer stmt.Close() + registryURL := pushEvent.Request.Host + repositoryName := pushEvent.Target.Repository + tag := pushEvent.Target.Tag + + if tag == "" { + tag = "latest" + } + imageName := registryURL + "/" + repositoryName + ":" + tag + size := pushEvent.Target.Size + shaID := pushEvent.Target.Digest + + // Marshaling the pushEvent into a JSON string + pushEventJSON, err := json.Marshal(pushEvent) + if err != nil { + log.Printf("Error while marshaling Azure Container Registry payload: %v", err) + return + } + + if _, err := stmt.Exec( + registryURL, + repositoryName, + tag, + imageName, + string(pushEventJSON), + pushEvent.Timestamp, + size, + shaID, + ); err != nil { + log.Fatal(err) + } + if err := tx.Commit(); err != nil { + log.Fatal(err) + } +} func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) { var ( diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index 12ee7754..65b8629a 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -145,6 +145,18 @@ const dockerHubBuildTable DBStatement = ` Event String ) engine=File(TabSeparated) ` +const azureContainerPushEventTable DBStatement = ` + CREATE TABLE IF NOT EXISTS azurecontainerpush ( + RegistryURL String, + RepositoryName String, + Tag String, + ImageName String, + Event String, + Timestamp String, + Size UInt64, + SHAID String + ) engine=File(TabSeparated) + ` const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)" const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)" @@ -160,3 +172,4 @@ const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_na const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)" const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)" +const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" diff --git a/client/pkg/clients/container_client.go b/client/pkg/clients/container_client.go index 413e2080..3688a102 100644 --- a/client/pkg/clients/container_client.go +++ b/client/pkg/clients/container_client.go @@ -49,6 +49,17 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) { } else if repoName == "Github_Registry" { conn.InsertContainerEventGithub(string(msg.Data)) log.Println("Inserted Github Container metrics:", string(msg.Data)) + } else if repoName == "Azure_Container_Registry" { + var pushEvent model.AzureContainerPushEventPayload + err := json.Unmarshal(msg.Data, &pushEvent) + if err != nil { + log.Printf("Error while unmarshaling Azure Container Registry payload: %v", err) + return + } + // Extract the necessary information from pushEvent and insert into ClickHouse + conn.InsertContainerEventAzure(pushEvent) + log.Println("Inserted Azure Container Registry metrics:", string(msg.Data)) } + }, nats.Durable(string(containerConsumer)), nats.ManualAck()) } diff --git a/docker-compose.yaml b/docker-compose.yaml index a384d7b3..d3271f13 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,7 +11,7 @@ services: - ch_ntw ch_server: - image: clickhouse/clickhouse-server:22.6 + image: clickhouse/clickhouse-server:23.6.2.18 ports: - '8123:8123' - '9000:9000' diff --git a/go.mod b/go.mod index 63450653..8188081c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/getkin/kin-openapi v0.118.0 github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-chi/chi/v5 v5.0.8 github.com/go-co-op/gocron v1.30.1 github.com/go-playground/webhooks/v6 v6.2.0 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index f4525b85..babc6d30 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,6 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= -github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-co-op/gocron v1.30.1 h1:tjWUvJl5KrcwpkEkSXFSQFr4F9h5SfV/m4+RX0cV2fs= github.com/go-co-op/gocron v1.30.1/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= diff --git a/model/azurecontainer.go b/model/azurecontainer.go new file mode 100644 index 00000000..2070412d --- /dev/null +++ b/model/azurecontainer.go @@ -0,0 +1,23 @@ +package model + +import "time" + +type AzureContainerPushEventPayload struct { + ID string `json:"id"` + Timestamp time.Time `json:"timestamp"` + Action string `json:"action"` + Target struct { + MediaType string `json:"mediaType"` + Size int32 `json:"size"` + Digest string `json:"digest"` + Length int32 `json:"length"` // Same as Size field + Repository string `json:"repository"` + Tag string `json:"tag"` + } `json:"target"` + Request struct { + ID string `json:"id"` + Host string `json:"host"` + Method string `json:"method"` + UserAgent string `json:"useragent"` + } `json:"request"` +} diff --git a/model/dockerhub.go b/model/dockercontainerevents.go similarity index 100% rename from model/dockerhub.go rename to model/dockercontainerevents.go