main update

Author: iamayushm

api/GrpcHandler.go

@@ -564,14 +564,22 @@ func (impl *GrpcHandlerImpl) GetWebhookData(ctx context.Context, req *pb.Webhook
 func (impl *GrpcHandlerImpl) GetAllWebhookEventConfigForHost(ctx context.Context, req *pb.WebhookEventConfigRequest) (
 	*pb.WebhookEventConfigResponse, error) {
 
-	res, err := impl.repositoryManager.GetAllWebhookEventConfigForHost(int(req.GitHostId))
+	var res []*git.WebhookEventConfig
+	var err error
+	reqModel := &git.WebhookEventConfigRequest{
+		GitHostId:   int(req.GitHostId),
+		GitHostName: req.GitHostName,
+		EventId:     int(req.EventId),
+	}
+	res, err = impl.repositoryManager.GetAllWebhookEventConfigForHost(reqModel)
 	if err != nil {
 		impl.logger.Errorw("error while fetching webhook event config",
 			"gitHostId", req.GitHostId,
 			"err", err)
 
 		return nil, err
 	}
+
 	webhookConfig := &pb.WebhookEventConfigResponse{}
 	if res == nil {
 		return webhookConfig, nil

api/RestHandler.go

@@ -429,7 +429,9 @@ func (handler RestHandlerImpl) GetAllWebhookEventConfigForHost(w http.ResponseWr
 	}
 	handler.logger.Infow("webhook event config request ", "req", request)
 
-	webhookEventConfigArr, err := handler.repositoryManager.GetAllWebhookEventConfigForHost(request.GitHostId)
+	var webhookEventConfigArr []*git.WebhookEventConfig
+	webhookEventConfigArr, err = handler.repositoryManager.GetAllWebhookEventConfigForHost(request)
+
 	if err != nil {
 		handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
 	} else {

go.mod

@@ -6,9 +6,9 @@ toolchain go1.22.4
 
 require (
 	github.com/caarlos0/env v3.5.0+incompatible
-	github.com/devtron-labs/common-lib v0.0.25-0.20240719070325-2cb5e69dc592
-	github.com/devtron-labs/protos v0.0.3-0.20240719073841-1c0c5d2079c6
-	github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
+	github.com/devtron-labs/common-lib v0.0.25-0.20240731080454-282e8ce8a2c5
+	github.com/devtron-labs/protos v0.0.3-0.20240726131238-dff2574e623d
+	github.com/gammazero/workerpool v1.1.3
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-pg/pg v6.15.1+incompatible
 	github.com/google/wire v0.6.0
@@ -39,7 +39,7 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8 // indirect
+	github.com/gammazero/deque v0.2.0 // indirect
 	github.com/gliderlabs/ssh v0.3.6 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect

go.sum

@@ -29,19 +29,18 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devtron-labs/common-lib v0.0.25-0.20240719070325-2cb5e69dc592 h1:b74H3z6+uWAJnmEZ34YoB/bZ3fdTxGxY88W3xF5WWO0=
-github.com/devtron-labs/common-lib v0.0.25-0.20240719070325-2cb5e69dc592/go.mod h1:UZGPt1ep9Tnd9Ak2sibGSiLr7p3ijO2/JLT+h+pqBuU=
-github.com/devtron-labs/protos v0.0.3-0.20240719073841-1c0c5d2079c6 h1:PC2j0/purK0ICQEQV0bBweap1BW6tFVvYJFy7QOvXNc=
-github.com/devtron-labs/protos v0.0.3-0.20240719073841-1c0c5d2079c6/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
+github.com/devtron-labs/common-lib v0.0.25-0.20240731080454-282e8ce8a2c5 h1:R0T3f9/xk6P6V1ncbAaY2EjRkyjl01wslfbOSnBC18Q=
+github.com/devtron-labs/common-lib v0.0.25-0.20240731080454-282e8ce8a2c5/go.mod h1:3GN9TABx4D+hVuF69vGYUUx+H8/WelcKw0lUt8aELok=
+github.com/devtron-labs/protos v0.0.3-0.20240726131238-dff2574e623d h1:uZk/CLrgRdPvb9/Eaio6+AkrDGdQaibNmD2DSwJZYEg=
+github.com/devtron-labs/protos v0.0.3-0.20240726131238-dff2574e623d/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/gammazero/deque v0.0.0-20190521012701-46e4ffb7a622/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
-github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8 h1:9eg1l6iiw3aLaU3P8RRoosiYCRoWDg8HzPiy8xzWKgg=
-github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
-github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab h1:BLCWHqLiTSzLZriw46b3HJLswyqY0E6B9y9tk4+7eFI=
-github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab/go.mod h1:ZObaTlXZGgqKXhhlk+zNvSOXT+h6VGThA0ZQxLqn8x0=
+github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA=
+github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
+github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q=
+github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc=
 github.com/gliderlabs/ssh v0.3.6 h1:ZzjlDa05TcFRICb3anf/dSPN3ewz1Zx6CMLPWgkm3b8=
 github.com/gliderlabs/ssh v0.3.6/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -163,8 +162,9 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=

internals/sql/WebhookEventRepository.go

@@ -26,6 +26,7 @@ type GitHostWebhookEvent struct {
 	tableName     struct{}  `sql:"git_host_webhook_event" pg:",discard_unknown_columns"`
 	Id            int       `sql:"id,pk"`
 	GitHostId     int       `sql:"git_host_id,notnull"`
+	GitHostName   string    `sql:"git_host_name,notnull"`
 	Name          string    `sql:"name,notnull"`
 	EventTypesCsv string    `sql:"event_types_csv,notnull"`
 	ActionType    string    `sql:"action_type,notnull"`
@@ -54,7 +55,9 @@ type GitHostWebhookEventSelectors struct {
 
 type WebhookEventRepository interface {
 	GetAllGitHostWebhookEventByGitHostId(gitHostId int) ([]*GitHostWebhookEvent, error)
+	GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*GitHostWebhookEvent, error)
 	GetWebhookEventConfigByEventId(eventId int) (*GitHostWebhookEvent, error)
+	Update(webhookEvent *GitHostWebhookEvent) error
 }
 
 type WebhookEventRepositoryImpl struct {
@@ -65,7 +68,7 @@ func NewWebhookEventRepositoryImpl(dbConnection *pg.DB) *WebhookEventRepositoryI
 	return &WebhookEventRepositoryImpl{dbConnection: dbConnection}
 }
 
-func (impl WebhookEventRepositoryImpl) GetAllGitHostWebhookEventByGitHostId(gitHostId int) ([]*GitHostWebhookEvent, error) {
+func (impl *WebhookEventRepositoryImpl) GetAllGitHostWebhookEventByGitHostId(gitHostId int) ([]*GitHostWebhookEvent, error) {
 	var gitHostWebhookEvents []*GitHostWebhookEvent
 	err := impl.dbConnection.Model(&gitHostWebhookEvents).
 		Column("git_host_webhook_event.*").
@@ -77,8 +80,20 @@ func (impl WebhookEventRepositoryImpl) GetAllGitHostWebhookEventByGitHostId(gitH
 		Select()
 	return gitHostWebhookEvents, err
 }
+func (impl *WebhookEventRepositoryImpl) GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*GitHostWebhookEvent, error) {
+	var gitHostWebhookEvents []*GitHostWebhookEvent
+	err := impl.dbConnection.Model(&gitHostWebhookEvents).
+		Column("git_host_webhook_event.*").
+		Relation("Selectors", func(q *orm.Query) (query *orm.Query, err error) {
+			return q.Where("is_active IS TRUE"), nil
+		}).
+		Where("git_host_name =? ", gitHostName).
+		Where("is_active = TRUE ").
+		Select()
+	return gitHostWebhookEvents, err
+}
 
-func (impl WebhookEventRepositoryImpl) GetWebhookEventConfigByEventId(eventId int) (*GitHostWebhookEvent, error) {
+func (impl *WebhookEventRepositoryImpl) GetWebhookEventConfigByEventId(eventId int) (*GitHostWebhookEvent, error) {
 	gitHostWebhookEvent := &GitHostWebhookEvent{}
 	err := impl.dbConnection.Model(gitHostWebhookEvent).
 		Column("git_host_webhook_event.*").
@@ -91,3 +106,11 @@ func (impl WebhookEventRepositoryImpl) GetWebhookEventConfigByEventId(eventId in
 
 	return gitHostWebhookEvent, err
 }
+
+func (impl *WebhookEventRepositoryImpl) Update(webhookEvent *GitHostWebhookEvent) error {
+	err := impl.dbConnection.Update(webhookEvent)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/RepoManages.go

@@ -47,7 +47,7 @@ type RepoManager interface {
 	RefreshGitMaterial(req *git.RefreshGitMaterialRequest) (*git.RefreshGitMaterialResponse, error)
 
 	GetWebhookAndCiDataById(id int, ciPipelineMaterialId int) (*git.WebhookAndCiData, error)
-	GetAllWebhookEventConfigForHost(gitHostId int) ([]*git.WebhookEventConfig, error)
+	GetAllWebhookEventConfigForHost(req *git.WebhookEventConfigRequest) ([]*git.WebhookEventConfig, error)
 	GetWebhookEventConfig(eventId int) (*git.WebhookEventConfig, error)
 	GetWebhookPayloadDataForPipelineMaterialId(request *git.WebhookPayloadDataRequest) (*git.WebhookPayloadDataResponse, error)
 	GetWebhookPayloadFilterDataForPipelineMaterialId(request *git.WebhookPayloadFilterDataRequest) (*git.WebhookPayloadFilterDataResponse, error)
@@ -860,18 +860,39 @@ func (impl RepoManagerImpl) GetWebhookAndCiDataById(id int, ciPipelineMaterialId
 	return webhookAndCiData, nil
 }
 
-func (impl RepoManagerImpl) GetAllWebhookEventConfigForHost(gitHostId int) ([]*git.WebhookEventConfig, error) {
-
+func (impl RepoManagerImpl) GetAllWebhookEventConfigForHost(req *git.WebhookEventConfigRequest) ([]*git.WebhookEventConfig, error) {
+	gitHostId := req.GitHostId
+	gitHostName := req.GitHostName
+	var webhookEventsFromDb []*sql.GitHostWebhookEvent
+	var err error
 	impl.logger.Debugw("Getting All webhook event config ", "gitHostId", gitHostId)
-
-	webhookEventsFromDb, err := impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostId(gitHostId)
-
-	if err != nil {
-		impl.logger.Errorw("error in getting webhook events", "gitHostId", gitHostId, "err", err)
-		return nil, err
+	if gitHostName != "" {
+		webhookEventsFromDb, err = impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostName(gitHostName)
+		if err != nil {
+			impl.logger.Errorw("error in getting webhook events", "gitHostName", gitHostName, "err", err)
+			return nil, err
+		}
+		if webhookEventsFromDb == nil || len(webhookEventsFromDb) == 0 {
+			webhookEventsFromDb, err = impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostId(gitHostId)
+			if err != nil {
+				impl.logger.Errorw("error in getting webhook events", "gitHostId", gitHostId, "err", err)
+				return nil, err
+			}
+		}
+	} else {
+		webhookEventsFromDb, err = impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostId(gitHostId)
+		if err != nil {
+			impl.logger.Errorw("error in getting webhook events", "gitHostId", gitHostId, "err", err)
+			return nil, err
+		}
 	}
 
 	// build events
+	return impl.convertSqlBeansToWebhookEventConfig(webhookEventsFromDb)
+
+}
+
+func (impl RepoManagerImpl) convertSqlBeansToWebhookEventConfig(webhookEventsFromDb []*sql.GitHostWebhookEvent) ([]*git.WebhookEventConfig, error) {
 	var webhookEvents []*git.WebhookEventConfig
 	for _, webhookEventFromDb := range webhookEventsFromDb {
 		webhookEvent := impl.webhookEventBeanConverter.ConvertFromWebhookEventSqlBean(webhookEventFromDb)

pkg/git/Bean.go

@@ -210,8 +210,9 @@ type WebhookDataRequest struct {
 }
 
 type WebhookEventConfigRequest struct {
-	GitHostId int `json:"gitHostId"`
-	EventId   int `json:"eventId"`
+	GitHostId   int    `json:"gitHostId"`
+	EventId     int    `json:"eventId"`
+	GitHostName string `json:"GitHostName"`
 }
 
 type RefreshGitMaterialRequest struct {
@@ -227,6 +228,7 @@ type RefreshGitMaterialResponse struct {
 type WebhookEvent struct {
 	PayloadId          int    `json:"payloadId"`
 	RequestPayloadJson string `json:"requestPayloadJson"`
+	GitHostName        string `json:"gitHostName"`
 	GitHostId          int    `json:"gitHostId"`
 	EventType          string `json:"eventType"`
 }

pkg/git/WebhookEventService.go

@@ -30,7 +30,8 @@ import (
 )
 
 type WebhookEventService interface {
-	GetAllGitHostWebhookEventByGitHostId(gitHostId int) ([]*sql.GitHostWebhookEvent, error)
+	GetAllGitHostWebhookEventByGitHostId(gitHostId int, gitHostName string) ([]*sql.GitHostWebhookEvent, error)
+	GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*sql.GitHostWebhookEvent, error)
 	GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error)
 	SaveWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error
 	UpdateWebhookParsedEventData(webhookEventParsedData *sql.WebhookEventParsedData) error
@@ -65,9 +66,30 @@ func NewWebhookEventServiceImpl(
 	}
 }
 
-func (impl WebhookEventServiceImpl) GetAllGitHostWebhookEventByGitHostId(gitHostId int) ([]*sql.GitHostWebhookEvent, error) {
+func (impl WebhookEventServiceImpl) GetAllGitHostWebhookEventByGitHostId(gitHostId int, gitHostName string) ([]*sql.GitHostWebhookEvent, error) {
 	impl.logger.Debugw("Getting All git host events", "gitHostId", gitHostId)
-	return impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostId(gitHostId)
+	events, err := impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostId(gitHostId)
+	if err != nil {
+		impl.logger.Errorw("Error in getting all webhook events", "err", err)
+		return nil, err
+	}
+	if events != nil && len(events) > 0 && gitHostName != "" {
+		//handle older events to update git_host_name
+		for _, event := range events {
+			if event.GitHostName == "" {
+				event.GitHostName = gitHostName
+				err = impl.webhookEventRepository.Update(event)
+				if err != nil {
+					impl.logger.Errorw("Error in updating webhook event with git_host_name", "git_host_name", gitHostName, "err", err)
+				}
+			}
+		}
+	}
+	return events, nil
+}
+func (impl WebhookEventServiceImpl) GetAllGitHostWebhookEventByGitHostName(gitHostName string) ([]*sql.GitHostWebhookEvent, error) {
+	impl.logger.Debugw("Getting All git host events", "gitHostName", gitHostName)
+	return impl.webhookEventRepository.GetAllGitHostWebhookEventByGitHostName(gitHostName)
 }
 
 func (impl WebhookEventServiceImpl) GetWebhookParsedEventDataByEventIdAndUniqueId(eventId int, uniqueId string) (*sql.WebhookEventParsedData, error) {

pkg/git/WebhookHandler.go

@@ -17,6 +17,7 @@
 package git
 
 import (
+	"github.com/devtron-labs/git-sensor/internals/sql"
 	"go.uber.org/zap"
 	"strings"
 	"time"
@@ -44,22 +45,36 @@ func (impl WebhookHandlerImpl) HandleWebhookEvent(webhookEvent *WebhookEvent) er
 	impl.logger.Debug("Webhook event came")
 
 	gitHostId := webhookEvent.GitHostId
+	gitHostName := webhookEvent.GitHostName
 	eventType := webhookEvent.EventType
 	payloadJson := webhookEvent.RequestPayloadJson
 	payloadId := webhookEvent.PayloadId
 
 	impl.logger.Debugw("webhook event request data", "gitHostId", gitHostId, "eventType", eventType)
 
-	// get all configured events from database for given git host Id
-	events, err := impl.webhookEventService.GetAllGitHostWebhookEventByGitHostId(gitHostId)
-	if err != nil {
-		impl.logger.Errorw("error in getting webhook events from db", "err", err, "gitHostId", gitHostId)
-		return err
+	var events []*sql.GitHostWebhookEvent
+	var err error
+	if gitHostName != "" {
+		// get all configured events from database for given git host Name
+		events, err = impl.webhookEventService.GetAllGitHostWebhookEventByGitHostName(gitHostName)
+		if err != nil {
+			impl.logger.Errorw("error in getting webhook events from db", "err", err, "gitHostId", gitHostId)
+			return err
+		}
 	}
 
 	if len(events) == 0 {
-		impl.logger.Warnw("webhook events not found for given gitHostId ", "gitHostId", gitHostId)
-		return nil
+		// retry with gitHostId and get all configured events from database for given git host Id
+		events, err = impl.webhookEventService.GetAllGitHostWebhookEventByGitHostId(gitHostId, gitHostName)
+		if err != nil {
+			impl.logger.Errorw("error in getting webhook events from db", "err", err, "gitHostId", gitHostId)
+			return err
+		}
+		if len(events) == 0 {
+			impl.logger.Warnw("webhook events not found for given gitHostId ", "gitHostId", gitHostId)
+			return nil
+		}
+
 	}
 
 	// operate for all matching event (match for eventType)

scripts/sql/12_gitlab_webhook.down.sql

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+DELETE FROM git_host_webhook_event_selectors where event_id in (select id from git_host_webhook_event where git_host_name='Gitlab_Devtron' and name in 'Pull Request', 'Tag Creation')
+DELETE FROM git_host_webhook_event where git_host_name='Gitlab_Devtron';
+ALTER TABLE git_host_webhook_event  DROP COLUMN git_host_name;
+ALTER TABLE git_host_webhook_event ALTER COLUMN git_host_id SET NOT NULL;