Skip to content

Commit 5ff5798

Browse files
prkhrkatprkhrkat
authored
Merge branch 'main' into cli-output-propagate
2 parents 2a99117 + 4b06414 commit 5ff5798

File tree

230 files changed

+32575
-8563
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

230 files changed

+32575
-8563
lines changed

‎Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.20-alpine3.17 AS build-env
1+
FROM golang:1.21-alpine3.17 AS build-env
22

33
RUN apk add --no-cache git gcc musl-dev
44
RUN apk add --update make

‎api/GrpcHandler.go

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,19 @@ func (impl *GrpcHandlerImpl) SaveGitProvider(ctx context.Context, req *pb.GitPro
7373

7474
// mapping req
7575
gitProvider := &sql.GitProvider{
76-
Id: int(req.Id),
77-
Name: req.Name,
78-
Url: req.Url,
79-
UserName: req.UserName,
80-
Password: req.Password,
81-
AccessToken: req.AccessToken,
82-
SshPrivateKey: req.SshPrivateKey,
83-
AuthMode: sql.AuthMode(req.AuthMode),
84-
Active: req.Active,
76+
Id: int(req.Id),
77+
Name: req.Name,
78+
Url: req.Url,
79+
UserName: req.UserName,
80+
Password: req.Password,
81+
AccessToken: req.AccessToken,
82+
SshPrivateKey: req.SshPrivateKey,
83+
AuthMode: sql.AuthMode(req.AuthMode),
84+
Active: req.Active,
85+
CaCert: req.CaCert,
86+
TlsCert: req.TlsCert,
87+
TlsKey: req.TlsKey,
88+
EnableTLSVerification: req.EnableTLSVerification,
8589
}
8690

8791
_, err := impl.repositoryManager.SaveGitProvider(gitProvider)
@@ -464,8 +468,8 @@ func (impl *GrpcHandlerImpl) RefreshGitMaterial(ctx context.Context, req *pb.Ref
464468

465469
func (impl *GrpcHandlerImpl) ReloadAllMaterial(ctx context.Context, req *pb.Empty) (*pb.Empty, error) {
466470
gitCtx := git.BuildGitContext(ctx)
467-
impl.repositoryManager.ReloadAllRepo(gitCtx)
468-
return &pb.Empty{}, nil
471+
err := impl.repositoryManager.ReloadAllRepo(gitCtx, nil)
472+
return &pb.Empty{}, err
469473
}
470474

471475
func (impl *GrpcHandlerImpl) ReloadMaterial(ctx context.Context, req *pb.ReloadMaterialRequest) (
@@ -560,14 +564,22 @@ func (impl *GrpcHandlerImpl) GetWebhookData(ctx context.Context, req *pb.Webhook
560564
func (impl *GrpcHandlerImpl) GetAllWebhookEventConfigForHost(ctx context.Context, req *pb.WebhookEventConfigRequest) (
561565
*pb.WebhookEventConfigResponse, error) {
562566

563-
res, err := impl.repositoryManager.GetAllWebhookEventConfigForHost(int(req.GitHostId))
567+
var res []*git.WebhookEventConfig
568+
var err error
569+
reqModel := &git.WebhookEventConfigRequest{
570+
GitHostId: int(req.GitHostId),
571+
GitHostName: req.GitHostName,
572+
EventId: int(req.EventId),
573+
}
574+
res, err = impl.repositoryManager.GetAllWebhookEventConfigForHost(reqModel)
564575
if err != nil {
565576
impl.logger.Errorw("error while fetching webhook event config",
566577
"gitHostId", req.GitHostId,
567578
"err", err)
568579

569580
return nil, err
570581
}
582+
571583
webhookConfig := &pb.WebhookEventConfigResponse{}
572584
if res == nil {
573585
return webhookConfig, nil

‎api/RestHandler.go

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ package api
1818

1919
import (
2020
"encoding/json"
21+
"fmt"
22+
"github.com/devtron-labs/git-sensor/bean"
2123
"github.com/devtron-labs/git-sensor/internals/sql"
2224
"github.com/devtron-labs/git-sensor/pkg"
2325
"github.com/devtron-labs/git-sensor/pkg/git"
2426
"github.com/gorilla/mux"
27+
"github.com/gorilla/schema"
2528
"go.uber.org/zap"
2629
"net/http"
2730
"strconv"
@@ -179,10 +182,30 @@ func (handler RestHandlerImpl) SavePipelineMaterial(w http.ResponseWriter, r *ht
179182
}
180183

181184
func (handler RestHandlerImpl) ReloadAllMaterial(w http.ResponseWriter, r *http.Request) {
182-
handler.logger.Infow("reload all pipelineMaterial request")
185+
handler.logger.Infow(bean.GetReloadAllLog("received reload all pipelineMaterial request"))
186+
decoder := schema.NewDecoder()
187+
decoder.IgnoreUnknownKeys(true)
188+
req := &bean.ReloadAllMaterialQuery{}
189+
err := decoder.Decode(req, r.URL.Query())
190+
if err != nil {
191+
handler.logger.Errorw(bean.GetReloadAllLog("invalid query params, ReloadAllMaterial"), "err", err, "query", r.URL.Query())
192+
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
193+
return
194+
}
195+
if req.Start < 0 || req.End < 0 || req.Start > req.End {
196+
handler.logger.Errorw(bean.GetReloadAllLog("invalid request, ReloadAllMaterial"), "query", r.URL.Query())
197+
handler.writeJsonResp(w, fmt.Errorf("invalid query params"), nil, http.StatusBadRequest)
198+
return
199+
}
183200
gitCtx := git.BuildGitContext(r.Context())
184-
handler.repositoryManager.ReloadAllRepo(gitCtx)
185-
handler.writeJsonResp(w, nil, "reloaded se logs for detail", http.StatusOK)
201+
err = handler.repositoryManager.ReloadAllRepo(gitCtx, req)
202+
if err != nil {
203+
handler.logger.Errorw(bean.GetReloadAllLog("error in request, ReloadAllMaterial"), "query", r.URL.Query(), "err", err)
204+
handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
205+
return
206+
}
207+
handler.logger.Infow(bean.GetReloadAllLog("Reloaded materials successfully!"), "query", r.URL.Query())
208+
handler.writeJsonResp(w, nil, "Reloaded materials successfully!", http.StatusOK)
186209
}
187210

188211
func (handler RestHandlerImpl) ReloadMaterials(w http.ResponseWriter, r *http.Request) {
@@ -406,7 +429,9 @@ func (handler RestHandlerImpl) GetAllWebhookEventConfigForHost(w http.ResponseWr
406429
}
407430
handler.logger.Infow("webhook event config request ", "req", request)
408431

409-
webhookEventConfigArr, err := handler.repositoryManager.GetAllWebhookEventConfigForHost(request.GitHostId)
432+
var webhookEventConfigArr []*git.WebhookEventConfig
433+
webhookEventConfigArr, err = handler.repositoryManager.GetAllWebhookEventConfigForHost(request)
434+
410435
if err != nil {
411436
handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
412437
} else {

‎bean/bean.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,20 @@
1616

1717
package bean
1818

19+
import "fmt"
20+
1921
type StartupConfig struct {
2022
RestPort int `env:"SERVER_REST_PORT" envDefault:"8080"`
2123
GrpcPort int `env:"SERVER_GRPC_PORT" envDefault:"8081"`
2224
}
25+
26+
const ReloadAllLogPrefix = "RELOAD_ALL_LOG"
27+
28+
func GetReloadAllLog(message string) string {
29+
return fmt.Sprintf("%s: %s", ReloadAllLogPrefix, message)
30+
}
31+
32+
type ReloadAllMaterialQuery struct {
33+
Start int `json:"start"`
34+
End int `json:"end"`
35+
}

‎config.md

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
# GITSENSOR CONFIGMAP
22

3-
| Key | Value | Description |
4-
|-----------------------------------|------------------------------------|-------------------------------|
5-
| PG_ADDR | postgresql-postgresql.devtroncd | PostgreSQL Server Address |
6-
| PG_USER | postgres | PostgreSQL User |
7-
| PG_DATABASE | git_sensor | PostgreSQL Database Name |
8-
| SENTRY_ENV | prod | Sentry Environment |
9-
| SENTRY_ENABLED | "false" | Sentry Enabled (boolean) |
10-
| POLL_DURATION | "1" | Polling Duration (in seconds) |
11-
| POLL_WORKER | "2" | Number of Polling Workers |
12-
| PG_LOG_QUERY | "false" | PostgreSQL Query Logging (boolean) |
13-
| COMMIT_STATS_TIMEOUT_IN_SEC | "2" | Commit Stats Timeout (in seconds) |
14-
| ENABLE_FILE_STATS | "false" | Enable File Stats (boolean) |
15-
| GIT_HISTORY_COUNT | "15" | Git History Count |
16-
| CLONING_MODE | FULL | Cloning Mode (Possible values: SHALLOW, FULL) |
3+
| Key | Value | Description |
4+
|-----------------------------|---------------------------------|---------------------------------------------------------------------|
5+
| PG_ADDR | postgresql-postgresql.devtroncd | PostgreSQL Server Address |
6+
| PG_USER | postgres | PostgreSQL User |
7+
| PG_DATABASE | git_sensor | PostgreSQL Database Name |
8+
| SENTRY_ENV | prod | Sentry Environment |
9+
| SENTRY_ENABLED | "false" | Sentry Enabled (boolean) |
10+
| POLL_DURATION | "1" | Polling Duration (in seconds) |
11+
| POLL_WORKER | "2" | Number of Polling Workers |
12+
| PG_LOG_QUERY | "false" | PostgreSQL Query Logging (boolean) |
13+
| COMMIT_STATS_TIMEOUT_IN_SEC | "2" | Commit Stats Timeout (in seconds) |
14+
| ENABLE_FILE_STATS | "false" | Enable File Stats (boolean) |
15+
| GIT_HISTORY_COUNT | "15" | Git History Count |
16+
| CLONING_MODE | FULL | Cloning Mode (Possible values: SHALLOW, FULL) |
17+
| USE_GIT_CLI | "false" | Use git cli commands directly for all git operations |
18+
| USE_GIT_CLI_ANALYTICS | "false" | Use git cli commands directly for getting commit data for analytics |

‎go.mod

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,28 @@
11
module github.com/devtron-labs/git-sensor
22

3-
go 1.20
3+
go 1.21
4+
5+
toolchain go1.22.4
46

57
require (
68
github.com/caarlos0/env v3.5.0+incompatible
7-
github.com/devtron-labs/common-lib v0.0.22-0.20240705073412-32e32c499160
8-
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1
9-
github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
9+
github.com/devtron-labs/common-lib v0.0.25-0.20240809073131-5cefb0e8a93a
10+
github.com/devtron-labs/protos v0.0.3-0.20240809072909-83171af34169
11+
github.com/gammazero/workerpool v1.1.3
1012
github.com/go-git/go-git/v5 v5.11.0
1113
github.com/go-pg/pg v6.15.1+incompatible
1214
github.com/google/wire v0.6.0
1315
github.com/gorilla/handlers v1.4.2
1416
github.com/gorilla/mux v1.8.0
17+
github.com/gorilla/schema v1.4.1
1518
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
1619
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
1720
github.com/prometheus/client_golang v1.16.0
1821
github.com/robfig/cron/v3 v3.0.1
1922
github.com/stretchr/testify v1.8.4
2023
github.com/tidwall/gjson v1.9.3
2124
go.uber.org/zap v1.21.0
22-
golang.org/x/sys v0.18.0
25+
golang.org/x/sys v0.22.0
2326
google.golang.org/grpc v1.58.3
2427
google.golang.org/protobuf v1.33.0
2528
gopkg.in/src-d/go-billy.v4 v4.3.2
@@ -36,7 +39,7 @@ require (
3639
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
3740
github.com/davecgh/go-spew v1.1.1 // indirect
3841
github.com/emirpasic/gods v1.18.1 // indirect
39-
github.com/gammazero/deque v0.0.0-20200124200322-7e84b94275b8 // indirect
42+
github.com/gammazero/deque v0.2.0 // indirect
4043
github.com/gliderlabs/ssh v0.3.6 // indirect
4144
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
4245
github.com/go-git/go-billy/v5 v5.5.0 // indirect
@@ -68,11 +71,12 @@ require (
6871
github.com/xanzy/ssh-agent v0.3.3 // indirect
6972
go.uber.org/atomic v1.10.0 // indirect
7073
go.uber.org/multierr v1.6.0 // indirect
71-
golang.org/x/crypto v0.21.0 // indirect
72-
golang.org/x/mod v0.14.0 // indirect
73-
golang.org/x/net v0.23.0 // indirect
74-
golang.org/x/text v0.14.0 // indirect
75-
golang.org/x/tools v0.17.0 // indirect
74+
golang.org/x/crypto v0.25.0 // indirect
75+
golang.org/x/mod v0.17.0 // indirect
76+
golang.org/x/net v0.27.0 // indirect
77+
golang.org/x/sync v0.7.0 // indirect
78+
golang.org/x/text v0.16.0 // indirect
79+
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
7680
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
7781
gopkg.in/fsnotify.v1 v1.4.7 // indirect
7882
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect

0 commit comments

Comments
 (0)