Initial commit (#1)

* Initial commit

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [shaharia-lab]

.github/linters/.golangci.yml

@@ -0,0 +1,16 @@
+---
+# See https://golangci-lint.run/usage/configuration/#config-file for more information
+run:
+  timeout: 5m
+linters:
+  disable-all: true
+  enable:
+    - gofmt
+    - revive
+    - goimports
+  fast: false
+linters-settings:
+  gofmt:
+    simplify: false
+issues:
+  exclude-use-default: false

.github/linters/.jscpd.json

@@ -0,0 +1,7 @@
+{
+  "threshold": 5,
+  "ignore": [
+    "**/*_test.go",
+    "**/*_mock.go"
+  ]
+}

.github/workflows/CI.yaml

@@ -0,0 +1,75 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  lint:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: write  # for golangci/golangci-lint-action to fetch pull requests
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.20'
+          check-latest: true
+          cache: true
+
+      - name: Lint Code Base
+        uses: github/super-linter/slim@v4
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          DEFAULT_BRANCH: master
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Go ^1.20
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.20'
+
+      - name: Test
+        run: go test ./... -covermode=atomic -coverprofile=coverage.out
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+
+      - name: Publish artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage.out
+          path: coverage.out
+
+  sonarcloud-scan:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: coverage.out
+
+      - name: SonarCloud Scan
+        uses: sonarsource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -0,0 +1,6 @@
+/.idea
+
+/out
+/vendor
+/coverage_*.txt
+/coverage.out

Makefile

@@ -0,0 +1,58 @@
+# Config for the binaries you want to build
+NAME=coco
+REPO=github.com/shaharia-lab/${NAME}
+VERSION ?= "dev-$(shell git rev-parse HEAD --short)"
+
+BINARY=$(NAME)
+BINARY_SRC=$(REPO)
+
+SRC_DIRS=pkg
+
+# Build configuration
+BUILD_DIR ?= $(CURDIR)/out
+BUILD_GOOS ?= $(shell go env GOOS)
+BUILD_GOARCH ?= $(shell go env GOARCH)
+GO_LINKER_FLAGS=-ldflags="-s -w -X main.version=$(VERSION)"
+
+# Build tweaks for windows
+ifeq (${BUILD_GOOS}, windows)
+BINARY := $(BINARY).exe
+endif
+
+# Other config
+NO_COLOR=\033[0m
+OK_COLOR=\033[32;01m
+ERROR_COLOR=\033[31;01m
+WARN_COLOR=\033[33;01m
+
+.PHONY: all clean deps build
+
+all: clean deps build
+
+# Install dependencies
+deps:
+	@printf "$(OK_COLOR)==> Installing dependencies$(NO_COLOR)\n"
+	@go mod vendor
+	@CGO_ENABLED=0 go generate
+
+# Builds the project
+build:
+	@mkdir -p ${BUILD_DIR}
+	@printf "$(OK_COLOR)==> Building ${BINARY} for ${BUILD_GOOS}/${BUILD_GOARCH}: $(NO_COLOR)\n"
+	CGO_ENABLED=0 GOARCH=${BUILD_GOARCH} GOOS=${BUILD_GOOS} \
+	  go build -o ${BUILD_DIR}/${BINARY} ${GO_LINKER_FLAGS} ${BINARY_SRC}
+	@printf "$(OK_COLOR)==> Building ${BINARY} for ${BUILD_GOOS}/${BUILD_GOARCH} succeed $(NO_COLOR)\n"
+
+test-unit:
+	@printf "$(OK_COLOR)==> Running unit tests$(NO_COLOR)\n"
+	@CGO_ENABLED=0 GOFLAGS=-mod=vendor go test -cover ./... -coverprofile=coverage_unit.txt -covermode=atomic
+
+# Added -p=1 to fix flakiness in integration DB tests
+test-integration: clean build
+	@printf "$(OK_COLOR)==> Running integration tests$(NO_COLOR)\n"
+	@CGO_ENABLED=0 GOFLAGS=-mod=vendor go test -tags integration -cover ./... -coverprofile=coverage_integration.txt -covermode=atomic
+
+# Cleans our project: deletes binaries
+clean:
+	@printf "$(OK_COLOR)==> Cleaning project$(NO_COLOR)\n"
+	if [ -d ${BUILD_DIR} ] ; then rm -rf ${BUILD_DIR}/* ; fi

github.go

@@ -0,0 +1,212 @@
+// Package cocogh to collect GitHub contents
+package cocogh
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/google/go-github/v57/github"
+	"github.com/shurcooL/githubv4"
+)
+
+// Paths is a struct to keep track of the status of file paths
+type Paths struct {
+	Added    []string
+	Removed  []string
+	Modified []string
+}
+
+// GitHubFilter is a filter used for GitHub file search.
+type GitHubFilter struct {
+	FilePath  string
+	FileTypes []string
+}
+
+// GitHubConfig holds the configuration for the GitHub client such as the repositories and branches to work with.
+type GitHubConfig struct {
+	Owner         string
+	Repositories  []string
+	DefaultBranch string
+	Filter        GitHubFilter
+}
+
+// GitHub stores a repo's GitHub client and its related configurations.
+type GitHub struct {
+	Configuration GitHubConfig
+
+	graphQLClient GraphQLClient
+	restClient    RESTClient
+}
+
+// GraphQLClient is an interface to help test the GitHub GraphQLClient.
+type GraphQLClient interface {
+	Query(ctx context.Context, q interface{}, variables map[string]interface{}) error
+}
+
+// RESTClient is an interface to help test the GitHub GraphQLClient.
+type RESTClient interface {
+	ListCommits(ctx context.Context, owner, repo string, opts *github.CommitsListOptions) ([]*github.RepositoryCommit, *github.Response, error)
+	GetCommit(ctx context.Context, owner, repo, sha string, opts *github.ListOptions) (*github.RepositoryCommit, *github.Response, error)
+}
+
+// GHQueryForListFiles holds the structure of the query to fetch all files from a repository.
+type GHQueryForListFiles struct {
+	Repository struct {
+		Object struct {
+			Tree struct {
+				Entries []struct {
+					Name string
+					Path string
+					Type string
+				}
+			} `graphql:"... on Tree"`
+		} `graphql:"object(expression: $expression)"`
+	} `graphql:"repository(owner: $owner, name: $name)"`
+}
+
+// NewGitHubClient creates a new GitHub with the given GraphQLClient and configuration.
+func NewGitHubClient(restClient RESTClient, graphQLClient GraphQLClient, configuration GitHubConfig) *GitHub {
+	return &GitHub{
+		restClient:    restClient,
+		graphQLClient: graphQLClient,
+		Configuration: configuration,
+	}
+}
+
+// GetFilePathsForRepositories fetches the file paths from all the repositories specified in the GitHub.
+func (c *GitHub) GetFilePathsForRepositories() ([]string, error) {
+	var files []string
+	for _, repo := range c.Configuration.Repositories {
+		fs, err := c.getFilePathsForRepo(c.Configuration.Owner, repo, fmt.Sprintf("%s:%s", c.Configuration.DefaultBranch, c.Configuration.Filter.FilePath))
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, fs...)
+	}
+
+	if len(c.Configuration.Filter.FileTypes) == 0 {
+		return files, nil
+	}
+
+	var filteredFiles []string
+	for i, file := range files {
+		if !c.hasFileType(file, c.Configuration.Filter.FileTypes) {
+			continue
+		}
+		filteredFiles = append(filteredFiles, files[i])
+	}
+
+	return filteredFiles, nil
+}
+
+// GetChangedFilePathsSince fetches the file paths from all repositories that have been changed in the specified duration (in hours).
+func (c *GitHub) GetChangedFilePathsSince(hoursSince int) (Paths, error) {
+	ctx := context.Background()
+
+	now := time.Now()
+	dayToHour := 24 * hoursSince
+	specifiedTime := now.Add(time.Hour * time.Duration(-dayToHour))
+
+	opt := &github.CommitsListOptions{
+		Since: specifiedTime,
+		Path:  c.Configuration.Filter.FilePath,
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+
+	var paths Paths
+
+	for _, repo := range c.Configuration.Repositories {
+		commitPaths, err := c.getChangedFilePathsForRepo(ctx, repo, opt)
+		if err != nil {
+			return Paths{}, err
+		}
+		paths.Added = append(paths.Added, commitPaths.Added...)
+		paths.Removed = append(paths.Removed, commitPaths.Removed...)
+		paths.Modified = append(paths.Modified, commitPaths.Modified...)
+	}
+
+	return paths, nil
+}
+
+// getFilePathsForRepo fetches the file paths in a GitHub repository.
+func (c *GitHub) getFilePathsForRepo(owner, name, expression string) ([]string, error) {
+	var query GHQueryForListFiles
+	variables := map[string]interface{}{
+		"owner":      githubv4.String(owner),
+		"name":       githubv4.String(name),
+		"expression": githubv4.String(expression),
+	}
+
+	err := c.graphQLClient.Query(context.Background(), &query, variables)
+	if err != nil {
+		return nil, err
+	}
+
+	var files []string
+	for _, entry := range query.Repository.Object.Tree.Entries {
+		if entry.Type == "blob" {
+			files = append(files, entry.Path)
+		} else if entry.Type == "tree" {
+			subFiles, err := c.getFilePathsForRepo(owner, name, expression+"/"+entry.Name)
+			if err != nil {
+				return nil, err
+			}
+			files = append(files, subFiles...)
+		}
+	}
+
+	return files, nil
+}
+
+// hasFileType determines if a filename ends with certain file types.
+func (c *GitHub) hasFileType(fileName string, fileTypes []string) bool {
+	for _, fileType := range fileTypes {
+		if strings.HasSuffix(fileName, fileType) {
+			return true
+		}
+	}
+	return false
+}
+
+// getChangedFilePathsForRepo fetches the file paths in a repository that have been changed.
+func (c *GitHub) getChangedFilePathsForRepo(ctx context.Context, repo string, opt *github.CommitsListOptions) (Paths, error) {
+	var paths Paths
+
+	commits, _, err := c.restClient.ListCommits(ctx, c.Configuration.Owner, repo, opt)
+	if err != nil {
+		return paths, err
+	}
+
+	directory := c.Configuration.Filter.FilePath
+
+	for _, commit := range commits {
+		commitDetails, _, err := c.restClient.GetCommit(ctx, c.Configuration.Owner, repo, *commit.SHA, nil)
+		if err != nil {
+			return paths, err
+		}
+
+		for _, file := range commitDetails.Files {
+			if strings.HasPrefix(file.GetFilename(), directory) {
+				switch file.GetStatus() {
+				case "removed":
+					paths.Removed = append(paths.Removed, file.GetFilename())
+				case "added":
+					paths.Added = append(paths.Added, file.GetFilename())
+				case "modified", "changed":
+					paths.Modified = append(paths.Modified, file.GetFilename())
+				case "renamed":
+					paths.Removed = append(paths.Removed, file.GetPreviousFilename())
+					paths.Added = append(paths.Added, file.GetFilename())
+				case "copied":
+					paths.Added = append(paths.Added, file.GetFilename())
+				}
+			}
+		}
+	}
+
+	return paths, nil
+}