diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index fe80e48..dc246b1 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -1,4 +1,4 @@ -name: golangci-lint +name: Lint on: push: branches: @@ -10,12 +10,11 @@ permissions: contents: read jobs: - golangci: + lint: strategy: matrix: go: [ '1.21.13' ] os: [ ubuntu-latest, macos-latest, windows-latest ] - name: lint runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c8a365..8f5ef28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ permissions: contents: write jobs: - build: + release: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/cmd/gbounty/bootstrap/update.go b/cmd/gbounty/bootstrap/update.go index a4aaea5..849658f 100644 --- a/cmd/gbounty/bootstrap/update.go +++ b/cmd/gbounty/bootstrap/update.go @@ -53,7 +53,7 @@ func CheckForUpdates() { updateApp := func() error { defer wg.Done(); return updateApp(update.app) } go die.OnErr(updateApp, "Could not update the application") } else { - pterm.Info.Printf("There is a new app version available: v%s (curr. %s)\n", + pterm.Info.Printf("There is a new app version available: %s (curr. %s)\n", update.app.latest.Version, update.app.current) pterm.Info.Println("Use --update or --update-app to update") } @@ -92,7 +92,7 @@ func updateApp(info updateNeeds) error { // Which, later, will be used to replace the binary. cmdPath, err := osext.Executable() if err != nil { - return fmt.Errorf("failed to get the executable's path: %s", err) //nolint:err113 + return fmt.Errorf("failed to get the executable's path: %s", err) //nolint:err113,errorlint } // When on Windows, the executable path might have the '.exe' suffix. @@ -103,14 +103,14 @@ func updateApp(info updateNeeds) error { // Check if the binary is a symlink. stat, err := os.Lstat(cmdPath) if err != nil { - return fmt.Errorf("failed to stat: %s - file may not exist: %s", cmdPath, err) + return fmt.Errorf("failed to stat: %s - file may not exist: %s", cmdPath, err) //nolint:err113 } // If it is, we resolve the symlink. if stat.Mode()&os.ModeSymlink != 0 { p, err := filepath.EvalSymlinks(cmdPath) if err != nil { - return fmt.Errorf("failed to resolve symlink: %s - for executable: %s", cmdPath, err) //nolint:err113 + return fmt.Errorf("failed to resolve symlink: %s - for executable: %s", cmdPath, err) //nolint:err113,errorlint } cmdPath = p } diff --git a/go.mod b/go.mod index 47c076d..c0b5183 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/afero v1.11.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect diff --git a/go.sum b/go.sum index dc2c2b2..79f5971 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/internal/interfaces.go b/internal/interfaces.go index e68d84f..aa1db67 100644 --- a/internal/interfaces.go +++ b/internal/interfaces.go @@ -1,4 +1,4 @@ -package internal +package scan import ( "github.com/bountysecurity/gbounty/internal/profile" diff --git a/internal/platform/filesystem/filesystem.go b/internal/platform/filesystem/filesystem.go new file mode 100644 index 0000000..fd9c2da --- /dev/null +++ b/internal/platform/filesystem/filesystem.go @@ -0,0 +1,639 @@ +package filesystem + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "sync" + + "github.com/spf13/afero" + + scan "github.com/bountysecurity/gbounty/internal" + "github.com/bountysecurity/gbounty/kit/logger" +) + +const ( + // FileStats is the name of the file where the scan stats are saved to. + FileStats = "stats.json" + + // FileErrors is the name of the file where the scan errors are saved to. + FileErrors = "errors.json" + + // FileMatches is the name of the file where the scan matches are saved to. + FileMatches = "matches.json" + + // FileTasks is the name of the file where the scan tasks are saved to. + FileTasks = "tasks.json" + + // FileTemplates is the name of the file where the scan templates are saved to. + FileTemplates = "templates.json" +) + +const maxCapacity = 50e6 + +// Afero must implement the [scan.FileSystem] interface. +var _ scan.FileSystem = &Afero{} + +// Afero is a [scan.FileSystem] implementation that uses the [afero.Fs] interface +// under the hood. So, it can rely on the [afero] package abstractions to either +// use a regular (disk-based) file-system, or a virtual one (like in-memory). +type Afero struct { + fs afero.Fs + basePath string + + statsMtx sync.Mutex + statsFile afero.File + + errorsMtx sync.Mutex + errorsFile afero.File + + matchesMtx sync.Mutex + matchesFile afero.File + + tasksMtx sync.Mutex + tasksFile afero.File + + templatesMtx sync.RWMutex + templatesFile afero.File +} + +// New creates a new [Afero] instance, using the given [afero.Fs] and the base path. +func New(fs afero.Fs, basePath string) (*Afero, error) { + err := fs.MkdirAll(basePath, 0o755) + if err != nil { + return nil, err + } + + statsFile, err := fs.OpenFile(statsFilePath(basePath), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return nil, err + } + + errorsFile, err := fs.OpenFile(errorsFilePath(basePath), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return nil, err + } + + matchesFile, err := fs.OpenFile(matchesFilePath(basePath), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return nil, err + } + + tasksFile, err := fs.OpenFile(tasksFilePath(basePath), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return nil, err + } + + templatesFile, err := fs.OpenFile(templatesFilePath(basePath), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o755) + if err != nil { + return nil, err + } + + return &Afero{ + fs: fs, + basePath: basePath, + + statsFile: statsFile, + errorsFile: errorsFile, + matchesFile: matchesFile, + tasksFile: tasksFile, + templatesFile: templatesFile, + }, nil +} + +// StoreStats stores the given [scan.Stats] into the file system. +func (a *Afero) StoreStats(ctx context.Context, stats *scan.Stats) error { + logger.For(ctx).Debug("Storing stats into the file system...") + + var err error + + a.statsMtx.Lock() + defer a.statsMtx.Unlock() + + if a.statsFile != nil { + _ = a.statsFile.Close() + } + + a.statsFile, err = a.fs.OpenFile(statsFilePath(a.basePath), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + return err + } + + bytes, err := json.Marshal(&stats) + if err != nil { + return err + } + + _, err = a.statsFile.Write(bytes) + + return err +} + +// LoadStats loads the [scan.Stats] from the file system. +func (a *Afero) LoadStats(ctx context.Context) (*scan.Stats, error) { + logger.For(ctx).Debug("Loading stats from the file system...") + + a.statsMtx.Lock() + defer a.statsMtx.Unlock() + + _, err := a.statsFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + bytes, err := io.ReadAll(a.statsFile) + if err != nil { + return nil, err + } + + if len(bytes) == 0 { + return nil, nil + } + + scanStats := scan.Stats{} + err = json.Unmarshal(bytes, &scanStats) + if err != nil { + return nil, err + } + + return &scanStats, nil +} + +// StoreError stores the given [scan.Error] into the file system. +func (a *Afero) StoreError(ctx context.Context, scanError scan.Error) error { + logger.For(ctx).Debug("Storing error into the file system...") + + bytes, err := json.Marshal(&scanError) + if err != nil { + return err + } + + a.errorsMtx.Lock() + defer a.errorsMtx.Unlock() + + _, err = a.errorsFile.WriteString(string(bytes) + "\n") + + return err +} + +// LoadErrors loads the [scan.Error] instances from the file system. +func (a *Afero) LoadErrors(ctx context.Context) ([]scan.Error, error) { + logger.For(ctx).Info("Loading errors from the file system...") + + a.errorsMtx.Lock() + defer a.errorsMtx.Unlock() + + // Get the current seek offset and defer reset + currSeekOffset, err := a.errorsFile.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + defer func() { _, _ = a.errorsFile.Seek(currSeekOffset, io.SeekStart) }() + + _, err = a.errorsFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var scanErrors []scan.Error + + scanner := bufio.NewScanner(a.errorsFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanError scan.Error + + err := json.Unmarshal(scanner.Bytes(), &scanError) + if err != nil { + return nil, err + } + + scanErrors = append(scanErrors, scanError) + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while loading errors: %s", err) + } + + return scanErrors, nil +} + +// ErrorsIterator returns a channel that iterates over the [scan.Error] instances. +// +// It also returns a function that can be used to close the iterator (see [scan.CloseFunc]). +// The channel is closed when the iterator is done (no more elements), when the [scan.CloseFunc] +// is called, or when the context is canceled. Thus, the context cancellation can also be used +// to stop the iteration. +// +// It is the "streaming fashion" equivalent of [LoadErrors()]. +func (a *Afero) ErrorsIterator(ctx context.Context) (chan scan.Error, scan.CloseFunc, error) { + logger.For(ctx).Info("Reading errors from the file system...") + + a.errorsMtx.Lock() + defer a.errorsMtx.Unlock() + + errorsFile, err := a.fs.OpenFile(errorsFilePath(a.basePath), os.O_RDONLY, 0o755) + if err != nil { + return nil, nil, err + } + + ch := make(chan scan.Error) + + go func() { + scanner := bufio.NewScanner(errorsFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanError scan.Error + + err := json.Unmarshal(scanner.Bytes(), &scanError) + if err != nil { + continue + } + + ch <- scanError + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while reading errors: %s", err) + } + + close(ch) + }() + + return ch, func() { _ = errorsFile.Close() }, nil +} + +// StoreMatch stores the given [scan.Match] into the file system. +func (a *Afero) StoreMatch(ctx context.Context, scanMatch scan.Match) error { + logger.For(ctx).Debug("Storing match into the file system...") + + bytes, err := json.Marshal(&scanMatch) + if err != nil { + return err + } + + a.matchesMtx.Lock() + defer a.matchesMtx.Unlock() + + _, err = a.matchesFile.WriteString(string(bytes) + "\n") + + return err +} + +// LoadMatches loads the [scan.Match] instances from the file system. +func (a *Afero) LoadMatches(ctx context.Context) ([]scan.Match, error) { + logger.For(ctx).Info("Loading matches from the file system...") + + a.matchesMtx.Lock() + defer a.matchesMtx.Unlock() + + // Get the current seek offset and defer reset + currSeekOffset, err := a.matchesFile.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + defer func() { _, _ = a.matchesFile.Seek(currSeekOffset, io.SeekStart) }() + + _, err = a.matchesFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var scanMatches []scan.Match + + scanner := bufio.NewScanner(a.matchesFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanMatch scan.Match + + err := json.Unmarshal(scanner.Bytes(), &scanMatch) + if err != nil { + return nil, err + } + + scanMatches = append(scanMatches, scanMatch) + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while loading matches: %s", err) + } + + return scanMatches, nil +} + +// MatchesIterator returns a channel that iterates over the [scan.Match] instances. +// +// It also returns a function that can be used to close the iterator (see [scan.CloseFunc]). +// The channel is closed when the iterator is done (no more elements), when the [scan.CloseFunc] +// is called, or when the context is canceled. Thus, the context cancellation can also be used +// to stop the iteration. +// +// It is the "streaming fashion" equivalent of [LoadMatches()]. +func (a *Afero) MatchesIterator(ctx context.Context) (chan scan.Match, scan.CloseFunc, error) { + logger.For(ctx).Debug("Reading matches from the file system...") + + a.matchesMtx.Lock() + defer a.matchesMtx.Unlock() + + matchesFile, err := a.fs.OpenFile(matchesFilePath(a.basePath), os.O_RDONLY, 0o755) + if err != nil { + return nil, nil, err + } + + ch := make(chan scan.Match) + + go func() { + scanner := bufio.NewScanner(matchesFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanMatch scan.Match + + err := json.Unmarshal(scanner.Bytes(), &scanMatch) + if err != nil { + continue + } + + ch <- scanMatch + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while reading matches: %s", err) + } + + close(ch) + }() + + return ch, func() { _ = matchesFile.Close() }, nil +} + +// StoreTaskSummary stores the given [scan.TaskSummary] into the file system. +func (a *Afero) StoreTaskSummary(ctx context.Context, scanTaskSummary scan.TaskSummary) error { + logger.For(ctx).Debug("Storing task summary into the file system...") + + bytes, err := json.Marshal(&scanTaskSummary) + if err != nil { + return err + } + + a.tasksMtx.Lock() + defer a.tasksMtx.Unlock() + + _, err = a.tasksFile.WriteString(string(bytes) + "\n") + + return err +} + +// LoadTasksSummaries loads the [scan.TaskSummary] instances from the file system. +func (a *Afero) LoadTasksSummaries(ctx context.Context) ([]scan.TaskSummary, error) { + logger.For(ctx).Info("Loading task summaries from the file system...") + + a.tasksMtx.Lock() + defer a.tasksMtx.Unlock() + + // Get the current seek offset and defer reset + currSeekOffset, err := a.tasksFile.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + defer func() { _, _ = a.tasksFile.Seek(currSeekOffset, io.SeekStart) }() + + _, err = a.tasksFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var scanTasks []scan.TaskSummary + + scanner := bufio.NewScanner(a.tasksFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanTask scan.TaskSummary + + err := json.Unmarshal(scanner.Bytes(), &scanTask) + if err != nil { + return nil, err + } + + scanTasks = append(scanTasks, scanTask) + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while loading task summaries: %s", err) + } + + return scanTasks, nil +} + +// TasksSummariesIterator returns a channel that iterates over the [scan.TaskSummary] instances. +// +// It also returns a function that can be used to close the iterator (see [scan.CloseFunc]). +// The channel is closed when the iterator is done (no more elements), when the [scan.CloseFunc] +// is called, or when the context is canceled. Thus, the context cancellation can also be used +// to stop the iteration. +// +// It is the "streaming fashion" equivalent of [LoadTasksSummaries()]. +func (a *Afero) TasksSummariesIterator(ctx context.Context) (chan scan.TaskSummary, scan.CloseFunc, error) { + logger.For(ctx).Info("Reading task summaries from the file system...") + + a.tasksMtx.Lock() + defer a.tasksMtx.Unlock() + + tasksFile, err := a.fs.OpenFile(tasksFilePath(a.basePath), os.O_RDONLY, 0o755) + if err != nil { + return nil, nil, err + } + + ch := make(chan scan.TaskSummary) + + go func() { + scanner := bufio.NewScanner(tasksFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanTask scan.TaskSummary + + err := json.Unmarshal(scanner.Bytes(), &scanTask) + if err != nil { + continue + } + + ch <- scanTask + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while reading task summaries: %s", err) + } + + close(ch) + }() + + return ch, func() { _ = tasksFile.Close() }, nil +} + +// StoreTemplate stores the given [scan.Template] into the file system. +func (a *Afero) StoreTemplate(ctx context.Context, scanTemplate scan.Template) error { + logger.For(ctx).Debug("Storing template into the file system...") + + bytes, err := json.Marshal(&scanTemplate) + if err != nil { + return err + } + + a.templatesMtx.Lock() + defer a.templatesMtx.Unlock() + + _, err = a.templatesFile.WriteString(string(bytes) + "\n") + + return err +} + +// LoadTemplates loads the [scan.Template] instances from the file system. +func (a *Afero) LoadTemplates(ctx context.Context) ([]scan.Template, error) { + logger.For(ctx).Info("Loading templates from the file system...") + + a.templatesMtx.Lock() + defer a.templatesMtx.Unlock() + + // Get the current seek offset and defer reset + currSeekOffset, err := a.templatesFile.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + defer func() { _, _ = a.templatesFile.Seek(currSeekOffset, io.SeekStart) }() + + _, err = a.templatesFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var scanTemplates []scan.Template + + scanner := bufio.NewScanner(a.templatesFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() { + var scanTemplate scan.Template + + err := json.Unmarshal(scanner.Bytes(), &scanTemplate) + if err != nil { + return nil, err + } + + scanTemplates = append(scanTemplates, scanTemplate) + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while loading templates: %s", err) + } + + return scanTemplates, nil +} + +// TemplatesIterator returns a channel that iterates over the [scan.Template] instances. +// +// It also returns a function that can be used to close the iterator (see [scan.CloseFunc]). +// The channel is closed when the iterator is done (no more elements), when the [scan.CloseFunc] +// is called, or when the context is canceled. Thus, the context cancellation can also be used +// to stop the iteration. +// +// It is the "streaming fashion" equivalent of [LoadTemplates()]. +func (a *Afero) TemplatesIterator(ctx context.Context) (chan scan.Template, error) { + logger.For(ctx).Info("Reading templates from the file system...") + + a.templatesMtx.RLock() + + templatesFile, err := a.fs.OpenFile(templatesFilePath(a.basePath), os.O_RDONLY, 0o755) + if err != nil { + a.templatesMtx.RUnlock() + return nil, err + } + + ch := make(chan scan.Template) + + go func() { + // Once the iterator finishes, we close the file. + defer func() { + if err := templatesFile.Close(); err != nil { + logger.For(ctx).Errorf("Error while closing templates file: %s", err) + } + }() + // Unlock the mutex protecting the file. + defer a.templatesMtx.RUnlock() + // And close the channel used as the iterator. + defer close(ch) + + scanner := bufio.NewScanner(templatesFile) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + for scanner.Scan() && ctx.Err() == nil { + var scanTemplate scan.Template + + err := json.Unmarshal(scanner.Bytes(), &scanTemplate) + if err != nil { + continue + } + + select { + case <-ctx.Done(): + case ch <- scanTemplate: + } + } + + if scanner.Err() != nil { + logger.For(ctx).Errorf("Error while reading templates: %s", err) + } + + if ctx.Err() != nil { + logger.For(ctx).Infof("Templates iterator was cancelled from context: %s", context.Cause(ctx)) + } + }() + + return ch, nil +} + +// Cleanup removes all the files from the file system. +func (a *Afero) Cleanup(ctx context.Context) error { + logger.For(ctx).Info("Removing files from the file system...") + toClose := []afero.File{a.statsFile, a.errorsFile, a.matchesFile, a.tasksFile, a.templatesFile} + for _, f := range toClose { + if err := f.Close(); err != nil { + logger.For(ctx).Errorf("Error while closing file '%s': %v", f.Name(), err) + } + } + return a.fs.RemoveAll(a.basePath) +} + +func statsFilePath(basePath string) string { + return fmt.Sprintf("%s/%s", basePath, FileStats) +} + +func errorsFilePath(basePath string) string { + return fmt.Sprintf("%s/%s", basePath, FileErrors) +} + +func matchesFilePath(basePath string) string { + return fmt.Sprintf("%s/%s", basePath, FileMatches) +} + +func tasksFilePath(basePath string) string { + return fmt.Sprintf("%s/%s", basePath, FileTasks) +} + +func templatesFilePath(basePath string) string { + return fmt.Sprintf("%s/%s", basePath, FileTemplates) +} diff --git a/internal/platform/filesystem/filesystem_test.go b/internal/platform/filesystem/filesystem_test.go new file mode 100644 index 0000000..8ee7ae0 --- /dev/null +++ b/internal/platform/filesystem/filesystem_test.go @@ -0,0 +1,503 @@ +package filesystem_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + scan "github.com/bountysecurity/gbounty/internal" + "github.com/bountysecurity/gbounty/internal/platform/filesystem" + "github.com/bountysecurity/gbounty/internal/request" + "github.com/bountysecurity/gbounty/internal/response" + "github.com/bountysecurity/gbounty/kit/ulid" +) + +func TestAfero_New(t *testing.T) { + t.Parallel() + + t.Run("no existing directory", func(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + _, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + assertEmptyFiles(t, fs, basePath) + }) + + t.Run("existing directory", func(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + stats := initializeTestData(t, aferoFS, fs, basePath) + assertNonEmptyFiles(t, fs, basePath, stats) + }) +} + +func TestAfero_LoadStats(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeStats(t, aferoFS) + + scanStats, err := aferoFS.LoadStats(context.Background()) + require.NoError(t, err) + + assert.EqualValues(t, &scan.Stats{}, scanStats) +} + +func TestAfero_LoadErrors(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeErrors(t, aferoFS) + + scanErrors, err := aferoFS.LoadErrors(context.Background()) + require.NoError(t, err) + + var numErrors int + for _, scanError := range scanErrors { + numErrors++ + + assert.Equal(t, dummyError(), scanError) + } + + assert.Equal(t, 5, numErrors) +} + +func TestAfero_ErrorsIterator(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeErrors(t, aferoFS) + + var numErrors int + + scanErrorsIterator, closeIt, err := aferoFS.ErrorsIterator(context.Background()) + require.NoError(t, err) + + for scanError := range scanErrorsIterator { + numErrors++ + + assert.Equal(t, dummyError(), scanError) + } + + closeIt() + + assert.Equal(t, 5, numErrors) +} + +func TestAfero_LoadMatches(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeMatches(t, aferoFS) + + scanMatches, err := aferoFS.LoadMatches(context.Background()) + require.NoError(t, err) + + var numMatches int + for _, scanMatch := range scanMatches { + numMatches++ + + assert.Equal(t, dummyMatch(), scanMatch) + } + + assert.Equal(t, 5, numMatches) +} + +func TestAfero_MatchesIterator(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeMatches(t, aferoFS) + + var numMatches int + + scanMatchesIterator, closeIt, err := aferoFS.MatchesIterator(context.Background()) + require.NoError(t, err) + + for scanMatch := range scanMatchesIterator { + numMatches++ + + assert.Equal(t, dummyMatch(), scanMatch) + } + + closeIt() + + assert.Equal(t, 5, numMatches) +} + +func TestAfero_LoadTasksSummaries(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeTaskSummaries(t, aferoFS) + + scanTasks, err := aferoFS.LoadTasksSummaries(context.Background()) + require.NoError(t, err) + + var numTasks int + for _, scanTask := range scanTasks { + numTasks++ + + assert.Equal(t, dummyTask(), scanTask) + } + + assert.Equal(t, 5, numTasks) +} + +func TestAfero_TasksSummariesIterator(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeTaskSummaries(t, aferoFS) + + var numTasks int + + scanTasksIterator, closeIt, err := aferoFS.TasksSummariesIterator(context.Background()) + require.NoError(t, err) + + for scanTask := range scanTasksIterator { + numTasks++ + + assert.Equal(t, dummyTask(), scanTask) + } + + closeIt() + + assert.Equal(t, 5, numTasks) +} + +func TestAfero_LoadTemplates(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeTemplates(t, aferoFS) + + scanTemplates, err := aferoFS.LoadTemplates(context.Background()) + require.NoError(t, err) + + var numTemplates int + for _, scanTemplate := range scanTemplates { + numTemplates++ + + assert.Equal(t, dummyTemplate(), scanTemplate) + } + + assert.Equal(t, 5, numTemplates) +} + +func TestAfero_TemplatesIterator(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeTemplates(t, aferoFS) + + var numTemplates int + + scanTemplatesIterator, err := aferoFS.TemplatesIterator(context.Background()) + require.NoError(t, err) + + for scanTemplate := range scanTemplatesIterator { + numTemplates++ + + assert.Equal(t, dummyTemplate(), scanTemplate) + } + + assert.Equal(t, 5, numTemplates) +} + +func TestConcurrentOps(t *testing.T) { + t.Parallel() + + fs, basePath := initializeFsTest() + + aferoFS, err := filesystem.New(fs, basePath) + require.NoError(t, err) + + storeSomeErrors(t, aferoFS) + + scanErrors, err := aferoFS.LoadErrors(context.Background()) + require.NoError(t, err) + + var numErrors int + for _, scanError := range scanErrors { + numErrors++ + + assert.Equal(t, dummyError(), scanError) + } + + assert.Equal(t, 5, numErrors) + + // Now we have tested the regular functionality. + // So, now we want to see what happens if new writes + // happens after the read (LoadErrors call above). + + additionalError1 := scan.Error{ + URL: "localhost:9090", + Requests: []*request.Request{dummyRequest()}, + Responses: []*response.Response{dummyResponse()}, + Err: "something went wrong", + } + + additionalError2 := scan.Error{ + URL: "localhost:9191", + Requests: []*request.Request{dummyRequest()}, + Responses: []*response.Response{dummyResponse()}, + Err: "something went wrong", + } + + require.NoError(t, aferoFS.StoreError(context.Background(), additionalError1)) + require.NoError(t, aferoFS.StoreError(context.Background(), additionalError2)) + + scanErrors, err = aferoFS.LoadErrors(context.Background()) + require.NoError(t, err) + + numErrors = 0 + for _, scanError := range scanErrors { + numErrors++ + + switch numErrors { + case 1, 2, 3, 4, 5: + assert.Equal(t, dummyError(), scanError) + case 6: + assert.Equal(t, additionalError1, scanError) + case 7: + assert.Equal(t, additionalError2, scanError) + } + } + + assert.Equal(t, 7, numErrors) +} + +func initializeFsTest() (*afero.MemMapFs, string) { + fs := &afero.MemMapFs{} + tmp := os.TempDir() + id := ulid.New() + + return fs, tmp + id +} + +func initializeTestData(t *testing.T, aferoFS *filesystem.Afero, fs afero.Fs, basePath string) []os.FileInfo { + t.Helper() + + storeSomeStats(t, aferoFS) + + statsInfo, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, filesystem.FileStats)) + require.NoError(t, err) + + storeSomeErrors(t, aferoFS) + + errorsInfo, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, filesystem.FileErrors)) + require.NoError(t, err) + + storeSomeMatches(t, aferoFS) + + matchesInfo, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, filesystem.FileMatches)) + require.NoError(t, err) + + storeSomeTaskSummaries(t, aferoFS) + + tasksInfo, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, filesystem.FileTasks)) + require.NoError(t, err) + + storeSomeTemplates(t, aferoFS) + + templatesInfo, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, filesystem.FileTemplates)) + require.NoError(t, err) + + return []os.FileInfo{statsInfo, errorsInfo, matchesInfo, tasksInfo, templatesInfo} +} + +func storeSomeStats(t *testing.T, aferoFS *filesystem.Afero) { + t.Helper() + + require.NoError(t, aferoFS.StoreStats(context.Background(), &scan.Stats{})) +} + +func storeSomeErrors(t *testing.T, aferoFS *filesystem.Afero) { + t.Helper() + + for i := 0; i < 5; i++ { + require.NoError(t, aferoFS.StoreError(context.Background(), dummyError())) + } +} + +func storeSomeMatches(t *testing.T, aferoFS *filesystem.Afero) { + t.Helper() + + for i := 0; i < 5; i++ { + require.NoError(t, aferoFS.StoreMatch(context.Background(), dummyMatch())) + } +} + +func storeSomeTaskSummaries(t *testing.T, aferoFS *filesystem.Afero) { + t.Helper() + + for i := 0; i < 5; i++ { + require.NoError(t, aferoFS.StoreTaskSummary(context.Background(), dummyTask())) + } +} + +func storeSomeTemplates(t *testing.T, aferoFS *filesystem.Afero) { + t.Helper() + + for i := 0; i < 5; i++ { + require.NoError(t, aferoFS.StoreTemplate(context.Background(), dummyTemplate())) + } +} + +func assertEmptyFiles(t *testing.T, fs afero.Fs, basePath string) { + t.Helper() + + toBeCreated := []string{filesystem.FileErrors, filesystem.FileMatches, filesystem.FileTasks} + + for _, f := range toBeCreated { + stat, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, f)) + require.NoError(t, err) + assert.Equal(t, int64(0), stat.Size()) + } +} + +func assertNonEmptyFiles(t *testing.T, fs afero.Fs, basePath string, stats []os.FileInfo) { + t.Helper() + + for _, stat := range stats { + currStat, err := fs.Stat(fmt.Sprintf("%s/%s", basePath, stat.Name())) + require.NoError(t, err) + assert.Equal(t, stat.Size(), currStat.Size()) + assert.NotEqual(t, stat.ModTime(), currStat.Size()) + } +} + +func dummyError() scan.Error { + return scan.Error{ + URL: "localhost:8080", + Requests: []*request.Request{dummyRequest()}, + Responses: []*response.Response{dummyResponse()}, + Err: "something went wrong", + } +} + +func dummyMatch() scan.Match { + return scan.Match{ + URL: "localhost:8080", + Requests: []*request.Request{dummyRequest()}, + Responses: []*response.Response{dummyResponse()}, + IssueName: "Issue for tests", + IssueSeverity: "Low", + IssueConfidence: "Tentative", + IssueParam: "URL Path", + ProfileType: "Active", + } +} + +func dummyTask() scan.TaskSummary { + return scan.TaskSummary{ + URL: "localhost:8080", + Requests: []*request.Request{dummyRequest()}, + Responses: []*response.Response{dummyResponse()}, + } +} + +func dummyTemplate() scan.Template { + return scan.Template{ + Request: *dummyRequest(), + } +} + +func dummyRequest() *request.Request { + return &request.Request{ + URL: "http://localhost:8080", + Method: "POST", + Path: "/search.php?test=query", + Proto: "HTTP/1.1", + Headers: map[string][]string{ + "Host": {"testphp.vulnweb.com"}, + "User-Agent": {"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0"}, + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"}, + "Accept-Language": {"es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"}, + "Accept-Encoding": {"gzip, deflate"}, + "Content-Type": {"application/x-www-form-urlencoded"}, + "Content-Length": {"26"}, + "Origin": {"http://testphp.vulnweb.com"}, + "Dnt": {"1"}, + "Connection": {"close"}, + "Referer": {"http://testphp.vulnweb.com/search.php?test=query"}, + "Upgrade-Insecure-Requests": {"1"}, + }, + Body: []byte(`searchFor=test&goButton=go + +`), + } +} + +func dummyResponse() *response.Response { + return &response.Response{ + Code: 404, + Status: "Not Found", + Proto: "HTTP/1.1", + Headers: map[string][]string{ + "Server": {"nginx/1.19.0"}, + "Date": {"Sun, 07 Feb 2021 23:44:49 GMT"}, + "Content-Type": {"text/html; charset=utf-8"}, + "Connection": {"close"}, + "Content-Length": {"150"}, + }, + Body: []byte(` +404 Not Found + +

404 Not Found

+
nginx/1.19.0
+ + +`), + } +} diff --git a/internal/stats.go b/internal/stats.go new file mode 100644 index 0000000..864df84 --- /dev/null +++ b/internal/stats.go @@ -0,0 +1,101 @@ +package scan + +import ( + "sync" + "time" +) + +// Stats is a structure that holds multiple stats about the [scan] process, +// such as the number of requests, the number of performed requests, etc. +type Stats struct { + NumOfTotalRequests int + NumOfPerformedRequests int + NumOfSucceedRequests int + NumOfFailedRequests int + NumOfSkippedRequests int + + NumOfRequestsToAnalyze int + NumOfResponsesToAnalyze int + + TemplatesEnded map[int]struct{} + + NumOfEntrypoints int + NumOfMatches int + + StartedAt time.Time + + sync.Mutex +} + +// NewStats creates a new instance of Stats. +func NewStats() *Stats { + return &Stats{ + StartedAt: time.Now(), + TemplatesEnded: make(map[int]struct{}), + } +} + +func (s *Stats) incrementTotalRequests(n int) { + s.Lock() + s.NumOfTotalRequests += n + s.Unlock() +} + +func (s *Stats) incrementSucceedRequests(n int) { + s.Lock() + s.NumOfPerformedRequests += n + s.NumOfSucceedRequests += n + s.Unlock() +} + +func (s *Stats) incrementFailedRequests(n int) { + s.Lock() + s.NumOfPerformedRequests += n + s.NumOfFailedRequests += n + s.Unlock() +} + +func (s *Stats) incrementSkippedRequests(n int) { + s.Lock() + s.NumOfSkippedRequests += n + s.Unlock() +} + +func (s *Stats) markTemplateAsEnded(i int) { + s.Lock() + s.TemplatesEnded[i] = struct{}{} + s.Unlock() +} + +func (s *Stats) isTemplateEnded(tpl Template) bool { + s.Lock() + defer s.Unlock() + + _, ok := s.TemplatesEnded[tpl.Idx] + + return ok +} + +func (s *Stats) incrementEntrypoints(n int) { + s.Lock() + s.NumOfEntrypoints += n + s.Unlock() +} + +func (s *Stats) incrementMatches(n int) { + s.Lock() + s.NumOfMatches += n + s.Unlock() +} + +func (s *Stats) incrementRequestsToAnalyze(n int) { + s.Lock() + s.NumOfRequestsToAnalyze += n + s.Unlock() +} + +func (s *Stats) incrementResponsesToAnalyze(n int) { + s.Lock() + s.NumOfResponsesToAnalyze += n + s.Unlock() +} diff --git a/internal/template.go b/internal/template.go index 4e060e4..0668933 100644 --- a/internal/template.go +++ b/internal/template.go @@ -1,4 +1,4 @@ -package internal +package scan import ( "github.com/bountysecurity/gbounty/internal/request" diff --git a/internal/types.go b/internal/types.go index 89b63af..cc2d956 100644 --- a/internal/types.go +++ b/internal/types.go @@ -1,5 +1,123 @@ -package internal +package scan + +import ( + "context" + "time" + + "github.com/bountysecurity/gbounty/internal/request" + "github.com/bountysecurity/gbounty/internal/response" + "github.com/bountysecurity/gbounty/kit/strings/occurrence" +) // CustomTokens is a type that represents a collection of pairs (key, value) // that can be used to replace certain tokens (i.e. placeholders) in a [request.Request]. type CustomTokens = map[string]string + +// TaskSummary represents a summary of a [scan] task, which corresponds to one of the +// iterations where one (or multiple) requests are targeted against a URL, and some +// checks are performed over the responses, looking for one (or multiple) [Match]. +type TaskSummary struct { + URL string + Requests []*request.Request + Responses []*response.Response +} + +// Match represents a match found during a [scan], containing the URL, +// the requests and responses that were made, and some other details associated +// with the match, like the profile's name and some information about the issue. +// +// There can be multiple [Match] per scan. +// See the `internal/match` package for further details. +type Match struct { + URL string + Requests []*request.Request + Responses []*response.Response + ProfileName string + ProfileTags []string + IssueName string + IssueSeverity string + IssueConfidence string + IssueDetail string + IssueBackground string + RemediationDetail string + RemediationBackground string + IssueParam string + ProfileType string + Payload string + Occurrences [][]occurrence.Occurrence + Grep string + At time.Time +} + +// Error represents an error that occurred during a [scan], containing the URL, +// the requests and responses that were made, and the error message. +// +// There can be multiple [Error] per scan. +type Error struct { + URL string + Requests []*request.Request + Responses []*response.Response + Err string +} + +// FileSystem defines the behavior expected from a [scan] file system, +// used to store and retrieve [Match], [Error], and [TaskSummary] instances. +type FileSystem interface { + FileSystemStats + FileSystemErrors + FileSystemMatches + FileSystemSummaries + FileSystemTemplates + Cleanup(context.Context) error +} + +// FileSystemStats defines the behavior expected from a [scan] file system +// to store and retrieve [Stats] instances. +type FileSystemStats interface { + StoreStats(context.Context, *Stats) error + LoadStats(context.Context) (*Stats, error) +} + +// FileSystemErrors defines the behavior expected from a [scan] file system +// to store and retrieve [Error] instances. +type FileSystemErrors interface { + StoreError(context.Context, Error) error + LoadErrors(context.Context) ([]Error, error) + ErrorsIterator(context.Context) (chan Error, CloseFunc, error) +} + +// FileSystemMatches defines the behavior expected from a [scan] file system +// to store and retrieve [Match] instances. +type FileSystemMatches interface { + StoreMatch(context.Context, Match) error + LoadMatches(context.Context) ([]Match, error) + MatchesIterator(context.Context) (chan Match, CloseFunc, error) +} + +// FileSystemSummaries defines the behavior expected from a [scan] file system +// to store and retrieve [TaskSummary] instances. +type FileSystemSummaries interface { + StoreTaskSummary(context.Context, TaskSummary) error + LoadTasksSummaries(context.Context) ([]TaskSummary, error) + TasksSummariesIterator(context.Context) (chan TaskSummary, CloseFunc, error) +} + +// FileSystemTemplates defines the behavior expected from a [scan] file system +// to store and retrieve [Template] instances. +type FileSystemTemplates interface { + StoreTemplate(context.Context, Template) error + LoadTemplates(context.Context) ([]Template, error) + + // TemplatesIterator returns a channel of Template (or an error), + // so the channel can be used as an iterator. + // The returned channel is closed when the iterator is done (no more elements) + // or when the context is canceled. + // Thus, the context cancellation can also be used to stop the iteration. + TemplatesIterator(context.Context) (chan Template, error) +} + +// CloseFunc is a function that can be used to close something that's open. +// For instance, a channel, a socket or a file descriptor. +// +// Internal details will vary depending on the function that returns it. +type CloseFunc func() diff --git a/kit/die/die.go b/kit/die/die.go index 71f7496..b72566c 100644 --- a/kit/die/die.go +++ b/kit/die/die.go @@ -18,7 +18,9 @@ func OnErr(fn func() error, mm ...string) { // OrRet is a helper function to handle errors, that runs the given function and, // in case of error, it prints the error message and exits (1) the program. // It is similar to [OnErr], but in case of success it returns the value. -func OrRet[T any](fn func() (T, error), mm ...string) (t T) { //nolint:ireturn +// +//nolint:ireturn +func OrRet[T any](fn func() (T, error), mm ...string) (t T) { v, err := fn() if err == nil { return v diff --git a/kit/selfupdate/detect.go b/kit/selfupdate/detect.go index cd679d6..6abc212 100644 --- a/kit/selfupdate/detect.go +++ b/kit/selfupdate/detect.go @@ -78,7 +78,7 @@ func findReleaseAndAsset( targetVersion semver.Version, ) (*github.RepositoryRelease, *github.ReleaseAsset, semver.Version, bool) { // Generate candidates - suffixes := make([]string, 0, 2*7*2) + suffixes := make([]string, 0, 2*7*2) //nolint:mnd for _, sep := range []rune{'_', '-'} { for _, ext := range []string{".zip", ".tar.gz", ".tgz", ".gzip", ".gz", ".tar.xz", ".xz", ""} { suffix := fmt.Sprintf("%s%c%s%s", runtime.GOOS, sep, runtime.GOARCH, ext)