Prepare for Ollama provider

cmd/eval-dev-quality/cmd/evaluate.go

@@ -30,7 +30,7 @@ import (
 type Evaluate struct {
 	// InstallToolsPath determines where tools for the evaluation are installed.
 	InstallToolsPath string `long:"install-tools-path" description:"Install tools for the evaluation into this path."`
-	// SymflowerBinaryPath overwites the Symflower binary path.
+	// SymflowerBinaryPath overwrites the Symflower binary path.
 	SymflowerBinaryPath string `long:"symflower-binary-path" description:"Overwrite the Symflower binary with this specific path instead of installing and using a global one." env:"SYMFLOWER_BINARY_PATH"`
 
 	// Languages determines which language should be used for the evaluation, or empty if all languages should be used.

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -45,8 +45,8 @@ func TestEvaluateExecute(t *testing.T) {
 	type testCase struct {
 		Name string
 
-		Before func(t *testing.T, resultPath string)
-		After  func(t *testing.T, resultPath string)
+		Before func(t *testing.T, logger *log.Logger, resultPath string)
+		After  func(t *testing.T, logger *log.Logger, resultPath string)
 
 		Arguments []string
 
@@ -59,20 +59,20 @@ func TestEvaluateExecute(t *testing.T) {
 		t.Run(tc.Name, func(t *testing.T) {
 			temporaryPath := t.TempDir()
 
-			if tc.Before != nil {
-				tc.Before(t, temporaryPath)
-			}
-			if tc.After != nil {
-				defer tc.After(t, temporaryPath)
-			}
-
 			logOutput, logger := log.Buffer()
 			defer func() {
 				if t.Failed() {
 					t.Logf("Logging output: %s", logOutput.String())
 				}
 			}()
 
+			if tc.Before != nil {
+				tc.Before(t, logger, temporaryPath)
+			}
+			if tc.After != nil {
+				defer tc.After(t, logger, temporaryPath)
+			}
+
 			arguments := append([]string{
 				"evaluate",
 				"--result-path", temporaryPath,
@@ -328,7 +328,7 @@ func TestEvaluateExecute(t *testing.T) {
 	validate(t, &testCase{
 		Name: "Current work directory contains a README.md",
 
-		Before: func(t *testing.T, resultPath string) {
+		Before: func(t *testing.T, logger *log.Logger, resultPath string) {
 			if err := os.Remove("README.md"); err != nil {
 				if osutil.IsWindows() {
 					require.Contains(t, err.Error(), "The system cannot find the file specified")
@@ -338,7 +338,7 @@ func TestEvaluateExecute(t *testing.T) {
 			}
 			require.NoError(t, os.WriteFile("README.md", []byte(""), 0644))
 		},
-		After: func(t *testing.T, resultPath string) {
+		After: func(t *testing.T, logger *log.Logger, resultPath string) {
 			require.NoError(t, os.Remove("README.md"))
 		},
 
@@ -361,15 +361,15 @@ func TestEvaluateExecute(t *testing.T) {
 	validate(t, &testCase{
 		Name: "Empty model responses are errors",
 
-		Before: func(t *testing.T, resultPath string) {
+		Before: func(t *testing.T, logger *log.Logger, resultPath string) {
 			// Setup provider and model mocking
 			modelMock := modeltesting.NewMockModelNamed(t, "empty-response")
 			providerMock := providertesting.NewMockProviderNamedWithModels(t, "testing", []model.Model{modelMock})
 			provider.Register(providerMock)
 
 			modelMock.On("GenerateTestsForFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("empty response from model"))
 		},
-		After: func(t *testing.T, resultPath string) {
+		After: func(t *testing.T, logger *log.Logger, resultPath string) {
 			delete(provider.Providers, "testing")
 		},
 

language/golang/language.go

@@ -1,6 +1,7 @@
 package golang
 
 import (
+	"context"
 	"errors"
 	"os"
 	"path/filepath"
@@ -81,7 +82,7 @@ var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statemen
 
 // Execute invokes the language specific testing on the given repository.
 func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage float64, err error) {
-	commandOutput, err := util.CommandWithResult(logger, &util.Command{
+	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			tools.SymflowerPath, "test",
 			"--language", "golang",

language/java/language.go

@@ -1,6 +1,7 @@
 package java
 
 import (
+	"context"
 	"errors"
 	"os"
 	"path/filepath"
@@ -91,7 +92,7 @@ var languageJavaCoverageMatch = regexp.MustCompile(`Total coverage (.+?)%`)
 
 // Execute invokes the language specific testing on the given repository.
 func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage float64, err error) {
-	commandOutput, err := util.CommandWithResult(logger, &util.Command{
+	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			tools.SymflowerPath, "test",
 			"--language", "java",

model/symflower/symflower.go

@@ -1,6 +1,8 @@
 package symflower
 
 import (
+	"context"
+
 	pkgerrors "github.com/pkg/errors"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
@@ -29,7 +31,7 @@ func (m *Model) ID() (id string) {
 
 // GenerateTestsForFile generates test files for the given implementation file in a repository.
 func (m *Model) GenerateTestsForFile(logger *log.Logger, language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) {
-	_, err = util.CommandWithResult(logger, &util.Command{
+	_, err = util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			tools.SymflowerPath, "unit-tests",
 			"--code-disable-fetch-dependencies",

tools/ollama.go

@@ -1,6 +1,7 @@
 package tools
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"path/filepath"
@@ -77,7 +78,7 @@ func (*ollama) Install(logger *log.Logger, installPath string) (err error) {
 
 	// Non-Windows binaries need to be made executable because the executable bit is not set for downloads.
 	if !osutil.IsWindows() {
-		if _, err := util.CommandWithResult(logger, &util.Command{
+		if _, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 			Command: []string{"chmod", "+x", ollamaInstallPath},
 		}); err != nil {
 			return pkgerrors.WithStack(pkgerrors.WithMessage(err, fmt.Sprintf("cannot make %s executable", ollamaInstallPath)))

tools/symflower.go

@@ -1,6 +1,7 @@
 package tools
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -45,7 +46,7 @@ func (*symflower) BinaryPath() string {
 
 // CheckVersion checks if the tool's version is compatible with the required version.
 func (*symflower) CheckVersion(logger *log.Logger, binaryPath string) (err error) {
-	symflowerVersionOutput, err := util.CommandWithResult(logger, &util.Command{
+	symflowerVersionOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{binaryPath, "version"},
 	})
 	if err != nil {
@@ -118,7 +119,7 @@ func (*symflower) Install(logger *log.Logger, installPath string) (err error) {
 
 	// Non-Windows binaries need to be made executable because the executable bit is not set for downloads.
 	if !osutil.IsWindows() {
-		if _, err := util.CommandWithResult(logger, &util.Command{
+		if _, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 			Command: []string{"chmod", "+x", symflowerInstallPath},
 		}); err != nil {
 			return pkgerrors.WithStack(pkgerrors.WithMessage(err, fmt.Sprintf("cannot make %s executable", symflowerInstallPath)))

util/exec.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"context"
 	"io"
 	"os/exec"
 	"strings"
@@ -21,11 +22,11 @@ type Command struct {
 }
 
 // CommandWithResult executes a command and returns its output, while printing the same output to the given logger.
-func CommandWithResult(logger *log.Logger, command *Command) (output string, err error) {
+func CommandWithResult(ctx context.Context, logger *log.Logger, command *Command) (output string, err error) {
 	logger.Printf("$ %s", strings.Join(command.Command, " "))
 
 	var writer bytesutil.SynchronizedBuffer
-	c := exec.Command(command.Command[0], command.Command[1:]...)
+	c := exec.CommandContext(ctx, command.Command[0], command.Command[1:]...)
 	if command.Directory != "" {
 		c.Dir = command.Directory
 	}