Support Windows

.github/workflows/go.yml

@@ -12,7 +12,10 @@ jobs:
     strategy:
       fail-fast: false # Run the whole matrix for maximum information, no matter if we fail in one entry early
       matrix:
-        os: ['ubuntu-latest', 'macOS-latest']
+        os:
+          - 'macOS-latest'
+          - 'ubuntu-latest'
+          - 'windows-latest'
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -39,7 +42,7 @@ jobs:
       - name: Install testing tools
         run: |
           make install-tools-testing
-          echo "PATH=$HOME/.eval-dev-quality/bin:$PATH" >> $GITHUB_ENV
+          go run scripts/gitlab-ci/main.go
 
       - name: Lint Build
         run: make lint-build-ci

cmd/eval-dev-quality/cmd/command_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
 	"github.com/symflower/eval-dev-quality/log"
@@ -28,22 +29,44 @@ func TestExecute(t *testing.T) {
 		})
 	}
 
-	validate(t, &testCase{
-		Name: "No arguments should show help",
+	if osutil.IsWindows() {
+		validate(t, &testCase{
+			Name: "No arguments should show help",
 
-		ExpectedOutput: bytesutil.StringTrimIndentations(`
-			Usage:
-			  eval-dev-quality [OPTIONS] [evaluate | install-tools]
+			ExpectedOutput: bytesutil.StringTrimIndentations(`
+				Usage:
+				  eval-dev-quality [OPTIONS] [evaluate | install-tools]
 
-			Command to manage, update and actually execute the ` + "`" + `eval-dev-quality` + "`" + `
-			evaluation benchmark.
+				Command to manage, update and actually execute the ` + "`" + `eval-dev-quality` + "`" + `
+				evaluation benchmark.
 
-			Help Options:
-			  -h, --help  Show this help message
+				Help Options:
+				  /?          Show this help message
+				  /h, /help   Show this help message
 
-			Available commands:
-			  evaluate       Run an evaluation, by default with all defined models, repositories and tasks.
-			  install-tools  Checks and installs all tools required for the evaluation benchmark.
-		`),
-	})
+				Available commands:
+				  evaluate       Run an evaluation, by default with all defined models, repositories and tasks.
+				  install-tools  Checks and installs all tools required for the evaluation benchmark.
+			`),
+		})
+	} else {
+		validate(t, &testCase{
+			Name: "No arguments should show help",
+
+			ExpectedOutput: bytesutil.StringTrimIndentations(`
+				Usage:
+				  eval-dev-quality [OPTIONS] [evaluate | install-tools]
+
+				Command to manage, update and actually execute the ` + "`" + `eval-dev-quality` + "`" + `
+				evaluation benchmark.
+
+				Help Options:
+				  -h, --help  Show this help message
+
+				Available commands:
+				  evaluate       Run an evaluation, by default with all defined models, repositories and tasks.
+				  install-tools  Checks and installs all tools required for the evaluation benchmark.
+			`),
+		})
+	}
 }

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -74,7 +74,7 @@ func TestEvaluateExecute(t *testing.T) {
 			Execute(logger, append([]string{
 				"evaluate",
 				"--result-path", temporaryPath,
-				"--testdata", "../../../testdata",
+				"--testdata", filepath.Join("..", "..", "..", "testdata"),
 			}, tc.Arguments...))
 
 			if tc.ExpectedOutputValidate != nil {
@@ -126,7 +126,7 @@ func TestEvaluateExecute(t *testing.T) {
 				"evaluation.csv": func(t *testing.T, filePath, data string) {
 					assert.Equal(t, bytesutil.StringTrimIndentations(`
 						model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-with-code
-						symflower/symbolic-execution,golang,golang/plain,14,10,1,1,1,1
+						symflower/symbolic-execution,golang,`+filepath.Join("golang", "plain")+`,14,10,1,1,1,1
 					`), data)
 				},
 				"evaluation.log": nil,
@@ -145,7 +145,7 @@ func TestEvaluateExecute(t *testing.T) {
 				"README.md": func(t *testing.T, filePath, data string) {
 					validateReportLinks(t, data, []string{"symflower_symbolic-execution"})
 				},
-				"symflower_symbolic-execution/golang/golang/plain.log": nil,
+				filepath.Join("symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
 			},
 		})
 		validate(t, &testCase{
@@ -168,8 +168,8 @@ func TestEvaluateExecute(t *testing.T) {
 				"evaluation.csv": func(t *testing.T, filePath, data string) {
 					assert.Equal(t, bytesutil.StringTrimIndentations(`
 						model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-with-code
-						symflower/symbolic-execution,golang,golang/plain,14,10,1,1,1,1
-						symflower/symbolic-execution,java,java/plain,14,10,1,1,1,1
+						symflower/symbolic-execution,golang,`+filepath.Join("golang", "plain")+`,14,10,1,1,1,1
+						symflower/symbolic-execution,java,`+filepath.Join("java", "plain")+`,14,10,1,1,1,1
 					`), data)
 				},
 				"golang-summed.csv": func(t *testing.T, filePath, data string) {
@@ -194,8 +194,8 @@ func TestEvaluateExecute(t *testing.T) {
 				"README.md": func(t *testing.T, filePath, data string) {
 					validateReportLinks(t, data, []string{"symflower_symbolic-execution"})
 				},
-				"symflower_symbolic-execution/golang/golang/plain.log": nil,
-				"symflower_symbolic-execution/java/java/plain.log":     nil,
+				filepath.Join("symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
+				filepath.Join("symflower_symbolic-execution", "java", "java", "plain.log"):     nil,
 			},
 		})
 	})
@@ -208,7 +208,7 @@ func TestEvaluateExecute(t *testing.T) {
 				Arguments: []string{
 					"--language", "golang",
 					"--model", "symflower/symbolic-execution",
-					"--repository", "golang/plain",
+					"--repository", filepath.Join("golang", "plain"),
 				},
 
 				ExpectedOutputValidate: func(t *testing.T, output string, resultPath string) {
@@ -224,7 +224,7 @@ func TestEvaluateExecute(t *testing.T) {
 					"evaluation.csv": func(t *testing.T, filePath, data string) {
 						assert.Equal(t, bytesutil.StringTrimIndentations(`
 							model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-with-code
-							symflower/symbolic-execution,golang,golang/plain,14,10,1,1,1,1
+							symflower/symbolic-execution,golang,`+filepath.Join("golang", "plain")+`,14,10,1,1,1,1
 						`), data)
 					},
 					"evaluation.log": nil,
@@ -243,15 +243,15 @@ func TestEvaluateExecute(t *testing.T) {
 					"README.md": func(t *testing.T, filePath, data string) {
 						validateReportLinks(t, data, []string{"symflower_symbolic-execution"})
 					},
-					"symflower_symbolic-execution/golang/golang/plain.log": nil,
+					filepath.Join("symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
 				},
 			})
 			validate(t, &testCase{
 				Name: "Multiple Languages",
 
 				Arguments: []string{
 					"--model", "symflower/symbolic-execution",
-					"--repository", "golang/plain",
+					"--repository", filepath.Join("golang", "plain"),
 				},
 
 				ExpectedOutputValidate: func(t *testing.T, output string, resultPath string) {
@@ -267,7 +267,7 @@ func TestEvaluateExecute(t *testing.T) {
 					"evaluation.csv": func(t *testing.T, filePath, data string) {
 						assert.Equal(t, bytesutil.StringTrimIndentations(`
 							model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-with-code
-							symflower/symbolic-execution,golang,golang/plain,14,10,1,1,1,1
+							symflower/symbolic-execution,golang,`+filepath.Join("golang", "plain")+`,14,10,1,1,1,1
 						`), data)
 					},
 					"evaluation.log": nil,
@@ -286,7 +286,7 @@ func TestEvaluateExecute(t *testing.T) {
 					"README.md": func(t *testing.T, filePath, data string) {
 						validateReportLinks(t, data, []string{"symflower_symbolic-execution"})
 					},
-					"symflower_symbolic-execution/golang/golang/plain.log": nil,
+					filepath.Join("symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
 				},
 			})
 		})
@@ -298,7 +298,11 @@ func TestEvaluateExecute(t *testing.T) {
 
 		Before: func(t *testing.T, resultPath string) {
 			if err := os.Remove("README.md"); err != nil {
-				require.Contains(t, err.Error(), "no such file or directory")
+				if osutil.IsWindows() {
+					require.Contains(t, err.Error(), "The system cannot find the file specified")
+				} else {
+					require.Contains(t, err.Error(), "no such file or directory")
+				}
 			}
 			require.NoError(t, os.WriteFile("README.md", []byte(""), 0644))
 		},
@@ -318,7 +322,7 @@ func TestEvaluateExecute(t *testing.T) {
 			"golang-summed.csv": nil,
 			"models-summed.csv": nil,
 			"README.md":         nil,
-			"symflower_symbolic-execution/golang/golang/plain.log": nil,
+			filepath.Join("symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
 		},
 	})
 
@@ -349,7 +353,7 @@ func TestEvaluateExecute(t *testing.T) {
 			"evaluation.csv": func(t *testing.T, filePath, data string) {
 				assert.Equal(t, bytesutil.StringTrimIndentations(`
 					model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-with-code
-					empty-response,golang,golang/plain,0,0,0,0,0,0
+					empty-response,golang,`+filepath.Join("golang", "plain")+`,0,0,0,0,0,0
 				`), data)
 			},
 			"evaluation.log": nil,
@@ -368,7 +372,7 @@ func TestEvaluateExecute(t *testing.T) {
 			"README.md": func(t *testing.T, filePath, data string) {
 				validateReportLinks(t, data, []string{"empty-response"})
 			},
-			"empty-response/golang/golang/plain.log": nil,
+			filepath.Join("empty-response", "golang", "golang", "plain.log"): nil,
 		},
 	})
 }

cmd/eval-dev-quality/cmd/install-tools_test.go

@@ -8,19 +8,24 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/zimmski/osutil"
 
 	"github.com/symflower/eval-dev-quality/log"
 )
 
 func TestInstallToolsExecute(t *testing.T) {
 	temporaryPath := t.TempDir()
 
-	chmodPath, err := exec.LookPath("chmod")
-	require.NoError(t, err)
-	t.Setenv("PATH", strings.Join([]string{temporaryPath, filepath.Dir(chmodPath)}, string(os.PathListSeparator)))
+	if osutil.IsWindows() {
+		t.Setenv("PATH", temporaryPath)
+	} else {
+		chmodPath, err := exec.LookPath("chmod")
+		require.NoError(t, err)
+		t.Setenv("PATH", strings.Join([]string{temporaryPath, filepath.Dir(chmodPath)}, string(os.PathListSeparator)))
+	}
 
 	t.Run("Tools are not yet installed", func(t *testing.T) {
-		symflowerPath, err := exec.LookPath("symflower")
+		symflowerPath, err := exec.LookPath("symflower" + osutil.BinaryExtension())
 		require.Error(t, err)
 		require.Empty(t, symflowerPath)
 	})
@@ -33,7 +38,7 @@ func TestInstallToolsExecute(t *testing.T) {
 		})
 
 		require.Contains(t, logOutput.String(), `Install "symflower" to`)
-		symflowerPath, err := exec.LookPath("symflower")
+		symflowerPath, err := exec.LookPath("symflower" + osutil.BinaryExtension())
 		require.NoError(t, err)
 		require.NotEmpty(t, symflowerPath)
 	})
@@ -46,7 +51,7 @@ func TestInstallToolsExecute(t *testing.T) {
 		})
 
 		require.NotContains(t, logOutput.String(), `Install "symflower" to`)
-		symflowerPath, err := exec.LookPath("symflower")
+		symflowerPath, err := exec.LookPath("symflower" + osutil.BinaryExtension())
 		require.NoError(t, err)
 		require.NotEmpty(t, symflowerPath)
 	})

evaluate/report/markdown.go

@@ -1,6 +1,7 @@
 package report
 
 import (
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -11,10 +12,11 @@ import (
 
 	pkgerrors "github.com/pkg/errors"
 	"github.com/wcharczuk/go-chart/v2"
+	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
-	"github.com/symflower/eval-dev-quality/provider"
+	"github.com/symflower/eval-dev-quality/model"
 )
 
 // Markdown holds the values for exporting a Markdown report.
@@ -49,12 +51,17 @@ type markdownTemplateContext struct {
 
 // ModelLogName formats a model name to match the logging structure.
 func (c markdownTemplateContext) ModelLogName(modelName string) string {
-	modelPath := filepath.Join(c.ModelLogsPath, strings.ReplaceAll(modelName, provider.ProviderModelSeparator, "_")) + string(os.PathSeparator)
+	modelPath := filepath.Join(c.ModelLogsPath, model.CleanModelNameForFileSystem(modelName)) + string(os.PathSeparator)
 	if !filepath.IsAbs(modelPath) {
 		// Ensure we reference the models relative to the Markdown file itself.
 		modelPath = "." + string(os.PathSeparator) + modelPath
 	}
 
+	if osutil.IsWindows() {
+		// Markdown should be able to handle "/" for file paths.
+		modelPath = strings.ReplaceAll(modelPath, "\\", "/")
+	}
+
 	return modelPath
 }
 
@@ -156,8 +163,13 @@ func (m Markdown) format(writer io.Writer, markdownFileDirectoryPath string) err
 		return pkgerrors.WithStack(err)
 	}
 	defer func() {
-		if err := svgFile.Close(); err != nil {
-			panic(err)
+		if e := svgFile.Close(); e != nil {
+			e = pkgerrors.WithStack(e)
+			if err == nil {
+				err = e
+			} else {
+				err = errors.Join(err, e)
+			}
 		}
 	}()
 
@@ -183,6 +195,16 @@ func (m Markdown) WriteToFile(path string) (err error) {
 	if err != nil {
 		return pkgerrors.WithStack(err)
 	}
+	defer func() {
+		if e := file.Close(); e != nil {
+			e = pkgerrors.WithStack(e)
+			if err == nil {
+				err = e
+			} else {
+				err = errors.Join(err, e)
+			}
+		}
+	}()
 
 	if err := m.format(file, filepath.Dir(path)); err != nil {
 		return pkgerrors.WithStack(err)

evaluate/report/markdown_test.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
@@ -42,7 +44,11 @@ func TestMarkdownWriteToFile(t *testing.T) {
 			assert.NoError(t, err)
 			expectedSVGContent, err := os.ReadFile(tc.ExpectedSVGFile)
 			require.NoError(t, err)
-			assert.Equal(t, string(expectedSVGContent), string(actualSVGContent))
+			expectedSVGContentCleaned := string(expectedSVGContent)
+			if osutil.IsWindows() {
+				expectedSVGContentCleaned = strings.ReplaceAll(expectedSVGContentCleaned, "\r", "")
+			}
+			assert.Equal(t, expectedSVGContentCleaned, string(actualSVGContent))
 		})
 	}
 
@@ -173,7 +179,11 @@ func TestBarChartModelsPerCategoriesSVG(t *testing.T) {
 
 			expectedSVGContent, err := os.ReadFile(tc.ExpectedFile)
 			require.NoError(t, err)
-			assert.Equal(t, string(expectedSVGContent), actualSVGContent.String())
+			expectedSVGContentCleaned := string(expectedSVGContent)
+			if osutil.IsWindows() {
+				expectedSVGContentCleaned = strings.ReplaceAll(expectedSVGContentCleaned, "\r", "")
+			}
+			assert.Equal(t, expectedSVGContentCleaned, actualSVGContent.String())
 		})
 	}