Add Tabular Diff for CSV files

modules/base/csv.go

@@ -0,0 +1,78 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+	"bytes"
+	"encoding/csv"
+	"regexp"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
+
+// CreateCsvReader creates a CSV reader with the given delimiter.
+func CreateCsvReader(rawBytes []byte, delimiter rune) *csv.Reader {
+	rd := csv.NewReader(bytes.NewReader(rawBytes))
+	rd.Comma = delimiter
+	rd.TrimLeadingSpace = true
+	return rd
+}
+
+// CreateCsvReaderAndGuessDelimiter creates a CSV reader with a guessed delimiter.
+func CreateCsvReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
+	delimiter := guessDelimiter(rawBytes)
+	return CreateCsvReader(rawBytes, delimiter)
+}
+
+// guessDelimiter scores the input CSV data against delimiters, and returns the best match.
+// Reads at most 10k bytes & 10 lines.
+func guessDelimiter(data []byte) rune {
+	maxLines := 10
+	maxBytes := util.Min(len(data), 1e4)
+	text := string(data[:maxBytes])
+	text = quoteRegexp.ReplaceAllLiteralString(text, "")
+	lines := strings.SplitN(text, "\n", maxLines+1)
+	lines = lines[:util.Min(maxLines, len(lines))]
+
+	delimiters := []rune{',', ';', '\t', '|'}
+	bestDelim := delimiters[0]
+	bestScore := 0.0
+	for _, delim := range delimiters {
+		score := scoreDelimiter(lines, delim)
+		if score > bestScore {
+			bestScore = score
+			bestDelim = delim
+		}
+	}
+
+	return bestDelim
+}
+
+// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV
+func scoreDelimiter(lines []string, delim rune) float64 {
+	countTotal := 0
+	countLineMax := 0
+	linesNotEqual := 0
+
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+
+		countLine := strings.Count(line, string(delim))
+		countTotal += countLine
+		if countLine != countLineMax {
+			if countLineMax != 0 {
+				linesNotEqual++
+			}
+			countLineMax = util.Max(countLine, countLineMax)
+		}
+	}
+
+	return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
+}

modules/base/csv_test.go

@@ -0,0 +1,40 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateCsvReader(t *testing.T) {
+	rd := CreateCsvReader([]byte{}, ',')
+	assert.Equal(t, ',', rd.Comma)
+}
+
+func TestCreateCsvReaderAndGuessDelimiter(t *testing.T) {
+	input := "a;b;c\n1;2;3\n4;5;6"
+
+	rd := CreateCsvReaderAndGuessDelimiter([]byte(input))
+	assert.Equal(t, ';', rd.Comma)
+}
+
+func TestGuessDelimiter(t *testing.T) {
+	var kases = map[string]rune{
+		"a":                         ',',
+		"1,2":                       ',',
+		"1;2":                       ';',
+		"1\t2":                      '\t',
+		"1|2":                       '|',
+		"1,2,3;4,5,6;7,8,9\na;b;c":  ';',
+		"\"1,2,3,4\";\"a\nb\"\nc;d": ';',
+		"<br/>":                     ',',
+	}
+
+	for k, v := range kases {
+		assert.EqualValues(t, guessDelimiter([]byte(k)), v)
+	}
+}

modules/markup/csv/csv.go

@@ -6,21 +6,16 @@ package markup
 
 import (
 	"bytes"
-	"encoding/csv"
 	"html"
 	"io"
-	"regexp"
-	"strings"
+	"strconv"
 
+	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/markup"
-	"code.gitea.io/gitea/modules/util"
 )
 
-var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
-
 func init() {
 	markup.RegisterParser(Parser{})
-
 }
 
 // Parser implements markup.Parser for orgmode
@@ -38,11 +33,27 @@ func (Parser) Extensions() []string {
 }
 
 // Render implements markup.Parser
-func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	rd := csv.NewReader(bytes.NewReader(rawBytes))
-	rd.Comma = p.bestDelimiter(rawBytes)
+func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+	rd := base.CreateCsvReaderAndGuessDelimiter(rawBytes)
 	var tmpBlock bytes.Buffer
-	tmpBlock.WriteString(`<table class="table">`)
+
+	writeField := func(element, class, field string) {
+		tmpBlock.WriteString("<")
+		tmpBlock.WriteString(element)
+		if len(class) > 0 {
+			tmpBlock.WriteString(" class=\"")
+			tmpBlock.WriteString(class)
+			tmpBlock.WriteString("\"")
+		}
+		tmpBlock.WriteString(">")
+		tmpBlock.WriteString(html.EscapeString(field))
+		tmpBlock.WriteString("</")
+		tmpBlock.WriteString(element)
+		tmpBlock.WriteString(">")
+	}
+
+	tmpBlock.WriteString(`<table class="data-table">`)
+	row := 1
 	for {
 		fields, err := rd.Read()
 		if err == io.EOF {
@@ -52,62 +63,19 @@ func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]strin
 			continue
 		}
 		tmpBlock.WriteString("<tr>")
+		element := "td"
+		if row == 1 {
+			element = "th"
+		}
+		writeField(element, "line-num", strconv.Itoa(row))
 		for _, field := range fields {
-			tmpBlock.WriteString("<td>")
-			tmpBlock.WriteString(html.EscapeString(field))
-			tmpBlock.WriteString("</td>")
+			writeField(element, "", field)
 		}
 		tmpBlock.WriteString("</tr>")
+
+		row++
 	}
 	tmpBlock.WriteString("</table>")
 
 	return tmpBlock.Bytes()
 }
-
-// bestDelimiter scores the input CSV data against delimiters, and returns the best match.
-// Reads at most 10k bytes & 10 lines.
-func (p Parser) bestDelimiter(data []byte) rune {
-	maxLines := 10
-	maxBytes := util.Min(len(data), 1e4)
-	text := string(data[:maxBytes])
-	text = quoteRegexp.ReplaceAllLiteralString(text, "")
-	lines := strings.SplitN(text, "\n", maxLines+1)
-	lines = lines[:util.Min(maxLines, len(lines))]
-
-	delimiters := []rune{',', ';', '\t', '|'}
-	bestDelim := delimiters[0]
-	bestScore := 0.0
-	for _, delim := range delimiters {
-		score := p.scoreDelimiter(lines, delim)
-		if score > bestScore {
-			bestScore = score
-			bestDelim = delim
-		}
-	}
-
-	return bestDelim
-}
-
-// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV
-func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) {
-	countTotal := 0
-	countLineMax := 0
-	linesNotEqual := 0
-
-	for _, line := range lines {
-		if len(line) == 0 {
-			continue
-		}
-
-		countLine := strings.Count(line, string(delim))
-		countTotal += countLine
-		if countLine != countLineMax {
-			if countLineMax != 0 {
-				linesNotEqual++
-			}
-			countLineMax = util.Max(countLine, countLineMax)
-		}
-	}
-
-	return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
-}

modules/markup/csv/csv_test.go

@@ -13,14 +13,10 @@ import (
 func TestRenderCSV(t *testing.T) {
 	var parser Parser
 	var kases = map[string]string{
-		"a":                         "<table class=\"table\"><tr><td>a</td></tr></table>",
-		"1,2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
-		"1;2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
-		"1\t2":                      "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
-		"1|2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
-		"1,2,3;4,5,6;7,8,9\na;b;c":  "<table class=\"table\"><tr><td>1,2,3</td><td>4,5,6</td><td>7,8,9</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>",
-		"\"1,2,3,4\";\"a\nb\"\nc;d": "<table class=\"table\"><tr><td>1,2,3,4</td><td>a\nb</td></tr><tr><td>c</td><td>d</td></tr></table>",
-		"<br/>":                     "<table class=\"table\"><tr><td>&lt;br/&gt;</td></tr></table>",
+		"a":        "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>",
+		"1,2":      "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>",
+		"1;2\n3;4": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr><tr><td class=\"line-num\">2</td><td>3</td><td>4</td></tr></table>",
+		"<br/>":    "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>&lt;br/&gt;</th></tr></table>",
 	}
 
 	for k, v := range kases {

modules/markup/sanitizer.go

@@ -69,6 +69,10 @@ func ReplaceSanitizer() {
 	// Allow icons, emojis, and chroma syntax on span
 	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^((icon(\s+[\p{L}\p{N}_-]+)+)|(emoji))$|^([a-z][a-z0-9]{0,2})$`)).OnElements("span")
 
+	// Allow data tables
+	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`data-table`)).OnElements("table")
+	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`line-num`)).OnElements("th", "td")
+
 	// Allow generally safe attributes
 	generalSafeAttrs := []string{"abbr", "accept", "accept-charset",
 		"accesskey", "action", "align", "alt",

options/locale/locale_en-US.ini

@@ -1813,6 +1813,7 @@ diff.whitespace_ignore_at_eol = Ignore changes in whitespace at EOL
 diff.stats_desc = <strong> %d changed files</strong> with <strong>%d additions</strong> and <strong>%d deletions</strong>
 diff.stats_desc_file = %d changes: %d additions and %d deletions
 diff.bin = BIN
+diff.bin_not_shown = Binary file not shown.
 diff.view_file = View File
 diff.file_before = Before
 diff.file_after = After

routers/repo/commit.go

@@ -335,9 +335,8 @@ func Diff(ctx *context.Context) {
 			return
 		}
 	}
-	setImageCompareContext(ctx, parentCommit, commit)
 	headTarget := path.Join(userName, repoName)
-	setPathsCompareContext(ctx, parentCommit, commit, headTarget)
+	setCompareContext(ctx, parentCommit, commit, headTarget)
 	ctx.Data["Title"] = commit.Summary() + " · " + base.ShortSha(commitID)
 	ctx.Data["Commit"] = commit
 	verification := models.ParseCommitWithSignature(commit)

routers/repo/compare.go

@@ -6,9 +6,12 @@ package repo
 
 import (
 	"bufio"
+	"encoding/csv"
 	"fmt"
 	"html"
+	"io/ioutil"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"code.gitea.io/gitea/models"
@@ -26,6 +29,16 @@ const (
 	tplBlobExcerpt base.TplName = "repo/diff/blob_excerpt"
 )
 
+// setCompareContext sets context data.
+func setCompareContext(ctx *context.Context, base *git.Commit, head *git.Commit, headTarget string) {
+	ctx.Data["BaseCommit"] = base
+	ctx.Data["HeadCommit"] = head
+
+	setPathsCompareContext(ctx, base, head, headTarget)
+	setImageCompareContext(ctx, base, head)
+	setCsvCompareContext(ctx)
+}
+
 // setPathsCompareContext sets context data for source and raw paths
 func setPathsCompareContext(ctx *context.Context, base *git.Commit, head *git.Commit, headTarget string) {
 	sourcePath := setting.AppSubURL + "/%s/src/commit/%s"
@@ -65,6 +78,49 @@ func setImageCompareContext(ctx *context.Context, base *git.Commit, head *git.Co
 	}
 }
 
+// setCsvCompareContext sets context data that is required by the CSV compare template
+func setCsvCompareContext(ctx *context.Context) {
+	ctx.Data["IsCsvFile"] = func(diffFile *gitdiff.DiffFile) bool {
+		extension := strings.ToLower(filepath.Ext(diffFile.Name))
+		return extension == ".csv" || extension == ".tsv"
+	}
+	ctx.Data["CreateCsvDiff"] = func(diffFile *gitdiff.DiffFile, baseCommit *git.Commit, headCommit *git.Commit) []*gitdiff.TableDiffSection {
+		if diffFile == nil || baseCommit == nil || headCommit == nil {
+			return nil
+		}
+
+		csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, error) {
+			blob, err := c.GetBlobByPath(diffFile.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			reader, err := blob.DataAsync()
+			if err != nil {
+				return nil, err
+			}
+			defer reader.Close()
+
+			b, err := ioutil.ReadAll(reader)
+			if err != nil {
+				return nil, err
+			}
+
+			return base.CreateCsvReaderAndGuessDelimiter(b), nil
+		}
+
+		baseReader, _ := csvReaderFromCommit(baseCommit)
+		headReader, _ := csvReaderFromCommit(headCommit)
+
+		sections, err := gitdiff.CreateCsvDiff(diffFile, baseReader, headReader)
+		if err != nil {
+			log.Error("RenderCsvDiff failed: %v", err)
+			return nil
+		}
+		return sections
+	}
+}
+
 // ParseCompareInfo parse compare info between two commit for preparing comparing references
 func ParseCompareInfo(ctx *context.Context) (*models.User, *models.Repository, *git.Repository, *git.CompareInfo, string, string) {
 	baseRepo := ctx.Repo.Repository
@@ -499,9 +555,8 @@ func PrepareCompareDiff(
 	ctx.Data["Username"] = headUser.Name
 	ctx.Data["Reponame"] = headRepo.Name
 
-	setImageCompareContext(ctx, baseCommit, headCommit)
 	headTarget := path.Join(headUser.Name, repo.Name)
-	setPathsCompareContext(ctx, baseCommit, headCommit, headTarget)
+	setCompareContext(ctx, baseCommit, headCommit, headTarget)
 
 	return false
 }