Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main'
Browse files Browse the repository at this point in the history
* upstream/main:
  Display image digest for container packages (go-gitea#21170)
  Use correct branch for .editorconfig error (go-gitea#21152)
  Passing command line arguments correctly by string slice (go-gitea#21168)
  Sort branches and tags by date descending (go-gitea#21136)
  Skip dirty check for team forms (go-gitea#21154)
  Add KaTeX rendering to Markdown. (go-gitea#20571)
  • Loading branch information
zjjhot committed Sep 15, 2022
2 parents 3569321 + ef40324 commit 46e0597
Show file tree
Hide file tree
Showing 36 changed files with 1,101 additions and 195 deletions.
3 changes: 3 additions & 0 deletions custom/conf/app.example.ini
Original file line number Diff line number Diff line change
Expand Up @@ -1262,6 +1262,9 @@ ROUTER = console
;; List of file extensions that should be rendered/edited as Markdown
;; Separate the extensions with a comma. To render files without any extension as markdown, just put a comma
;FILE_EXTENSIONS = .md,.markdown,.mdown,.mkd
;;
;; Enables math inline and block detection
;ENABLE_MATH = true

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand Down
1 change: 1 addition & 0 deletions docs/content/doc/advanced/config-cheat-sheet.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
- `CUSTOM_URL_SCHEMES`: Use a comma separated list (ftp,git,svn) to indicate additional
URL hyperlinks to be rendered in Markdown. URLs beginning in http and https are
always displayed
- `ENABLE_MATH`: **true**: Enables detection of `\(...\)`, `\[...\]`, `$...$` and `$$...$$` blocks as math blocks.

## Server (`server`)

Expand Down
3 changes: 2 additions & 1 deletion docs/content/doc/advanced/external-renderers.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,13 @@ RENDER_COMMAND = "timeout 30s pandoc +RTS -M512M -RTS -f rst"
IS_INPUT_FILE = false
```

If your external markup relies on additional classes and attributes on the generated HTML elements, you might need to enable custom sanitizer policies. Gitea uses the [`bluemonday`](https://godoc.org/github.com/microcosm-cc/bluemonday) package as our HTML sanitizier. The example below will support [KaTeX](https://katex.org/) output from [`pandoc`](https://pandoc.org/).
If your external markup relies on additional classes and attributes on the generated HTML elements, you might need to enable custom sanitizer policies. Gitea uses the [`bluemonday`](https://godoc.org/github.com/microcosm-cc/bluemonday) package as our HTML sanitizer. The example below could be used to support server-side [KaTeX](https://katex.org/) rendering output from [`pandoc`](https://pandoc.org/).

```ini
[markup.sanitizer.TeX]
; Pandoc renders TeX segments as <span>s with the "math" class, optionally
; with "inline" or "display" classes depending on context.
; - note this is different from the built-in math support in our markdown parser which uses <code>
ELEMENT = span
ALLOW_ATTR = class
REGEXP = ^\s*((math(\s+|$)|inline(\s+|$)|display(\s+|$)))+
Expand Down
2 changes: 2 additions & 0 deletions docs/content/doc/features/comparison.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ _Symbols used in table:_
| WebAuthn (2FA) ||||||| ? |
| Built-in CI/CD ||||||||
| Subgroups: groups within groups | [](https://github.com/go-gitea/gitea/issues/1872) |||||||
| Mermaid diagrams in Markdown ||||||||
| Math syntax in Markdown ||||||||

## Code management

Expand Down
3 changes: 2 additions & 1 deletion docs/content/page/index.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ You can try it out using [the online demo](https://try.gitea.io/).
- Environment variables
- Command line options
- Multi-language support ([21 languages](https://github.com/go-gitea/gitea/tree/main/options/locale))
- [Mermaid](https://mermaidjs.github.io/) Diagram support
- [Mermaid](https://mermaidjs.github.io/) diagrams in Markdown
- Math syntax in Markdown
- Mail service
- Notifications
- Registration confirmation
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ require (
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
mvdan.cc/xurls/v2 v2.4.0
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
xorm.io/builder v0.3.11
Expand Down Expand Up @@ -290,7 +291,6 @@ require (
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)

Expand Down
30 changes: 13 additions & 17 deletions modules/git/repo_branch_nogogit.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,42 +63,40 @@ func (repo *Repository) IsBranchExist(name string) bool {
// GetBranchNames returns branches from the repository, skipping skip initial branches and
// returning at most limit branches, or all branches if limit is 0.
func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) {
return callShowRef(repo.Ctx, repo.Path, BranchPrefix, "--heads", skip, limit)
return callShowRef(repo.Ctx, repo.Path, BranchPrefix, []string{BranchPrefix, "--sort=-committerdate"}, skip, limit)
}

// WalkReferences walks all the references from the repository
func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) {
return walkShowRef(ctx, repoPath, "", 0, 0, walkfn)
return walkShowRef(ctx, repoPath, nil, 0, 0, walkfn)
}

// WalkReferences walks all the references from the repository
// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty.
func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) {
var arg string
var args []string
switch refType {
case ObjectTag:
arg = "--tags"
args = []string{TagPrefix, "--sort=-taggerdate"}
case ObjectBranch:
arg = "--heads"
default:
arg = ""
args = []string{BranchPrefix, "--sort=-committerdate"}
}

return walkShowRef(repo.Ctx, repo.Path, arg, skip, limit, walkfn)
return walkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn)
}

// callShowRef return refs, if limit = 0 it will not limit
func callShowRef(ctx context.Context, repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) {
countAll, err = walkShowRef(ctx, repoPath, arg, skip, limit, func(_, branchName string) error {
branchName = strings.TrimPrefix(branchName, prefix)
func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs []string, skip, limit int) (branchNames []string, countAll int, err error) {
countAll, err = walkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error {
branchName = strings.TrimPrefix(branchName, trimPrefix)
branchNames = append(branchNames, branchName)

return nil
})
return branchNames, countAll, err
}

func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) {
func walkShowRef(ctx context.Context, repoPath string, extraArgs []string, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
Expand All @@ -107,10 +105,8 @@ func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, wal

go func() {
stderrBuilder := &strings.Builder{}
args := []string{"show-ref"}
if arg != "" {
args = append(args, arg)
}
args := []string{"for-each-ref", "--format=%(objectname) %(refname)"}
args = append(args, extraArgs...)
err := NewCommand(ctx, args...).Run(&RunOpts{
Dir: repoPath,
Stdout: stdoutWriter,
Expand Down Expand Up @@ -194,7 +190,7 @@ func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, wal
// GetRefsBySha returns all references filtered with prefix that belong to a sha commit hash
func (repo *Repository) GetRefsBySha(sha, prefix string) ([]string, error) {
var revList []string
_, err := walkShowRef(repo.Ctx, repo.Path, "", 0, 0, func(walkSha, refname string) error {
_, err := walkShowRef(repo.Ctx, repo.Path, nil, 0, 0, func(walkSha, refname string) error {
if walkSha == sha && strings.HasPrefix(refname, prefix) {
revList = append(revList, refname)
}
Expand Down
4 changes: 2 additions & 2 deletions modules/git/repo_branch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ func TestRepository_GetBranches(t *testing.T) {
assert.NoError(t, err)
assert.Len(t, branches, 2)
assert.EqualValues(t, 3, countAll)
assert.ElementsMatch(t, []string{"branch1", "branch2"}, branches)
assert.ElementsMatch(t, []string{"master", "branch2"}, branches)

branches, countAll, err = bareRepo1.GetBranchNames(0, 0)

assert.NoError(t, err)
assert.Len(t, branches, 3)
assert.EqualValues(t, 3, countAll)
assert.ElementsMatch(t, []string{"branch1", "branch2", "master"}, branches)
assert.ElementsMatch(t, []string{"master", "branch2", "branch1"}, branches)

branches, countAll, err = bareRepo1.GetBranchNames(5, 1)

Expand Down
2 changes: 1 addition & 1 deletion modules/git/repo_tag_nogogit.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (repo *Repository) IsTagExist(name string) bool {
// GetTags returns all tags of the repository.
// returning at most limit tags, or all if limit is 0.
func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) {
tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, "--tags", skip, limit)
tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, []string{TagPrefix, "--sort=-taggerdate"}, skip, limit)
return tags, err
}

Expand Down
84 changes: 84 additions & 0 deletions modules/markup/markdown/convertyaml.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package markdown

import (
"github.com/yuin/goldmark/ast"
east "github.com/yuin/goldmark/extension/ast"
"gopkg.in/yaml.v3"
)

func nodeToTable(meta *yaml.Node) ast.Node {
for {
if meta == nil {
return nil
}
switch meta.Kind {
case yaml.DocumentNode:
meta = meta.Content[0]
continue
default:
}
break
}
switch meta.Kind {
case yaml.MappingNode:
return mappingNodeToTable(meta)
case yaml.SequenceNode:
return sequenceNodeToTable(meta)
default:
return ast.NewString([]byte(meta.Value))
}
}

func mappingNodeToTable(meta *yaml.Node) ast.Node {
table := east.NewTable()
alignments := []east.Alignment{}
for i := 0; i < len(meta.Content); i += 2 {
alignments = append(alignments, east.AlignNone)
}

headerRow := east.NewTableRow(alignments)
valueRow := east.NewTableRow(alignments)
for i := 0; i < len(meta.Content); i += 2 {
cell := east.NewTableCell()

cell.AppendChild(cell, nodeToTable(meta.Content[i]))
headerRow.AppendChild(headerRow, cell)

if i+1 < len(meta.Content) {
cell = east.NewTableCell()
cell.AppendChild(cell, nodeToTable(meta.Content[i+1]))
valueRow.AppendChild(valueRow, cell)
}
}

table.AppendChild(table, east.NewTableHeader(headerRow))
table.AppendChild(table, valueRow)
return table
}

func sequenceNodeToTable(meta *yaml.Node) ast.Node {
table := east.NewTable()
alignments := []east.Alignment{east.AlignNone}
for _, item := range meta.Content {
row := east.NewTableRow(alignments)
cell := east.NewTableCell()
cell.AppendChild(cell, nodeToTable(item))
row.AppendChild(row, cell)
table.AppendChild(table, row)
}
return table
}

func nodeToDetails(meta *yaml.Node, icon string) ast.Node {
details := NewDetails()
summary := NewSummary()
summary.AppendChild(summary, NewIcon(icon))
details.AppendChild(details, summary)
details.AppendChild(details, nodeToTable(meta))

return details
}
15 changes: 3 additions & 12 deletions modules/markup/markdown/goldmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"code.gitea.io/gitea/modules/setting"
giteautil "code.gitea.io/gitea/modules/util"

meta "github.com/yuin/goldmark-meta"
"github.com/yuin/goldmark/ast"
east "github.com/yuin/goldmark/extension/ast"
"github.com/yuin/goldmark/parser"
Expand All @@ -32,20 +31,12 @@ type ASTTransformer struct{}

// Transform transforms the given AST tree.
func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
metaData := meta.GetItems(pc)
firstChild := node.FirstChild()
createTOC := false
ctx := pc.Get(renderContextKey).(*markup.RenderContext)
rc := &RenderConfig{
Meta: "table",
Icon: "table",
Lang: "",
}

if metaData != nil {
rc.ToRenderConfig(metaData)

metaNode := rc.toMetaNode(metaData)
rc := pc.Get(renderConfigKey).(*RenderConfig)
if rc.yamlNode != nil {
metaNode := rc.toMetaNode()
if metaNode != nil {
node.InsertBefore(node, firstChild, metaNode)
}
Expand Down
20 changes: 18 additions & 2 deletions modules/markup/markdown/markdown.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/common"
"code.gitea.io/gitea/modules/markup/markdown/math"
"code.gitea.io/gitea/modules/setting"
giteautil "code.gitea.io/gitea/modules/util"

Expand All @@ -38,6 +39,7 @@ var (
isWikiKey = parser.NewContextKey()
renderMetasKey = parser.NewContextKey()
renderContextKey = parser.NewContextKey()
renderConfigKey = parser.NewContextKey()
)

type limitWriter struct {
Expand Down Expand Up @@ -98,7 +100,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
languageStr := string(language)

preClasses := []string{"code-block"}
if languageStr == "mermaid" {
if languageStr == "mermaid" || languageStr == "math" {
preClasses = append(preClasses, "is-loading")
}

Expand All @@ -120,6 +122,9 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
}
}),
),
math.NewExtension(
math.Enabled(setting.Markdown.EnableMath),
),
meta.Meta,
),
goldmark.WithParserOptions(
Expand Down Expand Up @@ -167,7 +172,18 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
log.Error("Unable to ReadAll: %v", err)
return err
}
if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
buf = giteautil.NormalizeEOL(buf)

rc := &RenderConfig{
Meta: "table",
Icon: "table",
Lang: "",
}
buf, _ = ExtractMetadataBytes(buf, rc)

pc.Set(renderConfigKey, rc)

if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil {
log.Error("Unable to render: %v", err)
return err
}
Expand Down
42 changes: 42 additions & 0 deletions modules/markup/markdown/math/block_node.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package math

import "github.com/yuin/goldmark/ast"

// Block represents a display math block e.g. $$...$$ or \[...\]
type Block struct {
ast.BaseBlock
Dollars bool
Indent int
Closed bool
}

// KindBlock is the node kind for math blocks
var KindBlock = ast.NewNodeKind("MathBlock")

// NewBlock creates a new math Block
func NewBlock(dollars bool, indent int) *Block {
return &Block{
Dollars: dollars,
Indent: indent,
}
}

// Dump dumps the block to a string
func (n *Block) Dump(source []byte, level int) {
m := map[string]string{}
ast.DumpHelper(n, source, level, m, nil)
}

// Kind returns KindBlock for math Blocks
func (n *Block) Kind() ast.NodeKind {
return KindBlock
}

// IsRaw returns true as this block should not be processed further
func (n *Block) IsRaw() bool {
return true
}
Loading

0 comments on commit 46e0597

Please sign in to comment.