Fix some RenderShortcodes error cases

This issue fixes two cases where `{{__hugo_ctx` artifacts were left in the rendered output:

1. Inclusion when `.RenderShortcodes` is wrapped in HTML.
2. Inclusion of Markdown file without a trailing newline in some cases.

Closes gohugoio#12854
Updates gohugoio#12998

common/constants/constants.go

@@ -21,6 +21,7 @@ const (
 	ErrRemoteGetCSV  = "error-remote-getcsv"
 
 	WarnFrontMatterParamsOverrides = "warning-frontmatter-params-overrides"
+	WarnRenderShortcodesInHTML     = "warning-rendershortcodes-in-html"
 )
 
 // Field/method names with special meaning.

hugolib/collections_test.go

@@ -71,9 +71,9 @@ tags_weight: %d
 {{ $pageGroups := slice $cool $blue }}
 {{ $weighted := slice $wp1 $wp2 }}
 
-{{ printf "pages:%d:%T:%v/%v" (len $pages) $pages (index $pages 0) (index $pages 1) }}
-{{ printf "pageGroups:%d:%T:%v/%v" (len $pageGroups) $pageGroups (index (index $pageGroups 0).Pages 0) (index (index $pageGroups 1).Pages 0)}}
-{{ printf "weightedPages:%d::%T:%v" (len $weighted) $weighted $weighted | safeHTML }}
+{{ printf "pages:%d:%T:%s|%s" (len $pages) $pages (index $pages 0).Path (index $pages 1).Path }}
+{{ printf "pageGroups:%d:%T:%s|%s" (len $pageGroups) $pageGroups (index (index $pageGroups 0).Pages 0).Path (index (index $pageGroups 1).Pages 0).Path}}
+{{ printf "weightedPages:%d:%T" (len $weighted) $weighted | safeHTML }}
 
 `)
 	b.CreateSites().Build(BuildCfg{})
@@ -82,9 +82,9 @@ tags_weight: %d
 	c.Assert(len(b.H.Sites[0].RegularPages()), qt.Equals, 2)
 
 	b.AssertFileContent("public/index.html",
-		"pages:2:page.Pages:Page(/page1)/Page(/page2)",
-		"pageGroups:2:page.PagesGroup:Page(/page1)/Page(/page2)",
-		`weightedPages:2::page.WeightedPages:[WeightedPage(10,"Page") WeightedPage(20,"Page")]`)
+		"pages:2:page.Pages:/page1|/page2",
+		"pageGroups:2:page.PagesGroup:/page1|/page2",
+		`weightedPages:2:page.WeightedPages`)
 }
 
 func TestUnionFunc(t *testing.T) {
@@ -189,7 +189,7 @@ tags_weight: %d
 {{ $appendStrings := slice "a" "b" | append "c" "d" "e" }}
 {{ $appendStringsSlice := slice "a" "b" "c" | append (slice "c" "d") }}
 
-{{ printf "pages:%d:%T:%v/%v" (len $pages) $pages (index $pages 0) (index $pages 1)  }}
+{{ printf "pages:%d:%T:%s|%s" (len $pages) $pages (index $pages 0).Path (index $pages 1).Path  }}
 {{ printf "appendPages:%d:%T:%v/%v" (len $appendPages) $appendPages (index $appendPages 0).Kind (index $appendPages 8).Kind  }}
 {{ printf "appendStrings:%T:%v"  $appendStrings $appendStrings  }}
 {{ printf "appendStringsSlice:%T:%v"  $appendStringsSlice $appendStringsSlice }}
@@ -207,7 +207,7 @@ tags_weight: %d
 	c.Assert(len(b.H.Sites[0].RegularPages()), qt.Equals, 2)
 
 	b.AssertFileContent("public/index.html",
-		"pages:2:page.Pages:Page(/page2)/Page(/page1)",
+		"pages:2:page.Pages:/page2|/page1",
 		"appendPages:9:page.Pages:home/page",
 		"appendStrings:[]string:[a b c d e]",
 		"appendStringsSlice:[]string:[a b c c d]",

hugolib/integrationtest_builder.go

@@ -2,6 +2,7 @@ package hugolib
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -32,6 +33,7 @@ import (
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/afero"
+	"github.com/spf13/cast"
 	"golang.org/x/text/unicode/norm"
 	"golang.org/x/tools/txtar"
 )
@@ -294,6 +296,12 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s
 	}
 }
 
+func (s *IntegrationTestBuilder) AssertFileContentEquals(filename string, match string) {
+	s.Helper()
+	content := s.FileContent(filename)
+	s.Assert(content, qt.Equals, match, qt.Commentf(match))
+}
+
 func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) {
 	s.Helper()
 	content := s.FileContent(filename)
@@ -302,6 +310,16 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches
 	}
 }
 
+func (s *IntegrationTestBuilder) AssertNoRenderShortcodesArtifacts() {
+	s.Helper()
+	for _, p := range s.H.Pages() {
+		content, err := p.Content(context.Background())
+		s.Assert(err, qt.IsNil)
+		comment := qt.Commentf("Page: %s\n%s", p.Path(), content)
+		s.Assert(strings.Contains(cast.ToString(content), "__hugo_ctx"), qt.IsFalse, comment)
+	}
+}
+
 func (s *IntegrationTestBuilder) AssertPublishDir(matches ...string) {
 	s.AssertFs(s.fs.PublishDir, matches...)
 }
@@ -835,6 +853,11 @@ type IntegrationTestConfig struct {
 
 	// The files to use on txtar format, see
 	// https://pkg.go.dev/golang.org/x/exp/cmd/txtar
+	// There are some conentions used in this test setup.
+	// - §§§ can be used to wrap code fences.
+	// - §§ can be used to wrap multiline strings.
+	// - filenames prefixed with sourcefilename: will be read from the file system relative to the current dir.
+	// - filenames with a .png or .jpg extension will be treated as binary and base64 decoded.
 	TxtarString string
 
 	// COnfig to use as the base. We will also read the config from the txtar.

hugolib/menu_test.go

@@ -105,94 +105,6 @@ Menu Main:  {{ partial "menu.html" (dict "page" . "menu" "main") }}`,
 			"/sect3/|Sect3s|Sect3s|0|-|-|")
 }
 
-// related issue #7594
-func TestMenusSort(t *testing.T) {
-	b := newTestSitesBuilder(t).WithSimpleConfigFile()
-
-	b.WithTemplatesAdded("index.html", `
-{{ range $k, $v := .Site.Menus.main }}
-Default1|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-{{ range $k, $v := .Site.Menus.main.ByWeight }}
-ByWeight|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-{{ range $k, $v := (.Site.Menus.main.ByWeight).Reverse }}
-Reverse|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-{{ range $k, $v := .Site.Menus.main }}
-Default2|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-{{ range $k, $v := .Site.Menus.main.ByWeight }}
-ByWeight|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-{{ range $k, $v := .Site.Menus.main }}
-Default3|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }}
-`)
-
-	b.WithContent("_index.md", `
----
-title: Home
-menu:
-  main:
-    weight: 100
----`)
-
-	b.WithContent("blog/A.md", `
----
-title: "A"
-menu:
-  main:
-    weight: 10
----
-`)
-
-	b.WithContent("blog/B.md", `
----
-title: "B"
-menu:
-  main:
-    weight: 20
----
-`)
-	b.WithContent("blog/C.md", `
----
-title: "C"
-menu:
-  main:
-    weight: 30
----
-`)
-
-	b.Build(BuildCfg{})
-
-	b.AssertFileContent("public/index.html",
-		`Default1|0|10|A|/blog/a/|Page(/blog/a)
-        Default1|1|20|B|/blog/b/|Page(/blog/b)
-        Default1|2|30|C|/blog/c/|Page(/blog/c)
-        Default1|3|100|Home|/|Page(/)
-
-        ByWeight|0|10|A|/blog/a/|Page(/blog/a)
-        ByWeight|1|20|B|/blog/b/|Page(/blog/b)
-        ByWeight|2|30|C|/blog/c/|Page(/blog/c)
-        ByWeight|3|100|Home|/|Page(/)
-
-        Reverse|0|100|Home|/|Page(/)
-        Reverse|1|30|C|/blog/c/|Page(/blog/c)
-        Reverse|2|20|B|/blog/b/|Page(/blog/b)
-        Reverse|3|10|A|/blog/a/|Page(/blog/a)
-
-        Default2|0|10|A|/blog/a/|Page(/blog/a)
-        Default2|1|20|B|/blog/b/|Page(/blog/b)
-        Default2|2|30|C|/blog/c/|Page(/blog/c)
-        Default2|3|100|Home|/|Page(/)
-
-        ByWeight|0|10|A|/blog/a/|Page(/blog/a)
-        ByWeight|1|20|B|/blog/b/|Page(/blog/b)
-        ByWeight|2|30|C|/blog/c/|Page(/blog/c)
-        ByWeight|3|100|Home|/|Page(/)
-
-        Default3|0|10|A|/blog/a/|Page(/blog/a)
-        Default3|1|20|B|/blog/b/|Page(/blog/b)
-        Default3|2|30|C|/blog/c/|Page(/blog/c)
-        Default3|3|100|Home|/|Page(/)`,
-	)
-}
-
 func TestMenusFrontMatter(t *testing.T) {
 	b := newTestSitesBuilder(t).WithSimpleConfigFile()
 
@@ -437,8 +349,8 @@ url = "/blog/post3"
 	commonTempl := `
 Main: {{ len .Site.Menus.main }}
 {{ range .Site.Menus.main }}
-{{ .Title }}|HasMenuCurrent: {{ $.HasMenuCurrent "main" . }}|Page: {{ .Page }}
-{{ .Title }}|IsMenuCurrent: {{ $.IsMenuCurrent "main" . }}|Page: {{ .Page }}
+{{ .Title }}|HasMenuCurrent: {{ $.HasMenuCurrent "main" . }}|Page: {{ .Page.Path }}
+{{ .Title }}|IsMenuCurrent: {{ $.IsMenuCurrent "main" . }}|Page: {{ .Page.Path }}
 {{ end }}
 `
 
@@ -494,34 +406,34 @@ title: "Contact: With  No Menu Defined"
 
 	b.AssertFileContent("public/index.html", `
 Main: 5
-Home|HasMenuCurrent: false|Page: Page(/)
-Blog|HasMenuCurrent: false|Page: Page(/blog)
-My Post 2: With Menu Defined|HasMenuCurrent: false|Page: Page(/blog/post2)
-My Post 3|HasMenuCurrent: false|Page: Page(/blog/post3)
-Contact Us|HasMenuCurrent: false|Page: Page(/contact)
+Home|HasMenuCurrent: false|Page: /
+Blog|HasMenuCurrent: false|Page: /blog
+My Post 2: With Menu Defined|HasMenuCurrent: false|Page: /blog/post2
+My Post 3|HasMenuCurrent: false|Page: /blog/post3
+Contact Us|HasMenuCurrent: false|Page: /contact
 `)
 
 	b.AssertFileContent("public/blog/post1/index.html", `
-Home|HasMenuCurrent: false|Page: Page(/)
-Blog|HasMenuCurrent: true|Page: Page(/blog)
+Home|HasMenuCurrent: false|Page: /
+Blog|HasMenuCurrent: true|Page: /blog
 `)
 
 	b.AssertFileContent("public/blog/post2/index.html", `
-Home|HasMenuCurrent: false|Page: Page(/)
-Blog|HasMenuCurrent: true|Page: Page(/blog)
-Blog|IsMenuCurrent: false|Page: Page(/blog)
+Home|HasMenuCurrent: false|Page: /
+Blog|HasMenuCurrent: true|Page: /blog
+Blog|IsMenuCurrent: false|Page: /blog
 `)
 
 	b.AssertFileContent("public/blog/post3/index.html", `
-Home|HasMenuCurrent: false|Page: Page(/)
-Blog|HasMenuCurrent: true|Page: Page(/blog)
+Home|HasMenuCurrent: false|Page: /
+Blog|HasMenuCurrent: true|Page: /blog
 `)
 
 	b.AssertFileContent("public/contact/index.html", `
-Contact Us|HasMenuCurrent: false|Page: Page(/contact)
-Contact Us|IsMenuCurrent: true|Page: Page(/contact)
-Blog|HasMenuCurrent: false|Page: Page(/blog)
-Blog|IsMenuCurrent: false|Page: Page(/blog)
+Contact Us|HasMenuCurrent: false|Page: /contact
+Contact Us|IsMenuCurrent: true|Page: /contact
+Blog|HasMenuCurrent: false|Page: /blog
+Blog|IsMenuCurrent: false|Page: /blog
 `)
 }
 

hugolib/page.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -358,7 +359,18 @@ func (p *pageState) Site() page.Site {
 }
 
 func (p *pageState) String() string {
-	return fmt.Sprintf("Page(%s)", p.Path())
+	var sb strings.Builder
+	if p.File() != nil {
+		sb.WriteString(p.File().Filename())
+		if p.File().IsContentAdapter() {
+			// Also include the path.
+			sb.WriteString(":")
+			sb.WriteString(p.Path())
+		}
+	} else {
+		sb.WriteString(p.Path())
+	}
+	return sb.String()
 }
 
 // IsTranslated returns whether this content file is translated to