fix #8871 runnableExamples now preserves source code comments, litter…

…als, and all formatting

compiler/docgen.nim

@@ -17,7 +17,7 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables, nimpaths
+  pathutils, trees, tables, nimpaths, renderverbatim
 
 const
   exportSection = skField
@@ -494,8 +494,8 @@ proc runAllExamples(d: PDoc) =
       rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
       # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
-proc prepareExample(d: PDoc; n: PNode): string =
-  ## returns `rdoccmd` for this runnableExamples
+proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+  ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
   if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
   if n.len == 3:
@@ -512,10 +512,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
-
+  let ret = extractRunnableExamplesSource(d.conf, n)
   for a in n.lastSon: runnableExamples.add a
+  # we could also use `ret` instead here, to keep sources verbatim
   writeExample(d, runnableExamples, rdoccmd)
-  result = rdoccmd
+  result = (rdoccmd, ret)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -561,25 +562,30 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous
     if isRunnableExamples(n[0]) and
         n.len >= 2 and n.lastSon.kind == nkStmtList:
       previousIsRunnable = true
-      let rdoccmd = prepareExample(d, n)
+      let (rdoccmd, code) = prepareExample(d, n)
       var msg = "Example:"
       if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
           "\n\\textbf{$1}\n", [msg.rope])
       inc d.listingCounter
       let id = $d.listingCounter
       dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
-      # this is a rather hacky way to get rid of the initial indentation
-      # that the renderer currently produces:
-      var i = 0
-      var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList and
-          body.lastSon.kind == nkStmtList:
-        body = body.lastSon
-      for b in body:
-        if i > 0: dest.add "\n"
-        inc i
-        nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+      when true:
+        var dest2 = ""
+        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        dest.add dest2
+      else:
+        # this is a rather hacky way to get rid of the initial indentation
+        # that the renderer currently produces:
+        var i = 0
+        var body = n.lastSon
+        if body.len == 1 and body.kind == nkStmtList and
+            body.lastSon.kind == nkStmtList:
+          body = body.lastSon
+        for b in body:
+          if i > 0: dest.add "\n"
+          inc i
+          nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
   else: previousIsRunnable = false
 

compiler/msgs.nim

@@ -435,18 +435,21 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 

compiler/renderverbatim.nim

@@ -0,0 +1,65 @@
+import strutils
+from xmltree import addEscaped
+
+import ast, options, msgs
+import packages/docutils/highlite
+
+proc lastNodeRec(n: PNode): PNode =
+  result = n
+  while result.safeLen > 0: result = result[^1]
+
+proc isInIndentationBlock(src: string, indent: int): bool =
+  #[
+  we stop at the first de-indentation; there's an inherent ambiguity with non
+  doc comments since they can have arbitrary indentation, so we just take the
+  practical route and require a runnableExamples to keep its code (including non
+  doc comments) to its indentation level.
+  ]#
+  for j in 0..<indent:
+    if src.len <= j: return true
+    if src[j] == ' ': continue
+    return false
+  return true
+
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+  ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
+  ## we'd need to check performance impact to enable it for nimdoc.
+  let first = n.lastSon.info
+  let last = n.lastNodeRec.info
+  var ret = ""
+  var info = first
+  var indent = info.col
+  let numLines = numLines(conf, info.fileIndex).uint16
+  var lastNonemptyPos = 0
+  for line in first.line..<numLines:
+    info.line = line
+    let src = sourceLine(conf, info)
+    if line > last.line and not isInIndentationBlock(src, indent):
+      break
+    if line > first.line: ret.add "\n"
+    if src.len > indent:
+      ret.add src[indent..^1]
+      lastNonemptyPos = ret.len
+  ret = ret[0..<lastNonemptyPos]
+  return ret
+
+proc renderNimCode*(result: var string, code: string, isLatex = false) =
+  var toknizr: GeneralTokenizer
+  initGeneralTokenizer(toknizr, code)
+  var buf = ""
+  template append(kind, val) =
+    buf.setLen 0
+    buf.addEscaped(val)
+    let class = tokenClassToStr[kind]
+    if isLatex:
+      result.addf "\\span$1{$2}" % [class, buf]
+    else:
+      result.addf  "<span class=\"$1\">$2</span>" % [class, buf]
+
+  while true:
+    getNextToken(toknizr, langNim)
+    case toknizr.kind
+    of gtEof: break  # End Of File (or string)
+    else:
+      # TODO: avoid alloc; maybe toOpenArray
+      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))

nimdoc/testproject/expected/testproject.html

@@ -173,7 +173,9 @@ <h1 class="title">testproject</h1>
 <li>
   <a class="reference reference-toplevel" href="#18" id="68">Templates</a>
   <ul class="simple simple-toc-section">
-      <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
+      <li><a class="reference" href="#myfn.t"
+    title="myfn()"><wbr />myfn<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
     title="foo(a, b: SomeType)"><wbr />foo<span class="attachedType"></span></a></li>
 
   </ul>
@@ -187,13 +189,11 @@ <h1 class="title">testproject</h1>
 
   <p class="module-desc">This is the top level module.
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span>
-  <span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>
-
-<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
-<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
-<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
-  <span class="Keyword">discard</span></pre></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
+</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
+</span><span class="Comment"># bug #11078</span><span class="Whitespace">
+</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre></p>
   <div class="section" id="6">
 <h1><a class="toc-backref" href="#6">Imports</a></h1>
 <dl class="item">
@@ -331,6 +331,26 @@ <h1><a class="toc-backref" href="#17">Macros</a></h1>
 <div class="section" id="18">
 <h1><a class="toc-backref" href="#18">Templates</a></h1>
 <dl class="item">
+<a id="myfn.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#myfn.t"><span class="Identifier">myfn</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
+</span><span class="Comment">## line doc comment</span><span class="Whitespace">
+</span><span class="Comment"># bar</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
+</span><span class="LongComment">##[
+foo
+bar
+]##</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
+</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
+  </span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
+</span><span class="Comment"># should be in</span></pre>
+
+</dd>
 <a id="foo.t,SomeType,SomeType"></a>
 <dt><pre><span class="Keyword">template</span> <a href="#foo.t%2CSomeType%2CSomeType"><span class="Identifier">foo</span></a><span class="Other">(</span><span class="Identifier">a</span><span class="Other">,</span> <span class="Identifier">b</span><span class="Other">:</span> <a href="subdir/subdir_b/utils.html#SomeType"><span class="Identifier">SomeType</span></a><span class="Other">)</span></pre></dt>
 <dd>

nimdoc/testproject/expected/theindex.html

@@ -143,6 +143,10 @@ <h1 class="title">Index</h1>
 <li><a class="reference external"
           data-doc-search-tag="testproject: isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: isValid[T](x: T): bool</a></li>
           </ul></dd>
+<dt><a name="myfn" href="#myfn"><span>myfn:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: myfn()" href="testproject.html#myfn.t">testproject: myfn()</a></li>
+          </ul></dd>
 <dt><a name="someFunc" href="#someFunc"><span>someFunc:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()</a></li>

nimdoc/testproject/testproject.nim

@@ -9,6 +9,22 @@ runnableExamples:
   # bug #11078
   for x in "xx": discard
 
+template myfn*() =
+  runnableExamples:
+    import std/strutils
+    ## line doc comment
+    # bar
+    doAssert "'foo" == "'foo"
+    ##[
+    foo
+    bar
+    ]##
+    doAssert: not "foo".startsWith "ba"
+    block:
+      discard 0xff # elu par cette crapule
+    # should be in
+  # should be out
+
 const
   C_A* = 0x7FF0000000000000'f64
   C_B* = 0o377'i8