Skip to content

Commit

Permalink
fix nim-lang#8871 runnableExamples now preserves source code comments…
Browse files Browse the repository at this point in the history
…, litterals, and all formatting
  • Loading branch information
timotheecour committed May 27, 2020
1 parent cc65ae6 commit 52f8a2a
Show file tree
Hide file tree
Showing 7 changed files with 216 additions and 64 deletions.
1 change: 1 addition & 0 deletions compiler/ast.nim
Original file line number Diff line number Diff line change
Expand Up @@ -1029,6 +1029,7 @@ const
nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
nkLambdaKinds* = {nkLambda, nkDo}
declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
procDefs* = nkLambdaKinds + declarativeDefs

nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
Expand Down
139 changes: 91 additions & 48 deletions compiler/docgen.nim
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import
packages/docutils/rst, packages/docutils/rstgen,
json, xmltree, cgi, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, trees, tables, nimpaths
pathutils, trees, tables, nimpaths, renderverbatim

const
exportSection = skField
Expand Down Expand Up @@ -494,8 +494,8 @@ proc runAllExamples(d: PDoc) =
rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
# removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove

proc prepareExample(d: PDoc; n: PNode): string =
## returns `rdoccmd` for this runnableExamples
proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
## returns `rdoccmd` and source code for this runnableExamples
var rdoccmd = ""
if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
if n.len == 3:
Expand All @@ -512,10 +512,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
docComment,
newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
runnableExamples.info = n.info

let ret = extractRunnableExamplesSource(d.conf, n)
for a in n.lastSon: runnableExamples.add a
# we could also use `ret` instead here, to keep sources verbatim
writeExample(d, runnableExamples, rdoccmd)
result = rdoccmd
result = (rdoccmd, ret)
when false:
proc extractImports(n: PNode; result: PNode) =
if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
Expand All @@ -529,67 +530,110 @@ proc prepareExample(d: PDoc; n: PNode): string =
for imp in imports: runnableExamples.add imp
runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)

proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) =
proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) =
## this is a rather hacky way to get rid of the initial indentation
## that the renderer currently produces:
# deadcode
var i = 0
var body = n.lastSon
if body.len == 1 and body.kind == nkStmtList and
body.lastSon.kind == nkStmtList:
body = body.lastSon
for b in body:
if i > 0: dest.add "\n"
inc i
nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)

type RunnableState = enum
rsStart
rsComment
rsRunnable
rsDone

proc getAllRunnableExamplesImpl(d: PDoc; n, orig: PNode; dest: var Rope, state: RunnableState): RunnableState =
##[
previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though).
This is to ensure this works:
Simple state machine to tell whether we render runnableExamples and doc comments.
This is to ensure that we can interleave runnableExamples and doc comments freely;
the logic is easy to change but currently a doc comment following another doc comment
will not render, to avoid rendering in following case:
proc fn* =
runnableExamples: discard
## d1
runnableExamples: discard
## d2
## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
It also works with:
proc fn* =
## d0
runnableExamples: discard
## d1
etc
## internal explanation # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
discard # some code
]##
# xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test

# xxx: orig is deadcode
# owner check instead? this fails with the $nim/nimdoc/tester.nim test
# now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample)
# if n.info.fileIndex != orig.info.fileIndex: return

case n.kind
of nkCommentStmt:
if previousIsRunnable:
if state in {rsStart, rsRunnable}:
dest.add genRecComment(d, n)
previousIsRunnable = false
return rsComment
of nkCallKinds:
if isRunnableExamples(n[0]) and
n.len >= 2 and n.lastSon.kind == nkStmtList:
previousIsRunnable = true
let rdoccmd = prepareExample(d, n)
n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
let (rdoccmd, code) = prepareExample(d, n)
var msg = "Example:"
if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
"\n\\textbf{$1}\n", [msg.rope])
inc d.listingCounter
let id = $d.listingCounter
dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
# this is a rather hacky way to get rid of the initial indentation
# that the renderer currently produces:
var i = 0
var body = n.lastSon
if body.len == 1 and body.kind == nkStmtList and
body.lastSon.kind == nkStmtList:
body = body.lastSon
for b in body:
if i > 0: dest.add "\n"
inc i
nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
when true:
var dest2 = ""
renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
dest.add dest2
else:
renderNimCodeOld(d, n, dest)
dest.add(d.config.getOrDefault"doc.listing_end" % id)
else: previousIsRunnable = false

var previousIsRunnable2 = true
for i in 0..<n.safeLen:
getAllRunnableExamplesRec(d, n[i], orig, dest, previousIsRunnable2)

proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
var previousIsRunnable = true
getAllRunnableExamplesRec(d, n, n, dest, previousIsRunnable)
return rsRunnable
else: discard
return rsDone
# change this to `rsStart` if you want to keep generating doc comments
# and runnableExamples that occur after some code in routine

proc getRoutineBody(n: PNode): PNode =
## works around a transf bug D20200526T163511 where `result` messes AST
# TODO: try w templates and macro, see if they're affected
result = n[^1]
case result.kind
of nkSym:
result = n[^2]
case result.kind
of nkAsgn:
doAssert result[0].kind == nkSym
doAssert result.len == 2
result = result[1]
else: # eg: nkStmtList
discard
else:
discard

proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
var n = n
let orig = n
var state = rsStart
template fn(n2) =
state = getAllRunnableExamplesImpl(d, n2, orig, dest, state)
case n.kind
of routineDefs:
n = n.getRoutineBody
case n.kind
of nkCommentStmt, nkCallKinds: fn(n)
else:
for i in 0..<n.safeLen:
fn(n[i])
if state == rsDone: return
else: fn(n)

proc isVisible(d: PDoc; n: PNode): bool =
result = false
Expand Down Expand Up @@ -765,12 +809,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
var literal, plainName = ""
var kind = tkEof
var comm: Rope = nil
# skipping this (and doing it inside getAllRunnableExamples) would fix order in
# case of a runnableExample appearing before a doccomment, but would cause other
# issues
comm.add genRecComment(d, n)
if n.kind in declarativeDefs:
if n.kind in routineDefs:
getAllRunnableExamples(d, n, comm)
else:
comm.add genRecComment(d, n)

var r: TSrcGen
# Obtain the plain rendered string for hyperlink titles.
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
Expand Down
19 changes: 11 additions & 8 deletions compiler/msgs.nim
Original file line number Diff line number Diff line change
Expand Up @@ -435,18 +435,21 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
conf.m.fileInfos[fileIdx.int32].lines.add line

proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
if i.fileIndex.int32 < 0: return ""

if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
result = conf.m.fileInfos[fileIdx.int32].lines.len
if result == 0:
try:
for line in lines(toFullPathConsiderDirty(conf, i)):
addSourceLine conf, i.fileIndex, line.string
for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
addSourceLine conf, fileIdx, line.string
except IOError:
discard
assert i.fileIndex.int32 < conf.m.fileInfos.len
result = conf.m.fileInfos[fileIdx.int32].lines.len

proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
if i.fileIndex.int32 < 0: return ""
let num = numLines(conf, i.fileIndex)
# can happen if the error points to EOF:
if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
if i.line.int > num: return ""

result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]

Expand Down
65 changes: 65 additions & 0 deletions compiler/renderverbatim.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import strutils
from xmltree import addEscaped

import ast, options, msgs
import packages/docutils/highlite

proc lastNodeRec(n: PNode): PNode =
result = n
while result.safeLen > 0: result = result[^1]

proc isInIndentationBlock(src: string, indent: int): bool =
#[
we stop at the first de-indentation; there's an inherent ambiguity with non
doc comments since they can have arbitrary indentation, so we just take the
practical route and require a runnableExamples to keep its code (including non
doc comments) to its indentation level.
]#
for j in 0..<indent:
if src.len <= j: return true
if src[j] == ' ': continue
return false
return true

proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
## we'd need to check performance impact to enable it for nimdoc.
let first = n.lastSon.info
let last = n.lastNodeRec.info
var ret = ""
var info = first
var indent = info.col
let numLines = numLines(conf, info.fileIndex).uint16
var lastNonemptyPos = 0
for line in first.line..<numLines:
info.line = line
let src = sourceLine(conf, info)
if line > last.line and not isInIndentationBlock(src, indent):
break
if line > first.line: ret.add "\n"
if src.len > indent:
ret.add src[indent..^1]
lastNonemptyPos = ret.len
ret = ret[0..<lastNonemptyPos]
return ret

proc renderNimCode*(result: var string, code: string, isLatex = false) =
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
var buf = ""
template append(kind, val) =
buf.setLen 0
buf.addEscaped(val)
let class = tokenClassToStr[kind]
if isLatex:
result.addf "\\span$1{$2}" % [class, buf]
else:
result.addf "<span class=\"$1\">$2</span>" % [class, buf]

while true:
getNextToken(toknizr, langNim)
case toknizr.kind
of gtEof: break # End Of File (or string)
else:
# TODO: avoid alloc; maybe toOpenArray
append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))
36 changes: 28 additions & 8 deletions nimdoc/testproject/expected/testproject.html
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,9 @@ <h1 class="title">testproject</h1>
<li>
<a class="reference reference-toplevel" href="#18" id="68">Templates</a>
<ul class="simple simple-toc-section">
<li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
<li><a class="reference" href="#myfn.t"
title="myfn()"><wbr />myfn<span class="attachedType"></span></a></li>
<li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
title="foo(a, b: SomeType)"><wbr />foo<span class="attachedType"></span></a></li>

</ul>
Expand All @@ -187,13 +189,11 @@ <h1 class="title">testproject</h1>

<p class="module-desc">This is the top level module.
<p><strong class="examples_text">Example:</strong></p>
<pre class="listing"><span class="Keyword">import</span>
<span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>

<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
<span class="Keyword">discard</span></pre></p>
<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
</span><span class="Comment"># bug #11078</span><span class="Whitespace">
</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre></p>
<div class="section" id="6">
<h1><a class="toc-backref" href="#6">Imports</a></h1>
<dl class="item">
Expand Down Expand Up @@ -331,6 +331,26 @@ <h1><a class="toc-backref" href="#17">Macros</a></h1>
<div class="section" id="18">
<h1><a class="toc-backref" href="#18">Templates</a></h1>
<dl class="item">
<a id="myfn.t"></a>
<dt><pre><span class="Keyword">template</span> <a href="#myfn.t"><span class="Identifier">myfn</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
<dd>


<p><strong class="examples_text">Example:</strong></p>
<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
</span><span class="Comment">## line doc comment</span><span class="Whitespace">
</span><span class="Comment"># bar</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
</span><span class="LongComment">##[
foo
bar
]##</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
</span><span class="Comment"># should be in</span></pre>

</dd>
<a id="foo.t,SomeType,SomeType"></a>
<dt><pre><span class="Keyword">template</span> <a href="#foo.t%2CSomeType%2CSomeType"><span class="Identifier">foo</span></a><span class="Other">(</span><span class="Identifier">a</span><span class="Other">,</span> <span class="Identifier">b</span><span class="Other">:</span> <a href="subdir/subdir_b/utils.html#SomeType"><span class="Identifier">SomeType</span></a><span class="Other">)</span></pre></dt>
<dd>
Expand Down
4 changes: 4 additions & 0 deletions nimdoc/testproject/expected/theindex.html
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ <h1 class="title">Index</h1>
<li><a class="reference external"
data-doc-search-tag="testproject: isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: isValid[T](x: T): bool</a></li>
</ul></dd>
<dt><a name="myfn" href="#myfn"><span>myfn:</span></a></dt><dd><ul class="simple">
<li><a class="reference external"
data-doc-search-tag="testproject: myfn()" href="testproject.html#myfn.t">testproject: myfn()</a></li>
</ul></dd>
<dt><a name="someFunc" href="#someFunc"><span>someFunc:</span></a></dt><dd><ul class="simple">
<li><a class="reference external"
data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()</a></li>
Expand Down
16 changes: 16 additions & 0 deletions nimdoc/testproject/testproject.nim
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,22 @@ runnableExamples:
# bug #11078
for x in "xx": discard

template myfn*() =
runnableExamples:
import std/strutils
## line doc comment
# bar
doAssert "'foo" == "'foo"
##[
foo
bar
]##
doAssert: not "foo".startsWith "ba"
block:
discard 0xff # elu par cette crapule
# should be in
# should be out

const
C_A* = 0x7FF0000000000000'f64
C_B* = 0o377'i8
Expand Down

0 comments on commit 52f8a2a

Please sign in to comment.