From 52f8a2a9a838d44a0e2856e299c9c0f866c37f27 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 29 Feb 2020 16:30:31 -0800 Subject: [PATCH] fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting --- compiler/ast.nim | 1 + compiler/docgen.nim | 139 ++++++++++++------- compiler/msgs.nim | 19 +-- compiler/renderverbatim.nim | 65 +++++++++ nimdoc/testproject/expected/testproject.html | 36 +++-- nimdoc/testproject/expected/theindex.html | 4 + nimdoc/testproject/testproject.nim | 16 +++ 7 files changed, 216 insertions(+), 64 deletions(-) create mode 100644 compiler/renderverbatim.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 3fe75d1f37782..f4ed3db3d3606 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1029,6 +1029,7 @@ const nkFloatLiterals* = {nkFloatLit..nkFloat128Lit} nkLambdaKinds* = {nkLambda, nkDo} declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef} + routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef} procDefs* = nkLambdaKinds + declarativeDefs nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice} diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2072945973ce7..542f4a8c7faac 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -17,7 +17,7 @@ import packages/docutils/rst, packages/docutils/rstgen, json, xmltree, cgi, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths + pathutils, trees, tables, nimpaths, renderverbatim const exportSection = skField @@ -494,8 +494,8 @@ proc runAllExamples(d: PDoc) = rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string]) # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove -proc prepareExample(d: PDoc; n: PNode): string = - ## returns `rdoccmd` for this runnableExamples +proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] = + ## returns `rdoccmd` and source code for this runnableExamples var rdoccmd = "" if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid") if n.len == 3: @@ -512,10 +512,11 @@ proc prepareExample(d: PDoc; n: PNode): string = docComment, newTree(nkImportStmt, newStrNode(nkStrLit, d.filename))) runnableExamples.info = n.info - + let ret = extractRunnableExamplesSource(d.conf, n) for a in n.lastSon: runnableExamples.add a + # we could also use `ret` instead here, to keep sources verbatim writeExample(d, runnableExamples, rdoccmd) - result = rdoccmd + result = (rdoccmd, ret) when false: proc extractImports(n: PNode; result: PNode) = if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}: @@ -529,39 +530,57 @@ proc prepareExample(d: PDoc; n: PNode): string = for imp in imports: runnableExamples.add imp runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon) -proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) = +proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) = + ## this is a rather hacky way to get rid of the initial indentation + ## that the renderer currently produces: + # deadcode + var i = 0 + var body = n.lastSon + if body.len == 1 and body.kind == nkStmtList and + body.lastSon.kind == nkStmtList: + body = body.lastSon + for b in body: + if i > 0: dest.add "\n" + inc i + nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) + +type RunnableState = enum + rsStart + rsComment + rsRunnable + rsDone + +proc getAllRunnableExamplesImpl(d: PDoc; n, orig: PNode; dest: var Rope, state: RunnableState): RunnableState = ##[ - previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though). - This is to ensure this works: + Simple state machine to tell whether we render runnableExamples and doc comments. + This is to ensure that we can interleave runnableExamples and doc comments freely; + the logic is easy to change but currently a doc comment following another doc comment + will not render, to avoid rendering in following case: + proc fn* = runnableExamples: discard ## d1 runnableExamples: discard ## d2 - ## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment - - It also works with: - proc fn* = - ## d0 - runnableExamples: discard - ## d1 - - etc + ## internal explanation # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment + discard # some code ]## - # xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test + + # xxx: orig is deadcode + # owner check instead? this fails with the $nim/nimdoc/tester.nim test # now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample) # if n.info.fileIndex != orig.info.fileIndex: return + case n.kind of nkCommentStmt: - if previousIsRunnable: + if state in {rsStart, rsRunnable}: dest.add genRecComment(d, n) - previousIsRunnable = false + return rsComment of nkCallKinds: if isRunnableExamples(n[0]) and - n.len >= 2 and n.lastSon.kind == nkStmtList: - previousIsRunnable = true - let rdoccmd = prepareExample(d, n) + n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}: + let (rdoccmd, code) = prepareExample(d, n) var msg = "Example:" if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd dispA(d.conf, dest, "\n

$1

\n", @@ -569,27 +588,52 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"]) - # this is a rather hacky way to get rid of the initial indentation - # that the renderer currently produces: - var i = 0 - var body = n.lastSon - if body.len == 1 and body.kind == nkStmtList and - body.lastSon.kind == nkStmtList: - body = body.lastSon - for b in body: - if i > 0: dest.add "\n" - inc i - nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) + when true: + var dest2 = "" + renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex) + dest.add dest2 + else: + renderNimCodeOld(d, n, dest) dest.add(d.config.getOrDefault"doc.listing_end" % id) - else: previousIsRunnable = false - - var previousIsRunnable2 = true - for i in 0.. conf.m.fileInfos[i.fileIndex.int32].lines.len: return "" + if i.line.int > num: return "" result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1] diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim new file mode 100644 index 0000000000000..9513353cd92ac --- /dev/null +++ b/compiler/renderverbatim.nim @@ -0,0 +1,65 @@ +import strutils +from xmltree import addEscaped + +import ast, options, msgs +import packages/docutils/highlite + +proc lastNodeRec(n: PNode): PNode = + result = n + while result.safeLen > 0: result = result[^1] + +proc isInIndentationBlock(src: string, indent: int): bool = + #[ + we stop at the first de-indentation; there's an inherent ambiguity with non + doc comments since they can have arbitrary indentation, so we just take the + practical route and require a runnableExamples to keep its code (including non + doc comments) to its indentation level. + ]# + for j in 0.. last.line and not isInIndentationBlock(src, indent): + break + if line > first.line: ret.add "\n" + if src.len > indent: + ret.add src[indent..^1] + lastNonemptyPos = ret.len + ret = ret[0..$2" % [class, buf] + + while true: + getNextToken(toknizr, langNim) + case toknizr.kind + of gtEof: break # End Of File (or string) + else: + # TODO: avoid alloc; maybe toOpenArray + append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1)) diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index aa29c0c0dcbd4..257cb581b06b7 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -173,7 +173,9 @@

testproject

  • Templates @@ -187,13 +189,11 @@

    testproject

    This is the top level module.

    Example:

    -
    import
-  subdir / subdir_b / utils
-
-doAssert bar(3, 4) == 7
-foo(enumValueA, enumValueB)
-for x in "xx":
-  discard

    +
    import subdir / subdir_b / utils
+doAssert bar(3, 4) == 7
+foo(enumValueA, enumValueB)
+# bug #11078
+for x in "xx": discard

    Imports

    @@ -331,6 +331,26 @@

    Macros

    Templates

    + +
    template myfn()
    +
    + + +

    Example:

    +
    import std/strutils
+## line doc comment
+# bar
+doAssert "'foo" == "'foo"
+##[
+foo
+bar
+]##
+doAssert: not "foo".startsWith "ba"
+block:
+  discard 0xff # elu par cette crapule
+# should be in
    + +
    template foo(a, b: SomeType)
    diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index a76f5961f6b88..f78be11eeef16 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -143,6 +143,10 @@

    Index

  • testproject: isValid[T](x: T): bool
    • +
    myfn:
    someFunc:
    • testproject: someFunc()
      • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index d1fcf58cd83e4..b2976fe9a5495 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -9,6 +9,22 @@ runnableExamples: # bug #11078 for x in "xx": discard +template myfn*() = + runnableExamples: + import std/strutils + ## line doc comment + # bar + doAssert "'foo" == "'foo" + ##[ + foo + bar + ]## + doAssert: not "foo".startsWith "ba" + block: + discard 0xff # elu par cette crapule + # should be in + # should be out + const C_A* = 0x7FF0000000000000'f64 C_B* = 0o377'i8