Implement RFC 416: lazy symbol semantic checking: enable cyclic deps, remove need for fwd decls, etc

compiler/ast.nim

@@ -238,6 +238,11 @@ type
     sfGlobal,         # symbol is at global scope
 
     sfForward,        # symbol is forward declared
+    sfLazy,           # symbol is lazy declared
+    sfLazyForwardRequested, # semchecking was requested for a lazy symbol; PRTEMP RENAME
+    sfLazySemcheckInprogress,
+    sfLazySemcheckStarted,
+    sfLazyDeadSymTansf, # for transf.nim D20210904T200315
     sfWasForwarded,   # symbol had a forward declaration
                       # (implies it's too dangerous to patch its type signature)
     sfImportc,        # symbol is external; imported
@@ -869,6 +874,10 @@ type
       bitsize*: int
       alignment*: int # for alignment
     else: nil
+    lazyDecl*: PSym
+      # could be in routineKinds branch
+      # proc f(); proc f()=discard; => 2nd sym.lazyDecl = 1st sym
+      # this was resolved as an impl for a fwd proc; we shouldn't pick such a symbol and instead pick the fwd decl
     magic*: TMagic
     typ*: PType
     name*: PIdent
@@ -1206,6 +1215,9 @@ template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
 template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
+proc isVarargsUntyped*(x: PType): bool {.inline.} =
+  x.kind == tyVarargs and x[0].kind == tyUntyped
+
 proc getDeclPragma*(n: PNode): PNode =
   ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found.
   ## Currently only supports routineDefs + {nkTypeDef}.

compiler/astalgo.nim

@@ -31,6 +31,11 @@ proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
 proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
 proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
+proc isNotDup(s: PSym): bool =
+  s.lazyDecl == nil
+  # s.lazyDecl == nil and sfLazySemcheckStarted notin s.flags # PRTEMP
+  # sfLazySemcheckStarted
+
 proc typekinds*(t: PType) {.deprecated.} =
   var t = t
   var s = ""
@@ -527,6 +532,9 @@ proc value(this: var DebugPrinter; value: PSym) =
   this.value(value.name.s)
   this.key("id")
   this.value(value.id)
+  if value.ast!=nil and this.conf!=nil:
+    this.key("loc")
+    this.value(this.conf$value.ast.info)
   if value.kind in {skField, skEnumField, skParam}:
     this.key("position")
     this.value(value.position)
@@ -594,6 +602,7 @@ proc value(this: var DebugPrinter; value: PNode) =
   if this.conf != nil:
     this.key "info"
     this.value $lineInfoToStr(this.conf, value.info)
+    # TODO: loc this.value(this.conf$value.ast.info)
   if value.flags != {}:
     this.key "flags"
     this.value value.flags
@@ -640,23 +649,23 @@ proc value(this: var DebugPrinter; value: PNode) =
 
 
 proc debug(n: PSym; conf: ConfigRef) =
-  var this: DebugPrinter
+  var this = DebugPrinter(conf: conf)
   this.visited = initTable[pointer, int]()
   this.renderSymType = true
   this.useColor = not defined(windows)
   this.value(n)
   echo($this.res)
 
 proc debug(n: PType; conf: ConfigRef) =
-  var this: DebugPrinter
+  var this = DebugPrinter(conf: conf)
   this.visited = initTable[pointer, int]()
   this.renderSymType = true
   this.useColor = not defined(windows)
   this.value(n)
   echo($this.res)
 
 proc debug(n: PNode; conf: ConfigRef) =
-  var this: DebugPrinter
+  var this = DebugPrinter(conf: conf)
   this.visited = initTable[pointer, int]()
   #this.renderSymType = true
   this.useColor = not defined(windows)
@@ -802,7 +811,7 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym =
   while true:
     result = t.data[h]
     if result == nil: break
-    if result.name.id == name.id: break
+    if result.name.id == name.id and result.isNotDup: break
     h = nextTry(h, high(t.data))
 
 
@@ -816,7 +825,7 @@ proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym =
   var start = h
   result = tab.data[h]
   while result != nil:
-    if result.name.id == ti.name.id: break
+    if result.name.id == ti.name.id and result.isNotDup: break
     h = nextTry(h, high(tab.data))
     if h == start:
       result = nil
@@ -836,15 +845,18 @@ proc nextIdentExcluding*(ti: var TIdentIter, tab: TStrTable,
   var start = h
   result = tab.data[h]
   while result != nil:
-    if result.name.id == ti.name.id and not contains(excluding, result.id):
+    if result.name.id == ti.name.id and result.isNotDup and not contains(excluding, result.id):
       break
     h = nextTry(h, high(tab.data))
     if h == start:
       result = nil
       break
     result = tab.data[h]
   ti.h = nextTry(h, high(tab.data))
-  if result != nil and contains(excluding, result.id): result = nil
+  if result != nil:
+    # bugfix for pre-existing issue: avoid redundant check
+    assert not contains(excluding, result.id)
+    assert result.isNotDup
 
 proc firstIdentExcluding*(ti: var TIdentIter, tab: TStrTable, s: PIdent,
                           excluding: IntSet): PSym =

compiler/concepts.nim

@@ -14,6 +14,7 @@
 import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
 
 from magicsys import addSonSkipIntLit
+from modulegraphs import determineType2
 
 const
   logBindings = false
@@ -269,8 +270,10 @@ proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool =
 proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool =
   ## Walk the current scope, extract candidates which the same name as 'n[namePos]',
   ## 'n' is the nkProcDef or similar from the concept that we try to match.
+  determineType2 c.graph, n[namePos].sym
   let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds)
   for candidate in candidates:
+    determineType2(c.graph, candidate)
     #echo "considering ", typeToString(candidate.typ), " ", candidate.magic
     m.magic = candidate.magic
     if matchSym(c, candidate, n, m): return true

compiler/condsyms.nim

@@ -138,3 +138,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasHintAll")
   defineSymbol("nimHasTrace")
   defineSymbol("nimHasEffectsOf")
+  defineSymbol("nimHasLazySemcheck")

compiler/debugutils.nim

@@ -54,3 +54,5 @@ proc isCompilerDebug*(): bool =
       {.undef(nimCompilerDebug).}
       echo 'x'
   conf0.isDefined("nimCompilerDebug")
+
+include timn/exp/nim_compiler_debugutils

compiler/docgen.nim

@@ -1136,6 +1136,11 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
   ## which is implemented in ``docgen2.nim``.
   template genItemAux(skind) =
     genItem(d, n, n[namePos], skind, docFlags)
+  if n.kind in routineDefs:
+    if n[0].kind != nkSym:
+      # e.g. for: nim doc0 lib/system/threads.nim
+      assert d.conf.cmd == cmdDoc0, $d.conf.cmd
+    elif n[0].sym.typ == nil: return # lazy semchecking; we could also defer to epilogue
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)

compiler/docgen2.nim

@@ -78,11 +78,11 @@ proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassConte
 proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
 
-const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
+const docgen2Pass* = makePass(open = myOpen, process = processNode, closeEpilogue = close)
 const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
+                                 closeEpilogue = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
-                                  close = closeJson)
+                                  closeEpilogue = closeJson)
 
 proc finishDoc2Pass*(project: string) =
   discard

compiler/jsgen.nim

@@ -1391,6 +1391,10 @@ proc genCopyForParamIfNeeded(p: PProc, n: PNode) =
 
 proc genVarInit(p: PProc, v: PSym, n: PNode)
 
+proc checkClosureIteratorJs(p: PProc, n: PNode) =
+  if sfLazy notin n[0].sym.flags and n[0].sym.typ.callConv == TCallingConvention.ccClosure:
+    globalError(p.config, n.info, "Closure iterators are not supported by JS backend!")
+
 proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   var s = n.sym
   case s.kind
@@ -1440,6 +1444,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
     else:
       genProcForSymIfNeeded(p, s)
   else:
+    if s.kind == skIterator: checkClosureIteratorJs(p, s.ast)
     if s.loc.r == nil:
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     r.res = s.loc.r
@@ -2623,8 +2628,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
      nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt,
      nkMixinStmt, nkBindStmt: discard
   of nkIteratorDef:
-    if n[0].sym.typ.callConv == TCallingConvention.ccClosure:
-      globalError(p.config, n.info, "Closure iterators are not supported by JS backend!")
+    checkClosureIteratorJs(p, n)
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     var s = n[namePos].sym

compiler/lookups.nim

@@ -13,8 +13,6 @@ import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
   renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs
 
-proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
-
 proc noidentError(conf: ConfigRef; n, origin: PNode) =
   var m = ""
   if origin != nil:
@@ -70,10 +68,6 @@ proc openScope*(c: PContext): PScope {.discardable.} =
 proc rawCloseScope*(c: PContext) =
   c.currentScope = c.currentScope.parent
 
-proc closeScope*(c: PContext) =
-  ensureNoMissingOrUnusedSymbols(c, c.currentScope)
-  rawCloseScope(c)
-
 iterator allScopes*(scope: PScope): PScope =
   var current = scope
   while current != nil:
@@ -196,19 +190,26 @@ proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
     if result != nil: return result
   result = someSymFromImportTable(c, s, ambiguous)
 
-proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
+proc debugScopes*(conf: ConfigRef, scope: PScope; limit=0, max = int.high) {.deprecated.} =
   var i = 0
   var count = 0
-  for scope in allScopes(c.currentScope):
-    echo "scope ", i
+  for scope in allScopes(scope):
+    echo "scope $# depth: $#" % [$i, $scope.depthLevel]
     for h in 0..high(scope.symbols.data):
       if scope.symbols.data[h] != nil:
         if count >= max: return
-        echo count, ": ", scope.symbols.data[h].name.s
+        let s = scope.symbols.data[h]
+        var msg = $count & ": " & $s & $(s.flags, s.owner, s.kind, s.typ)
+        if s.ast!=nil:
+          msg.add " " & conf$s.ast.info
+        echo msg
         count.inc
     if i == limit: return
     inc i
 
+proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
+  debugScopes(c.config, c.currentScope, limit, max)
+
 proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
   result = @[]
   block outer:
@@ -268,30 +269,50 @@ proc getSymRepr*(conf: ConfigRef; s: PSym, getDeclarationPath = true): string =
     if getDeclarationPath:
       result.addDeclaredLoc(conf, s)
 
-proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
-  # check if all symbols have been used and defined:
+type EpilogueData = object
+  missingImpls: int
+  unusedSyms: seq[tuple[sym: PSym, key: string]]
+
+proc ensureNoMissingOrUnusedSymbol*(data: var EpilogueData, config: ConfigRef, s: PSym) =
+  if sfForward in s.flags and s.kind notin {skType, skModule}:
+    # too many 'implementation of X' errors are annoying
+    # and slow 'suggest' down:
+    if data.missingImpls == 0:
+      if false:
+        # PRTEMP
+        localError(config, s.info, "implementation of '$1' expected" %
+            getSymRepr(config, s, getDeclarationPath=false))
+    inc data.missingImpls
+  elif {sfUsed, sfExported} * s.flags == {}:
+    if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
+      # XXX: implicit type params are currently skTypes
+      # maybe they can be made skGenericParam as well.
+      if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
+         s.typ.kind != tyGenericParam:
+        data.unusedSyms.add (s, toFileLineCol(config, s.info))
+
+iterator getSymbols(scope: PScope): PSym =
   var it: TTabIter
   var s = initTabIter(it, scope.symbols)
-  var missingImpls = 0
-  var unusedSyms: seq[tuple[sym: PSym, key: string]]
   while s != nil:
-    if sfForward in s.flags and s.kind notin {skType, skModule}:
-      # too many 'implementation of X' errors are annoying
-      # and slow 'suggest' down:
-      if missingImpls == 0:
-        localError(c.config, s.info, "implementation of '$1' expected" %
-            getSymRepr(c.config, s, getDeclarationPath=false))
-      inc missingImpls
-    elif {sfUsed, sfExported} * s.flags == {}:
-      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
-        # XXX: implicit type params are currently skTypes
-        # maybe they can be made skGenericParam as well.
-        if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
-           s.typ.kind != tyGenericParam:
-          unusedSyms.add (s, toFileLineCol(c.config, s.info))
+    yield s
     s = nextIter(it, scope.symbols)
-  for (s, _) in sortedByIt(unusedSyms, it.key):
-    message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
+
+template ensureNoMissingOrUnusedSymbols*(conf: ConfigRef; syms: untyped) = # syms: iterable[PSym]
+  # check if all symbols have been used and defined:
+  var data: EpilogueData
+  for s in syms:
+    ensureNoMissingOrUnusedSymbol(data, conf, s)
+  for (s, _) in sortedByIt(data.unusedSyms, it.key):
+    # if s.getnimblePkgId == conf.mainPackageId:
+    if s.getModule.getnimblePkgId == conf.mainPackageId:
+      # see also foreignPackageNotes
+      message(conf, s.info, hintXDeclaredButNotUsed, s.name.s)
+
+proc closeScope*(c: PContext) =
+  if not c.config.isSemcheckUnusedSymbols:
+    ensureNoMissingOrUnusedSymbols(c.config, getSymbols(c.currentScope))
+  rawCloseScope(c)
 
 proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
                         conflictsWith: TLineInfo, note = errGenerated) =
@@ -330,13 +351,13 @@ proc addPrelimDecl*(c: PContext, sym: PSym) =
 
 from ic / ic import addHidden
 
-proc addInterfaceDeclAux(c: PContext, sym: PSym) =
+proc addInterfaceDeclAux(c: PContext, sym: PSym, isTopLevel: bool) =
   ## adds symbol to the module for either private or public access.
   if sfExported in sym.flags:
     # add to interface:
     if c.module != nil: exportSym(c, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
-  elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym):
+  elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevel:
     strTableAdd(semtabAll(c.graph, c.module), sym)
     if c.config.symbolFiles != disabledSf:
       addHidden(c.encoder, c.packedRepr, sym)
@@ -346,7 +367,7 @@ proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
   addDeclAt(c, scope, sym)
   if not scope.isShadowScope:
     # adding into a non-shadow scope, we need to handle exports, etc
-    addInterfaceDeclAux(c, sym)
+    addInterfaceDeclAux(c, sym, scope.isTopLevel)
 
 proc addInterfaceDecl*(c: PContext, sym: PSym) {.inline.} =
   ## adds a decl and the interface if appropriate
@@ -369,7 +390,13 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addOverloadableSymAt(c, scope, sym)
   if not scope.isShadowScope:
     # adding into a non-shadow scope, we need to handle exports, etc
-    addInterfaceDeclAux(c, sym)
+    addInterfaceDeclAux(c, sym, scope.isTopLevel)
+
+proc addInterfaceDeclSelect*(c: PContext, scope: PScope, sym: PSym) {.inline.} =
+  if sym.kind in OverloadableSyms:
+    addInterfaceOverloadableSymAt(c, scope, sym)
+  else:
+    addInterfaceDeclAt(c, scope, sym)
 
 proc openShadowScope*(c: PContext) =
   ## opens a shadow scope, just like any other scope except the depth is the
@@ -394,10 +421,7 @@ proc mergeShadowScope*(c: PContext) =
   let shadowScope = c.currentScope
   c.rawCloseScope
   for sym in shadowScope.symbols:
-    if sym.kind in OverloadableSyms:
-      c.addInterfaceOverloadableSymAt(c.currentScope, sym)
-    else:
-      c.addInterfaceDecl(sym)
+    addInterfaceDeclSelect(c, c.currentScope, sym)
 
 when false:
   # `nimfix` used to call `altSpelling` and prettybase.replaceDeprecated(n.info, ident, alt)
@@ -539,6 +563,7 @@ type
     checkAmbiguity, checkUndeclared, checkModule, checkPureEnumFields
 
 proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
+  # `determineType2` should not be called inside this; instead callers should selectively call it as needed.
   const allExceptModule = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage}
   case n.kind
   of nkIdent, nkAccQuoted: