From 7b31a81d4577a5d293e4f5bfa591848d93eb8320 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 20 Feb 2019 12:28:22 +0100 Subject: [PATCH 0001/1645] system.nim: avoid 'cannot prove init' warnings for the '..' constructors --- lib/system.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 268d6ccd3148..e8a04ce9d5d5 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -407,12 +407,11 @@ proc `..`*[T, U](a: T, b: U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDo ## and `b` are inclusive. Slices can also be used in the set constructor ## and in ordinal case statements, but then they are special-cased by the ## compiler. - result.a = a - result.b = b + result = HSlice[T, U](a: a, b: b) proc `..`*[T](b: T): HSlice[int, T] {.noSideEffect, inline, magic: "DotDot".} = ## unary `slice`:idx: operator that constructs an interval ``[default(int), b]`` - result.b = b + result = HSlice[int, T](a: 0, b: b) when not defined(niminheritable): {.pragma: inheritable.} From 2990a5ea5416520596b158a7fcb96106763e680f Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 20 Feb 2019 12:28:43 +0100 Subject: [PATCH 0002/1645] koch.nim: indentation uses 2 spaces --- koch.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/koch.nim b/koch.nim index ac95c15663c2..b776927ae228 100644 --- a/koch.nim +++ b/koch.nim @@ -192,11 +192,11 @@ proc zip(latest: bool; args: string) = ["tools/niminst/niminst".exe, VersionAsString]) proc ensureCleanGit() = - let (outp, status) = osproc.execCmdEx("git diff") - if outp.len != 0: - quit "Not a clean git repository; 'git diff' not empty!" - if status != 0: - quit "Not a clean git repository; 'git diff' returned non-zero!" + let (outp, status) = osproc.execCmdEx("git diff") + if outp.len != 0: + quit "Not a clean git repository; 'git diff' not empty!" + if status != 0: + quit "Not a clean git repository; 'git diff' returned non-zero!" proc xz(latest: bool; args: string) = ensureCleanGit() From ffd0f812471b78d15863b8299b7db35fa1951c77 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 20 Feb 2019 12:30:51 +0100 Subject: [PATCH 0003/1645] times.nim: avoid some stdlib dependencies --- lib/pure/times.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 09ec786b5890..cc6a61b21824 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -187,7 +187,7 @@ the same as 25 hours. ]## -import strutils, algorithm, math, options, strformat +import strutils, math, options include "system/inclrtl" @@ -1757,7 +1757,7 @@ proc `$`*(f: TimeFormat): string = proc raiseParseException(f: TimeFormat, input: string, msg: string) = raise newException(TimeParseError, - &"Failed to parse '{input}' with format '{f}'. {msg}") + "Failed to parse '" & input & "' with format '" & $f & "'. " & msg) proc parseInt(s: string, b: var int, start = 0, maxLen = int.high, allowSign = false): int = @@ -1809,7 +1809,7 @@ iterator tokens(f: string): tuple[kind: FormatTokenKind, token: string] = if i > f.high: raise newException(TimeFormatParseError, - &"Unclosed ' in time format string. " & + "Unclosed ' in time format string. " & "For a literal ', use ''.") i.inc yield (tkLiteral, token) @@ -1866,7 +1866,7 @@ proc stringToPattern(str: string): FormatPattern = of "zzzz": result = zzzz of "g": result = g else: raise newException(TimeFormatParseError, - &"'{str}' is not a valid pattern") + "'" & str & "' is not a valid pattern") proc initTimeFormat*(format: string): TimeFormat = ## Construct a new time format for parsing & formatting time types. @@ -2377,7 +2377,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local()): DateTime patIdx.inc else: if not parsePattern(input, pattern, inpIdx, parsed): - raiseParseException(f, input, &"Failed on pattern '{pattern}'") + raiseParseException(f, input, "Failed on pattern '" & $pattern & "'") patIdx.inc if inpIdx <= input.high: From 873539b50da56bbfe4c18283bfe239c29d52e81c Mon Sep 17 00:00:00 2001 From: narimiran Date: Wed, 20 Feb 2019 14:16:38 +0100 Subject: [PATCH 0004/1645] lib.rst: add link to `asyncstreams`, fixes #6383 [ci skip] --- doc/lib.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/lib.rst b/doc/lib.rst index a6d5e4e78cc4..6fb0abedc23d 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -213,6 +213,9 @@ Generic Operating System Services This module implements asynchronous file reading and writing using ``asyncdispatch``. +* `asyncstreams `_ + This module provides `FutureStream` - a future that acts as a queue. + * `distros `_ This module implements the basics for OS distribution ("distro") detection and the OS's native package manager. From 6c10d331db8c00accc7ac2884ac46c89ef72cc3b Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 20 Feb 2019 14:29:09 +0100 Subject: [PATCH 0005/1645] gc:destructors: slightly more stuff compiles --- compiler/semtypes.nim | 3 ++- lib/system.nim | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index a497e1eae4ee..9976e9302b36 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1613,7 +1613,8 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = if result.len > 0: var base = result[0] if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base) - if base.kind != tyGenericParam: + if not containsGenericType(base): + # base.kind != tyGenericParam: c.typesWithOps.add((result, result)) else: result = semContainer(c, n, tySequence, "seq", prev) diff --git a/lib/system.nim b/lib/system.nim index e8a04ce9d5d5..71d1458fde3a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2559,7 +2559,10 @@ proc `==`*[T](x, y: seq[T]): bool {.noSideEffect.} = else: when not defined(JS): proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = - result = cast[pointer](x) + when defined(gcDestructors): + result = cast[NimSeqV2[T]](x).p + else: + result = cast[pointer](x) if seqToPtr(x) == seqToPtr(y): return true From 43d570c17810fa9d8ab7605f3ce4dfd2120fdd63 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Feb 2019 07:57:29 +0100 Subject: [PATCH 0006/1645] gc:destructors: make system/repr compile --- compiler/ccgexprs.nim | 27 ++++++++++++++++++++++++++- lib/system.nim | 2 +- lib/system/repr.nim | 17 ++++++++++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 8ccca9813109..5cf6df847d4b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2607,6 +2607,28 @@ proc genConstSeq(p: BProc, n: PNode, t: PType): Rope = result = "(($1)&$2)" % [getTypeDesc(p.module, t), result] +proc genConstSeqV2(p: BProc, n: PNode, t: PType): Rope = + var data = rope"{" + for i in countup(0, n.len - 1): + if i > 0: data.addf(",$n", []) + data.add genConstExpr(p, n.sons[i]) + data.add("}") + + result = getTempName(p.module) + let payload = getTempName(p.module) + let base = t.skipTypes(abstractInst).sons[0] + + appcg(p.module, cfsData, + "static const struct {$n" & + " NI cap; void* allocator; $1 data[$2];$n" & + "} $3 = {$2, NIM_NIL, $4};$n" & + "static NIM_CONST struct {$n" & + " NI len;$n" & + " $6 p;$n" & + "} $5 = {$2, ($6)&$3};$n", [ + getTypeDesc(p.module, base), rope(len(n)), payload, data, + result, getTypeDesc(p.module, t)]) + proc genConstExpr(p: BProc, n: PNode): Rope = case n.kind of nkHiddenStdConv, nkHiddenSubConv: @@ -2618,7 +2640,10 @@ proc genConstExpr(p: BProc, n: PNode): Rope = of nkBracket, nkPar, nkTupleConstr, nkClosure: var t = skipTypes(n.typ, abstractInst) if t.kind == tySequence: - result = genConstSeq(p, n, n.typ) + if p.config.selectedGc == gcDestructors: + result = genConstSeqV2(p, n, n.typ) + else: + result = genConstSeq(p, n, n.typ) elif t.kind == tyProc and t.callConv == ccClosure and n.len > 1 and n.sons[1].kind == nkNilLit: # Conversion: nimcall -> closure. diff --git a/lib/system.nim b/lib/system.nim index 71d1458fde3a..df7f5e13b7ef 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3383,7 +3383,7 @@ when not defined(JS): #and not defined(nimscript): when not defined(nimscript) and hasAlloc: when not defined(gcDestructors): include "system/assign" - include "system/repr" + include "system/repr" when hostOS != "standalone" and not defined(nimscript): proc getCurrentException*(): ref Exception {.compilerRtl, inl, benign.} = diff --git a/lib/system/repr.nim b/lib/system/repr.nim index ff8f92404964..68d316b73c44 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -160,6 +160,21 @@ when not defined(useNimRtl): reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) add result, "]" + when defined(gcDestructors): + type + GenericSeq = object + len: int + p: pointer + PGenericSeq = ptr GenericSeq + const payloadOffset = sizeof(int) + sizeof(pointer) + # see seqs.nim: cap: int + # region: Allocator + + template payloadPtr(x: untyped): untyped = cast[PGenericSeq](x).p + else: + const payloadOffset = GenericSeqSize + template payloadPtr(x: untyped): untyped = x + proc reprSequence(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) = if p == nil: @@ -170,7 +185,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + GenericSeqSize + i*bs), + reprAux(result, cast[pointer](cast[ByteAddress](payloadPtr(p)) + payloadOffset + i*bs), typ.base, cl) add result, "]" From 418cbbb450d6986ac30ee3fec92fdb7f9cdac7ff Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Feb 2019 12:03:34 +0100 Subject: [PATCH 0007/1645] macros.nim: spacing --- lib/core/macros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 7ec1eefc6397..461afb963a08 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -254,7 +254,7 @@ else: # bootstrapping substitute proc getImpl*(symbol: NimNode): NimNode = symbol.symbol.getImpl - proc strValOld(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + proc strValOld(n: NimNode): string {.magic: "NStrVal", noSideEffect.} proc `$`*(s: NimSym): string {.magic: "IdentToStr", noSideEffect.} From f2f0b5d695fbde2f9a8537b30e8422b22416ddbe Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Feb 2019 12:03:47 +0100 Subject: [PATCH 0008/1645] system.nim: spacing --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index df7f5e13b7ef..441b7e2630ac 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3779,7 +3779,7 @@ template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool] bind instantiationInfo mixin failedAssertImpl when enabled: - # for stacktrace; fixes #8928 ; Note: `fullPaths = true` is correct + # for stacktrace; fixes #8928; Note: `fullPaths = true` is correct # here, regardless of --excessiveStackTrace {.line: instantiationInfo(fullPaths = true).}: if not cond: From 417d27c54406ac29ab00ac0b459aa7546af4b83a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Feb 2019 12:19:06 +0100 Subject: [PATCH 0009/1645] gc:destructors: progress --- compiler/ccgexprs.nim | 11 +++-------- compiler/ccgtypes.nim | 4 +++- tests/destructor/tgcdestructors.nim | 11 ++++++++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5cf6df847d4b..d459f6cbf810 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2614,20 +2614,15 @@ proc genConstSeqV2(p: BProc, n: PNode, t: PType): Rope = data.add genConstExpr(p, n.sons[i]) data.add("}") - result = getTempName(p.module) let payload = getTempName(p.module) let base = t.skipTypes(abstractInst).sons[0] appcg(p.module, cfsData, "static const struct {$n" & " NI cap; void* allocator; $1 data[$2];$n" & - "} $3 = {$2, NIM_NIL, $4};$n" & - "static NIM_CONST struct {$n" & - " NI len;$n" & - " $6 p;$n" & - "} $5 = {$2, ($6)&$3};$n", [ - getTypeDesc(p.module, base), rope(len(n)), payload, data, - result, getTypeDesc(p.module, t)]) + "} $3 = {$2, NIM_NIL, $4};$n", [ + getTypeDesc(p.module, base), rope(len(n)), payload, data]) + result = "{$1, ($2*)&$3}" % [rope(len(n)), getSeqPayloadType(p.module, t), payload] proc genConstExpr(p: BProc, n: PNode): Rope = case n.kind diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index c557123ac41c..37d0827de525 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -279,6 +279,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyString: case detectStrVersion(m) of 2: + discard cgsym(m, "NimStrPayload") discard cgsym(m, "NimStringV2") result = typeNameOrLiteral(m, typ, "NimStringV2") else: @@ -545,7 +546,8 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, let popExSym = magicsys.getCompilerProc(m.g.graph, "popCurrentExceptionEx") if lfDynamicLib in popExSym.loc.flags and sfImportc in popExSym.flags: # echo popExSym.flags, " ma flags ", popExSym.loc.flags - result = "extern " & getTypeDescAux(m, popExSym.typ, check) & " " & mangleName(m, popExSym) & ";\L" & result + result = "extern " & getTypeDescAux(m, popExSym.typ, check) & " " & + mangleName(m, popExSym) & ";\L" & result else: result = genProcHeader(m, popExSym) & ";\L" & result hasField = true diff --git a/tests/destructor/tgcdestructors.nim b/tests/destructor/tgcdestructors.nim index 60d7fc14f74d..1ae2b2549b65 100644 --- a/tests/destructor/tgcdestructors.nim +++ b/tests/destructor/tgcdestructors.nim @@ -1,6 +1,9 @@ discard """ cmd: '''nim c --gc:destructors $file''' - output: '''1 1''' + output: '''hi +ho +ha +1 1''' """ import allocators @@ -12,6 +15,12 @@ proc main = main() +const + test = @["hi", "ho", "ha"] + +for t in test: + echo t + #echo s let (a, d) = allocCounters() cprintf("%ld %ld\n", a, d) From a4543c1aac22e6da8afc7b22445f0cd56380a6ad Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 22 Feb 2019 12:06:30 +0100 Subject: [PATCH 0010/1645] gc:destructors: make strutils compile (but still crashes) --- compiler/ccgliterals.nim | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index 904d01e8112a..efbae3cb8e8d 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -51,8 +51,7 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope = # ------ Version 2: destructor based strings and seqs ----------------------- -proc genStringLiteralDataOnlyV2(m: BModule, s: string): Rope = - result = getTempName(m) +proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope) = addf(m.s[cfsData], "static const struct {$n" & " NI cap; void* allocator; NIM_CHAR data[$2+1];$n" & "} $1 = { $2, NIM_NIL, $3 };$n", @@ -61,24 +60,28 @@ proc genStringLiteralDataOnlyV2(m: BModule, s: string): Rope = proc genStringLiteralV2(m: BModule; n: PNode): Rope = let id = nodeTableTestOrSet(m.dataCache, n, m.labels) if id == m.labels: + let pureLit = getTempName(m) + genStringLiteralDataOnlyV2(m, n.strVal, pureLit) + result = getTempName(m) discard cgsym(m, "NimStrPayload") discard cgsym(m, "NimStringV2") # string literal not found in the cache: - let pureLit = genStringLiteralDataOnlyV2(m, n.strVal) - result = getTempName(m) addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", [result, rope(len(n.strVal)), pureLit]) else: - result = m.tmpBase & rope(id+1) + result = getTempName(m) + addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n", + [result, rope(len(n.strVal)), m.tmpBase & rope(id)]) proc genStringLiteralV2Const(m: BModule; n: PNode): Rope = let id = nodeTableTestOrSet(m.dataCache, n, m.labels) var pureLit: Rope if id == m.labels: + pureLit = getTempName(m) discard cgsym(m, "NimStrPayload") discard cgsym(m, "NimStringV2") # string literal not found in the cache: - pureLit = genStringLiteralDataOnlyV2(m, n.strVal) + genStringLiteralDataOnlyV2(m, n.strVal, pureLit) else: pureLit = m.tmpBase & rope(id) result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit] @@ -88,7 +91,9 @@ proc genStringLiteralV2Const(m: BModule; n: PNode): Rope = proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope = case detectStrVersion(m) of 0, 1: result = genStringLiteralDataOnlyV1(m, s) - of 2: result = genStringLiteralDataOnlyV2(m, s) + of 2: + result = getTempName(m) + genStringLiteralDataOnlyV2(m, s, result) else: localError(m.config, info, "cannot determine how to produce code for string literal") From cedf7847518907b76933c053575afaa50944ad45 Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 22 Feb 2019 12:28:00 +0100 Subject: [PATCH 0011/1645] better docs: options (#10720) --- lib/pure/options.nim | 280 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 226 insertions(+), 54 deletions(-) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index b827e1aa3b04..7a474e772101 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -7,21 +7,18 @@ # distribution, for details about the copyright. # -## Abstract -## ======== -## ## This module implements types which encapsulate an optional value. ## -## A value of type ``Option[T]`` either contains a value `x` (represented as -## ``some(x)``) or is empty (``none(T)``). +## A value of type `Option[T]` either contains a value `x` (represented as +## `some(x)`) or is empty (`none(T)`). ## ## This can be useful when you have a value that can be present or not. The -## absence of a value is often represented by ``nil``, but it is not always +## absence of a value is often represented by `nil`, but it is not always ## available, nor is it always a good solution. ## ## -## Tutorial -## ======== +## Basic usage +## =========== ## ## Let's start with an example: a procedure that finds the index of a character ## in a string. @@ -41,16 +38,12 @@ ## ## let found = "abc".find('c') ## assert found.isSome and found.get() == 2 -## -## The ``get`` operation demonstrated above returns the underlying value, or -## raises ``UnpackError`` if there is no value. Note that ``UnpackError`` inherits -## from ``system.Defect``, and should therefore never be catched. Instead, rely on -## checking if the option contains a value with ``isSome`` and ``isNone``. ## -## There is another option for obtaining the value: ``unsafeGet``, but you must -## only use it when you are absolutely sure the value is present (e.g. after -## checking ``isSome``). If you do not care about the tiny overhead that ``get`` -## causes, you should simply never use ``unsafeGet``. +## The `get` operation demonstrated above returns the underlying value, or +## raises `UnpackError` if there is no value. Note that `UnpackError` +## inherits from `system.Defect`, and should therefore never be caught. +## Instead, rely on checking if the option contains a value with +## `isSome <#isSome,Option[T]>`_ and `isNone <#isNone,Option[T]>`_ procs. ## ## How to deal with an absence of a value: ## @@ -79,8 +72,44 @@ type UnpackError* = object of Defect + +proc option*[T](val: T): Option[T] = + ## Can be used to convert a pointer type (`ptr` or `ref`) to an option type. + ## It converts `nil` to `None`. + ## + ## See also: + ## * `some <#some,T>`_ + ## * `none <#none,typedesc>`_ + runnableExamples: + type + Foo = ref object + a: int + b: string + var c: Foo + assert c.isNil + var d = option(c) + assert d.isNone + + result.val = val + when T isnot SomePointer: + result.has = true + proc some*[T](val: T): Option[T] = - ## Returns a ``Option`` that has this value. + ## Returns an `Option` that has the value `val`. + ## + ## See also: + ## * `option <#option,T>`_ + ## * `none <#none,typedesc>`_ + ## * `isSome <#isSome,Option[T]>`_ + runnableExamples: + var + a = some("abc") + b = some(42) + assert $type(a) == "Option[system.string]" + assert b.isSome + assert a.get == "abc" + assert $b == "Some(42)" + when T is SomePointer: assert val != nil result.val = val @@ -88,107 +117,240 @@ proc some*[T](val: T): Option[T] = result.has = true result.val = val -proc option*[T](val: T): Option[T] = - ## Can be used to convert a pointer type to an option type. It - ## converts ``nil`` to the none-option. - result.val = val - when T isnot SomePointer: - result.has = true - proc none*(T: typedesc): Option[T] = - ## Returns an ``Option`` for this type that has no value. + ## Returns an `Option` for this type that has no value. + ## + ## See also: + ## * `option <#option,T>`_ + ## * `some <#some,T>`_ + ## * `isNone <#isNone,Option[T]>`_ + runnableExamples: + var a = none(int) + assert a.isNone + assert $type(a) == "Option[system.int]" + # the default is the none type discard proc none*[T]: Option[T] = - ## Alias for ``none(T)``. + ## Alias for `none(T) proc <#none,typedesc>`_. none(T) proc isSome*[T](self: Option[T]): bool {.inline.} = + ## Checks if an `Option` contains a value. + runnableExamples: + var + a = some(42) + b = none(string) + assert a.isSome + assert not b.isSome + when T is SomePointer: self.val != nil else: self.has proc isNone*[T](self: Option[T]): bool {.inline.} = + ## Checks if an `Option` is empty. + runnableExamples: + var + a = some(42) + b = none(string) + assert not a.isNone + assert b.isNone when T is SomePointer: self.val == nil else: not self.has -proc unsafeGet*[T](self: Option[T]): T = - ## Returns the value of a ``some``. Behavior is undefined for ``none``. - assert self.isSome - self.val - proc get*[T](self: Option[T]): T = - ## Returns contents of the Option. If it is none, then an exception is + ## Returns contents of an `Option`. If it is `None`, then an exception is ## thrown. + ## + ## See also: + ## * `get proc <#get,Option[T],T>`_ with the default return value + runnableExamples: + let + a = some(42) + b = none(string) + assert a.get == 42 + doAssertRaises(UnpackError): + echo b.get + if self.isNone: raise newException(UnpackError, "Can't obtain a value from a `none`") self.val proc get*[T](self: Option[T], otherwise: T): T = - ## Returns the contents of this option or `otherwise` if the option is none. + ## Returns the contents of the `Option` or an `otherwise` value if + ## the `Option` is `None`. + runnableExamples: + var + a = some(42) + b = none(int) + assert a.get(9999) == 42 + assert b.get(9999) == 9999 + if self.isSome: self.val else: otherwise proc get*[T](self: var Option[T]): var T = - ## Returns contents of the Option. If it is none, then an exception is - ## thrown. + ## Returns contents of the `var Option`. If it is `None`, then an exception + ## is thrown. + runnableExamples: + let + a = some(42) + b = none(string) + assert a.get == 42 + doAssertRaises(UnpackError): + echo b.get + if self.isNone: raise newException(UnpackError, "Can't obtain a value from a `none`") return self.val proc map*[T](self: Option[T], callback: proc (input: T)) = - ## Applies a callback to the value in this Option + ## Applies a `callback` function to the value of the `Option`, if it has one. + ## + ## See also: + ## * `map proc <#map,Option[T],proc(T)_2>`_ for a version with a callback + ## which returns a value + ## * `filter proc <#filter,Option[T],proc(T)>`_ + runnableExamples: + var d = 0 + proc saveDouble(x: int) = + d = 2*x + + let + a = some(42) + b = none(int) + + b.map(saveDouble) + assert d == 0 + a.map(saveDouble) + assert d == 84 + if self.isSome: callback(self.val) proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] = - ## Applies a callback to the value in this Option and returns an option - ## containing the new value. If this option is None, None will be returned + ## Applies a `callback` function to the value of the `Option` and returns an + ## `Option` containing the new value. + ## + ## If the `Option` is `None`, `None` of the return type of the `callback` + ## will be returned. + ## + ## See also: + ## * `flatMap proc <#flatMap,Option[A],proc(A)>`_ for a version with a + ## callback which returns an `Option` + ## * `filter proc <#filter,Option[T],proc(T)>`_ + runnableExamples: + var + a = some(42) + b = none(int) + + proc isEven(x: int): bool = + x mod 2 == 0 + + assert $(a.map(isEven)) == "Some(true)" + assert $(b.map(isEven)) == "None[bool]" + if self.isSome: some[R]( callback(self.val) ) else: none(R) proc flatten*[A](self: Option[Option[A]]): Option[A] = - ## Remove one level of structure in a nested Option. + ## Remove one level of structure in a nested `Option`. + runnableExamples: + let a = some(some(42)) + assert $flatten(a) == "Some(42)" + if self.isSome: self.val else: none(A) proc flatMap*[A, B](self: Option[A], callback: proc (input: A): Option[B]): Option[B] = - ## Applies a callback to the value in this Option and returns an - ## option containing the new value. If this option is None, None will be - ## returned. Similar to ``map``, with the difference that the callback - ## returns an Option, not a raw value. This allows multiple procs with a - ## signature of ``A -> Option[B]`` (including A = B) to be chained together. + ## Applies a `callback` function to the value of the `Option` and returns an + ## `Option` containing the new value. + ## + ## If the `Option` is `None`, `None` of the return type of the `callback` + ## will be returned. + ## + ## Similar to `map`, with the difference that the `callback` returns an + ## `Option`, not a raw value. This allows multiple procs with a + ## signature of `A -> Option[B]` to be chained together. + ## + ## See also: + ## * `flatten proc <#flatten,Option[Option[A]]>`_ + ## * `filter proc <#filter,Option[T],proc(T)>`_ + runnableExamples: + proc doublePositives(x: int): Option[int] = + if x > 0: + return some(2*x) + else: + return none(int) + let + a = some(42) + b = none(int) + c = some(-11) + assert a.flatMap(doublePositives) == some(84) + assert b.flatMap(doublePositives) == none(int) + assert c.flatMap(doublePositives) == none(int) + map(self, callback).flatten() proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = - ## Applies a callback to the value in this Option. If the callback returns - ## `true`, the option is returned as a Some. If it returns false, it is - ## returned as a None. + ## Applies a `callback` to the value of the `Option`. + ## + ## If the `callback` returns `true`, the option is returned as `Some`. + ## If it returns `false`, it is returned as `None`. + ## + ## See also: + ## * `map proc <#map,Option[T],proc(T)_2>`_ + ## * `flatMap proc <#flatMap,Option[A],proc(A)>`_ + runnableExamples: + proc isEven(x: int): bool = + x mod 2 == 0 + let + a = some(42) + b = none(int) + c = some(-11) + assert a.filter(isEven) == some(42) + assert b.filter(isEven) == none(int) + assert c.filter(isEven) == none(int) + if self.isSome and not callback(self.val): none(T) else: self proc `==`*(a, b: Option): bool = - ## Returns ``true`` if both ``Option``s are ``none``, - ## or if they have equal values + ## Returns `true` if both `Option`s are `None`, + ## or if they are both `Some` and have equal values. + runnableExamples: + let + a = some(42) + b = none(int) + c = some(42) + d = none(int) + + assert a == c + assert b == d + assert not (a == b) + (a.isSome and b.isSome and a.val == b.val) or (not a.isSome and not b.isSome) proc `$`*[T](self: Option[T]): string = - ## Get the string representation of this option. If the option has a value, - ## the result will be `Some(x)` where `x` is the string representation of the contained value. - ## If the option does not have a value, the result will be `None[T]` where `T` is the name of - ## the type contained in the option. + ## Get the string representation of the `Option`. + ## + ## If the `Option` has a value, the result will be `Some(x)` where `x` + ## is the string representation of the contained value. + ## If the `Option` does not have a value, the result will be `None[T]` + ## where `T` is the name of the type contained in the `Option`. if self.isSome: result = "Some(" result.addQuoted self.val @@ -196,6 +358,16 @@ proc `$`*[T](self: Option[T]): string = else: result = "None[" & name(T) & "]" +proc unsafeGet*[T](self: Option[T]): T = + ## Returns the value of a `some`. Behavior is undefined for `none`. + ## + ## **Note:** Use it only when you are **absolutely sure** the value is present + ## (e.g. after checking `isSome <#isSome,Option[T]>`_). + ## Generally, using `get proc <#get,Option[T]>`_ is preferred. + assert self.isSome + self.val + + when isMainModule: import unittest, sequtils From 4fb73e6d8ff153743cff7c5385ee36a13dc3deec Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 22 Feb 2019 14:09:08 +0100 Subject: [PATCH 0012/1645] fix nimble package tests --- testament/important_packages.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index da43e34159d7..36437b82546c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -36,16 +36,15 @@ pkg "nimgame2", "nim c nimgame2/nimgame.nim", "", true pkg "nimongo", "nimble test_ci", "", true pkg "nimpy", "nim c -o:nimpyy nimpy.nim" pkg "nimsl", "nim c test.nim" -pkg "nimsvg", "nim c src/nimsvg.nim" +pkg "nimsvg" pkg "nimx", "nim c --threads:on test/main.nim", "", true pkg "parsetoml" pkg "patty" pkg "plotly", "nim c examples/all.nim", "", true pkg "protobuf", "nim c -o:protobuff src/protobuf.nim", "", true pkg "regex", "nim c src/regex" -pkg "rosencrantz" +pkg "rosencrantz", "nim c -o:rsncntz rosencrantz.nim" pkg "sdl1", "nim c src/sdl.nim" pkg "sdl2_nim", "nim c sdl2/sdl.nim" pkg "stint", "nim c -o:stintt stint.nim" pkg "zero_functional", "nim c test.nim" - From 15c208cd29313c0888900644f0d516541d3832fc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 22 Feb 2019 18:28:40 +0100 Subject: [PATCH 0013/1645] it's spelt callsite --- compiler/ast.nim | 2 +- compiler/evaltempl.nim | 2 +- compiler/semtempl.nim | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index fbe6132b3769..564a3406bc32 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -271,7 +271,7 @@ type # language; for interfacing with Objective C sfDiscardable, # returned value may be discarded implicitly sfOverriden, # proc is overriden - sfCallSideLineinfo# A flag for template symbols to tell the + sfCallsite # A flag for template symbols to tell the # compiler it should use line information from # the calling side of the macro, not from the # implementation. diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 43c038270e95..14388367a94d 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -186,7 +186,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; renderTree(result, {renderNoComments})) else: result = copyNode(body) - ctx.instLines = sfCallSideLineinfo in tmpl.flags + ctx.instLines = sfCallsite in tmpl.flags if ctx.instLines: result.info = n.info for i in countup(0, safeLen(body) - 1): diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index f180b5373afc..dae2833ec442 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -559,8 +559,9 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = else: s = semIdentVis(c, skTemplate, n.sons[0], {}) - if s.owner != nil and s.owner.name.s == "system" and s.name.s in ["!=", ">=", ">", "incl", "excl", "in", "notin", "isnot"]: - incl(s.flags, sfCallSideLineinfo) + if s.owner != nil and sfSystemModule in s.owner.flags and + s.name.s in ["!=", ">=", ">", "incl", "excl", "in", "notin", "isnot"]: + incl(s.flags, sfCallsite) styleCheckDef(c.config, s) onDef(n[0].info, s) From 721bf7188bfff3a3ae1db44bece57cca3dfe8461 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 22 Feb 2019 19:42:11 +0100 Subject: [PATCH 0014/1645] code cleanup: there is no tyOptRef --- compiler/ast.nim | 2 +- compiler/ccgtypes.nim | 13 +++++++------ compiler/jsgen.nim | 3 +-- compiler/semasgn.nim | 3 +-- compiler/semtypes.nim | 1 + compiler/types.nim | 9 ++++++--- compiler/vmdeps.nim | 2 +- lib/system/assign.nim | 16 +--------------- lib/system/channels.nim | 2 +- lib/system/deepcopy.nim | 2 +- lib/system/gc.nim | 12 ++++++------ lib/system/gc2.nim | 8 ++++---- lib/system/gc_ms.nim | 8 ++++---- lib/system/hti.nim | 2 +- 14 files changed, 36 insertions(+), 47 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 564a3406bc32..e691cc175aac 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -365,7 +365,7 @@ type tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, - tyOptAsRef, tySink, tyLent, + tyOwned, tySink, tyLent, tyVarargs, tyUncheckedArray # An array with boundaries [0,+∞] diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 37d0827de525..3d00b9b0af2c 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -94,7 +94,8 @@ proc scopeMangledParam(p: BProc; param: PSym) = const irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation, - tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred} + tyDistinct, tyRange, tyStatic, tyAlias, tySink, + tyInferred, tyOwned} proc typeName(typ: PType): Rope = let typ = typ.skipTypes(irrelevantForBackend) @@ -145,7 +146,7 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind = doAssert typ.isResolvedUserTypeClass return mapType(conf, typ.lastSon) of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal, - tyTypeDesc, tyAlias, tySink, tyInferred: + tyTypeDesc, tyAlias, tySink, tyInferred, tyOwned: result = mapType(conf, lastSon(typ)) of tyEnum: if firstOrd(conf, typ) < 0: @@ -158,7 +159,7 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind = of 8: result = ctInt64 else: result = ctInt32 of tyRange: result = mapType(conf, typ.sons[0]) - of tyPtr, tyVar, tyLent, tyRef, tyOptAsRef: + of tyPtr, tyVar, tyLent, tyRef: var base = skipTypes(typ.lastSon, typedescInst) case base.kind of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctPtrToArray @@ -633,7 +634,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope = excl(check, t.id) return case t.kind - of tyRef, tyOptAsRef, tyPtr, tyVar, tyLent: + of tyRef, tyPtr, tyVar, tyLent: var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and compileToCpp(m): "&" else: "*" var et = origTyp.skipTypes(abstractInst).lastSon @@ -844,7 +845,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope = of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)]) else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", [result, rope(getSize(m.config, t))]) - of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, + of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyOwned, tyUserTypeClass, tyUserTypeClassInst, tyInferred: result = getTypeDescAux(m, lastSon(t), check) else: @@ -1219,7 +1220,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) - of tyRef, tyOptAsRef: + of tyRef: genTypeInfoAux(m, t, t, result, info) if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 5263230fd5ab..53cfd5632db7 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -194,14 +194,13 @@ proc mapType(typ: PType): TJSTypeKind = tyAnd, tyOr, tyNot, tyAnything, tyVoid: result = etyNone of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst, - tySink: + tySink, tyOwned: result = mapType(typ.lastSon) of tyStatic: if t.n != nil: result = mapType(lastSon t) else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString - of tyOptAsRef: doAssert(false, "mapType") proc mapType(p: PProc; typ: PType): TJSTypeKind = result = mapType(typ) diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 41b0879e663e..6b36dd6fbe6e 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -269,9 +269,8 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = tyTypeDesc, tyGenericInvocation, tyForward: internalError(c.graph.config, c.info, "assignment requested for type: " & typeToString(t)) of tyOrdinal, tyRange, tyInferred, - tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink: + tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink, tyOwned: liftBodyAux(c, lastSon(t), body, x, y) - of tyOptAsRef: internalError(c.graph.config, "liftBodyAux") proc newProcType(info: TLineInfo; owner: PSym): PType = result = newType(tyProc, owner) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9976e9302b36..f570cdffa969 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1854,6 +1854,7 @@ proc processMagicType(c: PContext, m: PSym) = case m.name.s of "lent": setMagicType(c.config, m, tyLent, c.config.target.ptrSize) of "sink": setMagicType(c.config, m, tySink, szUncomputedSize) + of "owned": setMagicType(c.config, m, tyOwned, c.config.target.ptrSize) else: localError(c.config, m.info, errTypeExpected) else: localError(c.config, m.info, errTypeExpected) diff --git a/compiler/types.nim b/compiler/types.nim index 7034917d410c..91076fae3c32 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1054,7 +1054,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = a.sym.position == b.sym.position of tyGenericInvocation, tyGenericBody, tySequence, tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyLent, tySink, tyUncheckedArray, - tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyOpt: + tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyOpt, tyOwned: cycleCheck() if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n result = sameChildrenAux(a, b, c) @@ -1077,7 +1077,6 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = cycleCheck() result = sameTypeAux(a.lastSon, b.lastSon, c) of tyNone: result = false - of tyOptAsRef: result = false proc sameBackendType*(x, y: PType): bool = var c = initSameTypeClosure() @@ -1275,7 +1274,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil - of tyOptAsRef: result = t + of tyOwned: + if t.len == 1 and t.sons[0].kind in {tyRef, tyPtr}: + result = typeAllowedAux(marker, t.lastSon, skVar, flags+{taHeap}) + else: + result = t proc typeAllowed*(t: PType, kind: TSymKind; flags: TTypeAllowedFlags = {}): PType = # returns 'nil' on success and otherwise the part of the type that is diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index edbed5ca1609..6d9c56c0e8b4 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -294,7 +294,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; result.add atomicType("static", mNone) if t.n != nil: result.add t.n.copyTree - of tyOptAsRef: assert(false, "mapTypeToAstX") + of tyOwned: result = mapTypeToBracket("owned", mBuiltinType, t, info) proc opMapTypeToAst*(cache: IdentCache; t: PType; info: TLineInfo): PNode = result = mapTypeToAstX(cache, t, info, inst=false, allowRecursionX=true) diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 2b74e6682408..d4826fe0cddc 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -104,19 +104,6 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = cast[pointer](s +% i *% mt.base.size), mt.base, shallow) of tyRef: unsureAsgnRef(cast[PPointer](dest), cast[PPointer](s)[]) - of tyOptAsRef: - let s2 = cast[PPointer](src)[] - let d = cast[PPointer](dest) - if s2 == nil: - unsureAsgnRef(d, s2) - else: - when declared(usrToCell): - let realType = usrToCell(s2).typ - else: - let realType = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[] - else: mt.base - var z = newObj(realType, realType.base.size) - genericAssignAux(d, addr z, mt.base, shallow) else: copyMem(dest, src, mt.size) # copy raw bits @@ -143,7 +130,6 @@ when false: of tyPtr: k = "ptr" of tyRef: k = "ref" of tyVar: k = "var" - of tyOptAsRef: k = "optref" of tySequence: k = "seq" of tyProc: k = "proc" of tyPointer: k = "range" @@ -219,7 +205,7 @@ proc genericReset(dest: pointer, mt: PNimType) = var d = cast[ByteAddress](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind - of tyString, tyRef, tyOptAsRef, tySequence: + of tyString, tyRef, tySequence: unsureAsgnRef(cast[PPointer](dest), nil) of tyTuple: genericResetAux(dest, mt.node) diff --git a/lib/system/channels.nim b/lib/system/channels.nim index 057ea28436ae..e0eb9ae13cdd 100644 --- a/lib/system/channels.nim +++ b/lib/system/channels.nim @@ -147,7 +147,7 @@ proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, for i in 0..(mt.size div mt.base.size)-1: storeAux(cast[pointer](d +% i*% mt.base.size), cast[pointer](s +% i*% mt.base.size), mt.base, t, mode) - of tyRef, tyOptAsRef: + of tyRef: var s = cast[PPointer](src)[] var x = cast[PPointer](dest) if s == nil: diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 750da00cf197..1bdfe0467f73 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -124,7 +124,7 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = for i in 0..(mt.size div mt.base.size)-1: genericDeepCopyAux(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base, tab) - of tyRef, tyOptAsRef: + of tyRef: let s2 = cast[PPointer](src)[] if s2 == nil: unsureAsgnRef(cast[PPointer](dest), s2) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index d6d7da66e6ef..db5371156b3a 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -295,7 +295,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = for i in 0..n.len-1: # inlined for speed if n.sons[i].kind == nkSlot: - if n.sons[i].typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}: + if n.sons[i].typ.kind in {tyRef, tyString, tySequence}: doOperation(cast[PPointer](d +% n.sons[i].offset)[], op) else: forAllChildrenAux(cast[pointer](d +% n.sons[i].offset), @@ -312,7 +312,7 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind - of tyRef, tyOptAsRef, tyString, tySequence: # leaf: + of tyRef, tyString, tySequence: # leaf: doOperation(cast[PPointer](d)[], op) of tyObject, tyTuple: forAllSlotsAux(dest, mt.node, op) @@ -325,13 +325,13 @@ proc forAllChildren(cell: PCell, op: WalkOp) = gcAssert(cell != nil, "forAllChildren: cell is nil") gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: pointer not part of the heap") gcAssert(cell.typ != nil, "forAllChildren: cell.typ is nil") - gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type" + gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) else: case cell.typ.kind - of tyRef, tyOptAsRef: # common case + of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: var d = cast[ByteAddress](cellToUsr(cell)) @@ -406,7 +406,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 incTypeSize typ, size sysAssert(allocInv(gch.region), "rawNewObj begin") - gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" @@ -452,7 +452,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = # generates a new object and sets its reference counter to 1 incTypeSize typ, size sysAssert(allocInv(gch.region), "newObjRC1 begin") - gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) sysAssert(allocInv(gch.region), "newObjRC1 after collectCT") diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 522b2742b97f..458cfda492fd 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -268,7 +268,7 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind - of tyRef, tyOptAsRef, tyString, tySequence: # leaf: + of tyRef, tyString, tySequence: # leaf: doOperation(cast[PPointer](d)[], op) of tyObject, tyTuple: forAllSlotsAux(dest, mt.node, op) @@ -281,13 +281,13 @@ proc forAllChildren(cell: PCell, op: WalkOp) = gcAssert(cell != nil, "forAllChildren: 1") gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: 4" + gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) else: case cell.typ.kind - of tyRef, tyOptAsRef: # common case + of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: var d = cast[ByteAddress](cellToUsr(cell)) @@ -328,7 +328,7 @@ proc initGC() = proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 sysAssert(allocInv(gch.region), "rawNewObj begin") - gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index bd08eedf0013..f0b25c1892d7 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -236,7 +236,7 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind - of tyRef, tyOptAsRef, tyString, tySequence: # leaf: + of tyRef, tyString, tySequence: # leaf: doOperation(cast[PPointer](d)[], op) of tyObject, tyTuple: forAllSlotsAux(dest, mt.node, op) @@ -248,13 +248,13 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = proc forAllChildren(cell: PCell, op: WalkOp) = gcAssert(cell != nil, "forAllChildren: 1") gcAssert(cell.typ != nil, "forAllChildren: 2") - gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: 3" + gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 3" let marker = cell.typ.marker if marker != nil: marker(cellToUsr(cell), op.int) else: case cell.typ.kind - of tyRef, tyOptAsRef: # common case + of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: when not defined(gcDestructors): @@ -269,7 +269,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # generates a new object and sets its reference counter to 0 incTypeSize typ, size - gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") + gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") diff --git a/lib/system/hti.nim b/lib/system/hti.nim index 6e6f109deec6..e23c6418cf2c 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -56,7 +56,7 @@ type tyUInt16, tyUInt32, tyUInt64, - tyOptAsRef, tyUnused1, tyUnused2, + tyOwned, tyUnused1, tyUnused2, tyVarargsHidden, tyUncheckedArray, tyProxyHidden, From ea409fb15a62ff098e5de70efa780228b5d5aac0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Feb 2019 08:44:08 +0100 Subject: [PATCH 0015/1645] first steps in implementing 'owned' pointers; undocumented, do not use --- compiler/ccgexprs.nim | 8 ++++---- compiler/ccgtrav.nim | 6 +++--- compiler/ccgtypes.nim | 6 +++--- compiler/jsgen.nim | 16 +++++++++------- compiler/jstypes.nim | 2 +- compiler/semexprs.nim | 14 +++++++------- compiler/semstmts.nim | 14 +++++++------- compiler/semtypes.nim | 8 ++++---- compiler/semtypinst.nim | 2 +- compiler/sighashes.nim | 2 +- compiler/sigmatch.nim | 30 +++++++++++++++++++++++------- compiler/types.nim | 21 ++++++++++++--------- 12 files changed, 75 insertions(+), 54 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index d459f6cbf810..7e0437c39349 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -679,14 +679,14 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = #if e[0].kind != nkBracketExpr: # message(e.info, warnUser, "CAME HERE " & renderTree(e)) expr(p, e.sons[0], d) - if e.sons[0].typ.skipTypes(abstractInst).kind == tyRef: + if e.sons[0].typ.skipTypes(abstractInstOwned).kind == tyRef: d.storage = OnHeap else: var a: TLoc var typ = e.sons[0].typ if typ.kind in {tyUserTypeClass, tyUserTypeClassInst} and typ.isResolvedUserTypeClass: typ = typ.lastSon - typ = typ.skipTypes(abstractInst) + typ = typ.skipTypes(abstractInstOwned) if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr: initLocExprSingleUse(p, e[0][0], d) return @@ -726,7 +726,7 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = proc genAddr(p: BProc, e: PNode, d: var TLoc) = # careful 'addr(myptrToArray)' needs to get the ampersand: - if e.sons[0].typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}: + if e.sons[0].typ.skipTypes(abstractInstOwned).kind in {tyRef, tyPtr}: var a: TLoc initLocExpr(p, e.sons[0], a) putIntoDest(p, d, e, "&" & a.r, a.storage) @@ -1158,7 +1158,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) = let typ = a.t var b: TLoc initLoc(b, locExpr, a.lode, OnHeap) - let refType = typ.skipTypes(abstractInst) + let refType = typ.skipTypes(abstractInstOwned) assert refType.kind == tyRef let bt = refType.lastSon if sizeExpr.isNil: diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index c69bb2c805cb..0a2bbf93bd8a 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -67,7 +67,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) = var p = c.p case typ.kind of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred, - tySink: + tySink, tyOwned: genTraverseProc(c, accessor, lastSon(typ)) of tyArray: let arraySize = lengthOrd(c.p.config, typ.sons[0]) @@ -134,7 +134,7 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = var c: TTraversalClosure var p = newProc(nil, m) result = "Marker_" & getTypeName(m, origTyp, sig) - var typ = origTyp.skipTypes(abstractInst) + var typ = origTyp.skipTypes(abstractInstOwned) let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result] @@ -149,7 +149,7 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = if typ.kind == tySequence: genTraverseProcSeq(c, "a".rope, typ) else: - if skipTypes(typ.sons[0], typedescInst).kind == tyArray: + if skipTypes(typ.sons[0], typedescInst+{tyOwned}).kind == tyArray: # C's arrays are broken beyond repair: genTraverseProc(c, "a".rope, typ.sons[0]) else: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 3d00b9b0af2c..afe90544d1ba 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -115,7 +115,7 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope = t = t.lastSon else: break - let typ = if typ.kind in {tyAlias, tySink}: typ.lastSon else: typ + let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ if typ.loc.r == nil: typ.loc.r = typ.typeName & $sig else: @@ -296,7 +296,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyStatic: if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ) else: internalError(m.config, "tyStatic for getSimpleTypeDesc") - of tyGenericInst, tyAlias, tySink: + of tyGenericInst, tyAlias, tySink, tyOwned: result = getSimpleTypeDesc(m, lastSon typ) else: result = nil @@ -619,7 +619,7 @@ proc getSeqPayloadType(m: BModule; t: PType): Rope = proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope = # returns only the type's name - var t = origTyp.skipTypes(irrelevantForBackend) + var t = origTyp.skipTypes(irrelevantForBackend-{tyOwned}) if containsOrIncl(check, t.id): if not (isImportedCppType(origTyp) or isImportedCppType(t)): internalError(m.config, "cannot generate C type for: " & typeToString(origTyp)) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 53cfd5632db7..cad8fc9909d1 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -927,7 +927,7 @@ proc needsNoCopy(p: PProc; y: PNode): bool = return y.kind in nodeKindsNeedNoCopy or ((mapType(y.typ) != etyBaseIndex or (y.kind == nkSym and y.sym.kind == skParam)) and (skipTypes(y.typ, abstractInst).kind in - {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc} + IntegralTypes)) + {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc, tyOwned} + IntegralTypes)) proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = var a, b: TCompRes @@ -1120,7 +1120,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) = proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) = var ty = skipTypes(n.sons[0].typ, abstractVarRange) - if ty.kind in {tyRef, tyPtr, tyLent}: ty = skipTypes(ty.lastSon, abstractVarRange) + if ty.kind in {tyRef, tyPtr, tyLent, tyOwned}: ty = skipTypes(ty.lastSon, abstractVarRange) case ty.kind of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs: genArrayAddr(p, n, r) @@ -1340,7 +1340,8 @@ proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes; emitted: ptr int = add(r.res, ", ") add(r.res, a.res) if emitted != nil: inc emitted[] - elif n.typ.kind in {tyVar, tyPtr, tyRef, tyLent} and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex: + elif n.typ.kind in {tyVar, tyPtr, tyRef, tyLent, tyOwned} and + n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex: # this fixes bug #5608: let tmp = getTemp(p) add(r.res, "($1 = $2, $1[0]), $1[1]" % [tmp, a.rdLoc]) @@ -1540,7 +1541,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("0", indirect) of tyFloat..tyFloat128: result = putToSeq("0.0", indirect) - of tyRange, tyGenericInst, tyAlias, tySink: + of tyRange, tyGenericInst, tyAlias, tySink, tyOwned: result = createVar(p, lastSon(typ), indirect) of tySet: result = putToSeq("{}", indirect) @@ -1619,7 +1620,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = if n.kind == nkEmpty: if not isIndirect(v) and - v.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and mapType(p, v.typ) == etyBaseIndex: + v.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and mapType(p, v.typ) == etyBaseIndex: lineF(p, "var $1 = null;$n", [varName]) lineF(p, "var $1_Idx = 0;$n", [varName]) else: @@ -1807,7 +1808,8 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) = proc genOf(p: PProc, n: PNode, r: var TCompRes) = var x: TCompRes - let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyLent, tyTypeDesc}) + let t = skipTypes(n.sons[2].typ, + abstractVarRange+{tyRef, tyPtr, tyLent, tyTypeDesc, tyOwned}) gen(p, n.sons[1], x) if tfFinal in t.flags: r.res = "($1.m_type == $2)" % [x.res, genTypeInfo(p, t)] @@ -2172,7 +2174,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = resultSym = prc.ast.sons[resultPos].sym let mname = mangleName(p.module, resultSym) if not isindirect(resultSym) and - resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and + resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and mapType(p, resultSym.typ) == etyBaseIndex: resultAsgn = p.indentLine(("var $# = null;$n") % [mname]) resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index d86b09a03d25..743158505a8d 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -122,7 +122,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) = [name, genTypeInfo(p, typ.sons[0])]) proc genTypeInfo(p: PProc, typ: PType): Rope = - let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink}) + let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned}) result = "NTI$1" % [rope(t.id)] if containsOrIncl(p.g.typeInfoGenerated, t.id): return case t.kind diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 239dbad5441a..7bd40a95431e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -127,7 +127,7 @@ proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus = s = s.lastSon s = skipTypes(s, abstractVar-{tyTypeDesc}) var pointers = 0 - while (d != nil) and (d.kind in {tyPtr, tyRef}) and (d.kind == s.kind): + while (d != nil) and (d.kind in {tyPtr, tyRef, tyOwned}) and (d.kind == s.kind): d = d.lastSon s = s.lastSon inc pointers @@ -224,7 +224,7 @@ proc semConv(c: PContext, n: PNode): PNode = maybeLiftType(targetType, c, n[0].info) - if targetType.kind in {tySink, tyLent}: + if targetType.kind in {tySink, tyLent, tyOwned}: let baseType = semTypeNode(c, n.sons[1], nil).skipTypes({tyTypeDesc}) let t = newTypeS(targetType.kind, c) t.rawAddSonNoPropagationOfTypeFlags baseType @@ -997,7 +997,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, const tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass} - tyDotOpTransparent = {tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink} + tyDotOpTransparent = {tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink} proc readTypeParameter(c: PContext, typ: PType, paramName: PIdent, info: TLineInfo): PNode = @@ -1150,7 +1150,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = p = p.next if p != nil and p.selfSym != nil: var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, - tyAlias, tySink}) + tyAlias, tySink, tyOwned}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1282,7 +1282,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = return nil if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon - ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) + ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1353,7 +1353,7 @@ proc semDeref(c: PContext, n: PNode): PNode = checkSonsLen(n, 1, c.config) n.sons[0] = semExprWithType(c, n.sons[0]) result = n - var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyAlias, tySink}) + var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyAlias, tySink, tyOwned}) case t.kind of tyRef, tyPtr: n.typ = t.lastSon else: result = nil @@ -1372,7 +1372,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = # make sure we don't evaluate generic macros/templates n.sons[0] = semExprWithType(c, n.sons[0], {efNoEvaluateGeneric}) - var arr = skipTypes(n.sons[0].typ, {tyGenericInst, tyUserTypeClassInst, + var arr = skipTypes(n.sons[0].typ, {tyGenericInst, tyUserTypeClassInst, tyOwned, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) if arr.kind == tyStatic: if arr.base.kind == tyNone: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3827da220cfd..363049672017 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -391,18 +391,18 @@ proc makeDeref(n: PNode): PNode = var t = n.typ if t.kind in tyUserTypeClasses and t.isResolvedUserTypeClass: t = t.lastSon - t = skipTypes(t, {tyGenericInst, tyAlias, tySink}) + t = skipTypes(t, {tyGenericInst, tyAlias, tySink, tyOwned}) result = n if t.kind in {tyVar, tyLent}: result = newNodeIT(nkHiddenDeref, n.info, t.sons[0]) addSon(result, n) - t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink}) + t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink, tyOwned}) while t.kind in {tyPtr, tyRef}: var a = result let baseTyp = t.lastSon result = newNodeIT(nkHiddenDeref, n.info, baseTyp) addSon(result, a) - t = skipTypes(baseTyp, {tyGenericInst, tyAlias, tySink}) + t = skipTypes(baseTyp, {tyGenericInst, tyAlias, tySink, tyOwned}) proc fillPartialObject(c: PContext; n: PNode; typ: PType) = if n.len == 2: @@ -908,7 +908,7 @@ proc semRaise(c: PContext, n: PNode): PNode = n[0] = semExprWithType(c, n[0]) var typ = n[0].typ if not isImportedException(typ, c.config): - typ = typ.skipTypes({tyAlias, tyGenericInst}) + typ = typ.skipTypes({tyAlias, tyGenericInst, tyOwned}) if typ.kind != tyRef: localError(c.config, n.info, errExprCannotBeRaised) if typ.len > 0 and not isException(typ.lastSon): @@ -1059,7 +1059,7 @@ proc checkCovariantParamsUsages(c: PContext; genericType: PType) = of tyPtr, tyRef, tyVar, tyLent: if t.base.kind == tyGenericParam: return true return traverseSubTypes(c, t.base) - of tyDistinct, tyAlias, tySink: + of tyDistinct, tyAlias, tySink, tyOwned: return traverseSubTypes(c, t.lastSon) of tyGenericInst: internalAssert c.config, false @@ -1180,7 +1180,7 @@ proc checkForMetaFields(c: PContext; n: PNode) = let t = n.sym.typ case t.kind of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyLent, tyPtr, tyRef, - tyProc, tyGenericInvocation, tyGenericInst, tyAlias, tySink: + tyProc, tyGenericInvocation, tyGenericInst, tyAlias, tySink, tyOwned: let start = ord(t.kind in {tyGenericInvocation, tyGenericInst}) for i in start ..< t.len: checkMeta(c, n, t.sons[i]) @@ -1636,7 +1636,7 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) = if t != nil and t.kind == tyGenericInvocation: var x = skipTypes(t.sons[0], {tyVar, tyLent, tyPtr, tyRef, tyGenericInst, tyGenericInvocation, tyGenericBody, - tyAlias, tySink}) + tyAlias, tySink, tyOwned}) if x.kind == tyObject and t.len-1 == n.sons[genericParamsPos].len: foundObj = true x.methods.add((col,s)) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f570cdffa969..d4fcea0b4208 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -750,7 +750,7 @@ proc skipGenericInvocation(t: PType): PType {.inline.} = result = t if result.kind == tyGenericInvocation: result = result.sons[0] - while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias, tySink}: + while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias, tySink, tyOwned}: result = lastSon(result) proc addInheritedFields(c: PContext, check: var IntSet, pos: var int, @@ -955,7 +955,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, # disable the bindOnce behavior for the type class result = liftingWalk(paramType.base, true) - of tyAlias: + of tyAlias, tyOwned: result = liftingWalk(paramType.base) of tySequence, tySet, tyArray, tyOpenArray, @@ -1528,7 +1528,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = semRangeAux(c, n, prev) elif n[0].kind == nkNilLit and n.len == 2: result = semTypeNode(c, n.sons[1], prev) - if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes: + if result.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).kind in NilableTypes+GenericTypes: if tfNotNil in result.flags: result = freshType(result, prev) result.flags.excl(tfNotNil) @@ -1556,7 +1556,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = case n.len of 3: result = semTypeNode(c, n.sons[1], prev) - if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes+{tyForward} and + if result.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).kind in NilableTypes+GenericTypes+{tyForward} and n.sons[2].kind == nkNilLit: result = freshType(result, prev) result.flags.incl(tfNotNil) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index ceabd8e60dee..002f4f402a0a 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -574,7 +574,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = var r = replaceTypeVarsT(cl, result.sons[i]) if result.kind == tyObject: # carefully coded to not skip the precious tyGenericInst: - let r2 = r.skipTypes({tyAlias, tySink}) + let r2 = r.skipTypes({tyAlias, tySink, tyOwned}) if r2.kind in {tyPtr, tyRef}: r = skipTypes(r2, {tyPtr, tyRef}) result.sons[i] = r diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 3096d94a0c17..95240867f694 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -165,7 +165,7 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = c.hashType t.sons[i], flags else: c.hashType t.lastSon, flags - of tyAlias, tySink, tyUserTypeClasses, tyInferred: + of tyAlias, tySink, tyUserTypeClasses, tyInferred, tyOwned: c.hashType t.lastSon, flags of tyBool, tyChar, tyInt..tyUInt64: # no canonicalization for integral types, so that e.g. ``pid_t`` is diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 3eaac06e5105..cb71c1c81b40 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -188,7 +188,7 @@ proc sumGeneric(t: PType): int = case t.kind of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct, tyUncheckedArray, tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody, - tyLent: + tyLent, tyOwned: t = t.lastSon inc result of tyOr: @@ -476,7 +476,7 @@ proc skipToObject(t: PType; skipped: var SkippedPtr): PType = inc ptrs skipped = skippedPtr r = r.lastSon - of tyGenericBody, tyGenericInst, tyAlias, tySink: + of tyGenericBody, tyGenericInst, tyAlias, tySink, tyOwned: r = r.lastSon else: break @@ -919,7 +919,8 @@ proc inferStaticsInRange(c: var TCandidate, doInferStatic(lowerBound, upperBound.intVal + 1 - lengthOrd(c.c.config, concrete)) template subtypeCheck() = - if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}: + if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in { + tyRef, tyPtr, tyVar, tyLent, tyOwned}: result = isNone proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = @@ -927,11 +928,11 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = assert f.kind == a.kind template baseTypesCheck(lhs, rhs: PType): bool = - lhs.kind notin {tyPtr, tyRef, tyVar, tyLent} and + lhs.kind notin {tyPtr, tyRef, tyVar, tyLent, tyOwned} and typeRel(c, lhs, rhs, {trNoCovariance}) == isSubtype case f.kind - of tyRef, tyPtr: + of tyRef, tyPtr, tyOwned: return baseTypesCheck(f.base, a.base) of tyGenericInst: let body = f.base @@ -962,6 +963,9 @@ when false: of tyFloat64: greater({tyFloat128}) else: discard +template skipOwned(a) = + if a.kind == tyOwned: a = a.skipTypes({tyOwned, tyGenericInst}) + proc typeRelImpl(c: var TCandidate, f, aOrig: PType, flags: TTypeRelFlags = {}): TTypeRelation = # typeRel can be used to establish various relationships between types: @@ -1279,6 +1283,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, #internalError("forward type in typeRel()") result = isNone of tyNil: + skipOwned(a) if a.kind == f.kind: result = isEqual of tyTuple: if a.kind == tyTuple: result = recordRel(c, f, a) @@ -1298,7 +1303,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, #elif f.base.kind == tyAnything: result = isGeneric # issue 4435 elif c.coerceDistincts: result = typeRel(c, f.base, a) elif a.kind == tyNil and f.base.kind in NilableTypes: - result = f.allowsNil + result = f.allowsNil # XXX remove this typing rule, it is not in the spec elif c.coerceDistincts: result = typeRel(c, f.base, a) of tySet: if a.kind == tySet: @@ -1309,6 +1314,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, if result <= isConvertible: result = isNone # BUGFIX! of tyPtr, tyRef: + skipOwned(a) if a.kind == f.kind: # ptr[R, T] can be passed to ptr[T], but not the other way round: if a.len < f.len: return isNone @@ -1322,10 +1328,18 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, elif a.kind == tyNil: result = f.allowsNil else: discard of tyProc: + skipOwned(a) result = procTypeRel(c, f, a) if result != isNone and tfNotNil in f.flags and tfNotNil notin a.flags: result = isNilConversion + of tyOwned: + case a.kind + of tyOwned: + result = typeRel(c, lastSon(f), lastSon(a)) + of tyNil: result = f.allowsNil + else: discard of tyPointer: + skipOwned(a) case a.kind of tyPointer: if tfNotNil in f.flags and tfNotNil notin a.flags: @@ -1917,8 +1931,10 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, # this will be done earlier - we just have to # make sure that static types enter here - # XXX: weaken tyGenericParam and call it tyGenericPlaceholder + # Zahary: weaken tyGenericParam and call it tyGenericPlaceholder # and finally start using tyTypedesc for generic types properly. + # Araq: This would only shift the problems around, in 'proc p[T](x: T)' + # the T is NOT a typedesc. if a.kind == tyGenericParam and tfWildcard in a.flags: a.assignType(f) # put(m.bindings, f, a) diff --git a/compiler/types.nim b/compiler/types.nim index 91076fae3c32..0e2c3b65142d 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -52,20 +52,21 @@ const # TODO: Remove tyTypeDesc from each abstractX and (where necessary) # replace with typedescX abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, - tyTypeDesc, tyAlias, tyInferred, tySink, tyLent} + tyTypeDesc, tyAlias, tyInferred, tySink, tyLent, tyOwned} abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, - tyAlias, tyInferred, tySink, tyLent} + tyAlias, tyInferred, tySink, tyLent, tyOwned} abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc, - tyAlias, tyInferred, tySink} + tyAlias, tyInferred, tySink, tyOwned} abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tyInferred, tySink} abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tyInferred, tySink} + abstractInstOwned* = abstractInst + {tyOwned} skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc, tyAlias, - tyInferred, tySink, tyLent} + tyInferred, tySink, tyLent, tyOwned} # typedescX is used if we're sure tyTypeDesc should be included (or skipped) typedescPtrs* = abstractPtrs + {tyTypeDesc} - typedescInst* = abstractInst + {tyTypeDesc} + typedescInst* = abstractInst + {tyTypeDesc, tyOwned} type TTypeFieldResult* = enum @@ -323,7 +324,7 @@ proc canFormAcycleAux(marker: var IntSet, typ: PType, startId: int): bool = result = false if typ == nil: return if tfAcyclic in typ.flags: return - var t = skipTypes(typ, abstractInst-{tyTypeDesc}) + var t = skipTypes(typ, abstractInst+{tyOwned}-{tyTypeDesc}) if tfAcyclic in t.flags: return case t.kind of tyTuple, tyObject, tyRef, tySequence, tyArray, tyOpenArray, tyVarargs: @@ -399,8 +400,8 @@ const "int", "int8", "int16", "int32", "int64", "float", "float32", "float64", "float128", "uint", "uint8", "uint16", "uint32", "uint64", - "opt", "sink", - "lent ", "varargs[$1]", "UncheckedArray[$1]", "Error Type", + "owned", "sink", + "lent", "varargs[$1]", "UncheckedArray[$1]", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "inferred", "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", @@ -622,6 +623,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = typeToStr[t.kind] % typeToString(t.sons[0]) of tySink: result = "sink " & typeToString(t.sons[0]) + of tyOwned: + result = "owned " & typeToString(t.sons[0]) else: result = typeToStr[t.kind] result.addTypeFlags(t) @@ -1328,7 +1331,7 @@ proc baseOfDistinct*(t: PType): PType = result = copyType(t, t.owner, false) var parent: PType = nil var it = result - while it.kind in {tyPtr, tyRef}: + while it.kind in {tyPtr, tyRef, tyOwned}: parent = it it = it.lastSon if it.kind == tyDistinct and parent != nil: From e7878c0d0887b198583c96c854d6811e1a5907ad Mon Sep 17 00:00:00 2001 From: Miran Date: Sat, 23 Feb 2019 10:41:35 +0100 Subject: [PATCH 0016/1645] add tests for recently closed issues (#10722) --- tests/destructor/t7346.nim | 3 +- tests/enum/tenum.nim | 9 ++++ tests/iter/titer12.nim | 82 ++++++++++++++++++++++++++++++++++ tests/misc/tsizeof3.nim | 19 ++++++++ tests/misc/tunsignedconv.nim | 4 ++ tests/misc/tunsignedinc.nim | 6 +++ tests/pragmas/tdeprecated.nim | 10 +++++ tests/template/tmethodcall.nim | 24 ++++++++++ tests/types/tcast1.nim | 21 +++++++++ 9 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tests/iter/titer12.nim create mode 100644 tests/misc/tsizeof3.nim create mode 100644 tests/pragmas/tdeprecated.nim create mode 100644 tests/template/tmethodcall.nim create mode 100644 tests/types/tcast1.nim diff --git a/tests/destructor/t7346.nim b/tests/destructor/t7346.nim index ef2fd5b7994f..17f56681e6d5 100644 --- a/tests/destructor/t7346.nim +++ b/tests/destructor/t7346.nim @@ -7,4 +7,5 @@ type proc `=`(a: var Obj, b: Obj) = discard -let a: seq[Obj] = @[] \ No newline at end of file +let a: seq[Obj] = @[] # bug #7346 +let b = newSeq[Obj]() # bug #7345 diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index 72f1837f56bc..8cc578d3cce1 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -145,3 +145,12 @@ block toptions: optOverflowCheck, optAssert, optWarns, optHints, optLineDir, optStackTrace} compilerArgs: int gExitcode: int8 + + + +block nonzero: # bug #6959 + type SomeEnum = enum + A = 10 + B + C + let slice = SomeEnum.low..SomeEnum.high diff --git a/tests/iter/titer12.nim b/tests/iter/titer12.nim new file mode 100644 index 000000000000..f7fc64da4eb5 --- /dev/null +++ b/tests/iter/titer12.nim @@ -0,0 +1,82 @@ +discard """ +output: ''' +Selecting 2 +1.0 +Selecting 4 +2.0 +''' +""" + + +# bug #5522 +import macros, sugar, sequtils + +proc tryS(f: () -> void): void = + (try: f() except: discard) + +template trySTImpl(body: untyped): untyped = + tryS do() -> auto: + `body` + +macro tryST*(body: untyped): untyped = + var b = if body.kind == nnkDo: body[^1] else: body + result = quote do: + trySTImpl((block: + `b` + )) + +iterator testIt(): int {.closure.} = + for x in 0..10: + yield x + +var xs = newSeq[int]() +proc test = tryST do: + for x in testIt(): + xs.add(x) + +test() + +doAssert xs == toSeq(0..10) + + + +# bug #5690 +proc filter[T](it: (iterator : T), f: proc(x: T): bool): (iterator : T) = + return iterator (): T {.closure.} = + for x in it(): + if f(x): + yield x + +proc len[T](it : iterator : T) : Natural = + for i in it(): + result += 1 + +proc simpleSeqIterator(s :seq[int]) : iterator : int = + iterator it: int {.closure.} = + for x in s: + yield x + result = it + +let a = newSeq[int](99) + +doAssert len(simpleSeqIterator(a).filter(proc(x : int) : bool = true)) == 99 + + + +# bug #5340 +proc where[A](input: seq[A], filter: (A) -> bool): iterator (): A = + result = iterator (): A {.closure.} = + for item in input: + if filter(item): + yield item + +proc select[A,B](input: iterator(): A {.closure.}, selector: (A) -> B): iterator (): B {.closure.} = + result = iterator (): B = + for item in input(): + echo "Selecting " & $item + yield selector(item) + +let query = @[1,2,3,4].where(x=>x mod 2==0).select((x)=>x/2) + +for i in query(): + echo $i diff --git a/tests/misc/tsizeof3.nim b/tests/misc/tsizeof3.nim new file mode 100644 index 000000000000..e04ce8a0c1b4 --- /dev/null +++ b/tests/misc/tsizeof3.nim @@ -0,0 +1,19 @@ +discard """ +output: ''' +[0, 0, 0, 0, 0, 0, 48, 57] +''' +""" +# bug #7238 + +type ByteArrayBE*[N: static[int]] = array[N, byte] + ## A byte array that stores bytes in big-endian order + +proc toByteArrayBE*[T: SomeInteger](num: T): ByteArrayBE[sizeof(T)]= + ## Convert an integer (in native host endianness) to a big-endian byte array + ## Notice the result type + const N = T.sizeof + for i in 0 ..< N: + result[i] = byte(num shr ((N-1-i) * 8)) + +let a = 12345.toByteArrayBE +echo a diff --git a/tests/misc/tunsignedconv.nim b/tests/misc/tunsignedconv.nim index 956e014da032..efe5f1302432 100644 --- a/tests/misc/tunsignedconv.nim +++ b/tests/misc/tunsignedconv.nim @@ -39,3 +39,7 @@ var n32 = ar[v32] var n64 = ar[v64] +block t4176: + var yyy: uint8 = 0 + yyy = yyy - 127 + doAssert type(yyy) is uint8 diff --git a/tests/misc/tunsignedinc.nim b/tests/misc/tunsignedinc.nim index 60c0559b0f07..6d1b627d676d 100644 --- a/tests/misc/tunsignedinc.nim +++ b/tests/misc/tunsignedinc.nim @@ -26,3 +26,9 @@ block: var x = 123'u16 x -= 125 doAssert(x == 65534'u16) + +block t4175: + let i = 0u - 1u + const j = 0u - 1u + doAssert i == j + doAssert j + 1u == 0u diff --git a/tests/pragmas/tdeprecated.nim b/tests/pragmas/tdeprecated.nim new file mode 100644 index 000000000000..a5d07f727577 --- /dev/null +++ b/tests/pragmas/tdeprecated.nim @@ -0,0 +1,10 @@ +# bug #6436 +proc foo(size: int, T: typedesc): seq[T] {.deprecated.}= + result = newSeq[T](size) + +proc foo[T](size: int): seq[T]= + result = newSeq[T](size) + +let bar = foo[int](3) # Warning foo is deprecated + +doAssert bar == @[0, 0, 0] diff --git a/tests/template/tmethodcall.nim b/tests/template/tmethodcall.nim new file mode 100644 index 000000000000..d209443c8e1a --- /dev/null +++ b/tests/template/tmethodcall.nim @@ -0,0 +1,24 @@ +# bug #5909 +type + Vec2[T] = tuple + x,y: T + Vec2f = Vec2[float32] + +proc vec2f(x,y: float): Vec2f = + result.x = x + result.y = y + +proc `-`[T](a,b: Vec2[T]): Vec2[T] = + result.x = a.x - b.x + result.y = a.y - b.y + +proc foo[T](a: Vec2[T]): Vec2[T] = + result = a + +block: + # this being called foo is a problem when calling .foo() + var foo = true + + let a = vec2f(1.0,0.0) + let b = vec2f(3.0,1.0) + let c = (a - b).foo() # breaks diff --git a/tests/types/tcast1.nim b/tests/types/tcast1.nim new file mode 100644 index 000000000000..10e876ded5ec --- /dev/null +++ b/tests/types/tcast1.nim @@ -0,0 +1,21 @@ +discard """ +output: ''' +@[1.0, 2.0, 3.0] +@[1.0, 2.0, 3.0] +''' +""" + +# bug #6406 + +import sequtils + +proc remap1(s: seq[int], T: typedesc): seq[T] = + s.map do (x: int) -> T: + x.T + +proc remap2[T](s: seq[int], typ: typedesc[T]): seq[T] = + s.map do (x: int) -> T: + x.T + +echo remap1(@[1,2,3], float) +echo remap2(@[1,2,3], float) From 30ab7e6bdd779b6ef6c9a21507b6cf18f56024a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Sat, 23 Feb 2019 10:58:40 +0100 Subject: [PATCH 0017/1645] fixes #10678 (#10681) --- compiler/semtypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index d4fcea0b4208..e717c6e07d5a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -145,7 +145,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = proc semSet(c: PContext, n: PNode, prev: PType): PType = result = newOrPrevType(tySet, prev, c) - if sonsLen(n) == 2: + if sonsLen(n) == 2 and n.sons[1].kind != nkEmpty: var base = semTypeNode(c, n.sons[1], nil) addSonSkipIntLit(result, base) if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base) From adbabf145c109d7014f55227c429a933266dc2dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 23 Feb 2019 02:31:01 -0800 Subject: [PATCH 0018/1645] FFI at CT (#10150) * enable FFI at CT * rename useFFI=>nimHasLibFFI; improve formatting rawExecute traceCode * disable libffi on windows (works for win32, not yet win64) --- compiler/commands.nim | 2 +- compiler/evalffi.nim | 243 +++++++++++++++++++++--------------------- compiler/main.nim | 1 + compiler/options.nim | 6 +- compiler/vm.nim | 37 +++++-- compiler/vmdef.nim | 1 - compiler/vmgen.nim | 8 +- koch.nim | 7 ++ tests/vm/tevalffi.nim | 81 ++++++++++++++ 9 files changed, 250 insertions(+), 136 deletions(-) create mode 100644 tests/vm/tevalffi.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index af775f5cd2dd..56fe8f2057a5 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -34,7 +34,7 @@ bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") bootSwitch(usedNativeStacktrace, defined(nativeStackTrace) and nativeStackTraceSupported, "-d:nativeStackTrace") -bootSwitch(usedFFI, hasFFI, "-d:useFFI") +bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI") type TCmdLinePass* = enum diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index e863c89954bd..ab57457873da 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -9,43 +9,47 @@ ## This file implements the FFI part of the evaluator for Nim code. -import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs, os +import ast, astalgo, ropes, types, options, tables, dynlib, msgs, os, lineinfos +import pkg/libffi when defined(windows): const libcDll = "msvcrt.dll" -else: +elif defined(linux): const libcDll = "libc.so(.6|.5|)" +elif defined(osx): + const libcDll = "/usr/lib/libSystem.dylib" +else: + {.error: "`libcDll` not implemented on this platform".} type - TDllCache = tables.TTable[string, TLibHandle] + TDllCache = tables.Table[string, LibHandle] var - gDllCache = initTable[string, TLibHandle]() + gDllCache = initTable[string, LibHandle]() when defined(windows): var gExeHandle = loadLib(os.getAppFilename()) else: var gExeHandle = loadLib() -proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer = - result = cache[dll] +proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo): pointer = + if dll in cache: + return cache[dll] + var libs: seq[string] + libCandidates(dll, libs) + for c in libs: + result = loadLib(c) + if not result.isNil: break if result.isNil: - var libs: seq[string] = @[] - libCandidates(dll, libs) - for c in libs: - result = loadLib(c) - if not result.isNil: break - if result.isNil: - globalError(info, "cannot load: " & dll) - cache[dll] = result + globalError(conf, info, "cannot load: " & dll) + cache[dll] = result const nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon var myerrno {.importc: "errno", header: "".}: cint ## error variable -proc importcSymbol*(sym: PSym): PNode = - let name = ropeToStr(sym.loc.r) - +proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode = + let name = $sym.loc.r # the AST does not support untyped pointers directly, so we use an nkIntLit # that contains the address instead: result = newNodeIT(nkPtrLit, sym.info, sym.typ) @@ -57,28 +61,28 @@ proc importcSymbol*(sym: PSym): PNode = else: let lib = sym.annex if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: - globalError(sym.info, "dynlib needs to be a string lit for the REPL") + globalError(conf, sym.info, "dynlib needs to be a string lit") var theAddr: pointer - if lib.isNil and not gExehandle.isNil: + if (lib.isNil or lib.kind == libHeader) and not gExehandle.isNil: # first try this exe itself: theAddr = gExehandle.symAddr(name) # then try libc: if theAddr.isNil: - let dllhandle = gDllCache.getDll(libcDll, sym.info) + let dllhandle = getDll(conf, gDllCache, libcDll, sym.info) theAddr = dllhandle.symAddr(name) elif not lib.isNil: - let dllhandle = gDllCache.getDll(if lib.kind == libHeader: libcDll - else: lib.path.strVal, sym.info) + let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal + let dllhandle = getDll(conf, gDllCache, dll, sym.info) theAddr = dllhandle.symAddr(name) - if theAddr.isNil: globalError(sym.info, "cannot import: " & sym.name.s) + if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s) result.intVal = cast[ByteAddress](theAddr) -proc mapType(t: ast.PType): ptr libffi.TType = +proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType = if t == nil: return addr libffi.type_void case t.kind of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet: - case t.getSize + case getSize(conf, t) of 1: result = addr libffi.type_uint8 of 2: result = addr libffi.type_sint16 of 4: result = addr libffi.type_sint32 @@ -90,87 +94,87 @@ proc mapType(t: ast.PType): ptr libffi.TType = tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil: result = addr libffi.type_pointer of tyDistinct, tyAlias, tySink: - result = mapType(t.sons[0]) + result = mapType(conf, t.sons[0]) else: result = nil # too risky: #of tyFloat128: result = addr libffi.type_longdouble -proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI = +proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI = case cc of ccDefault: result = DEFAULT_ABI - of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI + of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI of ccCDecl: result = DEFAULT_ABI else: - globalError(info, "cannot map calling convention to FFI") + globalError(conf, info, "cannot map calling convention to FFI") template rd(T, p: untyped): untyped = (cast[ptr T](p))[] template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v template `+!`(x, y: untyped): untyped = cast[pointer](cast[ByteAddress](x) + y) -proc packSize(v: PNode, typ: PType): int = +proc packSize(conf: ConfigRef, v: PNode, typ: PType): int = ## computes the size of the blob case typ.kind of tyPtr, tyRef, tyVar, tyLent: if v.kind in {nkNilLit, nkPtrLit}: result = sizeof(pointer) else: - result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon) + result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon) of tyDistinct, tyGenericInst, tyAlias, tySink: - result = packSize(v, typ.sons[0]) + result = packSize(conf, v, typ.sons[0]) of tyArray: # consider: ptr array[0..1000_000, int] which is common for interfacing; # we use the real length here instead if v.kind in {nkNilLit, nkPtrLit}: result = sizeof(pointer) elif v.len != 0: - result = v.len * packSize(v.sons[0], typ.sons[1]) + result = v.len * packSize(conf, v.sons[0], typ.sons[1]) else: - result = typ.getSize.int + result = getSize(conf, typ).int -proc pack(v: PNode, typ: PType, res: pointer) +proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) -proc getField(n: PNode; position: int): PSym = +proc getField(conf: ConfigRef, n: PNode; position: int): PSym = case n.kind of nkRecList: for i in countup(0, sonsLen(n) - 1): - result = getField(n.sons[i], position) + result = getField(conf, n.sons[i], position) if result != nil: return of nkRecCase: - result = getField(n.sons[0], position) + result = getField(conf, n.sons[0], position) if result != nil: return for i in countup(1, sonsLen(n) - 1): case n.sons[i].kind of nkOfBranch, nkElse: - result = getField(lastSon(n.sons[i]), position) + result = getField(conf, lastSon(n.sons[i]), position) if result != nil: return - else: internalError(n.info, "getField(record case branch)") + else: internalError(conf, n.info, "getField(record case branch)") of nkSym: if n.sym.position == position: result = n.sym else: discard -proc packObject(x: PNode, typ: PType, res: pointer) = - internalAssert x.kind in {nkObjConstr, nkPar, nkTupleConstr} +proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) = + internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr} # compute the field's offsets: - discard typ.getSize + discard getSize(conf, typ) for i in countup(ord(x.kind == nkObjConstr), sonsLen(x) - 1): var it = x.sons[i] if it.kind == nkExprColonExpr: - internalAssert it.sons[0].kind == nkSym + internalAssert conf, it.sons[0].kind == nkSym let field = it.sons[0].sym - pack(it.sons[1], field.typ, res +! field.offset) + pack(conf, it.sons[1], field.typ, res +! field.offset) elif typ.n != nil: - let field = getField(typ.n, i) - pack(it, field.typ, res +! field.offset) + let field = getField(conf, typ.n, i) + pack(conf, it, field.typ, res +! field.offset) else: # XXX: todo - globalError(x.info, "cannot pack unnamed tuple") + globalError(conf, x.info, "cannot pack unnamed tuple") const maxPackDepth = 20 var packRecCheck = 0 -proc pack(v: PNode, typ: PType, res: pointer) = +proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) = template awr(T, v: untyped): untyped = wr(T, res, v) @@ -188,13 +192,13 @@ proc pack(v: PNode, typ: PType, res: pointer) = of tyUInt32: awr(uint32, v.intVal.uint32) of tyUInt64: awr(uint64, v.intVal.uint64) of tyEnum, tySet: - case v.typ.getSize + case getSize(conf, v.typ) of 1: awr(uint8, v.intVal.uint8) of 2: awr(uint16, v.intVal.uint16) of 4: awr(int32, v.intVal.int32) of 8: awr(int64, v.intVal.int64) else: - globalError(v.info, "cannot map value to FFI (tyEnum, tySet)") + globalError(conf, v.info, "cannot map value to FFI (tyEnum, tySet)") of tyFloat: awr(float, v.floatVal) of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) @@ -208,7 +212,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind in {nkStrLit..nkTripleStrLit}: awr(cstring, cstring(v.strVal)) else: - globalError(v.info, "cannot map pointer/proc value to FFI") + globalError(conf, v.info, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar, tyLent: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway @@ -218,44 +222,44 @@ proc pack(v: PNode, typ: PType, res: pointer) = else: if packRecCheck > maxPackDepth: packRecCheck = 0 - globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ)) inc packRecCheck - pack(v.sons[0], typ.lastSon, res +! sizeof(pointer)) + pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer)) dec packRecCheck awr(pointer, res +! sizeof(pointer)) of tyArray: - let baseSize = typ.sons[1].getSize + let baseSize = getSize(conf, typ.sons[1]) for i in 0 ..< v.len: - pack(v.sons[i], typ.sons[1], res +! i * baseSize) + pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize) of tyObject, tyTuple: - packObject(v, typ, res) + packObject(conf, v, typ, res) of tyNil: discard of tyDistinct, tyGenericInst, tyAlias, tySink: - pack(v, typ.sons[0], res) + pack(conf, v, typ.sons[0], res) else: - globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ)) -proc unpack(x: pointer, typ: PType, n: PNode): PNode +proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode -proc unpackObjectAdd(x: pointer, n, result: PNode) = +proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) = case n.kind of nkRecList: for i in countup(0, sonsLen(n) - 1): - unpackObjectAdd(x, n.sons[i], result) + unpackObjectAdd(conf, x, n.sons[i], result) of nkRecCase: - globalError(result.info, "case objects cannot be unpacked") + globalError(conf, result.info, "case objects cannot be unpacked") of nkSym: var pair = newNodeI(nkExprColonExpr, result.info, 2) pair.sons[0] = n - pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil) + pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil) #echo "offset: ", n.sym.name.s, " ", n.sym.offset result.add pair else: discard -proc unpackObject(x: pointer, typ: PType, n: PNode): PNode = +proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = # compute the field's offsets: - discard typ.getSize + discard getSize(conf, typ) # iterate over any actual field of 'n' ... if n is nil we need to create # the nkPar node: @@ -263,36 +267,36 @@ proc unpackObject(x: pointer, typ: PType, n: PNode): PNode = result = newNode(nkTupleConstr) result.typ = typ if typ.n.isNil: - internalError("cannot unpack unnamed tuple") - unpackObjectAdd(x, typ.n, result) + internalError(conf, "cannot unpack unnamed tuple") + unpackObjectAdd(conf, x, typ.n, result) else: result = n if result.kind notin {nkObjConstr, nkPar, nkTupleConstr}: - globalError(n.info, "cannot map value from FFI") + globalError(conf, n.info, "cannot map value from FFI") if typ.n.isNil: - globalError(n.info, "cannot unpack unnamed tuple") + globalError(conf, n.info, "cannot unpack unnamed tuple") for i in countup(ord(n.kind == nkObjConstr), sonsLen(n) - 1): var it = n.sons[i] if it.kind == nkExprColonExpr: - internalAssert it.sons[0].kind == nkSym + internalAssert conf, it.sons[0].kind == nkSym let field = it.sons[0].sym - it.sons[1] = unpack(x +! field.offset, field.typ, it.sons[1]) + it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1]) else: - let field = getField(typ.n, i) - n.sons[i] = unpack(x +! field.offset, field.typ, it) + let field = getField(conf, typ.n, i) + n.sons[i] = unpack(conf, x +! field.offset, field.typ, it) -proc unpackArray(x: pointer, typ: PType, n: PNode): PNode = +proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = if n.isNil: result = newNode(nkBracket) result.typ = typ - newSeq(result.sons, lengthOrd(typ).int) + newSeq(result.sons, lengthOrd(conf, typ).int) else: result = n if result.kind != nkBracket: - globalError(n.info, "cannot map value from FFI") - let baseSize = typ.sons[1].getSize + globalError(conf, n.info, "cannot map value from FFI") + let baseSize = getSize(conf, typ.sons[1]) for i in 0 ..< result.len: - result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i]) + result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i]) proc canonNodeKind(k: TNodeKind): TNodeKind = case k @@ -301,7 +305,7 @@ proc canonNodeKind(k: TNodeKind): TNodeKind = of nkStrLit..nkTripleStrLit: result = nkStrLit else: result = k -proc unpack(x: pointer, typ: PType, n: PNode): PNode = +proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = template aw(k, v, field: untyped): untyped = if n.isNil: result = newNode(k) @@ -313,7 +317,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = #echo "expected ", k, " but got ", result.kind #debug result return newNodeI(nkExceptBranch, n.info) - #globalError(n.info, "cannot map value from FFI") + #globalError(conf, n.info, "cannot map value from FFI") result.field = v template setNil() = @@ -344,13 +348,13 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).BiggestInt) of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).BiggestInt) of tyEnum: - case typ.getSize + case getSize(conf, typ) of 1: awi(nkIntLit, rd(uint8, x).BiggestInt) of 2: awi(nkIntLit, rd(uint16, x).BiggestInt) of 4: awi(nkIntLit, rd(int32, x).BiggestInt) of 8: awi(nkIntLit, rd(int64, x).BiggestInt) else: - globalError(n.info, "cannot map value from FFI (tyEnum, tySet)") + globalError(conf, n.info, "cannot map value from FFI (tyEnum, tySet)") of tyFloat: awf(nkFloatLit, rd(float, x)) of tyFloat32: awf(nkFloat32Lit, rd(float32, x)) of tyFloat64: awf(nkFloat64Lit, rd(float64, x)) @@ -371,15 +375,15 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = elif n == nil or n.kind == nkPtrLit: awi(nkPtrLit, cast[ByteAddress](p)) elif n != nil and n.len == 1: - internalAssert n.kind == nkRefTy - n.sons[0] = unpack(p, typ.lastSon, n.sons[0]) + internalAssert(conf, n.kind == nkRefTy) + n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0]) result = n else: - globalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ)) of tyObject, tyTuple: - result = unpackObject(x, typ, n) + result = unpackObject(conf, x, typ, n) of tyArray: - result = unpackArray(x, typ, n) + result = unpackArray(conf, x, typ, n) of tyCString, tyString: let p = rd(cstring, x) if p.isNil: @@ -389,12 +393,12 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyNil: setNil() of tyDistinct, tyGenericInst, tyAlias, tySink: - result = unpack(x, typ.lastSon, n) + result = unpack(conf, x, typ.lastSon, n) else: # XXX what to do with 'array' here? - globalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ)) -proc fficast*(x: PNode, destTyp: PType): PNode = +proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode = if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer, tyProc, tyCString, tyString, tySequence}: @@ -404,93 +408,94 @@ proc fficast*(x: PNode, destTyp: PType): PNode = result = newNodeIT(x.kind, x.info, destTyp) else: # we play safe here and allocate the max possible size: - let size = max(packSize(x, x.typ), packSize(x, destTyp)) + let size = max(packSize(conf, x, x.typ), packSize(conf, x, destTyp)) var a = alloc0(size) - pack(x, x.typ, a) + pack(conf, x, x.typ, a) # cast through a pointer needs a new inner object: let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1) else: x.copyTree y.typ = x.typ - result = unpack(a, destTyp, y) + result = unpack(conf, a, destTyp, y) dealloc a -proc callForeignFunction*(call: PNode): PNode = - internalAssert call.sons[0].kind == nkPtrLit +proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode = + internalAssert conf, call.sons[0].kind == nkPtrLit var cif: TCif var sig: TParamList # use the arguments' types for varargs support: for i in 1..call.len-1: - sig[i-1] = mapType(call.sons[i].typ) + sig[i-1] = mapType(conf, call.sons[i].typ) if sig[i-1].isNil: - globalError(call.info, "cannot map FFI type") + globalError(conf, call.info, "cannot map FFI type") let typ = call.sons[0].typ - if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1), - mapType(typ.sons[0]), sig) != OK: - globalError(call.info, "error in FFI call") + if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1), + mapType(conf, typ.sons[0]), sig) != OK: + globalError(conf, call.info, "error in FFI call") var args: TArgList let fn = cast[pointer](call.sons[0].intVal) for i in 1 .. call.len-1: var t = call.sons[i].typ - args[i-1] = alloc0(packSize(call.sons[i], t)) - pack(call.sons[i], t, args[i-1]) + args[i-1] = alloc0(packSize(conf, call.sons[i], t)) + pack(conf, call.sons[i], t, args[i-1]) let retVal = if isEmptyType(typ.sons[0]): pointer(nil) - else: alloc(typ.sons[0].getSize.int) + else: alloc(getSize(conf, typ.sons[0]).int) libffi.call(cif, fn, retVal, args) if retVal.isNil: result = newNode(nkEmpty) else: - result = unpack(retVal, typ.sons[0], nil) + result = unpack(conf, retVal, typ.sons[0], nil) result.info = call.info if retVal != nil: dealloc retVal for i in 1 .. call.len-1: - call.sons[i] = unpack(args[i-1], typ.sons[i], call[i]) + call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i]) dealloc args[i-1] -proc callForeignFunction*(fn: PNode, fntyp: PType, +proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType, args: var TNodeSeq, start, len: int, info: TLineInfo): PNode = - internalAssert fn.kind == nkPtrLit + internalAssert conf, fn.kind == nkPtrLit var cif: TCif var sig: TParamList for i in 0..len-1: var aTyp = args[i+start].typ if aTyp.isNil: - internalAssert i+1 < fntyp.len + internalAssert conf, i+1 < fntyp.len aTyp = fntyp.sons[i+1] args[i+start].typ = aTyp - sig[i] = mapType(aTyp) - if sig[i].isNil: globalError(info, "cannot map FFI type") + sig[i] = mapType(conf, aTyp) + if sig[i].isNil: globalError(conf, info, "cannot map FFI type") - if prep_cif(cif, mapCallConv(fntyp.callConv, info), cuint(len), - mapType(fntyp.sons[0]), sig) != OK: - globalError(info, "error in FFI call") + if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len), + mapType(conf, fntyp.sons[0]), sig) != OK: + globalError(conf, info, "error in FFI call") var cargs: TArgList let fn = cast[pointer](fn.intVal) for i in 0 .. len-1: let t = args[i+start].typ - cargs[i] = alloc0(packSize(args[i+start], t)) - pack(args[i+start], t, cargs[i]) + cargs[i] = alloc0(packSize(conf, args[i+start], t)) + pack(conf, args[i+start], t, cargs[i]) let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil) - else: alloc(fntyp.sons[0].getSize.int) + else: alloc(getSize(conf, fntyp.sons[0]).int) libffi.call(cif, fn, retVal, cargs) if retVal.isNil: result = newNode(nkEmpty) else: - result = unpack(retVal, fntyp.sons[0], nil) + result = unpack(conf, retVal, fntyp.sons[0], nil) result.info = info if retVal != nil: dealloc retVal for i in 0 .. len-1: let t = args[i+start].typ - args[i+start] = unpack(cargs[i], t, args[i+start]) + args[i+start] = unpack(conf, cargs[i], t, args[i+start]) dealloc cargs[i] + diff --git a/compiler/main.nim b/compiler/main.nim index 49c2666eac82..c1477a22bb89 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -107,6 +107,7 @@ when not defined(leanCompiler): proc interactivePasses(graph: ModuleGraph) = initDefines(graph.config.symbols) defineSymbol(graph.config.symbols, "nimscript") + # note: seems redundant with -d:nimHasLibFFI when hasFFI: defineSymbol(graph.config.symbols, "nimffi") registerPass(graph, verbosePass) registerPass(graph, semPass) diff --git a/compiler/options.nim b/compiler/options.nim index 0a25b1b96975..c9f884986bad 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -18,7 +18,7 @@ const hasTinyCBackend* = defined(tinyc) useEffectSystem* = true useWriteTracking* = false - hasFFI* = defined(useFFI) + hasFFI* = defined(nimHasLibFFI) copyrightYear* = "2018" type # please make sure we have under 32 options @@ -128,6 +128,10 @@ type forLoopMacros, caseStmtMacros, codeReordering, + compiletimeFFI, + ## This requires building nim with `-d:nimHasLibFFI` + ## which itself requires `nimble install libffi`, see #10150 + ## Note: this feature can't be localized with {.push.} SymbolFilesOption* = enum disabledSf, writeOnlySf, readOnlySf, v2Sf diff --git a/compiler/vm.nim b/compiler/vm.nim index 74f2a367dc85..7493b008d238 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -498,7 +498,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let ra = instr.regA when traceCode: - echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC + template regDescr(name, r): string = + let kind = if r < regs.len: $regs[r].kind else: "" + let ret = name & ": " & $r & " " & $kind + alignLeft(ret, 15) + echo "PC:$pc $opcode $ra $rb $rc" % [ + "pc", $pc, "opcode", alignLeft($c.code[pc].opcode, 15), + "ra", regDescr("ra", ra), "rb", regDescr("rb", instr.regB), + "rc", regDescr("rc", instr.regC)] case instr.opcode of opcEof: return regs[ra] @@ -1072,15 +1079,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = currentException: c.currentExceptionA, currentLineInfo: c.debug[pc])) elif sfImportc in prc.flags: - if allowFFI notin c.features: - globalError(c.config, c.debug[pc], "VM not allowed to do FFI") + if compiletimeFFI notin c.config.features: + globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`") # we pass 'tos.slots' instead of 'regs' so that the compiler can keep # 'regs' in a register: when hasFFI: let prcValue = c.globals.sons[prc.position-1] if prcValue.kind == nkEmpty: globalError(c.config, c.debug[pc], "cannot run " & prc.name.s) - let newValue = callForeignFunction(prcValue, prc.typ, tos.slots, + var slots2: TNodeSeq + slots2.setLen(tos.slots.len) + for i in 0..".} + +proc c_printf(frmt: cstring): cint {.importc: "printf", header: "", varargs, discardable.} + +const snprintfName = when defined(windows): "_snprintf" else: "snprintf" +proc c_snprintf*(buffer: pointer, buf_size: uint, format: cstring): cint {.importc: snprintfName, header: "", varargs .} + +proc c_malloc(size:uint):pointer {.importc:"malloc", header: "".} +proc c_free(p: pointer) {.importc:"free", header: "".} + +proc fun() = + block: # c_exp + var x = 0.3 + let b = c_exp(x) + let b2 = int(b*1_000_000) # avoids floating point equality + doAssert b2 == 1349858 + doAssert c_exp(0.3) == c_exp(x) + const x2 = 0.3 + doAssert c_exp(x2) == c_exp(x) + + block: # c_printf + c_printf("foo\n") + c_printf("foo:%d\n", 100) + c_printf("foo:%d\n", 101.cint) + c_printf("foo:%d:%d\n", 102.cint, 103.cint) + let temp = 104.cint + c_printf("foo:%d:%d:%d\n", 102.cint, 103.cint, temp) + var temp2 = 105.cint + c_printf("foo:%g:%s:%d:%d\n", 0.03, "asdf", 103.cint, temp2) + + block: # c_snprintf, c_malloc, c_free + let n: uint = 50 + var buffer2: pointer = c_malloc(n) + var s: cstring = "foobar" + var age: cint = 25 + let j = c_snprintf(buffer2, n, "s1:%s s2:%s age:%d pi:%g", s, s, age, 3.14) + c_printf("ret={%s}\n", buffer2) + c_free(buffer2) # not sure it has an effect + + block: # c_printf bug + var a = 123 + var a2 = a.addr + #[ + bug: different behavior between CT RT in this case: + at CT, shows foo2:a=123 + at RT, shows foo2:a=
+ ]# + if false: + c_printf("foo2:a=%d\n", a2) + +static: + fun() +fun() From 7ec187a8e11fa4a32aad9c722e49f5d3f0c27909 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Feb 2019 10:43:07 +0100 Subject: [PATCH 0019/1645] make the VM aware of tyOwned --- compiler/vmgen.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 44728758704d..e5e6d87350fc 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1337,7 +1337,7 @@ proc fitsRegister*(t: PType): bool = tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar} proc unneededIndirection(n: PNode): bool = - n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef + n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef proc canElimAddr(n: PNode): PNode = case n.sons[0].kind @@ -1397,7 +1397,7 @@ proc genDeref(c: PCtx, n: PNode, dest: var TDest, flags: TGenFlags) = c.gABC(n, opcNodeToReg, dest, dest) proc whichAsgnOpc(n: PNode): TOpcode = - case n.typ.skipTypes(abstractRange-{tyTypeDesc}).kind + case n.typ.skipTypes(abstractRange+{tyOwned}-{tyTypeDesc}).kind of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64: opcAsgnInt of tyString, tyCString: @@ -1717,7 +1717,7 @@ proc getNullValueAux(obj: PNode, result: PNode; conf: ConfigRef) = else: globalError(conf, result.info, "cannot create null element for: " & $obj) proc getNullValue(typ: PType, info: TLineInfo; conf: ConfigRef): PNode = - var t = skipTypes(typ, abstractRange+{tyStatic}-{tyTypeDesc}) + var t = skipTypes(typ, abstractRange+{tyStatic, tyOwned}-{tyTypeDesc}) case t.kind of tyBool, tyEnum, tyChar, tyInt..tyInt64: result = newNodeIT(nkIntLit, info, t) @@ -1864,7 +1864,7 @@ proc genSetConstr(c: PCtx, n: PNode, dest: var TDest) = proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) = if dest < 0: dest = c.getTemp(n.typ) - let t = n.typ.skipTypes(abstractRange-{tyTypeDesc}) + let t = n.typ.skipTypes(abstractRange+{tyOwned}-{tyTypeDesc}) if t.kind == tyRef: c.gABx(n, opcNew, dest, c.genType(t.sons[0])) else: From c5dbb0379fc431a01220224097f00323df6c9ced Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Feb 2019 11:34:39 +0100 Subject: [PATCH 0020/1645] disable compile-time FFI support --- koch.nim | 4 +--- tests/vm/tevalffi.nim | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index a6c2443ce648..3698db21249f 100644 --- a/koch.nim +++ b/koch.nim @@ -469,9 +469,7 @@ proc runCI(cmd: string) = ## build nimble early on to enable remainder to depend on it if needed kochExecFold("Build Nimble", "nimble") - when not defined(windows): - # pending https://github.com/Araq/libffi/pull/2 - # also, that PR works on win32 but not yet win64 + when false: execFold("nimble install -y libffi", "nimble install -y libffi") kochExecFold("boot -d:release -d:nimHasLibFFI", "boot -d:release -d:nimHasLibFFI") diff --git a/tests/vm/tevalffi.nim b/tests/vm/tevalffi.nim index 20852faca933..963d2a58e461 100644 --- a/tests/vm/tevalffi.nim +++ b/tests/vm/tevalffi.nim @@ -18,7 +18,7 @@ foo:102:103:104 foo:0.03:asdf:103:105 ret={s1:foobar s2:foobar age:25 pi:3.14} ''' - disabled: "windows" + disabled: "true" """ # re-enable for windows once libffi can be installed in koch.nim From 71df1b060b272f9c1b6e5a0f393b3a319705cc70 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 23 Feb 2019 11:52:52 +0100 Subject: [PATCH 0021/1645] Tighten the conversion from tyRange to scalar types (#10495) * Tighten the conversion from tyRange to scalar types. Introduce the `isIntConv` rule for unsigned types. Do not allow mixed-signedness conversions between ranges and scalar types. * More json adjustments --- compiler/sigmatch.nim | 13 +++++----- lib/pure/json.nim | 58 ++++++++++++++++++++++++++++++++++-------- tests/range/trange.nim | 13 ++++++++++ 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index cb71c1c81b40..71302e6bc458 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -385,18 +385,19 @@ proc handleRange(f, a: PType, min, max: TTypeKind): TTypeRelation = result = isFromIntLit elif f.kind == tyInt and k in {tyInt8..tyInt32}: result = isIntConv + elif f.kind == tyUInt and k in {tyUInt8..tyUInt32}: + result = isIntConv elif k >= min and k <= max: result = isConvertible - elif a.kind == tyRange and a.sons[0].kind in {tyInt..tyInt64, - tyUInt8..tyUInt32} and - a.n[0].intVal >= firstOrd(nil, f) and - a.n[1].intVal <= lastOrd(nil, f): + elif a.kind == tyRange and + # Make sure the conversion happens between types w/ same signedness + (f.kind in {tyInt..tyInt64} and a[0].kind in {tyInt..tyInt64} or + f.kind in {tyUInt8..tyUInt32} and a[0].kind in {tyUInt8..tyInt32}) and + a.n[0].intVal >= firstOrd(nil, f) and a.n[1].intVal <= lastOrd(nil, f): # passing 'nil' to firstOrd/lastOrd here as type checking rules should # not depent on the target integer size configurations! result = isConvertible else: result = isNone - #elif f.kind == tyInt and k in {tyInt..tyInt32}: result = isIntConv - #elif f.kind == tyUInt and k in {tyUInt..tyUInt32}: result = isIntConv proc isConvertibleToRange(f, a: PType): bool = # be less picky for tyRange, as that it is used for array indexing: diff --git a/lib/pure/json.nim b/lib/pure/json.nim index e387c516b8cd..bebeaf084e53 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -313,6 +313,24 @@ proc `%`*(s: string): JsonNode = result.kind = JString result.str = s +proc `%`*(n: uint): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + new(result) + result.kind = JInt + result.num = BiggestInt(n) + +proc `%`*(n: int): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + new(result) + result.kind = JInt + result.num = n + +proc `%`*(n: BiggestUInt): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + new(result) + result.kind = JInt + result.num = BiggestInt(n) + proc `%`*(n: BiggestInt): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. new(result) @@ -1757,17 +1775,37 @@ when isMainModule: # Generate constructors for range[T] types block: type - Q1 = range[0..10] - Q2 = range[0'i8..10'i8] - Q3 = range[0'u16..10'u16] + Q1 = range[0'u8 .. 50'u8] + Q2 = range[0'u16 .. 50'u16] + Q3 = range[0'u32 .. 50'u32] + Q4 = range[0'i8 .. 50'i8] + Q5 = range[0'i16 .. 50'i16] + Q6 = range[0'i32 .. 50'i32] + Q7 = range[0'f32 .. 50'f32] + Q8 = range[0'f64 .. 50'f64] + Q9 = range[0 .. 50] + X = object m1: Q1 m2: Q2 m3: Q3 - - let - obj = X(m1: 1, m2: 2'i8, m3: 3'u16) - jsonObj = %obj - desObj = to(jsonObj, type(obj)) - - doAssert(desObj == obj) + m4: Q4 + m5: Q5 + m6: Q6 + m7: Q7 + m8: Q8 + m9: Q9 + + let obj = X( + m1: Q1(42), + m2: Q2(42), + m3: Q3(42), + m4: Q4(42), + m5: Q5(42), + m6: Q6(42), + m7: Q7(42), + m8: Q8(42), + m9: Q9(42) + ) + + doAssert(obj == to(%obj, type(obj))) diff --git a/tests/range/trange.nim b/tests/range/trange.nim index bc48e95662d4..41804d0f26d4 100644 --- a/tests/range/trange.nim +++ b/tests/range/trange.nim @@ -105,3 +105,16 @@ block tcolors: return rgb(red(a) +! red(b), green(a) +! green(b), blue(a) +! blue(b)) rgb(34, 55, 255) + +block: + type + R8 = range[0'u8 .. 10'u8] + R16 = range[0'u16 .. 10'u16] + R32 = range[0'u32 .. 10'u32] + + var + x1 = R8(4) + x2 = R16(4) + x3 = R32(4) + + doAssert $x1 & $x2 & $x3 == "444" From f86835ce77f05133cdb8eeef0a7986a040a61954 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Feb 2019 11:49:21 +0100 Subject: [PATCH 0022/1645] manual: document implicit type conversions involving 'range'; refs #10495 --- doc/manual.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 7b0a30f6a4ac..a2d51cf082c7 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2288,7 +2288,7 @@ A type ``a`` is **implicitly** convertible to type ``b`` iff the following algorithm returns true: .. code-block:: nim - # XXX range types? + proc isImplicitlyConvertible(a, b: PType): bool = if isSubtype(a, b) or isCovariant(a, b): return true @@ -2316,6 +2316,18 @@ algorithm returns true: of string: result = b == cstring + +Implicit conversions are also performed for Nim's ``range`` type +constructor. + +Let ``a0``, ``b0`` of type ``T``. + +Let ``A = range[a0..b0]`` be the argument's type, ``F`` the formal +parameter's type. Then an implicit conversion from ``A`` to ``F`` +exists if ``a0 >= low(F) and b0 <= high(F)`` and both ``T`` and ``F`` +are signed integers or if both are unsigned integers. + + A type ``a`` is **explicitly** convertible to type ``b`` iff the following algorithm returns true: From e89aaaeaab40088eb57a0209e152e90ce16ed51c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 23 Feb 2019 12:05:07 +0100 Subject: [PATCH 0023/1645] Open a new scope for `static:` expr blocks (#10649) Bring this in line with how plain blocks are analysed and avoids codegen errors if one references variables defined in such a block. --- compiler/semexprs.nim | 6 +++++- tests/errmsgs/tstaticexprnotype.nim | 5 +++++ tests/errmsgs/tstaticexprscope.nim | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/tstaticexprnotype.nim create mode 100644 tests/errmsgs/tstaticexprscope.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7bd40a95431e..f66ec7062437 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -735,7 +735,11 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = # echo "SUCCESS evaluated at compile time: ", call.renderTree proc semStaticExpr(c: PContext, n: PNode): PNode = - let a = semExpr(c, n) + inc c.inStaticContext + openScope(c) + let a = semExprWithType(c, n) + closeScope(c) + dec c.inStaticContext if a.findUnresolvedStatic != nil: return a result = evalStaticExpr(c.module, c.graph, a, c.p.owner) if result.isNil: diff --git a/tests/errmsgs/tstaticexprnotype.nim b/tests/errmsgs/tstaticexprnotype.nim new file mode 100644 index 000000000000..8b6735ff80ea --- /dev/null +++ b/tests/errmsgs/tstaticexprnotype.nim @@ -0,0 +1,5 @@ +discard """ + action: reject +""" + +let x = static: discard diff --git a/tests/errmsgs/tstaticexprscope.nim b/tests/errmsgs/tstaticexprscope.nim new file mode 100644 index 000000000000..7af5bf9b37bb --- /dev/null +++ b/tests/errmsgs/tstaticexprscope.nim @@ -0,0 +1,11 @@ +discard """ + errmsg: "undeclared identifier: 'z'" + line: 11 +""" + +# Open a new scope for static expr blocks +block: + let a = static: + var z = 123 + 33 + echo z From 988412905837271ad8d8502e5337381f8e9624ce Mon Sep 17 00:00:00 2001 From: Neelesh Chandola Date: Sat, 23 Feb 2019 18:31:34 +0530 Subject: [PATCH 0024/1645] Tuple unpacking now works for `for` vars (#10152) * Tuple unpacking now works for `for` vars * Give error if length of tuple vars != length of tuple * Fix error message showing wrong tuple length * unpacking now works now for mutable items * Update changelog --- changelog.md | 1 + compiler/ccgexprs.nim | 2 +- compiler/lowerings.nim | 20 ++++++--- compiler/parser.nim | 39 +++++++++++------- compiler/semstmts.nim | 70 ++++++++++++++++++++++++-------- compiler/transf.nim | 30 ++++++++++---- tests/tuples/tfortupleunpack.nim | 39 ++++++++++++++++++ 7 files changed, 156 insertions(+), 45 deletions(-) create mode 100644 tests/tuples/tfortupleunpack.nim diff --git a/changelog.md b/changelog.md index 211855bfc289..7f6ad1ecb839 100644 --- a/changelog.md +++ b/changelog.md @@ -165,6 +165,7 @@ proc enumToString*(enums: openArray[enum]): string = - Pragma blocks are no longer eliminated from the typed AST tree to preserve pragmas for further analysis by macros - Custom pragmas are now supported for `var` and `let` symbols. +- Tuple unpacking is now supported for constants and for loop variables. ### Language changes diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7e0437c39349..f92f2e4de827 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -752,7 +752,7 @@ proc genTupleElem(p: BProc, e: PNode, d: var TLoc) = a: TLoc i: int initLocExpr(p, e.sons[0], a) - let tupType = a.t.skipTypes(abstractInst) + let tupType = a.t.skipTypes(abstractInst+{tyVar}) assert tupType.kind == tyTuple d.inheritLocation(a) discard getTypeDesc(p.module, a.t) # fill the record's fields.loc diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index d199abcc794e..2c1c3c48dd49 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -21,12 +21,20 @@ proc newDeref*(n: PNode): PNode {.inline.} = addSon(result, n) proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode = - result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes( - abstractInst).sons[i]) - addSon(result, copyTree(tup)) - var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt)) - lit.intVal = i - addSon(result, lit) + if tup.kind == nkHiddenAddr: + result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar})) + result.addSon(newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}).sons[i])) + addSon(result[0], tup[0]) + var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt)) + lit.intVal = i + addSon(result[0], lit) + else: + result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes( + abstractInst).sons[i]) + addSon(result, copyTree(tup)) + var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt)) + lit.intVal = i + addSon(result, lit) proc addVar*(father, v: PNode) = var vpart = newNodeI(nkIdentDefs, v.info, 3) diff --git a/compiler/parser.nim b/compiler/parser.nim index 01a3ce4d001a..260e57cdb251 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1153,18 +1153,26 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, result.addSon list parseSymbolList(p, list) +proc parseVarTuple(p: var TParser): PNode + proc parseFor(p: var TParser): PNode = #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt #| forExpr = forStmt - result = newNodeP(nkForStmt, p) getTokNoInd(p) - var a = identWithPragma(p) - addSon(result, a) - while p.tok.tokType == tkComma: - getTok(p) - optInd(p, a) - a = identWithPragma(p) + result = newNodeP(nkForStmt, p) + if p.tok.tokType == tkParLe: + addSon(result, parseVarTuple(p)) + else: + var a = identWithPragma(p) addSon(result, a) + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, a) + if p.tok.tokType == tkParLe: + addSon(result, parseVarTuple(p)) + break + a = identWithPragma(p) + addSon(result, a) eat(p, tkIn) addSon(result, parseExpr(p)) colcom(p, result) @@ -2048,14 +2056,15 @@ proc parseVarTuple(p: var TParser): PNode = addSon(result, p.emptyNode) # no type desc optPar(p) eat(p, tkParRi) - eat(p, tkEquals) - optInd(p, result) - addSon(result, parseExpr(p)) proc parseVariable(p: var TParser): PNode = #| colonBody = colcom stmt doBlocks? #| variable = (varTuple / identColonEquals) colonBody? indAndComment - if p.tok.tokType == tkParLe: result = parseVarTuple(p) + if p.tok.tokType == tkParLe: + result = parseVarTuple(p) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) else: result = parseIdentColonEquals(p, {withPragma, withDot}) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) @@ -2072,10 +2081,10 @@ proc parseConstant(p: var TParser): PNode = addSon(result, parseTypeDesc(p)) else: addSon(result, p.emptyNode) - eat(p, tkEquals) - optInd(p, result) - addSon(result, parseExpr(p)) - indAndComment(p, result) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) + indAndComment(p, result) proc parseBind(p: var TParser, k: TNodeKind): PNode = #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 363049672017..aa0230f2fc32 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -672,28 +672,66 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode = # and thus no tuple unpacking: if iter.kind != tyTuple or length == 3: if length == 3: - var v = symForVar(c, n.sons[0]) - if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) - # BUGFIX: don't use `iter` here as that would strip away - # the ``tyGenericInst``! See ``tests/compile/tgeneric.nim`` - # for an example: - v.typ = iterBase - n.sons[0] = newSymNode(v) - if sfGenSym notin v.flags: addForVarDecl(c, v) - elif v.owner == nil: v.owner = getCurrOwner(c) + if n.sons[0].kind == nkVarTuple: + var mutable = false + if iter.kind == tyVar: + iter = iter.skipTypes({tyVar}) + mutable = true + if sonsLen(n[0])-1 != sonsLen(iter): + localError(c.config, n[0].info, errWrongNumberOfVariables) + for i in 0 ..< sonsLen(n[0])-1: + var v = symForVar(c, n[0][i]) + if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) + if mutable: + v.typ = newTypeS(tyVar, c) + v.typ.sons.add iter[i] + else: + v.typ = iter.sons[i] + n.sons[0][i] = newSymNode(v) + if sfGenSym notin v.flags: addForVarDecl(c, v) + elif v.owner == nil: v.owner = getCurrOwner(c) + else: + var v = symForVar(c, n.sons[0]) + if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) + # BUGFIX: don't use `iter` here as that would strip away + # the ``tyGenericInst``! See ``tests/compile/tgeneric.nim`` + # for an example: + v.typ = iterBase + n.sons[0] = newSymNode(v) + if sfGenSym notin v.flags: addForVarDecl(c, v) + elif v.owner == nil: v.owner = getCurrOwner(c) else: localError(c.config, n.info, errWrongNumberOfVariables) elif length-2 != sonsLen(iter): localError(c.config, n.info, errWrongNumberOfVariables) else: for i in countup(0, length - 3): - var v = symForVar(c, n.sons[i]) - if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) - v.typ = iter.sons[i] - n.sons[i] = newSymNode(v) - if sfGenSym notin v.flags: - if not isDiscardUnderscore(v): addForVarDecl(c, v) - elif v.owner == nil: v.owner = getCurrOwner(c) + if n.sons[i].kind == nkVarTuple: + var mutable = false + if iter[i].kind == tyVar: + iter[i] = iter[i].skipTypes({tyVar}) + mutable = true + if sonsLen(n[i])-1 != sonsLen(iter[i]): + localError(c.config, n[i].info, errWrongNumberOfVariables) + for j in 0 ..< sonsLen(n[i])-1: + var v = symForVar(c, n[i][j]) + if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) + if mutable: + v.typ = newTypeS(tyVar, c) + v.typ.sons.add iter[i][j] + else: + v.typ = iter[i][j] + n.sons[i][j] = newSymNode(v) + if not isDiscardUnderscore(v): addForVarDecl(c, v) + elif v.owner == nil: v.owner = getCurrOwner(c) + else: + var v = symForVar(c, n.sons[i]) + if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal) + v.typ = iter.sons[i] + n.sons[i] = newSymNode(v) + if sfGenSym notin v.flags: + if not isDiscardUnderscore(v): addForVarDecl(c, v) + elif v.owner == nil: v.owner = getCurrOwner(c) inc(c.p.nestedLoopCounter) openScope(c) n.sons[length-1] = semExprBranch(c, n.sons[length-1], flags) diff --git a/compiler/transf.nim b/compiler/transf.nim index 071cb00ee788..a26598094c25 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -378,9 +378,15 @@ proc transformYield(c: PTransf, n: PNode): PTransNode = for i in countup(0, sonsLen(e) - 1): var v = e.sons[i] if v.kind == nkExprColonExpr: v = v.sons[1] - let lhs = c.transCon.forStmt.sons[i] - let rhs = transform(c, v) - add(result, asgnTo(lhs, rhs)) + if c.transCon.forStmt[i].kind == nkVarTuple: + for j in 0 ..< sonsLen(c.transCon.forStmt[i])-1: + let lhs = c.transCon.forStmt[i][j] + let rhs = transform(c, newTupleAccess(c.graph, v, j)) + add(result, asgnTo(lhs, rhs)) + else: + let lhs = c.transCon.forStmt.sons[i] + let rhs = transform(c, v) + add(result, asgnTo(lhs, rhs)) else: # Unpack the tuple into the loop variables # XXX: BUG: what if `n` is an expression with side-effects? @@ -389,9 +395,15 @@ proc transformYield(c: PTransf, n: PNode): PTransNode = let rhs = transform(c, newTupleAccess(c.graph, e, i)) add(result, asgnTo(lhs, rhs)) else: - let lhs = c.transCon.forStmt.sons[0] - let rhs = transform(c, e) - add(result, asgnTo(lhs, rhs)) + if c.transCon.forStmt.sons[0].kind == nkVarTuple: + for i in 0 ..< sonsLen(c.transCon.forStmt[0])-1: + let lhs = c.transCon.forStmt[0][i] + let rhs = transform(c, newTupleAccess(c.graph, e, i)) + add(result, asgnTo(lhs, rhs)) + else: + let lhs = c.transCon.forStmt.sons[0] + let rhs = transform(c, e) + add(result, asgnTo(lhs, rhs)) inc(c.transCon.yieldStmts) if c.transCon.yieldStmts <= 1: @@ -609,7 +621,11 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = var v = newNodeI(nkVarSection, n.info) for i in countup(0, length - 3): - addVar(v, copyTree(n.sons[i])) # declare new vars + if n[i].kind == nkVarTuple: + for j in 0 ..< sonsLen(n[i])-1: + addVar(v, copyTree(n[i][j])) # declare new vars + else: + addVar(v, copyTree(n.sons[i])) # declare new vars add(stmtList, v.PTransNode) # Bugfix: inlined locals belong to the invoking routine, not to the invoked diff --git a/tests/tuples/tfortupleunpack.nim b/tests/tuples/tfortupleunpack.nim new file mode 100644 index 000000000000..56cf30ebc9c2 --- /dev/null +++ b/tests/tuples/tfortupleunpack.nim @@ -0,0 +1,39 @@ +discard """ +output: ''' +123 +113283 +0 +123 +1 +113283 +@[(88, 99, 11), (88, 99, 11)] +@[(7, 6, -28), (7, 6, -28)] +''' +""" + +let t1 = (1, 2, 3) +let t2 = (11, 32, 83) +let s = @[t1, t2] + +for (a, b, c) in s: + echo a, b, c + +for i, (a, b, c) in s: + echo i + echo a, b, c + +var x = @[(1,2,3), (4,5,6)] + +for (a, b, c) in x.mitems: + a = 88 + b = 99 + c = 11 +echo x + +for i, (a, b, c) in x.mpairs: + a = 7 + b = 6 + c = -28 +echo x + + From f39aa1b40bd617e5ac13f67b38548e91d49bf107 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 23 Feb 2019 14:03:32 +0100 Subject: [PATCH 0025/1645] discard destroys its argument in-place (#9478) --- compiler/destroyer.nim | 4 ++++ tests/destructor/tdiscard.nim | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/destructor/tdiscard.nim diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index 0570873ea986..dd12a7966f73 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -601,6 +601,10 @@ proc p(n: PNode; c: var Con): PNode = of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef: result = n + of nkDiscardStmt: + result = n + if n[0].typ != nil and hasDestructor(n[0].typ): + result = genDestroy(c, n[0].typ, n[0]) of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv: result = copyNode(n) # Destination type diff --git a/tests/destructor/tdiscard.nim b/tests/destructor/tdiscard.nim new file mode 100644 index 000000000000..2e4a4b2856ef --- /dev/null +++ b/tests/destructor/tdiscard.nim @@ -0,0 +1,18 @@ +type + O = object + +var dCalls = 0 + +proc `=destroy`(x: var O) = inc dCalls +proc `=sink`(x: var O, y: O) = doAssert false + +proc newO(): O = discard + +proc main() = + doAssert dCalls == 0 + discard newO() + doAssert dCalls == 1 + discard newO() + doAssert dCalls == 2 + +main() From 28a83a838821cbe3efc8ddd412db966ca164ef5c Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 19 Feb 2019 22:55:35 +0000 Subject: [PATCH 0026/1645] Handle IPv6 in bindAddr #7633 Add test --- lib/pure/net.nim | 25 ++++++++++++------------- tests/stdlib/tnetbind.nim | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 tests/stdlib/tnetbind.nim diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 43284f8724f4..f468e1c5d37f 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -755,21 +755,20 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. ## Binds ``address``:``port`` to the socket. ## ## If ``address`` is "" then ADDR_ANY will be bound. + var realaddr = address + if realaddr == "": + case socket.domain + of AF_INET6: realaddr = "::" + of AF_INET: realaddr = "0.0.0.0" + else: + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") - if address == "": - var name: Sockaddr_in - name.sin_family = toInt(AF_INET).uint16 - name.sin_port = htons(port.uint16) - name.sin_addr.s_addr = htonl(INADDR_ANY) - if bindAddr(socket.fd, cast[ptr SockAddr](addr(name)), - sizeof(name).SockLen) < 0'i32: - raiseOSError(osLastError()) - else: - var aiList = getAddrInfo(address, port, socket.domain) - if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeAddrInfo(aiList) - raiseOSError(osLastError()) + var aiList = getAddrInfo(realaddr, port, socket.domain) + if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeAddrInfo(aiList) + raiseOSError(osLastError()) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var Socket, address: var string, flags = {SocketFlag.SafeDisconn}) {. diff --git a/tests/stdlib/tnetbind.nim b/tests/stdlib/tnetbind.nim new file mode 100644 index 000000000000..b2bcf4b05c40 --- /dev/null +++ b/tests/stdlib/tnetbind.nim @@ -0,0 +1,14 @@ +import net + +## Test for net.bindAddr + +proc test() = + # IPv4 TCP + newSocket(AF_INET, SOCK_STREAM, IPPROTO_TCP).bindAddr(Port(1900), "0.0.0.0") + newSocket(AF_INET, SOCK_STREAM, IPPROTO_TCP).bindAddr(Port(1901)) + + # IPv6 TCP + newSocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP).bindAddr(Port(1902), "::") + newSocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP).bindAddr(Port(1903)) + +test() From 68a82f100e66e1950064a353c209c7f0039327fc Mon Sep 17 00:00:00 2001 From: deech Date: Mon, 25 Feb 2019 04:21:14 -0600 Subject: [PATCH 0027/1645] Fixes #10727. (#10728) --- compiler/semgnrc.nim | 7 ++++++- tests/tuples/tfortupleunpack.nim | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 5972f3b550de..2810fca9e0db 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -352,7 +352,12 @@ proc semGenericStmt(c: PContext, n: PNode, openScope(c) n.sons[L - 2] = semGenericStmt(c, n.sons[L-2], flags, ctx) for i in countup(0, L - 3): - addTempDecl(c, n.sons[i], skForVar) + if (n.sons[i].kind == nkVarTuple): + for s in n.sons[i]: + if (s.kind == nkIdent): + addTempDecl(c,s,skForVar) + else: + addTempDecl(c, n.sons[i], skForVar) openScope(c) n.sons[L - 1] = semGenericStmt(c, n.sons[L-1], flags, ctx) closeScope(c) diff --git a/tests/tuples/tfortupleunpack.nim b/tests/tuples/tfortupleunpack.nim index 56cf30ebc9c2..9aeb7c5d623d 100644 --- a/tests/tuples/tfortupleunpack.nim +++ b/tests/tuples/tfortupleunpack.nim @@ -8,6 +8,7 @@ output: ''' 113283 @[(88, 99, 11), (88, 99, 11)] @[(7, 6, -28), (7, 6, -28)] +12 ''' """ @@ -36,4 +37,7 @@ for i, (a, b, c) in x.mpairs: c = -28 echo x - +proc test[n]() = + for (a,b) in @[(1,2)]: + echo a,b +test[string]() From 2074ad141603361b7a0f32f9d3b855f33587b49d Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 25 Feb 2019 11:33:18 +0100 Subject: [PATCH 0028/1645] strutils: document `parseEnum` [ci skip] --- lib/pure/strutils.nim | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index a946bc8c6a94..54d0e3f144ac 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1173,6 +1173,18 @@ proc parseEnum*[T: enum](s: string): T = ## ## Raises ``ValueError`` for an invalid value in `s`. The comparison is ## done in a style insensitive way. + runnableExamples: + type + MyEnum = enum + first = "1st", + second, + third = "3rd" + + doAssert parseEnum[MyEnum]("1_st") == first + doAssert parseEnum[MyEnum]("second") == second + doAssertRaises(ValueError): + echo parseEnum[MyEnum]("third") + for e in low(T)..high(T): if cmpIgnoreStyle(s, $e) == 0: return e @@ -1183,6 +1195,17 @@ proc parseEnum*[T: enum](s: string, default: T): T = ## ## Uses `default` for an invalid value in `s`. The comparison is done in a ## style insensitive way. + runnableExamples: + type + MyEnum = enum + first = "1st", + second, + third = "3rd" + + doAssert parseEnum[MyEnum]("1_st") == first + doAssert parseEnum[MyEnum]("second") == second + doAssert parseEnum[MyEnum]("last", third) == third + for e in low(T)..high(T): if cmpIgnoreStyle(s, $e) == 0: return e From bf4e688ca37ebfcedd64d2c8cb1854b287f56037 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 23 Feb 2019 20:58:14 +0100 Subject: [PATCH 0029/1645] make nimsuggest aware of tyOwned --- compiler/suggest.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index d501acd4667e..4be6407735a8 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -395,7 +395,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) suggestOperations(c, n, field, typ, outputs) else: let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias, tySink}) - typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}) + typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) if typ.kind == tyObject: var t = typ while true: From 1397ad7c5459d48c10cc19d487dc856bcb78e81a Mon Sep 17 00:00:00 2001 From: WhiteDuke Date: Mon, 25 Feb 2019 16:45:44 +0100 Subject: [PATCH 0030/1645] [random] add support for sets (#10532) * Support for sets * Rename 'rand' to 'sample' * Update random.nim --- lib/pure/random.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 86db7da497d0..4115d147b3c9 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -390,6 +390,18 @@ proc rand*[T](a: openArray[T]): T {.deprecated.} = ## Use `sample[T](openArray[T])<#sample,openArray[T]>`_ instead. result = a[rand(a.low..a.high)] +proc sample*[T](r: var Rand; s: set[T]): T = + ## returns a random element from a set + assert card(s) != 0 + var i = rand(r, card(s) - 1) + for e in s: + if i == 0: return e + dec(i) + +proc sample*[T](s: set[T]): T = + ## returns a random element from a set + sample(state, s) + proc sample*[T](r: var Rand; a: openArray[T]): T = ## Returns a random element from ``a`` using the given state. ## From fb863a147a79ab36306f884921b6693141a12870 Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 25 Feb 2019 16:53:05 +0100 Subject: [PATCH 0031/1645] use `initHashSet` and `toHashSet`, fixes #10730 (#10736) --- lib/pure/collections/sets.nim | 180 ++++++++++++++++++---------------- 1 file changed, 94 insertions(+), 86 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 5da5d92434cf..8583b72045e9 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -15,8 +15,8 @@ ## `hashed `_ and they don't contain duplicate entries. ## ## Common usages of sets: -## * removing duplicates from a container by converting it with `toSet proc -## <#toSet,openArray[A]>`_ (see also `sequtils.deduplicate proc +## * removing duplicates from a container by converting it with `toHashSet proc +## <#toHashSet,openArray[A]>`_ (see also `sequtils.deduplicate proc ## `_) ## * membership testing ## * mathematical operations on two sets, such as @@ -26,12 +26,12 @@ ## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ ## ## .. code-block:: -## echo toSet([9, 5, 1]) # {9, 1, 5} +## echo toHashSet([9, 5, 1]) # {9, 1, 5} ## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} ## ## let -## s1 = toSet([9, 5, 1]) -## s2 = toSet([3, 5, 7]) +## s1 = toHashSet([9, 5, 1]) +## s2 = toHashSet([3, 5, 7]) ## ## echo s1 + s2 # {9, 1, 3, 5, 7} ## echo s1 - s2 # {1, 9} @@ -64,7 +64,7 @@ type HashSet* {.myShallow.} [A] = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A],int>`_ or `initSet proc <#initSet,int>`_ + ## Use `init proc <#init,HashSet[A],int>`_ or `initHashSet proc <#initHashSet,int>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int @@ -230,8 +230,8 @@ proc init*[A](s: var HashSet[A], initialSize=64) = ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. ## ## See also: - ## * `initSet proc <#initSet,int>`_ - ## * `toSet proc <#toSet,openArray[A]>`_ + ## * `initHashSet proc <#initHashSet,int>`_ + ## * `toHashSet proc <#toHashSet,openArray[A]>`_ runnableExamples: var a: HashSet[int] assert(not a.isValid) @@ -242,7 +242,7 @@ proc init*[A](s: var HashSet[A], initialSize=64) = s.counter = 0 newSeq(s.data, initialSize) -proc initSet*[A](initialSize=64): HashSet[A] = +proc initHashSet*[A](initialSize=64): HashSet[A] = ## Wrapper around `init proc <#init,HashSet[A],int>`_ for initialization of ## hash sets. ## @@ -250,37 +250,45 @@ proc initSet*[A](initialSize=64): HashSet[A] = ## single line. ## ## See also: - ## * `toSet proc <#toSet,openArray[A]>`_ + ## * `toHashSet proc <#toHashSet,openArray[A]>`_ runnableExamples: - var a = initSet[int]() + var a = initHashSet[int]() assert a.isValid a.incl(3) assert len(a) == 1 result.init(initialSize) -proc toSet*[A](keys: openArray[A]): HashSet[A] = +proc toHashSet*[A](keys: openArray[A]): HashSet[A] = ## Creates a new hash set that contains the members of the given ## collection (seq, array, or string) `keys`. ## ## Duplicates are removed. ## ## See also: - ## * `initSet proc <#initSet,int>`_ + ## * `initHashSet proc <#initHashSet,int>`_ runnableExamples: let - a = toSet([5, 3, 2]) - b = toSet("abracadabra") + a = toHashSet([5, 3, 2]) + b = toHashSet("abracadabra") assert len(a) == 3 ## a == {2, 3, 5} assert len(b) == 5 ## b == {'a', 'b', 'c', 'd', 'r'} - result = initSet[A](rightSize(keys.len)) + result = initHashSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) +proc initSet*[A](initialSize=64): HashSet[A] {.deprecated: + "Deprecated since v0.20, use `initHashSet`"} = initHashSet[A](initialSize) + ## Deprecated since v0.20, use `initHashSet`. + +proc toSet*[A](keys: openArray[A]): HashSet[A] {.deprecated: + "Deprecated since v0.20, use `toHashSet`"} = toHashSet[A](keys) + ## Deprecated since v0.20, use `toHashSet`. + proc isValid*[A](s: HashSet[A]): bool = - ## Returns `true` if the set has been initialized (with `initSet proc - ## <#initSet,int>`_ or `init proc <#init,HashSet[A],int>`_). + ## Returns `true` if the set has been initialized (with `initHashSet proc + ## <#initHashSet,int>`_ or `init proc <#init,HashSet[A],int>`_). ## ## Most operations over an uninitialized set will crash at runtime and ## `assert `_ in debug builds. You can use this proc in @@ -320,7 +328,7 @@ proc contains*[A](s: HashSet[A], key: A): bool = ## * `incl proc <#incl,HashSet[A],A>`_ ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ runnableExamples: - var values = initSet[int]() + var values = initHashSet[int]() assert(not values.contains(2)) assert 2 notin values @@ -343,7 +351,7 @@ proc incl*[A](s: var HashSet[A], key: A) = ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ runnableExamples: - var values = initSet[int]() + var values = initHashSet[int]() values.incl(2) values.incl(2) assert values.len == 1 @@ -362,8 +370,8 @@ proc incl*[A](s: var HashSet[A], other: HashSet[A]) = ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ runnableExamples: var - values = toSet([1, 2, 3]) - others = toSet([3, 4, 5]) + values = toHashSet([1, 2, 3]) + others = toHashSet([3, 4, 5]) values.incl(others) assert values.len == 5 @@ -384,7 +392,7 @@ proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = ## * `incl proc <#incl,HashSet[A],HashSet[A]>`_ for including other set ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ runnableExamples: - var values = initSet[int]() + var values = initHashSet[int]() assert values.containsOrIncl(2) == false assert values.containsOrIncl(2) == true assert values.containsOrIncl(3) == false @@ -402,7 +410,7 @@ proc excl*[A](s: var HashSet[A], key: A) = ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ runnableExamples: - var s = toSet([2, 3, 6, 7]) + var s = toHashSet([2, 3, 6, 7]) s.excl(2) s.excl(2) assert s.len == 3 @@ -419,8 +427,8 @@ proc excl*[A](s: var HashSet[A], other: HashSet[A]) = ## * `missingOrExcl proc <#missingOrExcl,HashSet[A],A>`_ runnableExamples: var - numbers = toSet([1, 2, 3, 4, 5]) - even = toSet([2, 4, 6, 8]) + numbers = toHashSet([1, 2, 3, 4, 5]) + even = toHashSet([2, 4, 6, 8]) numbers.excl(even) assert len(numbers) == 3 ## numbers == {1, 3, 5} @@ -442,7 +450,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = ## * `excl proc <#excl,HashSet[A],HashSet[A]>`_ for excluding other set ## * `containsOrIncl proc <#containsOrIncl,HashSet[A],A>`_ runnableExamples: - var s = toSet([2, 3, 6, 7]) + var s = toHashSet([2, 3, 6, 7]) assert s.missingOrExcl(4) == true assert s.missingOrExcl(6) == false assert s.missingOrExcl(6) == true @@ -456,7 +464,7 @@ proc pop*[A](s: var HashSet[A]): A = ## See also: ## * `clear proc <#clear,HashSet[A]>`_ runnableExamples: - var s = toSet([2, 1]) + var s = toHashSet([2, 1]) assert s.pop == 1 assert s.pop == 2 doAssertRaises(KeyError, echo s.pop) @@ -477,7 +485,7 @@ proc clear*[A](s: var HashSet[A]) = ## See also: ## * `pop proc <#pop,HashSet[A]>`_ runnableExamples: - var s = toSet([3, 5, 7]) + var s = toHashSet([3, 5, 7]) clear(s) assert len(s) == 0 @@ -495,7 +503,7 @@ proc len*[A](s: HashSet[A]): int = runnableExamples: var a: HashSet[string] assert len(a) == 0 - let s = toSet([3, 5, 7]) + let s = toHashSet([3, 5, 7]) assert len(s) == 3 result = s.counter @@ -521,10 +529,10 @@ proc union*[A](s1, s2: HashSet[A]): HashSet[A] = ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = union(a, b) - assert c == toSet(["a", "b", "c"]) + assert c == toHashSet(["a", "b", "c"]) assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." @@ -546,14 +554,14 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = intersection(a, b) - assert c == toSet(["b"]) + assert c == toHashSet(["b"]) assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." - result = initSet[A](min(s1.data.len, s2.data.len)) + result = initHashSet[A](min(s1.data.len, s2.data.len)) for item in s1: if item in s2: incl(result, item) @@ -571,14 +579,14 @@ proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## * `symmetricDifference proc <#symmetricDifference,HashSet[A],HashSet[A]>`_ runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = difference(a, b) - assert c == toSet(["a"]) + assert c == toHashSet(["a"]) assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." - result = initSet[A]() + result = initHashSet[A]() for item in s1: if not contains(s2, item): incl(result, item) @@ -598,10 +606,10 @@ proc symmetricDifference*[A](s1, s2: HashSet[A]): HashSet[A] = ## * `difference proc <#difference,HashSet[A],HashSet[A]>`_ runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = symmetricDifference(a, b) - assert c == toSet(["a", "c"]) + assert c == toHashSet(["a", "c"]) assert s1.isValid, "The set `s1` needs to be initialized." assert s2.isValid, "The set `s2` needs to be initialized." @@ -630,8 +638,8 @@ proc disjoint*[A](s1, s2: HashSet[A]): bool = ## Returns `true` if the sets `s1` and `s2` have no items in common. runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) assert disjoint(a, b) == false assert disjoint(a, b - a) == true @@ -648,8 +656,8 @@ proc `<`*[A](s, t: HashSet[A]): bool = ## more elements than `s`. runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = intersection(a, b) assert c < a and c < b assert(not (a < a)) @@ -662,8 +670,8 @@ proc `<=`*[A](s, t: HashSet[A]): bool = ## have more members than `s`. That is, `s` can be equal to `t`. runnableExamples: let - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = intersection(a, b) assert c <= a and c <= b assert a <= a @@ -680,8 +688,8 @@ proc `==`*[A](s, t: HashSet[A]): bool = ## Returns true if both `s` and `t` have the same members and set size. runnableExamples: var - a = toSet([1, 2]) - b = toSet([2, 1]) + a = toHashSet([1, 2]) + b = toHashSet([2, 1]) assert a == b s.counter == t.counter and s <= t @@ -692,11 +700,11 @@ proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = ## You can use this proc to transform the elements from a set. runnableExamples: let - a = toSet([1, 2, 3]) + a = toHashSet([1, 2, 3]) b = a.map(proc (x: int): string = $x) - assert b == toSet(["1", "2", "3"]) + assert b == toHashSet(["1", "2", "3"]) - result = initSet[B]() + result = initHashSet[B]() for item in data: result.incl(op(item)) proc hash*[A](s: HashSet[A]): Hash = @@ -715,9 +723,9 @@ proc `$`*[A](s: HashSet[A]): string = ## **Examples:** ## ## .. code-block:: - ## echo toSet([2, 4, 5]) + ## echo toHashSet([2, 4, 5]) ## # --> {2, 4, 5} - ## echo toSet(["no", "esc'aping", "is \" provided"]) + ## echo toHashSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} assert s.isValid, "The set needs to be initialized." dollarImpl() @@ -743,7 +751,7 @@ iterator items*[A](s: HashSet[A]): A = ## type ## pair = tuple[a, b: int] ## var - ## a, b = initSet[pair]() + ## a, b = initHashSet[pair]() ## a.incl((2, 3)) ## a.incl((3, 2)) ## a.incl((2, 3)) @@ -917,7 +925,7 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = for key in items(keys): result.incl(key) proc isValid*[A](s: OrderedSet[A]): bool = - ## Returns `true` if the set has been initialized (with `initSet proc + ## Returns `true` if the set has been initialized (with `initHashSet proc ## <#initOrderedSet,int>`_ or `init proc <#init,OrderedSet[A],int>`_). ## ## Most operations over an uninitialized set will crash at runtime and @@ -982,7 +990,7 @@ proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) = ## * `containsOrIncl proc <#containsOrIncl,OrderedSet[A],A>`_ runnableExamples: var - values = toSet([1, 2, 3]) + values = toHashSet([1, 2, 3]) others = toOrderedSet([3, 4, 5]) values.incl(others) assert values.len == 5 @@ -1071,7 +1079,7 @@ proc len*[A](s: OrderedSet[A]): int {.inline.} = runnableExamples: var a: OrderedSet[string] assert len(a) == 0 - let s = toSet([3, 5, 7]) + let s = toHashSet([3, 5, 7]) assert len(s) == 3 result = s.counter @@ -1181,7 +1189,7 @@ when isMainModule and not defined(release): var options: HashSet[string] proc savePreferences(options: HashSet[string]) = assert options.isValid, "Pass an initialized set!" - options = initSet[string]() + options = initHashSet[string]() options.savePreferences block lenTest: @@ -1192,7 +1200,7 @@ when isMainModule and not defined(release): block setIterator: type pair = tuple[a, b: int] - var a, b = initSet[pair]() + var a, b = initHashSet[pair]() a.incl((2, 3)) a.incl((3, 2)) a.incl((2, 3)) @@ -1203,7 +1211,7 @@ when isMainModule and not defined(release): #echo b block setContains: - var values = initSet[int]() + var values = initHashSet[int]() assert(not values.contains(2)) values.incl(2) assert values.contains(2) @@ -1211,7 +1219,7 @@ when isMainModule and not defined(release): assert(not values.contains(2)) values.incl(4) - var others = toSet([6, 7]) + var others = toHashSet([6, 7]) values.incl(others) assert values.len == 3 @@ -1219,31 +1227,31 @@ when isMainModule and not defined(release): assert values.containsOrIncl(2) == false assert values.containsOrIncl(2) == true var - a = toSet([1, 2]) - b = toSet([1]) + a = toHashSet([1, 2]) + b = toHashSet([1]) b.incl(2) assert a == b block exclusions: - var s = toSet([2, 3, 6, 7]) + var s = toHashSet([2, 3, 6, 7]) s.excl(2) s.excl(2) assert s.len == 3 var - numbers = toSet([1, 2, 3, 4, 5]) - even = toSet([2, 4, 6, 8]) + numbers = toHashSet([1, 2, 3, 4, 5]) + even = toHashSet([2, 4, 6, 8]) numbers.excl(even) #echo numbers # --> {1, 3, 5} block toSeqAndString: - var a = toSet([2, 4, 5]) - var b = initSet[int]() + var a = toHashSet([2, 4, 5]) + var b = initHashSet[int]() for x in [2, 4, 5]: b.incl(x) assert($a == $b) #echo a - #echo toSet(["no", "esc'aping", "is \" provided"]) + #echo toHashSet(["no", "esc'aping", "is \" provided"]) #block orderedToSeqAndString: # echo toOrderedSet([2, 4, 5]) @@ -1251,32 +1259,32 @@ when isMainModule and not defined(release): block setOperations: var - a = toSet(["a", "b"]) - b = toSet(["b", "c"]) + a = toHashSet(["a", "b"]) + b = toHashSet(["b", "c"]) c = union(a, b) - assert c == toSet(["a", "b", "c"]) + assert c == toHashSet(["a", "b", "c"]) var d = intersection(a, b) - assert d == toSet(["b"]) + assert d == toHashSet(["b"]) var e = difference(a, b) - assert e == toSet(["a"]) + assert e == toHashSet(["a"]) var f = symmetricDifference(a, b) - assert f == toSet(["a", "c"]) + assert f == toHashSet(["a", "c"]) assert d < a and d < b assert((a < a) == false) assert d <= a and d <= b assert((a <= a)) # Alias test. - assert a + b == toSet(["a", "b", "c"]) - assert a * b == toSet(["b"]) - assert a - b == toSet(["a"]) - assert a -+- b == toSet(["a", "c"]) + assert a + b == toHashSet(["a", "b", "c"]) + assert a * b == toHashSet(["b"]) + assert a - b == toHashSet(["a"]) + assert a -+- b == toHashSet(["a", "c"]) assert disjoint(a, b) == false assert disjoint(a, b - a) == true block mapSet: - var a = toSet([1, 2, 3]) + var a = toHashSet([1, 2, 3]) var b = a.map(proc (x: int): string = $x) - assert b == toSet(["1", "2", "3"]) + assert b == toHashSet(["1", "2", "3"]) block isValidTest: var cards: OrderedSet[string] @@ -1361,7 +1369,7 @@ when isMainModule and not defined(release): b.incl(2) b.init assert b.len == 0 and b.isValid - b = initSet[int](4) + b = initHashSet[int](4) b.incl(2) assert b.len == 1 From 84f0a33bf0b0dc040c97205914d0b03c16e7c71a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 25 Feb 2019 19:00:52 +0100 Subject: [PATCH 0032/1645] make typeToString sane for sequence again --- compiler/types.nim | 2 +- tests/errmsgs/t8434.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/types.nim b/compiler/types.nim index 0e2c3b65142d..86e5e756ae83 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -430,7 +430,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = "" if t == nil: return if prefer in preferToResolveSymbols and t.sym != nil and - sfAnon notin t.sym.flags: + sfAnon notin t.sym.flags and t.kind != tySequence: if t.kind == tyInt and isIntLit(t): result = t.sym.name.s & " literal(" & $t.n.intVal & ")" elif t.kind == tyAlias and t.sons[0].kind != tyAlias: diff --git a/tests/errmsgs/t8434.nim b/tests/errmsgs/t8434.nim index 60fe2e2df194..b374681119ca 100644 --- a/tests/errmsgs/t8434.nim +++ b/tests/errmsgs/t8434.nim @@ -4,7 +4,7 @@ discard """ proc fun0[T1: int | float | object | array | seq](a1: T1; a2: int) first type mismatch at position: 1 - required type: T1: int or float or object or array or seq + required type: T1: int or float or object or array or seq[T] but expression 'byte(1)' is of type: byte expression: fun0(byte(1), 0) From d53ab9e5c857d47b8ea9643a2cf0235f3486afa0 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 26 Feb 2019 16:43:34 +0200 Subject: [PATCH 0033/1645] Prevent options from calling custom ref == operators (#10745) --- lib/pure/options.nim | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 7a474e772101..82ecedeb6d4b 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -111,7 +111,7 @@ proc some*[T](val: T): Option[T] = assert $b == "Some(42)" when T is SomePointer: - assert val != nil + assert(not val.isNil) result.val = val else: result.has = true @@ -146,7 +146,7 @@ proc isSome*[T](self: Option[T]): bool {.inline.} = assert not b.isSome when T is SomePointer: - self.val != nil + not self.val.isNil else: self.has @@ -159,7 +159,7 @@ proc isNone*[T](self: Option[T]): bool {.inline.} = assert not a.isNone assert b.isNone when T is SomePointer: - self.val == nil + self.val.isNil else: not self.has @@ -371,6 +371,16 @@ proc unsafeGet*[T](self: Option[T]): T = when isMainModule: import unittest, sequtils + # RefPerson is used to test that overloaded `==` operator is not called by + # options. It is defined here in the global scope, because otherwise the test + # will not even consider the `==` operator. Different bug? + type RefPerson = ref object + name: string + + proc `==`(a, b: RefPerson): bool = + assert(not a.isNil and not b.isNil) + a.name == b.name + suite "options": # work around a bug in unittest let intNone = none(int) @@ -489,3 +499,8 @@ when isMainModule: let noperson = none(Person) check($noperson == "None[Person]") + + test "Ref type with overloaded `==`": + let p = some(RefPerson.new()) + check p.isSome + From 287206f993e5989d417780f1e9a2287dcf34866e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Tue, 26 Feb 2019 15:45:01 +0100 Subject: [PATCH 0034/1645] minor fix for debug on symbols (#10742) --- compiler/astalgo.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 48a651632b99..292b5058fbf6 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -402,11 +402,11 @@ proc debugTree(conf: ConfigRef; n: PNode, indent: int, maxRecDepth: int; of nkSym: let s = n.sym var symStr = "" - symStr.add "\"kind\": " + symStr.add "\"kind\": \"" symStr.add $s.kind - symStr.add ", \"name\": " + symStr.add "\", \"name\": \"" symStr.add s.name.s - symStr.add ", \"id\": " + symStr.add "\", \"id\": " symStr.add s.id if s.kind in {skField, skEnumField, skParam}: symStr.add ", \"position\": " From ba38c05eb62a1b6e0b36b92886c43bed2cabd90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Tue, 26 Feb 2019 15:47:35 +0100 Subject: [PATCH 0035/1645] add gdb commands: koch, nim, nimble (#10741) * add gdb commands: koch, nim, nimble * make commands path independent --- tools/nim-gdb.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index 2abaf692658d..b6481a99d354 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -168,6 +168,60 @@ def invoke (self, arg, from_tty): DollarPrintCmd() + +################################################################################ +##### GDB Commands to invoke common nim tools. +################################################################################ + + +import subprocess, os + + +class KochCmd (gdb.Command): + """Command that invokes ``koch'', the build tool for the compiler.""" + + def __init__ (self): + super (KochCmd, self).__init__ ("koch", + gdb.COMMAND_USER, gdb.COMPLETE_FILENAME) + self.binary = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "koch") + + def invoke(self, argument, from_tty): + import os + subprocess.run([self.binary] + gdb.string_to_argv(argument)) + +KochCmd() + + +class NimCmd (gdb.Command): + """Command that invokes ``nim'', the nim compiler.""" + + def __init__ (self): + super (NimCmd, self).__init__ ("nim", + gdb.COMMAND_USER, gdb.COMPLETE_FILENAME) + self.binary = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "bin/nim") + + def invoke(self, argument, from_tty): + subprocess.run([self.binary] + gdb.string_to_argv(argument)) + +NimCmd() + + +class NimbleCmd (gdb.Command): + """Command that invokes ``nimble'', the nim package manager and build tool.""" + + def __init__ (self): + super (NimbleCmd, self).__init__ ("nimble", + gdb.COMMAND_USER, gdb.COMPLETE_FILENAME) + self.binary = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "bin/nimble") + + def invoke(self, argument, from_tty): + subprocess.run([self.binary] + gdb.string_to_argv(argument)) + +NimbleCmd() + ################################################################################ ##### Value pretty printers ################################################################################ From ca4b971bc81b2e751e0388d80896fde7079b1679 Mon Sep 17 00:00:00 2001 From: zah Date: Tue, 26 Feb 2019 16:48:55 +0200 Subject: [PATCH 0036/1645] Initial version of the hot-code reloading support for native targets (#10729) * squashed work by Zahary * squashing a ton of useful history... otherwise rebasing on top of upstream Nim after commit 82c009a2cbc5d07ab9a847f1c58228a20efaf219 would be impossible. * Code review changes; Working test suite (without code reloading enabled) * - documentation - implemented the HCR test - almost works... - fix the issue on Unix where for executable targets the source file for the main module of a project in nimcache was being overwritten with the binary itself (and thus the actual source code was lost) - fixing embedded paths to shared objects on unix (the "lib" prefix was being prepended to the entire path instead of just the filename) - other fixes - removing unnecessary includes since that file is already included in chcks.nim which is in turn included in system.nim (and previously was getting imported in chcks.nim but then system.nim improts something... and that breaks HCR (perhaps it could be fixed but it would be nice not to import anything in system)) * fix for clang & C++ - explicitly casting a function pointer to void* more stable mangling of parameter names when HCR is on the length of the static arrays in the DatInit functions is now part of the name of the variables, so when they get resized they get also recreated more stable mangling for inline functions - no longer depends on the module which first used them work on the new complicated HCR test - turned surprisingly complex - WIP test now successfully passes even when re-running `koch test` (previously when the nimcache wasn't cold that lead to errors) better documentation calling setStackBottomWith for PreMain passes over the HcrInit/DatInit/Init calls of all modules are now in the proper order (first all of one type, then all of the next). Also typeinfo globals are registered (created) in a single pass before the DatInit pass (because of the way generic instantiations are handled) Fix the test suite execution on macOs fix for being able to query the program arguments when using HCR on posix! other fixes * Bugfix: Fix a compilation error in C++ mode when a function pointer is converted to a raw pointer * basic documentation for the new hot code reloading semantics * Add change log entry * Don't re-execute the top-level statements while reloading JS code * fix a number of tests broken in a recent bugfix * Review changes * Added {.executeOnReload.} pragma that indicates top-level statements that should be executed on each reload. To make this work, I've modified the way the `if (hcr_init_) {...}` guards are produced in the init code. This still needs more work as the new guards seem to be inserted within the previously generated guards. This change also removes the need for `lastRegistedGlobal` in nimhcr. * Implemented the `signatureHash` magic and the `hasModuleChanged` API depending on it (the actual logic is not imlemented yet). * Add the "hcr" prefix to all HCR-related symbols in the system module. Added a new `hotcodereloading` module exporting the high-level API to the user. Besides being more hygienic, this was also required in order to make it possible to use macros in the high-level API. Without the split, `system` would have to import `macros`, which was going to produce the well-known init problems. * Attempted to solve the "GC markers problem". Crashes were expected with the previous code, because the GC markers were compiled as normal procs are registered in the GC. When their module is unloaded, dangling pointers will remain in the GC tables. To solve this issue, I don't register any GC markers when HCR is on, but I add them to the HCR globals metadata and I use a single marker registed in nimhcr during the initialization of the system module that will be responsible for marking all globals. * fix a compilation error * - implemented the hasModuleChanged functionality - tuples can be returned and broken into different vars in global scope - added comments for the closnig scopes of the if statements in the init proc - the new executeOnReload pragma works now! - other fixes * finally! fixing this hack in a proper way - declaring the destructor out of line (out of the class body) - we no longer need to forward-declare popCurrentExceptionEx * Force full module parsing This is a temporary hack that breaks some tests. I'll investigate later how these can be fixed. * tuples are now properly handled when global! * these comments mess up the codegen in debug mode when $n is not actually a new line (or something like that) - these labels are intended only for GOTO labels anyway... * "solved" the issue with the .pdb locks on windows when a binary is being debugged and hot code reloading is used at the same time * fixes after rebasing... * small fixes for the test * better handling of globals! no more compiler crashes for locals with the global pragma, also simplified code around loops in global scope which have local vars (actually globals) * we can now use the global pragma even for ... globals! * the right output * lets try those boehm GC tests * after the test is ran it will be at its starting state - no git modifications * clarification in the docs * removed unnecessary line directives for forward declarations of functions - they were causing trouble with hot code reloading when no semantic change propagates to the main module but a line directive got changed and thus the main module had to be recompiled since the .c code had changed * fixed bug! was inserting duplicate keys into the table and later was removing only 1 copy of all the duplicates (after a few reloads) * no longer breaking into DatInit code when not supposed to * fixes after rebasing * yet more fixes after rebasing * Update jssys.nim * Rework the HCR path-handling logic After reviewing the code more carefully, I've noticed that the old logic will be broken when the user overrides the '--out:f' compiler option. Besides fixing this issues, I took the opportunity to implement the missing '--outdir:d' option. Other changes: * ./koch test won't overwrite any HCR and RTL builds located in nim/lib * HCR and RTL are compiled with --threads:on by default * Clean up the globals registration logic * Handle non-flattened top-level stmtlists in JS as well * The HCR is not supported with the Boehm GC yet Also fixes some typos and the expected output of the HCR integration test * The GC marker procs are now properly used as trampolines * Fix the HCR integration test in release builds * Fix ./koch tools * this forward declaration doesn't seem to be necessary, and in fact breaks HCR because a 2nd function pointer is emitted for this externed/rtl func * the forward declaration I removed in the last commit was actually necessary * Attempt to make all tests green * Fix tgenscript * BAT file for running the HCR integration test on Windows [skip ci] * Fix the docgen tests * A final fix for Travis (hopefully) --- .gitignore | 4 +- changelog.md | 10 +- compiler/ast.nim | 12 +- compiler/ccgcalls.nim | 6 +- compiler/ccgexprs.nim | 22 + compiler/ccgmerge.nim | 2 +- compiler/ccgstmts.nim | 84 +++- compiler/ccgtrav.nim | 21 +- compiler/ccgtypes.nim | 151 ++++-- compiler/cgen.nim | 409 ++++++++++++---- compiler/cgendata.nim | 9 +- compiler/commands.nim | 20 +- compiler/condsyms.nim | 2 + compiler/docgen.nim | 14 +- compiler/docgen2.nim | 1 - compiler/extccomp.nim | 162 ++++-- compiler/jsgen.nim | 101 ++-- compiler/layouter.nim | 7 +- compiler/main.nim | 42 +- compiler/modulegraphs.nim | 2 + compiler/modules.nim | 5 +- compiler/nim.nim | 19 +- compiler/nodejs.nim | 1 + compiler/options.nim | 26 +- compiler/passes.nim | 4 +- compiler/pathutils.nim | 4 + compiler/pragmas.nim | 6 +- compiler/semexprs.nim | 5 +- compiler/semfold.nim | 2 - compiler/sighashes.nim | 16 +- compiler/transf.nim | 18 +- compiler/vm.nim | 11 +- compiler/vmdef.nim | 1 + compiler/vmgen.nim | 1 + compiler/wordrecg.nim | 2 + doc/advopt.txt | 1 + doc/nimc.rst | 98 +++- lib/core/hotcodereloading.nim | 27 + lib/core/macros.nim | 7 + lib/core/strs.nim | 2 +- lib/core/typeinfo.nim | 2 - lib/nimbase.h | 5 + lib/nimhcr.nim | 652 +++++++++++++++++++++++++ lib/nimhcr.nim.cfg | 5 + lib/nimrtl.nim.cfg | 1 + lib/pure/collections/sharedstrings.nim | 2 - lib/pure/os.nim | 2 +- lib/pure/reservedmem.nim | 241 +++++++++ lib/pure/strformat.nim | 6 +- lib/pure/strtabs.nim | 87 ++-- lib/system.nim | 20 +- lib/system/cgprocs.nim | 8 +- lib/system/chcks.nim | 2 +- lib/system/dyncalls.nim | 2 +- lib/system/excpt.nim | 4 +- lib/system/gc_common.nim | 8 +- lib/system/memory.nim | 6 +- lib/system/sysstr.nim | 2 +- lib/windows/winlean.nim | 4 + nimpretty/nimpretty.nim | 2 +- testament/categories.nim | 44 +- testament/tester.nim | 59 ++- tests/dll/nimhcr_0.nim | 4 + tests/dll/nimhcr_0_1.nim | 14 + tests/dll/nimhcr_0_2.nim | 18 + tests/dll/nimhcr_0_3.nim | 18 + tests/dll/nimhcr_0_4.nim | 19 + tests/dll/nimhcr_0_5.nim | 2 + tests/dll/nimhcr_0_6.nim | 4 + tests/dll/nimhcr_1.nim | 0 tests/dll/nimhcr_1_1.nim | 51 ++ tests/dll/nimhcr_1_2.nim | 4 + tests/dll/nimhcr_1_3.nim | 0 tests/dll/nimhcr_2.nim | 0 tests/dll/nimhcr_2_1.nim | 15 + tests/dll/nimhcr_2_2.nim | 7 + tests/dll/nimhcr_2_3.nim | 0 tests/dll/nimhcr_integration.nim | 152 ++++++ tests/dll/nimhcr_unit.nim | 147 ++++++ tests/dll/nimhcr_unit.nim.cfg | 2 + tests/dll/test_nimhcr_integration.bat | 10 + tests/dll/test_nimhcr_integration.sh | 17 + tests/types/tissues_types.nim | 7 + tools/kochdocs.nim | 2 +- 84 files changed, 2507 insertions(+), 487 deletions(-) create mode 100644 lib/core/hotcodereloading.nim create mode 100644 lib/nimhcr.nim create mode 100644 lib/nimhcr.nim.cfg create mode 100644 lib/pure/reservedmem.nim create mode 100644 tests/dll/nimhcr_0.nim create mode 100644 tests/dll/nimhcr_0_1.nim create mode 100644 tests/dll/nimhcr_0_2.nim create mode 100644 tests/dll/nimhcr_0_3.nim create mode 100644 tests/dll/nimhcr_0_4.nim create mode 100644 tests/dll/nimhcr_0_5.nim create mode 100644 tests/dll/nimhcr_0_6.nim create mode 100644 tests/dll/nimhcr_1.nim create mode 100644 tests/dll/nimhcr_1_1.nim create mode 100644 tests/dll/nimhcr_1_2.nim create mode 100644 tests/dll/nimhcr_1_3.nim create mode 100644 tests/dll/nimhcr_2.nim create mode 100644 tests/dll/nimhcr_2_1.nim create mode 100644 tests/dll/nimhcr_2_2.nim create mode 100644 tests/dll/nimhcr_2_3.nim create mode 100644 tests/dll/nimhcr_integration.nim create mode 100644 tests/dll/nimhcr_unit.nim create mode 100644 tests/dll/nimhcr_unit.nim.cfg create mode 100644 tests/dll/test_nimhcr_integration.bat create mode 100755 tests/dll/test_nimhcr_integration.sh diff --git a/.gitignore b/.gitignore index 38b7976120a9..9de4569bfd66 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,9 @@ dnimcache/ !/icons/*.o *.obj *.ilk +*.exp *.pdb +*.lib *.dll *.exe *.so @@ -19,7 +21,6 @@ dnimcache/ *.zip *.iss *.log -*.ilk *.pdb mapping.txt @@ -42,6 +43,7 @@ bin/* *.perspectivev3 *.swp .DS_Store +.tags project.xcworkspace/ xcuserdata/ diff --git a/changelog.md b/changelog.md index 7f6ad1ecb839..297a6c87bc94 100644 --- a/changelog.md +++ b/changelog.md @@ -136,6 +136,9 @@ proc enumToString*(enums: openArray[enum]): string = slightly. The `dumpLisp` macro in this module now outputs an indented proper Lisp, devoid of commas. +- Added `macros.signatureHash` that returns a stable identifier + derived from the signature of a symbol. + - In `strutils` empty strings now no longer matched as substrings anymore. @@ -192,6 +195,11 @@ proc enumToString*(enums: openArray[enum]): string = ### Compiler changes - The deprecated `fmod` proc is now unavailable on the VM'. - +- A new `--outdir` option was added. +- The compiled JavaScript file for the project produced by executing `nim js` + will no longer be placed in the nimcache directory. +- The `--hotCodeReloading` has been implemented for the native targets. + The compiler also provides a new more flexible API for handling the + hot code reloading events in the code. ### Bugfixes diff --git a/compiler/ast.nim b/compiler/ast.nim index e691cc175aac..607b497fb86b 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -227,7 +227,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # already 33 flags! + TSymFlag* = enum # already 34 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -276,6 +276,8 @@ type # the calling side of the macro, not from the # implementation. sfGenSym # symbol is 'gensym'ed; do not add to symbol table + sfNonReloadable # symbol will be left as-is when hot code reloading is on - + # meaning that it won't be renamed and/or changed in any way TSymFlags* = set[TSymFlag] @@ -468,6 +470,7 @@ type nfDefaultParam # an automatically inserter default parameter nfDefaultRefsParam # a default param value references another parameter # the flag is applied to proc default values and to calls + nfExecuteOnReload # A top-level statement that will be executed during reloads TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: beyond that) @@ -654,7 +657,7 @@ type mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, - mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, + mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNBindSym, mLocals, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, @@ -854,7 +857,7 @@ type offset*: int # offset of record field loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a - # reference to another object to safe space) + # reference to another object to save space) constraint*: PNode # additional constraints like 'lit|result'; also # misused for the codegenDecl pragma in the hope # it won't cause problems @@ -978,7 +981,8 @@ const PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfPreventCg, nfLL, - nfFromTemplate, nfDefaultRefsParam} + nfFromTemplate, nfDefaultRefsParam, + nfExecuteOnReload} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index d177e1f881a0..4c8fa7147c9b 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -185,7 +185,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) = var length = sonsLen(ri) for i in countup(1, length - 1): genParamLoop(params) - fixupCall(p, le, ri, d, op.r, params) + fixupCall(p, le, ri, d, rdLoc(op), params) proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = @@ -209,7 +209,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = genParamLoop(pl) template genCallPattern {.dirty.} = - lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc]) + lineF(p, cpsStmts, callPattern & ";$n", [rdLoc(op), pl, pl.addComma, rawProc]) let rawProc = getRawProcType(p, typ) let callPattern = if tfIterator in typ.flags: PatIter else: PatProc @@ -237,7 +237,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = assert(d.t != nil) # generate an assignment to d: var list: TLoc initLoc(list, locCall, d.lode, OnUnknown) - list.r = callPattern % [op.r, pl, pl.addComma, rawProc] + list.r = callPattern % [rdLoc(op), pl, pl.addComma, rawProc] genAssignment(p, d, list, {}) # no need for deep copying else: genCallPattern() diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f92f2e4de827..a40c60e6d094 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1534,6 +1534,7 @@ proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = var a: TLoc initLocExpr(p, n.sons[1], a) a.r = ropecg(p.module, frmt, [rdLoc(a)]) + a.flags = a.flags - {lfIndirect} # this flag should not be propagated here (not just for HCR) if d.k == locNone: getTemp(p, n.typ, d) genAssignment(p, d, a, {}) gcUsage(p.config, n) @@ -2034,8 +2035,28 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = genCall(p, e, d) of mNewString, mNewStringOfCap, mExit, mParseBiggestFloat: var opr = e.sons[0].sym + # Why would anyone want to set nodecl to one of these hardcoded magics? + # - not sure, and it wouldn't work if the symbol behind the magic isn't + # somehow forward-declared from some other usage, but it is *possible* if lfNoDecl notin opr.loc.flags: + let prc = magicsys.getCompilerProc(p.module.g.graph, $opr.loc.r) + # HACK: + # Explicitly add this proc as declared here so the cgsym call doesn't + # add a forward declaration - without this we could end up with the same + # 2 forward declarations. That happens because the magic symbol and the original + # one that shall be used have different ids (even though a call to one is + # actually a call to the other) so checking into m.declaredProtos with the 2 different ids doesn't work. + # Why would 2 identical forward declarations be a problem? + # - in the case of hot code-reloading we generate function pointers instead + # of forward declarations and in C++ it is an error to redefine a global + let wasDeclared = containsOrIncl(p.module.declaredProtos, prc.id) + # Make the function behind the magic get actually generated - this will + # not lead to a forward declaration! The genCall will lead to one. discard cgsym(p.module, $opr.loc.r) + # make sure we have pointer-initialising code for hot code reloading + if not wasDeclared and p.hcrOn: + addf(p.module.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [mangleDynLibProc(prc), getTypeDesc(p.module, prc.loc.t), getModuleDllPath(p.module, prc)]) genCall(p, e, d) of mReset: genReset(p, e) of mEcho: genEcho(p, e[1].skipConv) @@ -2292,6 +2313,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = proc expr(p: BProc, n: PNode, d: var TLoc) = p.currLineInfo = n.info + case n.kind of nkSym: var sym = n.sym diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 2d68a198e328..ccb5a7635db1 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -31,7 +31,7 @@ const cfsData: "NIM_merge_DATA", cfsProcs: "NIM_merge_PROCS", cfsInitProc: "NIM_merge_INIT_PROC", - cfsDatInitProc: "NIM_merge_DATINIT_PROC", + cfsDatInitProc: "NIM_merge_DATINIT_PROC", cfsTypeInit1: "NIM_merge_TYPE_INIT1", cfsTypeInit2: "NIM_merge_TYPE_INIT2", cfsTypeInit3: "NIM_merge_TYPE_INIT3", diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index bc873539772e..6dc10db3d626 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -15,18 +15,20 @@ const stringCaseThreshold = 8 # above X strings a hash-switch for strings is generated -proc registerGcRoot(p: BProc, v: PSym) = +proc getTraverseProc(p: BProc, v: Psym): Rope = if p.config.selectedGC in {gcMarkAndSweep, gcDestructors, gcV2, gcRefc} and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) - let prc = genTraverseProcForGlobal(p.module, v, v.info) - if sfThread in v.flags: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterThreadLocalMarker($1);$n", [prc]) - else: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterGlobalMarker($1);$n", [prc]) + result = genTraverseProcForGlobal(p.module, v, v.info) + +proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) = + if sfThread in v.flags: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc]) + else: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc]) proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false @@ -41,6 +43,10 @@ proc inExceptBlockLen(p: BProc): int = for x in p.nestedTryStmts: if x.inExcept: result.inc +proc startBlock(p: BProc, start: FormatStr = "{$n", + args: varargs[Rope]): int {.discardable.} +proc endBlock(p: BProc) + proc genVarTuple(p: BProc, n: PNode) = var tup, field: TLoc if n.kind != nkVarTuple: internalError(p.config, n.info, "genVarTuple") @@ -52,6 +58,32 @@ proc genVarTuple(p: BProc, n: PNode) = genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc)) return + # check only the first son + var forHcr = treatGlobalDifferentlyForHCR(p.module, n.sons[0].sym) + let hcrCond = if forHcr: getTempName(p.module) else: nil + var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]] + # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block) + let isGlobalInBlock = forHcr and p.blocks.len > 2 + # do not close and reopen blocks if this is a 'global' but inside of a block (if/while/block) + forHcr = forHcr and not isGlobalInBlock + + if forHcr: + # check with the boolean if the initializing code for the tuple should be ran + lineCg(p, cpsStmts, "if ($1)$n", hcrCond) + startBlock(p) + defer: + if forHcr: + # end the block where the tuple gets initialized + endBlock(p) + if forHcr or isGlobalInBlock: + # insert the registration of the globals for the different parts of the tuple at the + # start of the current scope (after they have been iterated) and init a boolean to + # check if any of them is newly introduced and the initializing code has to be ran + lineCg(p, cpsLocals, "NIM_BOOL $1 = NIM_FALSE;$n", hcrCond) + for curr in hcrGlobals: + lineCg(p, cpsLocals, "$1 |= hcrRegisterGlobal($4, \"$2\", sizeof($3), $5, (void**)&$2);$N", + hcrCond, curr.loc.r, rdLoc(curr.loc), getModuleDllPath(p.module, n.sons[0].sym), curr.tp) + genLineDir(p, n) initLocExpr(p, n.sons[L-1], tup) var t = tup.t.skipTypes(abstractInst) @@ -59,10 +91,13 @@ proc genVarTuple(p: BProc, n: PNode) = let vn = n.sons[i] let v = vn.sym if sfCompileTime in v.flags: continue + var traverseProc: Rope if sfGlobal in v.flags: assignGlobalVar(p, vn) genObjectInit(p, cpsInit, v.typ, v.loc, true) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: assignLocalVar(p, vn) initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[L-1])) @@ -73,6 +108,8 @@ proc genVarTuple(p: BProc, n: PNode) = if t.n.sons[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple") field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym)] putLocIntoDest(p, v.loc, field) + if forHcr or isGlobalInBlock: + hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc)) proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) @@ -242,6 +279,7 @@ proc genSingleVar(p: BProc, a: PNode) = genGotoVar(p, a.sons[2]) return var targetProc = p + var traverseProc: Rope if sfGlobal in v.flags: if v.flags * {sfImportc, sfExportc} == {sfImportc} and a.sons[2].kind == nkEmpty and @@ -270,7 +308,9 @@ proc genSingleVar(p: BProc, a: PNode) = # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc) if sfExportc in v.flags and p.module.g.generatedHeader != nil: genVarPrototype(p.module.g.generatedHeader, vn) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: let value = a.sons[2] let imm = isAssignedImmediately(p.config, value) @@ -302,6 +342,30 @@ proc genSingleVar(p: BProc, a: PNode) = assignLocalVar(p, vn) initLocalVar(p, v, imm) + if traverseProc == nil: traverseProc = ~"NULL" + # If the var is in a block (control flow like if/while or a block) in global scope just + # register the so called "global" so it can be used later on. There is no need to close + # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway. + var forHcr = treatGlobalDifferentlyForHCR(p.module, v) + if forHcr and targetProc.blocks.len > 3 and v.owner.kind == skModule: + # put it in the locals section - mainly because of loops which + # use the var in a call to resetLoc() in the statements section + lineCg(targetProc, cpsLocals, "hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1);$n", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + # nothing special left to do later on - let's avoid closing and reopening blocks + forHcr = false + + # we close and reopen the global if (nim_hcr_do_init_) blocks in the main Init function + # for the module so we can have globals and top-level code be interleaved and still + # be able to re-run it but without the top level code - just the init of globals + if forHcr: + lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + startBlock(targetProc) + defer: + if forHcr: + endBlock(targetProc) + if a.sons[2].kind != nkEmpty: genLineDir(targetProc, a) loadInto(targetProc, a.sons[0], a.sons[2], v.loc) diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 0a2bbf93bd8a..87e7c9d489ca 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -134,11 +134,13 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = var c: TTraversalClosure var p = newProc(nil, m) result = "Marker_" & getTypeName(m, origTyp, sig) - var typ = origTyp.skipTypes(abstractInstOwned) + let + hcrOn = m.hcrOn + typ = origTyp.skipTypes(abstractInstOwned) + markerName = if hcrOn: result & "_actual" else: result + header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [markerName] + t = getTypeDesc(m, typ) - let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result] - - let t = getTypeDesc(m, typ) lineF(p, cpsLocals, "$1 a;$n", [t]) lineF(p, cpsInit, "a = ($1)p;$n", [t]) @@ -155,18 +157,23 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = else: genTraverseProc(c, "(*a)".rope, typ.sons[0]) - let generatedProc = "$1 {$n$2$3$4}$n" % + let generatedProc = "$1 {$n$2$3$4}\n" % [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)] - m.s[cfsProcHeaders].addf("$1;$n", [header]) + m.s[cfsProcHeaders].addf("$1;\n", [header]) m.s[cfsProcs].add(generatedProc) + if hcrOn: + addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result]) + addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n", + [result, markerName, getModuleDllPath(m)]) + proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope = discard genTypeInfo(m, s.loc.t, info) var c: TTraversalClosure var p = newProc(nil, m) - var sLoc = s.loc.r + var sLoc = rdLoc(s.loc) result = getTempName(m) if sfThread in s.flags and emulatedThreadVars(m.config): diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index afe90544d1ba..063c02df9a5c 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -14,7 +14,7 @@ import sighashes from lowerings import createObj -proc genProcHeader(m: BModule, prc: PSym): Rope +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope proc isKeyword(w: PIdent): bool = # Nim and C++ share some keywords @@ -59,7 +59,23 @@ proc mangleParamName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: var res = s.name.s.mangle - if isKeyword(s.name) or m.g.config.cppDefines.contains(res): + # Take into account if HCR is on because of the following scenario: + # if a module gets imported and it has some more importc symbols in it, + # some param names might recieve the "_0" suffix to distinguish from what + # is newly available. That might lead to changes in the C code in nimcache + # that contain only a parameter name change, but that is enough to mandate + # recompilation of that source file and thus a new shared object will be + # relinked. That may lead to a module getting reloaded which wasn't intended + # and that may be fatal when parts of the current active callstack when + # performCodeReload() was called are from the module being reloaded + # unintentionally - example (3 modules which import one another): + # main => proxy => reloadable + # we call performCodeReload() in proxy to reload only changes in reloadable + # but there is a new import which introduces an importc symbol `socket` + # and a function called in main or proxy uses `socket` as a parameter name. + # That would lead to either needing to reload `proxy` or to overwrite the + # executable file for the main module, which is running (or both!) -> error. + if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): res.add "_0" result = res.rope s.loc.r = result @@ -507,6 +523,8 @@ proc fillObjectFields*(m: BModule; typ: PType) = var check = initIntSet() discard getRecordFields(m, typ, check) +proc mangleDynLibProc(sym: PSym): Rope + proc getRecordDesc(m: BModule, typ: PType, name: Rope, check: var IntSet): Rope = # declare the record: @@ -535,22 +553,13 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, appcg(m, result, " : public $1 {$n", [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)]) if typ.isException: - appcg(m, result, "virtual void raise() {throw *this;}$n") # required for polymorphic exceptions + appcg(m, result, "virtual void raise() { throw *this; }$n") # required for polymorphic exceptions if typ.sym.magic == mException: # Add cleanup destructor to Exception base class - appcg(m, result, "~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) - # hack: forward declare popCurrentExceptionEx() on top of type description, - # proper request to generate popCurrentExceptionEx not possible for 2 reasons: - # generated function will be below declared Exception type and circular dependency - # between Exception and popCurrentExceptionEx function - - let popExSym = magicsys.getCompilerProc(m.g.graph, "popCurrentExceptionEx") - if lfDynamicLib in popExSym.loc.flags and sfImportc in popExSym.flags: - # echo popExSym.flags, " ma flags ", popExSym.loc.flags - result = "extern " & getTypeDescAux(m, popExSym.typ, check) & " " & - mangleName(m, popExSym) & ";\L" & result - else: - result = genProcHeader(m, popExSym) & ";\L" & result + appcg(m, result, "~$1();$n", [name]) + # define it out of the class body and into the procs section so we don't have to + # artificially forward-declare popCurrentExceptionEx (very VERY troublesome for HCR) + appcg(m, cfsProcs, "inline $1::~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) hasField = true else: appcg(m, result, " {$n $1 Sup;$n", @@ -888,31 +897,43 @@ proc finishTypeDescriptions(m: BModule) = template cgDeclFrmt*(s: PSym): string = s.constraint.strVal -proc genProcHeader(m: BModule, prc: PSym): Rope = +proc isReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable notin prc.flags + +proc isNonReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable in prc.flags + +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope = var rettype, params: Rope - genCLineDir(result, prc.info, m.config) # using static is needed for inline procs if lfExportLib in prc.loc.flags: if isHeaderFile in m.flags: result.add "N_LIB_IMPORT " else: result.add "N_LIB_EXPORT " - elif prc.typ.callConv == ccInline: + elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc): result.add "static " elif {sfImportc, sfExportc} * prc.flags == {}: result.add "N_LIB_PRIVATE " var check = initIntSet() fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) genProcParams(m, prc.typ, rettype, params, check) + # handle the 2 options for hotcodereloading codegen - function pointer + # (instead of forward declaration) or header for function budy with "_actual" postfix + let asPtrStr = rope(if asPtr: "_PTR" else: "") + var name = prc.loc.r + if isReloadable(m, prc) and not asPtr: + add(name, "_actual") # careful here! don't access ``prc.ast`` as that could reload large parts of # the object graph! if prc.constraint.isNil: - addf(result, "$1($2, $3)$4", - [rope(CallingConvToStr[prc.typ.callConv]), rettype, prc.loc.r, + addf(result, "$1$2($3, $4)$5", + [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name, params]) else: - result = prc.cgDeclFrmt % [rettype, prc.loc.r, params] + let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name + result = prc.cgDeclFrmt % [rettype, asPtrStr, params] # ------------------ type info generation ------------------------------------- @@ -921,6 +942,9 @@ proc getNimNode(m: BModule): Rope = result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)] inc(m.typeNodes) +proc TINameForHcr(m: BModule, name: Rope): Rope = + return if m.hcrOn: "(*".rope & name & ")" else: name + proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope; info: TLineInfo) = var nimtypeKind: int @@ -930,19 +954,21 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; else: nimtypeKind = ord(typ.kind) + let nameHcr = TINameForHcr(m, name) + var size: Rope if tfIncompleteStruct in typ.flags: size = rope"void*" else: size = getTypeDesc(m, origType) addf(m.s[cfsTypeInit3], "$1.size = sizeof($2);$n" & "$1.kind = $3;$n" & "$1.base = $4;$n", - [name, size, rope(nimtypeKind), base]) + [nameHcr, size, rope(nimtypeKind), base]) # compute type flags for GC optimization var flags = 0 if not containsGarbageCollectedRef(typ): flags = flags or 1 if not canFormAcycle(typ): flags = flags or 2 #else MessageOut("can contain a cycle: " & typeToString(typ)) if flags != 0: - addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) + addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [nameHcr, rope(flags)]) discard cgsym(m, "TNimType") if isDefined(m.config, "nimTypeNames"): var typename = typeToString(if origType.typeInst != nil: origType.typeInst @@ -950,11 +976,17 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", - [name, makeCstring typename]) + [nameHcr, makeCstring typename]) discard cgsym(m, "nimTypeRoot") addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n", - [name]) - addf(m.s[cfsVars], "TNimType $1;$n", [name]) + [nameHcr]) + + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n", + [name, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsVars], "TNimType $1;$n", [name]) proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) = @@ -984,6 +1016,14 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope = var tmp = discriminatorTableName(m, objtype, d) result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)] +proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimNode** $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($3, \"$1\", sizeof(TNimNode*) * $2, NULL, (void**)&$1);$n", + [name, size, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [name, size]) + proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; info: TLineInfo) = case n.kind @@ -992,8 +1032,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; if L == 1: genObjectFields(m, typ, origType, n.sons[0], expr, info) elif L > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)]) + var tmp = getTempName(m) & "_" & $L + genTNimNodeArray(m, tmp, rope(L)) for i in countup(0, L-1): var tmp2 = getNimNode(m) addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2]) @@ -1066,7 +1106,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo var tmp = getNimNode(m) if not isImportedType(typ): genObjectFields(m, typ, origType, typ.n, tmp, info) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), tmp]) var t = typ.sons[0] while t != nil: t = t.skipTypes(skipPtrs) @@ -1078,8 +1118,8 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) var expr = getNimNode(m) var length = sonsLen(typ) if length > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)]) + var tmp = getTempName(m) & "_" & $length + genTNimNodeArray(m, tmp, rope(length)) for i in countup(0, length - 1): var a = typ.sons[i] var tmp2 = getNimNode(m) @@ -1094,7 +1134,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) else: addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2;$n", [expr, rope(length)]) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), expr]) proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # Type information for enumerations is quite heavy, so we do some @@ -1102,10 +1142,9 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # anyway. We generate a cstring array and a loop over it. Exceptional # positions will be reset after the loop. genTypeInfoAux(m, typ, typ, name, info) - var nodePtrs = getTempName(m) var length = sonsLen(typ.n) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", - [nodePtrs, rope(length)]) + var nodePtrs = getTempName(m) & "_" & $length + genTNimNodeArray(m, nodePtrs, rope(length)) var enumNames, specialCases: Rope var firstNimNode = m.typeNodes var hasHoles = false @@ -1134,17 +1173,17 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = add(m.s[cfsTypeInit3], specialCases) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n$4.node = &$1;$n", - [getNimNode(m), rope(length), nodePtrs, name]) + [getNimNode(m), rope(length), nodePtrs, TINameForHcr(m, name)]) if hasHoles: # 1 << 2 is {ntfEnumHole} - addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name]) + addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [TINameForHcr(m, name)]) proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = assert(typ.sons[0] != nil) genTypeInfoAux(m, typ, typ, name, info) var tmp = getNimNode(m) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n", - [tmp, rope(firstOrd(m.config, typ)), name]) + [tmp, rope(firstOrd(m.config, typ)), TINameForHcr(m, name)]) proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info) @@ -1169,19 +1208,29 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = let origType = t var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses) + let prefixTI = if m.hcrOn: "(" else: "(&" + let sig = hashType(origType) result = m.typeInfoMarker.getOrDefault(sig) if result != nil: - return "(&".rope & result & ")".rope + return prefixTI.rope & result & ")".rope - result = m.g.typeInfoMarker.getOrDefault(sig) - if result != nil: + proc declareNimType(m: BModule, str: Rope, ownerModule: PSym) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [str]) + addf(m.s[cfsTypeInit1], "\t$1 = (TNimType*)hcrGetGlobal($2, \"$1\");$n", + [str, getModuleDllPath(m, ownerModule)]) + else: + addf(m.s[cfsVars], "extern TNimType $1;$n", [str]) + + let marker = m.g.typeInfoMarker.getOrDefault(sig) + if marker.str != nil: discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) + declareNimType(m, marker.str, marker.owner) # also store in local type section: - m.typeInfoMarker[sig] = result - return "(&".rope & result & ")".rope + m.typeInfoMarker[sig] = marker.str + return prefixTI.rope & marker.str & ")".rope result = "NTI$1_" % [rope($sig)] m.typeInfoMarker[sig] = result @@ -1194,10 +1243,10 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = # reference the type info as extern here discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) - return "(&".rope & result & ")".rope + declareNimType(m, result, owner) + return prefixTI.rope & result & ")".rope - m.g.typeInfoMarker[sig] = result + m.g.typeInfoMarker[sig] = (str: result, owner: owner) case t.kind of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent: @@ -1219,12 +1268,12 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = if m.config.selectedGC != gcDestructors: if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyRef: genTypeInfoAux(m, t, t, result, info) if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info) of tyArray: genArrayInfo(m, t, result, info) of tySet: genSetInfo(m, t, result, info) @@ -1241,7 +1290,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = genDeepCopyProc(m, t.deepCopy, result) elif origType.deepCopy != nil: genDeepCopyProc(m, origType.deepCopy, result) - result = "(&".rope & result & ")".rope + result = prefixTI.rope & result & ")".rope proc genTypeSection(m: BModule, n: PNode) = discard diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cb186de10100..d8f426f05d17 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,8 +16,6 @@ import condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf -import system/indexerrors - when not defined(leanCompiler): import semparallel @@ -46,6 +44,9 @@ when options.hasTinyCBackend: # implementation +proc hcrOn(m: BModule): bool = m.config.hcrOn +proc hcrOn(p: BProc): bool = p.module.config.hcrOn + proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) @@ -92,6 +93,17 @@ proc useHeader(m: BModule, sym: PSym) = proc cgsym(m: BModule, name: string): Rope +proc getCFile(m: BModule): AbsoluteFile + +proc getModuleDllPath(m: BModule): Rope = + let (dir, name, ext) = splitFile(getCFile(m)) + let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext]) + return makeCString(dir.string & "/" & filename) + +proc getModuleDllPath(m: BModule, s: PSym): Rope = + return getModuleDllPath(findPendingModule(m, s)) + +# TODO: please document proc ropecg(m: BModule, frmt: FormatStr, args: varargs[Rope]): Rope = assert m != nil var i = 0 @@ -440,12 +452,18 @@ proc assignLocalVar(p: BProc, n: PNode) = include ccgthreadvars proc varInDynamicLib(m: BModule, sym: PSym) -proc mangleDynLibProc(sym: PSym): Rope + +proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool = + return m.hcrOn and {sfThread, sfGlobal} * s.flags == {sfGlobal} and + ({lfNoDecl, lfHeader} * s.loc.flags == {}) + # and s.owner.kind == skModule # owner isn't always a module (global pragma on local var) + # and s.loc.k == locGlobalVar # loc isn't always initialized when this proc is used proc assignGlobalVar(p: BProc, n: PNode) = let s = n.sym if s.loc.k == locNone: fillLoc(s.loc, locGlobalVar, n, mangleName(p.module, s), OnHeap) + if treatGlobalDifferentlyForHCR(p.module, s): incl(s.loc.flags, lfIndirect) if lfDynamicLib in s.loc.flags: var q = findPendingModule(p.module, s) @@ -463,8 +481,10 @@ proc assignGlobalVar(p: BProc, n: PNode) = var decl: Rope = nil var td = getTypeDesc(p.module, s.loc.t) if s.constraint.isNil: - if sfImportc in s.flags: add(decl, "extern ") + if p.hcrOn: add(decl, "static ") + elif sfImportc in s.flags: add(decl, "extern ") add(decl, td) + if p.hcrOn: add(decl, "*") if sfRegister in s.flags: add(decl, " register") if sfVolatile in s.flags: add(decl, " volatile") addf(decl, " $1;$n", [s.loc.r]) @@ -853,6 +873,14 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum = for i in 0.. 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) -proc getSomeInitName(m: PSym, suffix: string): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle - result.add suffix - -proc getInitName(m: PSym): Rope = - if sfMainModule in m.flags: - # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" - else: - result = getSomeInitName(m, "Init000") - -proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") - - proc registerModuleToMain(g: BModuleList; m: BModule) = + let + init = m.getInitName + datInit = m.getDatInitName + + if m.hcrOn: + var hcr_module_meta = "$nN_LIB_PRIVATE const char* hcr_module_list[] = {$n" % [] + let systemModulePath = getModuleDllPath(m, g.modules[g.graph.config.m.systemFileIdx.int].module) + let mainModulePath = getModuleDllPath(m, m.module) + if sfMainModule in m.module.flags: + addf(hcr_module_meta, "\t$1,$n", [systemModulePath]) + g.graph.importDeps.withValue(FileIndex(m.module.position), deps): + for curr in deps[]: + addf(hcr_module_meta, "\t$1,$n", [getModuleDllPath(m, g.modules[curr.int].module)]) + addf(hcr_module_meta, "\t\"\"};$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", + [($sigHash(m.module)).rope]) + if sfMainModule in m.module.flags: + add(g.mainModProcs, hcr_module_meta) + addf(g.mainModProcs, "static void* hcr_handle;$N", []) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [init]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [datInit]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void*, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*));$N", [m.getHcrInitName]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void);$N", []) + addf(g.mainModInit, "\t$1();$N", [init]) + addf(g.otherModsInit, "\thcrInit((void**)hcr_module_list, $1, $2, $3, hcr_handle, nimGetProcAddr);$n", + [mainModulePath, systemModulePath, datInit]) + addf(g.mainDatInit, "\t$1(hcr_handle, nimGetProcAddr);$N", [m.getHcrInitName]) + addf(g.mainDatInit, "\thcrAddModule($1);\n", [mainModulePath]) + addf(g.mainDatInit, "\tHcrCreateTypeInfos();$N", []) + # nasty nasty hack to get the command line functionality working with HCR + # register the 2 variables on behalf of the os module which might not even + # be loaded (in which case it will get collected but that is not a problem) + let osModulePath = ($systemModulePath).replace("stdlib_system", "stdlib_os").rope + addf(g.mainDatInit, "\thcrAddModule($1);\n", [osModulePath]) + add(g.mainDatInit, "\tint* cmd_count;\n") + add(g.mainDatInit, "\tchar*** cmd_line;\n") + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdCount\", sizeof(cmd_count), NULL, (void**)&cmd_count);$N", [osModulePath]) + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdLine\", sizeof(cmd_line), NULL, (void**)&cmd_line);$N", [osModulePath]) + add(g.mainDatInit, "\t*cmd_count = cmdCount;\n") + add(g.mainDatInit, "\t*cmd_line = cmdLine;\n") + else: + add(m.s[cfsInitProc], hcr_module_meta) + return + if m.s[cfsDatInitProc].len > 0: - let datInit = m.module.getDatInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) addf(g.mainDatInit, "\t$1();$N", [datInit]) @@ -1287,7 +1419,6 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = add(g.mainDatInit, ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")) if m.s[cfsInitProc].len > 0: - let init = m.module.getInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.module.flags: @@ -1302,10 +1433,14 @@ proc genDatInitCode(m: BModule) = ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed - var moduleDatInitRequired = false + var moduleDatInitRequired = m.hcrOn + + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), getDatInitName(m)] - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % - [getDatInitName(m.module)] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) for i in cfsTypeInit1..cfsDynLibInit: if m.s[i].len != 0: @@ -1319,45 +1454,73 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: add(m.s[cfsDatInitProc], prc) +# Very similar to the contents of symInDynamicLib - basically only the +# things needed for the hot code reloading runtime procs to be loaded +proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): Rope = + let prc = magicsys.getCompilerProc(m.g.graph, sym) + assert prc != nil + fillProcLoc(m, prc.ast[namePos]) + + var extname = prefix & sym + var tmp = mangleDynLibProc(prc) + prc.loc.r = tmp + prc.typ.sym = nil + + if not containsOrIncl(m.declaredThings, prc.id): + addf(m.s[cfsVars], "static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t)]) + + result = "\t$1 = ($2) $3($4, $5);$n" % + [tmp, getTypeDesc(m, prc.typ), getProcFunc.rope, handle.rope, makeCString(prefix & sym)] + proc genInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed appcg(m, m.s[cfsForwardTypes], frameDefines, [rope("#")]) - var moduleInitRequired = false - let initname = getInitName(m.module) - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % [initname] + var moduleInitRequired = m.hcrOn + let initname = getInitName(m) + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) if m.typeNodes > 0: - appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", - [m.typeNodesName, rope(m.typeNodes)]) + if m.hcrOn: + appcg(m, m.s[cfsTypeInit1], "\t#TNimNode* $1;$N", [m.typeNodesName]) + appcg(m, m.s[cfsTypeInit1], "\thcrRegisterGlobal($3, \"$1_$2\", sizeof(TNimNode) * $2, NULL, (void**)&$1);$N", + [m.typeNodesName, rope(m.typeNodes), getModuleDllPath(m, m.module)]) + else: + appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", + [m.typeNodesName, rope(m.typeNodes)]) if m.nimTypes > 0: appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", [m.nimTypesName, rope(m.nimTypes)]) + if m.hcrOn: + addf(prc, "\tint* nim_hcr_dummy_ = 0;$n" & + "\tNIM_BOOL nim_hcr_do_init_ = " & + "hcrRegisterGlobal($1, \"module_initialized_\", 1, NULL, (void**)&nim_hcr_dummy_);$n", + [getModuleDllPath(m, m.module)]) + + template writeSection(thing: untyped, section: TCProcSection, addHcrGuards = false) = + if m.thing.s(section).len > 0: + moduleInitRequired = true + if addHcrGuards: add(prc, "\tif (nim_hcr_do_init_) {\n\n") + add(prc, genSectionStart(section, m.config)) + add(prc, m.thing.s(section)) + add(prc, genSectionEnd(section, m.config)) + if addHcrGuards: add(prc, "\n\t} // nim_hcr_do_init_\n") + if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: # Give this small function its own scope addf(prc, "{$N", []) # Keep a bogus frame in case the code needs one add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - if m.preInitProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.preInitProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) - - if m.preInitProc.s(cpsInit).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.preInitProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - if m.preInitProc.s(cpsStmts).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.preInitProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(preInitProc, cpsLocals) + writeSection(preInitProc, cpsInit, m.hcrOn) + writeSection(preInitProc, cpsStmts) addf(prc, "}$N", []) # add new scope for following code, because old vcc compiler need variable @@ -1367,11 +1530,7 @@ proc genInitCode(m: BModule) = moduleInitRequired = true add(prc, initGCFrame(m.initProc)) - if m.initProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.initProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) + writeSection(initProc, cpsLocals) if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: moduleInitRequired = true @@ -1385,13 +1544,8 @@ proc genInitCode(m: BModule) = else: add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.initProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.initProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(initProc, cpsInit, m.hcrOn) + writeSection(initProc, cpsStmts) if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: add(prc, deinitFrame(m.initProc)) @@ -1407,6 +1561,19 @@ proc genInitCode(m: BModule) = # that would lead to a *nesting* of merge sections which the merger does # not support. So we add it to another special section: ``cfsInitProc`` + if m.hcrOn: + var procsToLoad = @["hcrRegisterProc", "hcrGetProc", "hcrRegisterGlobal", "hcrGetGlobal"] + + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, $1)(void* handle, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*)) {$N", [getHcrInitName(m)]) + if sfMainModule in m.module.flags: + # additional procs to load + procsToLoad.add("hcrInit") + procsToLoad.add("hcrAddModule") + # load procs + for curr in procsToLoad: + add(m.s[cfsInitProc], hcrGetProcLoadCode(m, curr, "", "handle", "getProcAddr")) + addf(m.s[cfsInitProc], "}$N$N", []) + for i, el in pairs(m.extensionLoaders): if el != nil: let ex = "NIM_EXTERNC N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" % @@ -1418,6 +1585,12 @@ proc genInitCode(m: BModule) = add(m.s[cfsInitProc], prc) genDatInitCode(m) + + if m.hcrOn: + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void) {$N", []) + add(m.s[cfsInitProc], m.hcrCreateTypeInfosProc) + addf(m.s[cfsInitProc], "}$N$N", []) + registerModuleToMain(m.g, m) proc genModule(m: BModule, cfile: Cfile): Rope = @@ -1445,7 +1618,7 @@ proc genModule(m: BModule, cfile: Cfile): Rope = if m.s[cfsInitProc].len > 0: moduleIsEmpty = false add(result, m.s[cfsInitProc]) - if m.s[cfsDatInitProc].len > 0: + if m.s[cfsDatInitProc].len > 0 or m.hcrOn: moduleIsEmpty = false add(result, m.s[cfsDatInitProc]) @@ -1565,6 +1738,25 @@ when false: readMergeInfo(getCFile(m), m) result = m +proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, inInitGuard) + else: + let stmtShouldExecute = n.kind in {nkVarSection, nkLetSection} or + nfExecuteOnReload in n.flags + if inInitGuard: + if stmtShouldExecute: + endBlock(p) + inInitGuard = false + else: + if not stmtShouldExecute: + line(p, cpsStmts, "if (nim_hcr_do_init_)\n") + startBlock(p) + inInitGuard = true + + genStmts(p, n) + proc myProcess(b: PPassContext, n: PNode): PNode = result = n if b == nil: return @@ -1573,8 +1765,11 @@ proc myProcess(b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! - let tranformed_n = transformStmt(m.g.graph, m.module, n) - genStmts(m.initProc, tranformed_n) + let transformed_n = transformStmt(m.g.graph, m.module, n) + if m.hcrOn: + addHcrInitGuards(m.initProc, transformed_n, m.inHcrInitGuard) + else: + genStmts(m.initProc, transformed_n) proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = result = true @@ -1674,7 +1869,24 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) genStmts(m.initProc, n) + if m.hcrOn: + # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init) + discard cgsym(m, "programResult") + if m.inHcrInitGuard: + endBlock(m.initProc) + if sfMainModule in m.module.flags: + if m.hcrOn: + # pull ("define" since they are inline when HCR is on) these functions in the main file + # so it can load the HCR runtime and later pass the library handle to the HCR runtime which + # will in turn pass it to the other modules it initializes so they can initialize the + # register/get procs so they don't have to have the definitions of these functions as well + discard cgsym(m, "nimLoadLibrary") + discard cgsym(m, "nimLoadLibraryError") + discard cgsym(m, "nimGetProcAddr") + discard cgsym(m, "procAddrError") + discard cgsym(m, "rawWrite") + # raise dependencies on behalf of genMainProc if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: discard cgsym(m, "initStackBottomWith") @@ -1709,14 +1921,11 @@ proc genForwardedProcs(g: BModuleList) = proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = let g = BModuleList(backend) + g.config = config + # we need to process the transitive closure because recursive module # deps are allowed (and the system module is processed in the wrong # order anyway) - g.config = config - let (outDir, _, _) = splitFile(config.outfile) - if not outDir.isEmpty: - createDir(outDir) - genForwardedProcs(g) for m in cgenModules(g): diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 50b484e31c4f..7b1e7c123395 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -98,6 +98,7 @@ type TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] + TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]] Codegenflag* = enum preventStackTrace, # true if stack traces need to be prevented @@ -118,7 +119,7 @@ type generatedHeader*: BModule breakPointId*: int breakpoints*: Rope # later the breakpoints are inserted into the main proc - typeInfoMarker*: TypeCache + typeInfoMarker*: TypeCacheWithOwner config*: ConfigRef graph*: ModuleGraph strVersion*, seqVersion*: int # version of the string/seq implementation to use @@ -150,6 +151,8 @@ type typeInfoMarker*: TypeCache # needed for generating type information initProc*: BProc # code for init procedure preInitProc*: BProc # code executed before the init proc + hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on + inHcrInitGuard*: bool # We are currently withing a HCR reloading guard. typeStack*: TTypeSeq # used for type generation dataCache*: TNodeTable typeNodes*, nimTypes*: int # used for type info generation @@ -189,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc = result.sigConflicts = initCountTable[string]() proc newModuleList*(g: ModuleGraph): BModuleList = - BModuleList(typeInfoMarker: initTable[SigHash, Rope](), config: g.config, - graph: g, nimtvDeclared: initIntSet()) + BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](), + config: g.config, graph: g, nimtvDeclared: initIntSet()) iterator cgenModules*(g: BModuleList): BModule = for m in g.modules_closed: diff --git a/compiler/commands.nim b/compiler/commands.nim index 56fe8f2057a5..1f46abca45b8 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -387,7 +387,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.nimcacheDir = processPath(conf, arg, info, true) of "out", "o": expectArg(conf, switch, arg, pass, info) - conf.outFile = AbsoluteFile arg + let f = splitFile(arg.expandTilde) + conf.outFile = RelativeFile f.name + conf.outDir = toAbsoluteDir f.dir + of "outdir": + expectArg(conf, switch, arg, pass, info) + conf.outDir = toAbsoluteDir arg.expandTilde of "docseesrcurl": expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg @@ -487,9 +492,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if optMemTracker in conf.options: defineSymbol(conf.symbols, "memtracker") else: undefSymbol(conf.symbols, "memtracker") of "hotcodereloading": - processOnOffSwitch(conf, {optHotCodeReloading}, arg, pass, info) - if optHotCodeReloading in conf.options: defineSymbol(conf.symbols, "hotcodereloading") - else: undefSymbol(conf.symbols, "hotcodereloading") + processOnOffSwitchG(conf, {optHotCodeReloading}, arg, pass, info) + if conf.hcrOn: + defineSymbol(conf.symbols, "hotcodereloading") + defineSymbol(conf.symbols, "useNimRtl") + else: + undefSymbol(conf.symbols, "hotcodereloading") + undefSymbol(conf.symbols, "useNimRtl") of "oldnewlines": case arg.normalize of "","on": @@ -787,7 +796,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: # support UNIX style filenames everywhere for portable build scripts: - config.projectName = unixToNativePath(p.key) + if config.projectName.len == 0: + config.projectName = unixToNativePath(p.key) config.arguments = cmdLineRest(p) result = true inc argsCount diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 5e7ce3a08184..57dd551324ea 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -82,7 +82,9 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasTypeof") defineSymbol("nimErrorProcCanHaveBody") defineSymbol("nimHasInstantiationOfInMacro") + defineSymbol("nimHasHotCodeReloading") defineSymbol("nimHasNilSeqs") + defineSymbol("nimHasSignatureHashInMacro") for f in low(Feature)..high(Feature): defineSymbol("nimHas" & $f) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5af4c464e16e..28d53533f7fe 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -100,14 +100,12 @@ proc parseRst(text, filename: string, proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile = if optWholeProject in conf.globalOptions: - # This is correct, for 'nim doc --project' we interpret the '--out' option as an - # absolute directory, not as a filename! - let d = if conf.outFile.isEmpty: conf.projectPath / dir else: AbsoluteDir(conf.outFile) + let d = if conf.outDir.isEmpty: conf.projectPath / dir else: conf.outDir createDir(d) result = d / changeFileExt(filename, ext) elif guessTarget: - let d = if not conf.outFile.isEmpty: splitFile(conf.outFile).dir - else: conf.projectPath + let d = if not conf.outDir.isEmpty: conf.outDir + else: conf.projectPath createDir(d) result = d / changeFileExt(filename, ext) else: @@ -952,9 +950,8 @@ proc genOutFile(d: PDoc): Rope = proc generateIndex*(d: PDoc) = if optGenIndex in d.conf.globalOptions: - let dir = if d.conf.outFile.isEmpty: d.conf.projectPath / RelativeDir"htmldocs" - elif optWholeProject in d.conf.globalOptions: AbsoluteDir(d.conf.outFile) - else: AbsoluteDir(d.conf.outFile.string.splitFile.dir) + let dir = if not d.conf.outDir.isEmpty: d.conf.outDir + else: d.conf.projectPath / RelativeDir"htmldocs" createDir(dir) let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename, d.conf.projectPath), IndexExt) @@ -995,6 +992,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) = "\" for writing") proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index f2ac6c1a9d77..2232601f4fbd 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -29,7 +29,6 @@ template shouldProcess(g): bool = template closeImpl(body: untyped) {.dirty.} = var g = PGen(p) let useWarning = sfMainModule notin g.module.flags - #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId if shouldProcess(g): body try: diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2c5af6433b3d..18a81ca9676d 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - lineinfos, std / sha1, streams, pathutils + lineinfos, std / sha1, streams, pathutils, sequtils, times type TInfoCCProp* = enum # properties of the C compiler: @@ -468,8 +468,8 @@ proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % cmd) -proc generateScript(conf: ConfigRef; projectFile: AbsoluteFile, script: Rope) = - let (_, name, _) = splitFile(projectFile) +proc generateScript(conf: ConfigRef; script: Rope) = + let (_, name, _) = splitFile(conf.outFile.string) let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name, platform.OS[conf.target.targetOS].scriptExt)) if writeRope(script, filename): @@ -551,14 +551,14 @@ proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler else: getCompilerExe(conf, compiler, AbsoluteFile"") -proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile): string = +proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string = var c = conf.cCompiler var options = cFileSpecificOptions(conf, cfile.cname) var exe = getConfigVar(conf, c, ".exe") if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) if needsExeExt(conf): exe = addFileExt(exe, "exe") - if optGenDynLib in conf.globalOptions and + if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and ospNeedsPIC in platform.OS[conf.target.targetOS].props: add(options, ' ' & CC[c].pic) @@ -648,12 +648,14 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) -proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, +proc compileCFiles(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, prettyCmds: var TStringSeq) = + var currIdx = 0 for it in list: # call the C compiler for the .c file: if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(conf, it) + var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1) + inc currIdx if optCompileOnly notin conf.globalOptions: add(cmds, compileCmd) let (_, name, _) = splitFile(it.cname) @@ -662,7 +664,8 @@ proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var add(script, compileCmd) add(script, "\n") -proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): string = +proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, + objfiles: string, isDllBuild: bool): string = if optGenStaticLib in conf.globalOptions: var libname: string if not conf.outFile.isEmpty: @@ -672,7 +675,7 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s else: libname = (libNameTmpl(conf) % splitFile(conf.projectName).name) result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname), - "objfiles", objfiles] + "objfiles", objfiles] else: var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe") if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler) @@ -684,28 +687,16 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s CC[conf.cCompiler].buildGui else: "" - var exefile, builddll: string - if optGenDynLib in conf.globalOptions: - exefile = platform.OS[conf.target.targetOS].dllFrmt % splitFile(projectfile).name - builddll = CC[conf.cCompiler].buildDll - else: - exefile = splitFile(projectfile).name & platform.OS[conf.target.targetOS].exeExt - builddll = "" - if not conf.outFile.isEmpty: - exefile = conf.outFile.string.expandTilde - if not exefile.isAbsolute(): - exefile = getCurrentDir() / exefile - if not noAbsolutePaths(conf): - if not exefile.isAbsolute(): - exefile = string(splitFile(projectfile).dir / RelativeFile(exefile)) + let builddll = if isDllBuild: CC[conf.cCompiler].buildDll else: "" + let exefile = quoteShell(output) + when false: if optCDebug in conf.globalOptions: writeDebugInfo(exefile.changeFileExt("ndb")) - exefile = quoteShell(exefile) # Map files are required by Nintendo Switch compilation. They are a list # of all function calls in the library and where they come from. - let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(projectFile).name & ".map")) + let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(output).name & ".map")) let linkOptions = getLinkOptions(conf) & " " & getConfigVar(conf, conf.cCompiler, ".options.linker") @@ -723,6 +714,46 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s "objfiles", objfiles, "exefile", exefile, "nim", quoteShell(getPrefixDir(conf)), "lib", quoteShell(conf.libpath)]) + # On windows the debug information for binaries is emitted in a separate .pdb + # file and the binaries (.dll and .exe) contain a full path to that .pdb file. + # This is a problem for hot code reloading because even when we copy the .dll + # and load the copy so the build process may overwrite the original .dll on + # the disk (windows locks the files of running binaries) the copy still points + # to the original .pdb (and a simple copy of the .pdb won't help). This is a + # problem when a debugger is attached to the program we are hot-reloading. + # This problem is nonexistent on Unix since there by default debug symbols + # are embedded in the binaries so loading a copy of a .so will be fine. There + # is the '/Z7' flag for the MSVC compiler to embed the debug info of source + # files into their respective .obj files but the linker still produces a .pdb + # when a final .dll or .exe is linked so the debug info isn't embedded. + # There is also the issue that even when a .dll is unloaded the debugger + # still keeps the .pdb for that .dll locked. This is a major problem and + # because of this we cannot just alternate between 2 names for a .pdb file + # when rebuilding a .dll - instead we need to accumulate differently named + # .pdb files in the nimcache folder - this is the easiest and most reliable + # way of being able to debug and rebuild the program at the same time. This + # is accomplished using the /PDB: flag (there also exists the + # /PDBALTPATH: flag). The only downside is that the .pdb files are + # atleast 5-10mb big and will quickly accumulate. There is a hacky solution: + # we could try to delete all .pdb files with a pattern and swallow exceptions. + # + # links about .pdb files and hot code reloading: + # https://ourmachinery.com/post/dll-hot-reloading-in-theory-and-practice/ + # https://ourmachinery.com/post/little-machines-working-together-part-2/ + # https://github.com/fungos/cr + # https://fungos.github.io/blog/2017/11/20/cr.h-a-simple-c-hot-reload-header-only-library/ + # on forcing the debugger to unlock a locked .pdb of an unloaded library: + # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/ + # and a bit about the .pdb format in case that is ever needed: + # https://github.com/crosire/blink + # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles + if conf.hcrOn and conf.cCompiler == ccVcc: + let t = now() + let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb" + result.add " /link /PDB:" & pdb + +template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string = + getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions) template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped): typed = try: @@ -799,7 +830,17 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = finally: removeFile(linkerArgs) -proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc getObjFilePath(conf: ConfigRef, f: CFile): string = + if noAbsolutePaths(conf): f.obj.extractFilename + else: f.obj.string + +proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): AbsoluteFile = + let basename = splitFile(objFile).name + let targetName = if isMain: basename & ".exe" + else: platform.OS[conf.target.targetOS].dllFrmt % basename + result = conf.nimcacheDir / RelativeFile(targetName) + +proc callCCompiler*(conf: ConfigRef) = var linkCmd: string if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: @@ -813,7 +854,7 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = when declared(echo): let cmd = prettyCmds[idx] if cmd != "": echo cmd - compileCFile(conf, conf.toCompile, script, cmds, prettyCmds) + compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) if optNoLinking notin conf.globalOptions: @@ -824,31 +865,64 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = add(objfiles, ' ') add(objfiles, quoteShell( addFileExt(objFile, CC[conf.cCompiler].objExt))) - for x in conf.toCompile: - let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string - add(objfiles, ' ') - add(objfiles, quoteShell(objFile)) - linkCmd = getLinkCmd(conf, projectfile, objfiles) - if optCompileOnly notin conf.globalOptions: - if defined(windows) and linkCmd.len > 8_000: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) + if conf.hcrOn: # lets assume that optCompileOnly isn't on + cmds = @[] + let mainFileIdx = conf.toCompile.len - 1 + for idx, x in conf.toCompile: + # don't relink each of the many binaries (one for each source file) if the nim code is + # cached because that would take too much time for small changes - the only downside to + # this is that if an external-to-link file changes the final target wouldn't be relinked + if x.flags.contains(CfileFlag.Cached): continue + # we pass each object file as if it is the project file - a .dll will be created for each such + # object file in the nimcache directory, and only in the case of the main project file will + # there be probably an executable (if the project is such) which will be copied out of the nimcache + let objFile = conf.getObjFilePath(x) + let buildDll = idx != mainFileIdx + let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll) + add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll)) + # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache + # for more info check the comment inside of getLinkCmd() where the /PDB: MSVC flag is used + if conf.cCompiler == ccVcc: + for pdb in walkFiles(objFile & ".*.pdb"): + discard tryRemoveFile(pdb) + # execute link commands in parallel - output will be a bit different + # if it fails than that from execLinkCmd() but that doesn't matter + prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) + execCmdsInParallel(conf, cmds, prettyCb) + # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination + if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached): + let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) + var src = conf.hcrLinkTargetName(mainObjFile, true) + var dst = conf.prepareToWriteOutput + copyFileWithPermissions(src.string, dst.string) + else: + for x in conf.toCompile: + let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string + add(objfiles, ' ') + add(objfiles, quoteShell(objFile)) + let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput + else: AbsoluteFile(conf.projectName) + linkCmd = getLinkCmd(conf, mainOutput, objfiles) + if optCompileOnly notin conf.globalOptions: + if defined(windows) and linkCmd.len > 8_000: + # Windows's command line limit is about 8K (don't laugh...) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) else: linkCmd = "" if optGenScript in conf.globalOptions: add(script, linkCmd) add(script, "\n") - generateScript(conf, projectfile, script) + generateScript(conf, script) #from json import escapeJson import json -proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc writeJsonBuildInstructions*(conf: ConfigRef) = template lit(x: untyped) = f.write x template str(x: untyped) = when compiles(escapeJson(x, buf)): @@ -895,7 +969,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = var buf = newStringOfCap(50) - let jsonFile = toGeneratedFile(conf, projectfile, "json") + let jsonFile = conf.nimcacheDir / RelativeFile(conf.projectName & ".json") var f: File if open(f, jsonFile.string, fmWrite): @@ -907,7 +981,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) lit "],\L\"linkcmd\": " - str getLinkCmd(conf, projectfile, objfiles) + str getLinkCmd(conf, conf.absOutFile, objfiles) lit "\L}\L" close(f) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index cad8fc9909d1..e3ca6830c141 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -83,7 +83,6 @@ type forwarded: seq[PSym] generatedSyms: IntSet typeInfoGenerated: IntSet - classes: seq[(PType, Rope)] unique: int # for temp identifier generation PProc = ref TProc @@ -133,7 +132,6 @@ proc newGlobals(): PGlobals = result.forwarded = @[] result.generatedSyms = initIntSet() result.typeInfoGenerated = initIntSet() - result.classes = @[] proc initCompRes(r: var TCompRes) = r.address = nil @@ -250,7 +248,7 @@ proc mangleName(m: BModule, s: PSym): Rope = result = rope(x) # From ES5 on reserved words can be used as object field names if s.kind != skField: - if optHotCodeReloading in m.config.options: + if m.config.hcrOn: # When hot reloading is enabled, we must ensure that the names # of functions and types will be preserved across rebuilds: add(result, idOrSig(s, m.module.name.s, m.sigConflicts)) @@ -1206,19 +1204,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = gen(p, n.sons[0].sons[0], r) else: internalError(p.config, n.sons[0].info, "genAddr: " & $n.sons[0].kind) -proc thisParam(p: PProc; typ: PType): PType = - discard - proc attachProc(p: PProc; content: Rope; s: PSym) = - let otyp = thisParam(p, s.typ) - if otyp != nil: - for i, cls in p.g.classes: - if sameType(cls[0], otyp): - add(p.g.classes[i][1], content) - return - p.g.classes.add((otyp, content)) - else: - add(p.g.code, content) + add(p.g.code, content) proc attachProc(p: PProc; s: PSym) = let newp = genProc(p, s) @@ -1460,9 +1447,6 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) = genArgs(p, n, r, 2) proc genCall(p: PProc, n: PNode, r: var TCompRes) = - if n.sons[0].kind == nkSym and thisParam(p, n.sons[0].typ) != nil: - genInfixCall(p, n, r) - return gen(p, n.sons[0], r) genArgs(p, n, r) if n.typ != nil: @@ -1606,13 +1590,14 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = s: Rope varCode: string varName = mangleName(p.module, v) - useReloadingGuard = sfGlobal in v.flags and optHotCodeReloading in p.config.options + useReloadingGuard = sfGlobal in v.flags and p.config.hcrOn if v.constraint.isNil: if useReloadingGuard: lineF(p, "var $1;$n", varName) lineF(p, "if ($1 === undefined) {$n", varName) varCode = $varName + inc p.extraIndent else: varCode = "var $2" else: @@ -1664,6 +1649,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, s]) if useReloadingGuard: + dec p.extraIndent lineF(p, "}$n") proc genVarStmt(p: PProc, n: PNode) = @@ -2204,7 +2190,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = else: result = ~"\L" - if optHotCodeReloading in p.config.options: + if p.config.hcrOn: # Here, we introduce thunks that create the equivalent of a jump table # for all global functions, because references to them may be stored # in JavaScript variables. The added indirection ensures that such @@ -2440,13 +2426,52 @@ proc genHeader(): Rope = "if (typeof Float64Array === 'undefined') Float64Array = Array;$n") % [rope(VersionAsString)] +proc addHcrInitGuards(p: PProc, n: PNode, + moduleLoadedVar: Rope, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, moduleLoadedVar, inInitGuard) + else: + let stmtShouldExecute = n.kind in { + nkProcDef, nkFuncDef, nkMethodDef,nkConverterDef, + nkVarSection, nkLetSection} or nfExecuteOnReload in n.flags + + if inInitGuard: + if stmtShouldExecute: + dec p.extraIndent + line(p, "}\L") + inInitGuard = false + else: + if not stmtShouldExecute: + lineF(p, "if ($1 == undefined) {$n", [moduleLoadedVar]) + inc p.extraIndent + inInitGuard = true + + genStmt(p, n) + proc genModule(p: PProc, n: PNode) = if optStackTrace in p.options: add(p.body, frameCreate(p, makeJSString("module " & p.module.module.name.s), makeJSString(toFilename(p.config, p.module.module.info)))) let n_transformed = transformStmt(p.module.graph, p.module.module, n) - genStmt(p, n_transformed) + if p.config.hcrOn and n.kind == nkStmtList: + let moduleSym = p.module.module + var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" & + idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts) + lineF(p, "var $1;$n", [moduleLoadedVar]) + var inGuardedBlock = false + + addHcrInitGuards(p, n_transformed, moduleLoadedVar, inGuardedBlock) + + if inGuardedBlock: + dec p.extraIndent + line(p, "}\L") + + lineF(p, "$1 = true;$n", [moduleLoadedVar]) + else: + genStmt(p, n_transformed) + if optStackTrace in p.options: add(p.body, frameDestroy(p)) @@ -2487,44 +2512,14 @@ proc getClassName(t: PType): Rope = if s.loc.r != nil: result = s.loc.r else: result = rope(s.name.s) -proc genClass(conf: ConfigRef; obj: PType; content: Rope; ext: string) = - let cls = getClassName(obj) - let t = skipTypes(obj, abstractPtrs) - let extends = if t.kind == tyObject and t.sons[0] != nil: - " extends " & getClassName(t.sons[0]) - else: nil - let result = ("` or `BrowserSync ` -to implement the actual reloading behavior in your project. +Once your code is compiled for hot reloading, the ``nim-livereload`` NPM +package provides a convenient solution for implementing the actual reloading +in the browser using a framework such as [LiveReload](http://livereload.com/) +or [BrowserSync](https://browsersync.io/). DynlibOverride diff --git a/lib/core/hotcodereloading.nim b/lib/core/hotcodereloading.nim new file mode 100644 index 000000000000..8b48b3d691dd --- /dev/null +++ b/lib/core/hotcodereloading.nim @@ -0,0 +1,27 @@ +when defined(hotcodereloading): + import + macros + + template beforeCodeReload*(body: untyped) = + hcrAddEventHandler(true, proc = body) {.executeOnReload.} + + template afterCodeReload*(body: untyped) = + hcrAddEventHandler(false, proc = body) {.executeOnReload.} + + macro hasModuleChanged*(module: typed): untyped = + if module.kind != nnkSym or module.symKind != nskModule: + error "hasModuleChanged expects a module symbol", module + return newCall(bindSym"hcrHasModuleChanged", newLit(module.signatureHash)) + + proc hasAnyModuleChanged*(): bool = hcrReloadNeeded() + + when not defined(JS): + template performCodeReload* = hcrPerformCodeReload() + else: + template performCodeReload* = discard +else: + template beforeCodeReload*(body: untyped) = discard + template afterCodeReload*(body: untyped) = discard + template hasModuleChanged*(module: typed): bool = false + proc hasAnyModuleChanged*(): bool = false + template performCodeReload*() = discard diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 461afb963a08..8e6b93a1164d 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -347,6 +347,13 @@ object doAssert(dumpTypeImpl(b) == t) doAssert(dumpTypeImpl(c) == t) +when defined(nimHasSignatureHashInMacro): + proc signatureHash*(n: NimNode): string {.magic: "NSigHash", noSideEffect.} + ## Returns a stable identifier derived from the signature of a symbol. + ## The signature combines many factors such as the type of the symbol, + ## the owning module of the symbol and others. The same identifier is + ## used in the back-end to produce the mangled symbol name. + proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} ## Version of ``getTypeImpl`` which takes a ``typedesc``. diff --git a/lib/core/strs.nim b/lib/core/strs.nim index ccbde76fe1fc..e55c884935a3 100644 --- a/lib/core/strs.nim +++ b/lib/core/strs.nim @@ -125,7 +125,7 @@ proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = if str == nil: toNimStr(str, 0) else: toNimStr(str, str.len) -proc nimToCStringConv(s: NimStringV2): cstring {.compilerProc, inline.} = +proc nimToCStringConv(s: NimStringV2): cstring {.compilerProc, nonReloadable, inline.} = if s.len == 0: result = cstring"" else: result = cstring(unsafeAddr s.p.data) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index d6dd16b54c2d..fe958c7f53a2 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -27,8 +27,6 @@ include "system/inclrtl.nim" include "system/hti.nim" -import system/indexerrors - {.pop.} type diff --git a/lib/nimbase.h b/lib/nimbase.h index ba427372602a..9fd475c85a11 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -354,7 +354,12 @@ typedef NU8 NU; # endif #endif +// for now there isn't an easy way for C code to reach the program result +// when hot code reloading is ON - users will have to: +// load the nimhcr.dll, get the hcrGetGlobal proc from there and use it +#ifndef NIM_HOT_CODE_RELOADING extern NI nim_program_result; +#endif typedef float NF32; typedef double NF64; diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim new file mode 100644 index 000000000000..f3afac34701e --- /dev/null +++ b/lib/nimhcr.nim @@ -0,0 +1,652 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This is the Nim hot code reloading run-time for the native targets. +## +## This minimal dynamic library is not subject to reloading when the +## `hotCodeReloading` build mode is enabled. It's responsible for providing +## a permanent memory location for all globals and procs within a program +## and orchestrating the reloading. For globals, this is easily achieved +## by storing them on the heap. For procs, we produce on the fly simple +## trampolines that can be dynamically overwritten to jump to a different +## target. In the host program, all globals and procs are first registered +## here with ``hcrRegisterGlobal`` and ``hcrRegisterProc`` and then the +## returned permanent locations are used in every reference to these symbols +## onwards. +## +## Detailed description: +## +## When code is compiled with the hotCodeReloading option for native targets +## a couple of things happen for all modules in a project: +## - the useNimRtl option is forced (including when building the HCR runtime too) +## - all modules of a target get built into separate shared libraries +## - the smallest granularity of reloads is modules +## - for each .c (or .cpp) in the corresponding nimcache folder of the project +## a shared object is built with the name of the source file + DLL extension +## - only the main module produces whatever the original project type intends +## (again in nimcache) and is then copied to its original destination +## - linking is done in parallel - just like compilation +## - function calls to functions from the same project go through function pointers: +## - with a few exceptions - see the nonReloadable pragma +## - the forward declarations of the original functions become function +## pointers as static globals with the same names +## - the original function definitions get suffixed with _actual +## - the function pointers get initialized with the address of the corresponding +## function in the DatInit of their module through a call to either hcrRegisterProc +## or hcrGetProc. When being registered, the _actual address is passed to +## hcrRegisterProc and a permanent location is returned and assigned to the pointer. +## This way the implementation (_actual) can change but the address for it +## will be the same - this works by just updating a jump instruction (trampoline). +## For functions from other modules hcrGetProc is used (after they are registered). +## - globals are initialized only once and their state is preserved +## - including locals with the {.global.} pragma +## - their definitions are changed into pointer definitions which are initialized +## in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the +## size of the type that this HCR runtime should allocate) and a bool is returned +## which when true triggers the initialization code for the global (only once). +## Globals from other modules: a global pointer coupled with a hcrGetGlobal call. +## - globals which have already been initialized cannot have their values changed +## by changing their initialization - use a handler or some other mechanism +## - new globals can be introduced when reloading +## - top-level code (global scope) is executed only once - at the first module load +## - the runtime knows every symbol's module owner (globals and procs) +## - both the RTL and HCR shared libraries need to be near the program for execution +## - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS) +## - the main module is responsible for initializing the HCR runtime +## - the main module loads the RTL and HCR shared objects +## - after that a call to hcrInit() is done in the main module which triggers +## the loading of all modules the main one imports, and doing that for the +## dependencies of each module recursively. Basically a DFS traversal. +## - then initialization takes place with several passes over all modules: +## - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc +## - HcrCreateTypeInfos - creates globals which will be referenced in the next pass +## - DatInit - usual dat init + register/get procs and get globals +## - Init - it does the following multiplexed operations: +## - register globals (if already registered - then just retrieve pointer) +## - execute top level scope (only if loaded for the first time) +## - when modules are loaded the originally built shared libraries get copied in +## the same folder and the copies are loaded instead of the original files +## - a module import tree is built in the runtime (and maintained when reloading) +## - hcrPerformCodeReload +## - named `performCodeReload`, requires the hotcodereloading module +## - explicitly called by the user - the current active callstack shouldn't contain +## any functions which are defined in modules that will be reloaded (or crash!). +## The reason is that old dynalic libraries get unloaded. +## Example: +## if A is the main module and it imports B, then only B is reloadable and only +## if when calling hcrPerformCodeReload there is no function defined in B in the +## current active callstack at the point of the call (it has to be done from A) +## - for reloading to take place the user has to have rebuilt parts of the application +## without changes affecting the main module in any way - it shouldn't be rebuilt. +## - to determine what needs to be reloaded the runtime starts traversing the import +## tree from the root and checks the timestamps of the loaded shared objects +## - modules that are no longer referenced are unloaded and cleaned up properly +## - symbols (procs/globals) that have been removed in the code are also cleaned up +## - so changing the init of a global does nothing, but removing it, reloading, +## and then re-introducing it with a new initializer works +## - new modules can be imported, and imports can also be reodereded/removed +## - hcrReloadNeeded() can be used to determine if any module needs reloading +## - named `hasAnyModuleChanged`, requires the hotcodereloading module +## - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload +## - require the hotcodereloading module +## - such handlers can be added and removed +## - before each reload all "beforeCodeReload" handlers are executed and after +## that all handlers (including "after") from the particular module are deleted +## - the order of execution is the same as the order of top-level code execution. +## Example: if A imports B which imports C, then all handlers in C will be executed +## first (from top to bottom) followed by all from B and lastly all from A +## - after the reload all "after" handlers are executed the same way as "before" +## - the handlers for a reloaded module are always removed when reloading and then +## registered when the top-level scope is executed (thanks to `executeOnReload`) +## +## TODO - after first merge in upstream Nim: +## +## - profile +## - build speed with and without hot code reloading - difference should be small +## - runtime degradation of HCR-enabled code - important!!! +## - ARM support for the trampolines +## - investigate: +## - rethink the closure iterators +## - ability to keep old versions of dynamic libraries alive +## - because of async server code +## - perhaps with refcounting of .dlls for unfinished closures +## - linking with static libs +## - all shared objects for each module will (probably) have to link to them +## - state in static libs gets duplicated +## - linking is slow and therefore iteration time suffers +## - have just a single .dll for all .nim files and bulk reload? +## - think about the compile/link/passC/passL/emit/injectStmt pragmas +## - if a passC pragma is introduced (either written or dragged in by a new +## import) the whole command line for compilation changes - for example: +## winlean.nim: {.passC: "-DWIN32_LEAN_AND_MEAN".} +## - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*' +## - everything thread-local related +## - tests +## - add a new travis build matrix entry which builds everything with HCR enabled +## - currently building with useNimRtl is problematic - lots of problems... +## - how to supply the nimrtl/nimhcr shared objects to all test binaries...? +## - think about building to C++ instead of only to C - added type safety +## - run tests through valgrind and the sanitizers! of HUGE importance! +## +## TODO - nice to have cool stuff: +## +## - separate handling of global state for much faster reloading and manipulation +## - imagine sliders in an IDE for tweaking variables +## - perhaps using shared memory +## - multi-dll projects - how everything can be reloaded..? +## - a single HCR instance shared across multiple .dlls +## - instead of having to call hcrPerformCodeReload from a function in each dll +## - which currently renders the main module of each dll not reloadable +## - ability to check with the current callstack if a reload is "legal" +## - if it is in any function which is in a module about to be reloaded ==> error +## - pragma annotations for files - to be excluded from dll shenanigans +## - for such file-global pragmas look at codeReordering or injectStmt +## - how would the initialization order be kept? messy... +## - per function exclude pragmas would be TOO messy and hard... +## - C code calling stable exportc interface of nim code (for bindings) +## - generate proxy functions with the stable names +## - in a non-reloadable part (the main binary) that call the function pointers +## - parameter passing/forwarding - how? use the same trampoline jumping? +## - extracting the dependencies for these stubs/proxies will be hard... +## - changing memory layout of types - detecting this..? +## - implement with registerType() call to HCR runtime...? +## - and checking if a previously registered type matches +## - issue an error +## - or let the user handle this by transferring the state properly +## - perhaps in the before/afterCodeReload handlers +## - optimization: calls to procs within a module (+inlined) to use the _actual versions +## - implement executeOnReload for global vars too - not just statements (and document!) +## - cleanup at shutdown - freeing all globals +## +## TODO - unimportant: +## +## - have a "bad call" trampoline that all no-longer-present functions are routed to call there +## - so the user gets some error msg if he calls a dangling pointer instead of a crash +## - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate +## - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h) +## - proper .json build file - but the format is different... multiple link commands... +## - avoid registering globals on each loop when using an iterator in global scope +## +## TODO - REPL: +## - proper way (as proposed by Zahary): +## - parse the input code and put everything in global scope except for +## statements with side effects only - those go in afterCodeReload blocks +## - my very hacky idea: just append to a closure iterator the new statements +## followed by a yield statement. So far I can think of 2 problems: +## - import and some other code cannot be written inside of a proc - +## has to be parsed and extracted in the outer scope +## - when new variables are created they are actually locals to the closure +## so the struct for the closure state grows in memory, but it has already +## been allocated when the closure was created with the previous smaller size. +## That would lead to working with memory outside of the initially allocated +## block. Perhaps something can be done about this - some way of re-allocating +## the state and transferring the old... + +when not defined(JS) and (defined(hotcodereloading) or + defined(createNimHcr) or + defined(testNimHcr)): + const + dllExt = when defined(windows): "dll" + elif defined(macosx): "dylib" + else: "so" + type + HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.} + HcrGcMarkerProc = proc () {.nimcall.} + HcrModuleInitializer* = proc () {.nimcall.} + +when defined(createNimHcr): + when system.appType != "lib": + {.error: "This file has to be compiled as a library!".} + + import os, tables, sets, times, strutils, reservedmem, dynlib + + template trace(args: varargs[untyped]) = + when defined(testNimHcr) or defined(traceHcr): + echo args + + proc sanitize(arg: Time): string = + when defined(testNimHcr): return "