change variables to be function-scoped

docs/language.md

@@ -262,41 +262,50 @@ you can do with variables, like:
 
 #### Scopes
 
-In general, side effects (i.e. functions with bang modifier) affect all following sibling and child
-expressions, but not the parent. This is like doing
+Rudi is roughly function-scoped, like JavaScript: Once a variable is defined using `set!`, it is
+available for the rest of the current program (or user-defined function).
+
+##### Variables
+
+A statement like `(set! $foo 42) (if (condition) (set! $foo 7))` would be equal to
 
 ```go
 foo := 42
 
 if condition {
-  foo := 7
+  foo = 7
 }
 
 println(foo) // prints 42
 ```
 
-in Go. Variables are meant to be helpers and are so scoped to the scope where they are defined:
+in Go. However function-scoping means that a variable is actually defined one it was set and is then
+valid for all subsequent expressions, so
 
 ```lisp
 (set! $var 42)
-$var             # 42
+(if (gt? $var 4) (set! $tooLarge true))
+$tooLarge
+```
 
-# This set! function would set the value for the entire positive branch of the "if" tuple,
-# but it will not leak outside of the "if".
+will yield `true`, since the `$tooLarge` variable is available even after `if` has finished.
 
-(if true
-  (set! $var "new-value"))    # "new-value"
-$var                          # 42
+##### User-Defined Functions
 
-# ... but the new variable is valid in its scope.
+Functions defined using `func!` form a sub-program. This means any scoped variable defined on the
+outside is not visible inside the function, the following is therefore invalid:
 
-(if true
-  (do
-    (set! $var "new-value")
-    (append $var "-suffix"))) # "new-value-suffix"
-$var                          # 42
+```lisp
+(set! $foo)
+(func! do-stuff [] (+ $foo 1))
+(do-stuff)
 ```
 
+User-defined functions run in their own scope, where only the arguments and global variables are
+available (though new, function-scoped variables can of course be defined).
+
+##### Global Document
+
 The exception from this rule is the global document. As the name implies, it is meant to be global
 and to allow for effective, readable Rudi code, there is only one document and it can be modified
 from anywhere.

pkg/builtin/core/functions.go

@@ -67,13 +67,12 @@ func ifElseFunction(ctx types.Context, test bool, yes, no ast.Expression) (any,
 // exported.
 func DoFunction(ctx types.Context, args ...ast.Expression) (any, error) {
 	var (
-		tupleCtx = ctx
-		result   any
-		err      error
+		result any
+		err    error
 	)
 
 	for _, arg := range args {
-		tupleCtx, result, err = ctx.Runtime().EvalExpression(tupleCtx, arg)
+		ctx, result, err = ctx.Runtime().EvalExpression(ctx, arg)
 		if err != nil {
 			return nil, err
 		}
@@ -285,7 +284,7 @@ func overwriteEverythingBangHandler(ctx types.Context, originalArgs []ast.Expres
 	// update the target, ignoring the path expression on the symbol.
 	if symbol.Variable != nil {
 		varName := string(*symbol.Variable)
-		ctx = ctx.WithVariable(varName, value)
+		ctx.SetVariable(varName, value)
 	} else {
 		ctx.GetDocument().Set(value)
 	}

pkg/builtin/core/functions_test.go

@@ -150,7 +150,7 @@ func TestSetFunction(t *testing.T) {
 			Expression: `(set! $var 1) $var`,
 			Expected:   int64(1),
 		},
-		// can overwrite variables on the top level
+		// can overwrite global variables
 		{
 			Expression: `(set! $myvar 12) $myvar`,
 			Variables:  testVariables(),
@@ -209,14 +209,14 @@ func TestSetFunction(t *testing.T) {
 			Variables:  testVariables(),
 			Expected:   []any{"first", 2, "third"},
 		},
-		// ...but not leak into upper scopes
+		// variables are program/rudifunc-scoped, so changes in sub expressions should modify the variable
 		{
 			Expression: `(set! $a 1) (if true (set! $a 2)) $a`,
-			Expected:   int64(1),
+			Expected:   int64(2),
 		},
 		{
 			Expression: `(set! $a 1) (if true (set! $b 2)) $b`,
-			Invalid:    true,
+			Expected:   int64(2),
 		},
 		// do not accidentally set a key without creating a new context
 		{
@@ -225,7 +225,7 @@ func TestSetFunction(t *testing.T) {
 		},
 		{
 			Expression: `(set! $a {foo "bar"}) (if true (set! $a.foo "updated")) $a.foo`,
-			Expected:   "bar",
+			Expected:   "updated",
 		},
 		// handle bad paths
 		{
@@ -258,8 +258,8 @@ func TestSetFunction(t *testing.T) {
 		},
 		{
 			Expression: `(set! $obj.aString "new value") $obj.aString`,
-			Expected:   "new value",
 			Variables:  testVariables(),
+			Expected:   "new value",
 		},
 		// add a new sub key
 		{
@@ -589,25 +589,27 @@ func TestDoFunction(t *testing.T) {
 			Expression: `(do 3)`,
 			Expected:   int64(3),
 		},
-
-		// test that the runtime context is inherited from one step to another
+		// do should not open a sub-scope, allowing all expressions to freely live in the same
+		// context
 		{
 			Expression: `(do (set! $var "foo") $var)`,
 			Expected:   "foo",
 		},
 		{
-			Expression: `(do (set! $var "foo") $var (set! $var "new") $var)`,
-			Expected:   "new",
+			Expression: `(do (set! $var "foo")) $var`,
+			Expected:   "foo",
 		},
-
-		// test that the runtime context doesn't leak
 		{
-			Expression: `(set! $var "outer") (do (set! $var "inner")) $var`,
-			Expected:   "outer",
+			Expression: `(set! $var "foo") (do (set! $var "bar")) $var`,
+			Expected:   "bar",
 		},
 		{
-			Expression: `(do (set! $var "inner")) $var`,
-			Invalid:    true,
+			Expression: `(do (set! $var "foo") (set! $var "new") $var)`,
+			Expected:   "new",
+		},
+		{
+			Expression: `(do (set! $var "foo") (set! $var "new") "dummy") $var`,
+			Expected:   "new",
 		},
 	}
 

pkg/builtin/lists/functions.go

@@ -61,10 +61,7 @@ func rangeVectorFunction(ctx types.Context, data []any, namingVec ast.Expression
 		return nil, fmt.Errorf("argument #1: not a valid naming vector: %w", err)
 	}
 
-	var (
-		result  any
-		loopCtx = ctx
-	)
+	var result any
 
 	for i, item := range data {
 		vars := map[string]any{
@@ -75,10 +72,12 @@ func rangeVectorFunction(ctx types.Context, data []any, namingVec ast.Expression
 			vars[loopIndexName] = i
 		}
 
-		// do not use separate contexts for each loop iteration, as the loop might build up a counter
-		loopCtx = loopCtx.WithVariables(vars)
+		// Do not use separate contexts for each loop iteration, as the loop might build up a counter,
+		// but only use the loop variables in a shallow scope, where only these two temporary variables
+		// are laid over the regular scoped/global variables.
+		scope := ctx.NewShallowScope(vars)
 
-		loopCtx, result, err = ctx.Runtime().EvalExpression(loopCtx, expr)
+		_, result, err = ctx.Runtime().EvalExpression(scope, expr)
 		if err != nil {
 			return nil, err
 		}
@@ -97,8 +96,7 @@ func rangeObjectFunction(ctx types.Context, data map[string]any, namingVec ast.E
 	}
 
 	var (
-		result  any
-		loopCtx = ctx
+		result any
 	)
 
 	for key, value := range data {
@@ -110,10 +108,7 @@ func rangeObjectFunction(ctx types.Context, data map[string]any, namingVec ast.E
 			vars[loopIndexName] = key
 		}
 
-		// do not use separate contexts for each loop iteration, as the loop might build up a counter
-		loopCtx = loopCtx.WithVariables(vars)
-
-		loopCtx, result, err = ctx.Runtime().EvalExpression(loopCtx, expr)
+		_, result, err = ctx.Runtime().EvalExpression(ctx.NewShallowScope(vars), expr)
 		if err != nil {
 			return nil, err
 		}
@@ -157,9 +152,7 @@ func mapVectorExpressionFunction(ctx types.Context, data []any, namingVec ast.Ex
 			vars[indexVarName] = index
 		}
 
-		ctx = ctx.WithVariables(vars)
-
-		return ctx.Runtime().EvalExpression(ctx, expr)
+		return ctx.Runtime().EvalExpression(ctx.NewShallowScope(vars), expr)
 	}
 
 	return mapVector(ctx, data, mapHandler)
@@ -219,9 +212,7 @@ func mapObjectExpressionFunction(ctx types.Context, data map[string]any, namingV
 			vars[keyVarName] = key
 		}
 
-		ctx = ctx.WithVariables(vars)
-
-		return ctx.Runtime().EvalExpression(ctx, expr)
+		return ctx.Runtime().EvalExpression(ctx.NewShallowScope(vars), expr)
 	}
 
 	return mapObject(ctx, data, mapHandler)
@@ -281,9 +272,7 @@ func filterVectorExpressionFunction(ctx types.Context, data []any, namingVec ast
 			vars[indexVarName] = index
 		}
 
-		ctx = ctx.WithVariables(vars)
-
-		return ctx.Runtime().EvalExpression(ctx, expr)
+		return ctx.Runtime().EvalExpression(ctx.NewShallowScope(vars), expr)
 	}
 
 	return filterVector(ctx, data, mapHandler)
@@ -349,9 +338,7 @@ func filterObjectExpressionFunction(ctx types.Context, data map[string]any, nami
 			vars[keyVarName] = key
 		}
 
-		ctx = ctx.WithVariables(vars)
-
-		return ctx.Runtime().EvalExpression(ctx, expr)
+		return ctx.Runtime().EvalExpression(ctx.NewShallowScope(vars), expr)
 	}
 
 	return filterObject(ctx, data, mapHandler)

pkg/builtin/rudifunc/functions.go

@@ -54,7 +54,9 @@ func funcBangHandler(ctx types.Context, originalArgs []ast.Expression, value any
 		panic("This should never happen: func! bang handler called with non-intermediate function.")
 	}
 
-	return ctx.WithRudispaceFunction(intermediate.name, intermediate), nil, nil
+	ctx.SetRudispaceFunction(intermediate.name, intermediate)
+
+	return ctx, nil, nil
 }
 
 type rudispaceFunc struct {
@@ -85,7 +87,9 @@ func (f rudispaceFunc) Evaluate(ctx types.Context, args []ast.Expression) (any,
 	}
 
 	// user-defined functions form a sub-program and all statements share the same context
-	funcCtx := ctx.WithVariables(funcArgs)
+	funcCtx := ctx.NewScope()
+	funcCtx.SetVariables(funcArgs)
+
 	runtime := ctx.Runtime()
 
 	var (