more permissive tracking of unbound references at phase > 0

The old expander was too permissive, losing track of unbound variables
that are part of an expansion. The new expander was too strict,
maintaining a complaint even if the bad reference doesn't appear in
the expansion. Look for a happy medium.

Author: mflatt

pkgs/racket-test-core/tests/racket/module.rktl

@@ -2162,6 +2162,41 @@ case of module-leve bindings; it doesn't cover local bindings.
 
   (define another 'x))
 
+;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Allow a reference to a never-defined variable in a `local-expand`
+;; or `syntax-local-bind-syntaxes` on the grounds that the result is
+;; not necessarily in the module's expansion. But keep track of
+;; missing variables encountered during
+;; `syntax-local-expand-expression`, since the opqaue result can be
+;; included without further inspection.
+
+(module im-ok-and-your-ok-local-expand racket/base
+  (require (for-syntax racket/base)
+           (for-meta 2 racket/base))
+  (begin-for-syntax
+    (define-syntax (m stx)
+      (local-expand #'(lambda () nonesuch) 'expression '())
+      #''ok)
+    (m)))
+
+(module im-ok-and-your-ok-syntax-local-bind-syntaxes racket/base
+  (require (for-syntax racket/base))
+  (define-syntax (m stx)
+    (syntax-local-bind-syntaxes (list #'x)
+                                #'(lambda () nonesuch)
+                                (syntax-local-make-definition-context))
+     #''ok)
+  (m))
+
+(syntax-test #'(module im-ok-and-your-ok-local-expand racket/base
+                 (require (for-syntax racket/base)
+                          (for-meta 2 racket/base))
+                 (begin-for-syntax
+                   (define-syntax (m stx)
+                     (syntax-local-expand-expression #'(lambda () nonesuch))
+                     #''ok)
+                   (m))))
+
 ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (report-errs)

racket/src/expander/expand/definition-context.rkt

@@ -195,7 +195,8 @@
                                    #:phase [phase (expand-context-phase ctx)]
                                    #:intdefs intdefs
                                    #:stop-ids [stop-ids #f]
-                                   #:to-parsed-ok? [to-parsed-ok? #f])
+                                   #:to-parsed-ok? [to-parsed-ok? #f]
+                                   #:track-to-be-defined? [track-to-be-defined? #f])
   (define same-kind? (or (eq? context
                               (expand-context-context ctx))
                          (and (list? context)
@@ -248,7 +249,18 @@
                 [just-once? #f]
                 [in-local-expand? #t]
                 [stops (free-id-set phase (or all-stop-ids null))]
-                [current-introduction-scopes null]))
+                [current-introduction-scopes null]
+                [need-eventually-defined (let ([ht (expand-context-need-eventually-defined ctx)])
+                                           (cond
+                                             [track-to-be-defined?
+                                              ;; maintain status quo and propagate tracking
+                                              ht]
+                                             [ht
+                                              ;; keep allowing unbound references, but don't track them
+                                              (make-hasheqv)]
+                                             [else
+                                              ;; keep disallowing unbound references
+                                              #f]))]))
 
 ;; ----------------------------------------
 

racket/src/expander/expand/local-expand.rkt

@@ -42,7 +42,8 @@
 (define (syntax-local-expand-expression s [opaque-only? #f])
   (define exp-s (do-local-expand 'syntax-local-expand-expression s 'expression null #f
                                  #:to-parsed-ok? opaque-only?
-                                 #:skip-log-exit? #t))
+                                 #:skip-log-exit? #t
+                                 #:track-to-be-defined? #t))
   (define ctx (get-current-expand-context))
   ;; Move introduction scope from the already-expanded syntax object to
   ;; its wrapper. The expander will later check that the wrapper ends up
@@ -68,6 +69,7 @@
                          #:lift-key [lift-key (and (or capture-lifts?
                                                        as-transformer?)
                                                    (generate-lift-key))]
+                         #:track-to-be-defined? [track-to-be-defined? #f]
                          #:skip-log-exit? [skip-log-exit? #f])
   (performance-region
    ['expand 'local-expand]
@@ -102,7 +104,8 @@
                                                 #:phase phase
                                                 #:intdefs intdefs
                                                 #:stop-ids stop-ids
-                                                #:to-parsed-ok? to-parsed-ok?))
+                                                #:to-parsed-ok? to-parsed-ok?
+                                                #:track-to-be-defined? track-to-be-defined?))
 
    (log-expand local-ctx 'enter-local s)
    (define input-s (add-intdef-scopes (flip-introduction-scopes s ctx) intdefs))

racket/src/expander/expand/module.rkt

@@ -29,6 +29,7 @@
          "def-id.rkt"
          "prepare.rkt"
          "log.rkt"
+         "syntax-id-error.rkt"
          "../compile/main.rkt"
          "../eval/top.rkt"
          "../eval/module.rkt"
@@ -366,7 +367,7 @@
                                            #:mpis-to-reset mpis-to-reset)))
 
      ;; Check that any tentatively allowed reference at phase >= 1 is ok
-     (check-defined-by-now need-eventually-defined self)
+     (check-defined-by-now need-eventually-defined self ctx)
 
      ;; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      ;; Pass 3: resolve provides at all phases
@@ -1009,7 +1010,7 @@
        (cons exp-body
              (loop tail? rest-bodys)))])))
 
-(define (check-defined-by-now need-eventually-defined self)
+(define (check-defined-by-now need-eventually-defined self ctx)
   ;; If `need-eventually-defined` is not empty, report an error
   (for ([(phase l) (in-hash need-eventually-defined)])
     (for ([id (in-list l)])
@@ -1019,7 +1020,9 @@
                    (module-binding? b)
                    (eq? (module-binding-sym b) (syntax-e id))
                    (eq? (module-binding-module b) self))
-        (raise-syntax-error #f "reference to an unbound identifier" id)))))
+        (raise-syntax-error #f "reference to an unbound identifier"
+                            id #f null
+                            (syntax-debug-info-string id ctx))))))
 
 ;; ----------------------------------------