integrate JuliaSyntax.jl (#2526)

* integrate JuliaSyntax.jl

* Delete test_bed.jl

* improve display

* update Parse.jl

* Move code to PlutoRunner

* Add tests

* No stacktrace for syntax error in <=1.9

* Update TypeScriptCheck.yml

* Update CodemirrorPlutoSetup.d.ts

* Update React.jl

* Update is_just_text.jl

* Update TypeScriptCheck.yml

* Fixes for the codemirror update

* Update TypeScriptCheck.yml

* Disable Forward compat test 🤔

* Set `z-index` for pluto-runarea to 19 (pluto-input.cm-editor.z-index - 1)

* Change Docs test to adapt to nightly

* tighten is_just_text

frontend/components/Cell.js

@@ -135,7 +135,31 @@ export const Cell = ({
     const remount = useMemo(() => () => setKey(key + 1))
     // cm_forced_focus is null, except when a line needs to be highlighted because it is part of a stack trace
     const [cm_forced_focus, set_cm_forced_focus] = useState(/** @type{any} */ (null))
+    const [cm_highlighted_range, set_cm_highlighted_range] = useState(null)
     const [cm_highlighted_line, set_cm_highlighted_line] = useState(null)
+    const [cm_diagnostics, set_cm_diagnostics] = useState([])
+
+    useEffect(() => {
+        const diagnosticListener = (e) => {
+            if (e.detail.cell_id === cell_id) {
+                set_cm_diagnostics(e.detail.diagnostics)
+            }
+        }
+        window.addEventListener("cell_diagnostics", diagnosticListener)
+        return () => window.removeEventListener("cell_diagnostics", diagnosticListener)
+    }, [cell_id])
+
+    useEffect(() => {
+        const highlightRangeListener = (e) => {
+            if (e.detail.cell_id == cell_id && e.detail.from != null && e.detail.to != null) {
+                set_cm_highlighted_range({ from: e.detail.from, to: e.detail.to })
+            } else {
+                set_cm_highlighted_range(null)
+            }
+        }
+        window.addEventListener("cell_highlight_range", highlightRangeListener)
+        return () => window.removeEventListener("cell_highlight_range", highlightRangeListener)
+    }, [cell_id])
 
     useEffect(() => {
         const focusListener = (e) => {
@@ -248,21 +272,21 @@ export const Cell = ({
             key=${cell_key}
             ref=${node_ref}
             class=${cl({
-                queued: queued || (waiting_to_run && is_process_ready),
-                running,
-                activate_animation,
-                errored,
-                selected,
-                code_differs: class_code_differs,
-                code_folded: class_code_folded,
-                skip_as_script,
-                running_disabled,
-                depends_on_disabled_cells,
-                depends_on_skipped_cells,
-                show_input,
-                shrunk: Object.values(logs).length > 0,
-                hooked_up: output?.has_pluto_hook_features ?? false,
-            })}
+        queued: queued || (waiting_to_run && is_process_ready),
+        running,
+        activate_animation,
+        errored,
+        selected,
+        code_differs: class_code_differs,
+        code_folded: class_code_folded,
+        skip_as_script,
+        running_disabled,
+        depends_on_disabled_cells,
+        depends_on_skipped_cells,
+        show_input,
+        shrunk: Object.values(logs).length > 0,
+        hooked_up: output?.has_pluto_hook_features ?? false,
+    })}
             id=${cell_id}
         >
             ${variables.map((name) => html`<span id=${encodeURI(name)} />`)}
@@ -274,14 +298,14 @@ export const Cell = ({
             <pluto-trafficlight></pluto-trafficlight>
             <button
                 onClick=${() => {
-                    pluto_actions.add_remote_cell(cell_id, "before")
-                }}
+            pluto_actions.add_remote_cell(cell_id, "before")
+        }}
                 class="add_cell before"
                 title="Add cell"
             >
                 <span></span>
             </button>
-            ${cell_api_ready ? html`<${CellOutput} errored=${errored} ...${output} cell_id=${cell_id} />` : html``}
+        ${cell_api_ready ? html`<${CellOutput} errored=${errored} ...${output} cell_id=${cell_id} />` : html``}
             <${CellInput}
                 local_code=${cell_input_local?.code ?? code}
                 remote_code=${code}
@@ -308,7 +332,8 @@ export const Cell = ({
                 set_show_logs=${set_show_logs}
                 set_cell_disabled=${set_cell_disabled}
                 cm_highlighted_line=${cm_highlighted_line}
-                set_cm_highlighted_line=${set_cm_highlighted_line}
+                cm_highlighted_range=${cm_highlighted_range}
+                cm_diagnostics=${cm_diagnostics}
                 onerror=${remount}
             />
             ${show_logs && cell_api_ready
@@ -320,8 +345,8 @@ export const Cell = ({
                 depends_on_disabled_cells=${depends_on_disabled_cells}
                 on_run=${on_run}
                 on_interrupt=${() => {
-                    pluto_actions.interrupt_remote(cell_id)
-                }}
+            pluto_actions.interrupt_remote(cell_id)
+        }}
                 set_cell_disabled=${set_cell_disabled}
                 runtime=${runtime}
                 running=${running}
@@ -331,42 +356,42 @@ export const Cell = ({
             />
             <button
                 onClick=${() => {
-                    pluto_actions.add_remote_cell(cell_id, "after")
-                }}
+            pluto_actions.add_remote_cell(cell_id, "after")
+        }}
                 class="add_cell after"
                 title="Add cell"
             >
                 <span></span>
             </button>
             ${skip_as_script
-                ? html`<div
+            ? html`<div
                       class="skip_as_script_marker"
                       title=${`This cell is directly flagged as disabled in file. Click to know more!`}
                       onClick=${(e) => {
-                          open_pluto_popup({
-                              type: "info",
-                              source_element: e.target,
-                              body: html`This cell is currently stored in the notebook file as a Julia <em>comment</em>, instead of <em>code</em>.<br />
+                    open_pluto_popup({
+                        type: "info",
+                        source_element: e.target,
+                        body: html`This cell is currently stored in the notebook file as a Julia <em>comment</em>, instead of <em>code</em>.<br />
                                   This way, it will not run when the notebook runs as a script outside of Pluto.<br />
                                   Use the context menu to enable it again`,
-                          })
-                      }}
+                    })
+                }}
                   ></div>`
-                : depends_on_skipped_cells
+            : depends_on_skipped_cells
                 ? html`<div
                       class="depends_on_skipped_marker"
                       title=${`This cell is indirectly flagged as disabled in file. Click to know more!`}
                       onClick=${(e) => {
-                          open_pluto_popup({
-                              type: "info",
-                              source_element: e.target,
-                              body: html`This cell is currently stored in the notebook file as a Julia <em>comment</em>, instead of <em>code</em>.<br />
+                        open_pluto_popup({
+                            type: "info",
+                            source_element: e.target,
+                            body: html`This cell is currently stored in the notebook file as a Julia <em>comment</em>, instead of <em>code</em>.<br />
                                   This way, it will not run when the notebook runs as a script outside of Pluto.<br />
                                   An upstream cell is <b> indirectly</b> <em>disabling in file</em> this one; enable
                                   <span onClick=${skip_as_script_jump} style="cursor: pointer; text-decoration: underline"> the upstream one</span> to affect
                                   this cell.`,
-                          })
-                      }}
+                        })
+                    }}
                   ></div>`
                 : null}
         </pluto-cell>
@@ -388,7 +413,7 @@ export const IsolatedCell = ({ cell_input: { cell_id, metadata }, cell_result: {
     return html`
         <pluto-cell ref=${node_ref} id=${cell_id} class=${hidden ? "hidden-cell" : "isolated-cell"}>
             ${cell_api_ready ? html`<${CellOutput} ...${output} cell_id=${cell_id} />` : html``}
-            ${show_logs ? html`<${Logs} logs=${Object.values(logs)} line_heights=${[15]} set_cm_highlighted_line=${() => {}} />` : null}
+            ${show_logs ? html`<${Logs} logs=${Object.values(logs)} line_heights=${[15]} set_cm_highlighted_line=${() => { }} />` : null}
         </pluto-cell>
     `
 }

frontend/components/CellInput.js

@@ -48,6 +48,7 @@ import {
     pythonLanguage,
     syntaxHighlighting,
     cssLanguage,
+    setDiagnostics,
 } from "../imports/CodemirrorPlutoSetup.js"
 
 import { markdown, html as htmlLang, javascript, sqlLang, python, julia_mixed } from "./CellInput/mixedParsers.js"
@@ -59,7 +60,7 @@ import { cell_movement_plugin, prevent_holding_a_key_from_doing_things_across_ce
 import { pluto_paste_plugin } from "./CellInput/pluto_paste_plugin.js"
 import { bracketMatching } from "./CellInput/block_matcher_plugin.js"
 import { cl } from "../common/ClassTable.js"
-import { HighlightLineFacet, highlightLinePlugin } from "./CellInput/highlight_line.js"
+import { HighlightLineFacet, HighlightRangeFacet, highlightLinePlugin, highlightRangePlugin } from "./CellInput/highlight_line.js"
 import { commentKeymap } from "./CellInput/comment_mixed_parsers.js"
 import { ScopeStateField } from "./CellInput/scopestate_statefield.js"
 import { mod_d_command } from "./CellInput/mod_d_command.js"
@@ -373,8 +374,10 @@ export const CellInput = ({
     set_show_logs,
     set_cell_disabled,
     cm_highlighted_line,
+    cm_highlighted_range,
     metadata,
     global_definition_locations,
+    cm_diagnostics,
 }) => {
     let pluto_actions = useContext(PlutoActionsContext)
     const { disabled: running_disabled, skip_as_script } = metadata
@@ -384,6 +387,7 @@ export const CellInput = ({
         set_error(null)
         throw to_throw
     }
+
     const notebook_id_ref = useRef(notebook_id)
     notebook_id_ref.current = notebook_id
 
@@ -394,6 +398,7 @@ export const CellInput = ({
     let nbpkg_compartment = useCompartment(newcm_ref, NotebookpackagesFacet.of(nbpkg))
     let global_definitions_compartment = useCompartment(newcm_ref, GlobalDefinitionsFacet.of(global_definition_locations))
     let highlighted_line_compartment = useCompartment(newcm_ref, HighlightLineFacet.of(cm_highlighted_line))
+    let highlighted_range_compartment = useCompartment(newcm_ref, HighlightRangeFacet.of(cm_highlighted_range))
     let editable_compartment = useCompartment(newcm_ref, EditorState.readOnly.of(disable_input))
 
     let on_change_compartment = useCompartment(
@@ -589,9 +594,11 @@ export const CellInput = ({
                     // Compartments coming from react state/props
                     nbpkg_compartment,
                     highlighted_line_compartment,
+                    highlighted_range_compartment,
                     global_definitions_compartment,
                     editable_compartment,
                     highlightLinePlugin(),
+                    highlightRangePlugin(),
 
                     // This is waaaay in front of the keys it is supposed to override,
                     // Which is necessary because it needs to run before *any* keymap,
@@ -713,6 +720,12 @@ export const CellInput = ({
                     // Wowww this has been enabled for some time now... wonder if there are issues about this yet ;) - DRAL
                     awesome_line_wrapping,
 
+                    // Reset diagnostics on change
+                    EditorView.updateListener.of((update) => {
+                        if (!update.docChanged) return
+                        update.view.dispatch(setDiagnostics(update.state, []))
+                    }),
+
                     on_change_compartment,
 
                     // This is my weird-ass extension that checks the AST and shows you where
@@ -778,6 +791,14 @@ export const CellInput = ({
         }
     }, [])
 
+    useEffect(() => {
+        if (newcm_ref.current == null) return
+        const cm = newcm_ref.current
+        const diagnostics = cm_diagnostics
+
+        cm.dispatch(setDiagnostics(cm.state, diagnostics))
+    }, [cm_diagnostics])
+
     // Effect to apply "remote_code" to the cell when it changes...
     // ideally this won't be necessary as we'll have actual multiplayer,
     // or something to tell the user that the cell is out of sync.

frontend/components/CellInput/highlight_line.js

@@ -4,6 +4,10 @@ const highlighted_line = Decoration.line({
     attributes: { class: "cm-highlighted-line" },
 })
 
+const highlighted_range = Decoration.mark({
+    attributes: { class: "cm-highlighted-range" },
+})
+
 /**
  * @param {EditorView} view
  */
@@ -17,6 +21,22 @@ function create_line_decorations(view) {
     return Decoration.set([highlighted_line.range(line.from, line.from)])
 }
 
+/**
+ * @param {EditorView} view
+ */
+function create_range_decorations(view) {
+    let range = view.state.facet(HighlightRangeFacet)
+    if (range == null) {
+        return Decoration.set([])
+    }
+    let { from, to } = range
+    if (from < 0 || from == to) {
+        return Decoration.set([])
+    }
+
+    return Decoration.set([highlighted_range.range(from, to)])
+}
+
 /**
  * @type Facet<number?, number?>
  */
@@ -25,6 +45,14 @@ export const HighlightLineFacet = Facet.define({
     compare: (a, b) => a === b,
 })
 
+/**
+ * @type Facet<{from: number, to: number}?, {from: number, to: number}?>
+ */
+export const HighlightRangeFacet = Facet.define({
+    combine: (values) => values[0],
+    compare: (a, b) => a === b,
+})
+
 export const highlightLinePlugin = () =>
     ViewPlugin.fromClass(
         class {
@@ -53,3 +81,33 @@ export const highlightLinePlugin = () =>
             decorations: (v) => v.decorations,
         }
     )
+
+
+export const highlightRangePlugin = () =>
+    ViewPlugin.fromClass(
+        class {
+            updateDecos(view) {
+                this.decorations = create_range_decorations(view)
+            }
+
+            /**
+             * @param {EditorView} view
+             */
+            constructor(view) {
+                this.decorations = Decoration.set([])
+                this.updateDecos(view)
+            }
+
+            /**
+             * @param {ViewUpdate} update
+             */
+            update(update) {
+                if (update.docChanged || update.state.facet(HighlightRangeFacet) !== update.startState.facet(HighlightRangeFacet)) {
+                    this.updateDecos(update.view)
+                }
+            }
+        },
+        {
+            decorations: (v) => v.decorations,
+        }
+    )

frontend/components/CellInput/pluto_autocomplete.js

@@ -131,7 +131,11 @@ let update_docs_from_autocomplete_selection = (on_update_doc_query) => {
         // The nice thing about this is that we can use the resulting state from the transaction,
         // without updating the actual state of the editor.
         let result_transaction = update.state.update({
-            changes: { from: selected_option.source.from, to: selected_option.source.to, insert: text_to_apply },
+            changes: {
+                from: selected_option.source.from,
+                to: Math.min(selected_option.source.to, update.state.doc.length),
+                insert: text_to_apply,
+            },
         })
 
         // So we can use `get_selected_doc_from_state` on our virtual state

frontend/components/CellOutput.js

@@ -1,6 +1,6 @@
 import { html, Component, useRef, useLayoutEffect, useContext } from "../imports/Preact.js"
 
-import { ErrorMessage } from "./ErrorMessage.js"
+import { ErrorMessage, ParseError } from "./ErrorMessage.js"
 import { TreeView, TableView, DivElement } from "./TreeView.js"
 
 import {
@@ -148,6 +148,9 @@ export const OutputBody = ({ mime, body, cell_id, persist_js_state = false, last
         case "application/vnd.pluto.table+object":
             return html`<${TableView} cell_id=${cell_id} body=${body} persist_js_state=${persist_js_state} />`
             break
+        case "application/vnd.pluto.parseerror+object":
+            return html`<div><${ParseError} cell_id=${cell_id} ...${body} /></div>`
+            break
         case "application/vnd.pluto.stacktrace+object":
             return html`<div><${ErrorMessage} cell_id=${cell_id} ...${body} /></div>`
             break