Skip to content

Commit

Permalink
Merge pull request #71 from PumasAI/mh/cell-multiple
Browse files Browse the repository at this point in the history 
Dynamic cell expansion
  • Loading branch information
@MichaelHatherly
MichaelHatherly authored Mar 20, 2024
2 parents 4d26705 + de96363 commit 9f8fbf0
Show file tree
Hide file tree
Showing 4 changed files with 310 additions and 105 deletions.
209 changes: 123 additions & 86 deletions src/server.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,8 +546,24 @@ function evaluate_raw_cells!(f::File, chunks::Vector, options::Dict; showprogres
header = "Running $(relpath(f.path, pwd()))"
@maybe_progress showprogress "$header" for (nth, chunk) in enumerate(chunks)
if chunk.type === :code

outputs = []
is_multiple = get(chunk.cell_options, "multiple", false) === true
# When we're not evaluating the code, or when there is a `multiple`
# cell output then we immediately splice in the cell code. The
# results of evaluating a `multiple` cell are added later on and are
# not considered direct outputs of this cell.
if !chunk.evaluate || is_multiple
push!(
cells,
(;
id = string(nth),
cell_type = chunk.type,
metadata = (;),
source = process_cell_source(chunk.source),
outputs = [],
execution_count = chunk.evaluate ? 1 : 0,
),
)
end

if chunk.evaluate
# Offset the line number by 1 to account for the triple backticks
Expand All @@ -558,105 +574,114 @@ function evaluate_raw_cells!(f::File, chunks::Vector, options::Dict; showprogres
$(chunk.line + 1),
$(chunk.cell_options),
))
remote = Malt.remote_eval_fetch(f.worker, expr)
processed = process_results(remote.results)

# Should this notebook cell be allowed to throw an error? When
# not allowed, we log all errors don't generate an output.
allow_error_cell = get(chunk.cell_options, "error", allow_error_global)

if isnothing(remote.error)
for display_result in remote.display_results
processed_display = process_results(display_result)
if !isempty(processed_display.data)
for (mth, remote) in enumerate(Malt.remote_eval_fetch(f.worker, expr))
outputs = []
processed = process_results(remote.results)

# Should this notebook cell be allowed to throw an error? When
# not allowed, we log all errors don't generate an output.
allow_error_cell = get(remote.cell_options, "error", allow_error_global)

if isnothing(remote.error)
for display_result in remote.display_results
processed_display = process_results(display_result)
if !isempty(processed_display.data)
push!(
outputs,
(;
output_type = "display_data",
processed_display.data,
processed_display.metadata,
),
)
end
if !isempty(processed_display.errors)
append!(outputs, processed_display.errors)
if !allow_error_cell
for each_error in processed_display.errors
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(each_error.traceback, "\n"))
@error "stopping notebook evaluation due to unexpected `show` error." file traceback
has_error = true
end
end
end
end
if !isempty(processed.data)
push!(
outputs,
(;
output_type = "display_data",
processed_display.data,
processed_display.metadata,
output_type = "execute_result",
execution_count = 1,
processed.data,
processed.metadata,
),
)
end
if !isempty(processed_display.errors)
append!(outputs, processed_display.errors)
if !allow_error_cell
for each_error in processed_display.errors
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(each_error.traceback, "\n"))
@error "stopping notebook evaluation due to unexpected `show` error." file traceback
has_error = true
end
end
else
# These are errors arising from evaluation of the contents
# of a code cell, not from the `show` output of the values,
# which is handled separately below.
push!(
outputs,
(;
output_type = "error",
ename = remote.error,
evalue = get(processed.data, "text/plain", ""),
traceback = remote.backtrace,
),
)
if !allow_error_cell
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(remote.backtrace, "\n"))
@error "stopping notebook evaluation due to unexpected cell error." file traceback
has_error = true
end
end
if !isempty(processed.data)
push!(

if !isempty(remote.output)
pushfirst!(
outputs,
(;
output_type = "execute_result",
execution_count = 1,
processed.data,
processed.metadata,
output_type = "stream",
name = "stdout",
text = remote.output,
),
)
end
else
# These are errors arising from evaluation of the contents
# of a code cell, not from the `show` output of the values,
# which is handled separately below.

# These are errors arising from the `show` output of the values
# generated by cells, not from the cell evaluation itself. So if
# something throws an error here then the user's `show` method
# has a bug.
if !isempty(processed.errors)
append!(outputs, processed.errors)
if !allow_error_cell
for each_error in processed.errors
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(each_error.traceback, "\n"))
@error "stopping notebook evaluation due to unexpected `show` error." file traceback
has_error = true
end
end
end
push!(
outputs,
cells,
(;
output_type = "error",
ename = remote.error,
evalue = get(processed.data, "text/plain", ""),
traceback = remote.backtrace,
id = is_multiple ? string(nth, "_", mth) : string(nth),
cell_type = chunk.type,
metadata = (;),
source = process_cell_source(
remote.code,
is_multiple ? remote.cell_options : Dict(),
),
outputs,
execution_count = 1,
),
)
if !allow_error_cell
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(remote.backtrace, "\n"))
@error "stopping notebook evaluation due to unexpected cell error." file traceback
has_error = true
end
end

if !isempty(remote.output)
pushfirst!(
outputs,
(; output_type = "stream", name = "stdout", text = remote.output),
)
end

# These are errors arising from the `show` output of the values
# generated by cells, not from the cell evaluation itself. So if
# something throws an error here then the user's `show` method
# has a bug.
if !isempty(processed.errors)
append!(outputs, processed.errors)
if !allow_error_cell
for each_error in processed.errors
file = "$(chunk.file):$(chunk.line)"
traceback = Text(join(each_error.traceback, "\n"))
@error "stopping notebook evaluation due to unexpected `show` error." file traceback
has_error = true
end
end
end
end

push!(
cells,
(;
id = string(nth),
cell_type = chunk.type,
metadata = (;),
source = process_cell_source(chunk.source),
outputs,
execution_count = chunk.evaluate ? 1 : 0,
),
)
elseif chunk.type === :markdown
marker = "{julia} "
source = chunk.source
Expand All @@ -667,7 +692,10 @@ function evaluate_raw_cells!(f::File, chunks::Vector, options::Dict; showprogres
if startswith(node.literal, marker)
source_code = replace(node.literal, marker => "")
expr = :(render($(source_code), $(chunk.file), $(chunk.line)))
remote = Malt.remote_eval_fetch(f.worker, expr)
# There should only ever be a single result from an
# inline evaluation since you can't pass cell
# options and so `multiple` will always be `false`.
remote = only(Malt.remote_eval_fetch(f.worker, expr))
if !isnothing(remote.error)
error("Error rendering inline code: $(remote.error)")
end
Expand Down Expand Up @@ -702,13 +730,22 @@ function evaluate_raw_cells!(f::File, chunks::Vector, options::Dict; showprogres
end

# All but the last line of a cell should contain a newline character to end it.
function process_cell_source(source::AbstractString)
# The optional `cell_options` argument is a dictionary of cell attributes which
# are written into the processed cell source when the cell is the result of an
# expansion of a `multiple` cell.
function process_cell_source(source::AbstractString, cell_options::Dict = Dict())
lines = collect(eachline(IOBuffer(source); keep = true))
if isempty(lines)
return []
else
if !isempty(lines)
lines[end] = rstrip(lines[end])
end
if isempty(cell_options)
return lines
else
yaml = YAML.write(cell_options)
return vcat(
String["#| $line" for line in eachline(IOBuffer(yaml); keep = true)],
lines,
)
end
end

Expand Down
78 changes: 59 additions & 19 deletions src/worker.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,30 +141,70 @@ function worker_init(f::File)
line::Integer,
cell_options::AbstractDict = Dict{String,Any}(),
)
captured, display_results = with_inline_display(cell_options) do
return _render_thunk(code, cell_options) do
Base.@invokelatest include_str(WORKSPACE[], code; file, line)
end
results = Base.@invokelatest render_mimetypes(
REPL.ends_with_semicolon(code) ? nothing : captured.value,
cell_options,
)
return (;
results,
display_results,
output = captured.output,
error = captured.error ? string(typeof(captured.value)) : nothing,
backtrace = collect(
eachline(
IOBuffer(
clean_bt_str(
captured.error,
captured.backtrace,
captured.value,
end

# Recursively render cell thunks. This might be an `include_str` call,
# which is the starting point for a source cell, or it may be a
# user-provided thunk that comes from a source cell with `multiple` set
# to `true`.
function _render_thunk(
thunk::Base.Callable,
code::AbstractString,
cell_options::AbstractDict = Dict{String,Any}(),
)
captured, display_results = with_inline_display(thunk, cell_options)
if get(cell_options, "multiple", false) === true
return collect(
# A cell expansion with `multiple` might itself also contain
# cells that expand to multiple cells, so we need to flatten
# the results to a single list of cells before passing back
# to the server. Cell expansion is recursive.
Base.Iterators.flatten(
map(captured.value) do cell
wrapped = function ()
return IOCapture.capture(
cell.thunk;
rethrow = InterruptException,
color = true,
)
end
# **The recursive call:**
return Base.@invokelatest _render_thunk(
wrapped,
get(cell, :code, ""),
get(Dict{String,Any}, cell, :options),
)
end,
),
)
else
results = Base.@invokelatest render_mimetypes(
REPL.ends_with_semicolon(code) ? nothing : captured.value,
cell_options,
)
return [(;
code,
cell_options,
results,
display_results,
output = captured.output,
error = captured.error ? string(typeof(captured.value)) : nothing,
backtrace = collect(
eachline(
IOBuffer(
clean_bt_str(
captured.error,
captured.backtrace,
captured.value,
),
),
),
),
),
)
)]
end
end

# Utilities:
Expand Down
49 changes: 49 additions & 0 deletions test/examples/cell_expansion.qmd
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
---
title: Cell Expansion
---

```{julia}
#| echo: false
#| multiple: true
[(;
thunk = function ()
println("print call")
display("display call")
"return value"
end,
code = """
# Fake code goes here.
""",
options = Dict("layout-ncol" => 2),
)]
```

```{julia}
#| multiple: true
[
(;
thunk = function ()
return [
(;
thunk = function ()
return [(; thunk = () -> 1, options = Dict("layout-ncol" => 1))]
end,
options = Dict("multiple" => true),
),
(;
thunk = function ()
return [(; thunk = () -> (display(2); 2)), (; thunk = () -> 3)]
end,
options = Dict("multiple" => true),
),
]
end,
options = Dict("multiple" => true),
),
(; thunk = () -> 4),
(;
thunk = () -> println("## Header"),
options = Dict("output" => "asis", "echo" => false),
),
]
```
Loading

0 comments on commit 9f8fbf0

Please sign in to comment.