Skip to content

Test: show context when a let testset errors #58727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+83 −10
Open

Test: show context when a let testset errors #58727

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,9 +214,10 @@ struct Error <: Result
orig_expr::String
value::String
backtrace::String
context::Union{Nothing, String}
source::LineNumberNode

function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode)
function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode, context::Union{Nothing, String}=nothing)
if test_type === :test_error
bt = scrub_exc_stack(bt, nothing, extract_file(source))
end
Expand Down Expand Up @@ -248,8 +249,14 @@ struct Error <: Result
string(orig_expr),
value,
bt_str,
context,
source)
end

# Internal constructor for creating Error with pre-processed values (used by ContextTestSet)
function Error(test_type::Symbol, orig_expr::String, value::String, backtrace::String, context::Union{Nothing, String}, source::LineNumberNode)
return new(test_type, orig_expr, value, backtrace, context, source)
end
end

function Base.show(io::IO, t::Error)
Expand All @@ -267,6 +274,9 @@ function Base.show(io::IO, t::Error)
elseif t.test_type === :test_error
println(io, " Test threw exception")
println(io, " Expression: ", t.orig_expr)
if t.context !== nothing
println(io, " Context: ", t.context)
end
# Capture error message and indent to match
join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n")
elseif t.test_type === :test_unbroken
Expand Down Expand Up @@ -751,13 +761,13 @@ function do_test(result::ExecutionResult, orig_expr)
Fail(:test, orig_expr, result.data, value, nothing, result.source, false)
else
# If the result is non-Boolean, this counts as an Error
Error(:test_nonbool, orig_expr, value, nothing, result.source)
Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing)
end
else
# The predicate couldn't be evaluated without throwing an
# exception, so that is an Error and not a Fail
@assert isa(result, Threw)
testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source)
testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source, nothing)
end
isa(testres, Pass) || trigger_test_failure_break(result)
record(get_testset(), testres)
Expand All @@ -770,11 +780,11 @@ function do_broken_test(result::ExecutionResult, orig_expr)
value = result.value
if isa(value, Bool)
if value
testres = Error(:test_unbroken, orig_expr, value, nothing, result.source)
testres = Error(:test_unbroken, orig_expr, value, nothing, result.source, nothing)
end
else
# If the result is non-Boolean, this counts as an Error
testres = Error(:test_nonbool, orig_expr, value, nothing, result.source)
testres = Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing)
end
end
record(get_testset(), testres)
Expand Down Expand Up @@ -1108,6 +1118,13 @@ function record(c::ContextTestSet, t::Fail)
context = t.context === nothing ? context : string(t.context, "\n ", context)
record(c.parent_ts, Fail(t.test_type, t.orig_expr, t.data, t.value, context, t.source, t.message_only))
end
function record(c::ContextTestSet, t::Error)
context = string(c.context_name, " = ", c.context)
context = t.context === nothing ? context : string(t.context, "\n ", context)
# Create a new Error with the same data but updated context using internal constructor
new_error = Error(t.test_type, t.orig_expr, t.value, t.backtrace, context, t.source)
record(c.parent_ts, new_error)
end

#-----------------------------------------------------------------------

Expand Down Expand Up @@ -1844,7 +1861,7 @@ function testset_beginend_call(args, tests, source)
if is_failfast_error(err)
get_testset_depth() > 1 ? rethrow() : failfast_print()
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
finally
copy!(default_rng(), default_rng_orig)
Expand Down Expand Up @@ -1932,7 +1949,7 @@ function testset_forloop(args, testloop, source)
if is_failfast_error(err)
get_testset_depth() > 1 ? rethrow() : failfast_print()
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
end
end
Expand Down
58 changes: 57 additions & 1 deletion stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ let retval_tests = @testset NoThrowTestSet begin
ts = Test.DefaultTestSet("Mock for testing retval of record(::DefaultTestSet, ::T <: Result) methods")
pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock"))
@test Test.record(ts, pass_mock) isa Test.Pass
error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock"))
error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock"), nothing)
@test Test.record(ts, error_mock; print_result=false) isa Test.Error
fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false)
@test Test.record(ts, fail_mock; print_result=false) isa Test.Fail
Expand Down Expand Up @@ -1892,3 +1892,59 @@ end
@test _escape_call(:((==).(x, y))) == (; func=Expr(:., esc(:(==))), args, kwargs, quoted_func=QuoteNode(Expr(:., :(==))))
end
end

@testset "Context display in @testset let blocks" begin
# Mock parent testset that just captures results
struct MockParentTestSet <: Test.AbstractTestSet
results::Vector{Any}
MockParentTestSet() = new([])
end
Test.record(ts::MockParentTestSet, t) = (push!(ts.results, t); t)
Test.finish(ts::MockParentTestSet) = ts

@testset "context shown when a context testset fails" begin
mock_parent1 = MockParentTestSet()
ctx_ts1 = Test.ContextTestSet(mock_parent1, :x, 42)

fail_result = Test.Fail(:test, "x == 99", "42 == 99", "42", nothing, LineNumberNode(1, :test), false)
Test.record(ctx_ts1, fail_result)

@test length(mock_parent1.results) == 1
recorded_fail = mock_parent1.results[1]
@test recorded_fail isa Test.Fail
@test recorded_fail.context !== nothing
@test occursin("x = 42", recorded_fail.context)
end

@testset "context shown when a context testset errors" begin
mock_parent2 = MockParentTestSet()
ctx_ts2 = Test.ContextTestSet(mock_parent2, :x, 42)

# Use internal constructor to create Error with pre-processed values
error_result = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test))
Test.record(ctx_ts2, error_result)

@test length(mock_parent2.results) == 1
recorded_error = mock_parent2.results[1]
@test recorded_error isa Test.Error
@test recorded_error.context !== nothing
@test occursin("x = 42", recorded_error.context)

# Context shows up in string representation
error_str = sprint(show, recorded_error)
@test occursin("Context:", error_str)
@test occursin("x = 42", error_str)

# Multiple variables context
mock_parent3 = MockParentTestSet()
ctx_ts3 = Test.ContextTestSet(mock_parent3, :(x, y), (42, "hello"))

error_result2 = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test))
Test.record(ctx_ts3, error_result2)

recorded_error2 = mock_parent3.results[1]
@test recorded_error2 isa Test.Error
@test recorded_error2.context !== nothing
@test occursin("(x, y) = (42, \"hello\")", recorded_error2.context)
end
end
4 changes: 2 additions & 2 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ cd(@__DIR__) do
# deserialization errors or something similar. Record this testset as Errored.
fake = Test.DefaultTestSet(testname)
fake.time_end = fake.time_start + duration
Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1)))
Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1), nothing))
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
Expand All @@ -423,7 +423,7 @@ cd(@__DIR__) do
for test in all_tests
(test in completed_tests) && continue
fake = Test.DefaultTestSet(test)
Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1)))
Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1), nothing))
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
Expand Down