Skip to content

Commit

Permalink
Add line info to test errors (JuliaLang#24181)
Browse files Browse the repository at this point in the history 
* add linfo to test error

* consolidate test a bit
  • Loading branch information
@KristofferC
KristofferC authored Oct 18, 2017
1 parent d810e08 commit c1a4df2
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 25 deletions.
61 changes: 37 additions & 24 deletions stdlib/Test/src/Test.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,12 @@ mutable struct Fail <: Result
orig_expr
data
value
source::LineNumberNode
end
function Base.show(io::IO, t::Fail)
print_with_color(Base.error_color(), io, "Test Failed\n"; bold = true)
print_with_color(Base.error_color(), io, "Test Failed"; bold = true)
print(io, " at ")
print_with_color(:default, io, t.source.file, ":", t.source.line, "\n"; bold = true)
print(io, " Expression: ", t.orig_expr)
if t.test_type == :test_throws_wrong
# An exception was thrown, but it was of the wrong type
Expand Down Expand Up @@ -120,9 +123,12 @@ mutable struct Error <: Result
orig_expr
value
backtrace
source::LineNumberNode
end
function Base.show(io::IO, t::Error)
print_with_color(Base.error_color(), io, "Error During Test\n"; bold = true)
print_with_color(Base.error_color(), io, "Error During Test"; bold = true)
print(io, " at ")
print_with_color(:default, io, t.source.file, ":", t.source.line, "\n"; bold = true)
if t.test_type == :test_nonbool
println(io, " Expression evaluated to non-Boolean")
println(io, " Expression: ", t.orig_expr)
Expand Down Expand Up @@ -175,14 +181,16 @@ abstract type ExecutionResult end
struct Returned <: ExecutionResult
value
data
source::LineNumberNode
end

struct Threw <: ExecutionResult
exception
backtrace
source::LineNumberNode
end

function eval_test(evaluated::Expr, quoted::Expr)
function eval_test(evaluated::Expr, quoted::Expr, source::LineNumberNode)
res = true
i = 1
evaled_args = evaluated.args
Expand Down Expand Up @@ -219,7 +227,7 @@ function eval_test(evaluated::Expr, quoted::Expr)
else
throw(ArgumentError("Unhandled expression type: $(evaluated.head)"))
end
Returned(res, quoted)
Returned(res, quoted, source)
end

const comparison_prec = Base.operator_precedence(:(==))
Expand Down Expand Up @@ -266,7 +274,7 @@ is a call expression and the rest are assignments (`k=v`).
macro test(ex, kws...)
test_expr!("@test", ex, kws...)
orig_ex = Expr(:inert, ex)
result = get_test_result(ex)
result = get_test_result(ex, __source__)
:(do_test($result, $orig_ex))
end

Expand All @@ -284,7 +292,7 @@ The `@test_broken f(args...) key=val...` form works as for the `@test` macro.
macro test_broken(ex, kws...)
test_expr!("@test_broken", ex, kws...)
orig_ex = Expr(:inert, ex)
result = get_test_result(ex)
result = get_test_result(ex, __source__)
# code to call do_test with execution result and original expr
:(do_broken_test($result, $orig_ex))
end
Expand All @@ -311,7 +319,7 @@ end
# In the special case of a comparison, e.g. x == 5, generate code to
# evaluate each term in the comparison individually so the results
# can be displayed nicely.
function get_test_result(ex)
function get_test_result(ex, source)
# Normalize non-dot comparison operator calls to :comparison expressions
is_splat = x -> isa(x, Expr) && x.head == :...
if isa(ex, Expr) && ex.head == :call && length(ex.args) == 3 &&
Expand All @@ -326,6 +334,7 @@ function get_test_result(ex)
testret = :(eval_test(
Expr(:comparison, $(escaped_terms...)),
Expr(:comparison, $(quoted_terms...)),
$(QuoteNode(source)),
))
elseif isa(ex, Expr) && ex.head == :call && ex.args[1] in (:isequal, :isapprox)
escaped_func = esc(ex.args[1])
Expand Down Expand Up @@ -368,15 +377,16 @@ function get_test_result(ex)
testret = :(eval_test(
Expr(:call, $escaped_func, Expr(:parameters, $(escaped_kwargs...)), $(escaped_args...)),
Expr(:call, $quoted_func),
$(QuoteNode(source)),
))
else
testret = :(Returned($(esc(ex)), nothing))
testret = :(Returned($(esc(ex)), nothing, $(QuoteNode(source))))
end
result = quote
try
$testret
catch _e
Threw(_e, catch_backtrace())
Threw(_e, catch_backtrace(), $(QuoteNode(source)))
end
end
Base.remove_linenums!(result)
Expand All @@ -398,16 +408,16 @@ function do_test(result::ExecutionResult, orig_expr)
testres = if isa(value, Bool)
# a true value Passes
value ? Pass(:test, nothing, nothing, value) :
Fail(:test, orig_expr, result.data, value)
Fail(:test, orig_expr, result.data, value, result.source)
else
# If the result is non-Boolean, this counts as an Error
Error(:test_nonbool, orig_expr, value, nothing)
Error(:test_nonbool, orig_expr, value, nothing, result.source)
end
else
# The predicate couldn't be evaluated without throwing an
# exception, so that is an Error and not a Fail
@assert isa(result, Threw)
testres = Error(:test_error, orig_expr, result.exception, result.backtrace)
testres = Error(:test_error, orig_expr, result.exception, result.backtrace, result.source)
end
record(get_testset(), testres)
end
Expand All @@ -418,7 +428,7 @@ function do_broken_test(result::ExecutionResult, orig_expr)
if isa(result, Returned)
value = result.value
if isa(value, Bool) && value
testres = Error(:test_unbroken, orig_expr, value, nothing)
testres = Error(:test_unbroken, orig_expr, value, nothing, result.source)
end
end
record(get_testset(), testres)
Expand All @@ -438,9 +448,9 @@ macro test_throws(extype, ex)
orig_ex = Expr(:inert, ex)
result = quote
try
Returned($(esc(ex)), nothing)
Returned($(esc(ex)), nothing, $(QuoteNode(__source__)))
catch _e
Threw(_e, nothing)
Threw(_e, nothing, $(QuoteNode(__source__)))
end
end
Base.remove_linenums!(result)
Expand Down Expand Up @@ -470,10 +480,10 @@ function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), @nosp
if success
testres = Pass(:test_throws, nothing, nothing, exc)
else
testres = Fail(:test_throws_wrong, orig_expr, extype, exc)
testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source)
end
else
testres = Fail(:test_throws_nothing, orig_expr, extype, nothing)
testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source)
end
record(get_testset(), testres)
end
Expand Down Expand Up @@ -509,6 +519,7 @@ macro test_warn(msg, expr)
end
eval(Base, Expr(:(=), :have_color, have_color))
@test ismatch_warn($(esc(msg)), read(fname, String))
eval(Base, Expr(:(=), :have_color, have_color))
ret
finally
eval(Base, Expr(:(=), :have_color, have_color))
Expand Down Expand Up @@ -642,7 +653,9 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error})
print(t)
# don't print the backtrace for Errors because it gets printed in the show
# method
isa(t, Error) || Base.show_backtrace(STDOUT, scrub_backtrace(backtrace()))
if !isa(t, Error)
Base.show_backtrace(STDOUT, scrub_backtrace(backtrace()))
end
println()
end
push!(ts.results, t)
Expand Down Expand Up @@ -903,16 +916,16 @@ macro testset(args...)
end

if tests.head == :for
return testset_forloop(args, tests)
return testset_forloop(args, tests, __source__)
else
return testset_beginend(args, tests)
return testset_beginend(args, tests, __source__)
end
end

"""
Generate the code for a `@testset` with a `begin`/`end` argument
"""
function testset_beginend(args, tests)
function testset_beginend(args, tests, source)
desc, testsettype, options = parse_testset_args(args[1:end-1])
if desc === nothing
desc = "test set"
Expand All @@ -938,7 +951,7 @@ function testset_beginend(args, tests)
catch err
# something in the test block threw an error. Count that as an
# error in this test set
record(ts, Error(:nontest_error, :(), err, catch_backtrace()))
record(ts, Error(:nontest_error, :(), err, catch_backtrace(), $(QuoteNode(source))))
end
pop_testset()
finish(ts)
Expand All @@ -949,7 +962,7 @@ end
"""
Generate the code for a `@testset` with a `for` loop argument
"""
function testset_forloop(args, testloop)
function testset_forloop(args, testloop, source)
# Pull out the loop variables. We might need them for generating the
# description and we'll definitely need them for generating the
# comprehension expression at the end
Expand Down Expand Up @@ -999,7 +1012,7 @@ function testset_forloop(args, testloop)
catch err
# Something in the test block threw an error. Count that as an
# error in this test set
record(ts, Error(:nontest_error, :(), err, catch_backtrace()))
record(ts, Error(:nontest_error, :(), err, catch_backtrace(), $(QuoteNode(source))))
end
end
quote
Expand Down
21 changes: 21 additions & 0 deletions stdlib/Test/test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,3 +631,24 @@ Test.print_test_results(Test.DefaultTestSet(""))'`), stderr=DevNull), String), "
@test orig == Base.GLOBAL_RNG
@test rand(orig) == rand()
end

@testset "file info in test errors" begin
f = tempname()

write(f,
"""
using Test
@testset begin
@test 1==2
@test_throws UndefVarError 1
@test_broken 1 == 1
end
""")

msg = read(pipeline(ignorestatus(`$(Base.julia_cmd()) --startup-file=no --color=no $f`), stderr=DevNull), String)
@test contains(msg, "at " * f * ":" * "3")
@test contains(msg, "at " * f * ":" * "4")
@test contains(msg, "at " * f * ":" * "5")

rm(f; force=true)
end
2 changes: 1 addition & 1 deletion test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ cd(dirname(@__FILE__)) do
# the test runner itself had some problem, so we may have hit a segfault,
# deserialization errors or something similar. Record this testset as Errored.
fake = Test.DefaultTestSet(res[1])
Test.record(fake, Test.Error(:test_error, res[1], res[2][1], []))
Test.record(fake, Test.Error(:test_error, res[1], res[2][1], [], LineNumberNode(1)))
Test.push_testset(fake)
Test.record(o_ts, fake)
Test.pop_testset()
Expand Down

0 comments on commit c1a4df2

Please sign in to comment.