Skip to content

Commit

Permalink
Some polishing on the visual tests. Changed CMake test variables and …
Browse files Browse the repository at this point in the history
…names. Update changelog.
  • Loading branch information
mikke89 committed Aug 13, 2020
1 parent ede2c68 commit 2392c8d
Show file tree
Hide file tree
Showing 22 changed files with 114 additions and 82 deletions.
9 changes: 3 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)

if(BUILD_TESTING)
set(RMLUI_TESTS_ENABLED ON)
option(BUILD_UNIT_TESTS "Enable to build UnitTests." ON)
option(BUILD_VISUAL_TESTS "Enable to build VisualTests." ON)
set(VISUAL_TESTS_DIRECTORIES "" CACHE STRING "Specify additional directories containing *.rml documents for the visual tests. Backslashes must be escaped. Separate multiple directories by semicolon.")
set(VISUAL_TESTS_INPUT_DIRECTORY "" CACHE PATH "Set the input directory for screenshot comparison performed by the VisualTests.")
set(VISUAL_TESTS_OUTPUT_DIRECTORY "" CACHE PATH "Set the output directory for screenshots generated by the VisualTests.")
option(BUILD_BENCHMARKS "Enable to build Benchmarks." ON)
set(VISUAL_TESTS_RML_DIRECTORIES "" CACHE PATH "Specify additional directories containing *.rml test documents for VisualTests. Separate multiple directories by semicolon.")
set(VISUAL_TESTS_COMPARE_DIRECTORY "" CACHE PATH "Set the input directory for screenshot comparison performed by VisualTests.")
set(VISUAL_TESTS_CAPTURE_DIRECTORY "" CACHE PATH "Set the output directory for screenshots generated by VisualTests.")
endif()
endif()

Expand Down
2 changes: 1 addition & 1 deletion Source/Debugger/LogSource.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ static const char* log_rcss = R"RCSS(body
{
width: 400dp;
height: 300dp;
min-width: 230dp;
min-width: 250dp;
min-height: 150dp;
top: 42dp;
left: 20dp;
Expand Down
103 changes: 51 additions & 52 deletions Tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,74 +32,73 @@ file(GLOB TestsCommon_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/Common/*.cpp
#===================================
# Unit Tests =======================
#===================================
if(BUILD_UNIT_TESTS)
file(GLOB UnitTests_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/UnitTests/*.h )
file(GLOB UnitTests_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/UnitTests/*.cpp )

add_executable(UnitTests ${UnitTests_HDR_FILES} ${UnitTests_SRC_FILES})
target_link_libraries(UnitTests RmlCore doctest::doctest)
set_target_properties(UnitTests PROPERTIES CXX_STANDARD 14)
file(GLOB UnitTests_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/UnitTests/*.h )
file(GLOB UnitTests_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/UnitTests/*.cpp )

add_common_target_options(UnitTests)
add_executable(UnitTests ${UnitTests_HDR_FILES} ${UnitTests_SRC_FILES})
target_link_libraries(UnitTests RmlCore doctest::doctest)
set_target_properties(UnitTests PROPERTIES CXX_STANDARD 14)

if(MSVC)
target_compile_definitions(UnitTests PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()
add_common_target_options(UnitTests)

doctest_discover_tests(UnitTests)
if(MSVC)
target_compile_definitions(UnitTests PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()

doctest_discover_tests(UnitTests)



#===================================
# Visual Tests =====================
#===================================
if(BUILD_VISUAL_TESTS)
file(GLOB VisualTests_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/VisualTests/*.h )
file(GLOB VisualTests_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/VisualTests/*.cpp )

if(WIN32)
add_executable(VisualTests WIN32 ${VisualTests_HDR_FILES} ${VisualTests_SRC_FILES})
else()
add_executable(VisualTests ${VisualTests_HDR_FILES} ${VisualTests_SRC_FILES})
endif()

target_link_libraries(VisualTests RmlCore RmlDebugger lodepng::lodepng ${sample_LIBRARIES})
set_target_properties(VisualTests PROPERTIES CXX_STANDARD 14)

# Enable compiler warnings
add_common_target_options(VisualTests)

if(VISUAL_TESTS_DIRECTORIES)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_DIRECTORIES="${VISUAL_TESTS_DIRECTORIES}")
endif()
if(VISUAL_TESTS_INPUT_DIRECTORY)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_INPUT_DIRECTORY="${VISUAL_TESTS_INPUT_DIRECTORY}")
endif()
if(VISUAL_TESTS_OUTPUT_DIRECTORY)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY="${VISUAL_TESTS_OUTPUT_DIRECTORY}")
endif()

if(MSVC)
target_compile_definitions(VisualTests PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()

file(GLOB VisualTests_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/VisualTests/*.h )
file(GLOB VisualTests_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/VisualTests/*.cpp )

if(WIN32)
add_executable(VisualTests WIN32 ${VisualTests_HDR_FILES} ${VisualTests_SRC_FILES})
else()
add_executable(VisualTests ${VisualTests_HDR_FILES} ${VisualTests_SRC_FILES})
endif()

target_link_libraries(VisualTests RmlCore RmlDebugger lodepng::lodepng ${sample_LIBRARIES})
set_target_properties(VisualTests PROPERTIES CXX_STANDARD 14)

# Enable compiler warnings
add_common_target_options(VisualTests)

if(VISUAL_TESTS_RML_DIRECTORIES)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_RML_DIRECTORIES="${VISUAL_TESTS_RML_DIRECTORIES}")
endif()
if(VISUAL_TESTS_COMPARE_DIRECTORY)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_COMPARE_DIRECTORY="${VISUAL_TESTS_COMPARE_DIRECTORY}")
endif()
if(VISUAL_TESTS_CAPTURE_DIRECTORY)
target_compile_definitions(VisualTests PRIVATE RMLUI_VISUAL_TESTS_CAPTURE_DIRECTORY="${VISUAL_TESTS_CAPTURE_DIRECTORY}")
endif()

if(MSVC)
target_compile_definitions(VisualTests PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()



#===================================
# Benchmarks =======================
#===================================
if(BUILD_BENCHMARKS)
file(GLOB Benchmarks_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/Benchmarks/*.h )
file(GLOB Benchmarks_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/Benchmarks/*.cpp )

add_executable(Benchmarks ${Benchmarks_HDR_FILES} ${Benchmarks_SRC_FILES} ${TestsCommon_HDR_FILES} ${TestsCommon_SRC_FILES})
target_link_libraries(Benchmarks RmlCore RmlDebugger doctest::doctest nanobench::nanobench ${sample_LIBRARIES})
set_target_properties(Benchmarks PROPERTIES CXX_STANDARD 14)
file(GLOB Benchmarks_HDR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/Benchmarks/*.h )
file(GLOB Benchmarks_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Source/Benchmarks/*.cpp )

add_executable(Benchmarks ${Benchmarks_HDR_FILES} ${Benchmarks_SRC_FILES} ${TestsCommon_HDR_FILES} ${TestsCommon_SRC_FILES})
target_link_libraries(Benchmarks RmlCore RmlDebugger doctest::doctest nanobench::nanobench ${sample_LIBRARIES})
set_target_properties(Benchmarks PROPERTIES CXX_STANDARD 14)

# Enable compiler warnings
add_common_target_options(Benchmarks)
# Enable compiler warnings
add_common_target_options(Benchmarks)

if(MSVC)
target_compile_definitions(Benchmarks PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()
endif()
if(MSVC)
target_compile_definitions(Benchmarks PUBLIC DOCTEST_CONFIG_USE_STD_HEADERS)
endif()
3 changes: 2 additions & 1 deletion Tests/Data/VisualTests/LICENSE.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
This software or document includes material copied and modified from the CSS specifications [1], in particular from examples there-in. In addition, parts of the CSS test suites [2] have been modified and included with this software. The license [3] of this material is restated below. Copyright © 2020 W3C® (MIT, ERCIM, Keio, Beihang).
This software or document includes material copied and modified from the CSS specifications [1], in particular from examples there-in. In addition, parts of the CSS test suites [2] have been modified and included with this software. The material is located in some of the *.rml files in the same directory as this file, and have been tagged with an XML 'link' node with a 'rel=source' attribute pointing to the exact place on the 'w3.org' or 'csswg.org' websites the material has been copied and modified from. The license [3] of this material is restated below. Copyright © 2020 W3C® (MIT, ERCIM, Keio, Beihang).

[1] https://drafts.csswg.org/
[2] https://www.w3.org/Style/CSS/Test/
[3] https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document
Expand Down
2 changes: 1 addition & 1 deletion Tests/Data/VisualTests/css1_clear.rml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://www.w3.org/Style/CSS/Test/CSS1/current/sec5526.htm"/>
<link rel="help" href="http://www.w3.org/TR/REC-CSS1#clear"/>
<meta name="description" content="This is the ACID1 test. Some minor modifications have been applied such as matching fonts and borders to the RmlUi syntax." />
<meta name="description" content="Basic CSS1 test of the 'clear' property." />
<style type="text/css">
@spritesheet theme
{
Expand Down
4 changes: 3 additions & 1 deletion Tests/Data/VisualTests/inline_block.rml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
<head>
<title>Inline-block</title>
<link type="text/rcss" href="../style.rcss"/>
<meta name="Description" content="Inline-block" />
<link rel="help" href="https://www.w3.org/TR/2011/REC-CSS2-20110607/visudet.html#inlineblock-width" />
<link rel="help" href="https://www.w3.org/TR/2011/REC-CSS2-20110607/visudet.html#leading" />
<meta name="Description" content="Elements with 'display: inline-block' should shrink-to-fit when no width is specified. The baseline of the element should be the baseline of its last line." />
<style>
body {
background: #ddd;
Expand Down
1 change: 1 addition & 0 deletions Tests/Data/VisualTests/position_01_normal_flow.rml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Normal flow</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content="Position demo" />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Relative positioning</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='The result is identical to normal flow, except that the "outer" text is shifted 12px upward, without affecting the flow of the "body" or "inner" text.' />
<style>
Expand Down
1 change: 1 addition & 0 deletions Tests/Data/VisualTests/position_03_floating_a_box.rml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Floating a box</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='The "inner" text lays out in an independent box on the right, causing the remaining "body" and "outer" text to flow around it.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Floating a box - Sibling</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='Identical to the previous example, save that there is now "sibling" text flowing with the "body" and "outer" text.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Floating a box - Clear</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='Now the "sibling" text moves down to below the "inner" text’s box, leaving blank space behind. The text following the "sibling" text flows after it as normal.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Absolute positioning</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='All of the text within #outer (the "outer" and "inner" text) moves down to an independent box in the lower right corner. The two halves of "body" text flow together.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Absolute positioning - Relative</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='The "inner" text is positioned in an independent box, relative to the top-left corner of the start of the "outer" text.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Absolute positioning - No relative</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='Same as before, except now the "inner text" is positioned relative to the top-left corner of the page itself.' />
<style>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<head>
<title>CSS Position: Absolute positioning - Change bars</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="source" href="https://drafts.csswg.org/css-position-3/#comparison" />
<link rel="help" href="https://drafts.csswg.org/css-position-3/#comparison" />
<meta name="description" content='The two red hyphens, indicating a change, sit in the left margin of the page on the line containing the word "THIS", regardless of what line that ends up being.' />
<style>
Expand Down
2 changes: 1 addition & 1 deletion Tests/Data/VisualTests/shrink_to_fit_01.rml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<title>Shrink-to-fit</title>
<link type="text/rcss" href="../style.rcss"/>
<link rel="help" href="https://www.w3.org/TR/CSS21/visudet.html#shrink-to-fit-float" />
<meta name="Description" content="Shrink-to-fit width" />
<meta name="Description" content="Floating elements, absolutely positioned elements, and inline-block elements should shrink-to-fit when their width is auto." />
<style>
body {
background: #ddd;
Expand Down
2 changes: 1 addition & 1 deletion Tests/Data/VisualTests/shrink_to_fit_02.rml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
</head>

<body id="test">
<p>The following boxes should all appear the same, with the background and border tightly packed around the word 'BOX'.</p>
<p>The background of the following boxes should pack tightly around their contents, with no line-breaks.</p>
<div style="position: relative; top: -50px;">
<div class="x" id="absolute">
Element Information
Expand Down
12 changes: 6 additions & 6 deletions Tests/Source/VisualTests/TestConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

Rml::String GetCompareInputDirectory()
{
#ifdef RMLUI_VISUAL_TESTS_INPUT_DIRECTORY
const Rml::String input_directory = Rml::String(RMLUI_VISUAL_TESTS_INPUT_DIRECTORY);
#ifdef RMLUI_VISUAL_TESTS_COMPARE_DIRECTORY
const Rml::String input_directory = Rml::String(RMLUI_VISUAL_TESTS_COMPARE_DIRECTORY);
#else
const Rml::String input_directory = Shell::FindSamplesRoot() + "../Tests/Output";
#endif
Expand All @@ -44,8 +44,8 @@ Rml::String GetCompareInputDirectory()

Rml::String GetCaptureOutputDirectory()
{
#ifdef RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY
const Rml::String output_directory = Rml::String(RMLUI_VISUAL_TESTS_OUTPUT_DIRECTORY);
#ifdef RMLUI_VISUAL_TESTS_CAPTURE_DIRECTORY
const Rml::String output_directory = Rml::String(RMLUI_VISUAL_TESTS_CAPTURE_DIRECTORY);
#else
const Rml::String output_directory = Shell::FindSamplesRoot() + "../Tests/Output";
#endif
Expand All @@ -58,8 +58,8 @@ Rml::StringList GetTestInputDirectories()

Rml::StringList directories = { samples_root + "../Tests/Data/VisualTests" };

#ifdef RMLUI_VISUAL_TESTS_DIRECTORIES
Rml::StringUtilities::ExpandString(directories, RMLUI_VISUAL_TESTS_DIRECTORIES, ';');
#ifdef RMLUI_VISUAL_TESTS_RML_DIRECTORIES
Rml::StringUtilities::ExpandString(directories, RMLUI_VISUAL_TESTS_RML_DIRECTORIES, ';');
#endif

return directories;
Expand Down
28 changes: 18 additions & 10 deletions Tests/Source/VisualTests/TestNavigator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,18 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
else
{
ComparisonResult result = CompareCurrentView();
const Rml::String compare_path = GetCompareInputDirectory() + '/' + GetImageFilenameFromCurrentTest();
if (result.success)
{
if (result.is_equal)
{
Rml::Log::Message(Rml::Log::LT_INFO, "%s compares EQUAL to the reference image.",
CurrentSuite().GetFilename().c_str());
Rml::Log::Message(Rml::Log::LT_INFO, "%s compares EQUAL to the reference image %s.",
CurrentSuite().GetFilename().c_str(), compare_path.c_str());
}
else
{
Rml::Log::Message(Rml::Log::LT_INFO, "%s compares NOT EQUAL to the reference image. See diff image written to %s.",
CurrentSuite().GetFilename().c_str(), GetCaptureOutputDirectory().c_str());
Rml::Log::Message(Rml::Log::LT_INFO, "%s compares NOT EQUAL to the reference image %s.\nSee diff image written to %s.",
CurrentSuite().GetFilename().c_str(), compare_path.c_str(), GetCaptureOutputDirectory().c_str());
}

if (!result.error_msg.empty())
Expand All @@ -141,15 +142,22 @@ void TestNavigator::ProcessEvent(Rml::Event& event)
{
Rml::Log::Message(Rml::Log::LT_ERROR, "Comparison of %s failed.\n%s", CurrentSuite().GetFilename().c_str(), result.error_msg.c_str());
}

}
}
else if (key_identifier == Rml::Input::KI_F7)
{
if (key_ctrl && key_shift)
{
StartTestSuiteIteration(IterationState::Capture);
}
else
CaptureCurrentView();
{
const Rml::String filepath = GetCaptureOutputDirectory() + '/' + GetImageFilenameFromCurrentTest();
if (CaptureCurrentView())
Rml::Log::Message(Rml::Log::LT_INFO, "Succesfully captured and saved screenshot to %s", filepath.c_str());
else
Rml::Log::Message(Rml::Log::LT_ERROR, "Could not capture screenshot to %s", filepath.c_str());
}
}
else if (key_identifier == Rml::Input::KI_F1)
{
Expand Down Expand Up @@ -330,15 +338,15 @@ void TestNavigator::LoadActiveTest()
UpdateGoToText();
}

static Rml::String ImageFilenameFromTest(const TestSuite& suite)
Rml::String TestNavigator::GetImageFilenameFromCurrentTest()
{
const Rml::String& filename = suite.GetFilename();
const Rml::String& filename = CurrentSuite().GetFilename();
return filename.substr(0, filename.rfind('.')) + ".png";
}

ComparisonResult TestNavigator::CompareCurrentView()
{
const Rml::String filename = ImageFilenameFromTest(CurrentSuite());
const Rml::String filename = GetImageFilenameFromCurrentTest();

ComparisonResult result = CompareScreenToPreviousCapture(shell_renderer, filename);

Expand All @@ -348,7 +356,7 @@ ComparisonResult TestNavigator::CompareCurrentView()

bool TestNavigator::CaptureCurrentView()
{
const Rml::String filename = ImageFilenameFromTest(CurrentSuite());
const Rml::String filename = GetImageFilenameFromCurrentTest();

bool result = CaptureScreenshot(shell_renderer, filename, 1060);

Expand Down
2 changes: 2 additions & 0 deletions Tests/Source/VisualTests/TestNavigator.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ class TestNavigator : public Rml::EventListener {

void UpdateGoToText(bool out_of_bounds = false);

Rml::String GetImageFilenameFromCurrentTest();

ShellRenderInterfaceOpenGL* shell_renderer;
Rml::Context* context;
TestViewer* viewer;
Expand Down
2 changes: 1 addition & 1 deletion Tests/Source/VisualTests/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ int main(int RMLUI_UNUSED_PARAMETER(argc), char** RMLUI_UNUSED_PARAMETER(argv))

if (files.empty())
{
Rml::Log::Message(Rml::Log::LT_WARNING, "Could not find any *.rml* files in directory '%s'. Ignoring.'", directory.c_str());
Rml::Log::Message(Rml::Log::LT_WARNING, "Could not find any *.rml files in directory '%s'. Ignoring.", directory.c_str());
}
else
{
Expand Down
Loading

0 comments on commit 2392c8d

Please sign in to comment.