From e194701e8604c942018e3f66d1e2c1e1a4580acc Mon Sep 17 00:00:00 2001 From: Pal Mezei Date: Fri, 9 Jul 2021 18:59:12 +1000 Subject: [PATCH] Running render tests via cmake (#827) * Running render tests. (#124) * Generating testsuite commands for windows. (#124) * Diffing tests, resaving scenes and generating thumbnails. (#124) * Fixing unit test build issues on windows. (#124) * Fixing testsuite issues when targeting a static USD build. (#124) * Using kick_params and generating test image optionally. (#124) * Using correct file check on windows. (#124) --- CMakeLists.txt | 2 +- cmake/utils/options.cmake | 8 ++ cmake/utils/test.cmake | 178 ++++++++++++++++++++++++++++-- docs/CMakeLists.txt | 2 +- docs/building.md | 1 + testsuite/CMakeLists.txt | 5 +- testsuite/test_0090/data/test.ass | 2 +- 7 files changed, 185 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4379bcbba7..61e626d984 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ project(arnold-usd) -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.20) if (UNIX AND NOT APPLE) set(LINUX TRUE) diff --git a/cmake/utils/options.cmake b/cmake/utils/options.cmake index 968b9ce1ae..9369c8c3d7 100644 --- a/cmake/utils/options.cmake +++ b/cmake/utils/options.cmake @@ -20,6 +20,7 @@ option(BUILD_USE_CUSTOM_BOOST "Using a custom boost layout." OFF) option(BUILD_BOOST_ALL_NO_LIB "Disable linking of boost libraries from boost headers." OFF) option(BUILD_DISABLE_CXX11_ABI "Disable the use of the new CXX11 ABI" OFF) option(BUILD_HEADERS_AS_SOURCES "Add headers are source files to the target to help when generating IDE projects." OFF) +set(USD_OVERRIDE_PLUGINPATH_NAME "PXR_PLUGINPATH_NAME" CACHE STRING "Override the plugin path name for the USD libraries. Used when running the testsuite with a static procedural") option(BUILD_SCHEMAS "Builds the USD Schemas" ON) option(BUILD_RENDER_DELEGATE "Builds the Render Delegate" ON) @@ -41,4 +42,11 @@ set(PREFIX_SCHEMA "schema" CACHE STRING "Directory to install the schemas under. set(PREFIX_BIN "bin" CACHE STRING "Directory to install the binaries under.") set(PREFIX_DOCS "docs" CACHE STRING "Directory to install the documentation under.") +set(TEST_DIFF_HARDFAIL "0.0157" CACHE STRING "Hard failure of an image comparison test.") +set(TEST_DIFF_FAIL "0.00001" CACHE STRING "Failure of an image comparison test.") +set(TEST_DIFF_FAILPERCENT "33.334" CACHE STRING "Failure percentage of an image comparison test.") +set(TEST_DIFF_WARNPERCENT "0.0" CACHE STRING "Warning percentage of an image comparison test.") +set(TEST_RESOLUTION "160 120" CACHE STRING "Resolution of unit tests.") +set(TEST_MAKE_THUMBNAILS "Enables the generation of test thumbnails." ON) + set(USD_PROCEDURAL_NAME "usd" CACHE STRING "Name of the usd procedural.") diff --git a/cmake/utils/test.cmake b/cmake/utils/test.cmake index 1edbb56155..0f1d2fa21f 100644 --- a/cmake/utils/test.cmake +++ b/cmake/utils/test.cmake @@ -90,6 +90,8 @@ function(add_unit_test) cmake_parse_arguments(_args "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN}) + # Common unit tests don't link a library, they include the sources directly. + # This is done, because libraries don't expose symbols from common utils. add_executable(${_args_TEST_NAME} "${CMAKE_SOURCE_DIR}/testsuite/${_args_TEST_NAME}/data/test.cpp") if (${_args_GTEST} OR ${_args_GMOCK}) target_include_directories(${_args_TEST_NAME} PUBLIC "${GTEST_INCLUDE_DIR}") @@ -114,6 +116,8 @@ function(add_unit_test) endif () add_test(NAME ${_args_TEST_NAME} COMMAND $) + # TODO(pal): Investigate if these environment sets do work. After running the render tests, looks like + # they might not work. Maybe we could generate a batch script here as well, that configures the env etc? if (WIN32) set_tests_properties(${_args_TEST_NAME} PROPERTIES ENVIRONMENT "PATH=${TEST_LIBRARY_PATHS}\;$") @@ -126,10 +130,11 @@ function(add_unit_test) endif () endfunction() +# Add new unit test for the render delegate library. function(add_render_delegate_unit_test) # We are ignoring the tests so the scripts that automatically scan the folders will skip these folders. ignore_test(${ARGN}) - if (NOT BUILD_RENDER_DELEGATE) + if (NOT BUILD_RENDER_DELEGATE OR USD_STATIC_BUILD) return() endif () foreach (_test_name ${ARGN}) @@ -139,10 +144,11 @@ function(add_render_delegate_unit_test) endforeach () endfunction() +# Add new unit test for the ndr library. function(add_ndr_unit_test) # We are ignoring the tests so the scripts that automatically scan the folders will skip these folders. ignore_test(${ARGN}) - if (NOT BUILD_NDR_PLUGIN) + if (NOT BUILD_NDR_PLUGIN OR USD_STATIC_BUILD) return() endif () foreach (_test_name ${ARGN}) @@ -152,6 +158,7 @@ function(add_ndr_unit_test) endforeach () endfunction() +# Add new unit test for the translator library. function(add_translator_unit_test) # We are ignoring the tests so the scripts that automatically scan the folders will skip these folders. ignore_test(${ARGN}) @@ -165,12 +172,13 @@ function(add_translator_unit_test) endforeach () endfunction() -function(discover_render_test dir) +function(discover_render_test test_name dir) # First we check if there is a README file and parse the list of parameters set(_resaved "") - set(_forceexpand OFF) + set(_force_expand OFF) set(_scene "test.ass") set(_readme "${dir}/README") + set(_kick_params "") # Readme should exists all the time. if (EXISTS "${_readme}") file(STRINGS @@ -178,19 +186,173 @@ function(discover_render_test dir) _tmp_match REGEX "PARAMS[ ]*\\:[ ]*{[^}]*}") if (_tmp_match MATCHES "'resaved'[ ]*\\:[ ]*'(ass|usda)'") - set(_resaved ${CMAKE_MATCH_1}) + set(_resaved "${CMAKE_MATCH_1}") endif () if (_tmp_match MATCHES "'forceexpand'[ ]*\\:[ ]*True") - set(_forceexpand ON) + set(_force_expand ON) endif () if (_tmp_match MATCHES "'scene'[ ]*\\:[ ]*'([^']+)'") set(_scene "${CMAKE_MATCH_1}") endif () + if (_tmp_match MATCHES "'kick_params'[ ]*\\:[ ]*'([^']+)'") + set(_kick_params "${CMAKE_MATCH_1}") + endif () + endif () + + # Determining the extension and the base file name. + if (_scene MATCHES "([a-z]+)\.([a-z]+)") + set(_scene_base "${CMAKE_MATCH_1}") + set(_scene_extension "${CMAKE_MATCH_2}") + else () + # We can't match a scene file name correct, exit. + return () + endif () + + set(_out_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") + make_directory("${_out_dir}") + + set(_input_scene "${dir}/data/${_scene}") + set(_input_reference "${dir}/ref/reference.tif") + set(_input_log "${dir}/ref/reference.log") + # Only supporting render tests for now. + if (NOT EXISTS "${_input_scene}") + return() + endif () + + set(_input_file "${_out_dir}/${_scene}") + set(_output_render "${_out_dir}/testrender.tif") + set(_output_difference "${_out_dir}/diff.tif") + set(_output_log "${_out_dir}/${test_name}.log") + + # Copying all the files from data to the output directory. + file(GLOB _data_files "${dir}/data/*") + file(COPY ${_data_files} DESTINATION "${_out_dir}") + file(COPY "${dir}/README" DESTINATION "${_out_dir}") + + # Since we need to execute multiple commands, and add_test supports a single command, we are generating a test + # command. + + # We can't have generator expressions in configure file, and the CONTENT field only takes a single variable, + # so we have to join with an newline string to convert a list to a single string that's correctly formatted. + # We need to use ARNOLD_PLUGIN_PATH instead of -l to load the procedural, otherwise our build won't get picked up. + if (WIN32) + set(_cmd + "del /f /q \"${_output_render}\"" + "del /f /q \"${_output_log}\"" + "del /f /q \"${_output_difference}\"" + "setx ARNOLD_PLUGIN_PATH \"$\"" + ) + else () + set(_cmd + "rm -f \"${_output_render}\"" + "rm -f \"${_output_log}\"" + "rm -f \"${_output_difference}\"" + "export ARNOLD_PLUGIN_PATH=\"$\"" + ) + endif () + # We have to setup the pxr plugin path to point at the original schema files if we are creating a static build + # of the procedural. This also could be overwritten in the USD build, and this information is not included in + # the USD library headers. + if (USD_STATIC_BUILD AND BUILD_PROCEDURAL) + if (WIN32) + list(APPEND _cmd "setx ${USD_OVERRIDE_PLUGINPATH_NAME} \"${USD_LIBRARY_DIR}/usd\"") + else () + list(APPEND _cmd "export ${USD_OVERRIDE_PLUGINPATH_NAME}=\"${USD_LIBRARY_DIR}/usd\"") + endif () + endif () + + set(_render_file "${_input_file}") + # If a test is resaved, we need to add a command that resaves the file and render that instead. + if (_resaved MATCHES ".+") + if (_forceexpand) + set(_cmd_force_expand "-forceexpand") + else () + set(_cmd_force_expand "") + endif () + set(_render_file "${_out_dir}/${_scene_base}_resaved.${_resaved}") + list(APPEND _cmd "\"${ARNOLD_KICK}\" -i \"${_input_file}\" -resave \"${_render_file}\" ${_cmd_force_expand}") + endif () + # Shared args for rendering tests and generating reference files. + set(_kick_args "-r ${TEST_RESOLUTION} -sm lambert -bs 16 -set driver_tiff.dither false -dp -dw -v 6 ${_kick_params}") + # Rendering the file. + list(APPEND _cmd "\"${ARNOLD_KICK}\" ${_kick_args} -i \"${_render_file}\" -o \"${_output_render}\" -logfile \"${_output_log}\"") + # TODO(pal): Investigate how best read existing channel definitions from the file. + if (TEST_MAKE_THUMBNAILS) + # Creating thumbnails for reference and new image. + list(APPEND _cmd "\"${ARNOLD_OIIOTOOL}\" \"${_output_render}\" --threads 1 --ch \"R,G,B\" -o ${_out_dir}/new.png") + list(APPEND _cmd "\"${ARNOLD_OIIOTOOL}\" \"${_input_reference}\" --threads 1 --ch \"R,G,B\" -o ${_out_dir}/ref.png") + # Adding diffing commands. + set(_cmd_thumbnails "--sub --abs --cmul 8 -ch \"R,G,B,A\" --dup --ch \"A,A,A,0\" --add -ch \"0,1,2\" -o dif.png") + else () + set(_cmd_thumbnails "") + endif () + # Diffing the result to the reference file. + list(APPEND _cmd "\"${ARNOLD_OIIOTOOL}\" --threads 1 --hardfail ${TEST_DIFF_HARDFAIL} --fail ${TEST_DIFF_FAIL} --failpercent ${TEST_DIFF_FAILPERCENT} --warnpercent ${TEST_DIFF_WARNPERCENT} --diff \"${_output_render}\" \"${_input_reference}\" ${_cmd_thumbnails}") + if (WIN32) + set(_cmd_ext ".bat") + else () + set(_cmd_ext ".sh") endif () + + # CMake has several generators that are support multiple configurations at once, like on Visual Sudio on windows. + # This means we have to generate several command files for each configuration, using different procedurals, otherwise + # we would get cmake warnings and potentially running the wrong binary for each test. We are still using the + # same output directory for each config, as the expectation is to work with a single config at any given time. + string(JOIN "\n" _cmd ${_cmd}) + file( + GENERATE OUTPUT "${_out_dir}/test_$${_cmd_ext}" + CONTENT "${_cmd}" + FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ + NEWLINE_STYLE UNIX + ) + + add_test( + NAME ${test_name} + COMMAND "${_out_dir}/test_$${_cmd_ext}" + WORKING_DIRECTORY "${_out_dir}" + ) + + if (WIN32) + set(_cmd_generate "setx ARNOLD_PLUGIN_PATH \"$\"") + else () + set(_cmd_generate "export ARNOLD_PLUGIN_PATH=\"$\"") + endif () + if (USD_STATIC_BUILD AND BUILD_PROCEDURAL) + if (WIN32) + list(APPEND _cmd_generate "setx ${USD_OVERRIDE_PLUGINPATH_NAME} \"${USD_LIBRARY_DIR}/usd\"") + else () + list(APPEND _cmd_generate "export ${USD_OVERRIDE_PLUGINPATH_NAME}=\"${USD_LIBRARY_DIR}/usd\"") + endif () + endif () + set(_generate_kick "\"${ARNOLD_KICK}\" ${_kick_args} -i \"${_input_scene}\" -o \"${_input_reference}\" -logfile \"${_input_log}\"") + if (WIN32) + list(APPEND _cmd_generate "if not exist \"${_input_reference}\" ${_generate_kick}") + else () + list(APPEND _cmd_generate "[[ -f \"${_input_reference}\" ]] || ${_generate_kick}") + endif () + string(JOIN "\n" _cmd_generate ${_cmd_generate}) + file( + GENERATE OUTPUT "${_out_dir}/test_generate_$${_cmd_ext}" + CONTENT "${_cmd_generate}" + FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ + NEWLINE_STYLE UNIX + ) + + # BYPRODUCTS is not really useful for us, as commands like make clean will remove any byproducts. + # We add a custom target that depends on the procedural, so we can render the reference image and output reference log. + add_custom_target( + ${test_name}_generate + "${_out_dir}/test_generate_$${_cmd_ext}" + DEPENDS ${USD_PROCEDURAL_NAME}_proc + SOURCES ${_data_files} + WORKING_DIRECTORY "${dir}/data" + ) endfunction() function(discover_render_tests) - if (NOT BUILD_PROCEDURAL AND NOT BUILD_USD_WRITER) + # We skip render tests if the procedural is not built for now. In the future we should be able to use the render + # delegate to render without + if (NOT BUILD_PROCEDURAL) return() endif () file(GLOB _subs RELATIVE "${CMAKE_SOURCE_DIR}/testsuite" "${CMAKE_SOURCE_DIR}/testsuite/*") @@ -201,7 +363,7 @@ function(discover_render_tests) endif () set(_sub "${CMAKE_SOURCE_DIR}/testsuite/${_iter}") if (IS_DIRECTORY "${_sub}") - discover_render_test("${_sub}") + discover_render_test("${_iter}" "${_sub}") endif () endforeach () endfunction() diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index b5b36b99d9..8931cdeaaf 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -1,4 +1,4 @@ -find_package(Doxygen REQUIRED) +find_package(Doxygen REQUIRED dot) set(DOXYGEN_IN "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile") set(DOXYGEN_OUT "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile") diff --git a/docs/building.md b/docs/building.md index feebc9b0f0..054094fe6b 100755 --- a/docs/building.md +++ b/docs/building.md @@ -241,6 +241,7 @@ We also support building the project with cmake to allow for greater flexibility - `USD_LIB_EXTENSION`: Extension of USD libraries. - `USD_STATIC_LIB_EXTENSION`: Extension of the static USD libraries. - `USD_LIB_PREFIX`: Prefix of USD libraries. +- `USD_OVERRIDE_PLUGINPATH_NAME`: The PXR_PLUGINPATH_NAME environment variable name of the used USD library. - `TBB_ROOT_DIR`: The base directory the of TBB installation. - `TBB_FOUND`: Set to ON if manual override of the TBB variables is required due to non-standard TBB installation layout. - `TBB_INCLUDE_DIRS`: Where to find TBB headers, optional. Use if not using a standard TBB installation layout. diff --git a/testsuite/CMakeLists.txt b/testsuite/CMakeLists.txt index 749bc6f6bf..37d4ce4a1e 100644 --- a/testsuite/CMakeLists.txt +++ b/testsuite/CMakeLists.txt @@ -2,8 +2,9 @@ include("${CMAKE_SOURCE_DIR}/cmake/utils/test.cmake") ignore_test(test_0011 test_0040 test_0101 test_0108) -add_render_delegate_unit_test(test_0039 test_0134 test_0136 test_0146 test_0147 test_0152 test_0153 test_0154 test_0155 test_0156) +add_render_delegate_unit_test(test_0039 test_0134 test_0146 test_0147 test_0152 test_0153 test_0154 test_0155 test_0156) add_ndr_unit_test(test_0044) -add_translator_unit_test(test_0045) +# test 136 requires only common utils, but we use the translator to automatically include headers, link libraries etc. +add_translator_unit_test(test_0045 test_0136) discover_render_tests() \ No newline at end of file diff --git a/testsuite/test_0090/data/test.ass b/testsuite/test_0090/data/test.ass index bb30412a20..5b0a4e14d4 100644 --- a/testsuite/test_0090/data/test.ass +++ b/testsuite/test_0090/data/test.ass @@ -29,7 +29,7 @@ driver_tiff persp_camera { - name /cameraTest/cameraTestShape + name /persp/perspShape matrix 0.733729839 0 -0.679441333 0 -0.175852343 0.965925813 -0.189903259 0