Supporting TECA_assets svn repo in TECA

.travis.yml

@@ -15,6 +15,7 @@ env:
     - TECA_DIR=/travis_teca_dir
     - TECA_PYTHON_VERSION=3
     - TECA_DATA_REVISION=83
+    - TECA_ASSETS_REVISION=3
   jobs:
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=TRUE
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=FALSE
@@ -54,6 +55,7 @@ install:
     docker exec teca_${DOCKER_IMAGE}_${IMAGE_VERSION} /bin/bash -c
     "export TECA_PYTHON_VERSION=${TECA_PYTHON_VERSION} &&
     export TECA_DATA_REVISION=${TECA_DATA_REVISION} &&
+    export TECA_ASSETS_REVISION=${TECA_ASSETS_REVISION} &&
     export REQUIRE_NETCDF_MPI=${REQUIRE_NETCDF_MPI} &&
     ${TECA_DIR}/test/travis_ci/install_${IMAGE_NAME}.sh";
     fi

CMake/teca_test.cmake

@@ -8,7 +8,7 @@
 #   WILL_FAIL -- flag whose presence indicates the test is expected to fail
 #   )
 function (teca_add_test T_NAME)
-    set(opt_args REQ_TECA_DATA WILL_FAIL)
+    set(opt_args REQ_TECA_DATA REQ_TECA_ASSETS WILL_FAIL)
     set(val_args EXEC_NAME)
     set(array_args SOURCES LIBS COMMAND FEATURES)
     cmake_parse_arguments(T "${opt_args}" "${val_args}" "${array_args}" ${ARGN})
@@ -33,7 +33,8 @@ function (teca_add_test T_NAME)
                 target_link_libraries(${EXEC_NAME} ${T_LIBS})
             endif()
         endif()
-        if ((T_REQ_TECA_DATA AND TECA_HAS_DATA) OR NOT T_REQ_TECA_DATA)
+        if (((T_REQ_TECA_DATA AND TECA_HAS_DATA) OR NOT T_REQ_TECA_DATA) AND
+            ((T_REQ_TECA_ASSETS AND TECA_HAS_ASSETS) OR NOT T_REQ_TECA_ASSETS))
             add_test(NAME ${T_NAME} COMMAND ${T_COMMAND}
                 WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
             if (T_WILL_FAIL)

CMakeLists.txt

@@ -93,6 +93,7 @@ set(REQUIRE_LIBXLSXWRITER FALSE CACHE BOOL "Forces build failure when libxlsxwri
 set(REQUIRE_PARAVIEW FALSE CACHE BOOL "Forces build failure when ParaView is missing")
 set(REQUIRE_VTK FALSE CACHE BOOL "Forces build failure when VTK is missing")
 set(REQUIRE_TECA_DATA ${BUILD_TESTING} CACHE BOOL "Forces build failure when TECA_data is missing")
+set(REQUIRE_TECA_ASSETS ${BUILD_TESTING} CACHE BOOL "Forces build failure when TECA_data is missing")
 
 # locate dependencies
 # configure for MPI
@@ -359,6 +360,33 @@ else()
 endif()
 set(TECA_HAS_DATA ${tmp} CACHE BOOL "TECA_data is present")
 
+# configure for TECA_assets
+set(TECA_ASSETS_ROOT CACHE PATH "Path to TECA_assets repository")
+if (REQUIRE_TECA_ASSETS)
+    # do we have access to a checkout of TECA_assets?
+    if (TECA_ASSETS_ROOT)
+        if (EXISTS "${TECA_ASSETS_ROOT}")
+            set(tmp ON)
+            message(STATUS "TECA_assets -- available")
+        else()
+            message(FATAL_ERROR "TECA_assets -- required but user-set path not found")
+        endif()
+    else()
+        set(tmp ON)
+        ExternalProject_Add(TECA_assets
+            SVN_REPOSITORY svn://svn.code.sf.net/p/teca/TECA_assets
+            UPDATE_COMMAND svn update
+            CONFIGURE_COMMAND "" BUILD_COMMAND ""
+            INSTALL_COMMAND "" LOG_DOWNLOAD 1)
+        ExternalProject_Get_property(TECA_assets SOURCE_DIR)
+        set(TECA_ASSETS_ROOT ${SOURCE_DIR})
+        message(STATUS "TECA_assets -- populating content at ${TECA_ASSETS_ROOT}")
+    endif()
+else()
+    message(STATUS "TECA_assets -- not available")
+endif()
+set(TECA_HAS_ASSETS ${tmp} CACHE BOOL "TECA_assets is present")
+
 # generate global configuration
 include(CMake/teca_test.cmake)
 include(CMake/teca_app.cmake)

doc/rtd/applications.rst

@@ -484,7 +484,7 @@ Example
 .. code-block:: bash
 
     mpiexec -np 10 ./bin/teca_tc_trajectory_scalars   \
-        --texture ../../TECA_data/earthmap4k.png      \
+        --texture ../../TECA_assets/earthmap4k.png      \
         tracks_1990s_3hr_mdd_4800.bin                 \
         traj_scalars_1990s_3hr_mdd_4800
 

doc/teca_users_guide.tex

@@ -251,6 +251,7 @@ \subsubsection{Installing TECA}
     -DCMAKE_INSTALL_PREFIX=${TECA_INSTALL_DIR} \
     -DBUILD_TESTING=ON \
     -DTECA_DATA_ROOT=${TECA_DATA_DIR} \
+    -DTECA_ASSETS_ROOT=${TECA_ASSETS_DIR} \
     ${TECA_SOURCE_DIR}
 
 make -j4 && make -j4 install
@@ -270,6 +271,7 @@ \subsubsection{Installing TECA}
     -DCMAKE_INSTALL_PREFIX=${TECA_INSTALL_DIR} \
     -DBUILD_TESTING=ON \
     -DTECA_DATA_ROOT=${TECA_DATA_DIR} \
+    -DTECA_ASSETS_ROOT=${TECA_ASSETS_DIR} \
     ${TECA_SOURCE_DIR}
 
 make -j4 && make -j4 install
@@ -710,7 +712,7 @@ \subsection{Command Line Arguments}
 \subsection{Example}
 \begin{minted}{bash}
 mpiexec -np 10 ./bin/teca_tc_trajectory_scalars   \
-    --texture ../../TECA_data/earthmap4k.png      \
+    --texture ../../TECA_assets/earthmap4k.png      \
     tracks_1990s_3hr_mdd_4800.bin                 \
     traj_scalars_1990s_3hr_mdd_4800
 \end{minted}

python/teca_py_config.i

@@ -42,6 +42,27 @@ const char *get_teca_data_root()
 #endif
 }
 
+// return true if the location of TECA_assets repo is known
+bool get_teca_has_assets()
+{
+#if defined(TECA_HAS_ASSETS)
+    return true;
+#else
+    return false;
+#endif
+}
+
+// return the path to a TECA_assets repo check out, or None if the
+// path is not known at compile time
+const char *get_teca_assets_root()
+{
+#if defined(TECA_HAS_ASSETS)
+    return TECA_ASSETS_ROOT;
+#else
+    return nullptr;
+#endif
+}
+
 // return true if TECA was compiled with regex support
 bool get_teca_has_regex()
 {

teca_config.h.in

@@ -15,11 +15,13 @@
 
 #cmakedefine TECA_VERSION_DESCR "@TECA_VERSION_DESCR@"
 #cmakedefine TECA_PYTHON_VERSION @TECA_PYTHON_VERSION@
-#cmakedefine TECA_DATA_ROOT "@TECA_DATA_ROOT@"
 
 #cmakedefine TECA_ENABLE_PROFILER
 
 #cmakedefine TECA_HAS_DATA
 #cmakedefine TECA_DATA_ROOT "@TECA_DATA_ROOT@"
 
+#cmakedefine TECA_HAS_ASSETS
+#cmakedefine TECA_ASSETS_ROOT "@TECA_ASSETS_ROOT@"
+
 #endif

test/apps/CMakeLists.txt

@@ -7,15 +7,16 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}
 
 teca_add_test(test_deeplabv3p_ar_detect_app_serial
     COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect_app.sh
-    ${CMAKE_BINARY_DIR}/${BIN_PREFIX} ${TECA_DATA_ROOT}
+    ${CMAKE_BINARY_DIR}/${BIN_PREFIX} ${TECA_DATA_ROOT} ${TECA_ASSETS_ROOT}
     FEATURES ${TECA_HAS_NETCDF}
-    REQ_TECA_DATA)
+    REQ_TECA_DATA REQ_TECA_ASSETS)
 
 teca_add_test(test_deeplabv3p_ar_detect_app_mpi
     COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect_app.sh
-    ${CMAKE_BINARY_DIR}/${BIN_PREFIX} ${TECA_DATA_ROOT} ${MPIEXEC} 2
+    ${CMAKE_BINARY_DIR}/${BIN_PREFIX} ${TECA_DATA_ROOT}
+    ${TECA_ASSETS_ROOT} ${MPIEXEC} 2
     FEATURES ${TECA_HAS_NETCDF_MPI} ${TECA_HAS_MPI} ${MPI4Py_FOUND}
-    REQ_TECA_DATA)
+    REQ_TECA_DATA REQ_TECA_ASSETS)
 
 teca_add_test(test_bayesian_ar_detect_app_serial
     COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test_bayesian_ar_detect_app.sh

test/apps/test_deeplabv3p_ar_detect_app.sh

@@ -2,32 +2,33 @@
 
 if [[ $# < 2 ]]
 then
-    echo "usage: test_deeplabv3p_ar_detect_app.sh [app prefix] " \
-         "[data root] [mpi exec] [test cores]"
+    echo "usage: test_deeplabv3p_ar_detect_app.sh [app prefix] "  \
+         "[data root] [assets root] [mpi exec] [test cores]"
     exit -1
 fi
 
 app_prefix=${1}
 data_root=${2}
+assets_root=${3}
 
-if [[ $# -eq 4 ]]
+if [[ $# -eq 5 ]]
 then
-    mpi_exec=${3}
-    test_cores=${4}
+    mpi_exec=${4}
+    test_cores=${5}
     launcher="${mpi_exec} -n ${test_cores}"
 fi
 
 set -x
 
 # run the app
-${launcher} ${app_prefix}/teca_deeplabv3p_ar_detect              \
-    --input_regex "${data_root}/ARTMIP_MERRA_2D_2017-05.*\.nc$"  \
-    --pytorch_deeplab_model ${data_root}/cascade_deeplab_IVT.pt  \
+${launcher} ${app_prefix}/teca_deeplabv3p_ar_detect               \
+    --input_regex "${data_root}/ARTMIP_MERRA_2D_2017-05.*\.nc$"   \
+    --pytorch_deeplab_model ${assets_root}/cascade_deeplab_IVT.pt \
     --output_file test_deeplabv3p_ar_detect_app_output_%t%.nc
 
 # run the diff
-${app_prefix}/teca_cartesian_mesh_diff                           \
-    "test_deeplabv3p_ar_detect_app_output.*\.nc"                 \
+${app_prefix}/teca_cartesian_mesh_diff                            \
+    "test_deeplabv3p_ar_detect_app_output.*\.nc"                  \
     "${data_root}/test_deeplabv3p_ar_detect_app_ref.nc"
 
 # clean up

test/python/CMakeLists.txt

@@ -309,7 +309,7 @@ teca_add_test(py_test_tc_activity
 teca_add_test(py_test_tc_trajectory_scalars_serial
     COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_trajectory_scalars.py
     "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin"
-    "${TECA_DATA_ROOT}/earthmap4k.png"
+    "${TECA_ASSETS_ROOT}/earthmap4k.png"
     "${TECA_DATA_ROOT}/py_test_tc_trajectory_scalars.bin" 0 -1
     FEATURES ${TECA_HAS_UDUNITS}
     REQ_TECA_DATA)
@@ -318,7 +318,7 @@ teca_add_test(py_test_tc_trajectory_scalars_mpi
     COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE}
     ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_trajectory_scalars.py
     "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin"
-    "${TECA_DATA_ROOT}/earthmap4k.png"
+    "${TECA_ASSETS_ROOT}/earthmap4k.png"
     "${TECA_DATA_ROOT}/py_test_tc_trajectory_scalars.bin" 0 -1
     FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4Py_FOUND}
     REQ_TECA_DATA)
@@ -356,28 +356,28 @@ teca_add_test(py_test_bayesian_ar_detect_mpi_threads
 
 teca_add_test(py_test_deeplabv3p_ar_detect_serial
     COMMAND  ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
-    "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt"
+    "${TECA_ASSETS_ROOT}/cascade_deeplab_IVT.pt"
     "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D_2017-05.*\.nc$"
     "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect" IVT 1
     FEATURES ${TECA_HAS_NETCDF}
-    REQ_TECA_DATA)
+    REQ_TECA_DATA REQ_TECA_ASSETS)
 
 teca_add_test(py_test_deeplabv3p_ar_detect_threads
     COMMAND  ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
-    "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt"
+    "${TECA_ASSETS_ROOT}/cascade_deeplab_IVT.pt"
     "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D_2017-05.*\.nc$"
     "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect" IVT -1
     FEATURES ${TECA_HAS_NETCDF}
-    REQ_TECA_DATA)
+    REQ_TECA_DATA REQ_TECA_ASSETS)
 
 teca_add_test(py_test_deeplabv3p_ar_detect_mpi_threads
     COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE}
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
-        "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt"
-        "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D_2017-05.*\.nc$"
-        "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect" IVT -1
+    ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
+    "${TECA_ASSETS_ROOT}/cascade_deeplab_IVT.pt"
+    "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D_2017-05.*\.nc$"
+    "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect" IVT -1
     FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} ${MPI4Py_FOUND}
-    REQ_TECA_DATA)
+    REQ_TECA_DATA REQ_TECA_ASSETS)
 
 teca_add_test(py_test_binary_stream
     COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE}

test/python/test_config.py

@@ -13,3 +13,5 @@
 sys.stderr.write('TECA_HAS_OPENSSL=%s\n'%(str(get_teca_has_openssl())))
 sys.stderr.write('TECA_HAS_DATA=%s\n'%(str(get_teca_has_data())))
 sys.stderr.write('TECA_DATA_ROOT=%s\n'%(get_teca_data_root()))
+sys.stderr.write('TECA_HAS_ASSETS=%s\n'%(str(get_teca_has_assets())))
+sys.stderr.write('TECA_ASSETS_ROOT=%s\n'%(get_teca_assets_root()))

test/travis_ci/ctest_linux.cmake

@@ -20,6 +20,7 @@ BUILD_TESTING=ON
 TECA_ENABLE_PROFILER=ON
 TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION}
 TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data
+TECA_ASSETS_ROOT=$ENV{DASHROOT}/TECA_assets
 TECA_TEST_CORES=2
 REQUIRE_OPENSSL=TRUE
 REQUIRE_BOOST=TRUE

test/travis_ci/ctest_osx.cmake

@@ -20,6 +20,7 @@ BUILD_TESTING=ON
 TECA_ENABLE_PROFILER=ON
 TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION}
 TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data
+TECA_ASSETS_ROOT=$ENV{DASHROOT}/TECA_assets
 TECA_TEST_CORES=2
 REQUIRE_OPENSSL=TRUE
 OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1

test/travis_ci/install_fedora_31.sh

@@ -37,3 +37,4 @@ pip3 install numpy mpi4py matplotlib torch
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets

test/travis_ci/install_osx.sh

@@ -34,3 +34,12 @@ do
   echo -n "."
   sleep 2s
 done
+
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets &
+svn_pid=$!
+
+while [ -n "$(ps -p $svn_pid -o pid=)" ]
+do
+  echo -n "."
+  sleep 2s
+done

test/travis_ci/install_ubuntu_14_04.sh

@@ -23,3 +23,4 @@ sudo tar -C /usr -x -z -f cmake-3.5.2-Linux-x86_64.tar.gz --strip-components=1
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets

test/travis_ci/install_ubuntu_20_04.sh

@@ -45,3 +45,4 @@ pip3 install numpy mpi4py matplotlib torch
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets