Updates to the way that the bundle script (and friends) find integtests

scripts/dst_set_integtest_env_vars.sh

@@ -0,0 +1,62 @@
+# This script needs to be sourced.
+#
+# Its purpose is to set values for various environmental variables that are
+# useful for looking up integration tests in our C++ and Python repositories.
+
+# check if a software environment has been set up
+if [[ "$DBT_AREA_ROOT" == "" ]]; then
+    echo
+    echo "Please set up a valid DUNE-DAQ software area before running this script."
+    return
+fi
+
+# check if this script has already been run, and exit early if yes
+if [[ "${DST_DAR_COPY}" != "" ]] && \
+   [[ "${DST_DAR_COPY}" == "${DBT_AREA_ROOT}" ]]; then
+    return
+fi
+export DST_DAR_COPY="${DBT_AREA_ROOT}"
+
+# default all of the env vars that we might set to empty strings
+export DST_LOCAL_CPP_SOURCE_DIR=""
+export DST_LOCAL_PYTHON_SOURCE_DIR=""
+export DST_LOCAL_PYTHON_VENV_DIR=""
+export DST_LOCAL_SPACK_INSTALL_DIR=""
+export DST_BASEREL_CPP_SOURCE_DIR=""
+export DST_BASEREL_PYTHON_VENV_DIR=""
+export DST_CORE_BASEREL_SPACK_DIR=""
+export DST_DETSPEC_BASEREL_SPACK_DIR=""
+
+if [[ "${DBT_AREA_ROOT}" != "" ]] && [[ "`echo ${DBT_AREA_ROOT} | grep '^/cvmfs'`" == "" ]] && \
+   [[ -w "${DBT_AREA_ROOT}" ]] && [[ -w "${DBT_AREA_ROOT}/sourcecode" ]]; then
+    export DST_LOCAL_CPP_SOURCE_DIR="${DBT_AREA_ROOT}/sourcecode"
+fi
+
+if [[ "${DBT_AREA_ROOT}" != "" ]] && [[ "`echo ${DBT_AREA_ROOT} | grep '^/cvmfs'`" == "" ]] && \
+   [[ -w "${DBT_AREA_ROOT}" ]] && [[ -w "${DBT_AREA_ROOT}/pythoncode" ]]; then
+    export DST_LOCAL_PYTHON_SOURCE_DIR="${DBT_AREA_ROOT}/pythoncode"
+fi
+
+if [[ "${DBT_AREA_ROOT}" != "" ]] && [[ "`echo ${DBT_AREA_ROOT} | grep '^/cvmfs'`" == "" ]] && \
+   [[ -w "${DBT_AREA_ROOT}" ]] && [[ -w "${DBT_AREA_ROOT}/.venv" ]]; then
+    export DST_LOCAL_PYTHON_VENV_DIR="`ls -1d ${DBT_AREA_ROOT}/.venv/lib/python*/site-packages 2>/dev/null | head -1`"
+fi
+
+if [[ "${DBT_AREA_ROOT}" != "" ]] && [[ "`echo ${DBT_AREA_ROOT} | grep '^/cvmfs'`" == "" ]] && \
+   [[ -w "${DBT_AREA_ROOT}" ]] && [[ -w "${DBT_AREA_ROOT}/install" ]]; then
+    export DST_LOCAL_SPACK_INSTALL_DIR="${DBT_AREA_ROOT}/install"
+fi
+
+dbt_info_output=`dbt-info release`
+detector_specific_release_dir=`echo "${dbt_info_output}" | grep 'Release dir' | awk '{print $3}'`
+detector_specific_release_name=`echo "${dbt_info_output}" | grep 'Release name:' | awk '{print $3}'`
+core_release_name=`echo "${dbt_info_output}" | grep 'Base release name:' | awk '{print $4}'`
+core_release_dir=`echo "${detector_specific_release_dir}" | sed "s/${detector_specific_release_name}/${core_release_name}/g"`
+
+export DST_BASEREL_CPP_SOURCE_DIR="${detector_specific_release_dir}/sourcecode"
+
+export DST_BASEREL_PYTHON_VENV_DIR="`ls -1d ${detector_specific_release_dir}/.venv/lib/python*/site-packages 2>/dev/null | head -1`"
+
+export DST_CORE_BASEREL_SPACK_DIR="`ls -1d ${core_release_dir}/spack-installation/opt/spack/linux-*/gcc-* 2>/dev/null | head -1`"
+
+export DST_DETSPEC_BASEREL_SPACK_DIR="`ls -1d ${detector_specific_release_dir}/spack-installation/opt/spack/linux-*/gcc-* 2>/dev/null | head -1`"

scripts/dunedaq_integtest_bundle.sh

@@ -185,6 +185,7 @@ let random_subset_count=0
 only_list_tests=""
 PYTEST_BASE_COMMAND=(pytest -s --tb=short)  # our core pytest command, with DAQ printout included and short pytest traceback
 PYTEST_OPTIONS=()
+BASEREL_PYTEST_SUMMARY_CHOICE="--no-summary"
 
 while true; do
     case "$1" in
@@ -311,6 +312,7 @@ while true; do
             if [[ $level -ge 6 ]]; then
                 # enable printout of Pytest 'skip' reasons and turn on drunc debugging
                 PYTEST_OPTIONS+=(-rs --dunerc-option log-level debug)
+                BASEREL_PYTEST_SUMMARY_CHOICE=""
             fi
             shift 2
             ;;
@@ -392,6 +394,7 @@ else
 fi
 
 # determine the list of tests
+source dst_set_integtest_env_vars.sh
 if [[ "${excluded_repo_names}" == "" ]]; then
     initial_integtest_list=(`list_available_integtests.sh ${requested_repo_list[@]} 2>/dev/null`)
 else
@@ -534,43 +537,74 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do
                 PYTEST_COMMAND+=("--junit-xml=${test_repo}_${test_name%.py}_results.xml" "--")
             fi
 
-            # First, check if the test is found in the Python virtual environment.
-            # This picks up tests from our Python-only software packages.
-            if [[ "`ls ${DBT_AREA_ROOT}/.venv/lib/python*/site-packages/${test_repo}/integtest/${test_name} 2>/dev/null`" != "" ]]; then
-                PYTEST_COMMAND+=(${DBT_AREA_ROOT}/.venv/lib/python*/site-packages/${test_repo}/integtest/${test_name})
-                "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
-
-            # Next, check if the test exists in the current working directory.
-            # This is a convenience for developers when they are working on an integtest
-            # in a C++ package (the test is found without rebuilding the software).
-            elif [[ -e "./${test_name}" ]]; then
-                PYTEST_COMMAND+=(./${test_name})
-                "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
-
-            # Next, check if the test exists in the local software area.
-            elif [[ -e "${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name}" ]]; then
-                if [[ -w "${DBT_AREA_ROOT}" ]]; then
-                    PYTEST_COMMAND+=(${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name})
+            # determine the repository type, to help determine where to look for the test
+            repo_type="`get_repo_type.sh ${test_repo}`"
+
+            # handle tests in Python repositories
+            if [[ "${repo_type}" == "Python" ]]; then
+
+                if [[ "${DST_LOCAL_PYTHON_VENV_DIR}" != "" ]]; then
+                    if [[ -e "${DST_LOCAL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name}" ]] && \
+                       [[ -e "${DST_LOCAL_PYTHON_SOURCE_DIR}/${test_repo}/src/${test_repo}/integtest/${test_name}" ]] && \
+                       [[ "`diff ${DST_LOCAL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name} ${DST_LOCAL_PYTHON_SOURCE_DIR}/${test_repo}/src/${test_repo}/integtest/${test_name}`" != "" ]]; then
+                        echo ""
+                        echo -e "\U1f7e1 WARNING: The version of ${test_name} in" | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: ${DST_LOCAL_PYTHON_SOURCE_DIR}/${test_repo}/src/${test_repo}/integtest" | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: does not match the version found in the local Python virtual environment (.venv dir)." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: This can happen when the Python package needs to be (re?)installed locally." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: Please try re-installing ${test_repo} (e.g. 'pip install .') or re-running 'dbt-build'." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                    elif [[ -e "${DST_LOCAL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name}" ]]; then
+                        PYTEST_COMMAND+=(${DST_LOCAL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name})
+                        "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
+                    else
+                        echo ""
+                        echo -e "\U1f7e1 WARNING: ${test_name} was not found in the local Python virtual environment (.venv dir)." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: This can happen when the Python package needs to be (re?)installed locally, or when" | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: it has been installed with the '-e' option. Please try re-installing the ${test_repo}" | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                        echo -e "\U1f7e1 WARNING: package without the '-e' option (e.g. 'pip install .') or re-running 'dbt-build'." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                    fi
+
+                elif [[ "${DST_BASEREL_PYTHON_VENV_DIR}" != "" ]] && \
+                     [[ -e "${DST_BASEREL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name}" ]]; then
+                    # remove any trailing "--" in PYTEST_COMMAND since we are adding more pytest options here
+                    if [[ "${PYTEST_COMMAND[-1]}" == "--" ]]; then
+                        unset 'PYTEST_COMMAND[-1]'
+                    fi
+                    PYTEST_COMMAND+=(-p no:cacheprovider ${BASEREL_PYTEST_SUMMARY_CHOICE} ${DST_BASEREL_PYTHON_VENV_DIR}/${test_repo}/integtest/${test_name})
                     "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
+
                 else
+                    echo ""
+                    echo -e "\U0001F534 ERROR: Unable to find ${test_name} in the ${test_repo} (Python) repo." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                    echo -e "\U0001F534 ERROR: This should not have happened. Please contact daqsystemtest developers." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                fi
+
+            # handle tests in C++ repositories
+            elif [[ "${repo_type}" == "CPP" ]]; then
+
+                if [[ "${DST_LOCAL_CPP_SOURCE_DIR}" != "" ]] && \
+                   [[ -e "${DST_LOCAL_CPP_SOURCE_DIR}/${test_repo}/integtest/${test_name}" ]]; then
+                    PYTEST_COMMAND+=(${DST_LOCAL_CPP_SOURCE_DIR}/${test_repo}/integtest/${test_name})
+                    "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
+                elif [[ "${DST_BASEREL_CPP_SOURCE_DIR}" != "" ]] && \
+                     [[ -e "${DST_BASEREL_CPP_SOURCE_DIR}/${test_repo}/integtest/${test_name}" ]]; then
                     # remove any trailing "--" in PYTEST_COMMAND since we are adding more pytest options here
                     if [[ "${PYTEST_COMMAND[-1]}" == "--" ]]; then
                         unset 'PYTEST_COMMAND[-1]'
                     fi
-                    PYTEST_COMMAND+=(-p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name})
+                    PYTEST_COMMAND+=(-p no:cacheprovider ${BASEREL_PYTEST_SUMMARY_CHOICE} ${DST_BASEREL_CPP_SOURCE_DIR}/${test_repo}/integtest/${test_name})
                     "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
+
+                else
+                    echo ""
+                    echo -e "\U0001F534 ERROR: Unable to find ${test_name} in the ${test_repo} (C++) repo." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                    echo -e "\U0001F534 ERROR: This should not have happened. Please contact daqsystemtest developers." | CaptureOutput ${ITGRUNNER_LOG_FILE}
                 fi
 
-            # Lastly, we assume that the test can be found in the installed software
-            # area (for C++ packages).
             else
-                share_envvar_name="${test_repo^^}_SHARE"  # double caret converts env var to uppercase
-                # remove any trailing "--" in PYTEST_COMMAND since we are adding more pytest options here
-                if [[ "${PYTEST_COMMAND[-1]}" == "--" ]]; then
-                    unset 'PYTEST_COMMAND[-1]'
-                fi
-                PYTEST_COMMAND+=(-p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name})
-                "${PYTEST_COMMAND[@]}" | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE}
+                echo ""
+                echo -e "\U0001F534 ERROR: Unexpected repo type for the '${test_repo}' repo: \"${repo_type}\"." | CaptureOutput ${ITGRUNNER_LOG_FILE}
+                echo -e "\U0001F534 ERROR: This should not have happened. Please contact daqsystemtest developers." | CaptureOutput ${ITGRUNNER_LOG_FILE}
             fi
             let pytest_return_code=${PIPESTATUS[0]}
 
@@ -598,8 +632,8 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do
                             script_pid=`jq -r .bundle_script_process_id ${info_file}`
                             individual_test_start_time=`jq -r .individual_test_start_time ${info_file}`
                             if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \
-                                   [[ ${script_pid} -eq ${CURRENT_PID} ]] && \
-                                   [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then
+                               [[ ${script_pid} -eq ${CURRENT_PID} ]] && \
+                               [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then
                                 current_pytest_rundir=$info_file
                                 break
                             fi
@@ -617,13 +651,24 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do
                                     echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535"
                                     echo -e "\U1F535 Please note that copied directories from failed tests typically get cleaned up after 26 hours, \U1F535"
                                     echo -e "\U1F535 or when 10 newer failures happen, whichever comes first. \U1F535"
-                                    cp -pR "${pytest_tmpdir}" "${new_dir}"
+                                    # 15-Oct-2026, KAB: added use of rsync with a size limit to avoid
+                                    # copying large files (e.g. large HDF5 data files)
+                                    if [[ "`which rsync 2>/dev/null`" != "" ]]; then
+                                        rsync -a --max-size=200m "${pytest_tmpdir}/" "${new_dir}"
+                                    else
+                                        cp -pR "${pytest_tmpdir}" "${new_dir}"
+                                    fi
                                     if [[ $? == 0 ]]; then
                                         was_successfully_copied="yes"
                                         # 18-Dec-2025, KAB: added the removal of the "current" symbolic links
                                         # from inside the copied directory (since they get broken in the copying)
                                         rm -f "${new_dir}/configcurrent"
                                         rm -f "${new_dir}/runcurrent"
+                                        # 15-Oct-2026, KAB: set permissions so that others can see files from failed tests
+                                        chmod go+rx ${new_dir}
+                                        chmod go+rx ${new_dir}/run*
+                                        chmod go+rx ${new_dir}/config*
+                                        chmod go+rx ${new_dir}/..
                                     fi
                                 fi
                             fi
@@ -680,8 +725,8 @@ echo ""                                                   | CaptureOutput ${ITGR
 date                                                      | CaptureOutput ${ITGRUNNER_LOG_FILE}
 echo "Log file is: ${ITGRUNNER_LOG_FILE}"                 | CaptureOutput ${ITGRUNNER_LOG_FILE}
 echo ""                                                   | CaptureOutput ${ITGRUNNER_LOG_FILE}
-summary_string="`egrep $'=====|\u2B95' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running'`"
-colorized_summary_string="`echo \"${summary_string}\" | sed 's/passed/passed \\\\U2705/' | sed 's/failed/failed \\\\U274c/' | sed 's/\(errors\?\)/\1 \\\\U1F6A8/' | sed 's/no tests ran/no tests ran \\\\U1F6A8/' | sed 's/skipped/skipped \\\\U1f7e1/'`"
+summary_string="`egrep $'=====|\u2B95|\U1f7e1 WARNING|\U0001F534 ERROR' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running|WARNING|ERROR'`"
+colorized_summary_string="`echo \"${summary_string}\" | sed 's/passed/passed \\\\U2705/' | sed 's/failed/failed \\\\U274c/' | sed 's/warning/warning \\\\U274c/' | sed 's/\(errors\?\)/\1 \\\\U1F6A8/' | sed 's/no tests ran/no tests ran \\\\U1F6A8/' | sed 's/skipped/skipped \\\\U1f7e1/'`"
 echo -e "${colorized_summary_string}" | CaptureOutput ${ITGRUNNER_LOG_FILE}
 
 # check again if the numad daemon is running

scripts/get_repo_type.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+if [[ $# -eq 0 ]] || [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then
+    echo
+    echo "Usage: `basename $0` <repo name>"
+    echo
+    echo "  Determines the type of the specified repo, \"CPP\" or \"Python\"."
+    echo "  Prints out \"Unknown\" if the type can't be determined."
+    echo
+    exit
+fi
+repo_name=$1
+
+# check if a software environment has been set up
+if [[ "$DBT_AREA_ROOT" == "" ]]; then
+    echo
+    echo "Please set up a valid DUNE-DAQ software area before running this script."
+    exit
+fi
+
+source dst_set_integtest_env_vars.sh
+
+if [[ "${DST_LOCAL_SPACK_INSTALL_DIR}" != "" ]] && \
+   [[ -e "${DST_LOCAL_SPACK_INSTALL_DIR}/${repo_name}" ]]; then
+    echo "CPP"
+elif [[ "${DST_CORE_BASEREL_SPACK_DIR}" != "" ]] && \
+     [[ "`ls -1d ${DST_CORE_BASEREL_SPACK_DIR}/${repo_name}* 2>/dev/null | head -1`" != "" ]]; then
+    echo "CPP"
+elif [[ "${DST_DETSPEC_BASEREL_SPACK_DIR}" != "" ]] && \
+     [[ "`ls -1d ${DST_DETSPEC_BASEREL_SPACK_DIR}/${repo_name}* 2>/dev/null | head -1`" != "" ]]; then
+    echo "CPP"
+elif [[ "${DST_LOCAL_PYTHON_VENV_DIR}" != "" ]] && \
+     [[ -e ${DST_LOCAL_PYTHON_VENV_DIR}/${repo_name} ]]; then
+    echo "Python"
+elif [[ "${DST_BASEREL_PYTHON_VENV_DIR}" != "" ]] && \
+     [[ -e ${DST_BASEREL_PYTHON_VENV_DIR}/${repo_name} ]]; then
+    echo "Python"
+else
+    echo "Unknown"
+fi