EESSI-install-software.sh

#!/bin/bash
#
# Script to install EESSI software stack (version set through init/eessi_defaults)

# see example parsing of command line arguments at
#   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
#   https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash

display_help() {
  echo "usage: $0 [OPTIONS]"
  echo "  --build-logs-dir       -  location to copy EasyBuild logs to for failed builds"
  echo "  -g | --generic         -  instructs script to build for generic architecture target"
  echo "  -h | --help            -  display this usage information"
  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
  echo "  --shared-fs-path       -  path to directory on shared filesystem that can be used"
  echo "  --skip-cuda-install    -  disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)"
}

function safe_module_use {
    # add a given non-empty directory to $MODULEPATH if and only if it is not yet in
    directory=${1}

    if [[ -z ${directory+x} ]]; then
        echo "safe_module_use: given directory unset or empty; not adding it to \$MODULEPATH (${MODULEPATH})"
        return
    fi
    if [[ ":${MODULEPATH}:" == *":${directory}:"* ]]; then
        echo "safe_module_use: directory '${directory}' is already in \$MODULEPATH (${MODULEPATH}); not adding it again"
        return
    else
        echo "safe_module_use: directory '${directory}' is not yet in \$MODULEPATH (${MODULEPATH}); adding it"
        module use ${directory}
    fi
}

POSITIONAL_ARGS=()

while [[ $# -gt 0 ]]; do
  case $1 in
    -g|--generic)
      EASYBUILD_OPTARCH="GENERIC"
      shift
      ;;
    -h|--help)
      display_help  # Call your function
      # no shifting needed here, we're done.
      exit 0
      ;;
    -x|--http-proxy)
      export http_proxy="$2"
      shift 2
      ;;
    -y|--https-proxy)
      export https_proxy="$2"
      shift 2
      ;;
    --build-logs-dir)
      export build_logs_dir="${2}"
      shift 2
      ;;
    --shared-fs-path)
      export shared_fs_path="${2}"
      shift 2
      ;;
    --skip-cuda-install)
      export skip_cuda_install=True
      shift 1
      ;;
    -*|--*)
      echo "Error: Unknown option: $1" >&2
      exit 1
      ;;
    *)  # No more options
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift
      ;;
  esac
done

set -- "${POSITIONAL_ARGS[@]}"

TOPDIR=$(dirname $(realpath $0))

source $TOPDIR/scripts/utils.sh

# Debugging aid, only active if EESSI_INTERACTIVE_BUILD is set (see bot/build.sh): start an interactive shell
# in the current environment. We are running inside 'startprefix' with stdin consumed by a here-string, so we
# talk to /dev/tty directly. The new shell inherits all exported variables (Prefix, Lmod state, ...); we
# additionally dump the non-exported variables, functions, aliases and shell options of this script into an
# rcfile, so that e.g. ${EB} and our helper functions are available too.
debug_shell() {
    local reason="$1"
    local rcfile
    rcfile=$(mktemp "${TMPDIR:-/tmp}/debug_shell_rc.XXXXXX")
    {
        # 'grep -v' filters out read-only variables (e.g. BASHOPTS, EUID), as trying to load those in a debug shell
        # would cause errors. If grep -v returns a non-zero output (i.e. the only variables are read-only variables)
        # the '|| true' ensures this still passes if 'set -e' is ever set
        declare -p 2>/dev/null | grep -v '^declare -[a-zA-Z-]*r[a-zA-Z-]* ' || true
        declare -f
        alias -p
        shopt -p
        # the debugging shell should not abort on errors, nor re-trigger our hooks
        echo "set +e +u; trap - EXIT"
        echo "PS1='[EESSI debug] \\w \\$ '"
        echo "cd '${PWD}'"
    } > "${rcfile}" 2>/dev/null
    echo_yellow ">> Dropping into an interactive debug shell: ${reason}"
    echo_yellow ">> Working directory: ${PWD}; exit the shell (Ctrl-D) to continue/finish"
    # Run the shell in a pty of its own (using 'script'), not directly on our terminal. An interactive bash takes over
    # the terminal (it makes its own process group the foreground group) and hands it back to our group when it exits.
    # This resolves issues with Ctrl+C no longer working after the debug shell has been exited (which was the case 
    # without 'script' / a pty).
    # See https://github.com/EESSI/software-layer-scripts/pull/321#issuecomment-6267183643 and
    # https://github.com/EESSI/software-layer-scripts/pull/321#issuecomment-6267991021 for details.
    # If script is ever no longer available from the compatibility layer, the same can be achieved with python3, see
    # https://github.com/EESSI/software-layer-scripts/pull/321/changes/8d617c7c44cbf51ccdb3c801d36c86c85bdfcbfc
    script -qc "bash --rcfile $(printf '%q' "${rcfile}") -i" /dev/null < /dev/tty > /dev/tty 2>&1
    rm -f "${rcfile}"
}

# The debug shell needs 'script', check this now rather than when we want the shell (possibly after a long build). This
# is done before defining the trap below, so that failing here does not try to start a debug shell.
if [[ -n "${EESSI_INTERACTIVE_BUILD}" ]] && ! command -v script > /dev/null; then
    fatal_error "EESSI_INTERACTIVE_BUILD=${EESSI_INTERACTIVE_BUILD} needs the 'script' command to start a debug shell, but it was not found. It is part of util-linux, and is normally available in the EESSI compatibility layer."
fi

# If EESSI_INTERACTIVE_BUILD=onfail, then define a trap so that if an error is encounted that would cause the shell to exit
# (e.g. fatal_error) we get dropped into the debug_shell as well. That will allow us to not only debug EasyBuild build 
# failures interactively, but anything resulting in fatal_error as well.
if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then
    trap 'ec_exit=$?; if [[ ${ec_exit} -ne 0 ]]; then debug_shell "script exiting with exit code ${ec_exit}"; fi' EXIT
fi

# honor $TMPDIR if it is already defined, use /tmp otherwise
if [[ -z $TMPDIR ]]; then
    export WORKDIR=/tmp/$USER
else
    export WORKDIR=$TMPDIR/$USER
fi

TMPDIR=$(mktemp -d)


# Get override subdir
DETECTION_PARAMETERS=''
GENERIC=0
EB='eb'
if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
    echo_yellow ">> GENERIC build requested, taking appropriate measures!"
    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
    GENERIC=1
    EB='eb --optarch=GENERIC'
fi

echo ">> Determining software subdirectory to use for current build host..."
if [[ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]]; then
  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
else
  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
  # Run in a subshell, so that minimal_eessi_env doesn't change the shell environment for the rest of this script
  (
      # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set
      source $TOPDIR/init/minimal_eessi_env

      # We've found that creation of new files and directories by unionfs (or any overlay fs) can fail if it
      # runs on top of CVMFS because the lower-dir provided by CVMFS is not fully initialized. It seems
      # these issues can be avoided simply by running an ls, to at least trigger the mount
      echo ">> Trigger automounting of the repo ($EESSI_CVMFS_REPO) before creating new files, to avoid overlay issues"
      ls -al $EESSI_CVMFS_REPO

      # make sure the the software and modules directory exist
      # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
      if [[ ! -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then
          if [[ ! -z "${EESSI_SITE_SOFTWARE_PREFIX}" ]]; then
              mkdir -p -v ${EESSI_SITE_SOFTWARE_PREFIX}/versions/${EESSI_VERSION_OVERRIDE:-${EESSI_VERSION}}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
          else
              fatal_error 'Requested a site installation, but $EESSI_SITE_SOFTWARE_PREFIX is somehow not set?! This should have been set by bot/build.sh.'
          fi
      else
          mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
      fi

      # If EESSI_ACCELERATOR_TARGET_OVERRIDE is defined, we are building for an accelerator target
      # In that case, make sure the modulepath for the accelerator subdir exists, otherwise the EESSI module will not
      # set EESSI_ACCELERATOR_TARGET and the if-condition later in this script which checks if EESSI_ACCELERATOR_TARGET
      # is equal to EESSI_ACCELERATOR_TARGET_OVERRIDE will fail 
      # See https://github.com/EESSI/software-layer-scripts/pull/59#issuecomment-3173593882
      if [[ -n $EESSI_ACCELERATOR_TARGET_OVERRIDE ]]; then
          # Note that ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
          # is only the correct path if EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE is not set
          if [[ -z $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE ]]; then
              if [[ ! -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then
                  mkdir -p -v ${EESSI_SITE_SOFTWARE_PREFIX}/versions/${EESSI_VERSION_OVERRIDE:-${EESSI_VERSION}}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
              else
                  mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
              fi
          else
              # At runtime, one might want to use a different CPU subdir for a given accelerator. E.g. one could use
              # a zen2 CPU subdir on a zen4 node if the required GPU software isn't available in the zen4 tree.
              # At build time, this doesn't make a lot of sense: we'd probably build in a CPU prefix that is different
              # from what the code will be optimized for, and we wouldn't want that
              # So this message _should_ never be printed...
              msg="When building the software subdirectory for the CPU should almost certainly be that of the host."
              msg="$msg If you think this is incorrect, please implement behaviour that makes sense in "
              msg="$msg EESSI-software-installation.sh, essentially replacing this error."
              fatal_error "$msg"
          fi
      fi
  )
fi

echo ">> Setting up environment..."

# If EESSI_VERSION is not set, source the defaults script to set it
if [[ -z ${EESSI_VERSION} ]]; then
    source $TOPDIR/init/eessi_defaults
fi

# If module command does not exist, use the one from the compat layer
command -v module
module_cmd_exists=$?
if [[ "$module_cmd_exists" -ne 0 ]]; then
    echo_green "No module command found, initializing lmod from the compatibility layer"
    # Minimal initalization of the lmod from the compat layer
    source $TOPDIR/init/lmod/bash
else
    echo_green "Module command found"
fi
ml_version_out=$TMPDIR/ml.out
ml --version &> $ml_version_out
if [[ $? -eq 0 ]]; then
    echo_green ">> Found Lmod ${LMOD_VERSION}"
else
    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
fi

# install any additional required scripts
# order is important: these are needed to install a full CUDA SDK in host_injections
# for now, this just reinstalls all scripts. Note the most elegant, but works

# the install_scripts.sh script relies on knowing the location of the PR diff
# assume there's only one diff file that corresponds to the PR patch file
pr_diff=$(ls [0-9]*.diff | head -n 1)
export PR_DIFF="$PWD/$pr_diff"

# Only run install_scripts.sh for software.eessi.io builds,
# i.e. skip it for site-installs (which also includes builds for dev.eessi.io)
if [[ -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then
    ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION} --eessi-version ${EESSI_VERSION}
fi

# Make sure we start with no modules and clean $MODULEPATH
echo ">> Setting up \$MODULEPATH..."
module --force purge
module unuse $MODULEPATH

# Initialize the EESSI environment
# Unset EESSI_SITE_SOFTWARE_PREFIX initially (& back it up for later restore), so that things like the CUDA
# drivers and libraries are installed in /cvmfs/software.eessi.io, and not in the EESSI_SITE_SOFTWARE_PREFIX
EESSI_SITE_SOFTWARE_PREFIX_BACKUP=${EESSI_SITE_SOFTWARE_PREFIX}
unset EESSI_SITE_SOFTWARE_PREFIX
module use $TOPDIR/init/modules
module load EESSI/$EESSI_VERSION

# make sure we're in Prefix environment by checking $SHELL
# We can only do this after loading the EESSI module, as we need ${EPREFIX}
if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
else
    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
fi

if [[ -d $EESSI_CVMFS_REPO ]]; then
    echo_green "$EESSI_CVMFS_REPO available, OK!"
else
    fatal_error "$EESSI_CVMFS_REPO is not available!"
fi
if [[ -n "$EESSI_CVMFS_REPO_OVERRIDE" && "$EESSI_CVMFS_REPO" != "$EESSI_CVMFS_REPO_OVERRIDE" ]]; then
    if [[ -d "$EESSI_CVMFS_REPO_OVERRIDE" ]]; then
        echo_green "$EESSI_CVMFS_REPO_OVERRIDE available, OK!"
    else
        fatal_error "$EESSI_CVMFS_REPO_OVERRIDE is not available!"
    fi
fi

# Check that EESSI_SOFTWARE_SUBDIR now matches EESSI_SOFTWARE_SUBDIR_OVERRIDE
if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
    fatal_error "Failed to determine software subdirectory?!"
elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
else
    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
fi

# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
# force Python's stdout and stderr streams to be unbuffered
export PYTHONUNBUFFERED=1

# if we run the script for the first time, e.g., to start building for a new
#   stack, we need to ensure certain files are present in
#   ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
# - .lmod/lmodrc.lua
# - .lmod/SitePackage.lua
if [[ -n "${EESSI_SITE_INSTALL_FORCE}" ]]; then
    # note: we need to use EESSI_SITE_SOFTWARE_PREFIX_BACKUP here, as we have temporarily unset EESSI_SITE_SOFTWARE_PREFIX above
    _eessi_software_path=${EESSI_SITE_SOFTWARE_PREFIX_BACKUP}/versions/${EESSI_VERSION_OVERRIDE:-${EESSI_VERSION}}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
else
    _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
fi
_lmod_cfg_dir=${_eessi_software_path}/.lmod
_lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua
if [[ ! -f ${_lmod_rc_file} ]]; then
    echo "Lmod file '${_lmod_rc_file}' does not exist yet; creating it..."
    command -V python3
    python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path}
fi
_lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua
if [[ ! -f ${_lmod_sitepackage_file} && -z "${EESSI_SITE_INSTALL_FORCE}" ]]; then
    echo "Lmod file '${_lmod_sitepackage_file}' does not exist yet; creating it..."
    command -V python3
    python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path}
fi

echo ">> Configuring EasyBuild..."

# Make sure EESSI-extend is not loaded, and configure location variables for a
#   CVMFS installation
module unload EESSI-extend
unset EESSI_USER_INSTALL
unset EESSI_PROJECT_INSTALL
if [[ -n "$EESSI_SITE_INSTALL_FORCE" ]]; then
    msg="Forcing a site installation (EESSI_SITE_INSTALL_FORCE=${EESSI_SITE_INSTALL_FORCE})"
    msg="$msg by setting EESSI_SITE_INSTALL=1 and unsetting EESSI_CVMFS_INSTALL"
    echo $msg
    export EESSI_SITE_INSTALL=1
    unset EESSI_CVMFS_INSTALL
else
    unset EESSI_SITE_INSTALL
    export EESSI_CVMFS_INSTALL=1
fi

# We now run 'source load_eessi_extend_module.sh' to load or install and load the
#   EESSI-extend module which sets up all build environment settings.
# The script requires the EESSI_VERSION given as argument, a couple of
#   environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the
#   function check_exit_code defined.
# NOTE 1, the script exits if those variables/functions are undefined.
# NOTE 2, loading the EESSI-extend module may adjust the value of EASYBUILD_INSTALLPATH,
#   e.g., to point to the installation directory for accelerators.
# NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the
#   EESSI-extend module itself needs to be installed.
export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
echo "EASYBUILD_INSTALLPATH set to $EASYBUILD_INSTALLPATH"

# If we have EESSI_ACCELERATOR_TARGET_OVERRIDE set (and non-empty), then this implies building for a GPU target
# (this must be set _before_ we load EESSI-extend).
# We also make sure that EESSI_ACCELERATOR_TARGET is also set as EESSI_ACCELERATOR_TARGET_OVERRIDE must
# be set before the EESSI module is loaded in order to set accelerator information.
if [[ -n "$EESSI_ACCELERATOR_TARGET_OVERRIDE" && -z "$EESSI_ACCELERATOR_TARGET" ]]; then
  fatal_error "EESSI module should've set EESSI_ACCELERATOR_TARGET ($EESSI_ACCELERATOR_TARGET) when EESSI_ACCELERATOR_TARGET_OVERRIDE ($EESSI_ACCELERATOR_TARGET_OVERRIDE) exported."
elif [[ -n "$EESSI_ACCELERATOR_TARGET_OVERRIDE" ]]; then
  export EESSI_ACCELERATOR_INSTALL=1
fi

# Make sure we use the hooks from the local software-layer-scripts clone:
export EESSI_EASYBUILD_HOOKS_OVERRIDE=$TOPDIR/eb_hooks.py
echo "Set EESSI_EASYBUILD_HOOKS_OVERRIDE to ${EESSI_EASYBUILD_HOOKS_OVERRIDE}"

echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION}
echo "DEBUG: after loading EESSI-extend //  EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"

# Install full CUDA SDK and cu* libraries in host_injections
# (This is done *before* configuring EasyBuild as it may rely on an older EB version)
# Hardcode this for now, see if it works
# TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
# Allow skipping CUDA SDK install in e.g. CI environments
echo "Going to install full CUDA SDK and cu* libraries under host_injections if necessary"
temp_install_storage=${TMPDIR}/temp_install_storage
# Make sure that we use the easystack files from the git repository, which has an additional subdirectory ${EESSI_VERSION} at the end
export NVIDIA_EASYSTACKS_DIRECTORY=${TOPDIR}/scripts/gpu_support/nvidia/easystacks/${EESSI_VERSION}
mkdir -p ${temp_install_storage}
if [[ -z "${skip_cuda_install}" ]] || [[ ! "${skip_cuda_install}" ]]; then
    ${TOPDIR}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \
        -t ${temp_install_storage} \
        --accept-cuda-eula \
        --accept-cudnn-eula
else
    echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed"
fi

# Install NVIDIA drivers in host_injections (if they exist)
if nvidia_gpu_available; then
    echo "Installing NVIDIA drivers for use in prefix shell..."
    # Site installs override EESSI_CVMFS_REPO, but link_nvidia_host_libraries should always use the usptream EESSI CVMFS repo
    EESSI_CVMFS_REPO=/cvmfs/software.eessi.io ${TOPDIR}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
fi

# Now that we are done with all installs that should go the /cvmfs/software.eessi.io
# Reload the EESSI and EESSI-extend modules if we're doing a site install for which EESSI_SITE_SOFTWARE_PREFIX is set
export EESSI_SITE_SOFTWARE_PREFIX=${EESSI_SITE_SOFTWARE_PREFIX_BACKUP}
if [[ ! -z "${EESSI_SITE_INSTALL}" && ! -z "${EESSI_SITE_SOFTWARE_PREFIX}" ]]; then
    echo "Doing a site install with EESSI_SITE_SOFTWARE_PREFIX '${EESSI_SITE_SOFTWARE_PREFIX}', so reloading EESSI and EESSI-extend"
    EESSI_VERSION_BACKUP=${EESSI_VERSION}
    module purge
    echo "Loading EESSI/${EESSI_VERSION_BACKUP}"
    module load EESSI/${EESSI_VERSION_BACKUP}
    # Use --ignore_cache in case the current build was the one that installed this EESSI-extend module
    echo "Loading EESSI-extend/${EESSI_VERSION}"
    module --ignore_cache load EESSI-extend/${EESSI_VERSION}-easybuild
    echo "EASYBUILD_INSTALLPATH=${EASYBUILD_INSTALLPATH}"
fi

if [[ ! -z "${shared_fs_path}" ]]; then
    shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
    echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
    export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
fi

# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available
if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
    CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all
    if [[ -d ${CPU_ONLY_MODULES_PATH} ]]; then
        module use ${CPU_ONLY_MODULES_PATH}
    else
        fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}"
    fi
fi

echo "DEBUG: adding path '$EASYBUILD_INSTALLPATH/modules/all' to MODULEPATH='${MODULEPATH}'"
#module use $EASYBUILD_INSTALLPATH/modules/all
safe_module_use $EASYBUILD_INSTALLPATH/modules/all
echo "DEBUG: after adding module path // MODULEPATH='${MODULEPATH}'"

if [[ -z ${MODULEPATH} ]]; then
    fatal_error "Failed to set up \$MODULEPATH?!"
else
    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
fi

# use PR patch file to determine in which easystack files stuff was added
# Note that we exclude the scripts/gpu_support/ dir, since those are not meant to be built in the
# software-layer, but they are helper easystacks for installing e.g. CUDA in host_injections
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | (grep -v "scripts/gpu_support/" || true)) 
if [[ -z "${changed_easystacks}" ]]; then
    echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
else

    # first process rebuilds, if any, then easystack files for new installations
    # "|| true" is used to make sure that the grep command always returns success
    rebuild_easystacks=$(echo "${changed_easystacks}" | (grep "/rebuilds/" || true))
    new_easystacks=$(echo "${changed_easystacks}" | (grep -v "/rebuilds/" || true))
    echo "When processing easystack files:"
    echo "EESSI_CVMFS_REPO=${EESSI_CVMFS_REPO}"
    echo "EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}"
    echo "EESSI_VERSION=${EESSI_VERSION}"
    echo "EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}"
    echo "EESSI_SOFTWARE_LAYER_VERSION_SUFFIX=${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}"
    for easystack_file in ${rebuild_easystacks} ${new_easystacks}; do
        echo "Checking if easystack file ${easystack_file} is for the current EESSI_CVMFS_REPO and EESSI_VERSION"

        # make sure that easystack file being picked up is for EESSI version that we're building for...
        # Preferentially check EESSI_CVMFS_REPO_OVERRIDE (since that is set for site builds), otherwise default to EESSI_CVMFS_REPO
        echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})/${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}/"
        if [[ $? -ne 0 ]]; then
            # Check if this was even an easystack file for the right repository
            echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}})"
            if [[ $? -ne 0 ]]; then
                echo_yellow "Easystack file ${easystack_file} is not intended for the repository ${EESSI_CVMFS_REPO_OVERRIDE:-${EESSI_CVMFS_REPO}}, skipping it..."
            else
                echo_yellow "Easystack file ${easystack_file} is not intended for EESSI version ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX}, skipping it..."
            fi
        else
            echo -e "Processing easystack file ${easystack_file}...\n\n"

            # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')

            # load EasyBuild module (will be installed if it's not available yet)
            source ${TOPDIR}/load_easybuild_module.sh ${eb_version}

            ${EB} --show-config

            echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."

            # full EasyBuild command for this easystack file (also shown in the debug shell messages)
            if [[ ${easystack_file} == *"/rebuilds/"* ]]; then
                eb_cmd="${EB} --easystack ${easystack_file} --rebuild"
                # for rebuilds on a CVMFS repository, we assume an overlay is used,
                # and we can safely enable the hook that removes the existing installation directory at the start of the build
                if [[ ${EASYBUILD_INSTALLPATH} == "/cvmfs"* ]]; then
                    echo "This is a rebuild, so setting EESSI_FORCE_EARLY_REMOVAL_OF_INSTALL_DIR=1"
                    export EESSI_FORCE_EARLY_REMOVAL_OF_INSTALL_DIR=1
                fi
            else
                eb_cmd="${EB} --easystack ${easystack_file} --robot"
            fi

            # If EESSI_INTERACTIVE_BUILD=before, you'll get a debug shell right before easybuild starts
            # You can execute ${eb_cmd} manually, or just exit the shell again
            # to let the script continue (e.g. because the item you want to debug is in the 2nd iteration of this for-loop over
            # easystack files)
            if [[ "${EESSI_INTERACTIVE_BUILD}" == "before" ]]; then
                debug_shell "environment is ready to run: ${eb_cmd}"
            fi

            if [[ -f ${easystack_file} ]]; then
                echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."

                ${eb_cmd}
                ec=$?

                # copy EasyBuild log file if EasyBuild exited with an error
                if [[ ${ec} -ne 0 ]]; then
                    # If easybuild had a non-zero exit code, and EESSI_INTERACTIVE_BUILD=onfail, you'll get a debug
                    # shell so you can debug the failure
                    if [[ "${EESSI_INTERACTIVE_BUILD}" == "onfail" ]]; then
                        debug_shell "EasyBuild failed with exit code ${ec} for command: ${eb_cmd}"
                    fi
                    eb_last_log=$(eb --last-log | grep ^/.*\.log)
                    # copy to current working directory if file exhists
                    if [[ -f ${eb_last_log} ]]; then
                        cp -a ${eb_last_log} .
                        echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
                        # copy to build logs dir (with context added)
                        copy_build_log "${eb_last_log}" "${build_logs_dir}"
                    else
                        fatal_error "Could not copy EasyBuild log file because ${eb_last_log} does not exist"
                    fi
                fi

                $TOPDIR/check_missing_installations.sh ${easystack_file} ${pr_diff}
            else
                fatal_error "Easystack file ${easystack_file} not found!"
            fi
        fi

    done
fi

echo "DEBUG: before creating/updating lmod config files // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod"
export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod"

# If this is a site install, the old method of checking if lmodrc.lua was updated doesn't work
# We simply skip that step for now - it's hardly ever changed anyway
if [[ -z "${EESSI_SITE_INSTALL}" ]]; then
    lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua"
    echo "DEBUG: lmod_rc_file='${lmod_rc_file}'"
    if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
        # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file
        lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/${EESSI_ACCELERATOR_TARGET}@@")
        echo "Path to lmodrc.lua changed to '${lmod_rc_file}'"
    fi
    lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?)
    if [[ ! -f $lmod_rc_file ]] || [[ ${lmodrc_changed} == '0' ]]; then
        echo ">> Creating/updating Lmod RC file (${lmod_rc_file})..."
        python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
        check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file"
    fi
else
    # For site builds, create_lmodrc.py will _never_ be in the pr_diff, but it _might_ have changed upstream
    # The only way to then trigger a redeploy is if we do something like
    # python3 $TOPDIR/create_lmodrc.py $TMDPIR
    # foo=$(diff $lmod_rc_file $TMPDIR/.lmod/lmodrc.lua)
    # if [[ -z $foo ]]; then
    #     python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
    # fi
    # in order to detect if our deployed lmodrc.lua is different from the one that _would_ be newly generated
    # However, we very rarely change these scripts anyway, so we don't implement this right now
    echo "WARNING: there is currently no mechanism to detect if the lmodrc.lua should be updated."
fi

# If this is a site install, don't install SitePackage.lua
if [[ -z "${EESSI_SITE_INSTALL}" ]]; then
    lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua"
    if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
        # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file
        lmod_sitepackage_file=$(echo ${lmod_sitepackage_file} | sed "s@/${EESSI_ACCELERATOR_TARGET}@@")
        echo "Path to SitePackage.lua changed to '${lmod_sitepackage_file}'"
    fi
    sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?)
    if [[ ! -f "$lmod_sitepackage_file" ]] || [[ "${sitepackage_changed}" == '0' ]]; then
        echo ">> Creating/updating Lmod SitePackage.lua (${lmod_sitepackage_file})..."
        python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH}
        check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file"
    fi
fi

echo ">> Cleaning up ${TMPDIR}..."
rm -r ${TMPDIR}