diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 213308554..27c1d0fcd 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -109,6 +109,36 @@ jobs: run: | docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/ + - name: Test pip requirements file (-e) + # One instance per distro is enough; the feature does not depend on the + # Salt version. This must run after "Test Bootstrap" and before the -R + # test below, which leaves the repo definition pointing at an + # unreachable host. + if: ${{ matrix.instance == 'stable-3007' }} + run: | + # A requirements file that can not be used must be rejected up front, + # before anything is installed + if docker exec ${{ github.run_id}}_salt-test sh /_w/btstrap/bootstrap-salt.sh -e /tmp/does-not-exist.txt stable 3007; then + echo "::error::-e with a missing requirements file did not fail" + exit 1 + fi + + # cowsay is small, pure Python and not a dependency of Salt + docker exec ${{ github.run_id}}_salt-test sh -c 'printf "# Salt Extensions\ncowsay==6.1\n" > /tmp/pip-requirements.txt' + docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh -e /tmp/pip-requirements.txt stable 3007 + + # The package must be installed into the Salt onedir + out=$(docker exec ${{ github.run_id}}_salt-test salt-pip show cowsay) + echo "$out" + echo "$out" | grep -q '^Version: 6.1$' || { echo "::error::cowsay 6.1 was not installed by -e"; exit 1; } + + # and the policy-rc.d used to hold back service starts on Debian and + # Ubuntu must not be left behind + if docker exec ${{ github.run_id}}_salt-test test -f /usr/sbin/policy-rc.d.salt-bootstrap-backup; then + echo "::error::the policy-rc.d backup was left behind" + exit 1 + fi + - name: Test custom repository URL (-R) # One instance per distro is enough; the repo definition does not depend # on the Salt version. Arch and ALT Linux have no salt.sources/salt.repo. diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index b4fa4feaf..0e9909244 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -71,3 +71,14 @@ jobs: - name: Test Bootstrap run: | pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + + - name: Test pip requirements file (-e) is rejected + # The macOS package starts the minion itself while it is installed, so + # there is no chance to install packages first. -e must fail up front + # instead of installing them after Salt has already started. + run: | + printf '# Salt Extensions\ncowsay==6.1\n' > "$RUNNER_TEMP/pip-requirements.txt" + if sudo sh ./bootstrap-salt.sh -e "$RUNNER_TEMP/pip-requirements.txt" onedir; then + echo "::error::-e was accepted on macOS" + exit 1 + fi diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 5ece3b0bb..6b684491b 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -76,6 +76,31 @@ jobs: run: | pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test -PipRequirements + # Runs after "Test Bootstrap": it needs Salt to be installed already, + # and it re-runs the installer, which keeps the service stopped. + run: | + $ErrorActionPreference = "Stop" + + # A requirements file that can not be used must be rejected up front + & .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion -PipRequirements "$env:RUNNER_TEMP\does-not-exist.txt" + if ( $LASTEXITCODE -eq 0 ) { throw "-PipRequirements with a missing file did not fail" } + + # cowsay is small, pure Python and not a dependency of Salt + $requirements = Join-Path $env:RUNNER_TEMP "pip-requirements.txt" + Set-Content -Path $requirements -Value "# Salt Extensions", "cowsay==6.1" + & .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion -PipRequirements $requirements + if ( $LASTEXITCODE -ne 0 ) { throw "-PipRequirements exited $LASTEXITCODE" } + + # The package must be installed into Salt + $saltPip = "$env:ProgramFiles\Salt Project\Salt\salt-pip.exe" + $out = & $saltPip show cowsay + $out + if ( ($out -join "`n") -notmatch "(?m)^Version: 6\.1\s*$" ) { throw "cowsay 6.1 was not installed" } + + # and the service must not have been started along the way + if ( (Get-Service salt-minion).Status -ne "Stopped" ) { throw "salt-minion is not stopped" } + - name: Test -ConfigureOnly # Regression test for #2138. Runs after "Test Bootstrap" since it # overwrites the minion config. diff --git a/README.rst b/README.rst index 56af780af..b580a6875 100644 --- a/README.rst +++ b/README.rst @@ -157,6 +157,20 @@ To view the latest options and descriptions for ``salt-bootstrap``, use ``-h`` a You can also do this by touching /tmp/disable_salt_checks on the target host. Default: \${BS_FALSE} -D Show debug output + -e Path to a pip requirements file listing PyPI packages (for example Salt + Extensions) to install into the Salt onedir with salt-pip. The packages + are installed after Salt is installed and before its services are + started, so they are available the first time Salt starts. Pin versions + in the file, for example "saltext-foo==1.2.3". Custom or private + package indexes can be set with "--index-url" or "--extra-index-url" + lines in the file, or with the PIP_INDEX_URL and PIP_EXTRA_INDEX_URL + environment variables. Prefer these over putting credentials on the + command line. Use absolute paths for local packages in the file, and + do not leave the file writable by others, it is installed as root. If + the minion "user" is not root, the file must be readable by that user. + Only supported for onedir installs on Linux. + Packages that need compiling or system libraries can be satisfied + with -p, for example: -p build-essential -p libmariadb-dev -f Force shallow cloning for git installations. This may result in an "n/a" in the version number. -F Allow copied files to overwrite existing (config, init.d, etc) @@ -417,6 +431,20 @@ per Salt version holding the installer for that version: the Artifactory API. For any other source, including FTP, SMB shares and local directories, the hash is **not** checked. Make sure you trust the source. +Installing PyPI packages, such as Salt Extensions, with ``-PipRequirements``. It +takes the path to a pip requirements file. The packages are installed with +``salt-pip`` after Salt is installed and before the ``salt-minion`` service is +first started, so they are available the first time Salt runs: + +.. code:: powershell + + & "$env:TEMP\bootstrap-salt.ps1" -PipRequirements C:\salt\extensions.txt + +Pin versions in the file (for example ``saltext-foo==1.2.3``). A custom or +private package index can be set with ``--index-url`` or ``--extra-index-url`` +lines in the file, or with the ``PIP_INDEX_URL`` and ``PIP_EXTRA_INDEX_URL`` +environment variables. Prefer those over putting credentials on the command line. + Using ``cygwin`` to install latest stable version: .. code:: console diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 56615bfe9..afb78d597 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -31,6 +31,12 @@ ./bootstrap-salt.ps1 -Minion minion-box -Master master-box -Version 3006.7 -RunService $false Specifies all the optional parameters in no particular order. +.EXAMPLE + ./bootstrap-salt.ps1 -PipRequirements C:\salt\extensions.txt + Installs the PyPI packages (for example Salt Extensions) listed in the + requirements file into Salt using salt-pip. They are installed after Salt + and before the salt-minion service is first started. + .NOTES All of the parameters are optional. The default should be the latest version. The architecture is dynamically determined by the script. @@ -45,6 +51,16 @@ Artifactory API. For any other source, including FTP, SMB shares and local directories, the hash is NOT checked. Make sure you trust the source. + -PipRequirements takes the path to a pip requirements file. Pin versions in + the file (for example "saltext-foo==1.2.3"). A custom or private package + index can be set with "--index-url" or "--extra-index-url" lines in the + file, or with the PIP_INDEX_URL and PIP_EXTRA_INDEX_URL environment + variables. Prefer those over putting credentials on the command line. The + contents of the file are never printed by this script. Use absolute paths + for local packages in the file, and do not leave the file in a location + others can write to, it is installed as administrator. If a package has to + be compiled, the build tools it needs must already be installed. + .LINK Salt Bootstrap GitHub Project (script home) - https://github.com/saltstack/salt-bootstrap Original Vagrant Provisioner Project - https://github.com/saltstack/salty-vagrant @@ -106,6 +122,18 @@ param( # installer [Switch]$ConfigureOnly, + [Parameter(Mandatory=$false, ValueFromPipeline=$True)] + [Alias("p")] + # Path to a pip requirements file listing PyPI packages (for example Salt + # Extensions) to install into Salt with salt-pip. They are installed after + # Salt is installed and before the salt-minion service is first started, so + # they are available the first time Salt runs. Pin versions in the file + # (for example "saltext-foo==1.2.3"). A custom or private index can be set + # with "--index-url" or "--extra-index-url" lines in the file, or with the + # PIP_INDEX_URL and PIP_EXTRA_INDEX_URL environment variables. The file's + # contents are never printed. Cannot be combined with -ConfigureOnly. + [String]$PipRequirements = "", + [Parameter(Mandatory=$false)] [Alias("h")] # Displays help for this script. @@ -465,6 +493,93 @@ function Get-FileHash { } } +function Resolve-PipRequirementsFile { + # Checks the file passed with -PipRequirements and returns its full path. + # Throws if it is not a file, or lists no packages (pip refuses to run + # with nothing to install). + param( + [Parameter(Mandatory=$true)] + [String] $Path + ) + + if ( !(Test-Path -LiteralPath $Path -PathType Leaf) ) { + throw "The requirements file does not exist: $Path" + } + + $fullPath = (Resolve-Path -LiteralPath $Path).ProviderPath + $package = Get-Content -LiteralPath $fullPath | + Where-Object { $_ -notmatch '^\s*(#|$)' } | + Select-Object -First 1 + if ( !$package ) { + throw "The requirements file does not list any packages: $fullPath" + } + + return $fullPath +} + +function Get-SaltPipPath { + # Returns the default location of salt-pip.exe. ProgramW6432 is the 64-bit + # Program Files even when this runs in a 32-bit PowerShell. + if ( $env:ProgramW6432 ) { + $programFiles = $env:ProgramW6432 + } else { + $programFiles = $env:ProgramFiles + } + return Join-Path $programFiles "Salt Project\Salt\salt-pip.exe" +} + +function Install-PipRequirements { + # Installs the packages in a requirements file with salt-pip and returns its + # exit code. Throws if salt-pip is missing. + # + # It runs from the Windows directory, not the file's directory: + # "python -m pip" puts the working directory at the front of sys.path, so a + # directory others can write to would let them run code as administrator by + # planting a module there. Nested -r files still resolve against the + # requirements file, but local package paths in it need to be absolute. + param( + [Parameter(Mandatory=$true)] + [String] $Path, + + [Parameter(Mandatory=$false)] + [String] $SaltPip = "" + ) + + if ( !$SaltPip ) { $SaltPip = Get-SaltPipPath } + if ( !(Test-Path -LiteralPath $SaltPip) ) { + throw "salt-pip was not found at: $SaltPip" + } + + Write-Verbose "salt-pip: $SaltPip" + Write-Verbose "Requirements file: $Path" + $process = Start-Process $SaltPip ` + -WorkingDirectory $env:SystemRoot ` + -ArgumentList "install -r `"$Path`"" ` + -NoNewWindow -Wait -PassThru + return $process.ExitCode +} + +#=============================================================================== +# Validate the pip requirements file +#=============================================================================== +# This is done before the elevation check below so a relative path is resolved +# from where the script was started. The elevated copy of the script starts in +# a different directory. +if ( $PipRequirements ) { + if ( $ConfigureOnly ) { + Write-Host "-PipRequirements cannot be used with -ConfigureOnly, no Salt is installed" -ForegroundColor Red + exit 1 + } + try { + $PipRequirements = Resolve-PipRequirementsFile -Path $PipRequirements + } catch { + Write-Host $_.Exception.Message -ForegroundColor Red + exit 1 + } + # Hand the resolved path to the elevated copy of the script + $PSBoundParameters["PipRequirements"] = $PipRequirements +} + #=============================================================================== # Check for Elevated Privileges #=============================================================================== @@ -551,6 +666,7 @@ Write-Verbose "repourl: $base_url" Write-Verbose "apiurl: $api_url" Write-Verbose "ConfDir: $ConfDir" Write-Verbose "RootDir: $RootDir" +Write-Verbose "PipRequirements: $PipRequirements" if ($RunService) { Write-Verbose "Windows service will be set to run" @@ -770,6 +886,29 @@ while ( ! $service ) { # If we get this far, the service was installed, we have a service object Write-Host "Success" -ForegroundColor Green +#=============================================================================== +# Install PyPI packages (for example Salt Extensions) +#=============================================================================== +# The installer ran with /start-service=0, so the service has not started yet. +# Installing here means the packages are in place the first time Salt runs. +if ( $PipRequirements ) { + Write-Host "Installing Python packages from the requirements file:" + try { + $pipExitCode = Install-PipRequirements -Path $PipRequirements + } catch { + Write-Host "Failed" -ForegroundColor Red + Write-Host $_.Exception.Message + exit 1 + } + if ( $pipExitCode -ne 0 ) { + Write-Host "Failed" -ForegroundColor Red + Write-Host "salt-pip exited with code $pipExitCode" + Write-Host "If a package has to be compiled, its build tools must already be installed" + exit 1 + } + Write-Host "Success" -ForegroundColor Green +} + #=============================================================================== # Configure the minion service #=============================================================================== diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 288aafc86..7c69d7bd0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -285,6 +285,11 @@ _SALT_MINION_ID="null" _SIMPLIFY_VERSION=$BS_TRUE _LIBCLOUD_MIN_VERSION="0.14.0" _EXTRA_PACKAGES="" +_PIP_REQUIREMENTS_FILE="null" +# Where salt-pip is looked for. Fixed, absolute paths: this runs as root, so it must not be found through PATH +_SALT_PIP_PATHS="/usr/bin/salt-pip /opt/saltstack/salt/salt-pip" +_START_GUARD_ACTIVE=$BS_FALSE +_POLICY_RC_D="/usr/sbin/policy-rc.d" _HTTP_PROXY="" _SALT_GIT_CHECKOUT_DIR=${BS_SALT_GIT_CHECKOUT_DIR:-${_TMP_DIR}/git/salt} _NO_DEPS=$BS_FALSE @@ -375,6 +380,20 @@ __usage() { You can also do this by touching ${_TMP_DIR}/disable_salt_checks on the target host. Default: \${BS_FALSE} -D Show debug output + -e Path to a pip requirements file listing PyPI packages (for example Salt + Extensions) to install into the Salt onedir with salt-pip. The packages + are installed after Salt is installed and before its services are + started, so they are available the first time Salt starts. Pin versions + in the file, for example "saltext-foo==1.2.3". Custom or private + package indexes can be set with "--index-url" or "--extra-index-url" + lines in the file, or with the PIP_INDEX_URL and PIP_EXTRA_INDEX_URL + environment variables. Prefer these over putting credentials on the + command line. Use absolute paths for local packages in the file, and + do not leave the file writable by others, it is installed as root. If + the minion "user" is not root, the file must be readable by that user. + Only supported for onedir installs on Linux. + Packages that need compiling or system libraries can be satisfied + with -p, for example: -p build-essential -p libmariadb-dev -f Force shallow cloning for git installations. This may result in an "n/a" in the version number. -F Allow copied files to overwrite existing (config, init.d, etc) @@ -446,7 +465,7 @@ __usage() { EOT } # ---------- end of function __usage ---------- -while getopts ':hvnDc:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:T:aqQyY' opt +while getopts ':hvnDc:e:g:Gx:k:s:MSWNXCPFUKIA:i:Lp:dH:bflV:J:j:rR:T:aqQyY' opt do case "${opt}" in @@ -455,6 +474,7 @@ do n ) _COLORS=0; __detect_color_support ;; D ) _ECHO_DEBUG=$BS_TRUE ;; c ) _TEMP_CONFIG_DIR="$OPTARG" ;; + e ) _PIP_REQUIREMENTS_FILE="$OPTARG" ;; g ) _SALT_REPO_URL=$OPTARG ;; G ) echowarn "The '-G' option is DEPRECATED and will be removed in the future stable release!" @@ -552,6 +572,11 @@ APT_ERR=$(mktemp ${_TMP_DIR}/apt_error.XXXXXX) __exit_cleanup() { EXIT_CODE=$? + # Never leave the policy-rc.d that blocks service starts behind, whatever way the script ends + if [ "$_START_GUARD_ACTIVE" -eq $BS_TRUE ]; then + __service_start_guard_off + fi + if [ "$ITYPE" = "git" ] && [ -d "${_SALT_GIT_CHECKOUT_DIR}" ]; then if [ $_KEEP_TEMP_FILES -eq $BS_FALSE ]; then # Clean up the checked out repository @@ -1878,6 +1903,165 @@ if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then echoinfo "Installing salt-cloud and required python3-libcloud package" fi +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __validate_pip_requirements +# DESCRIPTION: Validate the -e option before anything is installed, so a bad invocation fails fast instead of +# half way through an installation. On success _PIP_REQUIREMENTS_FILE holds an absolute path. +#---------------------------------------------------------------------------------------------------------------------- +__validate_pip_requirements() { + + [ "$_PIP_REQUIREMENTS_FILE" = "null" ] && return 0 + + if [ "$ITYPE" != "onedir" ]; then + echoerror "-e is only supported for onedir installs because it relies on salt-pip." + return 1 + fi + + if [ "$DISTRO_NAME_L" = "macosx" ]; then + echoerror "-e is not supported on macOS. The Salt package starts the minion itself while it is being" + echoerror "installed, so there is no opportunity to install packages before Salt starts." + return 1 + fi + + if [ "$_CONFIG_ONLY" -eq $BS_TRUE ]; then + echoerror "-e can not be used with -C, Salt is not installed in configuration only mode." + return 1 + fi + + if [ ! -f "$_PIP_REQUIREMENTS_FILE" ] || [ ! -r "$_PIP_REQUIREMENTS_FILE" ]; then + echoerror "The requirements file passed to -e does not exist or is not readable: $_PIP_REQUIREMENTS_FILE" + return 1 + fi + + # pip refuses to run with nothing to install, so catch an empty or comment only file here + if ! grep -Eqv '^[[:space:]]*(#|$)' "$_PIP_REQUIREMENTS_FILE"; then + echoerror "The requirements file passed to -e does not list any packages: $_PIP_REQUIREMENTS_FILE" + return 1 + fi + + # Whoever can change this file controls what is installed as root (any package, any index, any local path) + # (-L so a symlink is judged by its target, a symlink itself is always rwxrwxrwx) + if [ -n "$(find -L "$_PIP_REQUIREMENTS_FILE" -maxdepth 0 -perm -002 2>/dev/null)" ]; then + echoerror "The requirements file passed to -e is writable by everyone: $_PIP_REQUIREMENTS_FILE" + echoerror "Anyone could change what is installed as root. Fix it with: chmod o-w " + return 1 + fi + if [ -n "$(find -L "$_PIP_REQUIREMENTS_FILE" -maxdepth 0 -perm -020 2>/dev/null)" ]; then + echowarn "The requirements file passed to -e is writable by its group: $_PIP_REQUIREMENTS_FILE" + fi + + # Use an absolute path, salt-pip is run from a different working directory + _req_dir=$(cd "$(dirname "$_PIP_REQUIREMENTS_FILE")" && pwd) || return 1 + _PIP_REQUIREMENTS_FILE="${_req_dir}/$(basename "$_PIP_REQUIREMENTS_FILE")" + + return 0 +} # ---------- end of function __validate_pip_requirements ---------- + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __install_pip_requirements +# DESCRIPTION: Install the packages listed in the -e requirements file into the Salt onedir using salt-pip. Must run +# after Salt is installed and before its services are started. +#---------------------------------------------------------------------------------------------------------------------- +__install_pip_requirements() { + + [ "$_PIP_REQUIREMENTS_FILE" = "null" ] && return 0 + + # Only use salt-pip from its known locations, never one found through PATH, since this runs as root + _salt_pip="" + for _candidate in $_SALT_PIP_PATHS; do + if [ -x "$_candidate" ]; then + _salt_pip="$_candidate" + break + fi + done + + if [ -z "$_salt_pip" ]; then + echoerror "salt-pip was not found, can not install the packages from $_PIP_REQUIREMENTS_FILE" + return 1 + fi + + echoinfo "Installing the Python packages listed in $_PIP_REQUIREMENTS_FILE using $_salt_pip" + + # Run from / and not from the directory of the requirements file. "python -m pip" puts the working directory at the + # front of sys.path, so running from a directory that others can write to would let them run code as root by + # planting a module there. Nested -r files still resolve against the requirements file, local package paths in it + # need to be absolute. The file's contents are deliberately never echoed, it may hold index credentials. + if ! (cd / && "$_salt_pip" install -r "$_PIP_REQUIREMENTS_FILE"); then + echoerror "Failed to install the packages listed in $_PIP_REQUIREMENTS_FILE" + echoerror "If a package has to be compiled or needs system libraries, install those with -p, for example:" + echoerror " -p build-essential -p libmariadb-dev" + echoerror "If the minion user is not root, the requirements file must be readable by that user." + return 1 + fi + + return 0 +} # ---------- end of function __install_pip_requirements ---------- + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __start_guard_needed +# DESCRIPTION: Return 0 if package installs must be prevented from starting services. Only Debian and Ubuntu need +# this: their packages start the service as soon as it is installed, so Salt would already be running, +# without the -e packages, before they could be installed. The RPM packages and the other onedir +# installs only enable the services and leave starting them to this script. +#---------------------------------------------------------------------------------------------------------------------- +__start_guard_needed() { + + [ "$_PIP_REQUIREMENTS_FILE" != "null" ] || return 1 + + case "$DISTRO_NAME_L" in + debian|ubuntu) return 0 ;; + esac + + return 1 +} # ---------- end of function __start_guard_needed ---------- + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __service_start_guard_on +# DESCRIPTION: Install a policy-rc.d that denies starting services, which dpkg maintainer scripts honour. Any +# existing policy-rc.d is moved aside and put back by __service_start_guard_off. The exit trap also +# calls it, so the policy is removed however this script ends. +#---------------------------------------------------------------------------------------------------------------------- +__service_start_guard_on() { + + _POLICY_RC_D_BACKUP="${_POLICY_RC_D}.salt-bootstrap-backup" + + if [ -e "$_POLICY_RC_D" ] || [ -L "$_POLICY_RC_D" ]; then + mv -f "$_POLICY_RC_D" "$_POLICY_RC_D_BACKUP" || return 1 + fi + + if ! printf '#!/bin/sh\nexit 101\n' > "$_POLICY_RC_D" || ! chmod 755 "$_POLICY_RC_D"; then + __service_start_guard_off + return 1 + fi + + _START_GUARD_ACTIVE=$BS_TRUE + echodebug "Installed $_POLICY_RC_D to prevent services from starting during installation" + + return 0 +} # ---------- end of function __service_start_guard_on ---------- + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __service_start_guard_off +# DESCRIPTION: Remove the policy-rc.d installed by __service_start_guard_on and restore the original, if any. +#---------------------------------------------------------------------------------------------------------------------- +__service_start_guard_off() { + + rm -f "$_POLICY_RC_D" + + if [ -e "$_POLICY_RC_D_BACKUP" ] || [ -L "$_POLICY_RC_D_BACKUP" ]; then + mv -f "$_POLICY_RC_D_BACKUP" "$_POLICY_RC_D" + fi + + _START_GUARD_ACTIVE=$BS_FALSE + echodebug "Removed the policy-rc.d that prevented services from starting" + + return 0 +} # ---------- end of function __service_start_guard_off ---------- + +if ! __validate_pip_requirements; then + exit 1 +fi + if [ $_START_DAEMONS -eq $BS_FALSE ]; then echoinfo "Daemons will not be started" fi @@ -9102,6 +9286,17 @@ fi # Install Salt if [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then # Only execute function is not in config mode only + + # The packages on some distributions start the services as soon as they are installed. When there are packages to + # install with -e, hold that back until they are in place, see __start_guard_needed. + if __start_guard_needed; then + echoinfo "Preventing Salt services from starting until the packages from ${_PIP_REQUIREMENTS_FILE} are installed" + if ! __service_start_guard_on; then + echoerror "Failed to prevent Salt services from starting during installation" + exit 1 + fi + fi + echoinfo "Running ${INSTALL_FUNC}()" if ! ${INSTALL_FUNC}; then echoerror "Failed to run ${INSTALL_FUNC}()!!!" @@ -9118,6 +9313,21 @@ if [ "$POST_INSTALL_FUNC" != "null" ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then fi fi +# Install the PyPI packages requested with -e. This has to happen after Salt is installed, since it needs salt-pip, and +# before the services are started so the packages are available the first time Salt runs. +if [ "$_PIP_REQUIREMENTS_FILE" != "null" ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echoinfo "Running __install_pip_requirements()" + if ! __install_pip_requirements; then + echoerror "Failed to run __install_pip_requirements()!!!" + exit 1 + fi +fi + +# Services may start again now that everything they need is installed +if [ "$_START_GUARD_ACTIVE" -eq $BS_TRUE ]; then + __service_start_guard_off +fi + # Run any check services function, Only execute function if not in config mode only if [ "$CHECK_SERVICES_FUNC" != "null" ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then echoinfo "Running ${CHECK_SERVICES_FUNC}()" diff --git a/tests/powershell/bootstrap-salt.Tests.ps1 b/tests/powershell/bootstrap-salt.Tests.ps1 index 0180bf8aa..9fc95d4b2 100644 --- a/tests/powershell/bootstrap-salt.Tests.ps1 +++ b/tests/powershell/bootstrap-salt.Tests.ps1 @@ -147,3 +147,119 @@ Describe "Get-HashFromArtifactory" { Should -Invoke Invoke-RestMethod -Times 0 } } + +Describe "Resolve-PipRequirementsFile" { + BeforeAll { + $requirements = Join-Path $TestDrive "requirements.txt" + Set-Content -Path $requirements -Value "# Salt Extensions", "saltext-foo==1.2.3" + } + + It "returns the full path of a requirements file" { + Resolve-PipRequirementsFile -Path $requirements | + Should -Be (Resolve-Path $requirements).ProviderPath + } + + It "resolves a relative path" { + Push-Location $TestDrive + try { + Resolve-PipRequirementsFile -Path "./requirements.txt" | + Should -Be (Resolve-Path $requirements).ProviderPath + } finally { + Pop-Location + } + } + + It "accepts a file with an index option and a package" { + $file = Join-Path $TestDrive "index.txt" + Set-Content -Path $file -Value "--index-url https://pypi.example.com/simple", "saltext-foo" + { Resolve-PipRequirementsFile -Path $file } | Should -Not -Throw + } + + It "throws when the file does not exist" { + { Resolve-PipRequirementsFile -Path (Join-Path $TestDrive "missing.txt") } | + Should -Throw "*does not exist*" + } + + It "throws for a directory" { + { Resolve-PipRequirementsFile -Path $TestDrive } | Should -Throw "*does not exist*" + } + + It "throws when the file only has comments and blank lines" { + $file = Join-Path $TestDrive "empty.txt" + Set-Content -Path $file -Value "# nothing here", "", " ", " # still nothing" + { Resolve-PipRequirementsFile -Path $file } | Should -Throw "*does not list any packages*" + } +} + +Describe "Get-SaltPipPath" { + BeforeEach { + $savedW6432 = $env:ProgramW6432 + $savedProgramFiles = $env:ProgramFiles + } + + AfterEach { + $env:ProgramW6432 = $savedW6432 + $env:ProgramFiles = $savedProgramFiles + } + + It "uses the 64-bit Program Files when it is set" { + $env:ProgramW6432 = Join-Path $TestDrive "Program Files" + $env:ProgramFiles = Join-Path $TestDrive "Program Files (x86)" + Get-SaltPipPath | Should -Be (Join-Path $env:ProgramW6432 "Salt Project\Salt\salt-pip.exe") + } + + It "falls back to Program Files" { + $env:ProgramW6432 = "" + $env:ProgramFiles = Join-Path $TestDrive "Program Files" + Get-SaltPipPath | Should -Be (Join-Path $env:ProgramFiles "Salt Project\Salt\salt-pip.exe") + } +} + +Describe "Install-PipRequirements" { + BeforeAll { + $requirements = Join-Path $TestDrive "requirements.txt" + Set-Content -Path $requirements -Value "saltext-foo==1.2.3" + $saltPip = Join-Path $TestDrive "salt-pip.exe" + Set-Content -Path $saltPip -Value "" + # Always set on Windows. Only needed so the tests also run elsewhere. + $setSystemRoot = !$env:SystemRoot + if ( $setSystemRoot ) { $env:SystemRoot = Join-Path $TestDrive "Windows" } + } + + AfterAll { + if ( $setSystemRoot ) { $env:SystemRoot = $null } + } + + It "runs salt-pip install -r" { + Mock Start-Process { [pscustomobject]@{ ExitCode = 0 } } + Install-PipRequirements -Path $requirements -SaltPip $saltPip | Should -Be 0 + Should -Invoke Start-Process -Times 1 -ParameterFilter { + $FilePath -eq $saltPip -and + $ArgumentList -eq "install -r `"$requirements`"" + } + } + + # "python -m pip" puts the working directory first on sys.path, so running + # it from the directory of the file would let anyone who can write there + # run code as administrator + It "does not run salt-pip from the directory of the requirements file" { + Mock Start-Process { [pscustomobject]@{ ExitCode = 0 } } + Install-PipRequirements -Path $requirements -SaltPip $saltPip | Out-Null + Should -Invoke Start-Process -Times 1 -ParameterFilter { + $WorkingDirectory -eq $env:SystemRoot -and + $WorkingDirectory -ne (Split-Path $requirements -Parent) + } + } + + It "returns the exit code of salt-pip" { + Mock Start-Process { [pscustomobject]@{ ExitCode = 1 } } + Install-PipRequirements -Path $requirements -SaltPip $saltPip | Should -Be 1 + } + + It "throws when salt-pip is missing" { + Mock Start-Process { [pscustomobject]@{ ExitCode = 0 } } + { Install-PipRequirements -Path $requirements -SaltPip (Join-Path $TestDrive "missing.exe") } | + Should -Throw "*salt-pip was not found*" + Should -Invoke Start-Process -Times 0 + } +} diff --git a/tests/shell/pip_requirements_spec.sh b/tests/shell/pip_requirements_spec.sh new file mode 100644 index 000000000..e44e33877 --- /dev/null +++ b/tests/shell/pip_requirements_spec.sh @@ -0,0 +1,286 @@ +# shellcheck shell=sh + +Describe '-e pip requirements file' + Include "$(bootstrap_functions __validate_pip_requirements __install_pip_requirements __start_guard_needed __service_start_guard_on __service_start_guard_off)" + + BS_TRUE=1 + BS_FALSE=0 + + # Mocks: capture messages instead of printing colours + echoerror() { echo "error: $*" >&2; } + echodebug() { :; } + echoinfo() { echo "info: $*" >&2; } + echowarn() { echo "warn: $*" >&2; } + + setup() { + # The result must not depend on the umask of whoever runs the specs, files + # that are group writable make the validation warn + ORIG_UMASK=$(umask) + umask 022 + WORK=$(mktemp -d) + CALLS="$WORK/calls" + : > "$CALLS" + ITYPE="onedir" + DISTRO_NAME_L="debian" + _CONFIG_ONLY=$BS_FALSE + _PIP_REQUIREMENTS_FILE="null" + _START_GUARD_ACTIVE=$BS_FALSE + printf '# Salt Extensions\nsaltext-foo==1.2.3\n' > "$WORK/requirements.txt" + ORIG_PATH=$PATH + ORIG_PWD=$(pwd) + } + cleanup() { + umask "$ORIG_UMASK" + PATH=$ORIG_PATH + cd "$ORIG_PWD" || return 1 + rm -rf "$WORK" + } + BeforeEach setup + AfterEach cleanup + + Describe '__validate_pip_requirements' + It 'does nothing when -e was not passed' + When call __validate_pip_requirements + The status should be success + The stderr should eq '' + End + + It 'accepts a requirements file for a onedir install' + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be success + The variable _PIP_REQUIREMENTS_FILE should eq "$WORK/requirements.txt" + End + + It 'turns a relative path into an absolute one' + cd "$WORK" || return 1 + _PIP_REQUIREMENTS_FILE="./requirements.txt" + When call __validate_pip_requirements + The status should be success + The variable _PIP_REQUIREMENTS_FILE should eq "$WORK/requirements.txt" + End + + It 'rejects installs that are not onedir' + ITYPE="git" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'only supported for onedir installs' + End + + It 'rejects macOS' + DISTRO_NAME_L="macosx" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'not supported on macOS' + End + + It 'rejects configuration only mode' + _CONFIG_ONLY=$BS_TRUE + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'can not be used with -C' + End + + It 'rejects a file that does not exist' + _PIP_REQUIREMENTS_FILE="$WORK/missing.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'does not exist or is not readable' + End + + It 'rejects a directory' + _PIP_REQUIREMENTS_FILE="$WORK" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'does not exist or is not readable' + End + + It 'rejects a file with only comments and blank lines' + printf '# nothing here\n\n \n # still nothing\n' > "$WORK/empty.txt" + _PIP_REQUIREMENTS_FILE="$WORK/empty.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'does not list any packages' + End + + It 'accepts a file with an index option and a package' + printf -- '--index-url https://pypi.example.com/simple\nsaltext-foo\n' > "$WORK/index.txt" + _PIP_REQUIREMENTS_FILE="$WORK/index.txt" + When call __validate_pip_requirements + The status should be success + End + + It 'rejects a file that is writable by everyone' + chmod 666 "$WORK/requirements.txt" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be failure + The stderr should include 'writable by everyone' + End + + It 'warns about a file that is writable by its group but accepts it' + chmod 664 "$WORK/requirements.txt" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __validate_pip_requirements + The status should be success + The stderr should include 'writable by its group' + End + + It 'does not mistake a symlink for a writable file' + chmod 644 "$WORK/requirements.txt" + ln -s "$WORK/requirements.txt" "$WORK/link.txt" + _PIP_REQUIREMENTS_FILE="$WORK/link.txt" + When call __validate_pip_requirements + The status should be success + The stderr should eq '' + End + End + + Describe '__install_pip_requirements' + # A stand-in salt-pip that records how it was called + fake_salt_pip() { + mkdir -p "$WORK/bin" + cat > "$WORK/bin/salt-pip" <> "$CALLS" +echo "args=\$*" >> "$CALLS" +exit \${FAKE_SALT_PIP_EXIT:-0} +EOF + chmod +x "$WORK/bin/salt-pip" + _SALT_PIP_PATHS="$WORK/bin/salt-pip" + } + + It 'does nothing when -e was not passed' + When call __install_pip_requirements + The status should be success + End + + It 'runs salt-pip install -r' + fake_salt_pip + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __install_pip_requirements + The status should be success + The stderr should include 'Installing the Python packages' + The contents of file "$CALLS" should include "args=install -r $WORK/requirements.txt" + End + + # "python -m pip" puts the working directory first on sys.path, so running + # it from the directory of the file would let anyone who can write there + # run code as root + It 'does not run salt-pip from the directory of the requirements file' + fake_salt_pip + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __install_pip_requirements + The status should be success + The stderr should include 'Installing the Python packages' + The contents of file "$CALLS" should include "cwd=/" + The contents of file "$CALLS" should not include "cwd=$WORK" + End + + It 'does not look for salt-pip in the PATH' + fake_salt_pip + _SALT_PIP_PATHS="$WORK/not-there/salt-pip" + PATH="$WORK/bin:$PATH" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __install_pip_requirements + The status should be failure + The stderr should include 'salt-pip was not found' + End + + It 'does not print the contents of the file' + fake_salt_pip + printf -- '--index-url https://user:s3cret@pypi.example.com/simple\nsaltext-foo\n' > "$WORK/private.txt" + _PIP_REQUIREMENTS_FILE="$WORK/private.txt" + When call __install_pip_requirements + The status should be success + The stderr should not include 's3cret' + End + + It 'fails with a hint about -p when salt-pip fails' + fake_salt_pip + FAKE_SALT_PIP_EXIT=1 + export FAKE_SALT_PIP_EXIT + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __install_pip_requirements + The status should be failure + The stderr should include 'Failed to install the packages' + The stderr should include '-p build-essential' + End + + It 'fails when salt-pip can not be found' + _SALT_PIP_PATHS="$WORK/not-there/salt-pip" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __install_pip_requirements + The status should be failure + The stderr should include 'salt-pip was not found' + End + End + + Describe '__start_guard_needed' + It 'is needed on Debian when -e was passed' + DISTRO_NAME_L="debian" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __start_guard_needed + The status should be success + End + + It 'is needed on Ubuntu when -e was passed' + DISTRO_NAME_L="ubuntu" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __start_guard_needed + The status should be success + End + + It 'is not needed on RPM based distributions' + DISTRO_NAME_L="rocky_linux" + _PIP_REQUIREMENTS_FILE="$WORK/requirements.txt" + When call __start_guard_needed + The status should be failure + End + + It 'is not needed without -e' + DISTRO_NAME_L="debian" + When call __start_guard_needed + The status should be failure + End + End + + Describe 'service start guard' + setup_guard() { + _POLICY_RC_D="$WORK/policy-rc.d" + unset _POLICY_RC_D_BACKUP + } + BeforeEach setup_guard + + It 'installs an executable policy-rc.d that denies starting services' + When call __service_start_guard_on + The status should be success + The variable _START_GUARD_ACTIVE should eq 1 + The path "$WORK/policy-rc.d" should be executable + The contents of file "$WORK/policy-rc.d" should include 'exit 101' + End + + It 'removes the policy-rc.d again when there was none before' + __service_start_guard_on + When call __service_start_guard_off + The status should be success + The variable _START_GUARD_ACTIVE should eq 0 + The path "$WORK/policy-rc.d" should not be exist + The path "$WORK/policy-rc.d.salt-bootstrap-backup" should not be exist + End + + It 'puts an existing policy-rc.d back' + printf '#!/bin/sh\necho original\n' > "$WORK/policy-rc.d" + chmod 755 "$WORK/policy-rc.d" + __service_start_guard_on + When call __service_start_guard_off + The status should be success + The contents of file "$WORK/policy-rc.d" should include 'echo original' + The path "$WORK/policy-rc.d" should be executable + The path "$WORK/policy-rc.d.salt-bootstrap-backup" should not be exist + End + End +End