feature(bootstrap): add -e/-PipRequirements to install PyPI packagesbefore Salt starts

.github/workflows/test-linux.yml

@@ -109,6 +109,36 @@ jobs:
         run: |
           docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/
 
+      - name: Test pip requirements file (-e)
+        # One instance per distro is enough; the feature does not depend on the
+        # Salt version. This must run after "Test Bootstrap" and before the -R
+        # test below, which leaves the repo definition pointing at an
+        # unreachable host.
+        if: ${{ matrix.instance == 'stable-3007' }}
+        run: |
+          # A requirements file that can not be used must be rejected up front,
+          # before anything is installed
+          if docker exec ${{ github.run_id}}_salt-test sh /_w/btstrap/bootstrap-salt.sh -e /tmp/does-not-exist.txt stable 3007; then
+            echo "::error::-e with a missing requirements file did not fail"
+            exit 1
+          fi
+
+          # cowsay is small, pure Python and not a dependency of Salt
+          docker exec ${{ github.run_id}}_salt-test sh -c 'printf "# Salt Extensions\ncowsay==6.1\n" > /tmp/pip-requirements.txt'
+          docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh -e /tmp/pip-requirements.txt stable 3007
+
+          # The package must be installed into the Salt onedir
+          out=$(docker exec ${{ github.run_id}}_salt-test salt-pip show cowsay)
+          echo "$out"
+          echo "$out" | grep -q '^Version: 6.1$' || { echo "::error::cowsay 6.1 was not installed by -e"; exit 1; }
+
+          # and the policy-rc.d used to hold back service starts on Debian and
+          # Ubuntu must not be left behind
+          if docker exec ${{ github.run_id}}_salt-test test -f /usr/sbin/policy-rc.d.salt-bootstrap-backup; then
+            echo "::error::the policy-rc.d backup was left behind"
+            exit 1
+          fi
+
       - name: Test custom repository URL (-R)
         # One instance per distro is enough; the repo definition does not depend
         # on the Salt version. Arch and ALT Linux have no salt.sources/salt.repo.

.github/workflows/test-macos.yml

@@ -71,3 +71,14 @@ jobs:
       - name: Test Bootstrap
         run: |
           pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
+
+      - name: Test pip requirements file (-e) is rejected
+        # The macOS package starts the minion itself while it is installed, so
+        # there is no chance to install packages first. -e must fail up front
+        # instead of installing them after Salt has already started.
+        run: |
+          printf '# Salt Extensions\ncowsay==6.1\n' > "$RUNNER_TEMP/pip-requirements.txt"
+          if sudo sh ./bootstrap-salt.sh -e "$RUNNER_TEMP/pip-requirements.txt" onedir; then
+            echo "::error::-e was accepted on macOS"
+            exit 1
+          fi

.github/workflows/test-windows.yml

@@ -76,6 +76,31 @@ jobs:
         run: |
           pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/
 
+      - name: Test -PipRequirements
+        # Runs after "Test Bootstrap": it needs Salt to be installed already,
+        # and it re-runs the installer, which keeps the service stopped.
+        run: |
+          $ErrorActionPreference = "Stop"
+
+          # A requirements file that can not be used must be rejected up front
+          & .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion -PipRequirements "$env:RUNNER_TEMP\does-not-exist.txt"
+          if ( $LASTEXITCODE -eq 0 ) { throw "-PipRequirements with a missing file did not fail" }
+
+          # cowsay is small, pure Python and not a dependency of Salt
+          $requirements = Join-Path $env:RUNNER_TEMP "pip-requirements.txt"
+          Set-Content -Path $requirements -Value "# Salt Extensions", "cowsay==6.1"
+          & .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion -PipRequirements $requirements
+          if ( $LASTEXITCODE -ne 0 ) { throw "-PipRequirements exited $LASTEXITCODE" }
+
+          # The package must be installed into Salt
+          $saltPip = "$env:ProgramFiles\Salt Project\Salt\salt-pip.exe"
+          $out = & $saltPip show cowsay
+          $out
+          if ( ($out -join "`n") -notmatch "(?m)^Version: 6\.1\s*$" ) { throw "cowsay 6.1 was not installed" }
+
+          # and the service must not have been started along the way
+          if ( (Get-Service salt-minion).Status -ne "Stopped" ) { throw "salt-minion is not stopped" }
+
       - name: Test -ConfigureOnly
         # Regression test for #2138. Runs after "Test Bootstrap" since it
         # overwrites the minion config.

README.rst

@@ -157,6 +157,20 @@ To view the latest options and descriptions for ``salt-bootstrap``, use ``-h`` a
         You can also do this by touching /tmp/disable_salt_checks on the target
         host. Default: \${BS_FALSE}
     -D  Show debug output
+    -e  Path to a pip requirements file listing PyPI packages (for example Salt
+        Extensions) to install into the Salt onedir with salt-pip. The packages
+        are installed after Salt is installed and before its services are
+        started, so they are available the first time Salt starts. Pin versions
+        in the file, for example "saltext-foo==1.2.3". Custom or private
+        package indexes can be set with "--index-url" or "--extra-index-url"
+        lines in the file, or with the PIP_INDEX_URL and PIP_EXTRA_INDEX_URL
+        environment variables. Prefer these over putting credentials on the
+        command line. Use absolute paths for local packages in the file, and
+        do not leave the file writable by others, it is installed as root. If
+        the minion "user" is not root, the file must be readable by that user.
+        Only supported for onedir installs on Linux.
+        Packages that need compiling or system libraries can be satisfied
+        with -p, for example: -p build-essential -p libmariadb-dev
     -f  Force shallow cloning for git installations.
         This may result in an "n/a" in the version number.
     -F  Allow copied files to overwrite existing (config, init.d, etc)
@@ -417,6 +431,20 @@ per Salt version holding the installer for that version:
   the Artifactory API. For any other source, including FTP, SMB shares and local
   directories, the hash is **not** checked. Make sure you trust the source.
 
+Installing PyPI packages, such as Salt Extensions, with ``-PipRequirements``. It
+takes the path to a pip requirements file. The packages are installed with
+``salt-pip`` after Salt is installed and before the ``salt-minion`` service is
+first started, so they are available the first time Salt runs:
+
+.. code:: powershell
+
+  & "$env:TEMP\bootstrap-salt.ps1" -PipRequirements C:\salt\extensions.txt
+
+Pin versions in the file (for example ``saltext-foo==1.2.3``). A custom or
+private package index can be set with ``--index-url`` or ``--extra-index-url``
+lines in the file, or with the ``PIP_INDEX_URL`` and ``PIP_EXTRA_INDEX_URL``
+environment variables. Prefer those over putting credentials on the command line.
+
 Using ``cygwin`` to install latest stable version:
 
 .. code:: console

bootstrap-salt.ps1

@@ -31,6 +31,12 @@
     ./bootstrap-salt.ps1 -Minion minion-box -Master master-box -Version 3006.7 -RunService $false
     Specifies all the optional parameters in no particular order.
 
+.EXAMPLE
+    ./bootstrap-salt.ps1 -PipRequirements C:\salt\extensions.txt
+    Installs the PyPI packages (for example Salt Extensions) listed in the
+    requirements file into Salt using salt-pip. They are installed after Salt
+    and before the salt-minion service is first started.
+
 .NOTES
     All of the parameters are optional. The default should be the latest
     version. The architecture is dynamically determined by the script.
@@ -45,6 +51,16 @@
     Artifactory API. For any other source, including FTP, SMB shares and local
     directories, the hash is NOT checked. Make sure you trust the source.
 
+    -PipRequirements takes the path to a pip requirements file. Pin versions in
+    the file (for example "saltext-foo==1.2.3"). A custom or private package
+    index can be set with "--index-url" or "--extra-index-url" lines in the
+    file, or with the PIP_INDEX_URL and PIP_EXTRA_INDEX_URL environment
+    variables. Prefer those over putting credentials on the command line. The
+    contents of the file are never printed by this script. Use absolute paths
+    for local packages in the file, and do not leave the file in a location
+    others can write to, it is installed as administrator. If a package has to
+    be compiled, the build tools it needs must already be installed.
+
 .LINK
     Salt Bootstrap GitHub Project (script home) - https://github.com/saltstack/salt-bootstrap
     Original Vagrant Provisioner Project - https://github.com/saltstack/salty-vagrant
@@ -106,6 +122,18 @@ param(
     # installer
     [Switch]$ConfigureOnly,
 
+    [Parameter(Mandatory=$false, ValueFromPipeline=$True)]
+    [Alias("p")]
+    # Path to a pip requirements file listing PyPI packages (for example Salt
+    # Extensions) to install into Salt with salt-pip. They are installed after
+    # Salt is installed and before the salt-minion service is first started, so
+    # they are available the first time Salt runs. Pin versions in the file
+    # (for example "saltext-foo==1.2.3"). A custom or private index can be set
+    # with "--index-url" or "--extra-index-url" lines in the file, or with the
+    # PIP_INDEX_URL and PIP_EXTRA_INDEX_URL environment variables. The file's
+    # contents are never printed. Cannot be combined with -ConfigureOnly.
+    [String]$PipRequirements = "",
+
     [Parameter(Mandatory=$false)]
     [Alias("h")]
     # Displays help for this script.
@@ -465,6 +493,93 @@ function Get-FileHash {
     }
 }
 
+function Resolve-PipRequirementsFile {
+    # Checks the file passed with -PipRequirements and returns its full path.
+    # Throws if it is not a file, or lists no packages (pip refuses to run
+    # with nothing to install).
+    param(
+        [Parameter(Mandatory=$true)]
+        [String] $Path
+    )
+
+    if ( !(Test-Path -LiteralPath $Path -PathType Leaf) ) {
+        throw "The requirements file does not exist: $Path"
+    }
+
+    $fullPath = (Resolve-Path -LiteralPath $Path).ProviderPath
+    $package = Get-Content -LiteralPath $fullPath |
+        Where-Object { $_ -notmatch '^\s*(#|$)' } |
+        Select-Object -First 1
+    if ( !$package ) {
+        throw "The requirements file does not list any packages: $fullPath"
+    }
+
+    return $fullPath
+}
+
+function Get-SaltPipPath {
+    # Returns the default location of salt-pip.exe. ProgramW6432 is the 64-bit
+    # Program Files even when this runs in a 32-bit PowerShell.
+    if ( $env:ProgramW6432 ) {
+        $programFiles = $env:ProgramW6432
+    } else {
+        $programFiles = $env:ProgramFiles
+    }
+    return Join-Path $programFiles "Salt Project\Salt\salt-pip.exe"
+}
+
+function Install-PipRequirements {
+    # Installs the packages in a requirements file with salt-pip and returns its
+    # exit code. Throws if salt-pip is missing.
+    #
+    # It runs from the Windows directory, not the file's directory:
+    # "python -m pip" puts the working directory at the front of sys.path, so a
+    # directory others can write to would let them run code as administrator by
+    # planting a module there. Nested -r files still resolve against the
+    # requirements file, but local package paths in it need to be absolute.
+    param(
+        [Parameter(Mandatory=$true)]
+        [String] $Path,
+
+        [Parameter(Mandatory=$false)]
+        [String] $SaltPip = ""
+    )
+
+    if ( !$SaltPip ) { $SaltPip = Get-SaltPipPath }
+    if ( !(Test-Path -LiteralPath $SaltPip) ) {
+        throw "salt-pip was not found at: $SaltPip"
+    }
+
+    Write-Verbose "salt-pip: $SaltPip"
+    Write-Verbose "Requirements file: $Path"
+    $process = Start-Process $SaltPip `
+        -WorkingDirectory $env:SystemRoot `
+        -ArgumentList "install -r `"$Path`"" `
+        -NoNewWindow -Wait -PassThru
+    return $process.ExitCode
+}
+
+#===============================================================================
+# Validate the pip requirements file
+#===============================================================================
+# This is done before the elevation check below so a relative path is resolved
+# from where the script was started. The elevated copy of the script starts in
+# a different directory.
+if ( $PipRequirements ) {
+    if ( $ConfigureOnly ) {
+        Write-Host "-PipRequirements cannot be used with -ConfigureOnly, no Salt is installed" -ForegroundColor Red
+        exit 1
+    }
+    try {
+        $PipRequirements = Resolve-PipRequirementsFile -Path $PipRequirements
+    } catch {
+        Write-Host $_.Exception.Message -ForegroundColor Red
+        exit 1
+    }
+    # Hand the resolved path to the elevated copy of the script
+    $PSBoundParameters["PipRequirements"] = $PipRequirements
+}
+
 #===============================================================================
 # Check for Elevated Privileges
 #===============================================================================
@@ -551,6 +666,7 @@ Write-Verbose "repourl: $base_url"
 Write-Verbose "apiurl: $api_url"
 Write-Verbose "ConfDir: $ConfDir"
 Write-Verbose "RootDir: $RootDir"
+Write-Verbose "PipRequirements: $PipRequirements"
 
 if ($RunService) {
     Write-Verbose "Windows service will be set to run"
@@ -770,6 +886,29 @@ while ( ! $service ) {
 # If we get this far, the service was installed, we have a service object
 Write-Host "Success" -ForegroundColor Green
 
+#===============================================================================
+# Install PyPI packages (for example Salt Extensions)
+#===============================================================================
+# The installer ran with /start-service=0, so the service has not started yet.
+# Installing here means the packages are in place the first time Salt runs.
+if ( $PipRequirements ) {
+    Write-Host "Installing Python packages from the requirements file:"
+    try {
+        $pipExitCode = Install-PipRequirements -Path $PipRequirements
+    } catch {
+        Write-Host "Failed" -ForegroundColor Red
+        Write-Host $_.Exception.Message
+        exit 1
+    }
+    if ( $pipExitCode -ne 0 ) {
+        Write-Host "Failed" -ForegroundColor Red
+        Write-Host "salt-pip exited with code $pipExitCode"
+        Write-Host "If a package has to be compiled, its build tools must already be installed"
+        exit 1
+    }
+    Write-Host "Success" -ForegroundColor Green
+}
+
 #===============================================================================
 # Configure the minion service
 #===============================================================================