diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index 3eeb993738..1726587a39 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -103,10 +103,14 @@ runs: run: | echo "::group::AMI Build: Stage 2" + system=$([[ ${{ inputs.arch }} == arm64 ]] && echo aarch64-linux || echo x86_64-linux) + site_env_path=$(nix eval --raw "github:supabase/postgres/${{ inputs.packages_git_sha }}#packages.$system.site-env-${{ inputs.postgres_version }}.outPath") + nix run .#build-ami -- stage2 ${{ inputs.arch }} \ -var "ami_name=${{ inputs.ami_name_prefix }}" \ -var "git-head-version=${{ inputs.git_sha }}" \ -var "git_sha=${{ inputs.packages_git_sha }}" \ + -var "site_env_path=$site_env_path" \ -var "instance_type=${{ inputs.instance_type }}" \ -var "packer-execution-id=$PACKER_EXECUTION_ID" \ -var "postgres_major_version=${{ inputs.postgres_version }}" \ diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 6df0eb678f..aa110fe420 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -199,6 +199,19 @@ jobs: echo "Catalog uploaded to ${CATALOG_S3}" + - name: Publish site-env store path + run: | + GIT_SHA="${{ steps.resolve-git-sha.outputs.sha }}" + SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') + SITE_ENV_NAME="site-env-${POSTGRES_MAJOR_VERSION}" + SITE_ENV_PATH=$(nix eval --raw ".#${SITE_ENV_NAME}.outPath") + + aws s3 cp - "s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}/${SYSTEM}/${SITE_ENV_NAME}" <<< "$SITE_ENV_PATH" + + if [[ "$GITHUB_REF" == refs/heads/release/* ]]; then + aws s3 cp - "s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/latest/${SYSTEM}/${SITE_ENV_NAME}" <<< "$GIT_SHA" + fi + - name: Create release uses: softprops/action-gh-release@v3.0.3 with: diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 37e50623e0..73c37d8e43 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -695,7 +695,7 @@ jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - -#dynamic_library_path = '$libdir' +dynamic_library_path = '/nix/var/nix/profiles/site/lib/postgresql:$libdir' #gin_fuzzy_search_limit = 0 #------------------------------------------------------------------------------ diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 9c75e81dc7..f296a2c643 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -71,6 +71,12 @@ nix-env --set {{ postgres_env_path.stdout }} " + - name: Install and activate site env from nix binary cache + ansible.builtin.shell: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && + nix-env --profile /nix/var/nix/profiles/site --set {{ site_env_path }} && + /nix/var/nix/profiles/site/bin/activate + - name: Set ownership and permissions for file and dirs ansible.builtin.file: group: 'postgres' diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index e8470fc635..f7b7f525a8 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -59,6 +59,7 @@ function execute_stage2_playbook { ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ --extra-vars '{"stage2":true, "qemu":false}' \ --extra-vars "git_commit_sha=$GIT_SHA" \ + --extra-vars "site_env_path=$SITE_ENV_PATH" \ --extra-vars "psql_version=psql_$POSTGRES_MAJOR_VERSION" \ --extra-vars "postgresql_version=postgresql_$POSTGRES_MAJOR_VERSION" \ --extra-vars "postgresql_major_version=$POSTGRES_MAJOR_VERSION" \ diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index c064454b0d..4be1585248 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -155,6 +155,7 @@ function execute_stage2_playbook { ansible-playbook ./ansible/playbook.yml \ --extra-vars '{"stage2": true, "qemu": true}' \ --extra-vars "git_commit_sha=$GIT_SHA" \ + --extra-vars "site_env_path=$SITE_ENV_PATH" \ --extra-vars "postgresql_version=postgresql_$POSTGRES_MAJOR_VERSION" \ --extra-vars "postgresql_major_version=$POSTGRES_MAJOR_VERSION" \ --extra-vars "postgresql_major=$POSTGRES_MAJOR_VERSION" \ diff --git a/nix/checks.nix b/nix/checks.nix index 42053caef4..e1ae89624c 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -933,6 +933,7 @@ } ) // pkgs.lib.optionalAttrs (pkgs.stdenv.isLinux) { + site = import ./tests/site.nix { inherit self pkgs; }; inherit (self'.packages) postgresql_15_debug postgresql_15_src diff --git a/nix/docs/README.md b/nix/docs/README.md index 8efe8e1c6b..8dfd8d1415 100644 --- a/nix/docs/README.md +++ b/nix/docs/README.md @@ -28,6 +28,7 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md). - **[Update Extensions](./update-extension.md)** - How to update existing extensions - **[Update Nix Dependecies](./updating-dependencies.md)** - How to update the Nix dependencies - **[New Major PostgreSQL](./new-major-postgres.md)** - Adding support for new PostgreSQL versions +- **[Site Profile](./site-profile.md)** - The site profile deployed to instances ## Testing diff --git a/nix/docs/site-profile.md b/nix/docs/site-profile.md new file mode 100644 index 0000000000..bbbc07eaa3 --- /dev/null +++ b/nix/docs/site-profile.md @@ -0,0 +1,52 @@ +# Site profile + +The `site-env-15`, `site-env-17`, and `site-env-orioledb-17` package sets are deployed to `/nix/var/nix/profiles/site`. They hold supautils, and `activate`. + +## Publish + +`ami-release-nix.yml` uploads each store path as a string to the shared artifacts bucket, which is private, in the object `nix-catalog///site-env-`. Release branches also upload the git sha as a string to `nix-catalog/latest//site-env-`. The AMI bake installs the store path passed in as `site_env_path` and runs `activate`. + +## Update + +```sh +nix-env --profile /nix/var/nix/profiles/site --set +/nix/var/nix/profiles/site/bin/activate +``` + +`--set` downloads the closure from the binary cache if it is missing. It then creates the generation link `site--link` to the store path and repoints the `site` symlink at that link. If it fails, `nix-env` exits with 1 and the profile is unchanged. This holds when the path can't be fetched, isn't a store path, or the profile directory isn't writable. + +`activate` runs after the switch. It prints the library settings configured in `postgresql.conf`, and is idempotent. If it fails, the new generation stays active. Run `activate` again, or roll back. + +New sessions load supautils from the new profile. Open sessions keep the copy they loaded until they reconnect. Postgres does not restart. A restart applies the change to every session at once. The same holds after a rollback. + +## Roll back + +```sh +nix-env --profile /nix/var/nix/profiles/site --rollback +/nix/var/nix/profiles/site/bin/activate +``` + +Rollback switches to the previous generation. `activate` then runs from that generation and makes the machine match it. There is no separate rollback script. + +To go further back than the previous generation, install an older version by path. + +## Install any version by path + +```sh +nix-env --profile /nix/var/nix/profiles/site --set +/nix/var/nix/profiles/site/bin/activate +``` + +This is the way to roll back past the one kept generation. It is the same as an update with an older path. + +## `activate` safety + +`activate` must be idempotent, and the previous generation's `activate` must be able to converge the machine back. Safety shall be ensured by comparing snapshots in tests before and after an update and a rollback. + +## Test + +```sh +nix build .#checks..site -L +``` + +The `site` VM test covers loading supautils from the site profile, the `$libdir` fallthrough, and `activate`. diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index aff35bf9bc..fcab1c3fce 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -28,9 +28,10 @@ stdenv.mkDerivation rec { patches = [ ./patches/supautils-strtol-glibc-compat.patch ]; installPhase = '' - mkdir -p $out/lib + mkdir -p $out/lib/postgresql install -D *${postgresql.dlSuffix} -t $out/lib + ln -s ../supautils${postgresql.dlSuffix} $out/lib/postgresql/supautils${postgresql.dlSuffix} ''; meta = with lib; { diff --git a/nix/mkdocs.yml b/nix/mkdocs.yml index 1d360b3cf9..aee14f27bf 100644 --- a/nix/mkdocs.yml +++ b/nix/mkdocs.yml @@ -26,6 +26,7 @@ nav: - Update Extensions: update-extension.md - Update Nix Dependencies: updating-dependencies.md - New Major PostgreSQL: new-major-postgres.md + - Site Profile: site-profile.md - Testing: - Adding Tests: adding-tests.md - Migration Tests: migration-tests.md diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index c6c36a2bf0..6b01d3dbb0 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -50,6 +50,8 @@ install --mode 444 "$code" "$workdir/ovmf_code.fd" install --mode 644 "$vars" "$workdir/ovmf_vars.fd" # qemu writes to the vars pflash during boot git_sha=${GIT_SHA:-$(git rev-parse HEAD)} +system=$([[ $arch == arm64 ]] && echo aarch64-linux || echo x86_64-linux) +site_env_path=$(nix eval --raw "github:supabase/postgres/$git_sha#packages.$system.site-env-$postgres_major_version.outPath") postgres_version=$(yq -er ".postgres_release.postgres$postgres_major_version" ansible/postgres_release.yml) # Build the cloud-init seed ISO before invoking packer. @@ -64,6 +66,7 @@ PACKER_LOG=${PACKER_LOG:-1} packer build \ -var "git_sha=$git_sha" \ -var "machine=$machine" \ -var "postgres-version=$postgres_version" \ + -var "site_env_path=$site_env_path" \ -var "postgres_major_version=$postgres_major_version" \ -var "qemu_binary=$qemu" \ -var "workdir=$workdir" \ diff --git a/nix/packages/site-env.nix b/nix/packages/site-env.nix index 4fa580e99a..82bc1692e3 100644 --- a/nix/packages/site-env.nix +++ b/nix/packages/site-env.nix @@ -11,9 +11,37 @@ let makeSiteEnv = version: extraPaths: + let + supautils = self'.legacyPackages."psql_${version}".exts.supautils; + # Site profile activation script. MUST STAY IDEMPOTENT! + activate = pkgs.writeTextFile { + name = "activate"; + executable = true; + destination = "/bin/activate"; + # Runs on the host bash to keep Nix's bash and glibc out of the closure. + text = '' + #!/usr/bin/env bash + set -euo pipefail + echo "Activating site profile." + echo "Configured in /etc/postgresql/postgresql.conf (runtime could differ):" + for guc in dynamic_library_path session_preload_libraries; do + grep -h "^$guc" /etc/postgresql/postgresql.conf 2>/dev/null || echo "$guc is not set" + done + echo "Site profile activated." + ''; + checkPhase = '' + ${pkgs.shellcheck}/bin/shellcheck "$target" + ''; + }; + in pkgs.buildEnv { name = "site-env-${version}"; - paths = [ self'.legacyPackages."psql_${version}".exts.supautils ] ++ extraPaths; + paths = [ + supautils + activate + ] + ++ extraPaths; + passthru = { inherit activate; }; }; siteEnvs = { diff --git a/nix/tests/site.nix b/nix/tests/site.nix new file mode 100644 index 0000000000..6f10e85df4 --- /dev/null +++ b/nix/tests/site.nix @@ -0,0 +1,50 @@ +{ self, pkgs }: +let + system = pkgs.pkgsLinux.stdenv.hostPlatform.system; + site-env-17 = self.packages.${system}."site-env-17"; + psql_17 = self.legacyPackages.${system}."psql_17".bin; + psql_17_slim = self.legacyPackages.${system}."psql_17_slim".bin; + pgConf = pkgs.writeText "postgresql-test.conf" '' + dynamic_library_path = '/nix/var/nix/profiles/site/lib/postgresql:$libdir' + session_preload_libraries = 'supautils' + unix_socket_directories = '/tmp' + ''; +in +pkgs.testers.runNixOSTest { + name = "site"; + nodes.machine = + { ... }: + { + environment.systemPackages = [ site-env-17 ]; + virtualisation.additionalPaths = [ psql_17 ]; + users.users.postgres = { + isSystemUser = true; + group = "postgres"; + shell = pkgs.bash; + }; + users.groups.postgres = { }; + }; + testScript = '' + # site profile absent: supautils loads from $libdir + machine.succeed("install -d -o postgres -g postgres /tmp/pgdata-slim") + machine.succeed("su postgres -c '${psql_17_slim}/bin/initdb -D /tmp/pgdata-slim'") + machine.succeed("install -o postgres -g postgres ${pgConf} /tmp/pgdata-slim/postgresql.conf") + machine.succeed("su postgres -c '${psql_17_slim}/bin/pg_ctl -D /tmp/pgdata-slim -l /tmp/pg-slim.log start'") + machine.succeed("su postgres -c '${psql_17_slim}/bin/psql -h /tmp -d postgres -c \"select 1\"'") + machine.succeed("su postgres -c '${psql_17_slim}/bin/pg_ctl -D /tmp/pgdata-slim stop'") + + machine.succeed("nix-env --profile /nix/var/nix/profiles/site --set ${site-env-17}") + + # without $libdir in the path, supautils can only come from the site profile + machine.succeed("install -d -o postgres -g postgres /tmp/pgdata") + machine.succeed("su postgres -c '${psql_17}/bin/initdb -D /tmp/pgdata'") + machine.succeed("install -o postgres -g postgres ${pgConf} /tmp/pgdata/postgresql.conf") + machine.succeed("sed -i 's/:\\$libdir//' /tmp/pgdata/postgresql.conf") + machine.succeed("grep -qx \"dynamic_library_path = '/nix/var/nix/profiles/site/lib/postgresql'\" /tmp/pgdata/postgresql.conf") + machine.succeed("su postgres -c '${psql_17}/bin/pg_ctl -D /tmp/pgdata -l /tmp/pg.log start'") + machine.succeed("su postgres -c '${psql_17}/bin/psql -h /tmp -d postgres -c \"select 1\"'") + machine.succeed("su postgres -c '${psql_17}/bin/pg_ctl -D /tmp/pgdata stop'") + + machine.succeed("${site-env-17}/bin/activate") + ''; +} diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index ab854240f0..ca7de9a240 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -2,6 +2,10 @@ variable "git_sha" { type = string } +variable "site_env_path" { + type = string +} + variable "postgres-version" { type = string default = "" @@ -109,6 +113,7 @@ build { provisioner "shell" { environment_vars = [ "GIT_SHA=${var.git_sha}", + "SITE_ENV_PATH=${var.site_env_path}", "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}" ] use_env_var_file = true diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 7278ccd2bf..d76d2e6496 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -27,6 +27,10 @@ variable "git_sha" { default = env("GIT_SHA") } +variable "site_env_path" { + type = string +} + variable "postgres_major_version" { type = string default = "" @@ -140,6 +144,7 @@ build { provisioner "shell" { environment_vars = [ "GIT_SHA=${var.git_sha}", + "SITE_ENV_PATH=${var.site_env_path}", "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}" ] use_env_var_file = true