diff --git a/CHANGELOG.md b/CHANGELOG.md index b0ea1aafc..3b6bd9325 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* **role:mongodb**: The dump script is installed as `/usr/local/libexec/mongodb-dump`, since it only works when started by `mongodb-dump.service`. * **role:system_update**: On Debian and Ubuntu, the reboot notification lists the packages and services that asked for the reboot. * **role:system_update**: On Debian and Ubuntu, the weekly update removes the packages it left unused itself, typically replaced libraries, like unattended-upgrades does. Packages that were unused before stay installed. * **role:elasticsearch**: The query log of Elasticsearch 9.5 (`*_querylog.json`) is deleted after `elasticsearch__log4j2_retention_days`, like the other logs. @@ -328,6 +329,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +* **role:mongodb**: The password of the dump user is stored encrypted with `systemd-creds` on systemd 251 and newer instead of in plain text in `/etc/mongodb-dump-login.json`, so it is no longer readable in a backup of `/etc`. On RHEL 8 and Ubuntu 22.04 it stays in plain text, in `/etc/credstore/`. * **role:mariadb_server**: The login of the dump user is stored encrypted with `systemd-creds` on systemd 251 and newer instead of in plain text in `/etc/mariadb-dump-login.cnf`, so it is no longer readable in a backup of `/etc`. On RHEL 8 and Ubuntu 22.04 it stays in plain text, in `/etc/credstore/`. * **role:mongodb**: The daily dump no longer passes the password of the dump user on the command line, where every local user could read it in the process list. * **role:mariadb_server**: The daily dump and the log rotation no longer pass MariaDB passwords on the command line, where every local user could read them in the process list. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73cd7b47e..347c6df87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -551,12 +551,12 @@ A secret that a systemd service reads from a file (a daemon, or a script run by * Store it with `linuxfabrik.lfops.systemd_credential` and register the result. The module encrypts the credential into `/etc/credstore.encrypted/` with systemd 251 and newer, and otherwise writes it in plain text to `/etc/credstore/` (`0700` directory, `0600` file, root). * In the unit, add `LoadCredentialEncrypted=` only when the registered result is `encrypted`, so the unit loads the credential from the store the module chose. * The service reads `${CREDENTIALS_DIRECTORY:-/etc/credstore}/`. A script aborts with a hint to run the unit if the file is not readable: started by hand, outside the unit, it cannot decrypt the credential. -* Use a template rendered through `lookup('ansible.builtin.template', ...)` as `content`, below `templates/etc/credstore/`. It keeps the header and version line, but is never deployed itself, so there is no `backup: true` copy of the secret. +* Use a template rendered through `lookup('ansible.builtin.template', ...)` as `content`, below `templates/etc/credstore/`. It keeps the header and version line, but is never deployed itself, so there is no `backup: true` copy of the secret. Exception: build a JSON credential inline with `to_json` (e.g. `'{{ {"password": ...} | to_json }}'`), since ansible-core before 2.19 turns a rendered string that looks like a dict into one, and the module then stores its Python repr. * When moving an existing secret file to a credential, delete the old file and its `backup: true` copies. * Name the credential after the file it replaces, e.g. `mariadb-dump-login.cnf`. * Document in the role README where the secret ends up per platform, that a script has to run via `systemctl start `, and the TPM2 caveat: with a TPM2 chip, the credential is bound to the Secure Boot state, so after a firmware or Secure Boot change the unit cannot read it until the role runs again. * If the service only takes the secret inline in its config, the pattern does not apply without a wrapper that renders the config at start; keep the config file `0600` and owned by the service user. -* `roles/mariadb_server` (`mariadb-dump`) implements this pattern. +* `roles/mariadb_server` (`mariadb-dump`) and `roles/mongodb` (`mongodb-dump`) implement this pattern. #### OS-specific Variables @@ -684,7 +684,7 @@ Before adding an `allow` rule, work out what it buys an attacker **on top of** t * `apache_solr`: OpenJDK package per platform and Solr major version via `vars/.yml` (RHEL 10 and Debian 13 ship no Java 17), asserted per platform; `security.json` hashes computed on the controller with a host- and user-derived salt, so the file only changes with a password; `solr.in.sh` and `log4j2.xml` per Solr major (`9-`, `10-`), since Solr 10 renamed their variables. * `mariadb_server`: the dump login is a systemd credential, see "Secrets for systemd Services". -* `mongodb`: `state: skip` in `mongodb__databases` / `mongodb__users` leaves an entry untouched. +* `mongodb`: `state: skip` in `mongodb__databases` / `mongodb__users` leaves an entry untouched; the dump login is a systemd credential, see "Secrets for systemd Services". * `moodle`: patch tag discovered via `api.github.com/repos/moodle/moodle/tags`, first match of `^v`. * `nextcloud`: a state file skips install-only tasks after the initial installation; concise "Tags" README section. * `php`: present/absent split of `php__modules__combined_var` into two batched package calls. diff --git a/extensions/molecule/mongodb/inventory/group_vars/systems_under_test.yml b/extensions/molecule/mongodb/inventory/group_vars/systems_under_test.yml index d1699eb92..0dc0acaa2 100644 --- a/extensions/molecule/mongodb/inventory/group_vars/systems_under_test.yml +++ b/extensions/molecule/mongodb/inventory/group_vars/systems_under_test.yml @@ -7,11 +7,11 @@ mongodb__conf_security_authorization: true # Locks the instance as the dump user, so this checks the privileges the role grants it. mongodb__dump_method_file_based: true -# The special characters check that the login file escapes the password correctly for both -# mongodump and mongosh. +# The special characters check that the login escapes the password correctly for both +# mongodump and mongosh. verify.yml searches /etc for the marker `MoleculeDumpPw`. mongodb__dump_user: username: 'mongodb-dump' - password: 'linux"fab\rik #''x:@/%' + password: 'MoleculeDumpPw"fab\rik #''x:@/%' # A regular user, so that the idempotence step also covers the user loop. mongodb__users__group_var: diff --git a/extensions/molecule/mongodb/verify.yml b/extensions/molecule/mongodb/verify.yml index 6464a02de..a60aeaa7c 100644 --- a/extensions/molecule/mongodb/verify.yml +++ b/extensions/molecule/mongodb/verify.yml @@ -1,4 +1,4 @@ -# The dump user logs in with the password from /etc/mongodb-dump-login.json, so a dump that +# The dump user logs in with the password from the systemd credential, so a dump that # contains the test database proves that mongosh and mongodump both read it. The file-based # method locks the instance as the dump user, which proves the privileges the role grants it. - name: 'Verify MongoDB and its dump' @@ -113,3 +113,27 @@ ansible.builtin.assert: that: '"./molecule/test.bson" in __molecule__mongodb_dump_list_result["stdout_lines"]' fail_msg: '{{ __molecule__mongodb_dump_list_result["stdout_lines"] }}' + + # The dump above read the password from the systemd credential. With systemd 251 and newer it + # is encrypted, so the password must not be anywhere in /etc in plain text; older systemd + # keeps it in /etc/credstore only. + - name: 'systemctl show --property=Version --value' + ansible.builtin.command: 'systemctl show --property=Version --value' + register: '__molecule__systemd_version_result' + changed_when: false + + - name: 'grep --recursive --files-with-matches MoleculeDumpPw /etc' + ansible.builtin.command: 'grep --recursive --fixed-strings --files-with-matches MoleculeDumpPw /etc' + register: '__molecule__dump_password_files_result' + changed_when: false + failed_when: '__molecule__dump_password_files_result["rc"] > 1' + + - name: 'Assert that the dump password is only stored where expected' + ansible.builtin.assert: + that: '__molecule__dump_password_files_result["stdout_lines"] == __molecule__expected_files' + fail_msg: 'Found in: {{ __molecule__dump_password_files_result["stdout_lines"] }}' + vars: + __molecule__expected_files: '{{ + [] if (__molecule__systemd_version_result["stdout"] | regex_search("^\d+") | int) >= 251 + else ["/etc/credstore/mongodb-dump-login.json"] + }}' diff --git a/roles/mongodb/README.md b/roles/mongodb/README.md index f96a8725c..b05f7424e 100644 --- a/roles/mongodb/README.md +++ b/roles/mongodb/README.md @@ -16,7 +16,9 @@ This role is only compatible with the following MongoDB versions: ## How the Role Behaves * `mongodb-dump.service` runs with `Nice=19`, best-effort IO priority 7 and `CPUWeight=` / `IOWeight=` of `20` (systemd default: `100`), so the dump does not slow down the application on a busy host. The weights only take effect under contention. `IOWeight=` only has an effect on block devices that use the BFQ IO scheduler; with `none` or `mq-deadline`, as on a RHEL 9 VM with virtio disks, it is ignored. The IO class is deliberately not `idle`: the file-based method keeps the instance write-locked while it copies the data directory, and an idle copy could stretch that lock. -* The dump reads the password of `mongodb__dump_user` from `/etc/mongodb-dump-login.json`, which only root can read, so it does not show up in the process list. +* The password of `mongodb__dump_user` is stored as the systemd credential `mongodb-dump-login.json`. With systemd 251 and newer (RHEL 9 and newer, Debian 12 and newer, Ubuntu 24.04 and newer), it is encrypted in `/etc/credstore.encrypted/` with the host key `/var/lib/systemd/credential.secret`, which the default backup does not include, and `mongodb-dump.service` decrypts it to `/run/credentials/mongodb-dump.service/` while it runs. On RHEL 8 and Ubuntu 22.04, it stays in plain text in `/etc/credstore/`, readable by root only. Root can read it either way. The password does not show up in the process list. +* Run the dump with `systemctl start mongodb-dump.service`. Started directly, `/usr/local/libexec/mongodb-dump` cannot read the encrypted password. +* On a host with a TPM2 chip, the encrypted password is also bound to the Secure Boot state. After a firmware or Secure Boot update, the dump fails until the role runs again. * `mongodb-dump.service` fails if `mongodump` fails, so a broken dump shows up as a failed unit. * With `mongodb__dump_method_file_based`, `mongodb-dump.service` write-locks the instance (`db.fsyncLock()`) while it copies the data directory, and fails without copying anything if it cannot lock it. For this, the role grants `mongodb__dump_user` the MongoDB role `lfopsFsyncLock`, which only allows the `fsync` and `unlock` actions, in addition to the built-in `backup` role. Disabling the file-based method drops the role again. diff --git a/roles/mongodb/tasks/main.yml b/roles/mongodb/tasks/main.yml index b6ca0d1fc..d28b95ec2 100644 --- a/roles/mongodb/tasks/main.yml +++ b/roles/mongodb/tasks/main.yml @@ -395,28 +395,70 @@ group: 'root' mode: 0o600 - - name: 'Deploy /etc/mongodb-dump-login.json' - ansible.builtin.template: - backup: true - src: 'etc/mongodb-dump-login.json.j2' - dest: '/etc/mongodb-dump-login.json' - owner: 'root' - group: 'root' - mode: 0o600 # contains secrets + # encrypted with systemd 251 and newer, see the systemd_credential module. The service loads + # credentials on every start, so a changed one needs no restart. Read by `mongodump --config`, + # which rejects any key besides `password`, `uri` and `sslPEMKeyPassword`, and by mongosh. + # Built inline instead of from a template: ansible-core before 2.19 turns a rendered string + # that looks like a dict into one, which the module would then store as its Python repr; an + # expression ending in `to_json` is exempt. + - name: 'Store the dump login as the systemd credential mongodb-dump-login.json' + linuxfabrik.lfops.systemd_credential: + name: 'mongodb-dump-login.json' + content: '{{ {"password": mongodb__dump_user["password"]} | to_json }}' + register: '__mongodb__dump_login_credential_result' when: - 'mongodb__conf_security_authorization | bool' - 'mongodb__dump_user is defined' - 'mongodb__dump_user | length > 0' - - name: 'Deploy /usr/local/sbin/mongodb-dump' + # without a dump user, the dump logs in without a password + - name: 'Remove the systemd credential mongodb-dump-login.json' + linuxfabrik.lfops.systemd_credential: + name: 'mongodb-dump-login.json' + state: 'absent' + when: 'not (mongodb__conf_security_authorization | bool) or mongodb__dump_user is not defined or mongodb__dump_user | length == 0' + + # the plain-text predecessor of the credential, including the copies `backup: true` left + - name: 'find /etc -maxdepth 1 -name mongodb-dump-login.json*' + ansible.builtin.find: + paths: '/etc' + patterns: 'mongodb-dump-login.json*' + hidden: true + register: '__mongodb__dump_login_plain_result' + + - name: 'rm -f /etc/mongodb-dump-login.json*' + ansible.builtin.file: + path: '{{ item["path"] }}' + state: 'absent' + loop: '{{ __mongodb__dump_login_plain_result["files"] }}' + loop_control: + label: '{{ item["path"] }}' + + # /usr/local/libexec is missing on Debian before trixie and Ubuntu before 26.04, see + # roles/schedule_reboot. + - name: 'Ensure /usr/local/libexec exists' + ansible.builtin.file: + path: '/usr/local/libexec' + state: 'directory' + owner: 'root' + group: 'root' + mode: 0o755 + + # Only mongodb-dump.service can run the script, since only the unit gets the decrypted login. + - name: 'Deploy /usr/local/libexec/mongodb-dump' ansible.builtin.template: backup: true - src: 'usr/local/sbin/mongodb-dump.j2' - dest: '/usr/local/sbin/mongodb-dump' + src: 'usr/local/libexec/mongodb-dump.j2' + dest: '/usr/local/libexec/mongodb-dump' owner: 'root' group: 'root' mode: 0o755 + - name: 'rm -f /usr/local/sbin/mongodb-dump' + ansible.builtin.file: + path: '/usr/local/sbin/mongodb-dump' + state: 'absent' + - name: 'Deploy /etc/systemd/system/mongodb-dump.service' ansible.builtin.template: backup: true diff --git a/roles/mongodb/templates/etc/mongodb-dump-login.json.j2 b/roles/mongodb/templates/etc/mongodb-dump-login.json.j2 deleted file mode 100644 index e837ce1db..000000000 --- a/roles/mongodb/templates/etc/mongodb-dump-login.json.j2 +++ /dev/null @@ -1,2 +0,0 @@ -{#- 2026102701. JSON has no comments, so this file carries no ansible_managed header. It is read by `mongodump --config`, which rejects any key besides `password`, `uri` and `sslPEMKeyPassword`, and by mongosh in /usr/local/sbin/mongodb-dump. -#} -{"password": {{ mongodb__dump_user['password'] | to_json }}} diff --git a/roles/mongodb/templates/etc/mongodb-dump.conf.j2 b/roles/mongodb/templates/etc/mongodb-dump.conf.j2 index 33503f741..3d5885e09 100644 --- a/roles/mongodb/templates/etc/mongodb-dump.conf.j2 +++ b/roles/mongodb/templates/etc/mongodb-dump.conf.j2 @@ -1,11 +1,11 @@ # {{ ansible_managed }} -# 2026102702 +# 2026102901 HOST='127.0.0.1' PORT='{{ mongodb__conf_net_port }}' {% if mongodb__conf_security_authorization | bool and mongodb__dump_user is defined and mongodb__dump_user | length %} -# the password is in /etc/mongodb-dump-login.json +# the password is in the systemd credential mongodb-dump-login.json USERNAME='{{ mongodb__dump_user["username"] }}' AUTH_DATABASE='{{ mongodb__dump_user["auth_database"] | default("admin") }}' {% endif %} diff --git a/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 b/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 index 581618390..60e05cb6f 100644 --- a/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 +++ b/roles/mongodb/templates/etc/systemd/system/mongodb-dump.service.j2 @@ -1,13 +1,17 @@ # {{ ansible_managed }} -# 2026101301 +# 2026102901 [Unit] Description=MongoDB Dump Service [Service] -ExecStart=/usr/local/sbin/mongodb-dump +ExecStart=/usr/local/libexec/mongodb-dump Type=oneshot User=root +{% if __mongodb__dump_login_credential_result['encrypted'] | d(false) %} +# decrypted to $CREDENTIALS_DIRECTORY on every start +LoadCredentialEncrypted=mongodb-dump-login.json +{% endif %} # the dump must not slow down mongod and the applications behind it. Nice= and the CPU / IO # weights (default 100) only matter under contention, so an idle host dumps at full speed. # best-effort instead of idle: with `mongodb__dump_method_file_based` the instance stays diff --git a/roles/mongodb/templates/usr/local/sbin/mongodb-dump.j2 b/roles/mongodb/templates/usr/local/libexec/mongodb-dump.j2 similarity index 81% rename from roles/mongodb/templates/usr/local/sbin/mongodb-dump.j2 rename to roles/mongodb/templates/usr/local/libexec/mongodb-dump.j2 index a1915f4b2..15854bb86 100644 --- a/roles/mongodb/templates/usr/local/sbin/mongodb-dump.j2 +++ b/roles/mongodb/templates/usr/local/libexec/mongodb-dump.j2 @@ -2,7 +2,7 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# 2026102702 +# 2026102901 set -e @@ -13,28 +13,35 @@ umask 027 source /etc/mongodb-dump.conf # the password of the dump user, kept out of the command line (and the process list). mongodump -# reads it with `--config`, mongosh in mongosh_eval(). -login_file=/etc/mongodb-dump-login.json +# reads it with `--config`, mongosh in mongosh_eval(). With systemd 251 and newer, +# mongodb-dump.service decrypts it to $CREDENTIALS_DIRECTORY, older systemd keeps it in plain +# text in /etc/credstore. +login_file="${CREDENTIALS_DIRECTORY:-/etc/credstore}/mongodb-dump-login.json" auth_params=() if [[ -n "${USERNAME:-}" ]]; then + if [ ! -r "$login_file" ]; then + echo "Cannot read $login_file. Run the dump with 'systemctl start mongodb-dump.service'." >&2 + exit 1 + fi auth_params=(--authenticationDatabase "$AUTH_DATABASE" --username "$USERNAME") fi # run the JavaScript in $1 as the dump user. mongosh prompts for a password given without a -# value, so it logs in from the script; the user name comes through the environment instead of -# being quoted into the JavaScript. +# value, so it logs in from the script; the user name and the login file come through the +# environment instead of being quoted into the JavaScript. function mongosh_eval() { local login_js='' if [[ -n "${USERNAME:-}" ]]; then login_js=" - const login = JSON.parse(require('fs').readFileSync('$login_file', 'utf8')); + const login = JSON.parse(require('fs').readFileSync(process.env.MONGODB_DUMP_LOGIN_FILE, 'utf8')); if (!db.getSiblingDB(process.env.MONGODB_DUMP_AUTH_DATABASE).auth(process.env.MONGODB_DUMP_USERNAME, login.password).ok) { throw new Error('authentication failed'); } " fi - MONGODB_DUMP_AUTH_DATABASE="${AUTH_DATABASE:-}" MONGODB_DUMP_USERNAME="${USERNAME:-}" \ + MONGODB_DUMP_AUTH_DATABASE="${AUTH_DATABASE:-}" MONGODB_DUMP_LOGIN_FILE="$login_file" \ + MONGODB_DUMP_USERNAME="${USERNAME:-}" \ mongosh --quiet --host "$HOST" --port "$PORT" --eval "$login_js $1" }