From 8717fea45c32d506ecf661d318d6220ee10e3f45 Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 27 Oct 2026 16:54:00 +0100 Subject: [PATCH 1/2] feat(tools/follow-upstream): support per-series keys, .x series names and directory ignores `series_suffix` names a series `3.x` in the template header and in `supported_versions`. `series_gpg_key` sets another key for one series; with `sha1: true` the RPM check switches the container to DEFAULT:SHA1, as rpm on Rocky 9 rejects a key bound with a SHA-1 signature. An `ignore` entry ending in `/` ignores a whole directory. The RPM/DEB comparison skips templates based on a later release, e.g. one without `{series}` that follows the newest series only. --- tools/follow-upstream | 46 +++++++++++++++++--- tools/follow-upstream-container/fetch-rpm.sh | 6 +++ tools/follow-upstream.yml | 9 +++- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/tools/follow-upstream b/tools/follow-upstream index 69b791624..5d3692eef 100755 --- a/tools/follow-upstream +++ b/tools/follow-upstream @@ -57,7 +57,7 @@ import sys import yaml __author__ = 'Linuxfabrik GmbH, Zurich/Switzerland' -__version__ = '2026102703' +__version__ = '2026102704' REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) MANIFEST = os.path.join(REPO_ROOT, 'tools', 'follow-upstream.yml') @@ -122,6 +122,11 @@ def repo_series(config, version): return series_of(version, config.get('repo_series_parts', config['series_parts'])) +def series_name(config, series): + """The series as the templates and `supported_versions` name it: `3.x` for `3`.""" + return series + config.get('series_suffix', '') + + def find_app(manifest, name): if name in manifest: return name, manifest[name] @@ -212,7 +217,13 @@ def fetch(app, config, fmt, package, version): shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) series = repo_series(config, version) - gpg_key = expand(config['gpg_key'].replace('{series}', series)) + gpg_key = config['gpg_key'] + sha1 = False + # a series signed with another key than the current one (OpenSearch 2.x) + if series in config.get('series_gpg_key', {}): + gpg_key = config['series_gpg_key'][series]['url'] + sha1 = config['series_gpg_key'][series].get('sha1', False) + gpg_key = expand(gpg_key.replace('{series}', series)) if fmt == 'rpm': display = config['rpm_repo'].replace('{series}', series) env = {'REPO': expand(display)} @@ -232,6 +243,7 @@ def fetch(app, config, fmt, package, version): { 'EXTRACT': ' '.join(extra), 'GPG_KEY': gpg_key, + 'GPG_SHA1': '1' if sha1 else '', 'PACKAGE': package, 'REPO_DISPLAY': display, 'VERSION': version, @@ -273,6 +285,8 @@ def check_packaging(app, config, version): """ findings = 0 ignore = set(config.get('ignore', [])) + # an entry ending in `/` ignores everything below that directory + ignore_dirs = tuple(path for path in ignore if path.endswith('/')) fmts = ('rpm', 'deb') if 'deb_repo' in config else ('rpm',) for package in sorted({entry['package'] for entry in config['files']}): entries = [entry for entry in config['files'] if entry['package'] == package] @@ -285,6 +299,8 @@ def check_packaging(app, config, version): for fmt in fmts: for path in sorted(files[fmt] - known): + if path.startswith(ignore_dirs): + continue if not PATH_RE.fullmatch(path): path = f'a file with an unusual name below {roots[fmt]}' print( @@ -294,6 +310,13 @@ def check_packaging(app, config, version): findings += 1 for entry in entries: + # a template based on a later release does not take this one, e.g. one + # without `{series}` that follows the newest series only (OpenSearch's + # sysconfig and a 2.x release) + source, _ = source_template(config, entry, version) + base = read_based_on(source) if source else None + if base and version_key(base) > version_key(version): + continue rpm_path = entry['path'] deb_path = entry.get('deb_path', rpm_path) missing = False @@ -531,7 +554,12 @@ def update_file(app, config, entry, version, dry_run): with open(source, encoding='utf-8') as f: text = f.read() text = update_header( - text, entry['package'], version, old_series, new_series, bump=False + text, + entry['package'], + version, + series_name(config, old_series), + series_name(config, new_series), + bump=False, ) write_template(entry, target, text) return 'unchanged' @@ -559,7 +587,11 @@ def update_file(app, config, entry, version, dry_run): sys.exit(f'git merge-file failed: {merge.stderr}') conflicts = merge.returncode merged = update_header( - merge.stdout, entry['package'], version, old_series, new_series + merge.stdout, + entry['package'], + version, + series_name(config, old_series), + series_name(config, new_series), ) status = f'{conflicts} conflict(s)' if conflicts else 'merged cleanly' @@ -569,8 +601,10 @@ def update_file(app, config, entry, version, dry_run): return 'conflicts' if conflicts else 'merged' write_template(entry, target, merged) if source != target: - add_supported_version(entry['supported_versions'], new_series) - report_old_series(merged, old_series) + add_supported_version( + entry['supported_versions'], series_name(config, new_series) + ) + report_old_series(merged, series_name(config, old_series)) return 'conflicts' if conflicts else 'merged' diff --git a/tools/follow-upstream-container/fetch-rpm.sh b/tools/follow-upstream-container/fetch-rpm.sh index 5f5b2ed9c..2cdb656b3 100644 --- a/tools/follow-upstream-container/fetch-rpm.sh +++ b/tools/follow-upstream-container/fetch-rpm.sh @@ -15,6 +15,12 @@ if [ -z "${url}" ]; then fail "${PACKAGE} ${VERSION} not found in ${REPO_DISPLAY}" fi get "${GPG_KEY}" --output /var/tmp/key +# a key bound with a SHA-1 signature (OpenSearch 2.x) fails to import under the default +# crypto policy: "Hash algorithm SHA1 not available" (verified against rpm 4.16 on +# Rocky 9). gpgv in fetch-deb.sh accepts it as is. +if [ -n "${GPG_SHA1}" ]; then + update-crypto-policies --set DEFAULT:SHA1 >/dev/null +fi rpmkeys --import /var/tmp/key get "${url}" --output /var/tmp/package.rpm # `rpmkeys --checksig` also succeeds for an unsigned package, it then prints `digests OK` diff --git a/tools/follow-upstream.yml b/tools/follow-upstream.yml index e33e280aa..6d8464e21 100644 --- a/tools/follow-upstream.yml +++ b/tools/follow-upstream.yml @@ -15,10 +15,16 @@ # * `gpg_key`: the key the packages (RPM) and the repository metadata (DEB) are signed # with, the same one the `repo_*` role imports. `{series}` is replaced. A package that # is not signed with it is not extracted. +# * `series_gpg_key`: optional, `{: {url: , sha1: }}` for a series +# signed with another key than `gpg_key`. `sha1: true` lets the RPM check accept a key +# bound with a SHA-1 signature, which the default crypto policy of Rocky 9 rejects. # * `rpm_repo`: Yum repository serving every release of a series, `{series}` is replaced. # `tools/follow-upstream --check` looks for new releases here. # * `repo_series_parts`: optional, how many version components `{series}` has in the # repository URLs, if not `series_parts` (2: one repository per `26.04`). +# * `series_suffix`: optional, appended to the series in the template header and in +# `supported_versions` (`.x`: series `3` is named `3.x` there). `{series}` in paths +# and URLs stays without it. # # A repository URL can reference an environment variable, e.g. a customer token: # `https://example.com/repo-${EXAMPLE_TOKEN}/`. An application whose variable is not set @@ -34,7 +40,8 @@ # * `copies`: further templates that stay identical to `template`, e.g. the Debian # counterpart of a file the RPM ships under another path. They get the merge result. # * `ignore`: packaged files below /etc that LFOps deliberately does not template. -# Every other file a release ships is reported, so a file upstream adds is noticed. +# Every other file a release ships is reported, so a file upstream adds is noticed. An +# entry ending in `/` ignores every file below that directory. # # The comment above each application records how its install scripts treat the templated # files, for the release that was checked. `tools/follow-upstream` reports later changes From 01a44ae377470420289d81bf7972caf19c21d15f Mon Sep 17 00:00:00 2001 From: Navid Sassan Date: Tue, 27 Oct 2026 16:54:01 +0100 Subject: [PATCH 2/2] feat(tools/follow-upstream): follow OpenSearch releases --- tools/follow-upstream.yml | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tools/follow-upstream.yml b/tools/follow-upstream.yml index 6d8464e21..5ff2e6cf1 100644 --- a/tools/follow-upstream.yml +++ b/tools/follow-upstream.yml @@ -241,6 +241,65 @@ influxdb: path: '/etc/influxdb/influxdb.conf' template: 'roles/influxdb/templates/etc/influxdb/influxdb.conf.j2' +# One template per major version (`3.x-opensearch.yml.j2`); sysconfig serves both and +# follows the newest major only. The RPM ships the environment file as +# /etc/sysconfig/opensearch. The DEB of 3.x ships it as /etc/default/opensearch, plus an +# outdated copy as /etc/sysconfig/opensearch (2.x: the other way round). The systemd unit +# reads both, /etc/sysconfig last, and the opensearch role deploys /etc/sysconfig on +# every platform. +# +# Install scripts (opensearch 2.19.7 and 3.9.0, RPM and DEB): create the user, append +# the Performance Analyzer settings to jvm.options once, chown /etc/opensearch to +# opensearch, and on every installation and upgrade run the demo security configuration. +# It appends a demo `plugins.security` block to opensearch.yml, but quits if the file +# already sets a `plugins.security` key, which the template always does. +opensearch: + upstream: + - 'opensearch-project/OpenSearch' + series_parts: 1 + series_suffix: '.x' + deb_repo: 'https://artifacts.opensearch.org/releases/bundle/opensearch/{series}.x/apt stable main' + gpg_key: 'https://artifacts.opensearch.org/publickeys/opensearch-release.pgp' + series_gpg_key: + '2': + url: 'https://artifacts.opensearch.org/publickeys/opensearch.pgp' + sha1: true + rpm_repo: 'https://artifacts.opensearch.org/releases/bundle/opensearch/{series}.x/yum/' + files: + - package: 'opensearch' + path: '/etc/opensearch/opensearch.yml' + template: 'roles/opensearch/templates/etc/opensearch/{series}.x-opensearch.yml.j2' + supported_versions: 'roles/opensearch/vars/main.yml:__opensearch__supported_versions' + - package: 'opensearch' + path: '/etc/sysconfig/opensearch' + deb_path: '/etc/default/opensearch' + template: 'roles/opensearch/templates/etc/sysconfig/opensearch.j2' + ignore: + # the packaged defaults are used + - '/etc/init.d/opensearch' + - '/etc/opensearch/fips_java.security' + - '/etc/opensearch/jvm.options' + - '/etc/opensearch/log4j2.properties' + - '/etc/opensearch/opensearch-notifications-core/notifications-core.yml' + - '/etc/opensearch/opensearch-notifications/notifications.yml' + - '/etc/opensearch/opensearch-observability/observability.yml' + - '/etc/opensearch/opensearch-performance-analyzer/' # 2.x only + - '/etc/opensearch/opensearch-reports-scheduler/reports-scheduler.yml' + - '/etc/opensearch/opensearch-security/action_groups.yml' + - '/etc/opensearch/opensearch-security/allowlist.yml' + - '/etc/opensearch/opensearch-security/audit.yml' + - '/etc/opensearch/opensearch-security/config.yml' + - '/etc/opensearch/opensearch-security/nodes_dn.yml' + - '/etc/opensearch/opensearch-security/opensearch.yml.example' + - '/etc/opensearch/opensearch-security/roles.yml' + - '/etc/opensearch/opensearch-security/roles_mapping.yml' + - '/etc/opensearch/opensearch-security/tenants.yml' + - '/etc/opensearch/opensearch-security/whitelist.yml' # 2.x only + # Sigma detection rules of the Security Analytics plugin, about 2300 files + - '/etc/opensearch/opensearch-security-analytics/' + # the opensearch role writes it from scratch, it is no copy of the packaged file + - '/etc/opensearch/opensearch-security/internal_users.yml' + # Install scripts (proxysql 3.0.11, RPM): create the user and apply the systemd preset to # proxysql.service on the first installation, never touch the templated file. One # template serves 2.7 and 3.0. The DEB is not checked: the role supports RHEL only, and