feat(tools/follow-upstream): follow OpenSearch releases

tools/follow-upstream

@@ -57,7 +57,7 @@ import sys
 import yaml
 
 __author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
-__version__ = '2026102703'
+__version__ = '2026102704'
 
 REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 MANIFEST = os.path.join(REPO_ROOT, 'tools', 'follow-upstream.yml')
@@ -122,6 +122,11 @@ def repo_series(config, version):
     return series_of(version, config.get('repo_series_parts', config['series_parts']))
 
 
+def series_name(config, series):
+    """The series as the templates and `supported_versions` name it: `3.x` for `3`."""
+    return series + config.get('series_suffix', '')
+
+
 def find_app(manifest, name):
     if name in manifest:
         return name, manifest[name]
@@ -212,7 +217,13 @@ def fetch(app, config, fmt, package, version):
     shutil.rmtree(dest, ignore_errors=True)
     os.makedirs(dest)
     series = repo_series(config, version)
-    gpg_key = expand(config['gpg_key'].replace('{series}', series))
+    gpg_key = config['gpg_key']
+    sha1 = False
+    # a series signed with another key than the current one (OpenSearch 2.x)
+    if series in config.get('series_gpg_key', {}):
+        gpg_key = config['series_gpg_key'][series]['url']
+        sha1 = config['series_gpg_key'][series].get('sha1', False)
+    gpg_key = expand(gpg_key.replace('{series}', series))
     if fmt == 'rpm':
         display = config['rpm_repo'].replace('{series}', series)
         env = {'REPO': expand(display)}
@@ -232,6 +243,7 @@ def fetch(app, config, fmt, package, version):
         {
             'EXTRACT': ' '.join(extra),
             'GPG_KEY': gpg_key,
+            'GPG_SHA1': '1' if sha1 else '',
             'PACKAGE': package,
             'REPO_DISPLAY': display,
             'VERSION': version,
@@ -273,6 +285,8 @@ def check_packaging(app, config, version):
     """
     findings = 0
     ignore = set(config.get('ignore', []))
+    # an entry ending in `/` ignores everything below that directory
+    ignore_dirs = tuple(path for path in ignore if path.endswith('/'))
     fmts = ('rpm', 'deb') if 'deb_repo' in config else ('rpm',)
     for package in sorted({entry['package'] for entry in config['files']}):
         entries = [entry for entry in config['files'] if entry['package'] == package]
@@ -285,6 +299,8 @@ def check_packaging(app, config, version):
 
         for fmt in fmts:
             for path in sorted(files[fmt] - known):
+                if path.startswith(ignore_dirs):
+                    continue
                 if not PATH_RE.fullmatch(path):
                     path = f'a file with an unusual name below {roots[fmt]}'
                 print(
@@ -294,6 +310,13 @@ def check_packaging(app, config, version):
                 findings += 1
 
         for entry in entries:
+            # a template based on a later release does not take this one, e.g. one
+            # without `{series}` that follows the newest series only (OpenSearch's
+            # sysconfig and a 2.x release)
+            source, _ = source_template(config, entry, version)
+            base = read_based_on(source) if source else None
+            if base and version_key(base) > version_key(version):
+                continue
             rpm_path = entry['path']
             deb_path = entry.get('deb_path', rpm_path)
             missing = False
@@ -531,7 +554,12 @@ def update_file(app, config, entry, version, dry_run):
             with open(source, encoding='utf-8') as f:
                 text = f.read()
             text = update_header(
-                text, entry['package'], version, old_series, new_series, bump=False
+                text,
+                entry['package'],
+                version,
+                series_name(config, old_series),
+                series_name(config, new_series),
+                bump=False,
             )
             write_template(entry, target, text)
         return 'unchanged'
@@ -559,7 +587,11 @@ def update_file(app, config, entry, version, dry_run):
         sys.exit(f'git merge-file failed: {merge.stderr}')
     conflicts = merge.returncode
     merged = update_header(
-        merge.stdout, entry['package'], version, old_series, new_series
+        merge.stdout,
+        entry['package'],
+        version,
+        series_name(config, old_series),
+        series_name(config, new_series),
     )
 
     status = f'{conflicts} conflict(s)' if conflicts else 'merged cleanly'
@@ -569,8 +601,10 @@ def update_file(app, config, entry, version, dry_run):
         return 'conflicts' if conflicts else 'merged'
     write_template(entry, target, merged)
     if source != target:
-        add_supported_version(entry['supported_versions'], new_series)
-        report_old_series(merged, old_series)
+        add_supported_version(
+            entry['supported_versions'], series_name(config, new_series)
+        )
+        report_old_series(merged, series_name(config, old_series))
     return 'conflicts' if conflicts else 'merged'
 
 

tools/follow-upstream-container/fetch-rpm.sh

@@ -15,6 +15,12 @@ if [ -z "${url}" ]; then
     fail "${PACKAGE} ${VERSION} not found in ${REPO_DISPLAY}"
 fi
 get "${GPG_KEY}" --output /var/tmp/key
+# a key bound with a SHA-1 signature (OpenSearch 2.x) fails to import under the default
+# crypto policy: "Hash algorithm SHA1 not available" (verified against rpm 4.16 on
+# Rocky 9). gpgv in fetch-deb.sh accepts it as is.
+if [ -n "${GPG_SHA1}" ]; then
+    update-crypto-policies --set DEFAULT:SHA1 >/dev/null
+fi
 rpmkeys --import /var/tmp/key
 get "${url}" --output /var/tmp/package.rpm
 # `rpmkeys --checksig` also succeeds for an unsigned package, it then prints `digests OK`

tools/follow-upstream.yml

@@ -15,10 +15,16 @@
 # * `gpg_key`: the key the packages (RPM) and the repository metadata (DEB) are signed
 #   with, the same one the `repo_*` role imports. `{series}` is replaced. A package that
 #   is not signed with it is not extracted.
+# * `series_gpg_key`: optional, `{<series>: {url: <key>, sha1: <bool>}}` for a series
+#   signed with another key than `gpg_key`. `sha1: true` lets the RPM check accept a key
+#   bound with a SHA-1 signature, which the default crypto policy of Rocky 9 rejects.
 # * `rpm_repo`: Yum repository serving every release of a series, `{series}` is replaced.
 #   `tools/follow-upstream --check` looks for new releases here.
 # * `repo_series_parts`: optional, how many version components `{series}` has in the
 #   repository URLs, if not `series_parts` (2: one repository per `26.04`).
+# * `series_suffix`: optional, appended to the series in the template header and in
+#   `supported_versions` (`.x`: series `3` is named `3.x` there). `{series}` in paths
+#   and URLs stays without it.
 #
 # A repository URL can reference an environment variable, e.g. a customer token:
 # `https://example.com/repo-${EXAMPLE_TOKEN}/`. An application whose variable is not set
@@ -34,7 +40,8 @@
 #   * `copies`: further templates that stay identical to `template`, e.g. the Debian
 #     counterpart of a file the RPM ships under another path. They get the merge result.
 # * `ignore`: packaged files below /etc that LFOps deliberately does not template.
-#   Every other file a release ships is reported, so a file upstream adds is noticed.
+#   Every other file a release ships is reported, so a file upstream adds is noticed. An
+#   entry ending in `/` ignores every file below that directory.
 #
 # The comment above each application records how its install scripts treat the templated
 # files, for the release that was checked. `tools/follow-upstream` reports later changes
@@ -234,6 +241,65 @@ influxdb:
       path: '/etc/influxdb/influxdb.conf'
       template: 'roles/influxdb/templates/etc/influxdb/influxdb.conf.j2'
 
+# One template per major version (`3.x-opensearch.yml.j2`); sysconfig serves both and
+# follows the newest major only. The RPM ships the environment file as
+# /etc/sysconfig/opensearch. The DEB of 3.x ships it as /etc/default/opensearch, plus an
+# outdated copy as /etc/sysconfig/opensearch (2.x: the other way round). The systemd unit
+# reads both, /etc/sysconfig last, and the opensearch role deploys /etc/sysconfig on
+# every platform.
+#
+# Install scripts (opensearch 2.19.7 and 3.9.0, RPM and DEB): create the user, append
+# the Performance Analyzer settings to jvm.options once, chown /etc/opensearch to
+# opensearch, and on every installation and upgrade run the demo security configuration.
+# It appends a demo `plugins.security` block to opensearch.yml, but quits if the file
+# already sets a `plugins.security` key, which the template always does.
+opensearch:
+  upstream:
+    - 'opensearch-project/OpenSearch'
+  series_parts: 1
+  series_suffix: '.x'
+  deb_repo: 'https://artifacts.opensearch.org/releases/bundle/opensearch/{series}.x/apt stable main'
+  gpg_key: 'https://artifacts.opensearch.org/publickeys/opensearch-release.pgp'
+  series_gpg_key:
+    '2':
+      url: 'https://artifacts.opensearch.org/publickeys/opensearch.pgp'
+      sha1: true
+  rpm_repo: 'https://artifacts.opensearch.org/releases/bundle/opensearch/{series}.x/yum/'
+  files:
+    - package: 'opensearch'
+      path: '/etc/opensearch/opensearch.yml'
+      template: 'roles/opensearch/templates/etc/opensearch/{series}.x-opensearch.yml.j2'
+      supported_versions: 'roles/opensearch/vars/main.yml:__opensearch__supported_versions'
+    - package: 'opensearch'
+      path: '/etc/sysconfig/opensearch'
+      deb_path: '/etc/default/opensearch'
+      template: 'roles/opensearch/templates/etc/sysconfig/opensearch.j2'
+  ignore:
+    # the packaged defaults are used
+    - '/etc/init.d/opensearch'
+    - '/etc/opensearch/fips_java.security'
+    - '/etc/opensearch/jvm.options'
+    - '/etc/opensearch/log4j2.properties'
+    - '/etc/opensearch/opensearch-notifications-core/notifications-core.yml'
+    - '/etc/opensearch/opensearch-notifications/notifications.yml'
+    - '/etc/opensearch/opensearch-observability/observability.yml'
+    - '/etc/opensearch/opensearch-performance-analyzer/'  # 2.x only
+    - '/etc/opensearch/opensearch-reports-scheduler/reports-scheduler.yml'
+    - '/etc/opensearch/opensearch-security/action_groups.yml'
+    - '/etc/opensearch/opensearch-security/allowlist.yml'
+    - '/etc/opensearch/opensearch-security/audit.yml'
+    - '/etc/opensearch/opensearch-security/config.yml'
+    - '/etc/opensearch/opensearch-security/nodes_dn.yml'
+    - '/etc/opensearch/opensearch-security/opensearch.yml.example'
+    - '/etc/opensearch/opensearch-security/roles.yml'
+    - '/etc/opensearch/opensearch-security/roles_mapping.yml'
+    - '/etc/opensearch/opensearch-security/tenants.yml'
+    - '/etc/opensearch/opensearch-security/whitelist.yml'  # 2.x only
+    # Sigma detection rules of the Security Analytics plugin, about 2300 files
+    - '/etc/opensearch/opensearch-security-analytics/'
+    # the opensearch role writes it from scratch, it is no copy of the packaged file
+    - '/etc/opensearch/opensearch-security/internal_users.yml'
+
 # Install scripts (proxysql 3.0.11, RPM): create the user and apply the systemd preset to
 # proxysql.service on the first installation, never touch the templated file. One
 # template serves 2.7 and 3.0. The DEB is not checked: the role supports RHEL only, and