diff --git a/.github/workflows/build-ecr-aws-beta-manual.yaml b/.github/workflows/build-ecr-aws-beta-manual.yaml index e6ce1d4..fdea5bd 100644 --- a/.github/workflows/build-ecr-aws-beta-manual.yaml +++ b/.github/workflows/build-ecr-aws-beta-manual.yaml @@ -1,18 +1,13 @@ name: Push Bundled OpenHound + AWS Beta Image to ECR -run-name: AWS beta bundle — ${{ inputs.tag }} — ${{ inputs.source_ref }} +run-name: AWS beta bundle — ${{ inputs.tag }} — ${{ github.ref_name }} on: workflow_dispatch: inputs: tag: - description: "Development version, e.g. 0.4.1.dev9346005 (without ecr-enterprise)" + description: "Development version, e.g. 0.5.0.dev9346005" required: true type: string - source_ref: - description: "OpenHound source branch or commit to bundle" - required: true - default: main - type: string aws_commit: description: "Full 40-character openhound-aws source commit SHA to bundle" required: true @@ -62,17 +57,10 @@ jobs: id-token: write runs-on: ubuntu-latest steps: - - name: Checkout bundle build tooling + - name: Checkout OpenHound source and bundle build tooling uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # ratchet:actions/checkout@v6 with: ref: ${{ github.sha }} - path: bundle - persist-credentials: false - - - name: Checkout OpenHound source - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # ratchet:actions/checkout@v6 - with: - ref: ${{ inputs.source_ref }} path: source persist-credentials: false @@ -84,6 +72,7 @@ jobs: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }} tags: | type=raw,prefix=ecr-enterprise,value=${{ inputs.tag }},enable=true + type=raw,value=candidate - name: Set up QEMU uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # ratchet:docker/setup-qemu-action@v4 @@ -115,9 +104,7 @@ jobs: uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # ratchet:docker/build-push-action@v7 with: context: ./source - file: ./bundle/Dockerfile.aws-bundle - build-contexts: | - bundle=./bundle + file: ./source/Dockerfile.aws-bundle target: enterprise platforms: linux/amd64,linux/arm64 push: true @@ -127,4 +114,4 @@ jobs: OPENHOUND_VERSION=${{ inputs.tag }} OPENHOUND_AWS_COMMIT=${{ inputs.aws_commit }} secrets: | - OPENHOUND_AWS_GITHUB_TOKEN=${{ steps.openhound-aws-token.outputs.token }} + OPENHOUND_AWS_APP_TOKEN=${{ steps.openhound-aws-token.outputs.token }} diff --git a/Dockerfile.aws-bundle b/Dockerfile.aws-bundle index a78129a..d3975af 100644 --- a/Dockerfile.aws-bundle +++ b/Dockerfile.aws-bundle @@ -1,6 +1,5 @@ # Managed AWS beta bundle only; see deployments/aws-extension-bundle.md. -# The named "bundle" context supplies reviewed build tooling independently of -# the selected OpenHound source context. +# Source and build tooling come from the selected workflow revision. FROM rust:1.96.0-slim-bookworm@sha256:4732ca96fd086cb9be682050c3f0176288eebaac2b80aa2bcefccfaf198e1950 AS rust FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim@sha256:531f855bda2c73cd6ef67d56b733b357cea384185b3022bd09f05e002cd144ca AS builder @@ -19,16 +18,16 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=${OPENHOUND_VERSION} WORKDIR /app COPY . /app -COPY --from=bundle scripts/prepare_aws_bundle.py scripts/smoke_aws_bundle.py /app/scripts/ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --extra all --frozen --no-install-project \ && export SETUPTOOLS_SCM_PRETEND_VERSION="$(.venv/bin/python -c 'import os; from packaging.version import Version; print(Version(os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"]))')" \ && uv sync --extra all --frozen # The workflow's required aws_commit input selects this immutable source revision. -# Read access to the internal repository is supplied only as a BuildKit secret. +# A short-lived GitHub App installation token supplies read access to the internal +# repository only through a BuildKit secret. ARG OPENHOUND_AWS_COMMIT -RUN --mount=type=secret,id=OPENHOUND_AWS_GITHUB_TOKEN,required=true \ +RUN --mount=type=secret,id=OPENHOUND_AWS_APP_TOKEN,required=true \ .venv/bin/python scripts/prepare_aws_bundle.py "${OPENHOUND_AWS_COMMIT}" RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/usr/local/cargo/registry \ diff --git a/deployments/aws-extension-bundle.md b/deployments/aws-extension-bundle.md index d875d50..afe0d30 100644 --- a/deployments/aws-extension-bundle.md +++ b/deployments/aws-extension-bundle.md @@ -1,90 +1,87 @@ -# Managed OpenHound + AWS beta images (BED-9518) - -Use **Push Bundled OpenHound + AWS Beta Image to ECR** to publish updated managed -OpenHound/AWS beta collector images. This workflow is **manual-only**: pushes, -merges, pull requests, and releases do not trigger it. - -It uses the same repository-access rules, ECR variables, and AWS publishing -credentials as **Build image for ECR**. Repository writers can run it; it adds no -code-owner restriction, environment approval, or team-membership requirement. - -## Required configuration - -Keep the existing `ECR_REGISTRY` and `ECR_REPOSITORY` repository variables and -`BHE_AWS_ACCESS_KEY_ID` / `BHE_AWS_SECRET_ACCESS_KEY` secrets. Bundled images go to -the same ECR location as regular OpenHound images. - -The additional secret **`OPENHOUND_AWS_GITHUB_TOKEN`** needs **Contents: read** on -the internal `SpecterOps/openhound-aws` repository, with organization approval/SSO -authorization if required. The ordinary workflow `GITHUB_TOKEN` cannot normally -read that repository. The source token is mounted as a BuildKit secret, never -passed as a build argument or saved in an image layer. - -## Publish a beta image - -Commit references for the workflow inputs: - -- **`aws_commit`:** [Recent openhound-aws commits](https://github.com/SpecterOps/openhound-aws/commits/main/). - Open the desired commit and copy its **full 40-character SHA**, not the - abbreviated value shown in the history. Repository access is required. -- **`source_ref`:** [Recent OpenHound commits](https://github.com/SpecterOps/OpenHound/commits/main/). - Use the branch selector to view feature-branch history when needed. - -1. In GitHub Actions, select **Push Bundled OpenHound + AWS Beta Image to ECR**. -2. Use the workflow from **`main`**. Set `source_ref` to `main`, or to the beta - branch/commit containing the managed functionality you want to build. -3. Set `tag` to a development version such as `0.4.1.dev9346005`, without `v` or - `ecr-enterprise`. Use the existing numbering convention and coordinate versions - with other publishers to the shared repository. -4. Set **`aws_commit`** to the full 40-character source commit SHA from - `SpecterOps/openhound-aws` that you want to bundle. This input is required and - has no default; branches, tags, and abbreviated SHAs are rejected by the build. -5. Run the workflow. It builds the `enterprise` image for AMD64 and ARM64 and - publishes only this version tag: - - ```text - ${ECR_REGISTRY}/${ECR_REPOSITORY}:ecr-enterprise0.4.1.dev9346005 - ``` - -6. Update the managed beta collector deployment through the infrastructure team's - existing process. Publishing an image does not update a running deployment. - -The Dockerfile and preparation/smoke scripts come from the selected **workflow -revision**, independently of `source_ref`. This lets -the workflow on `main` build a feature branch that predates these bundle files. -The AWS source revision is selected separately through `aws_commit`. The selected -OpenHound source still needs to contain the desired managed runtime features. - -After the workflow merges to `main`, it can also be started with the GitHub CLI. -Set `AWS_COMMIT_SHA` to the desired full AWS source commit SHA first: +# Bundled OpenHound + AWS beta images + +Use **Push Bundled OpenHound + AWS Beta Image to ECR** to publish a bundled +OpenHound/AWS beta collector image. The workflow runs manually and uses the +configured GitHub App to access `SpecterOps/openhound-aws`. + +## Run in GitHub Actions + +1. Select **Push Bundled OpenHound + AWS Beta Image to ECR**, then **Run workflow**. +2. Under **Use workflow from**, select the OpenHound branch to build: `main` or + the feature branch containing the changes you want. This selects both the + OpenHound source and the bundle build tooling; the branch must include this + version of the workflow. +3. Set **`tag`** to a new development version such as `0.5.0.dev9346005`, without + `v` or `ecr-enterprise`. Coordinate versions with other publishers to the shared + ECR repository. +4. Set **`aws_commit`** to the full **40-character SHA** of the AWS extension + revision to bundle. Copy it from the desired commit in + [openhound-aws history](https://github.com/SpecterOps/openhound-aws/commits/main/). + This is required; the build rejects branches, tags, and abbreviated SHAs. +5. Leave **`verify_access_only`** unchecked and run the workflow. + +### Beta version convention + +Use development versions to distinguish beta builds from stable releases. For +example, `0.5.0.dev9346005` consists of: + +- `0.5.0`: the OpenHound base version. +- `.dev`: the Python development-release marker used for these beta builds. +- `9346`: the beta project's tracking identifier. +- `005`: the release counter, starting at `000`; `005` is the sixth beta image + in this sequence. + +Increment the counter for each release and use a new version for each published +build. This is a project naming convention; Python interprets the entire +`9346005` suffix as one development-release number. + +The workflow builds the `enterprise` image for AMD64 and ARM64 and publishes: + +```text +${ECR_REGISTRY}/${ECR_REPOSITORY}:ecr-enterprise0.5.0.dev9346005 +${ECR_REGISTRY}/${ECR_REPOSITORY}:candidate +``` + +Both tags point to the same multi-architecture image. Each successful publish +moves `candidate` to the newly published beta image. The version tag remains +available for that build. +The ECR repository must allow the `candidate` tag to be overwritten. + +Publishing does not update a running deployment. See the deployment guidance +below. + +To check GitHub App access without building or publishing, enable +**`verify_access_only`**. Supply the required `tag` and `aws_commit` inputs; this +mode only checks out the requested AWS revision. + +## Run with the GitHub CLI + +Set `AWS_COMMIT_SHA` to the desired full AWS source commit SHA, then run: ```sh gh workflow run build-ecr-aws-beta-manual.yaml --ref main \ - -f tag=0.4.1.dev9346005 -f source_ref=main -f aws_commit="$AWS_COMMIT_SHA" + -f tag=0.5.0.dev9346005 -f aws_commit="$AWS_COMMIT_SHA" ``` -## Include newer AWS extension changes - -Run the workflow with the desired **`aws_commit`** and a new development version. -Selecting a newer extension revision does not require editing files or merging a -SHA update to `main`. Each build fetches exactly the supplied commit instead of -automatically following the AWS repository's latest commit. Local Docker builds -must likewise pass `--build-arg OPENHOUND_AWS_COMMIT=`. - -When selecting a newer commit, consider compatibility with -`scripts/prepare_aws_bundle.py`. -The preparation script patches the temporary extension's OpenHound requirement -to the installed local version, raises its boto3 requirement to `>=1.43.90`, and -places `extension.yaml` inside the Python package. Changed upstream packaging may -require updating those patches. - -The build preserves OpenHound's installed, locked dependencies while resolving -the extension's declared requirements. It runs `uv pip check` and an offline smoke -check inside the final nonroot image on each architecture. These checks verify -extension discovery, metadata, and native imports only when the bundle is -manually built. No extra automatic workflow test suite is added. - -When bringing `main` into older beta branches, retain its standard -`build-ecr-manual.yaml` and `Dockerfile`, plus the separate bundle files. Replace -the old temporary in-place bundle modifications. Rebasing is not required just -to make the new workflow appear in Actions. +Replace `main` with the OpenHound branch to build. Add +`-f verify_access_only=true` for an access-only check. To include newer AWS +extension changes, run again with the desired SHA and a new development version. + +Bundle preparation updates the AWS extension's OpenHound requirements to the +selected OpenHound build version, regardless of the original version pins. Other +requirements, including boto3, are preserved and resolved against the frozen +OpenHound build environment. Incompatible requirements fail the build before the +image tag is published to ECR; reconcile the dependency requirements before +retrying. + +## Deployment guidance + +> [!WARNING] +> The linked guide includes examples specific to BHE. For OpenHound collectors, +> always explicitly pass `-p bhe-collector` to Commute commands. Verify the product +> and destination in the confirmation prompt before approving a deployment. + +For managed deployments, authorized operators should follow the internal +[Commute documentation](https://specterops.atlassian.net/wiki/spaces/BE/pages/356679809/Commute) +and the infrastructure team's current collector release guidance for setup, +target selection, promotion, deployment, and rollback procedures. diff --git a/scripts/prepare_aws_bundle.py b/scripts/prepare_aws_bundle.py index e8dda94..9ac3eec 100644 --- a/scripts/prepare_aws_bundle.py +++ b/scripts/prepare_aws_bundle.py @@ -9,22 +9,37 @@ from pathlib import Path from urllib.request import Request, urlopen +import tomlkit +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from packaging.version import Version def patch_packaging(destination: Path, local_version: str) -> None: - """Keep the selected OpenHound build installed; reject changed AWS metadata.""" + """Select the local OpenHound build while preserving other AWS requirements.""" metadata = destination / "pyproject.toml" - text = metadata.read_text() - replacements = { - '"openhound==0.2.13"': (f'"openhound=={Version(local_version)}"', 2), - '"boto3==1.43.68"': ('"boto3>=1.43.90"', 1), - } - for old, (new, expected_count) in replacements.items(): - if text.count(old) != expected_count: - raise ValueError(f"Unexpected AWS packaging metadata: {old}") - text = text.replace(old, new) - metadata.write_text(text) + document = tomlkit.parse(metadata.read_text()) + project = document.get("project", {}) + dependency_lists = [project.get("dependencies", [])] + dependency_lists.extend(project.get("optional-dependencies", {}).values()) + dependency_lists.append(document.get("build-system", {}).get("requires", [])) + dependency_lists.extend(document.get("dependency-groups", {}).values()) + selected_version = Version(local_version) + for dependencies in dependency_lists: + for index, dependency in enumerate(dependencies): + # Dependency groups may also contain include-group tables. + if not isinstance(dependency, str): + continue + requirement = Requirement(dependency) + if canonicalize_name(requirement.name) == "openhound": + extras = ( + f"[{','.join(sorted(requirement.extras))}]" + if requirement.extras + else "" + ) + marker = f"; {requirement.marker}" if requirement.marker else "" + dependencies[index] = f"openhound{extras}=={selected_version}{marker}" + metadata.write_text(tomlkit.dumps(document)) # Maturin's root-level include does not place the manifest inside the Python # package, where CollectorManager reads it via importlib.resources. shutil.copyfile( @@ -36,9 +51,9 @@ def patch_packaging(destination: Path, local_version: str) -> None: def main(commit: str) -> None: if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("OPENHOUND_AWS_COMMIT must be a full, pinned commit SHA") - token = Path("/run/secrets/OPENHOUND_AWS_GITHUB_TOKEN").read_text().strip() + token = Path("/run/secrets/OPENHOUND_AWS_APP_TOKEN").read_text().strip() if not token: - raise ValueError("OPENHOUND_AWS_GITHUB_TOKEN must not be empty") + raise ValueError("OPENHOUND_AWS_APP_TOKEN must not be empty") request = Request( f"https://api.github.com/repos/SpecterOps/openhound-aws/tarball/{commit}", headers={