From 1d108e288900b7d9f892165f39c5ff9360156652 Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Wed, 7 Oct 2026 15:26:53 -0500 Subject: [PATCH 1/6] Add Config with Local Storage Service --- .../workflow-gateway-don-cre-storage.toml | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml diff --git a/core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml b/core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml new file mode 100644 index 00000000000..4f4fcd3d0e1 --- /dev/null +++ b/core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml @@ -0,0 +1,105 @@ +# workflow-gateway-don-grpc-source.toml with workflows served by cre-storage-service (its local-stack on ctf). +# Start with `-e 4566` so the gateway can fetch artifacts from storage-s3. + +[chip_router] + image = "local-cre-chip-router:v1.0.1" + +[[blockchains]] + type = "anvil" + chain_id = "1337" + container_name = "anvil-1337" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[[blockchains]] + type = "anvil" + chain_id = "2337" + container_name = "anvil-2337" + port = "8546" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[jd] + csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44" # any random 32 byte hex string + # change to your version + image = "job-distributor:0.28.0" + +[fake] + port = 8171 + +[fake_http] + port = 8666 + +#[s3provider] +# # use all defaults +# port = 9000 +# console_port = 9001 + +[infra] + # either "docker" or "kubernetes" + type = "docker" + +[[nodesets]] + nodes = 4 + name = "workflow" + don_families = ["test-don-family"] + don_types = ["workflow"] + override_mode = "all" + http_port_range_start = 10100 + + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"] + registry_based_launch_allowlist = ["cron-trigger@1.0.0", "dontime@1.0.0"] + + # See ./examples/workflow-don-overrides.toml to learn how to override capability configs + + [nodesets.db] + image = "postgres:15.0" + port = 13000 + +[[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ +[CRE.WorkflowFetcher] +URL = '' + +[Capabilities.WorkflowRegistry.WorkflowStorage] +URL = 'storage-node-service:50052' +TLSEnabled = false +ArtifactStorageHost = 'storage-s3:4566' + +[[Capabilities.WorkflowRegistry.AdditionalSources]] +URL = 'storage-node-service:50052' +TLSEnabled = false +Name = 'cre-storage' +""" + +[[nodesets]] + nodes = 1 + name = "bootstrap-gateway" + don_families = ["test-don-family"] + don_types = ["bootstrap", "gateway"] + override_mode = "each" + http_port_range_start = 10300 + + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + supported_evm_chains = [1337, 2337] + + [nodesets.db] + image = "postgres:15.0" + port = 13200 + + [[nodesets.node_specs]] + roles = ["bootstrap", "gateway"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # 5002 is the web API capabilities port for incoming requests + # 15002 is the vault port for incoming requests + custom_ports = ["5002:5002","15002:15002"] + # image = "chainlink-tmp:latest" + user_config_overrides = "" From 0cf49a5dca07f2b7faddadfd37f850db0c78bcbc Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Thu, 8 Oct 2026 16:25:45 -0500 Subject: [PATCH 2/6] Local CRE: Central Service Composable Docker Compose configs/extensions/ includes extension stacks (e.g. cre-storage-service) that answer to role aliases on ctf; workflow-gateway-don-extensions.toml points workflow nodes at those roles. Replaces the storage-specific topology. Co-Authored-By: Claude Opus 5.5 --- .../environment/configs/extensions/README.md | 22 +++++++++++ .../configs/extensions/compose.yaml | 6 +++ .../configs/extensions/extensions.env | 6 +++ ...l => workflow-gateway-don-extensions.toml} | 30 ++++++++------- .../cre/environment/docs/TOPOLOGIES.md | 1 + .../workflow-gateway-don-extensions.md | 38 +++++++++++++++++++ 6 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 core/scripts/cre/environment/configs/extensions/README.md create mode 100644 core/scripts/cre/environment/configs/extensions/compose.yaml create mode 100644 core/scripts/cre/environment/configs/extensions/extensions.env rename core/scripts/cre/environment/configs/{workflow-gateway-don-cre-storage.toml => workflow-gateway-don-extensions.toml} (74%) create mode 100644 core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md diff --git a/core/scripts/cre/environment/configs/extensions/README.md b/core/scripts/cre/environment/configs/extensions/README.md new file mode 100644 index 00000000000..b567c9cf1cc --- /dev/null +++ b/core/scripts/cre/environment/configs/extensions/README.md @@ -0,0 +1,22 @@ +# Local CRE extensions + +Services run next to Local CRE from their own repos' compose files, listed in `compose.yaml`. Each joins the +`ctf` network and answers to role aliases that `workflow-gateway-don-extensions.toml` points the workflow nodes at. + +| Role (alias on `ctf`) | Port | Workflow node config | +| --- | --- | --- | +| `cre-workflow-source` | 50052 | `Capabilities.WorkflowRegistry.AdditionalSources` | +| `cre-artifact-api` | 50052 | `Capabilities.WorkflowRegistry.WorkflowStorage.URL` | +| `cre-artifact-store` | 4566 | `WorkflowStorage.ArtifactStorageHost`; start with `-e 4566` so the gateway can fetch from it | +| `cre-deploy-api` | 50051 | deploy tooling (`registryclient`) | + +One service per role. Inputs (`CRE_LOCAL_*`) describe this environment and default to `extensions.env`; set them in +the shell to override. In this topology workflows come from the extensions, not `env workflow deploy`; deploy tooling +runs in a container on `ctf`, since artifact URLs use `cre-artifact-store`. + +```bash +CTF_CONFIGS=configs/workflow-gateway-don-extensions.toml go run . env start -e 4566 +export CRE_STORAGE_COMPOSE=/local-stack/compose.yaml +docker compose -f configs/extensions/compose.yaml up -d --wait +docker restart $(docker ps --format '{{.Names}}' | grep '^workflow-node') +``` diff --git a/core/scripts/cre/environment/configs/extensions/compose.yaml b/core/scripts/cre/environment/configs/extensions/compose.yaml new file mode 100644 index 00000000000..85aab211cbd --- /dev/null +++ b/core/scripts/cre/environment/configs/extensions/compose.yaml @@ -0,0 +1,6 @@ +# Extension stacks started alongside Local CRE. Each joins ctf and answers to the roles in README.md. +name: cre-extensions + +include: + - path: ${CRE_STORAGE_COMPOSE:?path to cre-storage-service/local-stack/compose.yaml} + env_file: extensions.env diff --git a/core/scripts/cre/environment/configs/extensions/extensions.env b/core/scripts/cre/environment/configs/extensions/extensions.env new file mode 100644 index 00000000000..c98be145249 --- /dev/null +++ b/core/scripts/cre/environment/configs/extensions/extensions.env @@ -0,0 +1,6 @@ +CRE_NETWORK=ctf +CRE_NETWORK_EXTERNAL=true +CRE_LOCAL_REGISTRY_RPC_URL=http://anvil-1337:8545 +CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=3379446385462418246 +CRE_LOCAL_REGISTRY_ADDRESS=0x5FbDB2315678afecb367f032d93F642f64180aa3 +CRE_LOCAL_DON_FAMILY=test-don-family diff --git a/core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml b/core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml similarity index 74% rename from core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml rename to core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml index 4f4fcd3d0e1..83e867c3157 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-don-cre-storage.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml @@ -1,5 +1,9 @@ -# workflow-gateway-don-grpc-source.toml with workflows served by cre-storage-service (its local-stack on ctf). -# Start with `-e 4566` so the gateway can fetch artifacts from storage-s3. +# Workflow DON with gateway, reading workflows from extension stacks (configs/extensions). +# This topology is the same as workflow-gateway-don.toml but the workflow nodes use the extension roles: +# AdditionalSources at cre-workflow-source and WorkflowStorage at cre-artifact-api / cre-artifact-store. +# Start with `-e 4566` so the gateway can fetch artifacts from cre-artifact-store. +# +# Used by: configs/extensions/README.md [chip_router] image = "local-cre-chip-router:v1.0.1" @@ -43,14 +47,13 @@ don_families = ["test-don-family"] don_types = ["workflow"] override_mode = "all" - http_port_range_start = 10100 + http_port_range_start = 10000 + p2p_port_range_start = 12000 - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"] registry_based_launch_allowlist = ["cron-trigger@1.0.0", "dontime@1.0.0"] - # See ./examples/workflow-don-overrides.toml to learn how to override capability configs - [nodesets.db] image = "postgres:15.0" port = 13000 @@ -67,14 +70,14 @@ URL = '' [Capabilities.WorkflowRegistry.WorkflowStorage] -URL = 'storage-node-service:50052' +URL = 'cre-artifact-api:50052' TLSEnabled = false -ArtifactStorageHost = 'storage-s3:4566' +ArtifactStorageHost = 'cre-artifact-store:4566' [[Capabilities.WorkflowRegistry.AdditionalSources]] -URL = 'storage-node-service:50052' +URL = 'cre-workflow-source:50052' TLSEnabled = false -Name = 'cre-storage' +Name = 'cre-extensions' """ [[nodesets]] @@ -83,14 +86,15 @@ Name = 'cre-storage' don_families = ["test-don-family"] don_types = ["bootstrap", "gateway"] override_mode = "each" - http_port_range_start = 10300 + http_port_range_start = 10100 + p2p_port_range_start = 12100 - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } supported_evm_chains = [1337, 2337] [nodesets.db] image = "postgres:15.0" - port = 13200 + port = 13100 [[nodesets.node_specs]] roles = ["bootstrap", "gateway"] diff --git a/core/scripts/cre/environment/docs/TOPOLOGIES.md b/core/scripts/cre/environment/docs/TOPOLOGIES.md index 19fcd86d256..1c830ab38a4 100644 --- a/core/scripts/cre/environment/docs/TOPOLOGIES.md +++ b/core/scripts/cre/environment/docs/TOPOLOGIES.md @@ -11,6 +11,7 @@ This file is generated by `go run . topology generate`. Do not edit manually. | `configs/workflow-gateway-don-aptos.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-aptos.md)) | | `configs/workflow-gateway-don-cache-soak-test.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-cache-soak-test.md)) | | `configs/workflow-gateway-don-cache-test.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-cache-test.md)) | +| `configs/workflow-gateway-don-extensions.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-extensions.md)) | | `configs/workflow-gateway-don-grpc-source.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-grpc-source.md)) | | `configs/workflow-gateway-don.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don.md)) | | `configs/workflow-gateway-sharded-5-dons.toml` | `sharded` | 7 ([details](topologies/workflow-gateway-sharded-5-dons.md)) | diff --git a/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md b/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md new file mode 100644 index 00000000000..36b199863f7 --- /dev/null +++ b/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md @@ -0,0 +1,38 @@ +# DON Topology + +- Config: `configs/workflow-gateway-don-extensions.toml` +- Class: `single-don` +- Infra: `docker` + +## Capability Matrix + +This matrix is the source of truth for capability placement by DON. + +| Capability | `bootstrap-gateway` | `workflow` | +|---|---|---| +| `consensus` | `-` | `local` | +| `cron` | `-` | `local` | +| `don-time` | `-` | `local` | +| `evm` | `-` | `local (1337,2337)` | +| `http-action` | `-` | `local` | +| `http-trigger` | `-` | `local` | +| `vault` | `-` | `local` | + +## DONs + +### `bootstrap-gateway` + +- Types: `bootstrap`, `gateway` +- Nodes: `1` +- Roles: `bootstrap`, `gateway` +- EVM chains: `1337,2337` +- Exposes remote capabilities: `false` + +### `workflow` + +- Types: `workflow` +- Nodes: `4` +- Roles: `plugin` +- EVM chains: `1337,2337` +- Exposes remote capabilities: `false` + From c81043ebb4079d90be08e24a08976fbdbae36b2c Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Mon, 12 Oct 2026 08:50:39 -0500 Subject: [PATCH 3/6] Use Local CRE environment extensions for Local Central Services --- .../environment/configs/extensions/README.md | 18 ++- .../configs/extensions/compose.yaml | 6 - .../configs/extensions/extensions.env | 6 - .../configs/extensions/extensions.toml | 3 + .../workflow-gateway-don-extensions.toml | 109 --------------- .../cre/environment/docs/TOPOLOGIES.md | 1 - .../workflow-gateway-don-extensions.md | 38 ------ .../environment/environment/environment.go | 16 +++ .../cre/environment/environment/extensions.go | 126 ++++++++++++++++++ .../environment/extensions_test.go | 56 ++++++++ .../lib/cre/environment/config/config.go | 7 + system-tests/lib/cre/features/vault/vault.go | 15 +-- 12 files changed, 223 insertions(+), 178 deletions(-) delete mode 100644 core/scripts/cre/environment/configs/extensions/compose.yaml delete mode 100644 core/scripts/cre/environment/configs/extensions/extensions.env create mode 100644 core/scripts/cre/environment/configs/extensions/extensions.toml delete mode 100644 core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml delete mode 100644 core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md create mode 100644 core/scripts/cre/environment/environment/extensions.go create mode 100644 core/scripts/cre/environment/environment/extensions_test.go diff --git a/core/scripts/cre/environment/configs/extensions/README.md b/core/scripts/cre/environment/configs/extensions/README.md index b567c9cf1cc..9dfc275ab82 100644 --- a/core/scripts/cre/environment/configs/extensions/README.md +++ b/core/scripts/cre/environment/configs/extensions/README.md @@ -1,22 +1,20 @@ # Local CRE extensions -Services run next to Local CRE from their own repos' compose files, listed in `compose.yaml`. Each joins the -`ctf` network and answers to role aliases that `workflow-gateway-don-extensions.toml` points the workflow nodes at. +Centralized CRE services run from their own repos' compose files. With an `[extensions]` section (`extensions.toml`), +`env start` starts them on `ctf` after the DONs, passing the registry chain, its CapabilitiesRegistry and the workflow +DON family, and points the workflow nodes at the roles they answer to. `env stop` removes them. | Role (alias on `ctf`) | Port | Workflow node config | | --- | --- | --- | | `cre-workflow-source` | 50052 | `Capabilities.WorkflowRegistry.AdditionalSources` | | `cre-artifact-api` | 50052 | `Capabilities.WorkflowRegistry.WorkflowStorage.URL` | -| `cre-artifact-store` | 4566 | `WorkflowStorage.ArtifactStorageHost`; start with `-e 4566` so the gateway can fetch from it | +| `cre-artifact-store` | 4566 | `WorkflowStorage.ArtifactStorageHost`, allowed for the gateway | | `cre-deploy-api` | 50051 | deploy tooling (`registryclient`) | -One service per role. Inputs (`CRE_LOCAL_*`) describe this environment and default to `extensions.env`; set them in -the shell to override. In this topology workflows come from the extensions, not `env workflow deploy`; deploy tooling -runs in a container on `ctf`, since artifact URLs use `cre-artifact-store`. +`storage_compose` defaults to a `cre-storage-service` checkout next to this repo; its images are built locally (see its +`local-stack/README.md`). Workflows then come from the extensions, not `env workflow deploy`; deploy tooling runs in a +container on `ctf`, since artifact URLs use `cre-artifact-store`. ```bash -CTF_CONFIGS=configs/workflow-gateway-don-extensions.toml go run . env start -e 4566 -export CRE_STORAGE_COMPOSE=/local-stack/compose.yaml -docker compose -f configs/extensions/compose.yaml up -d --wait -docker restart $(docker ps --format '{{.Names}}' | grep '^workflow-node') +CTF_CONFIGS=configs/workflow-gateway-don.toml,configs/extensions/extensions.toml go run . env start ``` diff --git a/core/scripts/cre/environment/configs/extensions/compose.yaml b/core/scripts/cre/environment/configs/extensions/compose.yaml deleted file mode 100644 index 85aab211cbd..00000000000 --- a/core/scripts/cre/environment/configs/extensions/compose.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Extension stacks started alongside Local CRE. Each joins ctf and answers to the roles in README.md. -name: cre-extensions - -include: - - path: ${CRE_STORAGE_COMPOSE:?path to cre-storage-service/local-stack/compose.yaml} - env_file: extensions.env diff --git a/core/scripts/cre/environment/configs/extensions/extensions.env b/core/scripts/cre/environment/configs/extensions/extensions.env deleted file mode 100644 index c98be145249..00000000000 --- a/core/scripts/cre/environment/configs/extensions/extensions.env +++ /dev/null @@ -1,6 +0,0 @@ -CRE_NETWORK=ctf -CRE_NETWORK_EXTERNAL=true -CRE_LOCAL_REGISTRY_RPC_URL=http://anvil-1337:8545 -CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=3379446385462418246 -CRE_LOCAL_REGISTRY_ADDRESS=0x5FbDB2315678afecb367f032d93F642f64180aa3 -CRE_LOCAL_DON_FAMILY=test-don-family diff --git a/core/scripts/cre/environment/configs/extensions/extensions.toml b/core/scripts/cre/environment/configs/extensions/extensions.toml new file mode 100644 index 00000000000..e3dd9099c6d --- /dev/null +++ b/core/scripts/cre/environment/configs/extensions/extensions.toml @@ -0,0 +1,3 @@ +# Starts the extension stacks (README.md) with any topology. Add it after the topology in CTF_CONFIGS. +[extensions] + storage_compose = "../../../../../cre-storage-service/local-stack/compose.yaml" diff --git a/core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml b/core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml deleted file mode 100644 index 83e867c3157..00000000000 --- a/core/scripts/cre/environment/configs/workflow-gateway-don-extensions.toml +++ /dev/null @@ -1,109 +0,0 @@ -# Workflow DON with gateway, reading workflows from extension stacks (configs/extensions). -# This topology is the same as workflow-gateway-don.toml but the workflow nodes use the extension roles: -# AdditionalSources at cre-workflow-source and WorkflowStorage at cre-artifact-api / cre-artifact-store. -# Start with `-e 4566` so the gateway can fetch artifacts from cre-artifact-store. -# -# Used by: configs/extensions/README.md - -[chip_router] - image = "local-cre-chip-router:v1.0.1" - -[[blockchains]] - type = "anvil" - chain_id = "1337" - container_name = "anvil-1337" - docker_cmd_params = ["-b", "0.5", "--mixed-mining"] - -[[blockchains]] - type = "anvil" - chain_id = "2337" - container_name = "anvil-2337" - port = "8546" - docker_cmd_params = ["-b", "0.5", "--mixed-mining"] - -[jd] - csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44" # any random 32 byte hex string - # change to your version - image = "job-distributor:0.28.0" - -[fake] - port = 8171 - -[fake_http] - port = 8666 - -#[s3provider] -# # use all defaults -# port = 9000 -# console_port = 9001 - -[infra] - # either "docker" or "kubernetes" - type = "docker" - -[[nodesets]] - nodes = 4 - name = "workflow" - don_families = ["test-don-family"] - don_types = ["workflow"] - override_mode = "all" - http_port_range_start = 10000 - p2p_port_range_start = 12000 - - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"] - registry_based_launch_allowlist = ["cron-trigger@1.0.0", "dontime@1.0.0"] - - [nodesets.db] - image = "postgres:15.0" - port = 13000 - -[[nodesets.node_specs]] - roles = ["plugin"] - [nodesets.node_specs.node] - docker_ctx = "../../../.." - docker_file = "core/chainlink.Dockerfile" - docker_build_args = { "CL_IS_PROD_BUILD" = "false" } - # image = "chainlink-tmp:latest" - user_config_overrides = """ -[CRE.WorkflowFetcher] -URL = '' - -[Capabilities.WorkflowRegistry.WorkflowStorage] -URL = 'cre-artifact-api:50052' -TLSEnabled = false -ArtifactStorageHost = 'cre-artifact-store:4566' - -[[Capabilities.WorkflowRegistry.AdditionalSources]] -URL = 'cre-workflow-source:50052' -TLSEnabled = false -Name = 'cre-extensions' -""" - -[[nodesets]] - nodes = 1 - name = "bootstrap-gateway" - don_families = ["test-don-family"] - don_types = ["bootstrap", "gateway"] - override_mode = "each" - http_port_range_start = 10100 - p2p_port_range_start = 12100 - - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - supported_evm_chains = [1337, 2337] - - [nodesets.db] - image = "postgres:15.0" - port = 13100 - - [[nodesets.node_specs]] - roles = ["bootstrap", "gateway"] - [nodesets.node_specs.node] - docker_ctx = "../../../.." - docker_file = "core/chainlink.Dockerfile" - docker_build_args = { "CL_IS_PROD_BUILD" = "false" } - # 5002 is the web API capabilities port for incoming requests - # 15002 is the vault port for incoming requests - custom_ports = ["5002:5002","15002:15002"] - # image = "chainlink-tmp:latest" - user_config_overrides = "" diff --git a/core/scripts/cre/environment/docs/TOPOLOGIES.md b/core/scripts/cre/environment/docs/TOPOLOGIES.md index 1c830ab38a4..19fcd86d256 100644 --- a/core/scripts/cre/environment/docs/TOPOLOGIES.md +++ b/core/scripts/cre/environment/docs/TOPOLOGIES.md @@ -11,7 +11,6 @@ This file is generated by `go run . topology generate`. Do not edit manually. | `configs/workflow-gateway-don-aptos.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-aptos.md)) | | `configs/workflow-gateway-don-cache-soak-test.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-cache-soak-test.md)) | | `configs/workflow-gateway-don-cache-test.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-cache-test.md)) | -| `configs/workflow-gateway-don-extensions.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-extensions.md)) | | `configs/workflow-gateway-don-grpc-source.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don-grpc-source.md)) | | `configs/workflow-gateway-don.toml` | `single-don` | 2 ([details](topologies/workflow-gateway-don.md)) | | `configs/workflow-gateway-sharded-5-dons.toml` | `sharded` | 7 ([details](topologies/workflow-gateway-sharded-5-dons.md)) | diff --git a/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md b/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md deleted file mode 100644 index 36b199863f7..00000000000 --- a/core/scripts/cre/environment/docs/topologies/workflow-gateway-don-extensions.md +++ /dev/null @@ -1,38 +0,0 @@ -# DON Topology - -- Config: `configs/workflow-gateway-don-extensions.toml` -- Class: `single-don` -- Infra: `docker` - -## Capability Matrix - -This matrix is the source of truth for capability placement by DON. - -| Capability | `bootstrap-gateway` | `workflow` | -|---|---|---| -| `consensus` | `-` | `local` | -| `cron` | `-` | `local` | -| `don-time` | `-` | `local` | -| `evm` | `-` | `local (1337,2337)` | -| `http-action` | `-` | `local` | -| `http-trigger` | `-` | `local` | -| `vault` | `-` | `local` | - -## DONs - -### `bootstrap-gateway` - -- Types: `bootstrap`, `gateway` -- Nodes: `1` -- Roles: `bootstrap`, `gateway` -- EVM chains: `1337,2337` -- Exposes remote capabilities: `false` - -### `workflow` - -- Types: `workflow` -- Nodes: `4` -- Roles: `plugin` -- EVM chains: `1337,2337` -- Exposes remote capabilities: `false` - diff --git a/core/scripts/cre/environment/environment/environment.go b/core/scripts/cre/environment/environment/environment.go index bc843fac1b1..cace0fdab12 100644 --- a/core/scripts/cre/environment/environment/environment.go +++ b/core/scripts/cre/environment/environment/environment.go @@ -401,6 +401,12 @@ func startCmd() *cobra.Command { fmt.Fprintf(os.Stderr, "failed to track startup: %s\n", dxErr) } + if in.Extensions != nil { + if err := startExtensions(cmdContext, in.Extensions, output); err != nil { + return errors.Wrap(err, "failed to start extensions") + } + } + if withChipIngressStack { startChipIngressStackErr := startChipIngressStack( cmdContext, @@ -643,6 +649,10 @@ func stopCmd() *cobra.Command { return errors.Wrap(removeErr, "failed to remove environment containers. Please remove them manually") } + if err := stopExtensions(); err != nil { + framework.L.Warn().Msgf("failed to stop extensions: %s", err) + } + if allFlag { stopChipIngressStackErr := stopChipIngressStack() if stopChipIngressStackErr != nil { @@ -814,6 +824,9 @@ func defaultGatewayWhitelistConfig(in *envconfig.Config, extraAllowedPorts []int if in.FakeHTTP != nil { ports = append(ports, in.FakeHTTP.Port) } + if in.Extensions != nil { + ports = append(ports, artifactStorePort) + } return gateway.WhitelistConfig{ ExtraAllowedPorts: ports, @@ -861,6 +874,9 @@ func StartCLIEnvironment( BlockchainDeployers: blockchains_sets.NewDeployerSet(testLogger, in.Infra), ContractVersions: env.ContractVersions(), } + if in.Extensions != nil { + universalSetupInput.ConfigFactoryFunctions = append(universalSetupInput.ConfigFactoryFunctions, extensionsNodeConfig) + } ctx, cancel := context.WithTimeout(cmdContext, 20*time.Minute) defer cancel() diff --git a/core/scripts/cre/environment/environment/extensions.go b/core/scripts/cre/environment/environment/extensions.go new file mode 100644 index 00000000000..ccf6159ddf1 --- /dev/null +++ b/core/scripts/cre/environment/environment/extensions.go @@ -0,0 +1,126 @@ +package environment + +import ( + "context" + "os" + "os/exec" + "slices" + "strconv" + + "github.com/pelletier/go-toml/v2" + "github.com/pkg/errors" + + "github.com/smartcontractkit/chainlink-testing-framework/framework" + keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" + creenv "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment" + envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config" + coretoml "github.com/smartcontractkit/chainlink/v2/core/config/toml" + corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" +) + +const ( + extensionsStackName = "cre-extensions" + + workflowSourceRole = "cre-workflow-source:50052" + artifactAPIRole = "cre-artifact-api:50052" + artifactStorePort = 4566 +) + +var artifactStoreRole = "cre-artifact-store:" + strconv.Itoa(artifactStorePort) + +// extensionsNodeConfig points workflow DON nodes at the extension roles: workflows come from cre-workflow-source and +// artifacts from cre-artifact-api / cre-artifact-store. +func extensionsNodeConfig(input cre.GenerateConfigsInput, configs cre.NodeIndexToConfigOverride) (cre.NodeIndexToConfigOverride, error) { + if !input.DonMetadata.HasFlag(cre.WorkflowDON) { + return configs, nil + } + + out := make(cre.NodeIndexToConfigOverride, len(configs)) + for idx, raw := range configs { + var c corechainlink.Config + if err := toml.Unmarshal([]byte(raw), &c); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal config of node %d in DON %s", idx, input.DonMetadata.Name) + } + + c.CRE.WorkflowFetcher = &coretoml.WorkflowFetcherConfig{URL: new("")} + c.Capabilities.WorkflowRegistry.WorkflowStorage = coretoml.WorkflowStorage{ + URL: new(artifactAPIRole), + TLSEnabled: new(false), + ArtifactStorageHost: new(artifactStoreRole), + } + c.Capabilities.WorkflowRegistry.AdditionalSourcesConfig = append(c.Capabilities.WorkflowRegistry.AdditionalSourcesConfig, coretoml.AdditionalWorkflowSource{ + URL: new(workflowSourceRole), + TLSEnabled: new(false), + Name: new(extensionsStackName), + }) + + marshalled, err := toml.Marshal(c) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal config of node %d in DON %s", idx, input.DonMetadata.Name) + } + out[idx] = string(marshalled) + } + return out, nil +} + +// startExtensions starts the extension stacks on the environment's network, with the registry chain, its +// CapabilitiesRegistry and the workflow DON family as inputs. +func startExtensions(ctx context.Context, ext *envconfig.Extensions, output *creenv.SetupOutput) error { + inputs, err := extensionsInputs(output) + if err != nil { + return err + } + + _ = framework.RemoveTestStack(extensionsStackName) + + cmd := exec.CommandContext(ctx, "docker", "compose", "-p", extensionsStackName, "-f", ext.StorageCompose, "up", "-d", "--wait") //nolint:gosec // G204: compose file path from the developer's TOML config + cmd.Env = append(os.Environ(), inputs...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + return errors.Wrapf(cmd.Run(), "failed to start %s", ext.StorageCompose) +} + +func stopExtensions() error { + return framework.RemoveTestStack(extensionsStackName) +} + +func extensionsInputs(output *creenv.SetupOutput) ([]string, error) { + creEnv := output.CreEnvironment + selector := creEnv.RegistryChainSelector + + var rpcURL string + for _, bc := range creEnv.Blockchains { + if bc.ChainSelector() == selector && len(bc.CtfOutput().Nodes) > 0 { + rpcURL = bc.CtfOutput().Nodes[0].InternalHTTPUrl + } + } + if rpcURL == "" { + return nil, errors.Errorf("no RPC URL for registry chain %d", selector) + } + + capReg := contracts.MightGetAddressFromDataStore(creEnv.CldfEnvironment.DataStore, selector, + keystone_changeset.CapabilitiesRegistry.String(), creEnv.ContractVersions[keystone_changeset.CapabilitiesRegistry.String()], "") + if capReg == nil { + return nil, errors.Errorf("no CapabilitiesRegistry on registry chain %d", selector) + } + + var families []string + for _, don := range output.Dons.DonsWithFlag(cre.WorkflowDON) { + if f := don.DonFamily(); f != "" && !slices.Contains(families, f) { + families = append(families, f) + } + } + if len(families) != 1 { + return nil, errors.Errorf("extensions support exactly one workflow DON family, found %v", families) + } + + return []string{ + "CRE_NETWORK=" + framework.DefaultNetworkName, + "CRE_NETWORK_EXTERNAL=true", + "CRE_LOCAL_REGISTRY_RPC_URL=" + rpcURL, + "CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=" + strconv.FormatUint(selector, 10), + "CRE_LOCAL_REGISTRY_ADDRESS=" + capReg.Hex(), + "CRE_LOCAL_DON_FAMILY=" + families[0], + }, nil +} diff --git a/core/scripts/cre/environment/environment/extensions_test.go b/core/scripts/cre/environment/environment/extensions_test.go new file mode 100644 index 00000000000..4bec03833c6 --- /dev/null +++ b/core/scripts/cre/environment/environment/extensions_test.go @@ -0,0 +1,56 @@ +package environment + +import ( + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" +) + +const extensionsTestNodeConfig = ` +[CRE.WorkflowFetcher] +URL = 'file:///home/chainlink/workflows' + +[[Capabilities.WorkflowRegistry.AdditionalSources]] +URL = 'grpc-source:50051' +Name = 'existing' +` + +func TestExtensionsNodeConfig_workflowDON(t *testing.T) { + t.Parallel() + + out, err := extensionsNodeConfig(extensionsTestInput(cre.WorkflowDON), cre.NodeIndexToConfigOverride{0: extensionsTestNodeConfig, 1: extensionsTestNodeConfig}) + require.NoError(t, err) + require.Len(t, out, 2) + + var c corechainlink.Config + require.NoError(t, toml.Unmarshal([]byte(out[1]), &c)) + require.Empty(t, *c.CRE.WorkflowFetcher.URL) + + storage := c.Capabilities.WorkflowRegistry.WorkflowStorage + require.Equal(t, "cre-artifact-api:50052", *storage.URL) + require.Equal(t, "cre-artifact-store:4566", *storage.ArtifactStorageHost) + require.False(t, *storage.TLSEnabled) + + sources := c.Capabilities.WorkflowRegistry.AdditionalSourcesConfig + require.Len(t, sources, 2) + require.Equal(t, "existing", *sources[0].Name) + require.Equal(t, "cre-workflow-source:50052", *sources[1].URL) + require.False(t, *sources[1].TLSEnabled) +} + +func TestExtensionsNodeConfig_otherDONsUnchanged(t *testing.T) { + t.Parallel() + + configs := cre.NodeIndexToConfigOverride{0: extensionsTestNodeConfig} + out, err := extensionsNodeConfig(extensionsTestInput(cre.GatewayDON), configs) + require.NoError(t, err) + require.Equal(t, configs, out) +} + +func extensionsTestInput(flag cre.CapabilityFlag) cre.GenerateConfigsInput { + return cre.GenerateConfigsInput{DonMetadata: &cre.DonMetadata{Name: "don", Flags: []string{flag}}} +} diff --git a/system-tests/lib/cre/environment/config/config.go b/system-tests/lib/cre/environment/config/config.go index 3af59afaedb..76442bc050d 100644 --- a/system-tests/lib/cre/environment/config/config.go +++ b/system-tests/lib/cre/environment/config/config.go @@ -68,12 +68,19 @@ type Config struct { ChipRouter *ctfchiprouter.Input `toml:"chip_router" validate:"required"` S3ProviderInput *s3provider.Input `toml:"s3provider"` CapabilityConfigs map[string]cre.CapabilityConfig `toml:"capability_configs"` // capability flag -> capability config + Extensions *Extensions `toml:"extensions"` Addresses []string `toml:"addresses"` mu sync.Mutex loaded bool } +// Extensions are centralized CRE services started alongside the DONs, answering to fixed role aliases on the +// Docker network (see core/scripts/cre/environment/configs/extensions/README.md). +type Extensions struct { + StorageCompose string `toml:"storage_compose" validate:"required"` // path to cre-storage-service/local-stack/compose.yaml +} + // Validate performs validation checks on the configuration, ensuring all required fields // are present and all referenced capabilities are known to the system. func (c *Config) Validate(envDependencies cre.CLIEnvironmentDependencies) error { diff --git a/system-tests/lib/cre/features/vault/vault.go b/system-tests/lib/cre/features/vault/vault.go index 53c485a9310..a92915c6785 100644 --- a/system-tests/lib/cre/features/vault/vault.go +++ b/system-tests/lib/cre/features/vault/vault.go @@ -145,14 +145,13 @@ func updateNodeConfig(workerNode *cre.NodeMetadata, currentConfig string, regist return nil, errors.Wrapf(unmarshallErr, "failed to unmarshal config for node index %d", workerNode.Index) } - // enable workflow registry syncer - typedConfig.Capabilities.WorkflowRegistry = coretoml.WorkflowRegistry{ - Address: new(workflowRegistryAddress.Hex()), - NetworkID: new("evm"), - ChainID: new(strconv.FormatUint(registryChainID, 10)), - SyncStrategy: new("reconciliation"), - ContractVersion: new(wfRegVersion.String()), - } + // enable workflow registry syncer, keeping other WorkflowRegistry settings (e.g. WorkflowStorage, AdditionalSources) + wfReg := &typedConfig.Capabilities.WorkflowRegistry + wfReg.Address = new(workflowRegistryAddress.Hex()) + wfReg.NetworkID = new("evm") + wfReg.ChainID = new(strconv.FormatUint(registryChainID, 10)) + wfReg.SyncStrategy = new("reconciliation") + wfReg.ContractVersion = new(wfRegVersion.String()) typedConfig.CRE.Linking = &coretoml.LinkingConfig{ URL: new(strings.TrimPrefix(framework.HostDockerInternal(), "http://") + ":18124"), TLSEnabled: new(false), From d7b4730c613b8050b90aef4e5d7dc84b7a96640f Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Mon, 12 Oct 2026 09:45:52 -0500 Subject: [PATCH 4/6] Add pseudo-review updates --- .../environment/environment/environment.go | 10 ++- .../cre/environment/environment/extensions.go | 82 +++++++++++++++---- .../environment/extensions_test.go | 41 ++++++++++ .../cre/environment/environment/workflow.go | 3 + 4 files changed, 119 insertions(+), 17 deletions(-) diff --git a/core/scripts/cre/environment/environment/environment.go b/core/scripts/cre/environment/environment/environment.go index cace0fdab12..adc2e23197e 100644 --- a/core/scripts/cre/environment/environment/environment.go +++ b/core/scripts/cre/environment/environment/environment.go @@ -348,6 +348,14 @@ func startCmd() *cobra.Command { return errors.Wrap(err, "failed to validate test configuration") } + var extensionsDonFamily string + if in.Extensions != nil { + var extErr error + if extensionsDonFamily, extErr = validateExtensions(cmdContext, in); extErr != nil { + return errors.Wrap(extErr, "invalid extensions config") + } + } + topologySummary, _, topErr := generateTopologyArtifactsForLoadedConfig(in) if topErr != nil { framework.L.Warn().Err(topErr).Msg("failed to generate topology visualization artifacts") @@ -402,7 +410,7 @@ func startCmd() *cobra.Command { } if in.Extensions != nil { - if err := startExtensions(cmdContext, in.Extensions, output); err != nil { + if err := startExtensions(cmdContext, in.Extensions, extensionsDonFamily, output); err != nil { return errors.Wrap(err, "failed to start extensions") } } diff --git a/core/scripts/cre/environment/environment/extensions.go b/core/scripts/cre/environment/environment/extensions.go index ccf6159ddf1..d00d11ce85e 100644 --- a/core/scripts/cre/environment/environment/extensions.go +++ b/core/scripts/cre/environment/environment/extensions.go @@ -6,6 +6,7 @@ import ( "os/exec" "slices" "strconv" + "strings" "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" @@ -20,6 +21,8 @@ import ( corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" ) +// Role aliases the extension stacks answer to on ctf. They are a contract with those stacks' compose files; keep them in +// step with configs/extensions/README.md and cre-storage-service local-stack/README.md. const ( extensionsStackName = "cre-extensions" @@ -65,17 +68,66 @@ func extensionsNodeConfig(input cre.GenerateConfigsInput, configs cre.NodeIndexT return out, nil } +// validateExtensions rejects, before the environment starts, extensions config that can't work: a non-Docker provider, +// a missing compose file, other than one workflow DON family, or stack images that are neither local nor pullable. It +// returns the workflow DON family the stacks serve. +func validateExtensions(ctx context.Context, in *envconfig.Config) (string, error) { + if in.Infra == nil || !in.Infra.IsDocker() { + return "", errors.New("extensions require the docker infra provider") + } + if _, err := os.Stat(in.Extensions.StorageCompose); err != nil { + return "", errors.Wrap(err, "extensions.storage_compose") + } + + var families []string + for _, ns := range in.NodeSets { + if slices.Contains(ns.DONTypes, cre.WorkflowDON) && len(ns.DonFamilies) > 0 && !slices.Contains(families, ns.DonFamilies[0]) { + families = append(families, ns.DonFamilies[0]) + } + } + if len(families) != 1 { + return "", errors.Errorf("extensions support exactly one workflow DON family, found %v", families) + } + + return families[0], ensureExtensionsImages(ctx, in.Extensions) +} + +// ensureExtensionsImages pulls the stacks' images that aren't local, and names the ones that can't be pulled (images +// built from the stack's repo) instead of failing later in docker compose up. +func ensureExtensionsImages(ctx context.Context, ext *envconfig.Extensions) error { + cmd := docker(ctx, "compose", "-f", ext.StorageCompose, "config", "--images") + cmd.Env = append(os.Environ(), extensionsEnv("http://placeholder:8545", 0, "0x0000000000000000000000000000000000000000", "placeholder")...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return errors.Wrapf(err, "failed to list images in %s", ext.StorageCompose) + } + + images := strings.Fields(string(out)) + slices.Sort(images) + var missing []string + for _, image := range slices.Compact(images) { + if docker(ctx, "image", "inspect", image).Run() != nil && docker(ctx, "pull", "-q", image).Run() != nil { + missing = append(missing, image) + } + } + if len(missing) > 0 { + return errors.Errorf("images %v are neither local nor pullable; build them as described in the README next to %s", missing, ext.StorageCompose) + } + return nil +} + // startExtensions starts the extension stacks on the environment's network, with the registry chain, its // CapabilitiesRegistry and the workflow DON family as inputs. -func startExtensions(ctx context.Context, ext *envconfig.Extensions, output *creenv.SetupOutput) error { - inputs, err := extensionsInputs(output) +func startExtensions(ctx context.Context, ext *envconfig.Extensions, donFamily string, output *creenv.SetupOutput) error { + inputs, err := extensionsInputs(output, donFamily) if err != nil { return err } _ = framework.RemoveTestStack(extensionsStackName) - cmd := exec.CommandContext(ctx, "docker", "compose", "-p", extensionsStackName, "-f", ext.StorageCompose, "up", "-d", "--wait") //nolint:gosec // G204: compose file path from the developer's TOML config + cmd := docker(ctx, "compose", "-p", extensionsStackName, "-f", ext.StorageCompose, "up", "-d", "--wait") cmd.Env = append(os.Environ(), inputs...) cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr return errors.Wrapf(cmd.Run(), "failed to start %s", ext.StorageCompose) @@ -85,7 +137,7 @@ func stopExtensions() error { return framework.RemoveTestStack(extensionsStackName) } -func extensionsInputs(output *creenv.SetupOutput) ([]string, error) { +func extensionsInputs(output *creenv.SetupOutput, donFamily string) ([]string, error) { creEnv := output.CreEnvironment selector := creEnv.RegistryChainSelector @@ -105,22 +157,20 @@ func extensionsInputs(output *creenv.SetupOutput) ([]string, error) { return nil, errors.Errorf("no CapabilitiesRegistry on registry chain %d", selector) } - var families []string - for _, don := range output.Dons.DonsWithFlag(cre.WorkflowDON) { - if f := don.DonFamily(); f != "" && !slices.Contains(families, f) { - families = append(families, f) - } - } - if len(families) != 1 { - return nil, errors.Errorf("extensions support exactly one workflow DON family, found %v", families) - } + return extensionsEnv(rpcURL, selector, capReg.Hex(), donFamily), nil +} +func extensionsEnv(rpcURL string, selector uint64, registry, donFamily string) []string { return []string{ "CRE_NETWORK=" + framework.DefaultNetworkName, "CRE_NETWORK_EXTERNAL=true", "CRE_LOCAL_REGISTRY_RPC_URL=" + rpcURL, "CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=" + strconv.FormatUint(selector, 10), - "CRE_LOCAL_REGISTRY_ADDRESS=" + capReg.Hex(), - "CRE_LOCAL_DON_FAMILY=" + families[0], - }, nil + "CRE_LOCAL_REGISTRY_ADDRESS=" + registry, + "CRE_LOCAL_DON_FAMILY=" + donFamily, + } +} + +func docker(ctx context.Context, args ...string) *exec.Cmd { + return exec.CommandContext(ctx, "docker", args...) } diff --git a/core/scripts/cre/environment/environment/extensions_test.go b/core/scripts/cre/environment/environment/extensions_test.go index 4bec03833c6..b6b5d25fc8c 100644 --- a/core/scripts/cre/environment/environment/extensions_test.go +++ b/core/scripts/cre/environment/environment/extensions_test.go @@ -1,12 +1,16 @@ package environment import ( + "os" + "path/filepath" "testing" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config" + "github.com/smartcontractkit/chainlink/system-tests/lib/infra" corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" ) @@ -54,3 +58,40 @@ func TestExtensionsNodeConfig_otherDONsUnchanged(t *testing.T) { func extensionsTestInput(flag cre.CapabilityFlag) cre.GenerateConfigsInput { return cre.GenerateConfigsInput{DonMetadata: &cre.DonMetadata{Name: "don", Flags: []string{flag}}} } + +func TestValidateExtensions_rejectsBeforeStart(t *testing.T) { + t.Parallel() + + compose := filepath.Join(t.TempDir(), "compose.yaml") + require.NoError(t, os.WriteFile(compose, []byte("services: {}\n"), 0o600)) + workflowNodeSet := func(family string) *cre.NodeSet { + return &cre.NodeSet{DONTypes: []string{cre.WorkflowDON}, DonFamilies: []string{family}} + } + + tests := []struct { + name string + in *envconfig.Config + want string + }{ + {"kubernetes", &envconfig.Config{ + Infra: &infra.Provider{Type: infra.Kubernetes}, + Extensions: &envconfig.Extensions{StorageCompose: compose}, + }, "docker infra provider"}, + {"missing compose file", &envconfig.Config{ + Infra: &infra.Provider{Type: infra.Docker}, + Extensions: &envconfig.Extensions{StorageCompose: filepath.Join(t.TempDir(), "missing.yaml")}, + }, "extensions.storage_compose"}, + {"two workflow DON families", &envconfig.Config{ + Infra: &infra.Provider{Type: infra.Docker}, + NodeSets: []*cre.NodeSet{workflowNodeSet("zone-a"), workflowNodeSet("zone-b")}, + Extensions: &envconfig.Extensions{StorageCompose: compose}, + }, "exactly one workflow DON family"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := validateExtensions(t.Context(), tt.in) + require.ErrorContains(t, err, tt.want) + }) + } +} diff --git a/core/scripts/cre/environment/environment/workflow.go b/core/scripts/cre/environment/environment/workflow.go index db29b0a2303..b30cc96965c 100644 --- a/core/scripts/cre/environment/environment/workflow.go +++ b/core/scripts/cre/environment/environment/workflow.go @@ -126,6 +126,9 @@ func deployWorkflowCmd() *cobra.Command { if resolverErr != nil { return errors.Wrap(resolverErr, "failed to load local CRE state") } + if resolver != nil && resolver.cfg.Extensions != nil { + return errors.New("❌ this environment reads workflows from its extensions, not the file fetcher; deploy through cre-deploy-api instead (see configs/extensions/README.md)") + } defer func() { metaData := map[string]any{} From a42d81c9e0fbbc914ed2e158d3cafc24987182a4 Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Mon, 19 Oct 2026 09:54:49 -0500 Subject: [PATCH 5/6] Update to use CTF testcontainer --- .../environment/configs/extensions/README.md | 8 ++- .../cre/environment/environment/extensions.go | 69 ++++++++++++------- core/scripts/go.mod | 3 + core/scripts/go.sum | 4 +- 4 files changed, 53 insertions(+), 31 deletions(-) diff --git a/core/scripts/cre/environment/configs/extensions/README.md b/core/scripts/cre/environment/configs/extensions/README.md index 9dfc275ab82..a305676a4aa 100644 --- a/core/scripts/cre/environment/configs/extensions/README.md +++ b/core/scripts/cre/environment/configs/extensions/README.md @@ -1,10 +1,11 @@ # Local CRE extensions -Centralized CRE services run from their own repos' compose files. With an `[extensions]` section (`extensions.toml`), +Centralized CRE services run from their own repos' compose files, through their chainlink-testing-framework components +(`framework/components/dockercompose/cre_storage_service`). With an `[extensions]` section (`extensions.toml`), `env start` starts them on `ctf` after the DONs, passing the registry chain, its CapabilitiesRegistry and the workflow DON family, and points the workflow nodes at the roles they answer to. `env stop` removes them. -| Role (alias on `ctf`) | Port | Workflow node config | +| Role (alias on `ctf`) | Port on `ctf` | Workflow node config | | --- | --- | --- | | `cre-workflow-source` | 50052 | `Capabilities.WorkflowRegistry.AdditionalSources` | | `cre-artifact-api` | 50052 | `Capabilities.WorkflowRegistry.WorkflowStorage.URL` | @@ -13,7 +14,8 @@ DON family, and points the workflow nodes at the roles they answer to. `env stop `storage_compose` defaults to a `cre-storage-service` checkout next to this repo; its images are built locally (see its `local-stack/README.md`). Workflows then come from the extensions, not `env workflow deploy`; deploy tooling runs in a -container on `ctf`, since artifact URLs use `cre-artifact-store`. +container on `ctf`, since artifact URLs use `cre-artifact-store`. From the host, the roles are on free ports that +`env start` logs. ```bash CTF_CONFIGS=configs/workflow-gateway-don.toml,configs/extensions/extensions.toml go run . env start diff --git a/core/scripts/cre/environment/environment/extensions.go b/core/scripts/cre/environment/environment/extensions.go index d00d11ce85e..beca3686a8a 100644 --- a/core/scripts/cre/environment/environment/extensions.go +++ b/core/scripts/cre/environment/environment/extensions.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/smartcontractkit/chainlink-testing-framework/framework" + storage "github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose/cre_storage_service" keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" "github.com/smartcontractkit/chainlink/system-tests/lib/cre" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" @@ -21,17 +22,17 @@ import ( corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" ) -// Role aliases the extension stacks answer to on ctf. They are a contract with those stacks' compose files; keep them in -// step with configs/extensions/README.md and cre-storage-service local-stack/README.md. +// Role addresses the extension stacks answer to on ctf, as defined by their CTF components; configs/extensions/README.md +// lists them. const ( - extensionsStackName = "cre-extensions" + extensionsSourceName = "cre-extensions" - workflowSourceRole = "cre-workflow-source:50052" - artifactAPIRole = "cre-artifact-api:50052" - artifactStorePort = 4566 + workflowSourceRole = storage.WorkflowSourceRole + ":" + storage.NodeServicePort + artifactAPIRole = storage.ArtifactAPIRole + ":" + storage.NodeServicePort + artifactStoreRole = storage.ArtifactStoreRole + ":" + storage.ArtifactStorePort ) -var artifactStoreRole = "cre-artifact-store:" + strconv.Itoa(artifactStorePort) +var artifactStorePort = mustPort(storage.ArtifactStorePort) // extensionsNodeConfig points workflow DON nodes at the extension roles: workflows come from cre-workflow-source and // artifacts from cre-artifact-api / cre-artifact-store. @@ -56,7 +57,7 @@ func extensionsNodeConfig(input cre.GenerateConfigsInput, configs cre.NodeIndexT c.Capabilities.WorkflowRegistry.AdditionalSourcesConfig = append(c.Capabilities.WorkflowRegistry.AdditionalSourcesConfig, coretoml.AdditionalWorkflowSource{ URL: new(workflowSourceRole), TLSEnabled: new(false), - Name: new(extensionsStackName), + Name: new(extensionsSourceName), }) marshalled, err := toml.Marshal(c) @@ -96,7 +97,13 @@ func validateExtensions(ctx context.Context, in *envconfig.Config) (string, erro // built from the stack's repo) instead of failing later in docker compose up. func ensureExtensionsImages(ctx context.Context, ext *envconfig.Extensions) error { cmd := docker(ctx, "compose", "-f", ext.StorageCompose, "config", "--images") - cmd.Env = append(os.Environ(), extensionsEnv("http://placeholder:8545", 0, "0x0000000000000000000000000000000000000000", "placeholder")...) + // the stack's required inputs; their values don't affect the images + cmd.Env = append(os.Environ(), + "CRE_LOCAL_REGISTRY_RPC_URL=placeholder", + "CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=0", + "CRE_LOCAL_REGISTRY_ADDRESS=placeholder", + "CRE_LOCAL_DON_FAMILY=placeholder", + ) cmd.Stderr = os.Stderr out, err := cmd.Output() if err != nil { @@ -120,24 +127,30 @@ func ensureExtensionsImages(ctx context.Context, ext *envconfig.Extensions) erro // startExtensions starts the extension stacks on the environment's network, with the registry chain, its // CapabilitiesRegistry and the workflow DON family as inputs. func startExtensions(ctx context.Context, ext *envconfig.Extensions, donFamily string, output *creenv.SetupOutput) error { - inputs, err := extensionsInputs(output, donFamily) + in, err := storageInput(ext, output, donFamily) if err != nil { return err } - _ = framework.RemoveTestStack(extensionsStackName) + _ = storage.Stop() - cmd := docker(ctx, "compose", "-p", extensionsStackName, "-f", ext.StorageCompose, "up", "-d", "--wait") - cmd.Env = append(os.Environ(), inputs...) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - return errors.Wrapf(cmd.Run(), "failed to start %s", ext.StorageCompose) + out, err := storage.NewWithContext(ctx, in) + if err != nil { + return errors.Wrapf(err, "failed to start %s", ext.StorageCompose) + } + framework.L.Info(). + Str("cre-deploy-api", out.UserService.ExternalURL). + Str("cre-workflow-source", out.NodeService.ExternalURL). + Str("cre-artifact-store", out.ArtifactStore.ExternalURL). + Msg("Extensions started; reachable from the host at") + return nil } func stopExtensions() error { - return framework.RemoveTestStack(extensionsStackName) + return storage.Stop() } -func extensionsInputs(output *creenv.SetupOutput, donFamily string) ([]string, error) { +func storageInput(ext *envconfig.Extensions, output *creenv.SetupOutput, donFamily string) (*storage.Input, error) { creEnv := output.CreEnvironment selector := creEnv.RegistryChainSelector @@ -157,18 +170,22 @@ func extensionsInputs(output *creenv.SetupOutput, donFamily string) ([]string, e return nil, errors.Errorf("no CapabilitiesRegistry on registry chain %d", selector) } - return extensionsEnv(rpcURL, selector, capReg.Hex(), donFamily), nil + return &storage.Input{ + ComposeFile: ext.StorageCompose, + Network: framework.DefaultNetworkName, + RegistryRPCURL: rpcURL, + RegistryChainSelector: selector, + RegistryAddress: capReg.Hex(), + DonFamily: donFamily, + }, nil } -func extensionsEnv(rpcURL string, selector uint64, registry, donFamily string) []string { - return []string{ - "CRE_NETWORK=" + framework.DefaultNetworkName, - "CRE_NETWORK_EXTERNAL=true", - "CRE_LOCAL_REGISTRY_RPC_URL=" + rpcURL, - "CRE_LOCAL_REGISTRY_CHAIN_SELECTOR=" + strconv.FormatUint(selector, 10), - "CRE_LOCAL_REGISTRY_ADDRESS=" + registry, - "CRE_LOCAL_DON_FAMILY=" + donFamily, +func mustPort(port string) int { + p, err := strconv.Atoi(port) + if err != nil { + panic(errors.Wrapf(err, "invalid port %q", port)) } + return p } func docker(ctx context.Context, args ...string) *exec.Cmd { diff --git a/core/scripts/go.mod b/core/scripts/go.mod index d7d204ecf9d..4044b5ea550 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -706,3 +706,6 @@ exclude github.com/starknet-io/starknet-p2pspecs v0.0.0-00010101000000-000000000 // otel/opentelemetry-collector-contrib:0.123.0 rejects (expects int ms), so the collector // crash-loops and `obs up` produces no Beholder logs in Loki. Pin to v0.16.7 until CTF fixes it. replace github.com/smartcontractkit/chainlink-testing-framework/framework => github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7 + +// TODO: drop once chainlink-testing-framework#2869 is released, and bump dockercompose here and in system-tests/lib. +replace github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose => github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 9ba6a48e00b..f1948acd41f 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1770,8 +1770,8 @@ github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7 h1:2j1 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7/go.mod h1:nyOjn4ADJGqRMe3+4ZXSV+J/7nWb1H2Vx8Qk57eLRYA= github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter v1.0.4 h1:bGicxBPndwy9NeB79n+CgyNxA8aeWoMudC84krz6QGM= github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter v1.0.4/go.mod h1:TsZMdVIPeIBzFwVIUmU7jkXOTHSpyvCJGeLtjuBxa8E= -github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.23 h1:FYZZ2U6h2y4sITrEyTKPHTzjJrrsqCqN3zGqkpk7p3s= -github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.23/go.mod h1:S6VqtFkFyrNaPOodfiJTEHSu7wgj8bFfWtvfdFM+ztc= +github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e h1:t3snOdglptRHkkv5/z4dkfCpEsmb/SNq7bIt4BDVcIs= +github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e/go.mod h1:Iyo5qLQujzsQ32DXD3Kg82ftgGPMr1g4uUpz2UGt2hc= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.15.0 h1:qELAsHYwbNWfJDJsp+3yqAxvmBjB2/zLPZhR91t2cg8= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.15.0/go.mod h1:5xdlcXFhjagMeRA6q01FKEMv5OqvfVknI5qwXGriS5M= github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.9 h1:/5fgG2lZqqTJ7mr3mUtxbWiUxd/gmDHSEGUERIUPvyU= From 0b15cf3c9d99f2a70204a8943cc2f6d996425e57 Mon Sep 17 00:00:00 2001 From: Axel Kingsley Date: Thu, 22 Oct 2026 11:58:00 -0500 Subject: [PATCH 6/6] Use merged CTF cre_storage_service component --- core/scripts/cre/environment/environment/extensions.go | 2 +- core/scripts/go.mod | 5 +---- core/scripts/go.sum | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/scripts/cre/environment/environment/extensions.go b/core/scripts/cre/environment/environment/extensions.go index beca3686a8a..0a64d61682f 100644 --- a/core/scripts/cre/environment/environment/extensions.go +++ b/core/scripts/cre/environment/environment/extensions.go @@ -171,7 +171,7 @@ func storageInput(ext *envconfig.Extensions, output *creenv.SetupOutput, donFami } return &storage.Input{ - ComposeFile: ext.StorageCompose, + ComposeFile: "file://" + ext.StorageCompose, Network: framework.DefaultNetworkName, RegistryRPCURL: rpcURL, RegistryChainSelector: selector, diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 4044b5ea550..6ec2261c435 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -40,7 +40,7 @@ require ( github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20261005112317-b723176adfe8 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 - github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.23 + github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261022155844-32538607cda4 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 github.com/smartcontractkit/chainlink/deployment v0.0.0-20260126202327-6be9a05f0caf github.com/smartcontractkit/chainlink/system-tests/lib v0.0.0-00010101000000-000000000000 @@ -706,6 +706,3 @@ exclude github.com/starknet-io/starknet-p2pspecs v0.0.0-00010101000000-000000000 // otel/opentelemetry-collector-contrib:0.123.0 rejects (expects int ms), so the collector // crash-loops and `obs up` produces no Beholder logs in Loki. Pin to v0.16.7 until CTF fixes it. replace github.com/smartcontractkit/chainlink-testing-framework/framework => github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7 - -// TODO: drop once chainlink-testing-framework#2869 is released, and bump dockercompose here and in system-tests/lib. -replace github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose => github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e diff --git a/core/scripts/go.sum b/core/scripts/go.sum index f1948acd41f..1d60c27c888 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1770,8 +1770,8 @@ github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7 h1:2j1 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.7/go.mod h1:nyOjn4ADJGqRMe3+4ZXSV+J/7nWb1H2Vx8Qk57eLRYA= github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter v1.0.4 h1:bGicxBPndwy9NeB79n+CgyNxA8aeWoMudC84krz6QGM= github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter v1.0.4/go.mod h1:TsZMdVIPeIBzFwVIUmU7jkXOTHSpyvCJGeLtjuBxa8E= -github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e h1:t3snOdglptRHkkv5/z4dkfCpEsmb/SNq7bIt4BDVcIs= -github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261013162928-08054c38e73e/go.mod h1:Iyo5qLQujzsQ32DXD3Kg82ftgGPMr1g4uUpz2UGt2hc= +github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261022155844-32538607cda4 h1:zkbVIJML0Dkn4K45Butdp+5rTa8PYA56tSVTae7X7DM= +github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.24-0.20261022155844-32538607cda4/go.mod h1:Iyo5qLQujzsQ32DXD3Kg82ftgGPMr1g4uUpz2UGt2hc= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.15.0 h1:qELAsHYwbNWfJDJsp+3yqAxvmBjB2/zLPZhR91t2cg8= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.15.0/go.mod h1:5xdlcXFhjagMeRA6q01FKEMv5OqvfVknI5qwXGriS5M= github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.9 h1:/5fgG2lZqqTJ7mr3mUtxbWiUxd/gmDHSEGUERIUPvyU=