From 170b7a33247143a2f46a060f03bef051ac8478cf Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Fri, 23 Oct 2026 15:11:48 +0200 Subject: [PATCH 1/3] feat(dcp): grant the Gemini key per secret and lift prod job protection DCP v1.1.6 removes the serving service account's project-wide secretmanager.secretAccessor role. search_api_gemini_key reads the Gemini key through that role, so after the upgrade the service would start without GEMINI_API_KEY and /api/search/* would return 502. A per-secret grant on "${namespace}-gemini-api-key" keeps the access under both module versions, and the key script now waits for it. v1.1.6 also deletes the Cloud Run preprocessing job, which Terraform cannot do while stateless_deletion_protection is on. Prod turns it off here, on v1.1.5, so the v1.1.6 apply can complete. --- infra/dfo-prod/dcp/dfo.auto.tfvars | 9 ++++++--- infra/stacks/dcp/dfo.tf | 20 +++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/infra/dfo-prod/dcp/dfo.auto.tfvars b/infra/dfo-prod/dcp/dfo.auto.tfvars index b9a8eee..16cdb67 100644 --- a/infra/dfo-prod/dcp/dfo.auto.tfvars +++ b/infra/dfo-prod/dcp/dfo.auto.tfvars @@ -12,9 +12,12 @@ project_id = "prod-fin-observatory" namespace = "dfo-prod" region = "europe-west1" -stateful_deletion_protection = true -skip_container_restarts = true -stateless_deletion_protection = true +stateful_deletion_protection = true +skip_container_restarts = true + +# DCP v1.1.6 deletes the Cloud Run preprocessing job, and Terraform cannot +# delete it while protection is on. Set back to true once prod runs v1.1.6. +stateless_deletion_protection = false # ============================================================================= # Storage Module diff --git a/infra/stacks/dcp/dfo.tf b/infra/stacks/dcp/dfo.tf index 7798cd4..937ff57 100644 --- a/infra/stacks/dcp/dfo.tf +++ b/infra/stacks/dcp/dfo.tf @@ -37,13 +37,24 @@ resource "null_resource" "dc_key_guard" { } } +# The serving SA reads the Gemini key from Secret Manager as +# "${namespace}-gemini-api-key" through this per-secret grant, so the access +# holds whether or not the stack module grants secretmanager.secretAccessor +# project-wide. +resource "google_secret_manager_secret_iam_member" "serving_sa_gemini_key" { + project = var.project_id + secret_id = "${var.namespace}-gemini-api-key" + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${module.stack.datacommons_service_service_account_email}" +} + # Wire GEMINI_API_KEY onto the datacommons service so the ONE/DFO Search API # boots. Without it, custom_dc/api_server/start.sh deliberately skips launching # the FastAPI engine (it warm-loads Gemini at startup and would restart-loop), # so nginx returns 502 on every /api/search/* call while the DC site is fine. # -# The key lives in Secret Manager as "${namespace}-gemini-api-key", which the -# serving SA can already read through its project-level secretmanager.secretAccessor. +# The key lives in Secret Manager as "${namespace}-gemini-api-key", and the +# serving SA reads it through the per-secret secretAccessor grant above. # # The pinned stack module takes no input for adding secret env vars, so the key # goes on after the module deploys the service. The script header explains why it @@ -70,5 +81,8 @@ resource "null_resource" "search_api_gemini_key" { EOT } - depends_on = [module.stack] + depends_on = [ + module.stack, + google_secret_manager_secret_iam_member.serving_sa_gemini_key, + ] } From a9bf6608cadd63e696a36fed5309b34a207f4a24 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Fri, 23 Oct 2026 15:13:50 +0200 Subject: [PATCH 2/3] feat(dcp): move the dcp stack to Data Commons Platform 1.1.6 infra/stacks/dcp/main.tf, variables.tf and outputs.tf are upstream's 1.1.6 root files with the module ref pinned to v1.1.6. dcp_version defaults to 1.1.6, which picks the 1.1.6 ingestion images, the ingestion-1.1.6 Dataflow template and the new rollback template. Preprocessing runs as a Cloud Batch job with cpu and memory in milliCPU and MiB, so batch.googleapis.com joins the enabled APIs. The ingestion_prep_job_name output goes, since the Cloud Run preprocessing job no longer exists. The justfile pins CLI 1.1.6. seed-db is removed, init-db drops --init-only, and init-db and migrate-db run under the caller's application default credentials, which need roles/spanner.databaseAdmin and roles/spanner.databaseUser. docs/development.md and the dfo-dev dcp README state the same. --- docs/development.md | 4 +- infra/dfo-dev/dcp/dfo.README.md | 23 ++--- infra/dfo-dev/dcp/dfo.auto.tfvars | 5 +- infra/dfo-prod/dcp/dfo.auto.tfvars | 5 +- infra/stacks/dcp/dfo_override.tf | 5 +- infra/stacks/dcp/dfo_storage.tf | 7 ++ infra/stacks/dcp/main.tf | 71 ++++++++----- infra/stacks/dcp/outputs.tf | 49 ++++++++- infra/stacks/dcp/variables.tf | 159 +++++++++++++++++++++++++---- justfile | 76 ++++++-------- 10 files changed, 285 insertions(+), 119 deletions(-) diff --git a/docs/development.md b/docs/development.md index a06fb55..42197d2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -88,7 +88,7 @@ cd infra/dfo-dev/dcp terraform init && terraform apply ``` -After `apply` completes, grant the operator `roles/iam.serviceAccountTokenCreator` on the ingestion workflow service account (printed in the Terraform outputs) so the CLI can impersonate it for `init-db` and `ingest`: +After `apply` completes, grant the operator `roles/spanner.databaseAdmin` and `roles/spanner.databaseUser` on the dcp database for `init-db` and `migrate-db`, and `roles/iam.serviceAccountTokenCreator` on the ingestion workflow service account (printed in the Terraform outputs) so the CLI can impersonate it for `ingest`: ```bash gcloud iam service-accounts add-iam-policy-binding \ @@ -116,7 +116,7 @@ For a break-glass deploy, `just deploy ` applies an image directly agains ### Upgrading the DCP module ```bash -just dcp-scaffold 1.1.5 +just dcp-scaffold 1.1.6 ``` The recipe regenerates `main.tf`, `variables.tf`, `outputs.tf`, and `README.md` from the upstream CLI at the given version. It copies the three `.tf` files over the tracked files in `infra/stacks/dcp`, which every environment's dcp root links to, and `README.md` into `infra//dcp`. It leaves `backend.tf` and `terraform.tfvars` untouched, because the CLI's own `backend.tf` assumes one stack per instance and would point this stack at empty state. [infra/dfo-dev/dcp/dfo.README.md](../infra/dfo-dev/dcp/dfo.README.md) explains why. It fails on uncommitted edits to `backend.tf` or any `dfo.*` file, so those do not blend into the diff it prints. It also checks the file set the CLI emitted against the expected six, which catches upstream drift the copy loop would otherwise drop silently. diff --git a/infra/dfo-dev/dcp/dfo.README.md b/infra/dfo-dev/dcp/dfo.README.md index 32d8862..809b1ae 100644 --- a/infra/dfo-dev/dcp/dfo.README.md +++ b/infra/dfo-dev/dcp/dfo.README.md @@ -36,37 +36,32 @@ Two resources appear in every plan. `null_resource.search_api_gemini_key` carrie ## The database commands -Four `just` recipes touch the dcp database. Terraform grants none of the IAM they need, so grant it by hand before the first run. +Three `just` recipes touch the dcp database. Terraform grants none of the IAM they need, so grant it by hand before the first run. -| Recipe | What it does | Spanner role on your principal | Token creator on the workflow account | -| ----------------------- | --------------------------------------------------------------------------- | ------------------------------ | ------------------------------------- | -| `just init-db` | Creates the schema on a brand-new database | `roles/spanner.databaseReader` | yes | -| `just migrate-db` | Applies pending Spanner schema migrations | `roles/spanner.databaseAdmin` | yes | -| `just seed-db` | Re-applies Base DC's own geographic and schema seeds | none | yes | -| `just load data/` | Stages one of this repo's datasets into the ingestion bucket and ingests it | none | yes | +| Recipe | What it does | Spanner roles on your principal | Token creator on the workflow account | +| ----------------------- | --------------------------------------------------------------------------- | ----------------------------------------------------------- | ------------------------------------- | +| `just init-db` | Creates the schema on a brand-new database | `roles/spanner.databaseAdmin`, `roles/spanner.databaseUser` | no | +| `just migrate-db` | Applies pending Spanner schema migrations | `roles/spanner.databaseAdmin`, `roles/spanner.databaseUser` | no | +| `just load data/` | Stages one of this repo's datasets into the ingestion bucket and ingests it | none | yes | `just datasets` lists the values `load` accepts. `load` reaches the bucket named by the `storage_artifacts_bucket_name` output under your own credentials rather than the workflow account's, and it needs object write and object delete there. A permission error from any of these names the resource it denied rather than the principal it denied, which sends you to the wrong service account. Read the principal off the table first. -### init-db is safe only on a fresh database +### init-db on an existing database -`init-db`'s already-initialised check fails open. Any exception from the Spanner client or the table lookup returns "not initialised", and a permissions error is the default case, since Terraform grants `roles/spanner.databaseReader` to no principal. On that path `database/initialize` still fires against the live database. It runs under the impersonated workflow service account, so a 403 on your own principal does nothing to stop it. Nothing outside the ingestion-helper image documents what that call does to a populated database. - -The recipe passes `--init-only` and puts a `[confirm]` prompt in front of it. Answer that prompt only for a database you know is empty. +`init-db` skips a database whose `Node` table exists. If that check errors, the CLI treats the database as empty and applies the baseline schema under your own credentials. The schema's first statement, `CREATE TABLE Node`, then fails against an existing database before any DDL applies. ### Before running migrate-db Take a Spanner backup. Migrations are forward-only, the framework has no `downgrade()`, and Spanner has no transactional DDL rollback. The CLI asks whether you have taken a backup and does not take one for you. -Then wait for a `terraform apply` on this stack to finish. `apply.yml` runs asynchronously, so a merged PR leaves the apply still in flight. The ingestion-helper has to reach the matching image before the lock endpoints exist, and these commands read their targets from `terraform output -json`, which stays empty until the apply populates state. +Then wait for a `terraform apply` on this stack to finish. `apply.yml` runs asynchronously, so a merged PR leaves the apply still in flight. These commands read their targets from `terraform output -json`, which stays empty until the apply populates state. `migrate-db` takes the ingestion lock in Spanner and fails while an ingestion holds it. ### load replaces the whole input prefix Ingestion reads every file under `gs:///ingestion/input/` against a single `config.json`, so `load` clears that prefix before uploading. The clear runs under your own credentials and discards its own errors. Without object delete on the bucket it does nothing, the new dataset lands on top of the old files, and ingestion reads both. -`seed-db` and `load` both add data. `seed-db` re-applies Data Commons' own base seeds. `load` imports the datasets under `data/`. `seed-db` has no already-seeded check and no confirmation prompt, so it reseeds the moment you run it. - ## Migrator password rotation and recovery `datasets_migrator_password_version` in `dfo_datasets.tf` and `presentation_migrator_password_version` in `dfo_presentation.tf` each control the write-only password for a Secret Manager version and Cloud SQL migrator user. For normal rotation, increment the relevant counter in a PR. After the PR merges, CI applies the whole dcp stack. diff --git a/infra/dfo-dev/dcp/dfo.auto.tfvars b/infra/dfo-dev/dcp/dfo.auto.tfvars index a98aedd..b43886a 100644 --- a/infra/dfo-dev/dcp/dfo.auto.tfvars +++ b/infra/dfo-dev/dcp/dfo.auto.tfvars @@ -37,8 +37,9 @@ stateless_deletion_protection = false # dfo_storage.tf declares the artifacts bucket so it can carry lifecycle # rules, which the upstream module has no input for. The module receives # the name and passes it through to its consumers. -storage_create_artifacts_bucket = false -storage_artifacts_bucket_name = "dfo-dev-dc-artifacts-dev-fin-observatory" +storage_create_artifacts_bucket = false +storage_artifacts_bucket_name = "dfo-dev-dc-artifacts-dev-fin-observatory" +storage_artifacts_bucket_enable_versioning = false # ============================================================================= # Redis Module diff --git a/infra/dfo-prod/dcp/dfo.auto.tfvars b/infra/dfo-prod/dcp/dfo.auto.tfvars index 16cdb67..19160df 100644 --- a/infra/dfo-prod/dcp/dfo.auto.tfvars +++ b/infra/dfo-prod/dcp/dfo.auto.tfvars @@ -22,8 +22,9 @@ stateless_deletion_protection = false # ============================================================================= # Storage Module # ============================================================================= -storage_create_artifacts_bucket = false -storage_artifacts_bucket_name = "dfo-prod-dc-artifacts-prod-fin-observatory" +storage_create_artifacts_bucket = false +storage_artifacts_bucket_name = "dfo-prod-dc-artifacts-prod-fin-observatory" +storage_artifacts_bucket_enable_versioning = false # ============================================================================= # Redis Module diff --git a/infra/stacks/dcp/dfo_override.tf b/infra/stacks/dcp/dfo_override.tf index 855bff1..6292843 100644 --- a/infra/stacks/dcp/dfo_override.tf +++ b/infra/stacks/dcp/dfo_override.tf @@ -9,8 +9,9 @@ # # What it does: rewrites ONLY module.stack's auth_config, swapping the empty # var-sourced DC API key for the value read from Secret Manager in dfo.tf. The -# other two auth_config fields (google_maps_api_key, create_google_maps_key) -# are preserved from local.auth_config via merge(). +# other auth_config fields (google_maps_api_key, create_google_maps_key, +# google_maps_allowed_referrers) are preserved from local.auth_config via +# merge(). # # If you regenerate main.tf, this file and dfo.tf survive untouched and the # redirection keeps working. See dfo.README.md for the full story. diff --git a/infra/stacks/dcp/dfo_storage.tf b/infra/stacks/dcp/dfo_storage.tf index 0a997fa..e6f8fbe 100644 --- a/infra/stacks/dcp/dfo_storage.tf +++ b/infra/stacks/dcp/dfo_storage.tf @@ -4,6 +4,13 @@ # optional list, so rules added out of band on a module-owned bucket are # stripped on the module's next apply. The moved block below re-parents # the module's existing bucket onto this resource in place. +# +# Object versioning is off for this bucket. The module's +# storage_artifacts_bucket_enable_versioning flag only reaches the bucket +# the module creates, which storage_create_artifacts_bucket = false +# suppresses, so it is set to false in each environment's dfo.auto.tfvars +# and has no effect here. Versioning for this bucket would need a +# versioning block on the resource below. resource "google_storage_bucket" "dc_artifacts" { name = var.storage_artifacts_bucket_name project = var.project_id diff --git a/infra/stacks/dcp/main.tf b/infra/stacks/dcp/main.tf index 6327c82..866d9fc 100644 --- a/infra/stacks/dcp/main.tf +++ b/infra/stacks/dcp/main.tf @@ -40,14 +40,14 @@ resource "google_project_service" "apis" { "sqladmin.googleapis.com", "redis.googleapis.com", "secretmanager.googleapis.com", - "vpcaccess.googleapis.com", "artifactregistry.googleapis.com", "compute.googleapis.com" ], var.enable_spanner ? ["spanner.googleapis.com"] : [], var.enable_ingestion ? [ "workflows.googleapis.com", "workflowexecutions.googleapis.com", - "dataflow.googleapis.com" + "dataflow.googleapis.com", + "batch.googleapis.com" ] : [], var.spanner_enable_bigquery_connection ? [ "bigqueryconnection.googleapis.com", @@ -107,10 +107,26 @@ locals { website_search_scope = var.datacommons_services_website_search_scope } + network_config = { + enable = var.enable_network + enable_workload_vpc = var.enable_workload_vpc + prune_cloud_run_revisions = var.network_prune_cloud_run_revisions + create_vpc = var.network_create_vpc + network_name = var.network_name != "" ? var.network_name : "dc-vpc" + subnet_cidr = var.network_subnet_cidr + enable_cloud_nat = var.network_enable_cloud_nat + enable_flow_logs = var.network_enable_flow_logs + flow_sampling = var.network_flow_sampling + existing_network_id = var.network_existing_network_id + existing_subnet_id = var.network_existing_subnet_id + vpc_egress_mode = var.network_vpc_egress_mode + } + auth_config = { - google_datacommons_api_key = var.auth_google_datacommons_api_key - google_maps_api_key = var.auth_google_maps_api_key - create_google_maps_key = var.auth_create_google_maps_api_key + google_datacommons_api_key = var.auth_google_datacommons_api_key + google_maps_api_key = var.auth_google_maps_api_key + create_google_maps_key = var.auth_create_google_maps_api_key + google_maps_allowed_referrers = var.auth_google_maps_allowed_referrers } redis_config = { @@ -121,8 +137,8 @@ locals { location_id = var.redis_location_id alternative_location_id = var.redis_alternative_location_id replica_count = var.redis_replica_count - vpc_network_name = var.redis_vpc_network_name - vpc_connector_cidr = var.redis_vpc_connector_cidr + enable_auth = var.redis_enable_auth + enable_tls = var.redis_enable_tls } ingestion_config = { @@ -150,29 +166,34 @@ locals { workflow_lock_acquisition_timeout = var.ingestion_workflow_lock_acquisition_timeout helper_service_image = coalesce(var.ingestion_helper_service_image, "gcr.io/datcom-ci/datacommons-ingestion-helper:${var.dcp_version}") - # Dataflow Network & Scaling Configuration - dataflow_ip_configuration = var.ingestion_dataflow_ip_configuration - dataflow_subnetwork = var.ingestion_dataflow_subnetwork - dataflow_template_gcs_path = coalesce(var.ingestion_dataflow_template_gcs_path, "gs://datcom-templates/templates/flex/ingestion-${local.df_template_version}.json") - dataflow_max_workers = var.ingestion_dataflow_max_workers - dataflow_num_workers = var.ingestion_dataflow_num_workers - dataflow_worker_machine_type = var.ingestion_dataflow_worker_machine_type + # Worker Network Configuration (Dataflow & Cloud Batch) + worker_ip_configuration = var.ingestion_worker_ip_configuration + worker_subnetwork = var.ingestion_worker_subnetwork + + # Ingestion Dataflow Template & Worker Configuration + ingestion_dataflow_template_gcs_path = coalesce(var.ingestion_dataflow_template_gcs_path, "gs://datcom-templates/templates/flex/ingestion-${local.df_template_version}.json") + rollback_dataflow_template_gcs_path = coalesce(var.ingestion_rollback_dataflow_template_gcs_path, "gs://datcom-templates/templates/flex/rollback/rollback-${local.df_template_version}.json") + dataflow_max_workers = var.ingestion_dataflow_max_workers + dataflow_num_workers = var.ingestion_dataflow_num_workers + dataflow_worker_machine_type = var.ingestion_dataflow_worker_machine_type } } module "stack" { # DO NOT CHANGE the format of the source line below. - # The Data Commons CLI relies on matching 'source = "git::https://github.com/datacommonsorg/datacommons.git//infra/dcp/modules/stack?ref=v1.1.5"' to generate user scaffolding. - source = "git::https://github.com/datacommonsorg/datacommons.git//infra/dcp/modules/stack?ref=v1.1.5" - - global = local.global_config - spanner_config = local.spanner_config - storage_create_artifacts_bucket = var.storage_create_artifacts_bucket - storage_artifacts_bucket_name = var.storage_artifacts_bucket_name - datacommons_services_config = local.datacommons_services_config - auth_config = local.auth_config - redis_config = local.redis_config - ingestion_config = local.ingestion_config + # The Data Commons CLI relies on matching 'source = "git::https://github.com/datacommonsorg/datacommons.git//infra/dcp/modules/stack?ref=v1.1.6"' to generate user scaffolding. + source = "git::https://github.com/datacommonsorg/datacommons.git//infra/dcp/modules/stack?ref=v1.1.6" + + global = local.global_config + network_config = local.network_config + spanner_config = local.spanner_config + storage_create_artifacts_bucket = var.storage_create_artifacts_bucket + storage_artifacts_bucket_name = var.storage_artifacts_bucket_name + storage_artifacts_bucket_enable_versioning = var.storage_artifacts_bucket_enable_versioning + datacommons_services_config = local.datacommons_services_config + auth_config = local.auth_config + redis_config = local.redis_config + ingestion_config = local.ingestion_config depends_on = [google_project_service.apis] } diff --git a/infra/stacks/dcp/outputs.tf b/infra/stacks/dcp/outputs.tf index c4ffbc0..0c43b32 100644 --- a/infra/stacks/dcp/outputs.tf +++ b/infra/stacks/dcp/outputs.tf @@ -40,11 +40,6 @@ output "ingestion_service_url" { value = module.stack.ingestion_service_url } -output "ingestion_prep_job_name" { - description = "Name of the data ingestion pre-processing job" - value = module.stack.ingestion_prep_job_name -} - output "ingestion_workflow_service_account_email" { description = "Email of the service account used by the ingestion workflow" value = module.stack.ingestion_workflow_service_account_email @@ -55,6 +50,31 @@ output "storage_artifacts_bucket_name" { value = module.stack.storage_artifacts_bucket_name } +output "datacommons_services_image" { + description = "Container image URI used by the Data Commons serving service" + value = module.stack.datacommons_services_image +} + +output "ingestion_helper_image" { + description = "Container image URI used by the ingestion helper service" + value = module.stack.ingestion_helper_image +} + +output "ingestion_preprocessing_image" { + description = "Container image URI used by the Cloud Batch preprocessing job" + value = module.stack.ingestion_preprocessing_image +} + +output "ingestion_postprocessing_image" { + description = "Container image URI used by the Cloud Run postprocessing job" + value = module.stack.ingestion_postprocessing_image +} + +output "ingestion_dataflow_template_gcs_path" { + description = "GCS path to the Dataflow flex template specification" + value = module.stack.ingestion_dataflow_template_gcs_path +} + output "project_id" { description = "The GCP project ID where resources are deployed" value = var.project_id @@ -69,3 +89,22 @@ output "ingestion_input_path" { description = "The GCS root directory where input data files are stored." value = var.ingestion_input_path } + +output "ingestion_artifacts_path" { + description = "The GCS path to input files in the artifacts bucket." + value = var.ingestion_artifacts_path +} +output "network_id" { + description = "ID of the VPC network" + value = module.stack.network_id +} + +output "subnet_id" { + description = "ID of the private subnetwork" + value = module.stack.subnet_id +} + +output "subnet_url" { + description = "URL / self_link of the private subnetwork" + value = module.stack.subnet_url +} diff --git a/infra/stacks/dcp/variables.tf b/infra/stacks/dcp/variables.tf index 49dd7c4..ee68fb4 100644 --- a/infra/stacks/dcp/variables.tf +++ b/infra/stacks/dcp/variables.tf @@ -52,7 +52,7 @@ variable "billing_project_id" { variable "dcp_version" { description = "The version of the Data Commons Platform to deploy. This controls the default tag for Docker images and template paths if specific overrides are not provided." type = string - default = "1.1.5" + default = "1.1.6" } # ============================================================================= @@ -77,6 +77,12 @@ variable "auth_create_google_maps_api_key" { default = true } +variable "auth_google_maps_allowed_referrers" { + description = "A list of HTTP referrers allowed to use the Google Maps API key (e.g. ['https://example.com/*', 'http://localhost:*']). If empty, no browser referrer restrictions are enforced." + type = list(string) + default = [] +} + # ============================================================================= # Storage Module # ============================================================================= @@ -93,12 +99,109 @@ variable "storage_artifacts_bucket_name" { default = "" } +variable "storage_artifacts_bucket_enable_versioning" { + description = "Enable object versioning on the artifacts GCS bucket. Keeps historical versions of objects when overwritten or deleted." + type = bool + default = true +} + +# ============================================================================= +# Network Module +# ============================================================================= +# NOTE ON DISABLING / DECOMMISSIONING VPC: +# If you wish to disable or destroy an existing VPC network (enable_network = false), +# Google Cloud requires a two-stage apply to prevent "Subnetwork is in use" errors: +# +# Stage 1: Detach workloads and prune inactive Cloud Run revisions +# Set enable_workload_vpc = false (and enable_redis = false if enabled), +# and set network_prune_cloud_run_revisions = true, keeping enable_network = true. +# Run `terraform apply`. +# This detaches Cloud Run and Dataflow, and prunes older 0-traffic revisions +# that hold Direct VPC Egress (/28) IP reservations. +# +# Stage 2: Tear down the network infrastructure +# Set enable_network = false. Run `terraform apply`. +# The VPC, subnet, router, and NAT will now delete smoothly with no errors. +# ============================================================================= + +variable "enable_network" { + description = "Enable VPC networking infrastructure for DCP services and ingestion jobs. NOTE: To decommission, follow the two-stage process: set enable_workload_vpc = false first, apply, then set enable_network = false." + type = bool + default = false +} + +variable "enable_workload_vpc" { + description = "Whether compute workloads (Cloud Run services, jobs, Dataflow) attach to the VPC. Defaults to true so workloads automatically use the VPC when enable_network = true. Set to false in Stage 1 of VPC decommissioning to cleanly detach workloads before destroying network infrastructure in Stage 2." + type = bool + default = true +} + +variable "network_prune_cloud_run_revisions" { + description = "Prune inactive (0-traffic) Cloud Run revisions for DCP services (dc-datacommons-service and dc-ingestion-helper). Required when detaching workloads or destroying VPC to release Direct VPC Egress subnet IP reservations." + type = bool + default = false +} + +variable "network_create_vpc" { + description = "Whether to provision a new custom VPC and subnet. Set to false to attach to an existing/Shared VPC." + type = bool + default = true +} + +variable "network_name" { + description = "Name of the VPC network (used if network_create_vpc = true or for existing network lookup)." + type = string + default = "dc-vpc" +} + +variable "network_subnet_cidr" { + description = "CIDR range for the private subnet." + type = string + default = "10.0.0.0/24" +} + +variable "network_enable_cloud_nat" { + description = "Whether to provision a Cloud Router and Cloud NAT for secure outbound internet egress from private workers." + type = bool + default = false +} + +variable "network_existing_network_id" { + description = "Network ID or self_link when attaching to an existing/Shared VPC." + type = string + default = null +} + +variable "network_existing_subnet_id" { + description = "Subnet ID or self_link when attaching to an existing/Shared VPC." + type = string + default = null +} + +variable "network_enable_flow_logs" { + description = "Enable VPC Flow Logs on the private subnet for network visibility and security forensics." + type = bool + default = true +} + +variable "network_flow_sampling" { + description = "Sampling rate for VPC Flow Logs between 0.0 and 1.0 (default 1.0 = 100% of packets sampled, CIS GCP Benchmark standard)." + type = number + default = 1.0 +} + +variable "network_vpc_egress_mode" { + description = "VPC egress mode for Cloud Run services and jobs (PRIVATE_RANGES_ONLY or ALL_TRAFFIC). Set to ALL_TRAFFIC to route outbound internet traffic through Cloud NAT." + type = string + default = "PRIVATE_RANGES_ONLY" +} + # ============================================================================= # Redis Module # ============================================================================= variable "enable_redis" { - description = "Enable a Memorystore Redis instance for caching" + description = "Enable a Memorystore Redis instance for caching. NOTE: Cloud Memorystore instances only have private IP addresses and require VPC networking (enable_network = true) to be accessible from Cloud Run services." type = bool default = false } @@ -139,16 +242,16 @@ variable "redis_replica_count" { default = 1 } -variable "redis_vpc_network_name" { - description = "VPC network name" - type = string - default = "default" +variable "redis_enable_auth" { + description = "Enable Redis AUTH (password authentication) on the Memorystore instance. When true, the auto-generated AUTH password is stored in Secret Manager and mounted as REDIS_PASSWORD on DCP services." + type = bool + default = true } -variable "redis_vpc_connector_cidr" { - description = "CIDR range for the VPC Access Connector" - type = string - default = "10.13.0.0/28" +variable "redis_enable_tls" { + description = "Enable in-transit TLS encryption (SERVER_AUTHENTICATION) on the Memorystore instance. When true, the auto-generated CA certificate is passed as REDIS_CA_CERT on DCP services." + type = bool + default = true } # ============================================================================= @@ -339,15 +442,25 @@ variable "ingestion_preprocessing_job_image" { } variable "ingestion_preprocessing_job_cpu" { - description = "CPU limit for the pre-processing job container" + description = "CPU limit in milliCPUs for the pre-processing Cloud Batch container (e.g. '8000' for 8 vCPUs)" type = string - default = "8" + default = "8000" + + validation { + condition = can(regex("^[0-9]+$", var.ingestion_preprocessing_job_cpu)) && tonumber(var.ingestion_preprocessing_job_cpu) >= 1000 + error_message = "The ingestion_preprocessing_job_cpu must be an integer representing at least 1000 milliCPUs (1 vCPU, e.g. '8000')." + } } variable "ingestion_preprocessing_job_memory" { - description = "Memory limit for the pre-processing job container" + description = "Memory limit in MiB for the pre-processing Cloud Batch container (e.g. '32768' for 32GiB)" type = string - default = "32G" + default = "32768" + + validation { + condition = can(regex("^[0-9]+$", var.ingestion_preprocessing_job_memory)) && tonumber(var.ingestion_preprocessing_job_memory) >= 1024 + error_message = "The ingestion_preprocessing_job_memory must be an integer representing at least 1024 MiB (1 GiB, e.g. '32768')." + } } variable "ingestion_preprocessing_job_timeout" { @@ -429,17 +542,17 @@ variable "ingestion_helper_service_image" { } # ============================================================================= -# Ingestion - Dataflow Network Configuration +# Ingestion - Worker Network Configuration (Dataflow & Cloud Batch) # ============================================================================= -variable "ingestion_dataflow_ip_configuration" { - description = "IP configuration for Dataflow workers (WORKER_IP_UNSPECIFIED, WORKER_IP_PUBLIC, WORKER_IP_PRIVATE). Set to WORKER_IP_PRIVATE when a compute.vmExternalIpAccess org policy restricts VMs from obtaining external IPs." +variable "ingestion_worker_ip_configuration" { + description = "IP configuration for ingestion workers (Dataflow and Cloud Batch): WORKER_IP_UNSPECIFIED, WORKER_IP_PUBLIC, WORKER_IP_PRIVATE. Set to WORKER_IP_PRIVATE when a compute.vmExternalIpAccess org policy restricts VMs from obtaining external IPs. NOTE: WORKER_IP_PRIVATE requires enable_network = true or an explicitly configured ingestion_worker_subnetwork." type = string default = "WORKER_IP_UNSPECIFIED" } -variable "ingestion_dataflow_subnetwork" { - description = "Subnetwork for Dataflow workers. Required when ingestion_dataflow_ip_configuration is WORKER_IP_PRIVATE. Format: regions/{region}/subnetworks/{subnetwork}." +variable "ingestion_worker_subnetwork" { + description = "Subnetwork for ingestion workers (Dataflow and Cloud Batch). Automatically populated from the network module when enable_network = true. If enable_network = false and WORKER_IP_PRIVATE is used, this variable must be explicitly provided. Format: regions/{region}/subnetworks/{subnetwork} or full self_link." type = string default = "" } @@ -449,7 +562,13 @@ variable "ingestion_dataflow_subnetwork" { # ============================================================================= variable "ingestion_dataflow_template_gcs_path" { - description = "GCS path to the Dataflow Flex Template container spec. If not provided (null), defaults to the stable template defined in the workflow module (modules/ingestion/workflow/variables.tf)." + description = "GCS path to the ingestion Dataflow Flex Template container spec. If not provided (null), defaults to the stable ingestion template pinned to dcp_version." + type = string + default = null +} + +variable "ingestion_rollback_dataflow_template_gcs_path" { + description = "GCS path to the rollback Dataflow Flex Template container spec. If not provided (null), defaults to the stable rollback template pinned to dcp_version." type = string default = null } diff --git a/justfile b/justfile index e3525e1..a0f1500 100644 --- a/justfile +++ b/justfile @@ -24,15 +24,20 @@ STUDIO_TOKEN_SECRET := ENV + "-studio-api-token" # source ref). init-db and load both invoke datacommons-cli, and an unpinned # uvx call floats to whatever's latest on PyPI regardless of the module ref # this repo is actually running. -DCP_CLI_VERSION := "1.1.5" +DCP_CLI_VERSION := "1.1.6" # datacommons-cli finds its project, Spanner instance and database, # ingestion-helper URL and workflow service account by running -# `terraform output -json` in its own working directory. It offers no -# flag, environment variable, config file or directory walk for any of -# them, and just runs a recipe from this file's directory whatever -# directory the operator invoked it from. Hence the `cd {{TF_DIR}} &&` -# on every recipe that reaches the CLI. +# `terraform output -json` in its own working directory unless the admin +# group gets remote-state flags. --project-id with --instance-name assumes +# the state prefix terraform/state/, which misses this repo's +# terraform/state//dcp. --tf-state-location takes the exact state URI +# (gs:////default.tfstate) and would reach it, but the URI +# would repeat the bucket and prefix that backend.tf already holds. The +# recipes cd into the stack directory instead and read that backend. just +# runs a recipe from this file's directory whatever directory the operator +# invoked it from, hence the `cd {{TF_DIR}} &&` on every recipe that +# reaches the CLI. TF_DIR := "infra" / ENV / STACK # GCP Application Default Credentials (what Terraform's GCS backend uses) @@ -257,55 +262,32 @@ setup-github-apps *args: # ── Data ────────────────────────────────────── -# One-time bootstrap for a brand-new dcp database. Creates the schema. -# --init-only skips the seed step (now 'just seed-db') but still runs -# migrations, matching upstream's own guard message treating initialize, -# migrate and seed as independently runnable steps. -# The already-initialised check runs Spanner locally under your ADC and -# fails open on a permissions or connectivity error. On that path, -# initialize_database() still POSTs to the ingestion-helper's -# database/initialize. Migrations find nothing pending on an up-to-date -# schema and skip. The CLI reports "Successfully initialized Spanner -# database!" with no sign anything was unusual. What database/initialize -# does to an existing database isn't established from source. It runs -# server-side in the ingestion-helper image, under the impersonated -# workflow service account, so a 403 on your principal doesn't restrain it. -# Only run this against a known-fresh database. Needs -# roles/iam.serviceAccountTokenCreator on the ingestion workflow service -# account (init call) and Spanner read (e.g. roles/spanner.databaseReader) -# on your principal (initialised check). Terraform grants neither by -# default (see README.md). -[confirm("The already-initialised check fails open on a permissions or connectivity error. On that path this still calls the ingestion-helper's database/initialize against a possibly-live database and reports success regardless — migrations are safe (they skip if the schema is current), but database/initialize's own effect on an existing database isn't established. Continue only if this database is known to be fresh.")] +# One-time bootstrap for a brand-new dcp database. Applies the baseline +# schema, then every migration. The ingestion preprocessor creates the +# dc/g/Root node itself, so there is no seed step. The CLI talks to +# Spanner directly under your ADC and needs roles/spanner.databaseAdmin +# and roles/spanner.databaseUser on your own principal. Terraform grants +# neither (see README.md). The CLI skips a database whose Node table +# exists. If that check errors it treats the database as empty, and the +# baseline schema's first statement, CREATE TABLE Node, then fails +# against an existing database before any DDL applies. [doc("Bootstrap schema for a brand-new dcp database")] init-db: _check-gcloud - cd {{TF_DIR}} && uvx --from "datacommons-cli=={{DCP_CLI_VERSION}}" --with "datacommons-admin=={{DCP_CLI_VERSION}}" datacommons admin init-db --init-only + cd {{TF_DIR}} && uvx --from "datacommons-cli=={{DCP_CLI_VERSION}}" --with "datacommons-admin=={{DCP_CLI_VERSION}}" datacommons admin init-db # Migrate the dcp database schema forward. Run only after 'terraform -# apply' has finished, not just merged. The apply workflow fires -# asynchronously, and the migration locks through endpoints that exist -# only on the matching ingestion-helper image. Acquires that lock through -# impersonated credentials (needs roles/iam.serviceAccountTokenCreator on -# the ingestion workflow service account) but talks to Spanner as you -# (needs roles/spanner.databaseAdmin on your own principal). Terraform -# grants neither by default. Migrations are forward-only. The CLI's prompt -# recommends and confirms a backup but doesn't take one. Back up Spanner -# yourself before proceeding. +# apply' has finished. The apply workflow fires asynchronously after a +# merge, and the CLI reads its Spanner targets from Terraform outputs. +# The CLI runs under your ADC, takes the ingestion lock in +# Spanner directly, and fails if an ingestion holds it. It needs +# roles/spanner.databaseAdmin and roles/spanner.databaseUser on your own +# principal. Terraform grants neither. Migrations are forward-only. The +# CLI's prompt recommends and confirms a backup but doesn't take one. +# Back up Spanner yourself before proceeding. [doc("Migrate the dcp database schema forward")] migrate-db: _check-gcloud cd {{TF_DIR}} && uvx --from "datacommons-cli=={{DCP_CLI_VERSION}}" --with "datacommons-admin=={{DCP_CLI_VERSION}}" datacommons admin migrate-db -# Re-run the platform's base geographic and schema seeds. 'just load' -# stages and ingests a custom dataset. This only refreshes Base DC's own -# seed data. It goes through the ingestion-helper service, not Spanner -# directly. It needs roles/iam.serviceAccountTokenCreator on the ingestion -# workflow service account, and no Spanner role at all. Unguarded and -# unprompted, an unconditional reseed call with no already-seeded check -# and no confirmation. Typing the command is the only thing standing -# between you and it running. -[doc("Re-run Base DC's geographic and schema seeds")] -seed-db: _check-gcloud - cd {{TF_DIR}} && uvx --from "datacommons-cli=={{DCP_CLI_VERSION}}" --with "datacommons-admin=={{DCP_CLI_VERSION}}" datacommons admin seed-db - # Stage a dataset into the ingestion bucket + trigger ingestion: just load data/oecd-wage load dataset: _check-gcloud DCP_CLI_VERSION={{DCP_CLI_VERSION}} scripts/load.sh {{dataset}} {{ENV}} From 519c0f0ddbae18674698ef471df3c53acc06fc43 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Fri, 23 Oct 2026 15:14:05 +0200 Subject: [PATCH 3/3] chore(dcp): restore prod stateless deletion protection Prod runs Data Commons Platform 1.1.6 and its Cloud Run preprocessing job is gone, so stateless_deletion_protection returns to true. --- infra/dfo-prod/dcp/dfo.auto.tfvars | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/infra/dfo-prod/dcp/dfo.auto.tfvars b/infra/dfo-prod/dcp/dfo.auto.tfvars index 19160df..8ef5fe0 100644 --- a/infra/dfo-prod/dcp/dfo.auto.tfvars +++ b/infra/dfo-prod/dcp/dfo.auto.tfvars @@ -12,12 +12,9 @@ project_id = "prod-fin-observatory" namespace = "dfo-prod" region = "europe-west1" -stateful_deletion_protection = true -skip_container_restarts = true - -# DCP v1.1.6 deletes the Cloud Run preprocessing job, and Terraform cannot -# delete it while protection is on. Set back to true once prod runs v1.1.6. -stateless_deletion_protection = false +stateful_deletion_protection = true +skip_container_restarts = true +stateless_deletion_protection = true # ============================================================================= # Storage Module