From d41779ad06074c76b7c9a5b11640c517c042c409 Mon Sep 17 00:00:00 2001 From: "dfo-release[bot]" <336980964+dfo-release[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:22:09 +0100 Subject: [PATCH 1/3] Seed dev from ONEcampaign/dfo-pipelines@9fc00dfff9a2a277445da33e78a7f86f86e8770f Synced-from: ONEcampaign/dfo-pipelines@9fc00dfff9a2a277445da33e78a7f86f86e8770f Co-authored-by: Tilly Woodfield <22456167+tillywoodfield@users.noreply.github.com> --- .github/workflows/ci.yml | 56 ++-------------------------------------- README.md | 3 --- pyproject.toml | 4 +-- uv.lock | 12 ++++----- 4 files changed, 10 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8824dfa..91d4819 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,7 @@ on: jobs: lint: - # Mints a dfo-reader token, which prod keeps only in main-limited - # environments, so this runs in ONEcampaign only. The identical tree - # ran it here before it was synced. - if: github.repository_owner == 'ONEcampaign' && github.event.pull_request.draft != true + if: github.event.pull_request.draft != true runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -35,29 +32,6 @@ jobs: - name: Set up Python 3.13 run: uv python install '3.13' - # Temporary: dfo-ontology is a private repository, so uv cannot resolve the - # git dependency with the token a workflow is given. The lockfile pins it in - # ONEcampaign, so the token is read-only on that repository whichever - # organisation runs this. Rewriting the URL rather than depending over ssh - # keeps the dependency installable locally as it is declared, and narrowing - # it to ONEcampaign keeps the token off every other fetch. Remove these two - # steps once the repository is public. - - name: Mint a read-only token for dfo-ontology - id: reader - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.READER_APP_CLIENT_ID }} - private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - owner: ONEcampaign - repositories: dfo-ontology - permission-contents: read - - - name: Authenticate to private repositories - run: > - git config --global - url."https://x-access-token:${{ steps.reader.outputs.token }}@github.com/ONEcampaign/".insteadOf - "https://github.com/ONEcampaign/" - - name: Install dependencies run: uv sync --frozen @@ -82,10 +56,7 @@ jobs: test: - # Mints a dfo-reader token, which prod keeps only in main-limited - # environments, so this runs in ONEcampaign only. The identical tree - # ran it here before it was synced. - if: github.repository_owner == 'ONEcampaign' && github.event.pull_request.draft != true + if: github.event.pull_request.draft != true runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -104,29 +75,6 @@ jobs: - name: Set up Python 3.13 run: uv python install '3.13' - # Temporary: dfo-ontology is a private repository, so uv cannot resolve the - # git dependency with the token a workflow is given. The lockfile pins it in - # ONEcampaign, so the token is read-only on that repository whichever - # organisation runs this. Rewriting the URL rather than depending over ssh - # keeps the dependency installable locally as it is declared, and narrowing - # it to ONEcampaign keeps the token off every other fetch. Remove these two - # steps once the repository is public. - - name: Mint a read-only token for dfo-ontology - id: reader - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.READER_APP_CLIENT_ID }} - private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - owner: ONEcampaign - repositories: dfo-ontology - permission-contents: read - - - name: Authenticate to private repositories - run: > - git config --global - url."https://x-access-token:${{ steps.reader.outputs.token }}@github.com/ONEcampaign/".insteadOf - "https://github.com/ONEcampaign/" - - name: Install dependencies run: uv sync --frozen diff --git a/README.md b/README.md index 3a1eb03..b577729 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,6 @@ Data pipelines for the [Development Finance Observatory](https://developmentfina This project uses [uv](https://docs.astral.sh/uv/) for dependency management. Install uv first, and `uv sync` installs the Python version the project needs along with its dependencies. `dbt deps` then installs the dbt packages the models use. -> [!IMPORTANT] -> One dependency, `dfo-ontology`, is installed from a repository that is not yet public, so `uv sync` only succeeds with access to it. - ```bash uv sync uv run dbt deps --project-dir dbt --profiles-dir dbt diff --git a/pyproject.toml b/pyproject.toml index fa62e1e..daadd71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "universal-pathlib>=0.3.10", "pandas>=3.0.5", "imf-reader>=2.2.0", - "dcp-tools>=1.0.0a5", + "dcp-tools>=1.0.1", "dfo-ontology", "oda-data==2.8.1", "pyyaml>=6.0", @@ -158,4 +158,4 @@ show_missing = true precision = 2 [tool.uv.sources] -dfo-ontology = { git = "https://github.com/ONEcampaign/dfo-ontology", rev = "v0.2.1" } +dfo-ontology = { git = "https://github.com/DevelopmentFinanceObservatory/ontology", rev = "v0.2.1" } diff --git a/uv.lock b/uv.lock index 1eb916e..343fa02 100644 --- a/uv.lock +++ b/uv.lock @@ -735,7 +735,7 @@ wheels = [ [[package]] name = "dcp-tools" -version = "1.0.0a5" +version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "datacommons-admin" }, @@ -744,9 +744,9 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/f8/139937d17ce501e9fb19d476e18cca62cf1c5af25ef8bf48226a2a8086f3/dcp_tools-1.0.0a5.tar.gz", hash = "sha256:54c977e51894ccda8f775cf5c5ef946dbbcd3b990e187b0bee6cbefc330a9337", size = 44659, upload-time = "2026-09-10T10:09:09.032Z" } +sdist = { url = "https://files.pythonhosted.org/packages/37/60/15df64896af5efa8adba07ec9be4df6d35a181c8aaf507c4290eea70fada/dcp_tools-1.0.1.tar.gz", hash = "sha256:d73eb5c93095b47f601a6ffd92f8dd94b2ffe090797eef5e96415f61141c1244", size = 46732, upload-time = "2026-10-13T15:28:51.69Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/94/dd46a060069463c7e3738480d0efb6183e10257abdca861365c6c960d078/dcp_tools-1.0.0a5-py3-none-any.whl", hash = "sha256:4f01db79046cd475929fdeefe740a0ff4829ff31fef0b18225cf90c53d4c661c", size = 56904, upload-time = "2026-09-10T10:09:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/b4/24/d17e45df5db6b176ece631d5351c3bfe04015eba3a14c745b83dff429b97/dcp_tools-1.0.1-py3-none-any.whl", hash = "sha256:1686bbfbf0edef4db88d73e4a2e5d3f9d11a28c6f37bf4307c0c7a7c6e3e96d3", size = 56802, upload-time = "2026-10-13T15:28:50.709Z" }, ] [[package]] @@ -773,7 +773,7 @@ wheels = [ [[package]] name = "dfo-ontology" version = "0.1.0" -source = { git = "https://github.com/ONEcampaign/dfo-ontology?rev=v0.2.1#5d059b116c88985601516e61ba20c5a69f5bb537" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=v0.2.1#afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } dependencies = [ { name = "httpx" }, ] @@ -822,8 +822,8 @@ requires-dist = [ { name = "dagster-dbt", specifier = ">=0.29.10" }, { name = "dagster-duckdb", specifier = ">=0.29.10" }, { name = "dbt-duckdb", specifier = ">=1.10.1" }, - { name = "dcp-tools", specifier = ">=1.0.0a5" }, - { name = "dfo-ontology", git = "https://github.com/ONEcampaign/dfo-ontology?rev=v0.2.1" }, + { name = "dcp-tools", specifier = ">=1.0.1" }, + { name = "dfo-ontology", git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=v0.2.1" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "imf-reader", specifier = ">=2.2.0" }, { name = "oda-data", specifier = "==2.8.1" }, From f51dada254f870dbc1c0bced1efcd07776b67620 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Thu, 15 Oct 2026 10:41:33 +0200 Subject: [PATCH 2/3] build: take dfo-ontology from the DFO repo at a pinned commit, and drop the ONEcampaign guards dfo-ontology resolves to DevelopmentFinanceObservatory/ontology at afa964e452cc08dbfb3cf11379bbf9a9f32b14f4 (the tree of v0.2.1). The Codecov upload loses its owner guard, and the sync-guard workflow and the actionlint entries that only served it are removed. --- .github/actionlint.yaml | 9 ------- .github/workflows/ci.yml | 2 -- .github/workflows/sync-guard.yml | 45 -------------------------------- pyproject.toml | 2 +- uv.lock | 4 +-- 5 files changed, 3 insertions(+), 59 deletions(-) delete mode 100644 .github/actionlint.yaml delete mode 100644 .github/workflows/sync-guard.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml deleted file mode 100644 index 0a4aa26..0000000 --- a/.github/actionlint.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# actionlint 1.7.12 bundles metadata for actions/create-github-app-token@v3 that predates the -# `client-id` input, so it rejects `client-id` and asks for the deprecated `app-id`. The live -# v3 action.yml defines `client-id` and marks `app-id` deprecated. Drop these once actionlint -# ships current metadata. -paths: - .github/workflows/**/*.{yml,yaml}: - ignore: - - 'input "client-id" is not defined in action "actions/create-github-app-token@v3"' - - 'missing input "app-id" which is required by action "actions/create-github-app-token@v3"' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91d4819..f3e745e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,8 +100,6 @@ jobs: run: uv run pytest --cov='src/dfo_pipelines' --cov-report=xml --cov-report=term - name: Upload coverage - # Coverage reports to ONEcampaign's Codecov project only. - if: github.repository_owner == 'ONEcampaign' uses: codecov/codecov-action@v6 with: files: ./coverage.xml diff --git a/.github/workflows/sync-guard.yml b/.github/workflows/sync-guard.yml deleted file mode 100644 index 6635731..0000000 --- a/.github/workflows/sync-guard.yml +++ /dev/null @@ -1,45 +0,0 @@ -# sync-guard: the required check on main in every DevelopmentFinanceObservatory -# repo. It passes only a sync PR whose tree matches its ONEcampaign source -# commit, or a release App image pin. The rules live in -# DevelopmentFinanceObservatory/infra, in .github/actions/sync-guard and -# scripts/sync-guard.sh. -# -# pull_request_target runs this file and the action from the default branch, -# never from the PR, so a PR cannot edit the guard it faces. The job is -# prod-only. In ONEcampaign it skips before the private action is ever -# fetched. -# -# The job posts the sync-guard check run on the PR's head commit with the -# release App's token, and the ruleset accepts sync-guard only from that -# App. The token's secrets live in the sync-guard environment, which only -# main may use, so a workflow a PR adds cannot mint it. -name: sync-guard - -on: - pull_request_target: - types: [opened, synchronize, reopened] - branches: [main] - -permissions: - contents: read - -# A force-push cancels the run for the old head. Each run judges the head -# commit its own event names. -concurrency: - group: sync-guard-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - sync-guard: - # Named apart from the check run it posts, which is the required one. - name: sync-guard runner - if: github.repository_owner == 'DevelopmentFinanceObservatory' - runs-on: ubuntu-latest - environment: sync-guard - steps: - - uses: DevelopmentFinanceObservatory/infra/.github/actions/sync-guard@main - with: - reader-client-id: ${{ vars.READER_APP_CLIENT_ID }} - reader-private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - release-client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} - release-private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} diff --git a/pyproject.toml b/pyproject.toml index daadd71..674395d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,4 +158,4 @@ show_missing = true precision = 2 [tool.uv.sources] -dfo-ontology = { git = "https://github.com/DevelopmentFinanceObservatory/ontology", rev = "v0.2.1" } +dfo-ontology = { git = "https://github.com/DevelopmentFinanceObservatory/ontology", rev = "afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } diff --git a/uv.lock b/uv.lock index 343fa02..6b9a0db 100644 --- a/uv.lock +++ b/uv.lock @@ -773,7 +773,7 @@ wheels = [ [[package]] name = "dfo-ontology" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=v0.2.1#afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=afa964e452cc08dbfb3cf11379bbf9a9f32b14f4#afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } dependencies = [ { name = "httpx" }, ] @@ -823,7 +823,7 @@ requires-dist = [ { name = "dagster-duckdb", specifier = ">=0.29.10" }, { name = "dbt-duckdb", specifier = ">=1.10.1" }, { name = "dcp-tools", specifier = ">=1.0.1" }, - { name = "dfo-ontology", git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=v0.2.1" }, + { name = "dfo-ontology", git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "imf-reader", specifier = ">=2.2.0" }, { name = "oda-data", specifier = "==2.8.1" }, From c5417f14f05999e9b5b68c5b2cb3da76b29c8952 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Thu, 15 Oct 2026 12:07:21 +0200 Subject: [PATCH 3/3] build: repin DFO dependencies to their re-authored commits --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 674395d..bb0d370 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,4 +158,4 @@ show_missing = true precision = 2 [tool.uv.sources] -dfo-ontology = { git = "https://github.com/DevelopmentFinanceObservatory/ontology", rev = "afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } +dfo-ontology = { git = "https://github.com/DevelopmentFinanceObservatory/ontology", rev = "617bdb685e79022494132ef3614674c232b395fc" } diff --git a/uv.lock b/uv.lock index 6b9a0db..5358bef 100644 --- a/uv.lock +++ b/uv.lock @@ -773,7 +773,7 @@ wheels = [ [[package]] name = "dfo-ontology" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=afa964e452cc08dbfb3cf11379bbf9a9f32b14f4#afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=617bdb685e79022494132ef3614674c232b395fc#617bdb685e79022494132ef3614674c232b395fc" } dependencies = [ { name = "httpx" }, ] @@ -823,7 +823,7 @@ requires-dist = [ { name = "dagster-duckdb", specifier = ">=0.29.10" }, { name = "dbt-duckdb", specifier = ">=1.10.1" }, { name = "dcp-tools", specifier = ">=1.0.1" }, - { name = "dfo-ontology", git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=afa964e452cc08dbfb3cf11379bbf9a9f32b14f4" }, + { name = "dfo-ontology", git = "https://github.com/DevelopmentFinanceObservatory/ontology?rev=617bdb685e79022494132ef3614674c232b395fc" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "imf-reader", specifier = ">=2.2.0" }, { name = "oda-data", specifier = "==2.8.1" },