diff --git a/providers/ordinal/README.md b/providers/ordinal/README.md index 3e71149..d5be39e 100644 --- a/providers/ordinal/README.md +++ b/providers/ordinal/README.md @@ -47,8 +47,12 @@ verification is needed. > > These are **not** live captures. They use Ordinal's own placeholder data > (`Acme Inc`, `jane@example.com`, `550e8400-…` UUIDs) so they can't be -> mistaken for real account traffic, and `headers` are representative, not -> recorded (the docs don't enumerate delivery headers). +> mistaken for real account traffic. Each file says so itself: it carries a +> `source` block naming the docs page its body was read from and the date, +> and [`index.json`](./index.json) marks the version +> `provenance.latest.sourced_via: "docs"`. `headers` is `content-type` only: +> the docs state the body is JSON and document no other delivery header (no +> User-Agent, no event-type header, no signature). ## Event taxonomy diff --git a/providers/ordinal/index.json b/providers/ordinal/index.json index 1c247f4..16733f4 100644 --- a/providers/ordinal/index.json +++ b/providers/ordinal/index.json @@ -3,5 +3,11 @@ "configs": { "latest_version": "latest", "topic_identifier": "type" + }, + "provenance": { + "latest": { + "sourced_via": "docs", + "sourced_on": "2026-09-28" + } } } diff --git a/providers/ordinal/latest/campaign.approval.approved.json b/providers/ordinal/latest/campaign.approval.approved.json index 650dfef..eb74dcf 100644 --- a/providers/ordinal/latest/campaign.approval.approved.json +++ b/providers/ordinal/latest/campaign.approval.approved.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "campaign.approval.approved", @@ -42,5 +40,10 @@ }, "createdAt": "2025-02-26T16:15:00.000Z" }, - "topic": "campaign.approval.approved" + "topic": "campaign.approval.approved", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/campaign-approval-approved", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/campaign.approval.requested.json b/providers/ordinal/latest/campaign.approval.requested.json index 22eec27..87ebf2b 100644 --- a/providers/ordinal/latest/campaign.approval.requested.json +++ b/providers/ordinal/latest/campaign.approval.requested.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "campaign.approval.requested", @@ -46,5 +44,10 @@ }, "createdAt": "2025-02-26T14:00:00.000Z" }, - "topic": "campaign.approval.requested" + "topic": "campaign.approval.requested", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/campaign-approval-requested", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/invite.accepted.json b/providers/ordinal/latest/invite.accepted.json index 01e0adf..488d443 100644 --- a/providers/ordinal/latest/invite.accepted.json +++ b/providers/ordinal/latest/invite.accepted.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "invite.accepted", @@ -33,5 +31,10 @@ }, "createdAt": "2025-02-26T14:30:00.000Z" }, - "topic": "invite.accepted" + "topic": "invite.accepted", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/invite-accepted", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/invite.created.json b/providers/ordinal/latest/invite.created.json index a74826d..ce32ed2 100644 --- a/providers/ordinal/latest/invite.created.json +++ b/providers/ordinal/latest/invite.created.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "invite.created", @@ -27,5 +25,10 @@ }, "createdAt": "2025-02-26T11:00:00.000Z" }, - "topic": "invite.created" + "topic": "invite.created", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/invite-created", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.approval.approved.json b/providers/ordinal/latest/post.approval.approved.json index ec8e643..c60d402 100644 --- a/providers/ordinal/latest/post.approval.approved.json +++ b/providers/ordinal/latest/post.approval.approved.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.approval.approved", @@ -47,5 +45,10 @@ }, "createdAt": "2025-02-26T15:30:00.000Z" }, - "topic": "post.approval.approved" + "topic": "post.approval.approved", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-approval-approved", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.approval.requested.json b/providers/ordinal/latest/post.approval.requested.json index 1f6c13e..8f4322c 100644 --- a/providers/ordinal/latest/post.approval.requested.json +++ b/providers/ordinal/latest/post.approval.requested.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.approval.requested", @@ -51,5 +49,10 @@ }, "createdAt": "2025-02-26T14:00:00.000Z" }, - "topic": "post.approval.requested" + "topic": "post.approval.requested", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-approval-requested", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.archived.json b/providers/ordinal/latest/post.archived.json index ae8beb6..2aef884 100644 --- a/providers/ordinal/latest/post.archived.json +++ b/providers/ordinal/latest/post.archived.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.archived", @@ -85,5 +83,10 @@ }, "createdAt": "2025-02-26T15:00:00.000Z" }, - "topic": "post.archived" + "topic": "post.archived", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-archived", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.comment.created.json b/providers/ordinal/latest/post.comment.created.json index 67964c7..4ddca41 100644 --- a/providers/ordinal/latest/post.comment.created.json +++ b/providers/ordinal/latest/post.comment.created.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.comment.created", @@ -31,5 +29,10 @@ }, "createdAt": "2025-02-26T16:00:00.000Z" }, - "topic": "post.comment.created" + "topic": "post.comment.created", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-comment-created", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.content.edited.json b/providers/ordinal/latest/post.content.edited.json index 67b009f..9e4a11f 100644 --- a/providers/ordinal/latest/post.content.edited.json +++ b/providers/ordinal/latest/post.content.edited.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.content.edited", @@ -78,5 +76,10 @@ }, "createdAt": "2025-02-26T15:30:00.000Z" }, - "topic": "post.content.edited" + "topic": "post.content.edited", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-content-edited", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.created.json b/providers/ordinal/latest/post.created.json index cc2ea40..f8b9c88 100644 --- a/providers/ordinal/latest/post.created.json +++ b/providers/ordinal/latest/post.created.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.created", @@ -61,5 +59,10 @@ }, "createdAt": "2025-02-26T12:00:00.000Z" }, - "topic": "post.created" + "topic": "post.created", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-created", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.inline_comment.created.json b/providers/ordinal/latest/post.inline_comment.created.json index cfec506..06a7163 100644 --- a/providers/ordinal/latest/post.inline_comment.created.json +++ b/providers/ordinal/latest/post.inline_comment.created.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.inline_comment.created", @@ -39,5 +37,10 @@ }, "createdAt": "2025-02-26T16:05:00.000Z" }, - "topic": "post.inline_comment.created" + "topic": "post.inline_comment.created", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-inline-comment-created", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.permanently_deleted.json b/providers/ordinal/latest/post.permanently_deleted.json index cd88210..cc9d0d3 100644 --- a/providers/ordinal/latest/post.permanently_deleted.json +++ b/providers/ordinal/latest/post.permanently_deleted.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.permanently_deleted", @@ -61,5 +59,10 @@ }, "createdAt": "2025-03-15T10:00:00.000Z" }, - "topic": "post.permanently_deleted" + "topic": "post.permanently_deleted", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-permanently-deleted", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.publish_failed.json b/providers/ordinal/latest/post.publish_failed.json index c4a69f6..63fd42c 100644 --- a/providers/ordinal/latest/post.publish_failed.json +++ b/providers/ordinal/latest/post.publish_failed.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.publish_failed", @@ -36,5 +34,10 @@ }, "createdAt": "2025-03-01T14:00:05.000Z" }, - "topic": "post.publish_failed" + "topic": "post.publish_failed", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-publish-failed", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.published.json b/providers/ordinal/latest/post.published.json index c44e75a..3d773d7 100644 --- a/providers/ordinal/latest/post.published.json +++ b/providers/ordinal/latest/post.published.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.published", @@ -40,5 +38,10 @@ }, "createdAt": "2025-02-26T14:30:00.000Z" }, - "topic": "post.published" + "topic": "post.published", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-published", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.rescheduled.json b/providers/ordinal/latest/post.rescheduled.json index 1ce41bf..2e7e23c 100644 --- a/providers/ordinal/latest/post.rescheduled.json +++ b/providers/ordinal/latest/post.rescheduled.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.rescheduled", @@ -62,5 +60,10 @@ }, "createdAt": "2025-02-26T13:00:00.000Z" }, - "topic": "post.rescheduled" + "topic": "post.rescheduled", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-rescheduled", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.scheduled.json b/providers/ordinal/latest/post.scheduled.json index 0c1ce42..f9059b8 100644 --- a/providers/ordinal/latest/post.scheduled.json +++ b/providers/ordinal/latest/post.scheduled.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.scheduled", @@ -91,5 +89,10 @@ }, "createdAt": "2025-02-26T12:00:00.000Z" }, - "topic": "post.scheduled" + "topic": "post.scheduled", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-scheduled", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/post.unscheduled.json b/providers/ordinal/latest/post.unscheduled.json index 443fb95..e2ac4f1 100644 --- a/providers/ordinal/latest/post.unscheduled.json +++ b/providers/ordinal/latest/post.unscheduled.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "post.unscheduled", @@ -60,5 +58,10 @@ }, "createdAt": "2025-02-26T14:00:00.000Z" }, - "topic": "post.unscheduled" + "topic": "post.unscheduled", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/post-unscheduled", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/social_profile.connected.json b/providers/ordinal/latest/social_profile.connected.json index df91e3b..a4d0231 100644 --- a/providers/ordinal/latest/social_profile.connected.json +++ b/providers/ordinal/latest/social_profile.connected.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "social_profile.connected", @@ -29,5 +27,10 @@ }, "createdAt": "2025-02-26T10:00:00.000Z" }, - "topic": "social_profile.connected" + "topic": "social_profile.connected", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/social-profile-connected", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/social_profile.disconnected.json b/providers/ordinal/latest/social_profile.disconnected.json index 358b627..f06d6b7 100644 --- a/providers/ordinal/latest/social_profile.disconnected.json +++ b/providers/ordinal/latest/social_profile.disconnected.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "social_profile.disconnected", @@ -36,5 +34,10 @@ }, "createdAt": "2025-02-26T11:00:00.000Z" }, - "topic": "social_profile.disconnected" + "topic": "social_profile.disconnected", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/social-profile-disconnected", + "retrieved": "2026-09-28" + } } diff --git a/providers/ordinal/latest/social_profile.reconnect_needed.json b/providers/ordinal/latest/social_profile.reconnect_needed.json index 82bc1f0..a3b83a8 100644 --- a/providers/ordinal/latest/social_profile.reconnect_needed.json +++ b/providers/ordinal/latest/social_profile.reconnect_needed.json @@ -1,8 +1,6 @@ { "headers": { - "accept": "*/*", - "content-type": "application/json", - "user-agent": "Ordinal-Webhooks" + "content-type": "application/json" }, "body": { "type": "social_profile.reconnect_needed", @@ -23,5 +21,10 @@ }, "createdAt": "2025-02-26T09:00:00.000Z" }, - "topic": "social_profile.reconnect_needed" + "topic": "social_profile.reconnect_needed", + "source": { + "type": "vendor-documentation", + "url": "https://docs.tryordinal.com/integrations/webhooks/social-profile-reconnect-needed", + "retrieved": "2026-09-28" + } } diff --git a/scripts/ordinal/AGENTS.md b/scripts/ordinal/AGENTS.md index 0dac0d4..83a21dc 100644 --- a/scripts/ordinal/AGENTS.md +++ b/scripts/ordinal/AGENTS.md @@ -15,7 +15,8 @@ rather than live-capture, we scrape the docs: every event page ships one canonical example payload in a ```` ```json ```` fence. `docs.ts` discovers the pages from the docs index (`llms.txt`), extracts each example, derives the topic from the payload's own `type`, and writes -`.json` in the repo's `{ headers, body, topic }` shape. +`.json` in the repo's `{ headers, body, topic, source }` shape, and +records `provenance.latest` in `providers/ordinal/index.json`. ## Conventions to preserve @@ -31,9 +32,15 @@ example, derives the topic from the payload's own `type`, and writes generator can warn when the docs add/drop an event. The actual set written is whatever the docs publish. Keep `EXPECTED_TOPICS` in sync when the docs change. -- **Representative headers.** The docs don't enumerate delivery headers, so - `REPRESENTATIVE_HEADERS` is a small honest stand-in. Don't fabricate - signature/HMAC headers — Ordinal documents no signing scheme. +- **Mark every file as doc-sourced.** Each file carries a `source` block + (`type: "vendor-documentation"`, the page URL, the retrieval date). That + key is how this repo, and anything reading it, tells a documented example + from a capture — don't drop it. +- **Only documented headers.** `HEADERS` is `content-type` alone, because + the docs state the body is JSON and document no other delivery header. + Don't add a `user-agent`, `accept` or signature header: Ordinal documents + none, and transport headers on a doc example are exactly what makes it + read as a capture. - **No runtime deps.** Node built-ins + global `fetch`, run by the repo's root `ts-node`. The repo's `ts-node` target rejects iterating/spreading a `Set` — use `Array.from(...)` rather than `[...set]`. diff --git a/scripts/ordinal/README.md b/scripts/ordinal/README.md index 9175928..d17c4f7 100644 --- a/scripts/ordinal/README.md +++ b/scripts/ordinal/README.md @@ -23,12 +23,17 @@ every event in its docs, so we generate the samples from there instead. 1. Fetches the docs index, `https://docs.tryordinal.com/llms.txt`. 2. Discovers every `integrations/webhooks/` page (excluding `introduction` and `event-types`). -3. Fetches each page and extracts the single ```` ```json ```` example - payload it publishes. +3. Fetches each page's markdown (`.md`) and extracts the single + ```` ```json ```` example payload it publishes. 4. Derives the topic from the payload's own `type` and writes `providers/ordinal/latest/.json` in this repo's - `{ headers, body, topic }` shape. -5. Reconciles the discovered events against `EXPECTED_TOPICS` (a + `{ headers, body, topic, source }` shape, where `source` is + `{ type: "vendor-documentation", url, retrieved }` — the doc-sourced + marking described in the repo README. +5. Records `provenance.latest` (`sourced_via: "docs"`, `sourced_on` = the + oldest `retrieved` date in the directory) in + `providers/ordinal/index.json`. +6. Reconciles the discovered events against `EXPECTED_TOPICS` (a human-maintained safety net) and warns if the docs added or dropped an event. @@ -36,8 +41,9 @@ Deterministic and re-runnable, with no credentials. It fails loudly (non-zero exit) rather than writing a wrong payload if a page has no parseable example or no `type`. -`headers` on the generated files are representative, not recorded — the -docs don't enumerate delivery headers. Payloads use Ordinal's own +`headers` on the generated files is `content-type: application/json` +only — the docs state the body is JSON and document no other delivery +header, so none is invented. Payloads use Ordinal's own placeholder data (`Acme Inc`, `550e8400-…` UUIDs), so they're clearly examples, not real captures. See [`../../providers/ordinal/README.md`](../../providers/ordinal/README.md) diff --git a/scripts/ordinal/docs.ts b/scripts/ordinal/docs.ts index 220bc16..8efd4ff 100644 --- a/scripts/ordinal/docs.ts +++ b/scripts/ordinal/docs.ts @@ -10,8 +10,11 @@ // extract each example, and write it in this repo's // { headers, body, topic } shape. // -// Re-runnable and deterministic. The result is docs-sourced (clearly -// labelled in providers/ordinal/README.md). +// Re-runnable and deterministic. The result is docs-sourced, and every file +// says so itself: it carries a `source` block naming the page its body was +// read from and the date it was read, the marking this repo's README gives +// doc-sourced samples. index.json records the same fact one level up, as +// `provenance.latest.sourced_via: "docs"`. import * as fs from "fs"; import * as path from "path"; @@ -19,14 +22,22 @@ import * as path from "path"; const REPO_ROOT = path.resolve(__dirname, "..", ".."); const DOCS_BASE = "https://docs.tryordinal.com"; const LLMS_INDEX = `${DOCS_BASE}/llms.txt`; -const OUTPUT_DIR = path.join(REPO_ROOT, "providers", "ordinal", "latest"); - -// The docs do not enumerate delivery headers, so these are a -// representative (not recorded) set. -const REPRESENTATIVE_HEADERS = { - accept: "*/*", +const PROVIDER_DIR = path.join(REPO_ROOT, "providers", "ordinal"); +const VERSION = "latest"; +const OUTPUT_DIR = path.join(PROVIDER_DIR, VERSION); + +// The date this run read the docs, stamped into each file's `source`. +const RETRIEVED = new Date().toISOString().slice(0, 10); + +// A doc page gives a body, never a delivery, so the only header written is +// the one the docs state: "The request body is JSON" +// (https://docs.tryordinal.com/integrations/webhooks/introduction). +// Ordinal documents no vendor-set delivery header at all (no User-Agent, no +// event-type header, no delivery id, no signature), so none is written. An +// invented `user-agent` / `accept` pair is worse than none: it is what makes +// a hand-written fixture look like a capture. +const HEADERS = { "content-type": "application/json", - "user-agent": "Ordinal-Webhooks", }; // Pages under integrations/webhooks/ that are not event schemas. @@ -65,18 +76,28 @@ async function fetchText(url: string): Promise { return res.text(); } -/** Discover every webhook event-schema page URL from the docs index. */ +/** + * Discover every webhook event-schema page URL from the docs index. + * + * Returns the human page URL (no `.md`). The index has listed pages both + * with and without the `.md` suffix; the markdown is fetched from + * `.md` either way (see markdownUrl), and the page URL is what the + * sample's `source.url` cites. + */ function discoverEventPages(llms: string): string[] { const urls = new Set(); - const re = /https?:\/\/[^\s)]+\/integrations\/webhooks\/([a-z0-9-]+)\.md/gi; + const re = + /(https?:\/\/[^\s)]+\/integrations\/webhooks\/([a-z0-9-]+))(?:\.md)?(?=[\s)]|$)/gim; let m: RegExpExecArray | null; while ((m = re.exec(llms)) !== null) { - if (NON_EVENT_PAGES.has(m[1])) continue; - urls.add(m[0]); + if (NON_EVENT_PAGES.has(m[2])) continue; + urls.add(m[1]); } return Array.from(urls).sort(); } +const markdownUrl = (pageUrl: string): string => `${pageUrl}.md`; + /** Extract the single ```json example payload from a docs page. */ function extractExamplePayload(markdown: string, sourceUrl: string): any { const match = markdown.match(/```json[^\n]*\n([\s\S]*?)```/); @@ -92,12 +113,47 @@ function extractExamplePayload(markdown: string, sourceUrl: string): any { } } -function writeSample(topic: string, body: unknown): void { +function writeSample(topic: string, body: unknown, pageUrl: string): void { const file = path.join(OUTPUT_DIR, `${topic}.json`); - const out = { headers: REPRESENTATIVE_HEADERS, body, topic }; + const out = { + headers: HEADERS, + body, + topic, + source: { + type: "vendor-documentation", + url: pageUrl, + retrieved: RETRIEVED, + }, + }; fs.writeFileSync(file, JSON.stringify(out, null, 2) + "\n"); } +/** + * Record `provenance.latest` in index.json. `sourced_on` is the OLDEST + * `retrieved` date across the version's files, so a partial re-run (a page + * that failed keeps its earlier file) can never overstate freshness. + */ +function writeProvenance(): void { + const indexFile = path.join(PROVIDER_DIR, "index.json"); + const index = JSON.parse(fs.readFileSync(indexFile, "utf8")); + const dates = fs + .readdirSync(OUTPUT_DIR) + .filter((f) => f.endsWith(".json")) + .map((f) => { + const d = JSON.parse(fs.readFileSync(path.join(OUTPUT_DIR, f), "utf8")); + if (!d.source?.retrieved) { + throw new Error(`${f} has no source.retrieved — regenerate it`); + } + return d.source.retrieved as string; + }) + .sort(); + index.provenance = { + ...(index.provenance || {}), + [VERSION]: { sourced_via: "docs", sourced_on: dates[0] }, + }; + fs.writeFileSync(indexFile, JSON.stringify(index, null, 2) + "\n"); +} + async function main() { console.log("Ordinal docs → samples\n"); @@ -115,13 +171,13 @@ async function main() { const failures: string[] = []; for (const url of pages) { try { - const md = await fetchText(url); + const md = await fetchText(markdownUrl(url)); const body = extractExamplePayload(md, url); const topic = body?.type; if (typeof topic !== "string" || !topic) { throw new Error(`example payload has no string "type" (${url})`); } - writeSample(topic, body); + writeSample(topic, body, url); captured.push(topic); console.log(` ${topic}.json`); } catch (e) { @@ -130,6 +186,8 @@ async function main() { } } + writeProvenance(); + console.log(`3/3 Reconciling against the expected taxonomy...`); const expected = new Set(EXPECTED_TOPICS); const capturedSet = new Set(captured);