From de2182ea5efed55e883cffad5c12416c5c51eaf7 Mon Sep 17 00:00:00 2001 From: Devops Bot Date: Fri, 9 Oct 2026 16:08:24 +0000 Subject: [PATCH 1/4] docs: Add bounded distribution Excel example --- README.md | 5 + examples/distribution_report/README.md | 201 +++++ .../distribution_report.py | 778 ++++++++++++++++++ examples/distribution_report/requirements.txt | 3 + .../test_distribution_report.py | 496 +++++++++++ 5 files changed, 1483 insertions(+) create mode 100644 examples/distribution_report/README.md create mode 100644 examples/distribution_report/distribution_report.py create mode 100644 examples/distribution_report/requirements.txt create mode 100644 examples/distribution_report/test_distribution_report.py diff --git a/README.md b/README.md index 72ba28d..c3d7203 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,11 @@ The API doesn't stop there, explore a wide range of endpoints: Find full details at the [AskNews API documentation](https://docs.asknews.app). +## Examples + +- [Exact-domain distribution Excel report](examples/distribution_report/README.md): + bounded read-only export using your existing authorized credentials, with offline tests. + ## Support Join our [Discord](https://discord.gg/2Yw66XXEhY) to see what other people are building, and to get support with your projects. diff --git a/examples/distribution_report/README.md b/examples/distribution_report/README.md new file mode 100644 index 0000000..93d0d4a --- /dev/null +++ b/examples/distribution_report/README.md @@ -0,0 +1,201 @@ +# Exact-domain distribution Excel report + +Standalone, read-only Python **3.10+** example. Defaults: `apnews.com`, `2026-09`. +Run locally with **your existing authorized credentials**. No credential or +permission changes, customer extraction, or live authorization tests were made +to develop this example. Tests use synthetic responses only. + +## Install and run + +Keep `distribution_report.py` and `requirements.txt` together (or enter this +example directory in a checkout): + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements.txt +python distribution_report.py --domain apnews.com --month 2026-09 --output apnews-2026-09.xlsx --include-internal --allow-partial +``` + +On Windows use `.venv\Scripts\activate` instead. Output must be a new `.xlsx` +path; existing files are never overwritten. Save in a private local directory. +The temporary/output workbook uses restrictive permissions where the OS supports +POSIX modes. Filesystems must support hard links for atomic no-overwrite output. + +**Credentials:** provision the environment privately using your existing local +secret manager/session; do not put secrets in command arguments, source files, +chat, or shell history. This script reads one of: + +- `ASKNEWS_API_KEY`, passed to the SDK's `api_key` parameter; **or** +- `ASKNEWS_CLIENT_ID` + `ASKNEWS_CLIENT_SECRET`, passed to the SDK's OAuth client + credentials parameters. Optional `ASKNEWS_SCOPES` is a space-separated subset + of `distribution news internal`; default `distribution news`. Request only + scopes the existing client already has. The script does not grant scopes. + +These environment names are this example's adapter, not automatic SDK environment +loading. Do not set both authentication methods. No custom host, token URL, +organization impersonation headers, or credentials-on-CLI options are provided. +TLS verification stays enabled, redirects are disabled, and the SDK uses its +normal production API/auth hosts. All analytics requests are GET; SDK OAuth may +POST to its token endpoint. Request/response bodies, identities and credentials +are never logged. Do not enable HTTP debugging. + +`--include-internal` tries dashboard metrics with the same existing credentials. +**Admin status does not imply `internal` scope.** Omit the flag for the public +ranking/share plus news index report. A 403 stops the internal group immediately; +there is no loop probing its remaining endpoints. Public ranking 401/403 stops +the entire run without a workbook. Any 401 (including OAuth) is fatal without a +retry or credential-refresh loop. Optional-section 403s mark that section +unavailable; they are never retried or represented as zero. Use an already +suitably authorized credential or accept the documented gap—not an access change. + +### Exit codes and bounded requests + +- **0**: selected sections collected, **with calendar/snapshot coverage caveats**. + Does not mean an exact complete month or that internal sections were requested. +- **2**: a flagged partial workbook was saved with `--allow-partial`. +- **1**: fatal error or incomplete sections without `--allow-partial`; no new + workbook. Errors are deliberately sanitized. Inspect a flagged workbook's + Coverage sheet for section-specific failures, then address the identified + cause before rerunning. + +Defaults: page size **100**, at most **100 pages per metric**, **400 HTTP sends** +(including OAuth/retries), **900 seconds** of request-start/retry budget, +30-second HTTP operation timeout, serial requests with at least 1 second before +each send. A request already in flight may finish after the overall deadline; +httpx timeouts bound individual network operations, not a strict wall-clock +process deadline. Set an external process timeout if required. + +Optional bounded flags: `--page-size 1..100`, `--max-pages 1..1000`, +`--max-requests 1..4000`, `--max-seconds 1..3600`. API/transport transient failures +retry at most twice (three attempts total); retryable statuses are +408/429/500/502/503/504. `Retry-After` seconds and HTTP dates are honored. A wait +over 60 seconds, invalid Retry-After or exhausted budget stops that section +rather than retrying early. No concurrency or article-by-article fanout. Index +counts cost one additional bounded request per day (28–31). Request limits +include unsuccessful sends; no unrestricted all-history queries. + +## Workbook and interpretation + +| Sheet | Contents | +| --- | --- | +| Summary | Raw event aggregates when authorized, exported article sums, distinct retrieval articles when authorized, exported indexed-day count sum, explicit coverage status | +| Daily metrics | Server-grouped retrieval/citation/grounding/full-text events when authorized; daily unique retrieval articles; separate crawl-date index counts | +| Articles | Stable article UUID, literal URL, joined retrieval/citation/grounding counts | +| Weighted share | Exact-domain server weighted fraction formatted as percent; **not payout** | +| Coverage | Availability, pagination completion/caps, failure and reconciliation notes | +| Methodology | Requested and actual bounds, UTC acquisition timestamps, SDK/API behavior and limitations | + +Text cells are explicitly literal, including strings beginning with `=`, `+`, +`-`, `@`, and URLs; no formulas, hyperlinks or external workbook links are +created. Sheets have frozen headers, filters, widths and readable count/percent +formats. Missing fields/days are **blank/unavailable**, not zero. Zero is retained +when the server returns it; an absent article metric is zero only when that +positive-event ranking has exhausted pagination. A page's `total_count` is its +**event sum**, not a domain total or article count. Rankings are independently +paginated and joined by UUID, never by title or URL; conflicting UUID URLs fail +the export. Duplicate IDs/repeated pages are not summed. An exact page boundary +may require a final empty page. Old servers without explicit `page`/`next_page` +metadata are marked partial/unavailable rather than falsely reporting their +potentially top-N-capped cohort as complete. + +Monthly distinct article counts must **not** be computed by summing daily +uniques. Index counts describe current retained indexed articles filtered by +**crawl date**; they are not distribution events, publication-date activity, +or a historical inventory snapshot. Ranking endpoints do not supply titles, +publication dates, bias/sentiment or other enrichments: those remain unavailable; +this example does not add costly article fanout. Consumer identities, prompts, +queries, raw hits and billing/PII endpoints are never requested or exported. + +Only the requested exact domain is sent in filters. URLs with a different host +(including `www.` or other subdomains), embedded credentials, or nonstandard +ports cause the page to be discarded and flagged; returned share rows must match +exactly. Aggregate endpoints have no domain label in their response: isolation +there relies on their verified exact-domain filter contract. No parent-domain, +fuzzy or wildcard matching and no other publisher rows are exported. + +### Dates: important September limitation + +September 2026 is requested as `[2026-09-01T00:00:00Z, 2026-10-01T00:00:00Z)`. +Distribution queries accept integer seconds and compare both endpoints +inclusively. The script sends **1788220800 through 1790812799**, i.e. +`2026-09-30T23:59:59Z` inclusive. It **cannot include fractional-second events +after that instant**. Sending next-month midnight would both include an unwanted +instant and conflict with the public same-month restriction. Admin exemptions +are not used to pretend this is an exact half-open month. The warning is present +in every workbook and successful console output. + +Distribution dates are **event timestamps**, not article publication dates. +Daily distribution metrics use one monthly request grouped on the server, not +adjacent inclusive daily queries. Server bucket labels are treated as UTC; the +deployed database's timezone cannot be verified by this client. + +The index service's daily split can use adjacent inclusive endpoints. To avoid +double counting boundaries, this example requests each day separately from UTC +00:00:00 to 23:59:59 and validates the returned aware timestamps. Consequently +**index counts exclude the fractional remainder of EVERY day's final second**. +The retained-index backend can also combine recent hot/archive data or conceal +partial backend failures; client-side exact completeness is not established. +Missing/shifted/naive buckets are unavailable, never guessed or silently shifted. + +Requests have no shared database snapshot. Late events, cache TTLs, corrections +or backfills can move ranking offsets and change totals during acquisition. +Duplicate/page guards and ranking-vs-aggregate/daily reconciliation catch some, +not all, drift. Pagination exhaustion does not prove immutable completeness. +Weighted share reflects the server's current metric weights and publisher +multipliers for the selected period; weights/pool values are not exposed by this +response. **No revenue, payout, currency or dollar amount is inferred.** + +## Verified SDK/API compatibility + +Verified SDK source: `asknews==0.14.6`, commit +`77e434f3e08aa9372aed8fb4b72c3044a677b6a6`. + +- `AskNewsSDK(api_key=...)` or `(client_id=..., client_secret=..., scopes=...)` + and its supported httpx `event_hooks` constructor kwargs provide auth, + safe errors, throttling and request budgets. +- `sdk.distribution.get_domain_metrics(domain_names, start_date, end_date)` and + `get_domain_metrics_timeseries(...)` exist. Their current Pydantic models + omit `grounded`/`total_grounded` and would silently discard these fields. +- Ranking/share/unique aggregates have no named distribution wrappers. +- `sdk.news.get_index_counts(start_datetime, end_datetime, ..., domains=...)` + exists, returning `IndexCountsResponse.root` with `start/end/count` rows, + but its formatter drops the datetime timezone offset. + +For those reasons the example consistently uses the SDK's supported +**`sdk.client.request(method="GET", endpoint=..., query=...).content`** transport +with a fixed endpoint allowlist. It retains SDK authentication and serialization; +there is no direct `requests`, `httpx` client, curl fallback, SDK/API extension, +or private client member access. `httpx` types are used only for the SDK's +supported hooks and exceptions. The example never requests an alternate server. + +| SDK transport endpoint | Scope / contract | +| --- | --- | +| `/v1/distribution/articles/top_n_for_domains` | `distribution`; exact `domain_names`, integer dates, `metric=surface/citation/grounded`, `limit<=100`, 1-based `page`; `data` article IDs/URLs/raw counts, page sum, next page | +| `/v1/distribution/stats/hit_share` | `distribution`; exact domain, same dates, `is_publisher=true`; domain weighted fractions | +| `/v1/distribution/stats/metrics` | `distribution` + `internal`; raw surfaces/citations/grounded/full_text | +| `/v1/distribution/stats/metrics_timeseries` | same internal scope; day rows and metric totals | +| `/v1/distribution/articles/domain_hits_surface` | same internal scope; year/month; raw hits and monthly distinct surfaced articles | +| `/v1/distribution/articles/domain_hits_surface_timewindow` | same internal scope; daily hits/distinct; daily distinct sum is not monthly distinct | +| `/v1/index_counts` | `news`; exact domains, aware ISO start/end, `sampling=1d`, `time_filter=crawl_date`; index bucket counts | + +These are source-verified contracts, not evidence that any particular credential +or deployed version can use them. Endpoint/schema mismatch, missing additive +fields, pagination guards and API caps are explicit partial/unavailable outcomes. + +## Offline tests + +From this directory, after installing requirements: + +```bash +python -m pip install 'pytest>=8,<10' +python -m pytest test_distribution_report.py -q +``` + +Fixtures and httpx MockTransport exercise the **real SDK** without network or +real credentials. Tests cover pagination, repeated/capped pages, metric joining, +leap/year/UTC boundaries, exact-domain rejection, 401/403, retries/budgets, +missing fields, daily reconciliation, formula injection, workbook reopening, +totals, output collision and partial exit behavior. No real AP workbook or data +is included. Tests live with this optional example to avoid adding Excel +runtime/dev dependencies to the SDK package or changing its release workflows. diff --git a/examples/distribution_report/distribution_report.py b/examples/distribution_report/distribution_report.py new file mode 100644 index 0000000..fab78d1 --- /dev/null +++ b/examples/distribution_report/distribution_report.py @@ -0,0 +1,778 @@ +#!/usr/bin/env python3 +"""Bounded, read-only publisher Excel export. See README.md; Python 3.10+.""" + +from __future__ import annotations + +import argparse +import calendar +import logging +import math +import os +import re +import sys +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from urllib.parse import urlsplit +from uuid import UUID + +import httpx +from openpyxl import Workbook +from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter + +from asknews_sdk import AskNewsSDK +from asknews_sdk.errors import APIError + + +UTC = timezone.utc +METRICS = ("surface", "citation", "grounded") +FIELDS = ("surfaces", "citations", "grounded", "full_text") +BASE = "/v1/distribution/" +RANK = BASE + "articles/top_n_for_domains" +SHARE = BASE + "stats/hit_share" +TOTAL = BASE + "stats/metrics" +DAILY = BASE + "stats/metrics_timeseries" +UNIQUE = BASE + "articles/domain_hits_surface" +UNIQUE_DAILY = BASE + "articles/domain_hits_surface_timewindow" +INDEX = "/v1/index_counts" +ALLOWED = {RANK, SHARE, TOTAL, DAILY, UNIQUE, UNIQUE_DAILY, INDEX} +BOUNDARY = ( + "PARTIAL CALENDAR COVERAGE: inclusive integer bounds end at 23:59:59 UTC; " + "fractional events after that instant are not covered. Never an exact full-month claim." +) + + +class ReportError(Exception): + """Sanitized, operator-actionable message only; never include response bodies.""" + + +class AuthError(ReportError): + pass + + +class Denied(ReportError): + pass + + +def month_bounds(month): + if not re.fullmatch(r"20\d{2}-\d{2}|2100-\d{2}", month): + raise ReportError("Use YYYY-MM within 2000–2100.") + year, number = map(int, month.split("-")) + try: + start = datetime(year, number, 1, tzinfo=UTC) + following = start + timedelta(days=calendar.monthrange(year, number)[1]) + except ValueError: + raise ReportError("Invalid calendar month.") from None + return start, following - timedelta(seconds=1), following + + +def exact_domain(value): + value = value.lower() + if len(value) > 253 or not re.fullmatch( + r"(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+" + r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?", + value, + ): + raise ReportError("Use one bare exact DNS domain, without URL, wildcard or trailing dot.") + return value + + +def count(value): + if type(value) is not int or not 0 <= value <= 2**53 - 1: + raise ReportError("Missing/invalid count or count exceeds exact Excel numeric precision.") + return value + + +def utc_string(value): + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def retry_delay(header, attempt, now=None): + delay = 2**attempt + if header is not None: + try: + delay = float(header) + except (TypeError, ValueError): + try: + date = parsedate_to_datetime(header) + delay = (date - (now or datetime.now(UTC))).total_seconds() + except (TypeError, ValueError, OverflowError): + raise ReportError("Invalid Retry-After; stop and retry later.") from None + if not math.isfinite(delay) or delay > 60: + raise ReportError("Retry-After exceeds 60-second wait budget; retry later.") + return max(0, delay) + + +class Reader: + """All data reads use SDK transport; budgets count each HTTP send (including OAuth).""" + + def __init__(self, sdk, max_requests=400, max_seconds=900, sleep=time.sleep): + self.sdk = sdk + self.max_requests = max_requests + self.deadline = time.monotonic() + max_seconds + self.requests = 0 + self.sleep = sleep + + def before_request(self, request): + if self.requests >= self.max_requests or time.monotonic() + 1 >= self.deadline: + raise ReportError("Request/time budget reached; increase bounded limits explicitly.") + # Called by httpx for API and SDK-managed OAuth requests, not logged. + self.sleep(1.0) + self.requests += 1 + + def after_response(self, response): + # Stop before the SDK's built-in OAuth 401 refresh, including non-JSON errors. + if response.status_code == 401: + raise AuthError("401: verify existing authorized credentials; no retry.") + if response.status_code == 403: + raise Denied( + "403: required scope/domain authorization missing; admin is not internal scope. " + "Use existing authorized credentials; no access changed and no retry." + ) + # Surface even HTML/proxy errors before SDK JSON error parsing. + response.raise_for_status() + + def get(self, endpoint, query): + if endpoint not in ALLOWED: + raise ReportError("Endpoint not allowlisted.") + for attempt in range(3): + try: + return self.sdk.client.request(method="GET", endpoint=endpoint, query=query).content + except (APIError, httpx.HTTPStatusError) as exc: + status = exc.response.status_code + if status == 401: + raise AuthError( + "401: verify existing authorized credentials; no retry." + ) from None + if status == 403: + raise Denied( + "403: credential lacks required scope/domain authorization; admin is not " + "internal scope. Use an already-authorized credential; no access changed." + ) from None + if status not in (408, 429, 500, 502, 503, 504): + raise ReportError( + f"HTTP {status}: section unavailable; check API compatibility." + ) from None + delay = retry_delay(exc.response.headers.get("retry-after"), attempt) + except httpx.TransportError: + delay = 2**attempt + except (ValueError, TypeError, AttributeError): + raise ReportError("Unrecognized SDK/API response; verify compatibility.") from None + if attempt == 2: + raise ReportError("Transient request failed after three bounded attempts.") + if time.monotonic() + delay >= self.deadline: + raise ReportError("Retry would exceed time budget.") + self.sleep(delay) + raise ReportError("Request did not complete.") + + +@dataclass +class Ranking: + rows: dict = field(default_factory=dict) + complete: bool = False + note: str = "not requested" + pages: int = 0 + + +def parse_article(row, domain): + url = row.get("article_url") + if not isinstance(url, str) or len(url) > 8192: + raise ReportError("Invalid article URL.") + parts = urlsplit(url) + if ( + parts.scheme not in ("http", "https") + or parts.hostname != domain + or parts.username + or parts.password + or parts.port not in (None, 80, 443) + ): + raise ReportError("Exact-domain isolation failed; page discarded (no foreign data saved).") + # IDs are stable join keys; refuse URL-only joins because different IDs can share a URL. + try: + identity = str(UUID(row["article_id"])) + except (KeyError, ValueError, TypeError, AttributeError): + raise ReportError( + "Stable article_id missing/invalid; unsafe metric join refused." + ) from None + return identity, (url, count(row.get("hit_count"))) + + +def ranking(reader, query, domain, metric, page_size, max_pages): + result = Ranking() + try: + for page in range(1, max_pages + 1): + data = reader.get(RANK, dict(query, metric=metric, limit=page_size, page=page)) + rows = data["data"] + if not isinstance(rows, list) or len(rows) > page_size: + raise ReportError("Invalid ranking page size.") + if type(data.get("page")) is not int or data["page"] != page or "next_page" not in data: + raise ReportError( + "Pagination metadata missing/mismatched; legacy top-N may be capped." + ) + parsed = {} + for row in rows: + key, item = parse_article(row, domain) + if key in parsed or key in result.rows: + raise ReportError( + "Repeated article across pages; snapshot drift/page loop detected." + ) + parsed[key] = item + if count(data.get("total_count")) != sum(item[1] for item in parsed.values()): + raise ReportError("Page-only total_count does not reconcile.") + next_page = data["next_page"] + if next_page is not None and ( + type(next_page) is not int or next_page != page + 1 or not rows + ): + raise ReportError("Non-forward/empty pagination loop refused.") + result.rows.update(parsed) + result.pages += 1 + if next_page is None: + result.complete = True + result.note = "pagination exhausted; snapshot and month-end caveats still apply" + return result + raise ReportError("Page cap reached; exported counts are partial, not domain totals.") + except (AuthError, Denied): + raise + except ReportError as exc: + result.note = str(exc) + except (KeyError, TypeError, ValueError, AttributeError): + result.note = "Unexpected ranking schema; section partial/unavailable." + return result + + +def daily_values(payload, fields, start, following, total_prefix="total_"): + rows = payload["data"] + if not isinstance(rows, list) or len(rows) > 31: + raise ReportError("Invalid daily rows.") + parsed = {} + for row in rows: + day = row["day"] + if not isinstance(day, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", day): + raise ReportError("Daily bucket must be a UTC calendar date.") + date = datetime.strptime(day, "%Y-%m-%d").replace(tzinfo=UTC) + if not start <= date < following or day in parsed: + raise ReportError("Duplicate/out-of-window daily bucket.") + parsed[day] = {name: count(row[name]) if name in row else None for name in fields} + for name in fields: + if any(row[name] is None for row in parsed.values()) or total_prefix + name not in payload: + continue # Missing additive fields stay unavailable, never invented zeros. + if count(payload[total_prefix + name]) != sum(row[name] for row in parsed.values()): + raise ReportError("Daily sum does not match response total.") + return parsed + + +@dataclass +class Report: + domain: str + month: str + started: str = field(default_factory=lambda: utc_string(datetime.now(UTC))) + finished: str = "" + ranks: dict = field(default_factory=dict) + totals: dict = field(default_factory=dict) + daily: dict = field(default_factory=dict) + unique: dict = field(default_factory=dict) + unique_daily: dict = field(default_factory=dict) + index: dict = field(default_factory=dict) + share: float | None = None + coverage: list = field(default_factory=list) + issues: list = field(default_factory=list) + + def section(self, name, operation): + try: + value = operation() + self.coverage.append((name, "available", "See boundary/snapshot methodology")) + return value + except AuthError: + raise + except ReportError as exc: + message = str(exc) + except (KeyError, TypeError, ValueError, AttributeError): + message = "Unexpected schema; section unavailable." + self.coverage.append((name, "unavailable", message)) + self.issues.append(name + ": " + message) + return None + + +def parse_share(payload, domain): + rows = payload["data"] + if not isinstance(rows, list) or len(rows) != 1 or rows[0].get("domain") != domain: + raise ReportError("Missing/non-exact share row; no other publisher data saved.") + value = rows[0]["hit_share"] + if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1: + raise ReportError("Invalid weighted share.") + return value + + +def collect_internal(reader, report, query, start, following): + # One denied internal request stops the internal group; no 403 endpoint-probing loop. + raw = reader.get(TOTAL, query) + report.totals = {name: count(raw[name]) if name in raw else None for name in FIELDS} + if any(value is None for value in report.totals.values()): + report.issues.append("Internal totals: missing fields remain unavailable.") + report.coverage.append(("Internal totals", "available", "Raw event counts, not weighted")) + daily_payload = reader.get(DAILY, query) + report.daily = daily_values(daily_payload, FIELDS, start, following) + if any("total_" + name not in daily_payload for name in FIELDS): + report.issues.append("Daily response totals missing; completeness not established.") + for name in FIELDS: + values = [row[name] for row in report.daily.values()] + if any(value is None for value in values): + report.issues.append("Daily metric field missing: " + name) + elif report.totals[name] is not None and sum(values) != report.totals[name]: + report.issues.append("Daily/aggregate mismatch: " + name) + month_query = {"domain_names": [report.domain], "year": start.year, "month": start.month} + raw = reader.get(UNIQUE, month_query) + report.unique = {name: count(raw[name]) for name in ("hits", "surfaced")} + report.unique_daily = daily_values( + reader.get(UNIQUE_DAILY, month_query), ("hits", "surfaced"), start, following + ) + daily_hits = sum(row["hits"] for row in report.unique_daily.values()) + if daily_hits != report.unique["hits"] or report.unique["hits"] != report.totals["surfaces"]: + report.issues.append("Surface totals/daily/aggregate mismatch (snapshot drift).") + surface = report.ranks["surface"] + if surface.complete and len(surface.rows) != report.unique["surfaced"]: + report.issues.append("Monthly distinct/ranking article count mismatch (snapshot drift).") + for metric, name in zip(METRICS, FIELDS): + ranked = report.ranks[metric] + if ranked.complete and report.totals[name] is not None: + if sum(row[1] for row in ranked.rows.values()) != report.totals[name]: + report.issues.append(f"{metric}: ranking/raw aggregate mismatch (snapshot drift).") + return True + + +def collect_index(reader, report, start, following): + day = start + while day < following: + end = day + timedelta(days=1) - timedelta(seconds=1) + # Per-day disjoint requests avoid the server's adjacent inclusive bucket overlap. + payload = reader.get( + INDEX, + { + "domains": [report.domain], + "start_datetime": utc_string(day), + "end_datetime": utc_string(end), + "sampling": "1d", + "time_filter": "crawl_date", + }, + ) + if not isinstance(payload, list) or len(payload) != 1: + raise ReportError("Expected one index bucket; missing is not zero.") + row = payload[0] + for name, expected in (("start", day), ("end", end)): + date = datetime.fromisoformat(row[name].replace("Z", "+00:00")) + if date.tzinfo is None or date.astimezone(UTC) != expected: + raise ReportError("Index bucket timezone/boundaries differ; bucket discarded.") + report.index[day.date().isoformat()] = count(row["count"]) + day += timedelta(days=1) + return True + + +def collect(reader, domain, month, page_size=100, max_pages=100, include_internal=False): + start, end, following = month_bounds(month) + report = Report(domain, month) + query = { + "domain_names": [domain], + "start_date": int(start.timestamp()), + "end_date": int(end.timestamp()), + } + for metric in METRICS: + result = ranking(reader, query, domain, metric, page_size, max_pages) + report.ranks[metric] = result + report.coverage.append( + (metric + " ranking", "exhausted" if result.complete else "partial", result.note) + ) + if not result.complete: + report.issues.append(metric + ": " + result.note) + report.share = report.section( + "Weighted share", + lambda: parse_share(reader.get(SHARE, dict(query, is_publisher=True)), domain), + ) + if include_internal: + report.section( + "Internal group", lambda: collect_internal(reader, report, query, start, following) + ) + else: + report.coverage.append( + ( + "Internal group", + "not requested", + "Requires existing internal scope; use --include-internal", + ) + ) + report.section("Index counts", lambda: collect_index(reader, report, start, following)) + report.finished = utc_string(datetime.now(UTC)) + return report + + +def joined_articles(report): + joined = {} + for metric, ranked in report.ranks.items(): + for identity, (url, value) in ranked.rows.items(): + row = joined.setdefault(identity, {"url": url}) + if row["url"] != url: + raise ReportError("Article ID has conflicting URLs across metrics; join refused.") + row[metric] = value + for identity, row in sorted(joined.items()): + values = [] + for metric in METRICS: + # Absence means zero only for exhausted positive-event rankings. + values.append(row.get(metric, 0 if report.ranks[metric].complete else None)) + yield (identity, row["url"], *values) + + +def sheet(workbook, title, headers, rows): + ws = workbook.create_sheet(title) + for row in (headers, *rows): + ws.append( + [ + ILLEGAL_CHARACTERS_RE.sub("", value)[:32767] if isinstance(value, str) else value + for value in row + ] + ) + for cell in ws[ws.max_row]: + if isinstance(cell.value, str): + cell.value = ILLEGAL_CHARACTERS_RE.sub("", cell.value)[:32767] + cell.data_type = "s" # Explicit literal even for =,+,-,@ and URLs; no hyperlinks. + elif isinstance(cell.value, int): + cell.number_format = "#,##0" + cell.alignment = Alignment(vertical="top", wrap_text=True) + for cell in ws[1]: + cell.font = Font(color="FFFFFF", bold=True) + cell.fill = PatternFill("solid", fgColor="17365D") + ws.freeze_panes = "A2" + ws.auto_filter.ref = ws.dimensions + for index in range(1, len(headers) + 1): + ws.column_dimensions[get_column_letter(index)].width = min( + 70, + max( + 18, + max(len(str(ws.cell(r, index).value or "")) for r in range(1, ws.max_row + 1)) + 2, + ), + ) + return ws + + +def workbook(report): + start, end, following = month_bounds(report.month) + book = Workbook() + book.remove(book.active) + summary = [ + ("Domain (exact)", report.domain, "No subdomains or other publishers"), + ("Month", report.month, BOUNDARY), + ( + "Run status", + "PARTIAL" if report.issues else "COLLECTED WITH COVERAGE CAVEATS", + "Blank = unavailable/not requested; zero = observed or documented absence", + ), + ( + "Monthly unique retrieved articles", + report.unique.get("surfaced"), + "Internal month distinct; never sum daily uniques", + ), + ( + "Exported surface article IDs", + len(report.ranks["surface"].rows), + "Exported cohort only; not indexed article inventory", + ), + ( + "Indexed crawl-date counts (exported days)", + sum(report.index.values()) if report.index else None, + f"{len(report.index)} daily buckets; fractional-second gaps each day; " + "not publication/event dates", + ), + ] + for metric, name in zip(METRICS, FIELDS): + rank = report.ranks[metric] + summary.extend( + [ + ( + name + " raw aggregate", + report.totals.get(name), + "Internal endpoint, if authorized", + ), + ( + metric + " exported article sum", + sum(row[1] for row in rank.rows.values()) + if rank.complete or rank.rows + else None, + rank.note, + ), + ] + ) + summary.append( + ( + "full_text raw aggregate", + report.totals.get("full_text"), + "Separate metric, not retrievals or groundings", + ) + ) + sheet(book, "Summary", ("Metric", "Value", "Interpretation"), summary) + daily = [] + day = start + while day < following: + key = day.date().isoformat() + counts = report.daily.get(key, {}) + unique = report.unique_daily.get(key, {}) + daily.append( + ( + key, + *(counts.get(name) for name in FIELDS), + unique.get("hits"), + unique.get("surfaced"), + report.index.get(key), + "Missing buckets/fields blank, not zero", + ) + ) + day += timedelta(days=1) + sheet( + book, + "Daily metrics", + ( + "UTC day", + *FIELDS, + "Monthly-series hits", + "Daily distinct retrieved", + "Indexed crawl-date", + "Coverage", + ), + daily, + ) + sheet( + book, + "Articles", + ("Article ID", "URL (literal)", "Retrieval events", "Citation events", "Grounding events"), + list(joined_articles(report)), + ) + ws = sheet( + book, + "Weighted share", + ("Exact domain", "Weighted fraction", "Meaning"), + [ + ( + report.domain, + report.share, + "Publisher-only weighted pool fraction; metric weights/multipliers " + "are server-configured, not returned. NOT payout or dollars.", + ) + ], + ) + ws["B2"].number_format = "0.000000%" + sheet( + book, + "Coverage", + ("Section", "Status", "Details"), + report.coverage + [("Issue", "partial", issue) for issue in report.issues], + ) + sheet( + book, + "Methodology", + ("Item", "Value"), + [ + ("Requested start UTC", utc_string(start)), + ("Requested exclusive end UTC", utc_string(following)), + ("API inclusive start timestamp", int(start.timestamp())), + ("API inclusive end timestamp", int(end.timestamp())), + ("Calendar coverage", BOUNDARY), + ("Snapshot started UTC", report.started), + ("Snapshot finished UTC", report.finished), + ( + "Consistency", + "Sequential requests, no transactional snapshot. Late " + "events/backfills and ranking changes can alter pages; " + "duplicates/aggregate mismatches mark partial. Exhaustion is not " + "proof of immutable completeness.", + ), + ( + "Daily distribution", + "One whole-window server-grouped series, no adjacent query overlap. " + "Server date buckets assumed UTC; deployed database timezone not " + "verified. Missing days remain blank.", + ), + ( + "Index coverage", + "One bounded request per UTC day, inclusive 00:00:00–23:59:59; " + "fractional remainder excluded EVERY day to prevent overlapping " + "buckets. crawl_date, not publication dates or distribution events. " + "Current retained index, not historical inventory.", + ), + ( + "Index limits", + "No per-article fanout. Index service may combine hot/archive stores " + "for recent dates and mask backend failures; exact index completeness" + " cannot be established.", + ), + ( + "Pagination", + "Page size <=100; total_count is PAGE event sum, never full-domain " + "total. Explicit page/next_page required; missing metadata, repeats, " + "caps and failures mark partial.", + ), + ( + "Metric joining", + "Stable article UUID, never sum duplicate rows or join on title. " + "Missing metric is zero only after that ranking exhausts; otherwise " + "blank. Titles/publication dates/enrichments not supplied by ranking " + "endpoint.", + ), + ( + "Isolation", + "One exact domain in every query; reject non-exact article " + "hosts/share rows. No queries, consumer identities, billing data or " + "raw response dumps exported.", + ), + ( + "SDK compatibility", + "Verified against asknews 0.14.6. SDK authenticated client.request " + "GET transport preserves grounded fields omitted by current typed " + "distribution models; ranking/share/unique lack named wrappers. Index" + " transport retains UTC offsets stripped by named method.", + ), + ( + "Permissions", + "distribution for public rankings/share; internal + distribution for " + "dashboard aggregates; news for index. Admin status does not imply " + "internal scope. No scope discovery or access changes.", + ), + ( + "No money", + "Weighted share is not payout. No revenue pool, currency or dollars inferred.", + ), + ], + ) + return book + + +def save_workbook(book, output): + output = Path(output) + if output.exists(): + raise ReportError("Output already exists; choose a new path (no overwrite).") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=output.parent, suffix=".xlsx", delete=False) as handle: + temporary = Path(handle.name) + book.save(temporary) + os.link( + temporary, output + ) # Atomic publish, fails safely if destination appeared meanwhile. + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def credentials(environ): + key = environ.get("ASKNEWS_API_KEY") + client_id = environ.get("ASKNEWS_CLIENT_ID") + secret = environ.get("ASKNEWS_CLIENT_SECRET") + if key and (client_id or secret): + raise ReportError("Choose API key OR OAuth environment credentials, not both.") + if key: + return {"api_key": key} + if client_id and secret: + scopes = set(environ.get("ASKNEWS_SCOPES", "distribution news").split()) + if not scopes or not scopes <= {"distribution", "news", "internal"}: + raise ReportError( + "ASKNEWS_SCOPES must list existing distribution/news/internal scopes." + ) + return {"client_id": client_id, "client_secret": secret, "scopes": scopes} + raise ReportError( + "Set ASKNEWS_API_KEY or ASKNEWS_CLIENT_ID + ASKNEWS_CLIENT_SECRET " + "securely in your local environment." + ) + + +def bounded_int(low, high): + def convert(text): + value = int(text) + if not low <= value <= high: + raise argparse.ArgumentTypeError(f"must be {low}–{high}") + return value + + return convert + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--domain", default="apnews.com") + parser.add_argument("--month", default="2026-09") + parser.add_argument("--output", type=Path) + parser.add_argument( + "--include-internal", + action="store_true", + help="Try dashboard GETs only with existing authorized internal scope", + ) + parser.add_argument( + "--allow-partial", + action="store_true", + help="Save incomplete sections with explicit flags; exit 2", + ) + parser.add_argument("--page-size", type=bounded_int(1, 100), default=100) + parser.add_argument("--max-pages", type=bounded_int(1, 1000), default=100) + parser.add_argument("--max-requests", type=bounded_int(1, 4000), default=400) + parser.add_argument("--max-seconds", type=bounded_int(1, 3600), default=900) + args = parser.parse_args(argv) + # SDK errors/HTTP logging can contain auth requests or response bodies. Never enable debug. + logging.disable(logging.CRITICAL) + try: + domain = exact_domain(args.domain) + month_bounds(args.month) + output = args.output or Path(f"{domain}-{args.month}-distribution.xlsx") + if output.suffix.lower() != ".xlsx" or output.exists(): + raise ReportError("Choose a new .xlsx output path; existing files are not overwritten.") + auth = credentials(os.environ) + reader = Reader(None, args.max_requests, args.max_seconds) + with AskNewsSDK( + **auth, + retries=0, + timeout=30, + follow_redirects=False, + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) as sdk: + reader.sdk = sdk + report = collect( + reader, domain, args.month, args.page_size, args.max_pages, args.include_internal + ) + report.coverage.append( + ("HTTP sends", str(reader.requests), "Includes SDK-managed OAuth and bounded retries") + ) + report.coverage.append( + ( + "Configured limits", + "bounded", + f"page_size={args.page_size}; max_pages_per_metric={args.max_pages}; " + f"max_http_sends={args.max_requests}; " + f"request_start_budget_seconds={args.max_seconds}", + ) + ) + if report.issues and not args.allow_partial: + raise ReportError( + "Incomplete report; no workbook written. Use --allow-partial to save " + "flagged sections." + ) + save_workbook(workbook(report), output) + print(f"Saved {output}. {BOUNDARY}") + return 2 if report.issues else 0 + except (ReportError, OSError) as exc: + message = ( + str(exc) + if isinstance(exc, ReportError) + else "Cannot write workbook; check local path/permissions." + ) + print(message, file=sys.stderr) + return 1 + except Exception: + print( + "Unexpected SDK/schema failure; no report claimed complete. Verify " + "pinned dependencies; no sensitive diagnostics emitted.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/distribution_report/requirements.txt b/examples/distribution_report/requirements.txt new file mode 100644 index 0000000..5596b54 --- /dev/null +++ b/examples/distribution_report/requirements.txt @@ -0,0 +1,3 @@ +# Standalone example only; does not add dependencies to the SDK package. +asknews==0.14.6 +openpyxl==3.1.5 diff --git a/examples/distribution_report/test_distribution_report.py b/examples/distribution_report/test_distribution_report.py new file mode 100644 index 0000000..4217f37 --- /dev/null +++ b/examples/distribution_report/test_distribution_report.py @@ -0,0 +1,496 @@ +"""Synthetic only: no credentials or network. Run pytest in this directory.""" + +import importlib.util +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock +from uuid import UUID + +import httpx +import pytest +from openpyxl import load_workbook + +from asknews_sdk import AskNewsSDK + + +spec = importlib.util.spec_from_file_location( + "distribution_report", Path(__file__).with_name("distribution_report.py") +) +m = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = m +spec.loader.exec_module(m) +UTC = timezone.utc + + +@pytest.fixture(autouse=True) +def no_network(monkeypatch): + import socket + + monkeypatch.setattr( + socket.socket, "connect", Mock(side_effect=AssertionError("Network forbidden")) + ) + + +def article(n=1, value=3, url=None): + return { + "article_id": str(UUID(int=n)), + "article_url": url or f"https://apnews.com/article/{n}", + "hit_count": value, + } + + +def page(rows, number=1, next_page=None): + return { + "data": rows, + "total_count": sum(row["hit_count"] for row in rows), + "page": number, + "next_page": next_page, + } + + +class Fake: + def __init__(self, responses): + self.responses = iter(responses) + self.calls = [] + + def get(self, endpoint, query): + self.calls.append((endpoint, query)) + value = next(self.responses) + if isinstance(value, Exception): + raise value + return value + + +@pytest.mark.parametrize( + "month,days", [("2026-09", 30), ("2024-02", 29), ("2026-02", 28), ("2026-12", 31)] +) +def test_months_and_utc(month, days, monkeypatch): + monkeypatch.setenv("TZ", "Pacific/Honolulu") + start, end, following = m.month_bounds(month) + assert following - start == timedelta(days=days) + assert (following - end).total_seconds() == 1 + assert start.tzinfo == UTC + if month == "2026-09": + assert int(start.timestamp()) == 1788220800 + assert int(end.timestamp()) == 1790812799 + assert int(end.timestamp()) == int(following.timestamp()) - 1 + + +@pytest.mark.parametrize( + "month", ["2026-13", "2026-00", "2026-9", "1999-12", "2101-01", "2026-09Z"] +) +def test_invalid_month(month): + with pytest.raises(m.ReportError): + m.month_bounds(month) + + +@pytest.mark.parametrize( + "value", ["https://apnews.com", "*.apnews.com", "apnews.com.evil/", "apnews.com.", "a,b.com"] +) +def test_domain_input(value): + with pytest.raises(m.ReportError): + m.exact_domain(value) + + +def test_paginate_and_page_totals(): + reader = Fake([page([article()], next_page=2), page([article(2, 4)], 2)]) + ranked = m.ranking(reader, {}, "apnews.com", "surface", 1, 4) + assert ranked.complete and ranked.pages == 2 + assert sum(item[1] for item in ranked.rows.values()) == 7 + assert [call[1]["page"] for call in reader.calls] == [1, 2] + + +@pytest.mark.parametrize( + "second", + [ + page([article()], 2), + page([article(2)], 2, 2), + {"data": [], "total_count": 0}, + {"data": [article(2)], "total_count": 999, "page": 2, "next_page": None}, + ], +) +def test_page_guards_preserve_only_valid_pages(second): + ranked = m.ranking( + Fake([page([article()], next_page=2), second]), {}, "apnews.com", "surface", 1, 5 + ) + assert not ranked.complete + assert len(ranked.rows) == 1 + + +def test_caps_and_empty_final_page(): + ranked = m.ranking(Fake([page([article()], next_page=2)]), {}, "apnews.com", "surface", 1, 1) + assert not ranked.complete and "cap" in ranked.note + ranked = m.ranking( + Fake([page([article()], next_page=2), page([], 2)]), {}, "apnews.com", "surface", 1, 2 + ) + assert ranked.complete and len(ranked.rows) == 1 + + +@pytest.mark.parametrize( + "url", + [ + "https://other.com/a", + "https://www.apnews.com/a", + "https://apnews.com.evil/a", + "https://u:p@apnews.com/a", + '=HYPERLINK("https://evil")', + ], +) +def test_domain_isolation(url): + ranked = m.ranking(Fake([page([article(url=url)])]), {}, "apnews.com", "surface", 100, 1) + assert not ranked.complete and not ranked.rows + assert url not in ranked.note + + +def test_missing_id_is_not_url_joined(): + row = article() + del row["article_id"] + ranked = m.ranking(Fake([page([row])]), {}, "apnews.com", "surface", 100, 1) + assert not ranked.complete and not ranked.rows + + +def report_fixture(): + report = m.Report("apnews.com", "2026-09") + report.ranks = { + "surface": m.Ranking({str(UUID(int=1)): ("https://apnews.com/a", 4)}, True, "exhausted", 1), + "citation": m.Ranking( + {str(UUID(int=2)): ("https://apnews.com/b", 2)}, True, "exhausted", 1 + ), + "grounded": m.Ranking({}, False, "cap", 0), + } + return report + + +def test_join_missing_vs_zero_no_double_count(): + rows = list(m.joined_articles(report_fixture())) + assert rows[0][2:] == (4, 0, None) + assert rows[1][2:] == (0, 2, None) + + +def test_conflicting_identity_rejected(): + report = report_fixture() + report.ranks["citation"].rows[str(UUID(int=1))] = ("https://apnews.com/changed", 2) + with pytest.raises(m.ReportError, match="conflicting"): + list(m.joined_articles(report)) + + +def test_missing_daily_fields_and_absent_days_stay_unknown(): + start, _, end = m.month_bounds("2026-09") + result = m.daily_values( + {"data": [{"day": "2026-09-01", "surfaces": 0}], "total_surfaces": 0}, m.FIELDS, start, end + ) + assert result["2026-09-01"]["surfaces"] == 0 + assert result["2026-09-01"]["grounded"] is None + assert "2026-09-02" not in result + + +@pytest.mark.parametrize( + "days", [["2026-09-01", "2026-09-01"], ["2026-10-01"], ["2026-09-01T00:00:00-04:00"]] +) +def test_daily_guard(days): + start, _, end = m.month_bounds("2026-09") + with pytest.raises(m.ReportError): + m.daily_values( + {"data": [{"day": day, "surfaces": 1} for day in days]}, ["surfaces"], start, end + ) + + +def test_daily_reconciliation(): + start, _, end = m.month_bounds("2026-09") + with pytest.raises(m.ReportError, match="sum"): + m.daily_values( + {"data": [{"day": "2026-09-01", "surfaces": 3}], "total_surfaces": 4}, + ["surfaces"], + start, + end, + ) + + +def sdk_reader(handler, **kwargs): + reader = m.Reader(None, sleep=Mock(), **kwargs) + sdk = AskNewsSDK( + api_key="synthetic-not-real", + transport=httpx.MockTransport(handler), + retries=0, + follow_redirects=False, + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) + reader.sdk = sdk + return reader + + +@pytest.mark.parametrize("status,error", [(401, m.AuthError), (403, m.Denied)]) +def test_auth_no_retries_or_body_leak(status, error): + handler = Mock(return_value=httpx.Response(status, text="SECRET PII")) + reader = sdk_reader(handler) + with pytest.raises(error) as caught: + reader.get(m.RANK, {"domain_names": ["apnews.com"]}) + assert "SECRET" not in str(caught.value) + assert handler.call_count == 1 + reader.sdk.close() + + +def test_429_then_503_then_success_actual_sdk(): + handler = Mock( + side_effect=[ + httpx.Response(429, headers={"Retry-After": "3"}), + httpx.Response(503), + httpx.Response(200, json={"data": []}), + ] + ) + reader = sdk_reader(handler) + assert reader.get(m.RANK, {}) == {"data": []} + assert reader.requests == 3 + assert [call.args[0] for call in reader.sleep.call_args_list] == [1.0, 3, 1.0, 2, 1.0] + assert all(call.args[0].method == "GET" for call in handler.call_args_list) + reader.sdk.close() + + +def test_retry_after_date_and_budget(): + now = datetime(2026, 9, 1, tzinfo=UTC) + assert m.retry_delay("Tue, 01 Sep 2026 00:00:07 GMT", 0, now) == 7 + for value in ("600", "NaN", "infinity", "nonsense"): + with pytest.raises(m.ReportError): + m.retry_delay(value, 0, now) + reader = sdk_reader(lambda req: httpx.Response(503), max_requests=1) + with pytest.raises(m.ReportError, match="budget"): + reader.get(m.RANK, {}) + assert reader.requests == 1 + reader.sdk.close() + + +def test_transient_retry_bound_and_timeout_budget(): + reader = sdk_reader(lambda req: httpx.Response(503)) + with pytest.raises(m.ReportError, match="three"): + reader.get(m.RANK, {}) + assert reader.requests == 3 + reader.deadline = 0 + with pytest.raises(m.ReportError, match="budget"): + reader.get(m.RANK, {}) + reader.sdk.close() + + +def test_oauth_401_stops_before_sdk_refresh(): + handler = Mock(return_value=httpx.Response(401, text="secret")) + reader = m.Reader(None, sleep=Mock()) + sdk = AskNewsSDK( + client_id="fake", + client_secret="fake", + scopes={"distribution"}, + transport=httpx.MockTransport(handler), + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) + reader.sdk = sdk + with pytest.raises(m.AuthError): + reader.get(m.RANK, {}) + assert handler.call_count == 1 # token exchange only; no retry + sdk.close() + + +def test_index_disjoint_utc_and_leap_days(): + report = m.Report("apnews.com", "2024-02") + start, _, following = m.month_bounds(report.month) + calls = [] + + def get(endpoint, query): + calls.append(query) + assert endpoint == m.INDEX + assert query["domains"] == ["apnews.com"] + assert query["time_filter"] == "crawl_date" + return [{"start": query["start_datetime"], "end": query["end_datetime"], "count": 1}] + + m.collect_index(SimpleNamespace(get=get), report, start, following) + assert len(calls) == 29 and sum(report.index.values()) == 29 + for first, second in zip(calls, calls[1:]): + assert datetime.fromisoformat( + first["end_datetime"].replace("Z", "+00:00") + ) < datetime.fromisoformat(second["start_datetime"].replace("Z", "+00:00")) + assert all(query["end_datetime"].endswith("23:59:59Z") for query in calls) + + +@pytest.mark.parametrize("start_string", ["2026-09-01T00:00:00", "2026-09-01T00:00:00-04:00"]) +def test_index_rejects_naive_or_shifted_dates(start_string): + report = m.Report("apnews.com", "2026-09") + start, _, end = m.month_bounds(report.month) + with pytest.raises(m.ReportError, match="timezone"): + m.collect_index( + Fake([[{"start": start_string, "end": "2026-09-01T23:59:59Z", "count": 1}]]), + report, + start, + end, + ) + assert not report.index + + +def test_share_not_dollars_and_foreign_rejected(): + assert m.parse_share({"data": [{"domain": "apnews.com", "hit_share": 0}]}, "apnews.com") == 0 + with pytest.raises(m.ReportError): + m.parse_share({"data": [{"domain": "other.com", "hit_share": 0.4}]}, "apnews.com") + with pytest.raises(m.ReportError): + m.parse_share({"data": []}, "apnews.com") + + +def test_internal_forbidden_group_stops_without_zero(): + report = report_fixture() + reader = Fake([m.Denied("403: internal scope required")]) + start, _, following = m.month_bounds(report.month) + report.section("internal", lambda: m.collect_internal(reader, report, {}, start, following)) + assert len(reader.calls) == 1 and report.issues and not report.totals + + +def test_workbook_open_totals_injection_and_no_links(tmp_path): + report = report_fixture() + report.share = 0.25 + report.daily = {"2026-09-01": {"surfaces": 0, "citations": 2, "grounded": None}} + book = m.workbook(report) + payloads = [ + '=HYPERLINK("https://evil","x")', + "+SUM(A1)", + "-1+2", + "@SUM(A1)", + "\t=1+1", + "https://apnews.com/a?x==SUM(A1)", + "title\x01control", + ] + m.sheet(book, "Synthetic injection", ("Text",), [(value,) for value in payloads]) + path = tmp_path / "synthetic.xlsx" + m.save_workbook(book, path) + opened = load_workbook(path, data_only=False) + assert { + "Summary", + "Daily metrics", + "Articles", + "Weighted share", + "Coverage", + "Methodology", + } <= set(opened.sheetnames) + assert opened["Daily metrics"]["B2"].value == 0 + assert opened["Daily metrics"]["D2"].value is None + assert opened["Daily metrics"]["B3"].value is None + assert opened["Weighted share"]["B2"].value == 0.25 + assert sum(row[2] for row in opened["Articles"].iter_rows(min_row=2, values_only=True)) == 4 + assert opened["Synthetic injection"]["A2"].value == payloads[0] + for ws in opened: + assert ws.freeze_panes == "A2" and ws.auto_filter.ref + for row in ws: + assert all(cell.data_type != "f" and not cell.hyperlink for cell in row) + assert not opened._external_links + with pytest.raises(m.ReportError, match="exists"): + m.save_workbook(book, path) + + +def test_environment_auth_and_no_cli_credentials(): + assert m.credentials({"ASKNEWS_API_KEY": "synthetic"}) == {"api_key": "synthetic"} + assert m.credentials({"ASKNEWS_CLIENT_ID": "a", "ASKNEWS_CLIENT_SECRET": "b"})["scopes"] == { + "distribution", + "news", + } + for env in ( + {}, + {"ASKNEWS_CLIENT_SECRET": "b"}, + {"ASKNEWS_API_KEY": "a", "ASKNEWS_CLIENT_ID": "b"}, + ): + with pytest.raises(m.ReportError): + m.credentials(env) + + +def test_end_to_end_synthetic_sdk_requests_and_metrics(tmp_path): + def handler(request): + q = request.url.params + assert request.method == "GET" + assert q.get_list("domain_names") == ["apnews.com"] or q.get_list("domains") == [ + "apnews.com" + ] + path = request.url.path + if path == m.RANK: + return httpx.Response(200, json=page([article(value=3)])) + if path == m.SHARE: + return httpx.Response(200, json={"data": [{"domain": "apnews.com", "hit_share": 0.1}]}) + if path == m.TOTAL: + return httpx.Response(200, json=dict.fromkeys(m.FIELDS, 3)) + if path == m.DAILY: + return httpx.Response( + 200, + json={ + "data": [dict(day="2026-09-01", **dict.fromkeys(m.FIELDS, 3))], + **{"total_" + name: 3 for name in m.FIELDS}, + }, + ) + if path == m.UNIQUE: + return httpx.Response(200, json={"hits": 3, "surfaced": 1}) + if path == m.UNIQUE_DAILY: + return httpx.Response( + 200, + json={ + "data": [{"day": "2026-09-01", "hits": 3, "surfaced": 1}], + "total_hits": 3, + "total_surfaced": 1, + }, + ) + assert path == m.INDEX + return httpx.Response( + 200, json=[{"start": q["start_datetime"], "end": q["end_datetime"], "count": 2}] + ) + + reader = sdk_reader(handler) + report = m.collect(reader, "apnews.com", "2026-09", include_internal=True) + assert not report.issues and reader.requests == 38 + assert report.totals["grounded"] == 3 and len(report.index) == 30 + path = tmp_path / "complete-fixture.xlsx" + m.save_workbook(m.workbook(report), path) + assert load_workbook(path)["Articles"].max_row == 2 + reader.sdk.close() + + +def test_partial_output_is_explicit(tmp_path, monkeypatch, capsys): + report = report_fixture() + report.issues = ["synthetic partial"] + monkeypatch.setattr(m, "credentials", lambda env: {"api_key": "synthetic"}) + monkeypatch.setattr(m, "collect", lambda *a: report) + output = tmp_path / "partial.xlsx" + assert m.main(["--output", str(output)]) == 1 + assert not output.exists() + assert m.main(["--output", str(output), "--allow-partial"]) == 2 + assert output.exists() + assert "PARTIAL CALENDAR COVERAGE" in capsys.readouterr().out + + +def test_api_401_does_not_trigger_sdk_oauth_refresh(): + handler = Mock( + side_effect=[ + httpx.Response( + 200, json={"access_token": "synthetic", "expires_in": 3600, "token_type": "Bearer"} + ), + httpx.Response(401, text="sensitive body"), + ] + ) + reader = m.Reader(None, sleep=Mock()) + with AskNewsSDK( + client_id="fake", + client_secret="fake", + scopes={"distribution"}, + transport=httpx.MockTransport(handler), + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) as sdk: + reader.sdk = sdk + with pytest.raises(m.AuthError): + reader.get(m.RANK, {}) + assert handler.call_count == 2 + assert reader.requests == 2 + + +def test_partial_page_failure_and_numeric_precision(): + ranked = m.ranking( + Fake([page([article()], next_page=2), m.ReportError("Transient exhausted")]), + {}, + "apnews.com", + "surface", + 1, + 3, + ) + assert not ranked.complete and len(ranked.rows) == 1 + for value in (None, True, -1, 2**53, 1.5, "3"): + with pytest.raises(m.ReportError): + m.count(value) From 1b8e08a3496d6c219eb27c2f2b32396a40c832fc Mon Sep 17 00:00:00 2001 From: Devops Bot Date: Fri, 9 Oct 2026 16:16:38 +0000 Subject: [PATCH 2/4] docs: Align Excel example with publisher guide --- examples/distribution_report/README.md | 69 +++++++++++++++++-- .../distribution_report.py | 41 +++++++++-- .../test_distribution_report.py | 27 +++++++- 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/examples/distribution_report/README.md b/examples/distribution_report/README.md index 93d0d4a..72f62a3 100644 --- a/examples/distribution_report/README.md +++ b/examples/distribution_report/README.md @@ -5,6 +5,10 @@ Run locally with **your existing authorized credentials**. No credential or permission changes, customer extraction, or live authorization tests were made to develop this example. Tests use synthetic responses only. +Primary reference: the official [Publisher Analytics guide](https://docs.asknews.app/en/publisher) +and its [linked API reference](https://docs.asknews.app/en/reference#tag--distribution), +read with the live Python recipes on 2026-10-09. See the guide/source comparison below. + ## Install and run Keep `distribution_report.py` and `requirements.txt` together (or enter this @@ -26,11 +30,17 @@ POSIX modes. Filesystems must support hard links for atomic no-overwrite output. secret manager/session; do not put secrets in command arguments, source files, chat, or shell history. This script reads one of: -- `ASKNEWS_API_KEY`, passed to the SDK's `api_key` parameter; **or** +- `ASKNEWS_API_KEY`, passed to the SDK's `api_key` parameter. The publisher guide + uses an **existing publisher organization key** (`ank_org_...`) with + `distribution` + `news` scopes. **Personal API keys cannot access distribution**; + ownership/admin privileges do not turn a personal key into a publisher key; **or** - `ASKNEWS_CLIENT_ID` + `ASKNEWS_CLIENT_SECRET`, passed to the SDK's OAuth client credentials parameters. Optional `ASKNEWS_SCOPES` is a space-separated subset of `distribution news internal`; default `distribution news`. Request only scopes the existing client already has. The script does not grant scopes. + OAuth is an additional source-verified SDK/admin authentication option, not + the publisher guide's organization-key recipe; it still needs endpoint and + exact-domain authorization. Do not create or change credentials to run this example. These environment names are this example's adapter, not automatic SDK environment loading. Do not set both authentication methods. No custom host, token URL, @@ -60,9 +70,13 @@ suitably authorized credential or accept the documented gap—not an access chan cause before rerunning. Defaults: page size **100**, at most **100 pages per metric**, **400 HTTP sends** -(including OAuth/retries), **900 seconds** of request-start/retry budget, -30-second HTTP operation timeout, serial requests with at least 1 second before -each send. A request already in flight may finish after the overall deadline; +(including OAuth/retries), **2400 seconds** of request-start/retry budget, +30-second HTTP operation timeout, serial requests with **6 seconds before every +distribution send** (including retries and admin/internal requests) and 1 second +before other/news/auth sends. This follows guide Recipe 3 and stays below the +published **0.2 distribution requests/second** limit (burst 5, concurrency 2). +The longer bounded default budget accommodates the slower polite pagination. +A request already in flight may finish after the overall deadline; httpx timeouts bound individual network operations, not a strict wall-clock process deadline. Set an external process timeout if required. @@ -103,8 +117,10 @@ Monthly distinct article counts must **not** be computed by summing daily uniques. Index counts describe current retained indexed articles filtered by **crawl date**; they are not distribution events, publication-date activity, or a historical inventory snapshot. Ranking endpoints do not supply titles, -publication dates, bias/sentiment or other enrichments: those remain unavailable; -this example does not add costly article fanout. Consumer identities, prompts, +publication dates, bias/sentiment or other enrichments: those are not collected +by this event-count example. The guide's Recipe 4 supports these via a separate +batched article lookup (see below); they are not unsupported API features. +This example performs no additional article enrichment fanout. Consumer identities, prompts, queries, raw hits and billing/PII endpoints are never requested or exported. Only the requested exact domain is sent in filters. URLs with a different host @@ -146,6 +162,47 @@ Weighted share reflects the server's current metric weights and publisher multipliers for the selected period; weights/pool values are not exposed by this response. **No revenue, payout, currency or dollar amount is inferred.** +## Publisher guide / source cross-check + +- **Recipes 1–3 match:** `AskNewsSDK(api_key=...)`, authenticated + `sdk.client.request(...)`, `response.content`, exact domain names, same-UTC-month + integer dates, page size <=100, 1-based page/next_page, page-only event sums and + an optional empty final page. The script also guards caps/duplicates/drift. +- **Rate correction:** the guide and API configuration say **0.2 req/s**, not + 1 req/s. This example now follows the guide's six-second distribution delay. + The guide's deterministic tie ordering does not create a transactional snapshot. +- **Share wording:** the guide calls `hit_share` a share of publisher traffic. + Inspected service code applies metric weights and publisher multipliers. The + workbook therefore says **weighted share**, never unweighted event percentage + or dollars; it cannot reconstruct weights from this response. +- **Metrics not listed in the guide's parameter table:** the linked public + API reference (version **0.32.4** when checked) includes + `metric=surface|citation|grounded` on article rankings. Retrievals are `surface` + events, citations and groundings are independent event counts, not publication + counts. Internal raw models also have `full_text`, which is a separate metric. +- **Public versus internal:** the guide FAQ explicitly excludes dashboard-only + daily/query/domain lookup endpoints from the public API. The linked schema + confirms `distribution` + `internal` on the optional metric/unique routes; + neither the guide's organization-key recipe nor admin status promises access. +- **Index counts:** not one of the guide's three recipes. The linked reference + exposes `/v1/index_counts` with `news` scope, and inspected publisher-key + allowlist explicitly includes it. This source-verified extra is still + deployment/authorization-dependent; denial is recorded, not bypassed. +- **Recipe 4 is supported, but deliberately not requested:** verified SDK method + `sdk.news.get_articles(article_ids: List[str] | List[UUID], full_text=False)` + returns `List[ArticleResponse]` and GETs `/v1/news`. The guide batches at 100 IDs + for title, publication metadata, classification, sentiment and reporting voice. + This scoped event-count report does not fetch that additional metadata, entities, + summaries or article bodies, even in batches. Its article sheet is IDs/URLs/counts; + do not interpret absent metadata as an API/SDK limitation. No high-cost per-article fanout. +- **Date completeness:** the guide's examples use integer timestamps and do not + establish fractional-second month-end coverage; the source's inclusive bounds + and this report's explicit coverage warnings remain necessary. + +The guide links are also recorded as **literal text** in each workbook's +Methodology sheet. No guidance here changes existing permissions, SDK/API +functionality or production settings. + ## Verified SDK/API compatibility Verified SDK source: `asknews==0.14.6`, commit diff --git a/examples/distribution_report/distribution_report.py b/examples/distribution_report/distribution_report.py index fab78d1..dc77d93 100644 --- a/examples/distribution_report/distribution_report.py +++ b/examples/distribution_report/distribution_report.py @@ -32,6 +32,8 @@ UTC = timezone.utc METRICS = ("surface", "citation", "grounded") FIELDS = ("surfaces", "citations", "grounded", "full_text") +PUBLISHER_GUIDE = "https://docs.asknews.app/en/publisher" +API_REFERENCE = "https://docs.asknews.app/en/reference#tag--distribution" BASE = "/v1/distribution/" RANK = BASE + "articles/top_n_for_domains" SHARE = BASE + "stats/hit_share" @@ -111,7 +113,7 @@ def retry_delay(header, attempt, now=None): class Reader: """All data reads use SDK transport; budgets count each HTTP send (including OAuth).""" - def __init__(self, sdk, max_requests=400, max_seconds=900, sleep=time.sleep): + def __init__(self, sdk, max_requests=400, max_seconds=2400, sleep=time.sleep): self.sdk = sdk self.max_requests = max_requests self.deadline = time.monotonic() + max_seconds @@ -119,10 +121,13 @@ def __init__(self, sdk, max_requests=400, max_seconds=900, sleep=time.sleep): self.sleep = sleep def before_request(self, request): - if self.requests >= self.max_requests or time.monotonic() + 1 >= self.deadline: + # Publisher guide Recipe 3: 6s keeps distribution below its 0.2 req/s limit. + # Apply even to admin/internal calls; do not assume rate-limit exemptions. + delay = 6.0 if request.url.path.startswith(BASE) else 1.0 + if self.requests >= self.max_requests or time.monotonic() + delay >= self.deadline: raise ReportError("Request/time budget reached; increase bounded limits explicitly.") # Called by httpx for API and SDK-managed OAuth requests, not logged. - self.sleep(1.0) + self.sleep(delay) self.requests += 1 def after_response(self, response): @@ -573,6 +578,34 @@ def workbook(report): "Methodology", ("Item", "Value"), [ + ("Official publisher guide", PUBLISHER_GUIDE), + ("Linked API reference", API_REFERENCE), + ( + "Publisher authentication", + "Guide uses existing publisher organization API keys (ank_org_...) with " + "distribution/news scopes; personal API keys cannot access distribution. " + "OAuth is an SDK-supported admin alternative only when already authorized. " + "Admin status never implies internal scope.", + ), + ( + "Guide/source differences", + "Guide describes traffic share broadly; implementation is weighted, not raw " + "traffic or payout. Guide omits metric selector and index counts; public API " + "reference/source support them. Named SDK internal DTOs still drop grounded.", + ), + ( + "Optional guide enrichment", + "Recipe 4 supports news.get_articles(article_ids=..., full_text=False) in " + "batches of 100. This bounded event-count report does not request metadata " + "enrichment, summaries or article bodies; unavailable here does not mean " + "unavailable from the API.", + ), + ( + "Polite requests", + "Guide limit: distribution 0.2 requests/sec, burst 5, concurrency 2. " + "This report is serial and waits 6s before every distribution send, including " + "retries; other/auth sends wait 1s. Retry-After is additional when present.", + ), ("Requested start UTC", utc_string(start)), ("Requested exclusive end UTC", utc_string(following)), ("API inclusive start timestamp", int(start.timestamp())), @@ -714,7 +747,7 @@ def main(argv=None): parser.add_argument("--page-size", type=bounded_int(1, 100), default=100) parser.add_argument("--max-pages", type=bounded_int(1, 1000), default=100) parser.add_argument("--max-requests", type=bounded_int(1, 4000), default=400) - parser.add_argument("--max-seconds", type=bounded_int(1, 3600), default=900) + parser.add_argument("--max-seconds", type=bounded_int(1, 3600), default=2400) args = parser.parse_args(argv) # SDK errors/HTTP logging can contain auth requests or response bodies. Never enable debug. logging.disable(logging.CRITICAL) diff --git a/examples/distribution_report/test_distribution_report.py b/examples/distribution_report/test_distribution_report.py index 4217f37..57bd2e5 100644 --- a/examples/distribution_report/test_distribution_report.py +++ b/examples/distribution_report/test_distribution_report.py @@ -243,7 +243,7 @@ def test_429_then_503_then_success_actual_sdk(): reader = sdk_reader(handler) assert reader.get(m.RANK, {}) == {"data": []} assert reader.requests == 3 - assert [call.args[0] for call in reader.sleep.call_args_list] == [1.0, 3, 1.0, 2, 1.0] + assert [call.args[0] for call in reader.sleep.call_args_list] == [6.0, 3, 6.0, 2, 6.0] assert all(call.args[0].method == "GET" for call in handler.call_args_list) reader.sdk.close() @@ -494,3 +494,28 @@ def test_partial_page_failure_and_numeric_precision(): for value in (None, True, -1, 2**53, 1.5, "3"): with pytest.raises(m.ReportError): m.count(value) + + +def test_publisher_guide_rate_limit_pacing(): + reader = m.Reader(None, sleep=Mock()) + for endpoint in (m.RANK, m.INDEX, "/oauth2/token"): + reader.before_request(httpx.Request("GET", "https://api.asknews.app" + endpoint)) + assert [call.args[0] for call in reader.sleep.call_args_list] == [6.0, 1.0, 1.0] + assert reader.requests == 3 + + +def test_rate_delay_must_fit_remaining_budget(): + reader = m.Reader(None, max_seconds=5, sleep=Mock()) + with pytest.raises(m.ReportError, match="budget"): + reader.before_request(httpx.Request("GET", "https://api.asknews.app" + m.RANK)) + assert reader.requests == 0 and reader.sleep.call_count == 0 + + +def test_official_guide_in_workbook_methodology(): + book = m.workbook(report_fixture()) + rows = dict(book["Methodology"].iter_rows(min_row=2, values_only=True)) + assert rows["Official publisher guide"] == "https://docs.asknews.app/en/publisher" + assert "6s" in rows["Polite requests"] + assert "personal API keys cannot access distribution" in rows["Publisher authentication"] + assert "batches of 100" in rows["Optional guide enrichment"] + assert "weighted" in rows["Guide/source differences"] From e2e3adf1e3c672887d2b8b70c69d108f7f741a50 Mon Sep 17 00:00:00 2001 From: Devops Bot Date: Fri, 9 Oct 2026 16:26:39 +0000 Subject: [PATCH 3/4] fix: Preserve report retry deadlines and daily coverage --- examples/distribution_report/README.md | 26 ++- .../distribution_report.py | 73 ++++++-- .../test_distribution_report.py | 171 +++++++++++++++++- 3 files changed, 249 insertions(+), 21 deletions(-) diff --git a/examples/distribution_report/README.md b/examples/distribution_report/README.md index 72f62a3..735c6f6 100644 --- a/examples/distribution_report/README.md +++ b/examples/distribution_report/README.md @@ -83,9 +83,15 @@ process deadline. Set an external process timeout if required. Optional bounded flags: `--page-size 1..100`, `--max-pages 1..1000`, `--max-requests 1..4000`, `--max-seconds 1..3600`. API/transport transient failures retry at most twice (three attempts total); retryable statuses are -408/429/500/502/503/504. `Retry-After` seconds and HTTP dates are honored. A wait -over 60 seconds, invalid Retry-After or exhausted budget stops that section -rather than retrying early. No concurrency or article-by-article fanout. Index +408/429/500/502/503/504. `Retry-After` seconds and HTTP dates set a **reader-wide +not-before deadline**, including on the final failed attempt of a section. +Every later send (other metrics, share, internal, index or OAuth) must honor that +deadline as well as its normal pacing; a section boundary never resets it. +A wait over 60 seconds, unusable/negative Retry-After, or exhausted request/time +budget **permanently stops all further sends for the run**, not just the current +section. Remaining sections are recorded unavailable/partial without network +requests. Previously collected data can only be saved via `--allow-partial` +(exit 2); otherwise no workbook is written. No concurrency or article-by-article fanout. Index counts cost one additional bounded request per day (28–31). Request limits include unsuccessful sends; no unrestricted all-history queries. @@ -113,8 +119,14 @@ may require a final empty page. Old servers without explicit `page`/`next_page` metadata are marked partial/unavailable rather than falsely reporting their potentially top-N-capped cohort as complete. +The daily distinct response requires `hits` and `surfaced` on every returned +row plus both `total_hits` and `total_surfaced`. Missing fields remain blank; +missing/invalid fields or totals mark the internal group partial/unavailable and +require `--allow-partial` (exit 2), never an unflagged successful export. +`total_surfaced` reconciles the **sum of daily distinct counts only**. Monthly distinct article counts must **not** be computed by summing daily -uniques. Index counts describe current retained indexed articles filtered by +uniques; the independent monthly `surfaced` value is preserved. Index counts +describe current retained indexed articles filtered by **crawl date**; they are not distribution events, publication-date activity, or a historical inventory snapshot. Ranking endpoints do not supply titles, publication dates, bias/sentiment or other enrichments: those are not collected @@ -253,6 +265,10 @@ Fixtures and httpx MockTransport exercise the **real SDK** without network or real credentials. Tests cover pagination, repeated/capped pages, metric joining, leap/year/UTC boundaries, exact-domain rejection, 401/403, retries/budgets, missing fields, daily reconciliation, formula injection, workbook reopening, -totals, output collision and partial exit behavior. No real AP workbook or data +totals, output collision and partial exit behavior. Reviewer regressions cover +cross-section/final-attempt Retry-After, unusable headers, exhausted wait/request +time budgets, missing daily distinct fields/totals, explicit partial CLI behavior, +and preserving monthly distinct counts when daily distinct counts overlap. +No real AP workbook or data is included. Tests live with this optional example to avoid adding Excel runtime/dev dependencies to the SDK package or changing its release workflows. diff --git a/examples/distribution_report/distribution_report.py b/examples/distribution_report/distribution_report.py index dc77d93..b4de626 100644 --- a/examples/distribution_report/distribution_report.py +++ b/examples/distribution_report/distribution_report.py @@ -99,6 +99,8 @@ def retry_delay(header, attempt, now=None): if header is not None: try: delay = float(header) + if delay < 0: + raise ReportError("Invalid negative Retry-After; stop and retry later.") except (TypeError, ValueError): try: date = parsedate_to_datetime(header) @@ -119,15 +121,33 @@ def __init__(self, sdk, max_requests=400, max_seconds=2400, sleep=time.sleep): self.deadline = time.monotonic() + max_seconds self.requests = 0 self.sleep = sleep + self.not_before = 0.0 + self.stopped_reason = None + + def stop(self, reason): + self.stopped_reason = reason + raise ReportError("All further sends stopped: " + reason) + + def defer(self, delay): + # Shared across every section, including after a section's final failed attempt. + self.not_before = max(self.not_before, time.monotonic() + delay) + if self.not_before >= self.deadline: + self.stop("Retry-After/backoff would exceed time budget.") def before_request(self, request): + if self.stopped_reason is not None: + self.stop(self.stopped_reason) # Publisher guide Recipe 3: 6s keeps distribution below its 0.2 req/s limit. # Apply even to admin/internal calls; do not assume rate-limit exemptions. - delay = 6.0 if request.url.path.startswith(BASE) else 1.0 - if self.requests >= self.max_requests or time.monotonic() + delay >= self.deadline: - raise ReportError("Request/time budget reached; increase bounded limits explicitly.") + pacing = 6.0 if request.url.path.startswith(BASE) else 1.0 + now = time.monotonic() + delay = max(pacing, self.not_before - now) + if self.requests >= self.max_requests or now + delay >= self.deadline: + self.stop("Request/time budget reached; increase bounded limits explicitly.") # Called by httpx for API and SDK-managed OAuth requests, not logged. self.sleep(delay) + if time.monotonic() >= self.deadline: + self.stop("Time budget expired while waiting.") self.requests += 1 def after_response(self, response): @@ -146,6 +166,8 @@ def get(self, endpoint, query): if endpoint not in ALLOWED: raise ReportError("Endpoint not allowlisted.") for attempt in range(3): + if self.stopped_reason is not None: + self.stop(self.stopped_reason) try: return self.sdk.client.request(method="GET", endpoint=endpoint, query=query).content except (APIError, httpx.HTTPStatusError) as exc: @@ -163,16 +185,18 @@ def get(self, endpoint, query): raise ReportError( f"HTTP {status}: section unavailable; check API compatibility." ) from None - delay = retry_delay(exc.response.headers.get("retry-after"), attempt) + try: + delay = retry_delay(exc.response.headers.get("retry-after"), attempt) + except ReportError as error: + # A section-local failure cannot release a shared-quota restriction. + self.stop(str(error)) except httpx.TransportError: delay = 2**attempt except (ValueError, TypeError, AttributeError): raise ReportError("Unrecognized SDK/API response; verify compatibility.") from None + self.defer(delay) if attempt == 2: raise ReportError("Transient request failed after three bounded attempts.") - if time.monotonic() + delay >= self.deadline: - raise ReportError("Retry would exceed time budget.") - self.sleep(delay) raise ReportError("Request did not complete.") @@ -289,8 +313,10 @@ class Report: def section(self, name, operation): try: + issues_before = len(self.issues) value = operation() - self.coverage.append((name, "available", "See boundary/snapshot methodology")) + status = "partial" if len(self.issues) > issues_before else "available" + self.coverage.append((name, status, "See coverage issues and boundary methodology")) return value except AuthError: raise @@ -313,6 +339,17 @@ def parse_share(payload, domain): return value +def validate_distinct_coverage(report, payload): + for name in ("hits", "surfaced"): + if "total_" + name not in payload: + report.issues.append("Daily distinct response total missing: total_" + name) + else: + # Validate totals even when missing row fields prevent sum reconciliation. + count(payload["total_" + name]) + if any(row[name] is None for row in report.unique_daily.values()): + report.issues.append("Daily distinct field missing: " + name) + + def collect_internal(reader, report, query, start, following): # One denied internal request stops the internal group; no 403 endpoint-probing loop. raw = reader.get(TOTAL, query) @@ -333,12 +370,16 @@ def collect_internal(reader, report, query, start, following): month_query = {"domain_names": [report.domain], "year": start.year, "month": start.month} raw = reader.get(UNIQUE, month_query) report.unique = {name: count(raw[name]) for name in ("hits", "surfaced")} - report.unique_daily = daily_values( - reader.get(UNIQUE_DAILY, month_query), ("hits", "surfaced"), start, following - ) - daily_hits = sum(row["hits"] for row in report.unique_daily.values()) - if daily_hits != report.unique["hits"] or report.unique["hits"] != report.totals["surfaces"]: - report.issues.append("Surface totals/daily/aggregate mismatch (snapshot drift).") + unique_daily_payload = reader.get(UNIQUE_DAILY, month_query) + report.unique_daily = daily_values(unique_daily_payload, ("hits", "surfaced"), start, following) + validate_distinct_coverage(report, unique_daily_payload) + daily_hits = [row["hits"] for row in report.unique_daily.values()] + if all(value is not None for value in daily_hits): + if sum(daily_hits) != report.unique["hits"]: + report.issues.append("Surface totals/daily mismatch (snapshot drift).") + if report.unique["hits"] != report.totals["surfaces"]: + report.issues.append("Surface totals/aggregate mismatch (snapshot drift).") + # total_surfaced reconciles daily rows only; NEVER replace the monthly distinct count. surface = report.ranks["surface"] if surface.complete and len(surface.rows) != report.unique["surfaced"]: report.issues.append("Monthly distinct/ranking article count mismatch (snapshot drift).") @@ -604,7 +645,9 @@ def workbook(report): "Polite requests", "Guide limit: distribution 0.2 requests/sec, burst 5, concurrency 2. " "This report is serial and waits 6s before every distribution send, including " - "retries; other/auth sends wait 1s. Retry-After is additional when present.", + "retries; other/auth sends wait 1s. A shared Retry-After deadline applies " + "across sections, including final attempts. Unusable headers or waits/budgets " + "that prevent compliance permanently stop all further sends for this run.", ), ("Requested start UTC", utc_string(start)), ("Requested exclusive end UTC", utc_string(following)), diff --git a/examples/distribution_report/test_distribution_report.py b/examples/distribution_report/test_distribution_report.py index 57bd2e5..4b18e65 100644 --- a/examples/distribution_report/test_distribution_report.py +++ b/examples/distribution_report/test_distribution_report.py @@ -243,7 +243,7 @@ def test_429_then_503_then_success_actual_sdk(): reader = sdk_reader(handler) assert reader.get(m.RANK, {}) == {"data": []} assert reader.requests == 3 - assert [call.args[0] for call in reader.sleep.call_args_list] == [6.0, 3, 6.0, 2, 6.0] + assert [call.args[0] for call in reader.sleep.call_args_list] == [6.0, 6.0, 6.0] assert all(call.args[0].method == "GET" for call in handler.call_args_list) reader.sdk.close() @@ -519,3 +519,172 @@ def test_official_guide_in_workbook_methodology(): assert "personal API keys cannot access distribution" in rows["Publisher authentication"] assert "batches of 100" in rows["Optional guide enrichment"] assert "weighted" in rows["Guide/source differences"] + + +@pytest.mark.parametrize("header", ["120", "not-a-date", "NaN", "infinity", "-1"]) +def test_unusable_retry_after_stops_all_sections(header, monkeypatch): + clock, sent = [0.0], [] + monkeypatch.setattr(m.time, "monotonic", lambda: clock[0]) + + def handler(request): + sent.append((request.url.path, clock[0])) + return httpx.Response(429, headers={"Retry-After": header}) + + reader = sdk_reader(handler) + reader.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + try: + report = m.collect(reader, "apnews.com", "2026-09", include_internal=True) + assert reader.stopped_reason and len(sent) == 1 + assert report.issues and report.share is None and not report.index + assert all(not ranked.complete for ranked in report.ranks.values()) + # Changing sections, removing request hooks or extending time cannot reopen the reader. + reader.deadline += 1000 + with pytest.raises(m.ReportError, match="stopped"): + reader.get(m.INDEX, {}) + assert len(sent) == 1 + finally: + reader.sdk.close() + + +def test_retry_after_final_attempt_survives_section_and_endpoint_change(monkeypatch): + clock, sent = [0.0], [] + monkeypatch.setattr(m.time, "monotonic", lambda: clock[0]) + + def handler(request): + sent.append((request.url.path, clock[0])) + return httpx.Response(429, headers={"Retry-After": "10"}) + + reader = sdk_reader(handler) + reader.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + try: + report = m.collect(reader, "apnews.com", "2026-09") + assert len(sent) == 15 and report.issues + assert sent[0][1] == 6 + assert sent[3][1] - sent[2][1] >= 10 # final surface attempt -> citation + assert sent[12][0] == m.INDEX # restriction also follows shared reader into news + assert all(b[1] - a[1] >= 10 for a, b in zip(sent, sent[1:])) + assert reader.not_before == sent[-1][1] + 10 + finally: + reader.sdk.close() + + +@pytest.mark.parametrize("budget", [15, 20]) +def test_retry_after_or_wait_budget_stops_future_sends(budget, monkeypatch): + clock, sent = [0.0], [] + monkeypatch.setattr(m.time, "monotonic", lambda: clock[0]) + + def handler(request): + sent.append(clock[0]) + return httpx.Response(429, headers={"Retry-After": "10"}) + + reader = sdk_reader(handler, max_seconds=budget) + reader.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + try: + report = m.collect(reader, "apnews.com", "2026-09") + assert report.issues and reader.stopped_reason + assert sent == ([6.0] if budget == 15 else [6.0, 16.0]) + finally: + reader.sdk.close() + + +def test_budget_expiring_during_wait_stops_before_send(monkeypatch): + clock = [0.0] + monkeypatch.setattr(m.time, "monotonic", lambda: clock[0]) + handler = Mock(return_value=httpx.Response(200, json={})) + reader = sdk_reader(handler, max_seconds=10) + reader.sleep = lambda seconds: clock.__setitem__(0, clock[0] + 20) + try: + with pytest.raises(m.ReportError, match="expired"): + reader.get(m.RANK, {}) + assert reader.stopped_reason and handler.call_count == 0 + with pytest.raises(m.ReportError, match="stopped"): + reader.get(m.INDEX, {}) + assert handler.call_count == 0 + finally: + reader.sdk.close() + + +def distinct_fixture_handler(unique_daily): + def handler(request): + path, query = request.url.path, request.url.params + if path == m.RANK: + payload = page([article(value=3)]) + elif path == m.SHARE: + payload = {"data": [{"domain": "apnews.com", "hit_share": 0.1}]} + elif path == m.TOTAL: + payload = dict.fromkeys(m.FIELDS, 3) + elif path == m.DAILY: + payload = { + "data": [dict(day="2026-09-01", **dict.fromkeys(m.FIELDS, 3))], + **{"total_" + name: 3 for name in m.FIELDS}, + } + elif path == m.UNIQUE: + payload = {"hits": 3, "surfaced": 1} + elif path == m.UNIQUE_DAILY: + payload = unique_daily + else: + assert path == m.INDEX + payload = [{"start": query["start_datetime"], "end": query["end_datetime"], "count": 2}] + return httpx.Response(200, json=payload) + + return handler + + +@pytest.mark.parametrize( + "unique_daily", + [ + {"data": [{"day": "2026-09-01", "hits": 3}], "total_hits": 3, "total_surfaced": 1}, + {"data": [{"day": "2026-09-01", "surfaced": 1}], "total_hits": 3, "total_surfaced": 1}, + {"data": [{"day": "2026-09-01", "hits": 3, "surfaced": 1}], "total_surfaced": 1}, + {"data": [{"day": "2026-09-01", "hits": 3, "surfaced": 1}], "total_hits": 3}, + {"data": [{"day": "2026-09-01", "hits": 3, "surfaced": 1}]}, + {"data": []}, + { + "data": [{"day": "2026-09-01", "hits": 3, "surfaced": None}], + "total_hits": 3, + "total_surfaced": 1, + }, + {"data": [{"day": "2026-09-01", "hits": 3}], "total_hits": 3, "total_surfaced": None}, + ], +) +def test_missing_distinct_schema_requires_partial_cli(unique_daily, tmp_path, monkeypatch): + reader = sdk_reader(distinct_fixture_handler(unique_daily)) + try: + report = m.collect(reader, "apnews.com", "2026-09", include_internal=True) + finally: + reader.sdk.close() + assert report.issues + assert all(row[1] != "available" for row in report.coverage if row[0] == "Internal group") + monkeypatch.setattr(m, "collect", lambda *args: report) + monkeypatch.setattr(m, "credentials", lambda env: {"api_key": "synthetic-not-real"}) + path = tmp_path / "requires-partial.xlsx" + assert m.main(["--output", str(path)]) == 1 and not path.exists() + assert m.main(["--output", str(path), "--allow-partial"]) == 2 + book = load_workbook(path) + summary = {row[0]: row[1] for row in book["Summary"].iter_rows(min_row=2, values_only=True)} + assert summary["Run status"] == "PARTIAL" + assert summary["Monthly unique retrieved articles"] == 1 + for row in unique_daily["data"]: + if row.get("hits") is None: + assert book["Daily metrics"]["F2"].value is None + if row.get("surfaced") is None: + assert book["Daily metrics"]["G2"].value is None + + +def test_daily_distinct_sum_is_never_monthly_distinct(): + payload = { + "data": [ + {"day": "2026-09-01", "hits": 2, "surfaced": 1}, + {"day": "2026-09-02", "hits": 1, "surfaced": 1}, + ], + "total_hits": 3, + "total_surfaced": 2, + } + reader = sdk_reader(distinct_fixture_handler(payload)) + try: + report = m.collect(reader, "apnews.com", "2026-09", include_internal=True) + finally: + reader.sdk.close() + assert not report.issues + assert sum(row["surfaced"] for row in report.unique_daily.values()) == 2 + assert report.unique["surfaced"] == 1 From b6a1d6680916cd17d8f28dfdbf685b7b921b40f1 Mon Sep 17 00:00:00 2001 From: Devops Bot Date: Sat, 10 Oct 2026 12:32:29 +0000 Subject: [PATCH 4/4] fix: Add bounded date ranges and named article enrichment --- examples/distribution_report/README.md | 450 ++++++++---------- .../distribution_report.py | 269 +++++++++-- .../test_distribution_report.py | 445 ++++++++++++++++- 3 files changed, 861 insertions(+), 303 deletions(-) diff --git a/examples/distribution_report/README.md b/examples/distribution_report/README.md index 735c6f6..e35fc05 100644 --- a/examples/distribution_report/README.md +++ b/examples/distribution_report/README.md @@ -1,274 +1,222 @@ # Exact-domain distribution Excel report -Standalone, read-only Python **3.10+** example. Defaults: `apnews.com`, `2026-09`. -Run locally with **your existing authorized credentials**. No credential or -permission changes, customer extraction, or live authorization tests were made -to develop this example. Tests use synthetic responses only. +Standalone read-only Python **3.10+** utility, default domain `apnews.com`. +Select an explicit **inclusive `--start` / exclusive `--stop`** range, or use +`--month YYYY-MM` as an optional, mutually exclusive shorthand. There is no +implicit date range. No SDK runtime, release, permission or production changes +are needed to run the utility. Development and tests use synthetic data only. -Primary reference: the official [Publisher Analytics guide](https://docs.asknews.app/en/publisher) -and its [linked API reference](https://docs.asknews.app/en/reference#tag--distribution), -read with the live Python recipes on 2026-10-09. See the guide/source comparison below. +Reference: [Publisher Analytics](https://docs.asknews.app/en/publisher), including +its rendered Python Recipes 1–4, re-read **2026-10-10**, and the +[API reference](https://docs.asknews.app/en/reference#tag--distribution). -## Install and run +## Install and safe commands -Keep `distribution_report.py` and `requirements.txt` together (or enter this -example directory in a checkout): +Keep the script and requirements together; use your existing authorized local +credentials. Never put credentials in CLI arguments, source, chat or shell history. ```bash python3 -m venv .venv . .venv/bin/activate python -m pip install -r requirements.txt -python distribution_report.py --domain apnews.com --month 2026-09 --output apnews-2026-09.xlsx --include-internal --allow-partial + +# Test one week first: Sep 1 inclusive through Sep 8 exclusive, UTC. +python distribution_report.py --domain apnews.com --start 2026-09-01T00:00:00Z --stop 2026-09-08T00:00:00Z --output apnews-sep01-07.xlsx + +# All September (same precision caveats below). +python distribution_report.py --domain apnews.com --start 2026-09-01T00:00:00Z --stop 2026-10-01T00:00:00Z --output apnews-september.xlsx + +# Equivalent month shorthand; do not combine with --start or --stop. +python distribution_report.py --domain apnews.com --month 2026-09 --output apnews-month.xlsx ``` -On Windows use `.venv\Scripts\activate` instead. Output must be a new `.xlsx` -path; existing files are never overwritten. Save in a private local directory. -The temporary/output workbook uses restrictive permissions where the OS supports -POSIX modes. Filesystems must support hard links for atomic no-overwrite output. - -**Credentials:** provision the environment privately using your existing local -secret manager/session; do not put secrets in command arguments, source files, -chat, or shell history. This script reads one of: - -- `ASKNEWS_API_KEY`, passed to the SDK's `api_key` parameter. The publisher guide - uses an **existing publisher organization key** (`ank_org_...`) with - `distribution` + `news` scopes. **Personal API keys cannot access distribution**; - ownership/admin privileges do not turn a personal key into a publisher key; **or** -- `ASKNEWS_CLIENT_ID` + `ASKNEWS_CLIENT_SECRET`, passed to the SDK's OAuth client - credentials parameters. Optional `ASKNEWS_SCOPES` is a space-separated subset - of `distribution news internal`; default `distribution news`. Request only - scopes the existing client already has. The script does not grant scopes. - OAuth is an additional source-verified SDK/admin authentication option, not - the publisher guide's organization-key recipe; it still needs endpoint and - exact-domain authorization. Do not create or change credentials to run this example. - -These environment names are this example's adapter, not automatic SDK environment -loading. Do not set both authentication methods. No custom host, token URL, -organization impersonation headers, or credentials-on-CLI options are provided. -TLS verification stays enabled, redirects are disabled, and the SDK uses its -normal production API/auth hosts. All analytics requests are GET; SDK OAuth may -POST to its token endpoint. Request/response bodies, identities and credentials -are never logged. Do not enable HTTP debugging. - -`--include-internal` tries dashboard metrics with the same existing credentials. -**Admin status does not imply `internal` scope.** Omit the flag for the public -ranking/share plus news index report. A 403 stops the internal group immediately; -there is no loop probing its remaining endpoints. Public ranking 401/403 stops -the entire run without a workbook. Any 401 (including OAuth) is fatal without a -retry or credential-refresh loop. Optional-section 403s mark that section -unavailable; they are never retried or represented as zero. Use an already -suitably authorized credential or accept the documented gap—not an access change. - -### Exit codes and bounded requests - -- **0**: selected sections collected, **with calendar/snapshot coverage caveats**. - Does not mean an exact complete month or that internal sections were requested. -- **2**: a flagged partial workbook was saved with `--allow-partial`. -- **1**: fatal error or incomplete sections without `--allow-partial`; no new - workbook. Errors are deliberately sanitized. Inspect a flagged workbook's - Coverage sheet for section-specific failures, then address the identified - cause before rerunning. - -Defaults: page size **100**, at most **100 pages per metric**, **400 HTTP sends** -(including OAuth/retries), **2400 seconds** of request-start/retry budget, -30-second HTTP operation timeout, serial requests with **6 seconds before every -distribution send** (including retries and admin/internal requests) and 1 second -before other/news/auth sends. This follows guide Recipe 3 and stays below the -published **0.2 distribution requests/second** limit (burst 5, concurrency 2). -The longer bounded default budget accommodates the slower polite pagination. -A request already in flight may finish after the overall deadline; -httpx timeouts bound individual network operations, not a strict wall-clock -process deadline. Set an external process timeout if required. - -Optional bounded flags: `--page-size 1..100`, `--max-pages 1..1000`, -`--max-requests 1..4000`, `--max-seconds 1..3600`. API/transport transient failures -retry at most twice (three attempts total); retryable statuses are -408/429/500/502/503/504. `Retry-After` seconds and HTTP dates set a **reader-wide -not-before deadline**, including on the final failed attempt of a section. -Every later send (other metrics, share, internal, index or OAuth) must honor that -deadline as well as its normal pacing; a section boundary never resets it. -A wait over 60 seconds, unusable/negative Retry-After, or exhausted request/time -budget **permanently stops all further sends for the run**, not just the current -section. Remaining sections are recorded unavailable/partial without network -requests. Previously collected data can only be saved via `--allow-partial` -(exit 2); otherwise no workbook is written. No concurrency or article-by-article fanout. Index -counts cost one additional bounded request per day (28–31). Request limits -include unsuccessful sends; no unrestricted all-history queries. - -## Workbook and interpretation +Windows activation: `.venv\Scripts\activate`. Files must be new `.xlsx` paths +in a private directory. Atomic no-overwrite publication requires hard-link +support; the temporary/output file has restrictive POSIX permissions where +supported. Existing files are never overwritten, including a concurrent creator. + +Add `--include-internal` **only with already-authorized internal scope** to request +the existing optional dashboard sections. Add `--allow-partial` only if you want +a flagged workbook despite failed/missing sections (exit 2). Neither flag grants +access. The week command does not request any unrelated whole-month analytics. + +### Authentication + +Set privately via your existing local secret manager/session, choosing one: + +- `ASKNEWS_API_KEY`: the guide uses an existing **publisher organization key** + (`ank_org_...`) with `distribution` and `news` scopes. Personal keys cannot + access distribution; ownership/admin status does not change that. +- `ASKNEWS_CLIENT_ID` + `ASKNEWS_CLIENT_SECRET`: SDK OAuth alternative for an + already-authorized client. `ASKNEWS_SCOPES` is an optional space-separated + subset of `distribution news internal`, default `distribution news`. + Admin status does **not** imply internal scope. + +These environment names are utility adapters, not automatic SDK environment +loading. No credentials are created, read from other sessions, or modified. +No custom host, token URL, impersonation header or credential CLI is offered. +SDK default production hosts, TLS verification and disabled redirects apply. +Data calls are GETs; OAuth may POST to the SDK token endpoint. Errors are sanitized; +no raw bodies, credentials or HTTP debugging are emitted. + +Public ranking 401/403 aborts with no workbook. Any 401, including token exchange +or article enrichment, is fatal: no retry or SDK OAuth-refresh loop. A 403 is +never retried; a denied internal request stops that group without probing the +remaining endpoints. Optional-section denials are unavailable, never zero. + +## Range semantics and limitations + +- Require both ISO8601 calendar datetimes with seconds and explicit `Z` or + `±HH:MM` offset; normalize to **UTC before validation or any requests**. + Reject naive/date-only/invalid input, nonzero fractional seconds, stop <= start, + mixed month/range options and missing bounds. Whole-second precision is the + supported input contract; no silent rounding or expansion to a month. +- Only one UTC calendar month is supported. The inclusive integer end is + **exclusive stop minus one second** and must be in the start's UTC month. + Thus Sep 1–Oct 1 midnight works, but Sep 1–Oct 1 00:00:01 does not. + Cross-month weighted-share aggregation is deliberately not invented. +- Inclusive integer-second server filters cannot represent a truly half-open + continuous-time range: events **after stop - 1s and before stop are omitted**. + For September the API bounds are `1788220800..1790812799`; the final fractional + second is not covered. The workbook and console never claim exact completeness. +- Rankings, weighted share, internal aggregates and their daily series all use + these same selected bounds. Distribution dates are **event times**, not article + publication dates. Server daily buckets are accepted for intersecting UTC dates, + including clipped first/last days. Database bucket timezone is assumed UTC, + not live-verified. No daily query overlap or daily-share averaging is used. +- The two internal distinct routes accept **year/month only**. For partial months, + neither route is called: monthly and daily distinct cells stay blank, and + Coverage explicitly says unsupported. This expected limitation alone does not + force exit 2. No whole-month values are labelled weekly, no sum of daily uniques + is presented as range unique. Full-month requests retain both original routes + and reconcile their response fields/totals; monthly distinct remains independent. +- Index requests use **crawl_date**, not event/publication date, clipped to each + intersecting UTC day: `[clipped start, clipped stop - 1s]` inclusive. This repeats + the fractional gap at **every daily stop**, avoiding adjacent inclusive overlap. + A one-second clipped segment has equal server endpoints and cannot produce an + index bucket; no request is sent, its cell stays blank and the run is partial. + Missing/naive/shifted buckets are unavailable. Counts describe the **current + retained index**, not an immutable historical inventory. Backend hot/archive + combination or masked partial failures prevent proof of completeness. +- Enrichment looks up only the UUID union of valid rows from the selected event + rankings. `/v1/news` has no date filter: an older article used during the range + correctly retains its older publication date. Do not filter away those events. + Missing retained-index articles/metadata stay blank and never erase event counts. + +No shared snapshot exists across sections/pages. Cache TTLs, late events, +backfills and offset changes can cause drift; checks catch some, not all of it. +Pagination exhaustion does not prove immutable completeness. + +## SDK methods and honest transport fallbacks + +Pinned **asknews 0.14.6**; no SDK upgrade/release is needed for this correction. + +| Operation | Implementation and rationale | +| --- | --- | +| Article enrichment (Recipe 4) | **`sdk.news.get_articles(article_ids=batch, full_text=False)`**, one batch per <=100 unique IDs; actual named SDK method, not a locally invented SDK wrapper | +| Rankings/share (Recipes 1–3) | `sdk.client.request(...).content`, exactly the guide's authenticated transport; no named helpers in 0.14.6 | +| Internal aggregate/daily metrics | Narrow transport fallback: existing `sdk.distribution.get_domain_metrics` and `get_domain_metrics_timeseries` DTOs silently drop `grounded`/`total_grounded` | +| Internal monthly/daily distinct | Transport: no named helpers; year/month-only contract, full-month requests only | +| Index counts | Transport preserves explicit UTC offsets; named `sdk.news.get_index_counts` serializes datetimes without offsets | + +There are no duplicate named-plus-transport calls just to demonstrate methods. +Transport is fixed-allowlist SDK transport, not an independent HTTP client or +private SDK access. Named enrichment shares the same hooks, retries and budgets. +The guide's broad “traffic share” language does not expose service weighting: +this workbook calls it **weighted share**, never raw-event percentage or payout. +Rank metric selector `surface|citation|grounded` and index counts are reference/ +source-verified extras; internal dashboard routes require existing internal scope. +No assertion is made that every deployed credential has access to these routes. + +## Workbook | Sheet | Contents | | --- | --- | -| Summary | Raw event aggregates when authorized, exported article sums, distinct retrieval articles when authorized, exported indexed-day count sum, explicit coverage status | -| Daily metrics | Server-grouped retrieval/citation/grounding/full-text events when authorized; daily unique retrieval articles; separate crawl-date index counts | -| Articles | Stable article UUID, literal URL, joined retrieval/citation/grounding counts | -| Weighted share | Exact-domain server weighted fraction formatted as percent; **not payout** | -| Coverage | Availability, pagination completion/caps, failure and reconciliation notes | -| Methodology | Requested and actual bounds, UTC acquisition timestamps, SDK/API behavior and limitations | - -Text cells are explicitly literal, including strings beginning with `=`, `+`, -`-`, `@`, and URLs; no formulas, hyperlinks or external workbook links are -created. Sheets have frozen headers, filters, widths and readable count/percent -formats. Missing fields/days are **blank/unavailable**, not zero. Zero is retained -when the server returns it; an absent article metric is zero only when that -positive-event ranking has exhausted pagination. A page's `total_count` is its -**event sum**, not a domain total or article count. Rankings are independently -paginated and joined by UUID, never by title or URL; conflicting UUID URLs fail -the export. Duplicate IDs/repeated pages are not summed. An exact page boundary -may require a final empty page. Old servers without explicit `page`/`next_page` -metadata are marked partial/unavailable rather than falsely reporting their -potentially top-N-capped cohort as complete. - -The daily distinct response requires `hits` and `surfaced` on every returned -row plus both `total_hits` and `total_surfaced`. Missing fields remain blank; -missing/invalid fields or totals mark the internal group partial/unavailable and -require `--allow-partial` (exit 2), never an unflagged successful export. -`total_surfaced` reconciles the **sum of daily distinct counts only**. -Monthly distinct article counts must **not** be computed by summing daily -uniques; the independent monthly `surfaced` value is preserved. Index counts -describe current retained indexed articles filtered by -**crawl date**; they are not distribution events, publication-date activity, -or a historical inventory snapshot. Ranking endpoints do not supply titles, -publication dates, bias/sentiment or other enrichments: those are not collected -by this event-count example. The guide's Recipe 4 supports these via a separate -batched article lookup (see below); they are not unsupported API features. -This example performs no additional article enrichment fanout. Consumer identities, prompts, -queries, raw hits and billing/PII endpoints are never requested or exported. - -Only the requested exact domain is sent in filters. URLs with a different host -(including `www.` or other subdomains), embedded credentials, or nonstandard -ports cause the page to be discarded and flagged; returned share rows must match -exactly. Aggregate endpoints have no domain label in their response: isolation -there relies on their verified exact-domain filter contract. No parent-domain, -fuzzy or wildcard matching and no other publisher rows are exported. - -### Dates: important September limitation - -September 2026 is requested as `[2026-09-01T00:00:00Z, 2026-10-01T00:00:00Z)`. -Distribution queries accept integer seconds and compare both endpoints -inclusively. The script sends **1788220800 through 1790812799**, i.e. -`2026-09-30T23:59:59Z` inclusive. It **cannot include fractional-second events -after that instant**. Sending next-month midnight would both include an unwanted -instant and conflict with the public same-month restriction. Admin exemptions -are not used to pretend this is an exact half-open month. The warning is present -in every workbook and successful console output. - -Distribution dates are **event timestamps**, not article publication dates. -Daily distribution metrics use one monthly request grouped on the server, not -adjacent inclusive daily queries. Server bucket labels are treated as UTC; the -deployed database's timezone cannot be verified by this client. - -The index service's daily split can use adjacent inclusive endpoints. To avoid -double counting boundaries, this example requests each day separately from UTC -00:00:00 to 23:59:59 and validates the returned aware timestamps. Consequently -**index counts exclude the fractional remainder of EVERY day's final second**. -The retained-index backend can also combine recent hot/archive data or conceal -partial backend failures; client-side exact completeness is not established. -Missing/shifted/naive buckets are unavailable, never guessed or silently shifted. - -Requests have no shared database snapshot. Late events, cache TTLs, corrections -or backfills can move ranking offsets and change totals during acquisition. -Duplicate/page guards and ranking-vs-aggregate/daily reconciliation catch some, -not all, drift. Pagination exhaustion does not prove immutable completeness. -Weighted share reflects the server's current metric weights and publisher -multipliers for the selected period; weights/pool values are not exposed by this -response. **No revenue, payout, currency or dollar amount is inferred.** - -## Publisher guide / source cross-check - -- **Recipes 1–3 match:** `AskNewsSDK(api_key=...)`, authenticated - `sdk.client.request(...)`, `response.content`, exact domain names, same-UTC-month - integer dates, page size <=100, 1-based page/next_page, page-only event sums and - an optional empty final page. The script also guards caps/duplicates/drift. -- **Rate correction:** the guide and API configuration say **0.2 req/s**, not - 1 req/s. This example now follows the guide's six-second distribution delay. - The guide's deterministic tie ordering does not create a transactional snapshot. -- **Share wording:** the guide calls `hit_share` a share of publisher traffic. - Inspected service code applies metric weights and publisher multipliers. The - workbook therefore says **weighted share**, never unweighted event percentage - or dollars; it cannot reconstruct weights from this response. -- **Metrics not listed in the guide's parameter table:** the linked public - API reference (version **0.32.4** when checked) includes - `metric=surface|citation|grounded` on article rankings. Retrievals are `surface` - events, citations and groundings are independent event counts, not publication - counts. Internal raw models also have `full_text`, which is a separate metric. -- **Public versus internal:** the guide FAQ explicitly excludes dashboard-only - daily/query/domain lookup endpoints from the public API. The linked schema - confirms `distribution` + `internal` on the optional metric/unique routes; - neither the guide's organization-key recipe nor admin status promises access. -- **Index counts:** not one of the guide's three recipes. The linked reference - exposes `/v1/index_counts` with `news` scope, and inspected publisher-key - allowlist explicitly includes it. This source-verified extra is still - deployment/authorization-dependent; denial is recorded, not bypassed. -- **Recipe 4 is supported, but deliberately not requested:** verified SDK method - `sdk.news.get_articles(article_ids: List[str] | List[UUID], full_text=False)` - returns `List[ArticleResponse]` and GETs `/v1/news`. The guide batches at 100 IDs - for title, publication metadata, classification, sentiment and reporting voice. - This scoped event-count report does not fetch that additional metadata, entities, - summaries or article bodies, even in batches. Its article sheet is IDs/URLs/counts; - do not interpret absent metadata as an API/SDK limitation. No high-cost per-article fanout. -- **Date completeness:** the guide's examples use integer timestamps and do not - establish fractional-second month-end coverage; the source's inclusive bounds - and this report's explicit coverage warnings remain necessary. - -The guide links are also recorded as **literal text** in each workbook's -Methodology sheet. No guidance here changes existing permissions, SDK/API -functionality or production settings. - -## Verified SDK/API compatibility - -Verified SDK source: `asknews==0.14.6`, commit -`77e434f3e08aa9372aed8fb4b72c3044a677b6a6`. - -- `AskNewsSDK(api_key=...)` or `(client_id=..., client_secret=..., scopes=...)` - and its supported httpx `event_hooks` constructor kwargs provide auth, - safe errors, throttling and request budgets. -- `sdk.distribution.get_domain_metrics(domain_names, start_date, end_date)` and - `get_domain_metrics_timeseries(...)` exist. Their current Pydantic models - omit `grounded`/`total_grounded` and would silently discard these fields. -- Ranking/share/unique aggregates have no named distribution wrappers. -- `sdk.news.get_index_counts(start_datetime, end_datetime, ..., domains=...)` - exists, returning `IndexCountsResponse.root` with `start/end/count` rows, - but its formatter drops the datetime timezone offset. - -For those reasons the example consistently uses the SDK's supported -**`sdk.client.request(method="GET", endpoint=..., query=...).content`** transport -with a fixed endpoint allowlist. It retains SDK authentication and serialization; -there is no direct `requests`, `httpx` client, curl fallback, SDK/API extension, -or private client member access. `httpx` types are used only for the SDK's -supported hooks and exceptions. The example never requests an alternate server. - -| SDK transport endpoint | Scope / contract | -| --- | --- | -| `/v1/distribution/articles/top_n_for_domains` | `distribution`; exact `domain_names`, integer dates, `metric=surface/citation/grounded`, `limit<=100`, 1-based `page`; `data` article IDs/URLs/raw counts, page sum, next page | -| `/v1/distribution/stats/hit_share` | `distribution`; exact domain, same dates, `is_publisher=true`; domain weighted fractions | -| `/v1/distribution/stats/metrics` | `distribution` + `internal`; raw surfaces/citations/grounded/full_text | -| `/v1/distribution/stats/metrics_timeseries` | same internal scope; day rows and metric totals | -| `/v1/distribution/articles/domain_hits_surface` | same internal scope; year/month; raw hits and monthly distinct surfaced articles | -| `/v1/distribution/articles/domain_hits_surface_timewindow` | same internal scope; daily hits/distinct; daily distinct sum is not monthly distinct | -| `/v1/index_counts` | `news`; exact domains, aware ISO start/end, `sampling=1d`, `time_filter=crawl_date`; index bucket counts | - -These are source-verified contracts, not evidence that any particular credential -or deployed version can use them. Endpoint/schema mismatch, missing additive -fields, pagination guards and API caps are explicit partial/unavailable outcomes. +| Summary | Selected UTC bounds, raw aggregates if authorized, exported event sums, monthly distinct only where supported, sum of available clipped index windows | +| Daily metrics | Range-filtered daily events if authorized, optional full-month daily distinct, separate clipped crawl-date index counts; missing cells blank | +| Articles | UUID and literal ranking URL; joined retrieval/citation/grounding counts; **title, eng_title, pub_date, language, country, classification, sentiment, entities, keywords, reporting_voice, provocative, page_rank, key_points, bias, content_type**, enrichment status | +| Weighted share | Selected range's exact-domain server weighted fraction, not money | +| Coverage | Exhaustion/caps/errors, unsupported sections, send counts and limits | +| Methodology | Source links, precise bounds, UTC acquisition timestamps and caveats | + +Enrichment is joined by **UUID**, not return order/title/URL, and validates both +article URL host and metadata `domain_url` against the exact requested domain. +Unknown/duplicate IDs or foreign/subdomain URLs discard the entire batch; prior +valid batches survive as flagged partial data. No foreign metadata is exported. +Missing IDs mark partial; absent optional fields stay blank. List/object values +are literal JSON. No full-text bodies are fetched or exported. + +All text cells are literal—even `=`, `+`, `-`, `@` and URLs—with no formulas, +hyperlinks or external workbook links. Control characters are removed and Excel +cell limits applied. No consumer identities, prompts, raw hits, billing data or +raw response dumps are fetched/exported. One bare exact domain is sent; no fuzzy, +wildcard or parent-domain matching. Aggregate isolation relies on the verified +server domain filter because responses have no domain labels. + +Missing article metrics are zero **only after that positive-event ranking +exhausts**; otherwise blank. Duplicate ranking IDs/pages are not summed and +conflicting UUID URLs fail export. `total_count` is the **page event sum**, not a +domain total. Both daily distinct row fields and response totals are validated; +`total_surfaced` reconciles daily uniques only, never replaces monthly distinct. + +## Bounded requests and precise counts + +For a successful collection without retries: + +**Data sends = P_surface + P_citation + P_grounded + 1 + I × (2 + 2F) ++ ceil(U / 100) + D** + +- `P_metric`: pages actually needed for that ranking (including a terminal empty + page when supplied; empty rankings still cost one request). +- `1`: weighted-share call. +- `I`: 1 with `--include-internal`, else 0; `F`: 1 for a full UTC month, else 0. + Two date-range aggregates remain available for partial-month requests; the + two monthly-only distinct calls are added only for a full month. +- `U`: number of **unique validated UUIDs across all three rankings**. Zero IDs + means zero enrichment calls; batches are not repeated per metric. +- `D`: number of clipped UTC-day index windows of **at least two seconds**. + Usually 7 for the week or 30 for September; partial days count as windows. + One-second clips are skipped/flagged as described above. + +Actual HTTP sends additionally include **OAuth token sends + extra retry sends**. +Failures/denials/caps can stop sections early; this is not a promise all planned +calls will happen. With one page per metric and 1–100 unique IDs: week **12** +public / **14** internal data sends; September **35** public / **39** internal. + +Defaults: page size 100, max 100 pages **per metric**, **400 total HTTP sends**, +**2400 seconds request-start budget**, 30-second individual HTTP operation timeout. +Explicit bounded flags: `--page-size 1..100`, `--max-pages 1..1000`, +`--max-requests 1..4000`, `--max-seconds 1..3600`. +Enrichment now consumes budget too; high-UUID walks may need explicitly larger +limits even when rankings fit. No unrestricted all-history or per-article fanout. + +All calls are serial. Every distribution send waits **6 seconds** (guide limit +0.2 req/s, burst 5, concurrency 2); other/news/auth sends wait 1 second. Transient +408/429/500/502/503/504 or transport failures get at most three attempts total. +Retry-After seconds/HTTP dates establish **reader-wide not-before state**, including +on the last retry, across enrichment, index, distribution and OAuth sends. +Invalid/negative/unusable headers, waits >60 seconds or exhausted request/time +budgets permanently stop all further sends. No section can reset that state. +In-flight operations can finish after the request-start deadline; use an external +process timeout if a hard wall-clock limit is required. + +Exit **0** means collected with documented precision/snapshot/unsupported-section +caveats, not exact completeness. **2** means flagged partial output explicitly +allowed by `--allow-partial`. **1** means failure/no new workbook. Inspect Coverage +before interpreting any sums as totals; partial sums describe exported data only. ## Offline tests -From this directory, after installing requirements: - ```bash python -m pip install 'pytest>=8,<10' python -m pytest test_distribution_report.py -q ``` -Fixtures and httpx MockTransport exercise the **real SDK** without network or -real credentials. Tests cover pagination, repeated/capped pages, metric joining, -leap/year/UTC boundaries, exact-domain rejection, 401/403, retries/budgets, -missing fields, daily reconciliation, formula injection, workbook reopening, -totals, output collision and partial exit behavior. Reviewer regressions cover -cross-section/final-attempt Retry-After, unusable headers, exhausted wait/request -time budgets, missing daily distinct fields/totals, explicit partial CLI behavior, -and preserving monthly distinct counts when daily distinct counts overlap. -No real AP workbook or data -is included. Tests live with this optional example to avoid adding Excel -runtime/dev dependencies to the SDK package or changing its release workflows. +Tests forbid sockets and exercise real SDK methods with MockTransport: named +helper invocation/batches, UUID/domain validation and partial enrichment, +week/month/offset/partial-day ranges, invalid inputs before credentials/network, +monthly-only omissions, pacing/backpressure and caps including OAuth/news, +prior pagination/distinct regressions, workbook reopening, injection/no links, +missing versus zero, no false totals and atomic no-overwrite. +No live extraction or real AP workbook is included. Optional example dependencies +and tests are not added to the SDK package or its CI workflows. diff --git a/examples/distribution_report/distribution_report.py b/examples/distribution_report/distribution_report.py index b4de626..196141e 100644 --- a/examples/distribution_report/distribution_report.py +++ b/examples/distribution_report/distribution_report.py @@ -5,6 +5,7 @@ import argparse import calendar +import json import logging import math import os @@ -42,10 +43,30 @@ UNIQUE = BASE + "articles/domain_hits_surface" UNIQUE_DAILY = BASE + "articles/domain_hits_surface_timewindow" INDEX = "/v1/index_counts" +# Narrow transport fallbacks only: no ranking/share/unique helpers; internal DTOs +# discard grounded; get_index_counts strips UTC offsets. Recipe 4 uses a named helper. ALLOWED = {RANK, SHARE, TOTAL, DAILY, UNIQUE, UNIQUE_DAILY, INDEX} +ENRICH_FIELDS = ( + "title", + "eng_title", + "pub_date", + "language", + "country", + "classification", + "sentiment", + "entities", + "keywords", + "reporting_voice", + "provocative", + "page_rank", + "key_points", + "bias", + "content_type", +) BOUNDARY = ( - "PARTIAL CALENDAR COVERAGE: inclusive integer bounds end at 23:59:59 UTC; " - "fractional events after that instant are not covered. Never an exact full-month claim." + "PARTIAL CALENDAR COVERAGE: integer-second inclusive API end = exclusive stop minus 1s; " + "events after that instant and before stop are omitted. Index repeats this gap at " + "each clipped UTC-day stop. Never an exact full-range/full-month claim." ) @@ -73,6 +94,71 @@ def month_bounds(month): return start, following - timedelta(seconds=1), following +@dataclass(frozen=True) +class Window: + start: datetime + stop: datetime + + def __post_init__(self): + for value in (self.start, self.stop): + if value.tzinfo is None or value.utcoffset() is None or value.microsecond: + raise ReportError("Use timezone-aware datetimes at whole-second precision.") + object.__setattr__(self, "start", self.start.astimezone(UTC)) + object.__setattr__(self, "stop", self.stop.astimezone(UTC)) + if self.stop <= self.start: + raise ReportError("--stop must be later than --start (exclusive stop).") + if not 2000 <= self.start.year <= 2100: + raise ReportError("Range must be within 2000–2100.") + if (self.start.year, self.start.month) != (self.end.year, self.end.month): + raise ReportError("Cross-month UTC ranges are unsupported; select one month at a time.") + + @property + def end(self): + return self.stop - timedelta(seconds=1) + + @property + def full_month(self): + start, _, stop = month_bounds(self.start.strftime("%Y-%m")) + return (self.start, self.stop) == (start, stop) + + +def parse_datetime(value): + # Explicit calendar date, seconds and numeric offset/Z only, not date-only/naive input. + if not re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})", value + ): + raise ReportError("Use ISO8601 YYYY-MM-DDTHH:MM:SSZ or an explicit ±HH:MM offset.") + # datetime.fromisoformat normalizes malformed offsets such as +01:60; reject them. + if value[-1] != "Z" and (int(value[-5:-3]) > 23 or int(value[-2:]) > 59): + raise ReportError("Invalid ISO8601 timezone offset.") + try: + date = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + raise ReportError("Invalid ISO8601 calendar datetime.") from None + if date.microsecond: + raise ReportError("Fractional-second inputs are unsupported; use whole seconds explicitly.") + return date.astimezone(UTC) + + +def select_window(month=None, start=None, stop=None): + if month is not None: + if start is not None or stop is not None: + raise ReportError("--month is mutually exclusive with --start/--stop.") + first, _, following = month_bounds(month) + return Window(first, following) + if start is None or stop is None: + raise ReportError("Supply both --start and --stop, or the optional --month shorthand.") + return Window(parse_datetime(start), parse_datetime(stop)) + + +def day_windows(start, stop): + current = start + while current < stop: + following = min(current.replace(hour=0, minute=0, second=0) + timedelta(days=1), stop) + yield current, following + current = following + + def exact_domain(value): value = value.lower() if len(value) > 253 or not re.fullmatch( @@ -113,7 +199,7 @@ def retry_delay(header, attempt, now=None): class Reader: - """All data reads use SDK transport; budgets count each HTTP send (including OAuth).""" + """Shared safety for named SDK methods and narrow transport fallbacks, including OAuth.""" def __init__(self, sdk, max_requests=400, max_seconds=2400, sleep=time.sleep): self.sdk = sdk @@ -165,11 +251,20 @@ def after_response(self, response): def get(self, endpoint, query): if endpoint not in ALLOWED: raise ReportError("Endpoint not allowlisted.") + return self.read( + lambda: self.sdk.client.request(method="GET", endpoint=endpoint, query=query).content + ) + + def articles(self, ids): + # Real published SDK helper, not a local imitation; no full-text body fetch. + return self.read(lambda: self.sdk.news.get_articles(article_ids=ids, full_text=False)) + + def read(self, operation): for attempt in range(3): if self.stopped_reason is not None: self.stop(self.stopped_reason) try: - return self.sdk.client.request(method="GET", endpoint=endpoint, query=query).content + return operation() except (APIError, httpx.HTTPStatusError) as exc: status = exc.response.status_code if status == 401: @@ -262,7 +357,7 @@ def ranking(reader, query, domain, metric, page_size, max_pages): result.pages += 1 if next_page is None: result.complete = True - result.note = "pagination exhausted; snapshot and month-end caveats still apply" + result.note = "pagination exhausted; snapshot and range-end caveats still apply" return result raise ReportError("Page cap reached; exported counts are partial, not domain totals.") except (AuthError, Denied): @@ -284,7 +379,7 @@ def daily_values(payload, fields, start, following, total_prefix="total_"): if not isinstance(day, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", day): raise ReportError("Daily bucket must be a UTC calendar date.") date = datetime.strptime(day, "%Y-%m-%d").replace(tzinfo=UTC) - if not start <= date < following or day in parsed: + if not start.replace(hour=0, minute=0, second=0) <= date < following or day in parsed: raise ReportError("Duplicate/out-of-window daily bucket.") parsed[day] = {name: count(row[name]) if name in row else None for name in fields} for name in fields: @@ -299,6 +394,8 @@ def daily_values(payload, fields, start, following, total_prefix="total_"): class Report: domain: str month: str + window: Window | None = None + enrichment: dict = field(default_factory=dict) started: str = field(default_factory=lambda: utc_string(datetime.now(UTC))) finished: str = "" ranks: dict = field(default_factory=dict) @@ -311,6 +408,10 @@ class Report: coverage: list = field(default_factory=list) issues: list = field(default_factory=list) + @property + def selected(self): + return self.window or select_window(month=self.month) + def section(self, name, operation): try: issues_before = len(self.issues) @@ -367,6 +468,27 @@ def collect_internal(reader, report, query, start, following): report.issues.append("Daily metric field missing: " + name) elif report.totals[name] is not None and sum(values) != report.totals[name]: report.issues.append("Daily/aggregate mismatch: " + name) + if Window(start, following).full_month: + collect_distinct(reader, report, start, following) + else: + report.coverage.append( + ( + "Distinct retrieval counts", + "unsupported for partial month", + "Both distinct routes accept year/month only; not requested. Monthly and daily " + "distinct cells remain blank. Never sum daily uniques into range uniques.", + ) + ) + for metric, name in zip(METRICS, FIELDS): + ranked = report.ranks[metric] + if ranked.complete and report.totals[name] is not None: + if sum(row[1] for row in ranked.rows.values()) != report.totals[name]: + report.issues.append(f"{metric}: ranking/raw aggregate mismatch (snapshot drift).") + return True + + +def collect_distinct(reader, report, start, following): + # These routes have year/month ONLY, not arbitrary date bounds. month_query = {"domain_names": [report.domain], "year": start.year, "month": start.month} raw = reader.get(UNIQUE, month_query) report.unique = {name: count(raw[name]) for name in ("hits", "surfaced")} @@ -383,19 +505,17 @@ def collect_internal(reader, report, query, start, following): surface = report.ranks["surface"] if surface.complete and len(surface.rows) != report.unique["surfaced"]: report.issues.append("Monthly distinct/ranking article count mismatch (snapshot drift).") - for metric, name in zip(METRICS, FIELDS): - ranked = report.ranks[metric] - if ranked.complete and report.totals[name] is not None: - if sum(row[1] for row in ranked.rows.values()) != report.totals[name]: - report.issues.append(f"{metric}: ranking/raw aggregate mismatch (snapshot drift).") - return True def collect_index(reader, report, start, following): - day = start - while day < following: - end = day + timedelta(days=1) - timedelta(seconds=1) - # Per-day disjoint requests avoid the server's adjacent inclusive bucket overlap. + for day, stop in day_windows(start, following): + end = stop - timedelta(seconds=1) + if end == day: + # Server create_sub_ranges emits no bucket for equal endpoints. + report.issues.append("Index: one-second clipped window unsupported; left blank.") + continue + # UTC-day windows clipped to selected bounds; no adjacent inclusive overlap. + # Transport intentionally preserves UTC offsets lost by the named index helper. payload = reader.get( INDEX, { @@ -414,13 +534,13 @@ def collect_index(reader, report, start, following): if date.tzinfo is None or date.astimezone(UTC) != expected: raise ReportError("Index bucket timezone/boundaries differ; bucket discarded.") report.index[day.date().isoformat()] = count(row["count"]) - day += timedelta(days=1) return True def collect(reader, domain, month, page_size=100, max_pages=100, include_internal=False): - start, end, following = month_bounds(month) - report = Report(domain, month) + window = month if isinstance(month, Window) else select_window(month=month) + start, end, following = window.start, window.end, window.stop + report = Report(domain, start.strftime("%Y-%m"), window=window) query = { "domain_names": [domain], "start_date": int(start.timestamp()), @@ -450,6 +570,7 @@ def collect(reader, domain, month, page_size=100, max_pages=100, include_interna "Requires existing internal scope; use --include-internal", ) ) + report.section("Article enrichment", lambda: collect_enrichment(reader, report)) report.section("Index counts", lambda: collect_index(reader, report, start, following)) report.finished = utc_string(datetime.now(UTC)) return report @@ -471,6 +592,46 @@ def joined_articles(report): yield (identity, row["url"], *values) +def collect_enrichment(reader, report): + # Union ONLY the validated range-event cohorts; publication date need not be in range. + ids = [row[0] for row in joined_articles(report)] + for offset in range(0, len(ids), 100): + batch = ids[offset : offset + 100] + articles = reader.articles(batch) + if not isinstance(articles, list) or len(articles) > len(batch): + raise ReportError("Invalid enrichment batch size; batch discarded.") + parsed = {} + for article in articles: + data = article.model_dump(mode="json", exclude_unset=True) + identity, _ = parse_article(dict(data, hit_count=0), report.domain) + if ( + identity not in batch + or identity in parsed + or data.get("domain_url") != report.domain + ): + raise ReportError("Enrichment ID/domain isolation failed; batch discarded.") + parsed[identity] = {name: data.get(name) for name in ENRICH_FIELDS} + report.enrichment.update(parsed) # only publish after validating the entire batch + if len(parsed) != len(batch): + report.issues.append( + f"Article enrichment: {len(batch) - len(parsed)} requested IDs absent in batch; " + "retained-index/access gaps remain blank, not zero." + ) + return True + + +def enriched_rows(report): + for row in joined_articles(report): + data = report.enrichment.get(row[0], {}) + values = [] + for name in ENRICH_FIELDS: + value = data.get(name) + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False, sort_keys=True) + values.append(value) + yield (*row, *values, "returned" if row[0] in report.enrichment else "unavailable") + + def sheet(workbook, title, headers, rows): ws = workbook.create_sheet(title) for row in (headers, *rows): @@ -504,12 +665,14 @@ def sheet(workbook, title, headers, rows): def workbook(report): - start, end, following = month_bounds(report.month) + window = report.selected + start, end, following = window.start, window.end, window.stop book = Workbook() book.remove(book.active) summary = [ ("Domain (exact)", report.domain, "No subdomains or other publishers"), - ("Month", report.month, BOUNDARY), + ("Requested start UTC (inclusive)", utc_string(start), BOUNDARY), + ("Requested stop UTC (exclusive)", utc_string(following), "Never expanded to month"), ( "Run status", "PARTIAL" if report.issues else "COLLECTED WITH COVERAGE CAVEATS", @@ -518,7 +681,7 @@ def workbook(report): ( "Monthly unique retrieved articles", report.unique.get("surfaced"), - "Internal month distinct; never sum daily uniques", + "Full-month requests only; partial-month unsupported/blank; never sum daily uniques", ), ( "Exported surface article IDs", @@ -526,9 +689,9 @@ def workbook(report): "Exported cohort only; not indexed article inventory", ), ( - "Indexed crawl-date counts (exported days)", + "Indexed crawl-date counts (exported windows)", sum(report.index.values()) if report.index else None, - f"{len(report.index)} daily buckets; fractional-second gaps each day; " + f"{len(report.index)} clipped daily buckets; fractional-second gaps each window; " "not publication/event dates", ), ] @@ -559,8 +722,7 @@ def workbook(report): ) sheet(book, "Summary", ("Metric", "Value", "Interpretation"), summary) daily = [] - day = start - while day < following: + for day, stop in day_windows(start, following): key = day.date().isoformat() counts = report.daily.get(key, {}) unique = report.unique_daily.get(key, {}) @@ -571,10 +733,12 @@ def workbook(report): unique.get("hits"), unique.get("surfaced"), report.index.get(key), - "Missing buckets/fields blank, not zero", + "Missing buckets/fields blank, not zero; " + + utc_string(day) + + " <= requested time < " + + utc_string(stop), ) ) - day += timedelta(days=1) sheet( book, "Daily metrics", @@ -591,8 +755,16 @@ def workbook(report): sheet( book, "Articles", - ("Article ID", "URL (literal)", "Retrieval events", "Citation events", "Grounding events"), - list(joined_articles(report)), + ( + "Article ID", + "URL (literal)", + "Retrieval events", + "Citation events", + "Grounding events", + *ENRICH_FIELDS, + "Enrichment coverage", + ), + list(enriched_rows(report)), ) ws = sheet( book, @@ -635,11 +807,13 @@ def workbook(report): "reference/source support them. Named SDK internal DTOs still drop grounded.", ), ( - "Optional guide enrichment", - "Recipe 4 supports news.get_articles(article_ids=..., full_text=False) in " - "batches of 100. This bounded event-count report does not request metadata " - "enrichment, summaries or article bodies; unavailable here does not mean " - "unavailable from the API.", + "Guide article enrichment", + "Recipe 4: this report calls sdk.news.get_articles " + "(article_ids=..., full_text=False) in " + "batches of 100, joined by UUID with exact-domain validation. " + "Only IDs from selected " + "range-event rankings are enriched; older publication dates are legitimate. " + "Missing metadata/retained-index gaps stay blank. No article body fetched.", ), ( "Polite requests", @@ -671,7 +845,8 @@ def workbook(report): ), ( "Index coverage", - "One bounded request per UTC day, inclusive 00:00:00–23:59:59; " + "One bounded request per clipped UTC day, inclusive start through " + "clipped stop - 1s; " "fractional remainder excluded EVERY day to prevent overlapping " "buckets. crawl_date, not publication dates or distribution events. " "Current retained index, not historical inventory.", @@ -692,8 +867,8 @@ def workbook(report): "Metric joining", "Stable article UUID, never sum duplicate rows or join on title. " "Missing metric is zero only after that ranking exhausts; otherwise " - "blank. Titles/publication dates/enrichments not supplied by ranking " - "endpoint.", + "blank. Metadata comes from named news.get_articles, not ranking fields. " + "Missing enrichments never erase event counts or become zero.", ), ( "Isolation", @@ -703,7 +878,8 @@ def workbook(report): ), ( "SDK compatibility", - "Verified against asknews 0.14.6. SDK authenticated client.request " + "Verified against asknews 0.14.6; uses named sdk.news.get_articles for enrichment. " + "Narrow authenticated client.request " "GET transport preserves grounded fields omitted by current typed " "distribution models; ranking/share/unique lack named wrappers. Index" " transport retains UTC offsets stripped by named method.", @@ -711,7 +887,7 @@ def workbook(report): ( "Permissions", "distribution for public rankings/share; internal + distribution for " - "dashboard aggregates; news for index. Admin status does not imply " + "dashboard aggregates; news for enrichment/index. Admin status does not imply " "internal scope. No scope discovery or access changes.", ), ( @@ -775,7 +951,9 @@ def convert(text): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--domain", default="apnews.com") - parser.add_argument("--month", default="2026-09") + parser.add_argument("--month", help="Optional YYYY-MM shorthand, exclusive with --start/--stop") + parser.add_argument("--start", help="Inclusive timezone-aware ISO8601 datetime, whole seconds") + parser.add_argument("--stop", help="Exclusive timezone-aware ISO8601 datetime, whole seconds") parser.add_argument("--output", type=Path) parser.add_argument( "--include-internal", @@ -796,8 +974,11 @@ def main(argv=None): logging.disable(logging.CRITICAL) try: domain = exact_domain(args.domain) - month_bounds(args.month) - output = args.output or Path(f"{domain}-{args.month}-distribution.xlsx") + window = select_window(args.month, args.start, args.stop) + label = ( + window.start.strftime("%Y%m%dT%H%M%SZ") + "-" + window.stop.strftime("%Y%m%dT%H%M%SZ") + ) + output = args.output or Path(f"{domain}-{label}-distribution.xlsx") if output.suffix.lower() != ".xlsx" or output.exists(): raise ReportError("Choose a new .xlsx output path; existing files are not overwritten.") auth = credentials(os.environ) @@ -811,7 +992,7 @@ def main(argv=None): ) as sdk: reader.sdk = sdk report = collect( - reader, domain, args.month, args.page_size, args.max_pages, args.include_internal + reader, domain, window, args.page_size, args.max_pages, args.include_internal ) report.coverage.append( ("HTTP sends", str(reader.requests), "Includes SDK-managed OAuth and bounded retries") diff --git a/examples/distribution_report/test_distribution_report.py b/examples/distribution_report/test_distribution_report.py index 4b18e65..8388fa1 100644 --- a/examples/distribution_report/test_distribution_report.py +++ b/examples/distribution_report/test_distribution_report.py @@ -41,6 +41,31 @@ def article(n=1, value=3, url=None): } +def metadata(n=1, **overrides): + data = { + "article_id": str(UUID(int=n)), + "article_url": f"https://apnews.com/article/{n}", + "domain_url": "apnews.com", + "classification": ["World"], + "country": "US", + "source_id": "synthetic", + "page_rank": 3, + "eng_title": f"English {n}", + "title": f"Title {n}", + "entities": {"Organization": ["Synthetic"]}, + "keywords": ["test"], + "language": "en", + "pub_date": "2026-08-01T02:00:00Z", + "summary": "synthetic", + "sentiment": 0, + "key_points": ["Point"], + "reporting_voice": "Objective", + "provocative": "low", + } + data.update(overrides) + return data + + def page(rows, number=1, next_page=None): return { "data": rows, @@ -400,10 +425,12 @@ def test_end_to_end_synthetic_sdk_requests_and_metrics(tmp_path): def handler(request): q = request.url.params assert request.method == "GET" + path = request.url.path + if path == "/v1/news": + return httpx.Response(200, json=[metadata()]) assert q.get_list("domain_names") == ["apnews.com"] or q.get_list("domains") == [ "apnews.com" ] - path = request.url.path if path == m.RANK: return httpx.Response(200, json=page([article(value=3)])) if path == m.SHARE: @@ -436,7 +463,7 @@ def handler(request): reader = sdk_reader(handler) report = m.collect(reader, "apnews.com", "2026-09", include_internal=True) - assert not report.issues and reader.requests == 38 + assert not report.issues and reader.requests == 39 assert report.totals["grounded"] == 3 and len(report.index) == 30 path = tmp_path / "complete-fixture.xlsx" m.save_workbook(m.workbook(report), path) @@ -450,9 +477,9 @@ def test_partial_output_is_explicit(tmp_path, monkeypatch, capsys): monkeypatch.setattr(m, "credentials", lambda env: {"api_key": "synthetic"}) monkeypatch.setattr(m, "collect", lambda *a: report) output = tmp_path / "partial.xlsx" - assert m.main(["--output", str(output)]) == 1 + assert m.main(["--month", "2026-09", "--output", str(output)]) == 1 assert not output.exists() - assert m.main(["--output", str(output), "--allow-partial"]) == 2 + assert m.main(["--month", "2026-09", "--output", str(output), "--allow-partial"]) == 2 assert output.exists() assert "PARTIAL CALENDAR COVERAGE" in capsys.readouterr().out @@ -517,7 +544,7 @@ def test_official_guide_in_workbook_methodology(): assert rows["Official publisher guide"] == "https://docs.asknews.app/en/publisher" assert "6s" in rows["Polite requests"] assert "personal API keys cannot access distribution" in rows["Publisher authentication"] - assert "batches of 100" in rows["Optional guide enrichment"] + assert "batches of 100" in rows["Guide article enrichment"] assert "weighted" in rows["Guide/source differences"] @@ -607,7 +634,9 @@ def test_budget_expiring_during_wait_stops_before_send(monkeypatch): def distinct_fixture_handler(unique_daily): def handler(request): path, query = request.url.path, request.url.params - if path == m.RANK: + if path == "/v1/news": + payload = [metadata()] + elif path == m.RANK: payload = page([article(value=3)]) elif path == m.SHARE: payload = {"data": [{"domain": "apnews.com", "hit_share": 0.1}]} @@ -658,8 +687,8 @@ def test_missing_distinct_schema_requires_partial_cli(unique_daily, tmp_path, mo monkeypatch.setattr(m, "collect", lambda *args: report) monkeypatch.setattr(m, "credentials", lambda env: {"api_key": "synthetic-not-real"}) path = tmp_path / "requires-partial.xlsx" - assert m.main(["--output", str(path)]) == 1 and not path.exists() - assert m.main(["--output", str(path), "--allow-partial"]) == 2 + assert m.main(["--month", "2026-09", "--output", str(path)]) == 1 and not path.exists() + assert m.main(["--month", "2026-09", "--output", str(path), "--allow-partial"]) == 2 book = load_workbook(path) summary = {row[0]: row[1] for row in book["Summary"].iter_rows(min_row=2, values_only=True)} assert summary["Run status"] == "PARTIAL" @@ -688,3 +717,403 @@ def test_daily_distinct_sum_is_never_monthly_distinct(): assert not report.issues assert sum(row["surfaced"] for row in report.unique_daily.values()) == 2 assert report.unique["surfaced"] == 1 + + +@pytest.mark.parametrize( + "start,stop,days,full", + [ + ("2026-09-01T00:00:00Z", "2026-09-08T00:00:00Z", 7, False), + ("2026-09-01T00:00:00Z", "2026-10-01T00:00:00Z", 30, True), + ("2026-09-01T02:00:00+02:00", "2026-10-01T02:00:00+02:00", 30, True), + ("2026-09-01T23:00:00-02:00", "2026-09-03T02:30:00Z", 2, False), + ("2026-09-05T12:00:00Z", "2026-09-05T12:01:00Z", 1, False), + ], +) +def test_explicit_window_normalized_utc(start, stop, days, full): + window = m.select_window(start=start, stop=stop) + assert window.start.tzinfo == UTC and window.stop.tzinfo == UTC + assert window.full_month == full + assert window.end == window.stop - timedelta(seconds=1) + assert len(list(m.day_windows(window.start, window.stop))) == days + + +@pytest.mark.parametrize( + "kwargs", + [ + {}, + {"start": "2026-09-01T00:00:00Z"}, + {"stop": "2026-09-08T00:00:00Z"}, + {"month": "2026-09", "start": "2026-09-01T00:00:00Z"}, + {"month": "2026-09", "stop": "2026-09-08T00:00:00Z"}, + *[ + {"start": value, "stop": "2026-09-08T00:00:00Z"} + for value in ( + "2026-09-01", + "2026-09-01T00:00:00", + "2026-09-01 00:00:00Z", + "2026-09-01T00:00:00.1Z", + "2026-09-01T00:00:00+24:00", + "2026-09-01T00:00:00+01:60", + "2026-09-31T00:00:00Z", + "2026-09-08T00:00:00Z", + "2026-09-09T00:00:00Z", + "2026-08-31T23:59:59Z", + "1999-12-31T00:00:00Z", + ) + ], + {"start": "2026-09-01T00:00:00Z", "stop": "2026-10-01T00:00:01Z"}, + ], +) +def test_invalid_range_fails_before_credentials_or_sdk(kwargs, monkeypatch, capsys): + auth = Mock(side_effect=AssertionError("Credentials must not be read")) + sdk = Mock(side_effect=AssertionError("SDK must not be constructed")) + monkeypatch.setattr(m, "credentials", auth) + monkeypatch.setattr(m, "AskNewsSDK", sdk) + argv = [part for name, value in kwargs.items() for part in ("--" + name, value)] + assert m.main(argv) == 1 + assert auth.call_count == sdk.call_count == 0 + assert capsys.readouterr().err + + +def test_week_and_partial_day_all_endpoints_and_workbook(tmp_path): + for start, stop, days in ( + ("2026-09-01T00:00:00Z", "2026-09-08T00:00:00Z", 7), + ("2026-09-01T12:34:56Z", "2026-09-03T06:07:08Z", 3), + ): + window = m.select_window(start=start, stop=stop) + sent = [] + + def handler(request, window=window, sent=sent): + path, q = request.url.path, request.url.params + sent.append(path) + assert path not in (m.UNIQUE, m.UNIQUE_DAILY) + if path in (m.RANK, m.SHARE, m.TOTAL, m.DAILY): + assert int(q["start_date"]) == int(window.start.timestamp()) + assert int(q["end_date"]) == int(window.end.timestamp()) + assert q.get_list("domain_names") == ["apnews.com"] + if path == m.RANK: + payload = page([article()]) + elif path == m.SHARE: + payload = {"data": [{"domain": "apnews.com", "hit_share": 0.2}]} + elif path == m.TOTAL: + payload = dict.fromkeys(m.FIELDS, 3) + elif path == m.DAILY: + payload = { + "data": [dict(day="2026-09-01", **dict.fromkeys(m.FIELDS, 3))], + **{"total_" + name: 3 for name in m.FIELDS}, + } + elif path == "/v1/news": + assert q.get_list("article_ids") == [str(UUID(int=1))] + assert q["full_text"].lower() == "false" + payload = [metadata()] + else: + assert path == m.INDEX + begin, end = ( + m.parse_datetime(q["start_datetime"]), + m.parse_datetime(q["end_datetime"]), + ) + assert window.start <= begin <= end < window.stop + assert begin.date() == end.date() + payload = [{"start": q["start_datetime"], "end": q["end_datetime"], "count": 2}] + return httpx.Response(200, json=payload) + + reader = sdk_reader(handler) + try: + report = m.collect(reader, "apnews.com", window, include_internal=True) + assert not report.issues + assert reader.requests == 3 + 1 + 2 + 1 + days + finally: + reader.sdk.close() + assert not report.unique and not report.unique_daily + assert len(report.enrichment) == 1 and len(report.index) == days + assert any(row[1] == "unsupported for partial month" for row in report.coverage) + path = tmp_path / f"window-{days}.xlsx" + m.save_workbook(m.workbook(report), path) + book = load_workbook(path) + summary = {row[0]: row[1] for row in book["Summary"].iter_rows(min_row=2, values_only=True)} + assert summary["Requested start UTC (inclusive)"] == start + assert summary["Requested stop UTC (exclusive)"] == stop + assert summary["Monthly unique retrieved articles"] is None + assert summary["Indexed crawl-date counts (exported windows)"] == 2 * days + assert book["Daily metrics"].max_row == days + 1 + for row in book["Daily metrics"].iter_rows(min_row=2, values_only=True): + assert row[5] is None and row[6] is None + assert book["Daily metrics"]["B2"].value == 3 # clipped first day is accepted + values = list(book["Articles"].values) + enriched = dict(zip(values[0], values[1])) + assert enriched["pub_date"].startswith("2026-08-01") # event cohort, not pub cohort + assert enriched["title"] == "Title 1" and enriched["Retrieval events"] == 3 + + +def test_named_method_batches_uuid_join_missing_enrichment_and_injection(tmp_path, monkeypatch): + report = m.Report("apnews.com", "2026-09") + rows = {str(UUID(int=i)): (f"https://apnews.com/article/{i}", i) for i in range(1, 202)} + report.ranks = {metric: m.Ranking(rows.copy(), True, "exhausted") for metric in m.METRICS} + batches = [] + + def handler(request): + assert request.url.path == "/v1/news" and request.method == "GET" + assert request.url.params["full_text"].lower() == "false" + ids = request.url.params.get_list("article_ids") + batches.append(ids) + payload = [ + metadata(UUID(value).int, title="=1+1", keywords=["@SUM(A1)"], key_points=["-1+2"]) + for value in reversed(ids) + if UUID(value).int != 2 + ] + return httpx.Response(200, json=payload) + + reader = sdk_reader(handler) + named = Mock(wraps=reader.sdk.news.get_articles) + monkeypatch.setattr(reader.sdk.news, "get_articles", named) + try: + report.section("Article enrichment", lambda: m.collect_enrichment(reader, report)) + assert named.call_count == reader.requests == 3 + assert [len(ids) for ids in batches] == [100, 100, 1] + assert len({value for batch in batches for value in batch}) == 201 + for call, batch in zip(named.call_args_list, batches): + assert call.kwargs == {"article_ids": batch, "full_text": False} + finally: + reader.sdk.close() + assert len(report.enrichment) == 200 and report.issues + path = tmp_path / "enriched.xlsx" + m.save_workbook(m.workbook(report), path) + book = load_workbook(path) + values = list(book["Articles"].values) + data = {row[0]: dict(zip(values[0], row)) for row in values[1:]} + assert data[str(UUID(int=2))]["title"] is None + assert data[str(UUID(int=2))]["Retrieval events"] == 2 + assert data[str(UUID(int=2))]["Enrichment coverage"] == "unavailable" + assert data[str(UUID(int=101))]["eng_title"] == "English 101" + assert data[str(UUID(int=101))]["Retrieval events"] == 101 + assert data[str(UUID(int=1))]["title"] == "=1+1" + assert data[str(UUID(int=1))]["bias"] is None + assert sum(row[2] for row in values[1:]) == sum(range(1, 202)) + for ws in book: + for row in ws: + assert all(cell.data_type != "f" and not cell.hyperlink for cell in row) + assert not book._external_links + + +@pytest.mark.parametrize( + "bad", + [ + metadata(article_url="https://www.apnews.com/x"), + metadata(domain_url="other.com"), + metadata(3), + metadata(article_url="https://other.com/x"), + metadata(article_url="https://u:p@apnews.com/x"), + ], +) +def test_enrichment_domain_and_unrequested_ids_discard_whole_batch(bad): + report = report_fixture() + reader = sdk_reader(lambda req: httpx.Response(200, json=[metadata(2), bad])) + try: + report.section("Article enrichment", lambda: m.collect_enrichment(reader, report)) + finally: + reader.sdk.close() + assert report.issues and not report.enrichment + assert "other.com" not in str(report.issues) + + +def test_enrichment_duplicate_and_later_batch_failure_preserves_valid_data(): + report = report_fixture() + reader = sdk_reader(lambda req: httpx.Response(200, json=[metadata(), metadata()])) + try: + report.section("Article enrichment", lambda: m.collect_enrichment(reader, report)) + finally: + reader.sdk.close() + assert not report.enrichment and report.issues + rows = {str(UUID(int=i)): (f"https://apnews.com/article/{i}", 1) for i in range(1, 102)} + report = m.Report("apnews.com", "2026-09") + report.ranks = {metric: m.Ranking(rows, True) for metric in m.METRICS} + + def handler(request): + ids = request.url.params.get_list("article_ids") + if len(ids) == 1: + return httpx.Response(403, text="SECRET") + return httpx.Response(200, json=[metadata(UUID(value).int) for value in ids]) + + reader = sdk_reader(handler) + try: + report.section("Article enrichment", lambda: m.collect_enrichment(reader, report)) + assert reader.requests == 2 + finally: + reader.sdk.close() + assert len(report.enrichment) == 100 and report.issues + assert "SECRET" not in str(report.issues) + + +@pytest.mark.parametrize("status,error", [(401, m.AuthError), (403, m.Denied)]) +def test_named_method_auth_stops_without_retry(status, error): + reader = sdk_reader(lambda req: httpx.Response(status, text="SECRET")) + try: + with pytest.raises(error) as exc: + reader.articles([str(UUID(int=1))]) + assert reader.requests == 1 and "SECRET" not in str(exc.value) + finally: + reader.sdk.close() + + +def test_named_method_shared_backpressure_and_global_caps(monkeypatch): + clock, sent = [0.0], [] + monkeypatch.setattr(m.time, "monotonic", lambda: clock[0]) + + def handler(request): + sent.append((request.url.path, clock[0])) + return httpx.Response(429, headers={"Retry-After": "10"}) + + reader = sdk_reader(handler, max_requests=4) + reader.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + try: + with pytest.raises(m.ReportError, match="three"): + reader.articles([str(UUID(int=1))]) + with pytest.raises(m.ReportError, match="budget"): + reader.get(m.INDEX, {}) + assert [stamp for _, stamp in sent] == [1, 11, 21, 31] + with pytest.raises(m.ReportError, match="stopped"): + reader.articles([str(UUID(int=1))]) + assert len(sent) == 4 + finally: + reader.sdk.close() + + +def test_named_oauth_token_count_and_no_refresh(): + handler = Mock( + side_effect=[ + httpx.Response( + 200, json={"access_token": "synthetic", "expires_in": 3600, "token_type": "Bearer"} + ), + httpx.Response(401, text="SECRET"), + ] + ) + reader = m.Reader(None, sleep=Mock()) + with AskNewsSDK( + client_id="fake", + client_secret="fake", + scopes={"news"}, + retries=0, + transport=httpx.MockTransport(handler), + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) as sdk: + reader.sdk = sdk + with pytest.raises(m.AuthError): + reader.articles([str(UUID(int=1))]) + assert reader.requests == handler.call_count == 2 + + +def test_one_second_index_clip_is_blank_without_request(): + report = report_fixture() + window = m.select_window(start="2026-09-01T23:59:59Z", stop="2026-09-02T00:00:00Z") + reader = Fake([]) + m.collect_index(reader, report, window.start, window.stop) + assert not reader.calls and not report.index and report.issues + + +def test_empty_cohort_never_requests_enrichment(): + report = report_fixture() + report.ranks = {metric: m.Ranking({}, True) for metric in m.METRICS} + reader = SimpleNamespace(articles=Mock(side_effect=AssertionError("No batch expected"))) + assert m.collect_enrichment(reader, report) + reader.articles.assert_not_called() + + +def test_index_partial_days_exact_clips(): + window = m.select_window(start="2026-09-01T14:34:56+02:00", stop="2026-09-03T06:07:08Z") + report, calls = report_fixture(), [] + + def get(endpoint, query): + calls.append((query["start_datetime"], query["end_datetime"])) + return [{"start": query["start_datetime"], "end": query["end_datetime"], "count": 0}] + + m.collect_index(SimpleNamespace(get=get), report, window.start, window.stop) + assert calls == [ + ("2026-09-01T12:34:56Z", "2026-09-01T23:59:59Z"), + ("2026-09-02T00:00:00Z", "2026-09-02T23:59:59Z"), + ("2026-09-03T00:00:00Z", "2026-09-03T06:07:07Z"), + ] + assert list(report.index.values()) == [0, 0, 0] + + +def test_explicit_month_equals_shorthand(): + assert m.select_window(month="2026-09") == m.select_window( + start="2026-09-01T00:00:00Z", stop="2026-10-01T00:00:00Z" + ) + + +def test_enrichment_missing_schema_sanitized_partial_and_cli_gate(tmp_path, monkeypatch): + report = report_fixture() + reader = sdk_reader(lambda req: httpx.Response(200, json=[{"title": "SECRET"}])) + try: + report.section("Article enrichment", lambda: m.collect_enrichment(reader, report)) + finally: + reader.sdk.close() + assert report.issues and not report.enrichment and "SECRET" not in str(report.issues) + monkeypatch.setattr(m, "credentials", lambda env: {"api_key": "synthetic"}) + captured = [] + + def collect(reader, domain, window, *args): + captured.append(window) + report.window = window + return report + + monkeypatch.setattr(m, "collect", collect) + output = tmp_path / "missing-enrichment.xlsx" + argv = [ + "--start", + "2026-09-01T02:00:00+02:00", + "--stop", + "2026-09-08T00:00:00Z", + "--output", + str(output), + ] + assert m.main(argv) == 1 and not output.exists() + assert m.main([*argv, "--allow-partial"]) == 2 + assert captured[0].start == datetime(2026, 9, 1, tzinfo=UTC) + assert captured[0].stop == datetime(2026, 9, 8, tzinfo=UTC) + book = load_workbook(output) + assert book["Articles"]["F2"].value is None + assert book["Articles"]["C2"].value == 4 + + +@pytest.mark.parametrize("header", ["120", "not-a-date"]) +def test_named_unusable_backpressure_stops_every_send(header): + handler = Mock(return_value=httpx.Response(429, headers={"Retry-After": header})) + reader = sdk_reader(handler) + try: + with pytest.raises(m.ReportError, match="stopped"): + reader.articles([str(UUID(int=1))]) + with pytest.raises(m.ReportError, match="stopped"): + reader.get(m.RANK, {}) + assert reader.requests == handler.call_count == 1 + finally: + reader.sdk.close() + + +def test_named_time_and_oauth_request_caps(): + handler = Mock(return_value=httpx.Response(200, json=[])) + reader = sdk_reader(handler, max_seconds=1) + try: + with pytest.raises(m.ReportError, match="budget"): + reader.articles([str(UUID(int=1))]) + assert handler.call_count == reader.requests == 0 + finally: + reader.sdk.close() + handler = Mock( + return_value=httpx.Response( + 200, json={"access_token": "synthetic", "expires_in": 3600, "token_type": "Bearer"} + ) + ) + reader = m.Reader(None, max_requests=1, sleep=Mock()) + with AskNewsSDK( + client_id="fake", + client_secret="fake", + scopes={"news"}, + retries=0, + transport=httpx.MockTransport(handler), + event_hooks={"request": [reader.before_request], "response": [reader.after_response]}, + ) as sdk: + reader.sdk = sdk + with pytest.raises(m.ReportError, match="budget"): + reader.articles([str(UUID(int=1))]) + assert handler.call_count == reader.requests == 1 # token only; no unbudgeted news send