From 8f7889f4b34f06c6539eca228d92d609f148a297 Mon Sep 17 00:00:00 2001 From: lenitpx Date: Tue, 13 Oct 2026 07:56:45 -0400 Subject: [PATCH 1/3] add an explanation file to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5b6d43a6d..b835a9df9 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,4 @@ CLAUDE.md # come from prod outright. See tests/search-eval/README.md. Labeling sheets are # authoring scratch. tests/search-eval/labeling-sheet-*.md +/components/LegislatorProfile/TabComponents/billsTab/BillsTab.explanation.md From b9881fd453f1f28ccabace1da9b8d0ec64ddcbc5 Mon Sep 17 00:00:00 2001 From: lenitpx Date: Wed, 14 Oct 2026 09:35:34 -0400 Subject: [PATCH 2/3] add memberBillSummary cron job tests and skeletonize outward with Copilot --- functions/src/index.ts | 3 +- functions/src/members/index.ts | 1 + .../src/members/memberBillSummary.test.ts | 75 +++++++++++ functions/src/members/memberBillSummary.ts | 119 ++++++++++++++++++ functions/src/members/memberBillSummaryJob.ts | 73 +++++++++++ 5 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 functions/src/members/memberBillSummary.test.ts create mode 100644 functions/src/members/memberBillSummary.ts create mode 100644 functions/src/members/memberBillSummaryJob.ts diff --git a/functions/src/index.ts b/functions/src/index.ts index 0c8a0cbd3..0900b60ad 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -31,7 +31,8 @@ export { export { createMemberSearchIndex, fetchMemberBatch, - startMemberBatches + startMemberBatches, + buildMemberBillSummaries } from "./members" export { completePhoneVerificationV2, finishSignupV2 } from "./profile" export { checkSearchIndexVersion, searchHealthCheck } from "./search" diff --git a/functions/src/members/index.ts b/functions/src/members/index.ts index a8fb1b2c5..66d4f263e 100644 --- a/functions/src/members/index.ts +++ b/functions/src/members/index.ts @@ -1,2 +1,3 @@ export * from "./scrapeMembers" export * from "./createMemberSearchIndex" +export * from "./memberBillSummaryJob" diff --git a/functions/src/members/memberBillSummary.test.ts b/functions/src/members/memberBillSummary.test.ts new file mode 100644 index 000000000..2b38b0ff6 --- /dev/null +++ b/functions/src/members/memberBillSummary.test.ts @@ -0,0 +1,75 @@ +import { + buildMemberBillSummary, + MemberBillSummaryInput +} from "./memberBillSummary" + +const makeBill = ( + id: string, + primaryTopic?: string, + subtopic?: string +): MemberBillSummaryInput => ({ + id, + content: { Title: `Title ${id}` }, + topics: [primaryTopic, subtopic] + .filter((topic): topic is string => !!topic) + .map(topic => ({ topic })) +}) + +describe("buildMemberBillSummary", () => { + it("groups unique existing bills and sorts recent sponsored bills by numeric bill ID", () => { + const sponsoredIds = [ + "HD9", + "HD10", + "HD11", + "HD12", + "HD13", + "HD14", + "HD10", + "HD99" + ] + const cosponsoredIds = ["HD10", "HD15", "HD16"] + const bills = [ + makeBill("HD9", "Education", "K-12"), + makeBill("HD10", "Education", "Higher education"), + makeBill("HD11", "Education", "K-12"), + makeBill("HD12", "Healthcare", "Mental health"), + makeBill("HD13", "Education", "Higher education"), + makeBill("HD14", "Education", "K-12"), + makeBill("HD15", "Healthcare", "Mental health"), + makeBill("HD16", "Government", "Voting") + ] + + const summary = buildMemberBillSummary(sponsoredIds, cosponsoredIds, bills) + + expect(summary.all.count).toBe(8) + expect(summary.sponsored.count).toBe(6) + expect(summary.cosponsored.count).toBe(2) + expect(summary.recentSponsoredBills).toEqual([ + { id: "HD14", title: "Title HD14", topic: "K-12" }, + { id: "HD13", title: "Title HD13", topic: "Higher education" }, + { id: "HD12", title: "Title HD12", topic: "Mental health" }, + { id: "HD11", title: "Title HD11", topic: "K-12" }, + { id: "HD10", title: "Title HD10", topic: "Higher education" } + ]) + expect(summary.all.topics).toEqual([ + { + topic: "Education", + count: 5, + subtopics: [ + { topic: "K-12", count: 3 }, + { topic: "Higher education", count: 2 } + ] + }, + { + topic: "Healthcare", + count: 2, + subtopics: [{ topic: "Mental health", count: 2 }] + }, + { + topic: "Government", + count: 1, + subtopics: [{ topic: "Voting", count: 1 }] + } + ]) + }) +}) diff --git a/functions/src/members/memberBillSummary.ts b/functions/src/members/memberBillSummary.ts new file mode 100644 index 000000000..7a6197e2a --- /dev/null +++ b/functions/src/members/memberBillSummary.ts @@ -0,0 +1,119 @@ +export type BillTopicSummary = { + topic: string + count: number + subtopics: { topic: string; count: number }[] +} + +export type BillGroupSummary = { + count: number + topics: BillTopicSummary[] +} + +export type RecentSponsoredBill = { + id: string + title: string + topic: string +} + +export type MemberBillSummary = { + all: BillGroupSummary + sponsored: BillGroupSummary + cosponsored: BillGroupSummary + recentSponsoredBills: RecentSponsoredBill[] +} + +export type MemberBillSummaryInput = { + id: string + content: { Title: string } + topics?: { topic: string }[] +} + +export function buildMemberBillSummary( + sponsoredBillIds: string[], + cosponsoredBillIds: string[], + bills: MemberBillSummaryInput[] +): MemberBillSummary { + const sponsoredIds = new Set(sponsoredBillIds) + const cosponsoredIds = new Set(cosponsoredBillIds) + const billById = new Map(bills.map(bill => [bill.id, bill])) + const allIds = new Set([...sponsoredIds, ...cosponsoredIds]) + const allBills = getBills(allIds, billById) + const sponsoredBills = getBills(sponsoredIds, billById) + const cosponsoredBills = getBills( + new Set([...cosponsoredIds].filter(id => !sponsoredIds.has(id))), + billById + ) + + return { + all: summarizeGroup(allBills), + sponsored: summarizeGroup(sponsoredBills), + cosponsored: summarizeGroup(cosponsoredBills), + recentSponsoredBills: [...sponsoredBills] + .sort(compareBillIdsDescending) + .slice(0, 5) + .map(bill => ({ + id: bill.id, + title: bill.content.Title, + topic: bill.topics?.[1]?.topic ?? "" + })) + } +} + +function getBills( + ids: Set, + billsById: Map +): MemberBillSummaryInput[] { + return [...ids] + .map(id => billsById.get(id)) + .filter((bill): bill is MemberBillSummaryInput => bill !== undefined) +} + +function summarizeGroup(bills: MemberBillSummaryInput[]): BillGroupSummary { + const groups = new Map< + string, + { count: number; subtopics: Map } + >() + + for (const bill of bills) { + const topic = bill.topics?.[0]?.topic + if (!topic) continue + + const group = groups.get(topic) ?? { count: 0, subtopics: new Map() } + group.count += 1 + + const subtopic = bill.topics?.[1]?.topic + if (subtopic) { + group.subtopics.set(subtopic, (group.subtopics.get(subtopic) ?? 0) + 1) + } + + groups.set(topic, group) + } + + const topics = [...groups] + .map(([topic, group]) => ({ + topic, + count: group.count, + subtopics: [...group.subtopics] + .sort(([, countA], [, countB]) => countB - countA) + .slice(0, 4) + .map(([subtopic, count]) => ({ topic: subtopic, count })) + })) + .sort((groupA, groupB) => groupB.count - groupA.count) + .slice(0, 5) + + return { count: bills.length, topics } +} + +function compareBillIdsDescending( + billA: MemberBillSummaryInput, + billB: MemberBillSummaryInput +): number { + const numberA = Number(billA.id.match(/(\d+)$/)?.[1]) + const numberB = Number(billB.id.match(/(\d+)$/)?.[1]) + + if (Number.isFinite(numberA) && Number.isFinite(numberB)) { + return numberB - numberA || billB.id.localeCompare(billA.id) + } + + return billB.id.localeCompare(billA.id) +} diff --git a/functions/src/members/memberBillSummaryJob.ts b/functions/src/members/memberBillSummaryJob.ts new file mode 100644 index 000000000..cbb87597a --- /dev/null +++ b/functions/src/members/memberBillSummaryJob.ts @@ -0,0 +1,73 @@ +import { onSchedule } from "firebase-functions/v2/scheduler" +import { + MemberBillSummaryInput, + buildMemberBillSummary +} from "./memberBillSummary" +import { FieldValue, db } from "../firebase" +import { currentGeneralCourt } from "../shared" + +export const buildMemberBillSummaries = onSchedule( + { schedule: "every 24 hours", timeoutSeconds: 540 }, + async () => { + const court = currentGeneralCourt + const [memberSnapshot, billSnapshot] = await Promise.all([ + db + .collection(`/generalCourts/${court}/members`) + .select("content.SponsoredBills", "content.CoSponsoredBills") + .get(), + db + .collection(`/generalCourts/${court}/bills`) + .select("content.Title", "topics") + .get() + ]) + + const bills = billSnapshot.docs + .map(toBillSummaryInput) + .filter((bill): bill is MemberBillSummaryInput => bill !== null) + const writer = db.bulkWriter() + + memberSnapshot.docs.forEach(member => { + const summary = buildMemberBillSummary( + getStringArray(member.get("content.SponsoredBills")), + getStringArray(member.get("content.CoSponsoredBills")), + bills + ) + + writer.set( + db.doc(`/generalCourts/${court}/memberBillSummaries/${member.id}`), + { ...summary, generatedAt: FieldValue.serverTimestamp() } + ) + }) + + await writer.close() + } +) + +function getStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : [] +} + +function toBillSummaryInput( + snapshot: FirebaseFirestore.QueryDocumentSnapshot +): MemberBillSummaryInput | null { + const title = snapshot.get("content.Title") + if (typeof title !== "string") return null + + const rawTopics: unknown = snapshot.get("topics") + const topics = Array.isArray(rawTopics) + ? rawTopics + .filter( + (topic): topic is { topic: string } => + typeof topic?.topic === "string" + ) + .map(({ topic }) => ({ topic })) + : undefined + + return { + id: snapshot.id, + content: { Title: title }, + topics + } +} From 78008088da9559d4312bc356fac984e5c7a995fc Mon Sep 17 00:00:00 2001 From: lenitpx Date: Tue, 20 Oct 2026 20:14:11 -0400 Subject: [PATCH 3/3] revise query and tie to daily jobs --- functions/src/bills/bills.ts | 1 + functions/src/scraper.test.ts | 56 ++++++++- functions/src/scraper.ts | 201 ++++++++++++++++++++++++------- functions/src/scraperBatching.ts | 66 ++++++++++ next-env.d.ts | 2 +- 5 files changed, 281 insertions(+), 45 deletions(-) create mode 100644 functions/src/scraperBatching.ts diff --git a/functions/src/bills/bills.ts b/functions/src/bills/bills.ts index 58aeb8a71..cf6e806d7 100644 --- a/functions/src/bills/bills.ts +++ b/functions/src/bills/bills.ts @@ -21,6 +21,7 @@ export const { fetchBatch: fetchBillBatch, startBatches: startBillBatches } = startBatchSchedule: "every 3 hours", fetchBatchTimeout: 240, startBatchTimeout: 240, + trackFullRefresh: true, fetchResource: async (court: number, id: string, current) => { const { content, pdfTextExtraction } = await getDocumentWithPdfTextFallback(court, id) diff --git a/functions/src/scraper.test.ts b/functions/src/scraper.test.ts index 44499612e..44b14d5e4 100644 --- a/functions/src/scraper.test.ts +++ b/functions/src/scraper.test.ts @@ -1,4 +1,8 @@ -import { chooseBatches } from "./scraper" +import { + chooseBatches, + isRefreshCycleComplete, + selectBatchRun +} from "./scraperBatching" import { last, flattenDeep } from "lodash" const listOfNumbers = (size: number) => new Array(size).fill(undefined).map((_, i) => i.toString()) @@ -75,4 +79,54 @@ describe("chooseBatches", () => { }) expect(ids).toEqual([["c", "d", "e", "a", "b"]]) }) + + it("marks the cycle complete when a selection reaches the end of the IDs", () => { + const selection = selectBatchRun({ + ids: ["a", "b", "c", "d", "e"], + docsPerBatch: 1, + numBatches: 2, + startAfterId: "c" + }) + + expect(selection.batches).toEqual([["d"], ["e"]]) + expect(selection.nextLastId).toBe("e") + expect(selection.cycleComplete).toBe(true) + }) + + it("starts a new pass at the first ID after the previous pass reached the end", () => { + const selection = selectBatchRun({ + ids: ["a", "b", "c", "d", "e"], + docsPerBatch: 1, + numBatches: 2, + startAfterId: "e" + }) + + expect(selection.batches).toEqual([["a"], ["b"]]) + expect(selection.nextLastId).toBe("b") + expect(selection.cycleComplete).toBe(false) + }) + + it("does not complete a refresh until the cycle is closed and every batch finishes", () => { + expect( + isRefreshCycleComplete({ + cycleClosed: false, + completedBatches: 5, + expectedBatches: 5 + }) + ).toBe(false) + expect( + isRefreshCycleComplete({ + cycleClosed: true, + completedBatches: 4, + expectedBatches: 5 + }) + ).toBe(false) + expect( + isRefreshCycleComplete({ + cycleClosed: true, + completedBatches: 5, + expectedBatches: 5 + }) + ).toBe(true) + }) }) diff --git a/functions/src/scraper.ts b/functions/src/scraper.ts index 8c572aa44..f9fbd4251 100644 --- a/functions/src/scraper.ts +++ b/functions/src/scraper.ts @@ -5,11 +5,19 @@ import { onSchedule } from "firebase-functions/v2/scheduler" import { last } from "lodash" import { db, DocumentData, FieldValue, Timestamp } from "./firebase" import { currentGeneralCourt } from "./shared" +import { + chooseBatches, + isRefreshCycleComplete, + selectBatchRun +} from "./scraperBatching" + +export { chooseBatches } from "./scraperBatching" /** Batch documents trigger the batch fetch function to scrape `ids` */ type Batch = { court: number ids: string[] + cycleId?: number } /** List all ids of the resources to scrape. Falsey values will be filtered out. @@ -69,7 +77,8 @@ export function createScraper({ resourcesPerBatch, batchesPerRun, startBatchTimeout, - fetchBatchTimeout + fetchBatchTimeout, + trackFullRefresh = false }: { resourceName: string listIds: ListIds @@ -79,31 +88,40 @@ export function createScraper({ fetchBatchTimeout: number resourcesPerBatch: number batchesPerRun: number + trackFullRefresh?: boolean }) { const startBatches = onSchedule( { schedule: startBatchSchedule, timeoutSeconds: startBatchTimeout }, async () => { - const scraper = await db.doc(`/scrapers/${resourceName}`).get(), - lastId = scraper.data()?.lastId ?? "", - court = currentGeneralCourt, - ids = await listIds(court), - batchedIds = chooseBatches({ - startAfterId: lastId, + const scraperRef = db.doc(`/scrapers/${resourceName}`) + const court = currentGeneralCourt + const ids = await listIds(court) + + if (trackFullRefresh) { + await startTrackedBatchRun({ + scraperRef, + resourceName, + court, + ids, docsPerBatch: resourcesPerBatch, - numBatches: batchesPerRun, - ids + numBatches: batchesPerRun }) + return + } - const writer = db.bulkWriter(), - batches = scraper.ref.collection("batches") - - // Step the document ID for the next round of batches - await scraper.ref.set({ lastId: last(last(batchedIds)) }, { merge: true }) + const scraper = await scraperRef.get() + const batchedIds = chooseBatches({ + startAfterId: scraper.data()?.lastId ?? "", + docsPerBatch: resourcesPerBatch, + numBatches: batchesPerRun, + ids + }) + const writer = db.bulkWriter() + const batches = scraperRef.collection("batches") - // Delete old batch documents + await scraperRef.set({ lastId: last(last(batchedIds)) }, { merge: true }) await db.recursiveDelete(batches, writer) - // Trigger batch execution by writing batch documents batchedIds.forEach(ids => { const batch: Batch = { court, ids } writer.create(batches.doc(), batch) @@ -120,7 +138,8 @@ export function createScraper({ const fetchBatch = onDocumentCreated( { document: `/scrapers/${resourceName}/batches/{batchId}`, - timeoutSeconds: fetchBatchTimeout + timeoutSeconds: fetchBatchTimeout, + retry: trackFullRefresh }, async event => { const snap = event.data @@ -153,6 +172,7 @@ export function createScraper({ logger.warn( `Could not fetch resource ${resourceName}/${id}: ${e.message}` ) + if (trackFullRefresh) throw e } } else { throw e @@ -161,41 +181,136 @@ export function createScraper({ } await writer.close() + + if (trackFullRefresh && batch.cycleId !== undefined) { + await completeTrackedBatch({ + scraperRef: db.doc(`/scrapers/${resourceName}`), + batchRef: snap.ref, + completedCycleRef: db.doc( + `/scrapers/${resourceName}/completedCycles/${batch.cycleId}` + ), + cycleId: batch.cycleId, + court + }) + } } ) return { startBatches, fetchBatch } } +async function startTrackedBatchRun({ + scraperRef, + resourceName, + court, + ids, + docsPerBatch, + numBatches +}: { + scraperRef: FirebaseFirestore.DocumentReference + resourceName: string + court: number + ids: (string | undefined | null)[] + docsPerBatch: number + numBatches: number +}) { + const batchCollection = scraperRef.collection("batches") + + await db.runTransaction(async transaction => { + const scraperSnapshot = await transaction.get(scraperRef) + const state = scraperSnapshot.data() ?? {} + let cycleId = typeof state.cycleId === "number" ? state.cycleId : 1 + let expectedBatches = + typeof state.expectedBatches === "number" ? state.expectedBatches : 0 + let completedBatches = + typeof state.completedBatches === "number" ? state.completedBatches : 0 + let cycleClosed = state.cycleClosed === true + + if (cycleClosed) { + if (completedBatches < expectedBatches) return + cycleId += 1 + expectedBatches = 0 + completedBatches = 0 + cycleClosed = false + } + + const selection = selectBatchRun({ + startAfterId: typeof state.lastId === "string" ? state.lastId : "", + docsPerBatch, + numBatches, + ids + }) + if (selection.batches.length === 0) return + + selection.batches.forEach(batchIds => { + transaction.create(batchCollection.doc(), { + court, + ids: batchIds, + cycleId + }) + }) + + transaction.set( + scraperRef, + { + lastId: selection.nextLastId, + cycleId, + expectedBatches: expectedBatches + selection.batches.length, + completedBatches, + cycleClosed: selection.cycleComplete + }, + { merge: true } + ) + }) +} + +async function completeTrackedBatch({ + scraperRef, + batchRef, + completedCycleRef, + cycleId, + court +}: { + scraperRef: FirebaseFirestore.DocumentReference + batchRef: FirebaseFirestore.DocumentReference + completedCycleRef: FirebaseFirestore.DocumentReference + cycleId: number + court: number +}) { + await db.runTransaction(async transaction => { + const [batchSnapshot, scraperSnapshot] = await Promise.all([ + transaction.get(batchRef), + transaction.get(scraperRef) + ]) + if (!batchSnapshot.exists) return + + const state = scraperSnapshot.data() ?? {} + const completedBatches = + (typeof state.completedBatches === "number" + ? state.completedBatches + : 0) + 1 + const expectedBatches = + typeof state.expectedBatches === "number" ? state.expectedBatches : 0 + const cycleClosed = state.cycleClosed === true + + transaction.delete(batchRef) + transaction.set(scraperRef, { completedBatches }, { merge: true }) + + if ( + isRefreshCycleComplete({ + cycleClosed, + completedBatches, + expectedBatches + }) + ) { + transaction.create(completedCycleRef, { court, cycleId }) + } + }) +} + function missingResource(e: AxiosError) { return ( e.response?.status === 400 && !!e.response.statusText.match(/the requested .* could not be found/i) ) } - -export function chooseBatches({ - startAfterId, - docsPerBatch, - numBatches, - ids: rawIds -}: { - startAfterId: string - docsPerBatch: number - numBatches: number - ids: (string | undefined | null)[] -}): string[][] { - const allIds = rawIds.filter(Boolean).sort() as string[], - startIndex = allIds.findIndex(id => id > startAfterId), - count = Math.min(docsPerBatch * numBatches, allIds.length), - after = allIds.slice(startIndex, startIndex + count), - wrapped = allIds.slice(0, count - after.length), - ids = [...after, ...wrapped], - batchedIds = [] - - for (let start = 0; start < ids.length; start += docsPerBatch) { - batchedIds.push(ids.slice(start, start + docsPerBatch)) - } - - return batchedIds -} diff --git a/functions/src/scraperBatching.ts b/functions/src/scraperBatching.ts new file mode 100644 index 000000000..9d4f06432 --- /dev/null +++ b/functions/src/scraperBatching.ts @@ -0,0 +1,66 @@ +import { last } from "lodash" + +export function chooseBatches({ + startAfterId, + docsPerBatch, + numBatches, + ids: rawIds +}: { + startAfterId: string + docsPerBatch: number + numBatches: number + ids: (string | undefined | null)[] +}): string[][] { + return selectBatchRun({ + startAfterId, + docsPerBatch, + numBatches, + ids: rawIds + }).batches +} + +export function selectBatchRun({ + startAfterId, + docsPerBatch, + numBatches, + ids: rawIds +}: { + startAfterId: string + docsPerBatch: number + numBatches: number + ids: (string | undefined | null)[] +}) { + const allIds = [...new Set(rawIds.filter(Boolean) as string[])].sort() + const nextIndex = allIds.findIndex(id => id > startAfterId) + const startIndex = nextIndex === -1 ? 0 : nextIndex + const count = Math.min(docsPerBatch * numBatches, allIds.length) + const selected = [ + ...allIds.slice(startIndex, startIndex + count), + ...allIds.slice(0, Math.max(0, count - (allIds.length - startIndex))) + ] + const batches: string[][] = [] + + for (let start = 0; start < selected.length; start += docsPerBatch) { + batches.push(selected.slice(start, start + docsPerBatch)) + } + + return { + batches, + nextLastId: last(selected), + cycleComplete: selected.length > 0 && startIndex + count >= allIds.length + } +} + +export function isRefreshCycleComplete({ + cycleClosed, + completedBatches, + expectedBatches +}: { + cycleClosed: boolean + completedBatches: number + expectedBatches: number +}): boolean { + return ( + cycleClosed && expectedBatches > 0 && completedBatches >= expectedBatches + ) +} diff --git a/next-env.d.ts b/next-env.d.ts index a4a7b3f5c..4f11a03dc 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -2,4 +2,4 @@ /// // NOTE: This file should not be edited -// see https://nextjs.org/docs/pages/building-your-application/configuring/typescript for more information. +// see https://nextjs.org/docs/basic-features/typescript for more information.