Create Bill Summaries Query Tied To Daily Jobs

.gitignore

@@ -108,3 +108,4 @@ CLAUDE.md
 # come from prod outright. See tests/search-eval/README.md. Labeling sheets are
 # authoring scratch.
 tests/search-eval/labeling-sheet-*.md
+/components/LegislatorProfile/TabComponents/billsTab/BillsTab.explanation.md

functions/src/bills/bills.ts

@@ -21,6 +21,7 @@ export const { fetchBatch: fetchBillBatch, startBatches: startBillBatches } =
     startBatchSchedule: "every 3 hours",
     fetchBatchTimeout: 240,
     startBatchTimeout: 240,
+    trackFullRefresh: true,
     fetchResource: async (court: number, id: string, current) => {
       const { content, pdfTextExtraction } =
         await getDocumentWithPdfTextFallback(court, id)

functions/src/index.ts

@@ -31,7 +31,8 @@ export {
 export {
   createMemberSearchIndex,
   fetchMemberBatch,
-  startMemberBatches
+  startMemberBatches,
+  buildMemberBillSummaries
 } from "./members"
 export { completePhoneVerificationV2, finishSignupV2 } from "./profile"
 export { checkSearchIndexVersion, searchHealthCheck } from "./search"

functions/src/members/index.ts

@@ -1,2 +1,3 @@
 export * from "./scrapeMembers"
 export * from "./createMemberSearchIndex"
+export * from "./memberBillSummaryJob"

functions/src/members/memberBillSummary.test.ts

@@ -0,0 +1,75 @@
+import {
+  buildMemberBillSummary,
+  MemberBillSummaryInput
+} from "./memberBillSummary"
+
+const makeBill = (
+  id: string,
+  primaryTopic?: string,
+  subtopic?: string
+): MemberBillSummaryInput => ({
+  id,
+  content: { Title: `Title ${id}` },
+  topics: [primaryTopic, subtopic]
+    .filter((topic): topic is string => !!topic)
+    .map(topic => ({ topic }))
+})
+
+describe("buildMemberBillSummary", () => {
+  it("groups unique existing bills and sorts recent sponsored bills by numeric bill ID", () => {
+    const sponsoredIds = [
+      "HD9",
+      "HD10",
+      "HD11",
+      "HD12",
+      "HD13",
+      "HD14",
+      "HD10",
+      "HD99"
+    ]
+    const cosponsoredIds = ["HD10", "HD15", "HD16"]
+    const bills = [
+      makeBill("HD9", "Education", "K-12"),
+      makeBill("HD10", "Education", "Higher education"),
+      makeBill("HD11", "Education", "K-12"),
+      makeBill("HD12", "Healthcare", "Mental health"),
+      makeBill("HD13", "Education", "Higher education"),
+      makeBill("HD14", "Education", "K-12"),
+      makeBill("HD15", "Healthcare", "Mental health"),
+      makeBill("HD16", "Government", "Voting")
+    ]
+
+    const summary = buildMemberBillSummary(sponsoredIds, cosponsoredIds, bills)
+
+    expect(summary.all.count).toBe(8)
+    expect(summary.sponsored.count).toBe(6)
+    expect(summary.cosponsored.count).toBe(2)
+    expect(summary.recentSponsoredBills).toEqual([
+      { id: "HD14", title: "Title HD14", topic: "K-12" },
+      { id: "HD13", title: "Title HD13", topic: "Higher education" },
+      { id: "HD12", title: "Title HD12", topic: "Mental health" },
+      { id: "HD11", title: "Title HD11", topic: "K-12" },
+      { id: "HD10", title: "Title HD10", topic: "Higher education" }
+    ])
+    expect(summary.all.topics).toEqual([
+      {
+        topic: "Education",
+        count: 5,
+        subtopics: [
+          { topic: "K-12", count: 3 },
+          { topic: "Higher education", count: 2 }
+        ]
+      },
+      {
+        topic: "Healthcare",
+        count: 2,
+        subtopics: [{ topic: "Mental health", count: 2 }]
+      },
+      {
+        topic: "Government",
+        count: 1,
+        subtopics: [{ topic: "Voting", count: 1 }]
+      }
+    ])
+  })
+})

functions/src/members/memberBillSummary.ts

@@ -0,0 +1,119 @@
+export type BillTopicSummary = {
+  topic: string
+  count: number
+  subtopics: { topic: string; count: number }[]
+}
+
+export type BillGroupSummary = {
+  count: number
+  topics: BillTopicSummary[]
+}
+
+export type RecentSponsoredBill = {
+  id: string
+  title: string
+  topic: string
+}
+
+export type MemberBillSummary = {
+  all: BillGroupSummary
+  sponsored: BillGroupSummary
+  cosponsored: BillGroupSummary
+  recentSponsoredBills: RecentSponsoredBill[]
+}
+
+export type MemberBillSummaryInput = {
+  id: string
+  content: { Title: string }
+  topics?: { topic: string }[]
+}
+
+export function buildMemberBillSummary(
+  sponsoredBillIds: string[],
+  cosponsoredBillIds: string[],
+  bills: MemberBillSummaryInput[]
+): MemberBillSummary {
+  const sponsoredIds = new Set(sponsoredBillIds)
+  const cosponsoredIds = new Set(cosponsoredBillIds)
+  const billById = new Map(bills.map(bill => [bill.id, bill]))
+  const allIds = new Set([...sponsoredIds, ...cosponsoredIds])
+  const allBills = getBills(allIds, billById)
+  const sponsoredBills = getBills(sponsoredIds, billById)
+  const cosponsoredBills = getBills(
+    new Set([...cosponsoredIds].filter(id => !sponsoredIds.has(id))),
+    billById
+  )
+
+  return {
+    all: summarizeGroup(allBills),
+    sponsored: summarizeGroup(sponsoredBills),
+    cosponsored: summarizeGroup(cosponsoredBills),
+    recentSponsoredBills: [...sponsoredBills]
+      .sort(compareBillIdsDescending)
+      .slice(0, 5)
+      .map(bill => ({
+        id: bill.id,
+        title: bill.content.Title,
+        topic: bill.topics?.[1]?.topic ?? ""
+      }))
+  }
+}
+
+function getBills(
+  ids: Set<string>,
+  billsById: Map<string, MemberBillSummaryInput>
+): MemberBillSummaryInput[] {
+  return [...ids]
+    .map(id => billsById.get(id))
+    .filter((bill): bill is MemberBillSummaryInput => bill !== undefined)
+}
+
+function summarizeGroup(bills: MemberBillSummaryInput[]): BillGroupSummary {
+  const groups = new Map<
+    string,
+    { count: number; subtopics: Map<string, number> }
+  >()
+
+  for (const bill of bills) {
+    const topic = bill.topics?.[0]?.topic
+    if (!topic) continue
+
+    const group = groups.get(topic) ?? { count: 0, subtopics: new Map() }
+    group.count += 1
+
+    const subtopic = bill.topics?.[1]?.topic
+    if (subtopic) {
+      group.subtopics.set(subtopic, (group.subtopics.get(subtopic) ?? 0) + 1)
+    }
+
+    groups.set(topic, group)
+  }
+
+  const topics = [...groups]
+    .map(([topic, group]) => ({
+      topic,
+      count: group.count,
+      subtopics: [...group.subtopics]
+        .sort(([, countA], [, countB]) => countB - countA)
+        .slice(0, 4)
+        .map(([subtopic, count]) => ({ topic: subtopic, count }))
+    }))
+    .sort((groupA, groupB) => groupB.count - groupA.count)
+    .slice(0, 5)
+
+  return { count: bills.length, topics }
+}
+
+function compareBillIdsDescending(
+  billA: MemberBillSummaryInput,
+  billB: MemberBillSummaryInput
+): number {
+  const numberA = Number(billA.id.match(/(\d+)$/)?.[1])
+  const numberB = Number(billB.id.match(/(\d+)$/)?.[1])
+
+  if (Number.isFinite(numberA) && Number.isFinite(numberB)) {
+    return numberB - numberA || billB.id.localeCompare(billA.id)
+  }
+
+  return billB.id.localeCompare(billA.id)
+}

functions/src/members/memberBillSummaryJob.ts

@@ -0,0 +1,73 @@
+import { onSchedule } from "firebase-functions/v2/scheduler"
+import {
+  MemberBillSummaryInput,
+  buildMemberBillSummary
+} from "./memberBillSummary"
+import { FieldValue, db } from "../firebase"
+import { currentGeneralCourt } from "../shared"
+
+export const buildMemberBillSummaries = onSchedule(
+  { schedule: "every 24 hours", timeoutSeconds: 540 },
+  async () => {
+    const court = currentGeneralCourt
+    const [memberSnapshot, billSnapshot] = await Promise.all([
+      db
+        .collection(`/generalCourts/${court}/members`)
+        .select("content.SponsoredBills", "content.CoSponsoredBills")
+        .get(),
+      db
+        .collection(`/generalCourts/${court}/bills`)
+        .select("content.Title", "topics")
+        .get()
+    ])
+
+    const bills = billSnapshot.docs
+      .map(toBillSummaryInput)
+      .filter((bill): bill is MemberBillSummaryInput => bill !== null)
+    const writer = db.bulkWriter()
+
+    memberSnapshot.docs.forEach(member => {
+      const summary = buildMemberBillSummary(
+        getStringArray(member.get("content.SponsoredBills")),
+        getStringArray(member.get("content.CoSponsoredBills")),
+        bills
+      )
+
+      writer.set(
+        db.doc(`/generalCourts/${court}/memberBillSummaries/${member.id}`),
+        { ...summary, generatedAt: FieldValue.serverTimestamp() }
+      )
+    })
+
+    await writer.close()
+  }
+)
+
+function getStringArray(value: unknown): string[] {
+  return Array.isArray(value)
+    ? value.filter((item): item is string => typeof item === "string")
+    : []
+}
+
+function toBillSummaryInput(
+  snapshot: FirebaseFirestore.QueryDocumentSnapshot
+): MemberBillSummaryInput | null {
+  const title = snapshot.get("content.Title")
+  if (typeof title !== "string") return null
+
+  const rawTopics: unknown = snapshot.get("topics")
+  const topics = Array.isArray(rawTopics)
+    ? rawTopics
+        .filter(
+          (topic): topic is { topic: string } =>
+            typeof topic?.topic === "string"
+        )
+        .map(({ topic }) => ({ topic }))
+    : undefined
+
+  return {
+    id: snapshot.id,
+    content: { Title: title },
+    topics
+  }
+}

functions/src/scraper.test.ts

@@ -1,4 +1,8 @@
-import { chooseBatches } from "./scraper"
+import {
+  chooseBatches,
+  isRefreshCycleComplete,
+  selectBatchRun
+} from "./scraperBatching"
 import { last, flattenDeep } from "lodash"
 const listOfNumbers = (size: number) =>
   new Array(size).fill(undefined).map((_, i) => i.toString())
@@ -75,4 +79,54 @@ describe("chooseBatches", () => {
     })
     expect(ids).toEqual([["c", "d", "e", "a", "b"]])
   })
+
+  it("marks the cycle complete when a selection reaches the end of the IDs", () => {
+    const selection = selectBatchRun({
+      ids: ["a", "b", "c", "d", "e"],
+      docsPerBatch: 1,
+      numBatches: 2,
+      startAfterId: "c"
+    })
+
+    expect(selection.batches).toEqual([["d"], ["e"]])
+    expect(selection.nextLastId).toBe("e")
+    expect(selection.cycleComplete).toBe(true)
+  })
+
+  it("starts a new pass at the first ID after the previous pass reached the end", () => {
+    const selection = selectBatchRun({
+      ids: ["a", "b", "c", "d", "e"],
+      docsPerBatch: 1,
+      numBatches: 2,
+      startAfterId: "e"
+    })
+
+    expect(selection.batches).toEqual([["a"], ["b"]])
+    expect(selection.nextLastId).toBe("b")
+    expect(selection.cycleComplete).toBe(false)
+  })
+
+  it("does not complete a refresh until the cycle is closed and every batch finishes", () => {
+    expect(
+      isRefreshCycleComplete({
+        cycleClosed: false,
+        completedBatches: 5,
+        expectedBatches: 5
+      })
+    ).toBe(false)
+    expect(
+      isRefreshCycleComplete({
+        cycleClosed: true,
+        completedBatches: 4,
+        expectedBatches: 5
+      })
+    ).toBe(false)
+    expect(
+      isRefreshCycleComplete({
+        cycleClosed: true,
+        completedBatches: 5,
+        expectedBatches: 5
+      })
+    ).toBe(true)
+  })
 })