diff --git a/functions/src/index.ts b/functions/src/index.ts index 0c8a0cbd3..0fdf0ada7 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -63,6 +63,7 @@ export { export { scrapeElections } from "./legislators" export { transcriptionV2 } from "./webhooks" +export { scrapeHouseRollCalls, scrapeSenateRollCalls } from "./votes" export { matchOcpfMembersV2 } from "./ocpf/matchOcpfMembers" export { scrapeOcpfFinanceV2 } from "./ocpf/scrapeOcpfFinance" diff --git a/functions/src/malegislature.ts b/functions/src/malegislature.ts index df0d93f01..3ffbc473b 100644 --- a/functions/src/malegislature.ts +++ b/functions/src/malegislature.ts @@ -12,8 +12,10 @@ import { SessionContent, SpecialEventContent } from "./events/types" +import { APIDocument, APIRollCall } from "./votes/types" import { Timestamp } from "./firebase" import { MemberContent } from "./members/types" +import { RollCallSummary } from "./votes/types" /** * The MA Legislature website's SSL certificate only contains the first @@ -91,7 +93,7 @@ export async function getDocument({ }: { id: string court: number -}): Promise { +}): Promise { const response = await request({ url: `/GeneralCourts/${court}/Documents/${id}`, method: "GET", @@ -250,3 +252,25 @@ export async function getSimilarBills(court: number, id: string) { }) return SimilarBills.check(data) } + +export async function getRollCalls(court: number, branch: string) { + const data = await request({ + url: `GeneralCourts/${court}/Branches/${branch}/RollCalls`, + method: "GET", + timeout: 60_000 + }) + return Array(RollCallSummary).check(data) +} + +export async function getRollCall( + court: number, + branch: string, + number: number +): Promise { + const data = await request({ + url: `GeneralCourts/${court}/Branches/${branch}/RollCalls/${number}`, + method: "GET", + timeout: 60_000 + }) + return APIRollCall.check(data) +} diff --git a/functions/src/votes/index.ts b/functions/src/votes/index.ts new file mode 100644 index 000000000..870eb4ab1 --- /dev/null +++ b/functions/src/votes/index.ts @@ -0,0 +1,6 @@ +export { + scrapeHouseRollCalls, + scrapeSenateRollCalls, + HouseRollCallScraper, + SenateRollCallScraper +} from "./scrapers" diff --git a/functions/src/votes/scrapeRollCall.test.ts b/functions/src/votes/scrapeRollCall.test.ts new file mode 100644 index 000000000..5b851575a --- /dev/null +++ b/functions/src/votes/scrapeRollCall.test.ts @@ -0,0 +1,51 @@ +import { + getHouseRollCall, + HouseRollCallResult, + translateNames +} from "./scrapeRollCall" + +import fs from "fs" +import path from "path" + +const FIXTURES_DIR = path.resolve(__dirname, "../../../tests/fixtures/votes") + +const pdf = fs.readFileSync(path.join(FIXTURES_DIR, "HouseRollCall19.pdf")) + +global.fetch = jest.fn().mockResolvedValue({ + ok: true, + status: 200, + arrayBuffer: async () => + pdf.buffer.slice(pdf.byteOffset, pdf.byteOffset + pdf.byteLength) +}) as jest.Mock + +describe("getHouseRollCall test", () => { + it("HouseRollCall19", async () => { + const legislators_file = fs.readFileSync( + path.join(FIXTURES_DIR, "193legislators.json"), + "utf8" + ) + const legislators: [string, string][] = JSON.parse(legislators_file) + + const actualRead = await getHouseRollCall(193, 19) + const expectedRead = JSON.parse( + fs.readFileSync( + path.join(FIXTURES_DIR, "HouseRollCall19Read.json"), + "utf8" + ) + ) + expectedRead.time = new Date(expectedRead.time) + expect(actualRead).toEqual(expectedRead) + const expectedTranslation = JSON.parse( + fs.readFileSync( + path.join(FIXTURES_DIR, "HouseRollCall19Translation.json"), + "utf8" + ) + ) + const read = actualRead + const names = read.votes.map( + vote => [vote.initial, vote.lastName] as [string | null, string] + ) + const actualTranslation = translateNames(names, legislators) + expect(actualTranslation).toEqual(expectedTranslation) + }) +}) diff --git a/functions/src/votes/scrapeRollCall.ts b/functions/src/votes/scrapeRollCall.ts new file mode 100644 index 000000000..85e12b974 --- /dev/null +++ b/functions/src/votes/scrapeRollCall.ts @@ -0,0 +1,351 @@ +import pdfParse from "pdf-parse" +import { RollCallResponse } from "./types" +import { db } from "../firebase" +import { timeZone } from "../malegislature" +import { DateTime } from "luxon" + +type HouseRollCallVote = { + lastName: string + initial: string | null + vote: RollCallResponse + afterVote: boolean + highlighted: boolean +} + +function parseTime(timeStr: string): Date | string { + const time = DateTime.fromFormat(timeStr, "M/d/yyyy h:mm a", { + zone: timeZone + }) + + if (!time.isValid) { + return `Invalid date/time: ${time.invalidExplanation}` + } + + return time.toJSDate() +} + +function parseVoteLines( + lines: string[], + expectedVoteCount: number +): HouseRollCallVote[] | string { + const results = [] + let present = 0 + const votes: ("Yea" | "Nay" | "Not Voting" | "Present")[] = [] + for (const line of lines) { + if (line.match(/^[YNPX]*$/)) { + for (const char of line) { + if (char === "Y") { + votes.push("Yea") + } else if (char === "N") { + votes.push("Nay") + } else if (char === "X") { + votes.push("Not Voting") + } else if (char === "P") { + votes.push("Present") + present += 1 + } + } + } + } + if (votes.length !== expectedVoteCount + present) { + return `Expected ${expectedVoteCount} votes; found ${votes.length} votes` + } + let nextVote = 0 + let i = 0 + for (; i < lines.length; i++) { + if (nextVote >= votes.length) { + break + } + let line = lines[i] + if (line.match(/^[YNPX]*$/)) { + continue + } + let entry = { + afterVote: false, + highlighted: false + } + if (line[line.length - 1] == "*") { + entry.afterVote = true + line = line.slice(0, -1).trimEnd() + } + if (/^--.*--$/.test(line)) { + line = line.slice(2, -2).trim() + entry.highlighted = true + } + + let initial + let lastName + const nameMatch = line.match(/^([^,]*),? ([a-zA-Z])\.$/) + if (nameMatch !== null) { + initial = nameMatch[2] + lastName = nameMatch[1] + } else { + initial = null + lastName = line + } + results.push({ vote: votes[nextVote], lastName, initial, ...entry }) + nextVote += 1 + } + + return results +} + +type RollCall = { + votes: HouseRollCallVote[] + bill: string | null + question: string + yeas: number + nays: number + time: Date + nv: number +} +export type HouseRollCallResult = RollCall & { downloadUrl: string } + +function parseQuestion(number: number, input: string) { + const trim1 = "MASSACHUSETTS HOUSE OF REPRESENTATIVES" + const trim2 = "Yea and Nay" + const trim3 = `No. ${number}` + if (!input.includes(trim1)) { + return "Could not find MASSACHUSETTS HOUSE OF REPRESENTATIVES." + } + if (!input.includes(trim2)) { + return "Could not find Yea and Nay." + } + if (!input.includes(trim3)) { + return `Could not find ${trim3}.` + } + input = input.replace(trim1, " ").replace(trim2, " ").replace(trim3, " ") + + const search1 = /([0-9]+\/[0-9]+\/[0-9]+ [0-9]+:[0-9]+ (AM|PM))/ + const search2 = /([0-9]+) YEAS([0-9]+) NAYS([0-9]+) N\/V/ + const search3 = /(H|S)\. ([0-9]+)( , )?/ + + // Find all required values + const timeMatch = input.match(search1) + const voteMatch = input.match(search2) + const billMatch = input.match(search3) + + // Make sure every required value was found + if (!timeMatch) { + return "Could not find the date/time." + } + + if (!voteMatch) { + return "Could not find the Yea/Nay vote counts." + } + + // Extract the values + const timeStr = timeMatch[1] + const yeas = Number(voteMatch[1]) + const nays = Number(voteMatch[2]) + const nv = Number(voteMatch[3]) + let billNumber + if (!billMatch) { + billNumber = null + } else { + billNumber = billMatch[1] + billMatch[2] + } + + // Remove everything we've extracted + let question = input + .replace(search1, "") + .replace(search2, "") + .replace(search3, "") + .replace(/\s+/g, " ") + .trim() + + return { + question, + timeStr, + billNumber, + yeas, + nays, + nv + } +} + +function parseVotes(number: number, text: string): RollCall | string { + const lines = text + .split(/\r?\n/) + .map(line => line.trim()) + .filter(Boolean) + const untilFirstVote = lines.findIndex( + line => line.match(/^[YNPX]+$/) !== null + ) + const lastIntroLine = untilFirstVote - 2 + + const result = parseQuestion( + number, + lines.slice(0, lastIntroLine + 1).join(" ") + ) + if (typeof result === "string") { + return result + } + const { question, timeStr, billNumber, yeas, nays, nv } = result + const time = parseTime(timeStr) + if (typeof time === "string") { + return `Error interpreting time string ${timeStr}: ${time}` + } + + const results = parseVoteLines( + lines.slice(untilFirstVote - 1), + yeas + nays + nv + ) + if (typeof results === "string") { + return `Error interpreting pdf for house roll call ${number}: ${results}` + } + + return { + votes: results, + bill: billNumber, + question, + time, + yeas: Number(yeas), + nays: Number(nays), + nv: Number(nv) + } +} + +function houseDownloadUrl(court: number, number: number): string { + return `https://malegislature.gov/RollCall/${court}/HouseRollCall${number}.pdf` +} + +export function senateDownloadUrl(court: number, number: number): string { + return `https://malegislature.gov/RollCall/${court}/SenateRollCall${number}.pdf` +} + +export async function getHouseRollCall( + court: number, + number: number +): Promise { + const url = houseDownloadUrl(court, number) + const pdf = await fetch(url) + if (pdf.status === 404) { + return null + } + const result = await pdfParse(Buffer.from(await pdf.arrayBuffer())) + const votes = parseVotes(number, result.text) + if (typeof votes === "string") { + return votes + } + return { downloadUrl: url, ...votes } +} + +// Used to cross-reference legislator names to ids +// Returns a [legislator name as present on pdfs, legislator id][] | error +export async function getHouseLegislators( + court: number +): Promise<[string, string][] | string> { + const snapshot = await db.collection(`generalCourts/${court}/members`).get() + + const result = [] + for (const doc of snapshot.docs) { + const data = doc.data() + if (data.content.Branch === "House") { + // Cut off affixes (Jr., III) + const match = data.content.Name.match(/^(.*), [^\s]+$/) + let name: string = data.content.Name + if (match !== null) { + name = match[1] + } + if (data.content.LeadershipPosition === "Speaker of the House") { + name = "The Speaker" + } + result.push([name, data.id] as [string, string]) + } + } + return result +} + +function normalize(s: string): string { + return s + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[ʼʻ’‘ꞌˈ′ʹ՚]/g, "'") +} + +// Arguments: +// [first initial | null, last name][] +// [legislator name from database, legislator id][] +// Returns: +// corresponding legislator id for each item in argument 1[] | error +export function translateNames( + searching: [string | null, string][], + legislatorNames: [string, string][] +): string[] | string { + const searchMap = new Map() + let i = 0 + for (let [initial, lastName] of searching) { + lastName = normalize(lastName) + // Normalize Mr., Mrs. Speaker + if (lastName.endsWith(" Speaker")) { + lastName = "The Speaker" + } + searchMap.set(`${initial}, ${lastName}`, i) + i += 1 + } + + const translations = new Array(searching.length) + let j = 0 + for (let [name, id] of legislatorNames) { + name = normalize(name) + const initial = name[0] + for (let i = name.length - 1; i >= -1; i--) { + if (i === -1 || name[i] === " ") { + const namePart = name.slice(i + 1) + const getInitial = searchMap.get(`${initial}, ${namePart}`) + if (getInitial !== undefined) { + if (translations[getInitial] !== undefined) { + return `two legislators matched ${namePart}, ${initial}: ${translations[getInitial]}, ${id}` + } + translations[getInitial] = id + j += 1 + break + } + const getNoInitial = searchMap.get(`null, ${namePart}`) + if (getNoInitial !== undefined) { + if (translations[getNoInitial] !== undefined) { + return `two legislators matched ${namePart}: ${translations[getNoInitial]}, ${id}` + } + translations[getNoInitial] = id + j += 1 + break + } + } + } + } + if (j !== searching.length) { + const missing = [] + if (j > searching.length) { + return "multiple database legislators matched with a single rollcall legislator" + } + for (let i = 0; i < searching.length; i++) { + if (!translations[i]) { + missing.push(searching[i]) + } + } + return `couldn't find all legislators (${JSON.stringify( + missing + )} remaining)` + } + return translations +} + +export async function fetchSenateBillNumber( + url: string +): Promise { + const pdf = await fetch(url) + if (pdf.status === 404) { + return null + } + const result = await pdfParse(Buffer.from(await pdf.arrayBuffer())) + const text = result.text + text.split("\n") + for (const line of text.split("\n")) { + const match = line.match(/(S|H)\.([0-9]+)/) + if (match) { + return `${match[1]}${match[2]}` + } + } + return null +} diff --git a/functions/src/votes/scrapers.ts b/functions/src/votes/scrapers.ts new file mode 100644 index 000000000..789d119a8 --- /dev/null +++ b/functions/src/votes/scrapers.ts @@ -0,0 +1,298 @@ +import type { MemoryOption } from "firebase-functions/v2" +import { onSchedule } from "firebase-functions/v2/scheduler" +import { db } from "../firebase" +import { + fetchSenateBillNumber, + getHouseLegislators, + getHouseRollCall, + senateDownloadUrl, + translateNames +} from "./scrapeRollCall" +import * as functions from "firebase-functions" +import { currentGeneralCourt } from "../shared" +import { RollCallVote, HouseRollCall, SenateRollCall, Branch } from "./types" +import { BulkWriter, Timestamp } from "firebase-admin/firestore" +import { getRollCall } from "../malegislature" + +async function getHighestProcessedRollCall(branch: Branch): Promise { + const lastVote = await db + .collection("votes") + .where("type", "==", "RollCall") + .where("branch", "==", branch) + .where("generalCourtNumber", "==", currentGeneralCourt) + .orderBy("rollCallNumber", "desc") + .select("rollCallNumber") + .limit(1) + .get() + + // Roll call numbering starts at 1 + const number = lastVote.empty ? 0 : lastVote.docs[0].get("rollCallNumber") + return number +} + +export class SenateRollCallScraper { + private schedule + private timeout + private memory + + constructor( + schedule: string = "every 24 hours", + timeout: number = 480, + memory: MemoryOption = "256MiB" + ) { + this.schedule = schedule + this.timeout = timeout + this.memory = memory + } + + get function() { + return onSchedule( + { + schedule: this.schedule, + timeoutSeconds: this.timeout, + memory: this.memory, + maxInstances: 1 + }, + () => this.run() + ) + } + + async addRollCall( + writer: BulkWriter, + court: number, + rollCallNumber: number + ): Promise { + let rollCall + try { + rollCall = await getRollCall(court, "Senate", rollCallNumber) + } catch (e) { + return `fetch error: ${e}` + } + const pdfURL = senateDownloadUrl(court, rollCallNumber) + const bill = await fetchSenateBillNumber(pdfURL) + const rollCallRewritten: SenateRollCall = { + type: "RollCall", + generalCourtNumber: court, + branch: "Senate", + bill, + questionMotion: rollCall.QuestionMotion ?? null, + downloadUrl: + rollCall.DownloadUrl ?? + `http://malegislature.gov/api/DownloadRollCall?Branch=Senate&generalCourtNumber=${court}&rollCallNumber=${rollCallNumber}`, + rollCallNumber + } + + const docId = `senate-${court}-${rollCallNumber}` + writer.set(db.collection("votes").doc(docId), rollCallRewritten) + + for (let i = 0; i < (rollCall.Yeas ?? []).length; i++) { + const memberCode = rollCall?.Yeas?.[i]?.MemberCode + if (!memberCode) { + continue + } + const voteRewritten: RollCallVote = { + response: "Yea", + branch: "Senate", + memberCode: memberCode!, + generalCourtNumber: court, + rollCallNumber + } + writer.set( + db.collection(`votes/${docId}/vote`).doc(memberCode), + voteRewritten + ) + } + for (let i = 0; i < (rollCall.Nays ?? []).length; i++) { + const memberCode = rollCall?.Nays?.[i]?.MemberCode + if (!memberCode) { + continue + } + const voteRewritten: RollCallVote = { + response: "Nay", + branch: "Senate", + memberCode: memberCode!, + generalCourtNumber: court, + rollCallNumber + } + writer.set( + db.collection(`votes/${docId}/vote`).doc(memberCode), + voteRewritten + ) + } + // The senate PDFs, members who are listed as absent + // are listed in the section "ABSENT OR NOT VOTING", + // so absent is switch to "Not Voting" (the more general + // description) here. + for (let i = 0; i < (rollCall.Absent ?? []).length; i++) { + const memberCode = rollCall?.Absent?.[i]?.MemberCode + if (!memberCode) { + continue + } + const voteRewritten: RollCallVote = { + response: "Not Voting", + branch: "Senate", + memberCode: memberCode!, + generalCourtNumber: court, + rollCallNumber + } + writer.set( + db.collection(`votes/${docId}/vote`).doc(memberCode), + voteRewritten + ) + } + return "success" + } + + private async run() { + let rollCallNumber = (await getHighestProcessedRollCall("Senate")) + 1 + + const writer = db.bulkWriter() + + let rollcall = await this.addRollCall( + writer, + currentGeneralCourt, + rollCallNumber + ) + while (rollcall === "success") { + rollCallNumber += 1 + + rollcall = await this.addRollCall( + writer, + currentGeneralCourt, + rollCallNumber + ) + } + + await writer.close() + + if (!rollcall.startsWith("fetch error")) { + functions.logger.error( + `Error collecting house rollcall ${rollCallNumber}: ${rollcall}` + ) + } + } +} + +export class HouseRollCallScraper { + private schedule + private timeout + private memory + private legislators: [string, string][] | null + + constructor( + schedule: string = "every 24 hours", + timeout: number = 480, + memory: MemoryOption = "256MiB" + ) { + this.schedule = schedule + this.timeout = timeout + this.memory = memory + this.legislators = null + } + + get function() { + return onSchedule( + { + schedule: this.schedule, + timeoutSeconds: this.timeout, + memory: this.memory, + maxInstances: 1 + }, + () => this.run() + ) + } + + async addRollCall( + writer: BulkWriter, + court: number, + rollCallNumber: number + ): Promise { + if (!this.legislators) { + const legislators = await getHouseLegislators(court) + if (typeof legislators === "string") { + this.legislators = [] + return `error collecting legislators: ${legislators}` + } else { + if (!legislators.length) { + return `error no legislators found in database` + } + this.legislators = legislators + } + } + if (this.legislators!.length === 0) { + return "no legislators available" + } + const rollCall = await getHouseRollCall(court, rollCallNumber) + if (!rollCall) { + return "fetch error" + } + if (typeof rollCall === "string") { + return rollCall + } + const rollcallRewritten: HouseRollCall = { + type: "RollCall", + rollCallNumber, + generalCourtNumber: court, + branch: "House", + bill: rollCall.bill, + time: Timestamp.fromDate(rollCall.time), + questionMotion: rollCall.question, + downloadUrl: rollCall.downloadUrl + } + const docId = `house-${court}-${rollCallNumber}` + writer.set(db.collection("votes").doc(docId), rollcallRewritten) + const names = rollCall.votes.map( + vote => [vote.initial, vote.lastName] as [string | null, string] + ) + const ids = translateNames(names, this.legislators) + if (typeof ids === "string") { + return `matching error: ${ids}` + } + for (let i = 0; i < rollCall.votes.length; i++) { + const voteRewritten: RollCallVote = { + response: rollCall.votes[i].vote, + branch: "House", + memberCode: ids[i], + generalCourtNumber: court, + rollCallNumber: rollCallNumber + } + writer.set( + db.collection(`votes/${docId}/vote`).doc(ids[i]), + voteRewritten + ) + } + return "success" + } + + private async run() { + let rollCallNumber = (await getHighestProcessedRollCall("House")) + 1 + + const writer = db.bulkWriter() + + let rollcall = await this.addRollCall( + writer, + currentGeneralCourt, + rollCallNumber + ) + + while ((this.legislators ?? []).length && rollcall === "success") { + rollCallNumber += 1 + rollcall = await this.addRollCall( + writer, + currentGeneralCourt, + rollCallNumber + ) + } + + await writer.close() + + if (!rollcall.startsWith("fetch error")) { + functions.logger.error( + `Error collecting house rollcall ${rollCallNumber}: ${rollcall}` + ) + } + } +} + +export const scrapeHouseRollCalls = new HouseRollCallScraper().function +export const scrapeSenateRollCalls = new SenateRollCallScraper().function diff --git a/functions/src/votes/types.ts b/functions/src/votes/types.ts new file mode 100644 index 000000000..40794d77f --- /dev/null +++ b/functions/src/votes/types.ts @@ -0,0 +1,232 @@ +import { Timestamp } from "firebase-admin/firestore" +import { + Array, + Boolean, + Literal, + InstanceOf, + Number, + Optional, + Partial, + Record, + Static, + String, + Union, + Null +} from "runtypes" + +// Types in our database + +export type CommitteeVoteResponse = Static +export const CommitteeVoteResponse = Union( + Literal("Favorable"), + Literal("Adverse"), + Literal("ReserveRight"), + Literal("NoVote") +) + +export type Branch = Static +export const Branch = Union(Literal("Senate"), Literal("House")) + +export type VoteType = Static +export const VoteType = Union(Literal("RollCall"), Literal("Committee")) + +export type CommitteeVote = Static +export const CommitteeVote = Record({ + response: CommitteeVoteResponse, + memberCode: String +}) + +export type CommitteeVotes = Static +export const CommitteeVotes = Record({ + type: VoteType, + generalCourtNumber: Number, + bill: String, + committee: String +}) + +// The API states "Absent" votes, which theoretically excludes "Present" +// votes. A "Present" vote is where a member explicitly states they are present +// without voting, which can affect the quorum. +// For "Not Voting", we cannot determine whether the member was absent or whether +// they were physically present but made no statement from the roll call record. +// Since it is additional information, "Present" votes are separated from "Not Voting" +// (unknown status), but they can also be considered an example of not voting. +export type RollCallResponse = Static +export const RollCallResponse = Union( + Literal("Yea"), + Literal("Nay"), + Literal("Not Voting"), + Literal("Present") +) + +export type RollCallVote = Static +export const RollCallVote = Record({ + response: RollCallResponse, + memberCode: String, + generalCourtNumber: Number, + rollCallNumber: Number, + branch: Branch +}) + +export type SenateRollCall = Static +export const SenateRollCall = Record({ + type: VoteType, + generalCourtNumber: Number, + branch: Branch, + bill: Union(String, Null), + rollCallNumber: Number, + questionMotion: Union(String, Null), + downloadUrl: String +}) + +export type HouseRollCall = Static +export const HouseRollCall = Record({ + type: VoteType, + generalCourtNumber: Number, + branch: Branch, + time: InstanceOf(Timestamp), + bill: Union(String, Null), + rollCallNumber: Number, + questionMotion: String, + downloadUrl: String +}) + +// Types from MA Legislature API + +export type LegislativeMemberSummary = Static +export const LegislativeMemberSummary = Record({ + GeneralCourtNumber: Number, + MemberCode: Optional(String), + Details: Optional(String) +}) + +export type RollCallSummary = Static +export const RollCallSummary = Record({ + GeneralCourtNumber: Number, + Branch: Optional(String), + RollCallNumber: Number, + Details: Optional(String) +}) + +export type APIRollCall = Static +export const APIRollCall = Record({ + GeneralCourtNumber: Number, + Branch: Optional(String), + QuestionMotion: Optional(Union(String, Null)), + RollCallNumber: Number, + Yeas: Optional(Array(LegislativeMemberSummary)), + Nays: Optional(Array(LegislativeMemberSummary)), + Absent: Optional(Array(LegislativeMemberSummary)), + DownloadUrl: Optional(String) +}) + +export type FiscalAmount = Static +export const FiscalAmount = Record({ + FiscalType: Optional(String), + Amount: Optional(String) +}) + +export type CommitteeVoteRecord = Static +export const CommitteeVoteRecord = Record({ + Favorable: Optional(Array(LegislativeMemberSummary)), + Adverse: Optional(Array(LegislativeMemberSummary)), + ReserveRight: Optional(Array(LegislativeMemberSummary)), + NoVoteRecorded: Optional(Array(LegislativeMemberSummary)) +}) + +export type BillSponsor = Static +export const BillSponsor = Record({ + Id: Optional(String), + Name: Optional(String), + /** + * Type of the Bill Sponsor: + * 1 = Legislative Member + * 2 = Committee + * 3 = Public Request + * 4 = Special Request + */ + Type: Number, + /** + * Only Committees and Legislative Members will have further details. + */ + Details: Optional(String), + ResponseDate: Optional(String) +}) + +export type CommitteeVoteBill = Static +export const CommitteeVoteBill = Record({ + BillNumber: Optional(String), + DocketNumber: Optional(String), + Title: Optional(String), + PrimarySponsor: Optional(BillSponsor), + Cosponsors: Optional(Array(BillSponsor)), + JointSponsor: Optional(BillSponsor), + GeneralCourtNumber: Number, + Details: Optional(String), + IsDocketBookOnly: Boolean +}) + +export type Committee = Static +export const Committee = Record({ + CommitteeCode: Optional(String), + GeneralCourtNumber: Optional(Number), + Details: Optional(String) +}) + +export type APICommitteeVote = Static +export const APICommitteeVote = Record({ + Question: Optional(String), + Bill: Optional(CommitteeVoteBill), + Committee: Optional(Committee), + Date: String, + Vote: Optional(Array(CommitteeVoteRecord)) +}) + +export type CommitteeRecommendation = Static +export const CommitteeRecommendation = Record({ + Action: Optional(String), + FiscalAmounts: Optional(Array(FiscalAmount)), + Committee: Optional(Committee), + Votes: Optional(Array(APICommitteeVote)) +}) + +export type Attachment = Static +export const Attachment = Record({ + Description: Optional(String), + DownloadUrl: Optional(String) +}) + +export type AmendmentSummary = Static +export const AmendmentSummary = Record({ + GeneralCourtNumber: Number +}).And( + Partial({ + AmendmentNumber: String, + ParentBillNumber: String, + Branch: String, + Details: String + }) +) + +export type APIDocument = Static +export const APIDocument = Record({ + GeneralCourtNumber: Number +}).And( + Partial({ + Title: String, + BillNumber: String, + DocketNumber: String, + PrimarySponsor: BillSponsor, + Cosponsors: Array(BillSponsor), + JointSponsor: BillSponsor, + BillHistory: String, + LegislationTypeName: String, + Pinslip: String, + DocumentText: String, + EmergencyPreamble: String, + RollCalls: Array(RollCallSummary), + Attachments: Array(Attachment), + CommitteeRecommendations: Array(CommitteeRecommendation), + Amendments: Array(AmendmentSummary) + }) +) diff --git a/scripts/firebase-admin/backfillRollCalls.ts b/scripts/firebase-admin/backfillRollCalls.ts new file mode 100644 index 000000000..1c85fee58 --- /dev/null +++ b/scripts/firebase-admin/backfillRollCalls.ts @@ -0,0 +1,104 @@ +import { Record, Number, Optional, String } from "runtypes" +import { Script } from "./types" +import { + HouseRollCallScraper, + SenateRollCallScraper +} from "functions/src/votes" + +const Args = Record({ + court: Number, + endCourt: Optional(Number), + number: Optional(Number), + endNumber: Optional(Number), + branch: Optional(String) +}) + +export const script: Script = async ({ db, args }) => { + let { court, endCourt, number, endNumber, branch } = Args.check(args) + + endCourt = endCourt ?? court + if (number && !endNumber) { + endNumber = number + } + if (!number) { + number = 1 + } + const writer = db.bulkWriter() + const houseScraper = new HouseRollCallScraper() + const senateScraper = new SenateRollCallScraper() + for (; court <= endCourt; court++) { + if (!branch) { + for ( + let rollCallNumber = number; + !endNumber || rollCallNumber <= endNumber; + rollCallNumber++ + ) { + const result = await senateScraper.addRollCall( + writer, + court, + rollCallNumber + ) + if (result !== "success") { + console.log( + `Error fetching senate rollcall ${rollCallNumber} for court ${court}: ${result}` + ) + return + } + } + for ( + let rollCallNumber = number; + !endNumber || rollCallNumber <= endNumber; + rollCallNumber++ + ) { + const result = await houseScraper.addRollCall( + writer, + court, + rollCallNumber + ) + if (result !== "success") { + console.log( + `Error fetching house rollcall ${rollCallNumber} for court ${court}: ${result}` + ) + return + } + } + } else if (branch === "Senate") { + for ( + let rollCallNumber = number; + !endNumber || rollCallNumber <= endNumber; + rollCallNumber++ + ) { + const result = await senateScraper.addRollCall( + writer, + court, + rollCallNumber + ) + if (result !== "success") { + console.log( + `Error fetching senate rollcall ${rollCallNumber} for court ${court}: ${result}` + ) + return + } + } + } else if (branch === "House") { + for ( + let rollCallNumber = number; + !endNumber || rollCallNumber <= endNumber; + rollCallNumber++ + ) { + const result = await houseScraper.addRollCall( + writer, + court, + rollCallNumber + ) + if (result !== "success") { + console.log( + `Error fetching house rollcall ${rollCallNumber} for court ${court}: ${result}` + ) + return + } + } + } + } + await writer.close() +} diff --git a/tests/fixtures/votes/193legislators.json b/tests/fixtures/votes/193legislators.json new file mode 100644 index 000000000..7aef00f6a --- /dev/null +++ b/tests/fixtures/votes/193legislators.json @@ -0,0 +1,646 @@ +[ + [ + "Adrian C. Madaro", + "ACM1" + ], + [ + "Antonio F. D. Cabral", + "AFC1" + ], + [ + "Alice Hanlon Peisch", + "AHP1" + ], + [ + "Angelo J. Puppolo", + "AJP1" + ], + [ + "Adam J. Scanlon", + "AJS1" + ], + [ + "Angelo L. D'Emilia", + "ALD1" + ], + [ + "Aaron L. Saunders", + "ALS1" + ], + [ + "Ann-Margaret Ferrante", + "AMF1" + ], + [ + "Aaron Michlewitz", + "AMM1" + ], + [ + "Alyson M. Sullivan-Almeida", + "AMS2" + ], + [ + "Adrianne Pasateri Ramos", + "APR1" + ], + [ + "Andres X. Vargas", + "AXV1" + ], + [ + "Alan Silvia", + "A_S1" + ], + [ + "Brandy Fluker Oakley", + "BFO1" + ], + [ + "Bradley H. Jones", + "BHJ1" + ], + [ + "Bruce J. Ayers", + "BJA1" + ], + [ + "Bud L. Williams", + "BLW1" + ], + [ + "Brian M. Ashe", + "BMA1" + ], + [ + "Brian W. Murray", + "BWM1" + ], + [ + "Carol A. Doherty", + "CAD1" + ], + [ + "Carole A. Fiola", + "CAF1" + ], + [ + "Christopher J. Worrell", + "CJW1" + ], + [ + "Carmine Lawrence Gentile", + "CLG1" + ], + [ + "Colleen M. Garry", + "CMG1" + ], + [ + "Christopher M. Markey", + "CMM1" + ], + [ + "Christine P. Barber", + "CPB2" + ], + [ + "Christopher Richard Flanagan", + "CRF1" + ], + [ + "Carlos González", + "C_G1" + ], + [ + "Christopher Hendricks", + "C_H1" + ], + [ + "Chynah Tyler", + "C_T1" + ], + [ + "Dylan A. Fernandes", + "DAF1" + ], + [ + "David Henry Argosky LeBoeuf", + "DAL1" + ], + [ + "Danillo A. Sena", + "DAS1" + ], + [ + "Denise C. Garlick", + "DCG1" + ], + [ + "Daniel Cahill", + "DFC1" + ], + [ + "David F. DeCoste", + "DFD1" + ], + [ + "Donald H. Wong", + "DHW1" + ], + [ + "David K. Muradian", + "DKM1" + ], + [ + "Daniel M. Donahue", + "DMD1" + ], + [ + "David M. Rogers", + "DMR1" + ], + [ + "David Paul Linsky", + "DPL1" + ], + [ + "Donald R. Berthiaume", + "DRB1" + ], + [ + "Daniel R. Carey", + "DRC1" + ], + [ + "David T. Vieira", + "DTV1" + ], + [ + "Danielle W. Gregoire", + "DWG1" + ], + [ + "David Biele", + "D_B1" + ], + [ + "David Allen Robertson", + "D_R1" + ], + [ + "Dawne Shand", + "D_S1" + ], + [ + "Estela A. Reyes", + "EAR1" + ], + [ + "Edward R. Philips", + "ERP1" + ], + [ + "Edward F. Coppinger", + "E_C1" + ], + [ + "Erika Uyterhoeven", + "E_U1" + ], + [ + "Frank A. Moran", + "FAM1" + ], + [ + "Francisco E. Paulino", + "FEP1" + ], + [ + "F. Jay Barrows", + "FJB1" + ], + [ + "Gerard J. Cassidy", + "G_C2" + ], + [ + "Hannah Kane", + "HEK1" + ], + [ + "Jessica Ann Giannino", + "JAG1" + ], + [ + "Judith A. Garcia", + "JAG2" + ], + [ + "Jerald A. Parisella", + "JAP1" + ], + [ + "Jennifer Balinsky Armini", + "JBA1" + ], + [ + "James C. Arena-DeRosa", + "JCD1" + ], + [ + "Joseph D. McKenna", + "JDM1" + ], + [ + "Jonathan D. Zlotnik", + "JDZ1" + ], + [ + "John H. Rogers", + "JHR1" + ], + [ + "John J. Lawn", + "JJL2" + ], + [ + "John J. Mahoney", + "JJM1" + ], + [ + "James J. O'Day", + "JJO1" + ], + [ + "James K. Hawkins", + "JKH1" + ], + [ + "James M. Murphy", + "JMM1" + ], + [ + "Jeffrey N. Roy", + "JNR1" + ], + [ + "Jack Patrick Lewis", + "JPL1" + ], + [ + "Jeffrey Rosario Turco", + "JRT1" + ], + [ + "Josh S. Cutler", + "JSC1" + ], + [ + "James Arciero", + "J_A1" + ], + [ + "John Barrett", + "J_B1" + ], + [ + "Jay D. Livingstone", + "J_L1" + ], + [ + "Joan Meschino", + "J_M1" + ], + [ + "Jon Santiago", + "J_S2" + ], + [ + "Kip A. Diggs", + "KAD1" + ], + [ + "Kevin G. Honan", + "KGH1" + ], + [ + "Kenneth I. Gordon", + "KIG1" + ], + [ + "Kate Lipper-Garabedian", + "KLG1" + ], + [ + "Kimberly N. Ferguson", + "KNF1" + ], + [ + "Kathleen R. LaNatra", + "KPL1" + ], + [ + "Kelly W. Pease", + "KWP1" + ], + [ + "Kate Donaghue", + "K_D1" + ], + [ + "Kate Hogan", + "K_H1" + ], + [ + "Kay Khan", + "K_K1" + ], + [ + "Kristin Kassner", + "K_K2" + ], + [ + "Lenny Mirra", + "L_M1" + ], + [ + "Lindsay N. Sabadosa", + "L_S1" + ], + [ + "Marjorie C. Decker", + "MCD1" + ], + [ + "Mark J. Cusack", + "MJC1" + ], + [ + "Michael J. Finn", + "MJF1" + ], + [ + "Michael J. Moran", + "MJM1" + ], + [ + "Mathew J. Muratore", + "MJM2" + ], + [ + "Michael J. Soter", + "MJS3" + ], + [ + "Michelle M. DuBois", + "MMD1" + ], + [ + "Michael P. Kushmerek", + "MPK1" + ], + [ + "Margaret R. Scarsdale", + "MRS1" + ], + [ + "Michael S. Day", + "MSD1" + ], + [ + "Mary S. Keefe", + "MSK1" + ], + [ + "Marcus S. Vaughn", + "MSV1" + ], + [ + "Marc T. Lombardo", + "MTL1" + ], + [ + "Mike Connolly", + "M_C1" + ], + [ + "Michelle L. Ciccolo", + "M_C2" + ], + [ + "Manny Cruz", + "M_C3" + ], + [ + "Mindy Domb", + "M_D2" + ], + [ + "Meghan Kilcoyne", + "M_K1" + ], + [ + "Nicholas A. Boldyga", + "NAG1" + ], + [ + "Norman J. Orrall", + "NJO1" + ], + [ + "Natalie M. Blais", + "NMB1" + ], + [ + "Natalie M. Higgins", + "N_H1" + ], + [ + "Orlando Ramos", + "O_R1" + ], + [ + "Patricia A. Duffy", + "PAD1" + ], + [ + "Patricia A. Haddad", + "PAH1" + ], + [ + "Paul A. Schmid", + "PAS1" + ], + [ + "Paul J. Donato", + "PJD1" + ], + [ + "Peter J. Durant", + "PJD2" + ], + [ + "Patrick Joseph Kearney", + "PJK1" + ], + [ + "Paul K. Frost", + "PKF1" + ], + [ + "Peter Capano", + "PLC1" + ], + [ + "Priscila S. Sousa", + "PSS1" + ], + [ + "Paul McMurtry", + "P_M1" + ], + [ + "Rita A. Mendes", + "RAM1" + ], + [ + "Ruth B. Balser", + "RBB1" + ], + [ + "Russell E. Holmes", + "REH1" + ], + [ + "Rodney M. Elliott", + "RME1" + ], + [ + "Richard M. Haggerty", + "RMH1" + ], + [ + "Ryan M. Hamilton", + "RMH2" + ], + [ + "Rob Consalvo", + "R_C1" + ], + [ + "The Speaker", + "R_M1" + ], + [ + "Rady Mom", + "R_M2" + ], + [ + "Shirley B. Arriaga", + "SBA1" + ], + [ + "Steven C. Owens", + "SCO1" + ], + [ + "Steven G. Xiarhos", + "SGX1" + ], + [ + "Sarah K. Peake", + "SKP1" + ], + [ + "Susannah M. Whipps", + "SLG1" + ], + [ + "Sally P. Kerans", + "SPK1" + ], + [ + "Steven S. Howitt", + "SSH1" + ], + [ + "Susan Williams Gifford", + "SWG1" + ], + [ + "Simon Cataldo", + "S_C1" + ], + [ + "Sean Garballey", + "S_G1" + ], + [ + "Steven Ultrino", + "S_G2" + ], + [ + "Samantha Montaño", + "S_M1" + ], + [ + "Tricia Farley-Bouvier", + "TFB1" + ], + [ + "Thomas P. Walsh", + "TJW1" + ], + [ + "Thomas M. Stanley", + "TMS1" + ], + [ + "Todd M. Smola", + "TMS2" + ], + [ + "Tram T. Nguyen", + "TTN1" + ], + [ + "Tackey Chan", + "T_C1" + ], + [ + "Tommy Vitolo", + "T_V1" + ], + [ + "Vanna Howard", + "V_H1" + ], + [ + "William C. Galvin", + "WCG1" + ], + [ + "William J. Driscoll", + "WJD1" + ], + [ + "William M. Straus", + "WMS1" + ], + [ + "Smitty Pignatelli", + "WSP1" + ], + [ + "Daniel J. Hunt", + "djh1" + ], + [ + "Daniel J. Ryan", + "djr1" + ], + [ + "Joseph W. McGonagle", + "jwm1" + ] +] \ No newline at end of file diff --git a/tests/fixtures/votes/HouseRollCall19.pdf b/tests/fixtures/votes/HouseRollCall19.pdf new file mode 100644 index 000000000..534f84838 Binary files /dev/null and b/tests/fixtures/votes/HouseRollCall19.pdf differ diff --git a/tests/fixtures/votes/HouseRollCall19Read.json b/tests/fixtures/votes/HouseRollCall19Read.json new file mode 100644 index 000000000..fc18bf643 --- /dev/null +++ b/tests/fixtures/votes/HouseRollCall19Read.json @@ -0,0 +1,1117 @@ +{ + "downloadUrl": "https://malegislature.gov/RollCall/193/HouseRollCall19.pdf", + "votes": [ + { + "vote": "Yea", + "lastName": "Mr. Speaker", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Duffy", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Linsky", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Sousa", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Moran", + "initial": "M", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Elliott", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Lipper-Garabedian", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Stanley", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Hogan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Farley-Bouvier", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Livingstone", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Straus", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Peisch", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Fernandes", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Madaro", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Turco", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Moran", + "initial": "F", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ferrante", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Mahoney", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Tyler", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Peake", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Finn", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Markey", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ultrino", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Arciero", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Fiola", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "McGonagle", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Uyterhoeven", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Arena-DeRosa", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Flanagan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "McMurtry", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Vargas", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Armini", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Fluker Oakley", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Mendes", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Vitolo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Arriaga", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Galvin", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Meschino", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Walsh", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ashe", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Garballey", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Michlewitz", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Williams", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ayers", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Garcia", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Mom", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Worrell", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Balser", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Garlick", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Montano", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Zlotnik", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Barber", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Garry", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Murphy", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Jones", + "initial": null, + "afterVote": false, + "highlighted": true + }, + { + "vote": "Yea", + "lastName": "Barrett", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Gentile", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Murray", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ferguson", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Biele", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Giannino", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Nguyen", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Frost", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Blais", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Gonzalez", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "O’Day", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Gifford", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cabral", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Gordon", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Owens", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Barrows", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cahill", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Gregoire", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Parisella", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Berthiaume", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Capano", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Haddad", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Paulino", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Boldyga", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Carey", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Haggerty", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Philips", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "D'Emilia", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cassidy", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Hamilton", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Pignatelli", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "DeCoste", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cataldo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Hawkins", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Puppolo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Durant", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Chan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Hendricks", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ramos", + "initial": "A", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Howitt", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ciccolo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Higgins", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ramos", + "initial": "O", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kane", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Connolly", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Holmes", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Reyes", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Lombardo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Consalvo", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Honan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Robertson", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "McKenna", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cruz", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Howard", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Rogers", + "initial": "D", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Muradian", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cusack", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Hunt", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Rogers", + "initial": "J", + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Muratore", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Cutler", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kassner", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Roy", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Orrall", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Day", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kearney", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Ryan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Pease", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Decker", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Keefe", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Sabadosa", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Smola", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Diggs", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kerans", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Saunders", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Soter", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Doherty", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Khan", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Scanlon", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Sullivan-Almeida", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Domb", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kilcoyne", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Scarsdale", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Vaughn", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Donaghue", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Kushmerek", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Schmid", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Vieira", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Donahue", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "LaNatra", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Sena", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Wong", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Donato", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Lawn", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Shand", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Xiarhos", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Driscoll", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "LeBoeuf", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Silvia", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Whipps", + "initial": null, + "afterVote": false, + "highlighted": true + }, + { + "vote": "Yea", + "lastName": "DuBois", + "initial": null, + "afterVote": false, + "highlighted": false + }, + { + "vote": "Yea", + "lastName": "Lewis", + "initial": null, + "afterVote": false, + "highlighted": false + } + ], + "bill": "H3900", + "question": "Consol. \"B\" H.H.S. and Elder Affairs", + "time": "2023-04-25T01:06:00.000Z", + "yeas": 158, + "nays": 0, + "nv": 0 +} \ No newline at end of file diff --git a/tests/fixtures/votes/HouseRollCall19Translation.json b/tests/fixtures/votes/HouseRollCall19Translation.json new file mode 100644 index 000000000..ac88bc4ef --- /dev/null +++ b/tests/fixtures/votes/HouseRollCall19Translation.json @@ -0,0 +1,160 @@ +[ + "R_M1", + "PAD1", + "DPL1", + "PSS1", + "MJM1", + "RME1", + "KLG1", + "TMS1", + "K_H1", + "TFB1", + "J_L1", + "WMS1", + "AHP1", + "DAF1", + "ACM1", + "JRT1", + "FAM1", + "AMF1", + "JJM1", + "C_T1", + "SKP1", + "MJF1", + "CMM1", + "S_G2", + "J_A1", + "CAF1", + "jwm1", + "E_U1", + "JCD1", + "CRF1", + "P_M1", + "AXV1", + "JBA1", + "BFO1", + "RAM1", + "T_V1", + "SBA1", + "WCG1", + "J_M1", + "TJW1", + "BMA1", + "S_G1", + "AMM1", + "BLW1", + "BJA1", + "JAG2", + "R_M2", + "CJW1", + "RBB1", + "DCG1", + "S_M1", + "JDZ1", + "CPB2", + "CMG1", + "JMM1", + "BHJ1", + "J_B1", + "CLG1", + "BWM1", + "KNF1", + "D_B1", + "JAG1", + "TTN1", + "PKF1", + "NMB1", + "C_G1", + "JJO1", + "SWG1", + "AFC1", + "KIG1", + "SCO1", + "FJB1", + "DFC1", + "DWG1", + "JAP1", + "DRB1", + "PLC1", + "PAH1", + "FEP1", + "NAG1", + "DRC1", + "RMH1", + "ERP1", + "ALD1", + "G_C2", + "RMH2", + "WSP1", + "DFD1", + "S_C1", + "JKH1", + "AJP1", + "PJD2", + "T_C1", + "C_H1", + "APR1", + "SSH1", + "M_C2", + "N_H1", + "O_R1", + "HEK1", + "M_C1", + "REH1", + "EAR1", + "MTL1", + "R_C1", + "KGH1", + "D_R1", + "JDM1", + "M_C3", + "V_H1", + "DMR1", + "DKM1", + "MJC1", + "djh1", + "JHR1", + "MJM2", + "JSC1", + "K_K2", + "JNR1", + "NJO1", + "MSD1", + "PJK1", + "djr1", + "KWP1", + "MCD1", + "MSK1", + "L_S1", + "TMS2", + "KAD1", + "SPK1", + "ALS1", + "MJS3", + "CAD1", + "K_K1", + "AJS1", + "AMS2", + "M_D2", + "M_K1", + "MRS1", + "MSV1", + "K_D1", + "MPK1", + "PAS1", + "DTV1", + "DMD1", + "KPL1", + "DAS1", + "DHW1", + "PJD1", + "JJL2", + "D_S1", + "SGX1", + "WJD1", + "DAL1", + "A_S1", + "SLG1", + "MMD1", + "JPL1" +]