From 43bebdf7a7c0c2b6981cbcb8f6e389ce67421b95 Mon Sep 17 00:00:00 2001 From: shigahi Date: Fri, 25 Sep 2026 19:35:36 +0200 Subject: [PATCH 1/3] Add rendering regression check scripts/render-check.mjs generates a worker for a public Notion test page, runs it with wrangler dev and checks database views, images, PDF and file blocks, Mermaid, anchor links, database items and browser history in Chromium. Each check also runs against notion.site, and only failures that happen through Worknot alone are reported. The daily monitor workflow runs it and opens an issue on failure. Refresh the Notion baseline for client 23.13.20260925.1117, which the check passes. Closes #149 Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/notion-monitor.yml | 70 +++++++ README.md | 2 + baselines/notion-snapshot.json | 4 +- package-lock.json | 30 +++ package.json | 4 +- scripts/render-check.mjs | 287 +++++++++++++++++++++++++++ 6 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 scripts/render-check.mjs diff --git a/.github/workflows/notion-monitor.yml b/.github/workflows/notion-monitor.yml index d9397b1..deaaf3e 100644 --- a/.github/workflows/notion-monitor.yml +++ b/.github/workflows/notion-monitor.yml @@ -82,3 +82,73 @@ jobs: }); console.log('Created new issue'); } + + render-check: + name: Render Check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: "npm" + + - name: Install dependencies + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Run render check + id: render + run: | + npm run check:render 2>&1 | tee render-output.txt + exit "${PIPESTATUS[0]}" + continue-on-error: true + + - name: Create or update issue if rendering broke + if: steps.render.outcome == 'failure' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const output = fs.readFileSync('render-output.txt', 'utf8'); + const date = new Date().toISOString().slice(0, 10); + const body = [ + '## Rendering regression detected', + '', + 'The daily render check (`npm run check:render`) found features that work on notion.site but break through a generated Worknot worker.', + '', + '```', + output.split('\n').filter((line) => /^(PASS|FAIL|NOTION|SKIP)/.test(line)).join('\n'), + '```', + '', + `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + ].join('\n'); + + const existing = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: 'render-regression', + state: 'open', + per_page: 1 + }); + + if (existing.data.length > 0) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.data[0].number, + body: `### Update ${date}\n\n` + body + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Rendering regression detected - ${date}`, + body: body, + labels: ['render-regression'] + }); + } diff --git a/README.md b/README.md index 0dac76f..d6121f5 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,8 @@ Worknot includes automated monitoring for Notion client-side changes that could When changes are detected, an issue is automatically created. See `scripts/notion-monitor.sh` for details. +The same workflow runs a rendering check (`npm run check:render`, see `scripts/render-check.mjs`). It generates a worker for a public [test page](https://succinct-scar-f20.notion.site/3e6558d7de1781bb89a9fb6adc4012d6), runs it with `wrangler dev`, and loads the page in Chromium. It checks database views, images, PDF and file blocks, Mermaid, anchor links, database items and browser history. Every check also runs against notion.site, and only failures that happen through Worknot alone are reported. To run it locally, run `npx playwright install chromium` first. + ## License MIT diff --git a/baselines/notion-snapshot.json b/baselines/notion-snapshot.json index 515c450..da41ea7 100644 --- a/baselines/notion-snapshot.json +++ b/baselines/notion-snapshot.json @@ -1,6 +1,6 @@ { - "notion_version": "23.13.20260925.0853", - "app_js_filename": "app-ab692f49bf1f3e24.js", + "notion_version": "23.13.20260925.1117", + "app_js_filename": "app-601ea34bf19afcb2.js", "asset_js_count": 1, "public_domain_name_count": 2, "notion_site_literal_count": 2, diff --git a/package-lock.json b/package-lock.json index 26ec260..4c83947 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@types/react": "^19.2.18", "@types/react-dom": "^19.3.0", "@vitejs/plugin-react": "^6.1.1", + "playwright": "^1.63.0", "typescript": "^7.0.2", "vite": "^8.3.0", "wrangler": "^4.131.2" @@ -3270,6 +3271,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.28", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", diff --git a/package.json b/package.json index b45331c..d816f89 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@types/react": "^19.2.18", "@types/react-dom": "^19.3.0", "@vitejs/plugin-react": "^6.1.1", + "playwright": "^1.63.0", "typescript": "^7.0.2", "vite": "^8.3.0", "wrangler": "^4.131.2" @@ -33,6 +34,7 @@ "build": "vite build", "preview": "vite preview", "dev": "wrangler dev", - "deploy": "wrangler deploy" + "deploy": "wrangler deploy", + "check:render": "node scripts/render-check.mjs" } } diff --git a/scripts/render-check.mjs b/scripts/render-check.mjs new file mode 100644 index 0000000..1031da8 --- /dev/null +++ b/scripts/render-check.mjs @@ -0,0 +1,287 @@ +// Rendering regression check for the generated worker (#149). +// Generates a worker for the public Notion test page, runs it with `wrangler dev` +// and loads the page through it in Chromium. Every check also runs against the +// page on notion.site, so failures that Notion itself has are reported separately. +// +// Usage: npm run check:render (RUNS=3 to retry flaky Notion responses more) +// Requires Chromium for Playwright: npx playwright install chromium + +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { chromium } from "playwright"; +import { createServer } from "vite"; + +const DOMAIN = "render-check.example.com"; +const NOTION_SITE = "https://succinct-scar-f20.notion.site"; +const ROOT_PAGE = "3e6558d7de1781bb89a9fb6adc4012d6"; +const SUBPAGE = "3e6558d7de17817da2dcd0227d6062e0"; +const DB_ITEM = "3e6558d7de1781258e84de9616a91c0e"; +const PORT = Number(process.env.PORT || 8799); +const RUNS = Number(process.env.RUNS || 2); + +const DB_VIEWS = ["table", "board", "gallery", "list", "calendar", "timeline"]; + +const TARGETS = { + worker: { + origin: `https://${DOMAIN}`, + root: "/", + isRoot: (p) => p === "/", + isSubpage: (p) => p === "/subpage", + }, + notion: { + origin: NOTION_SITE, + root: `/${ROOT_PAGE}`, + isRoot: (p) => p.endsWith(ROOT_PAGE), + isSubpage: (p) => p.endsWith(SUBPAGE), + }, +}; + +async function generateWorker(dir) { + const vite = await createServer({ + server: { middlewareMode: true }, + appType: "custom", + logLevel: "error", + }); + try { + const { default: code } = await vite.ssrLoadModule("/src/code.ts"); + const script = code({ + myDomain: DOMAIN, + notionUrl: `${NOTION_SITE}/${ROOT_PAGE}`, + slugs: [["subpage", `${NOTION_SITE}/${SUBPAGE}`]], + pageTitle: "", + pageDescription: "", + googleFont: "", + customScript: "", + customCss: "", + optionImage: {}, + pageMetadata: {}, + structuredData: { enabled: false, schemaType: "WebPage" }, + branding: {}, + socialPreview: {}, + seo: {}, + analytics: {}, + caching: { enabled: false }, + customHtml: {}, + custom404: {}, + subdomainRedirects: [], + redirectRules: [], + rss: { enabled: false }, + i18n: { enabled: false, defaultLocale: "en" }, + ogImageGeneration: { enabled: false }, + }); + // wrangler dev listens on a local port; keep upstream fetches on the default port + const patched = script.replace( + "let url = new URL(request.url);", + "let url = new URL(request.url); url.port = '';", + ); + if (patched === script) throw new Error("Could not patch the worker for local use"); + fs.writeFileSync(path.join(dir, "worker.js"), patched); + fs.writeFileSync( + path.join(dir, "wrangler.jsonc"), + JSON.stringify({ name: "worknot-render-check", main: "worker.js", compatibility_date: "2026-03-11" }), + ); + } finally { + await vite.close(); + } +} + +function startWrangler(dir) { + const bin = path.resolve("node_modules/.bin/wrangler"); + const proc = spawn(bin, ["dev", "--local-protocol", "https", "--port", String(PORT)], { + cwd: dir, + env: { ...process.env, WRANGLER_SEND_METRICS: "false" }, + }); + return new Promise((resolve, reject) => { + let log = ""; + const timer = setTimeout(() => reject(new Error(`wrangler dev did not start:\n${log}`)), 90000); + const onData = (data) => { + log += data; + if (log.includes("Ready on")) { + clearTimeout(timer); + resolve(proc); + } + }; + proc.stdout.on("data", onData); + proc.stderr.on("data", onData); + proc.on("exit", (code) => reject(new Error(`wrangler dev exited (${code}):\n${log}`))); + }); +} + +async function openPage(browser, target) { + const context = await browser.newContext({ + ignoreHTTPSErrors: true, + serviceWorkers: "block", + viewport: { width: 1280, height: 1000 }, + }); + const failures = []; + const origin = new URL(target.origin).host; + context.on("response", (response) => { + const url = new URL(response.url()); + // 401s are expected for account-only endpoints on public pages + if (url.host === origin && response.status() >= 400 && response.status() !== 401) { + failures.push(`${response.status()} ${url.pathname}`); + } + }); + await context.route(`https://${DOMAIN}/**`, async (route) => { + const request = route.request(); + const url = new URL(request.url()); + try { + const response = await route.fetch({ + url: `https://127.0.0.1:${PORT}${url.pathname}${url.search}`, + headers: request.headers(), + maxRedirects: 0, + }); + await route.fulfill({ response }); + } catch { + await route.abort().catch(() => {}); + } + }); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message.slice(0, 200))); + await page.goto(target.origin + target.root, { waitUntil: "domcontentloaded", timeout: 60000 }); + await page + .waitForFunction(() => document.querySelectorAll(".notion-collection_view-block").length > 0, null, { + timeout: 40000, + }) + .catch(() => {}); + await page.waitForTimeout(4000); + return { context, page, errors, failures, target }; +} + +const state = (page) => + page.evaluate(() => ({ + host: location.host, + path: location.pathname, + h1: document.querySelector("h1")?.innerText.trim(), + text: document.body.innerText, + })); + +// Each check returns an error message, or null when it passes. +const CHECKS = { + "page renders": async ({ page }) => { + const s = await state(page); + if (s.text.includes("couldn’t be found")) return "Notion shows 'page not found'"; + return s.h1 === "Worknot Render Check" ? null : `unexpected heading: ${s.h1}`; + }, + "database views": async ({ page }) => { + const missing = []; + for (const view of DB_VIEWS) { + if (!(await page.$(`.notion-${view}-view`))) missing.push(view); + } + return missing.length ? `missing views: ${missing.join(", ")}` : null; + }, + "database rows and formula": async ({ page }) => { + const text = (await state(page)).text; + return ["Item Alpha", "Item Beta", "Item Gamma", "40", "60"].every((t) => text.includes(t)) + ? null + : "rows or formula values are missing"; + }, + "images load": async ({ page }) => { + const broken = await page.evaluate(() => + [...document.querySelectorAll("img")] + .filter((img) => img.getAttribute("src")?.startsWith("/image/") && !(img.complete && img.naturalWidth > 0)) + .map((img) => img.getAttribute("src").slice(0, 80)), + ); + const count = await page.$$eval("img[src^='/image/']", (imgs) => imgs.length); + if (count < 2) return `expected cover and uploaded image, found ${count}`; + return broken.length ? `broken images: ${broken.join(", ")}` : null; + }, + "pdf and file blocks": async ({ page }) => { + if (!(await page.$(".notion-pdf-block"))) return "PDF block missing"; + if (!(await page.$(".notion-file-block"))) return "file block missing"; + return null; + }, + "mermaid diagram": async ({ page }) => + (await page.$(".notion-code-block svg")) ? null : "mermaid diagram not rendered", + "embeds and bookmarks": async ({ page }) => { + const embeds = await page.$$(".notion-embed-block, .notion-bookmark-block, .notion-tweet-block"); + return embeds.length ? null : "skip: no embed or bookmark blocks on the test page"; + }, + "anchor link": async ({ page, target }) => { + await page.click(".notion-table_of_contents-block a >> text=Anchor target", { timeout: 10000 }); + await page.waitForTimeout(1500); + const s = await state(page); + return target.isRoot(s.path) && s.h1 === "Worknot Render Check" ? null : `navigated away to ${s.path}`; + }, + "database item stays on domain": async ({ page, target }) => { + await page.click(`.notion-collection_view-block a[href*="${DB_ITEM}"]`, { timeout: 10000 }); + await page.waitForTimeout(5000); + const s = await state(page); + if (s.host !== new URL(target.origin).host) return `left the site: ${s.host}`; + return s.text.includes("Item Beta body text") ? null : "item page did not render"; + }, + "navigation and history": async ({ page, target }) => { + await page.waitForSelector(".notion-page-content", { timeout: 40000 }); + await page.click(".notion-page-content a >> text=Render Check Subpage", { timeout: 10000 }); + await page.waitForTimeout(5000); + let s = await state(page); + if (!target.isSubpage(s.path) || s.h1 !== "Render Check Subpage") return `after click: ${s.path} "${s.h1}"`; + await page.goBack(); + await page.waitForTimeout(4000); + s = await state(page); + if (!target.isRoot(s.path) || s.h1 !== "Worknot Render Check") return `after back: ${s.path} "${s.h1}"`; + await page.goForward(); + await page.waitForTimeout(4000); + s = await state(page); + return target.isSubpage(s.path) && s.h1 === "Render Check Subpage" ? null : `after forward: ${s.path} "${s.h1}"`; + }, + "no errors": async ({ errors, failures }) => { + const problems = [...errors, ...failures]; + return problems.length ? problems.slice(0, 5).join("; ") : null; + }, +}; + +const GROUPS = [ + ["page renders", "database views", "database rows and formula", "images load", "pdf and file blocks", "mermaid diagram", "embeds and bookmarks", "anchor link", "no errors"], + // Checks that navigate run on a page of their own + ["database item stays on domain"], + ["navigation and history"], +]; + +async function runChecks(browser, target) { + const results = Object.fromEntries(Object.keys(CHECKS).map((name) => [name, []])); + for (let run = 0; run < RUNS; run++) { + for (const group of GROUPS) { + const session = await openPage(browser, target); + for (const name of group) { + results[name].push(await CHECKS[name](session).catch((e) => e.message.split("\n")[0])); + } + await session.context.close(); + } + // Retry only while some check has not passed yet + if (!Object.values(results).some((r) => r.every((e) => e && !e.startsWith("skip:")))) break; + } + return results; +} + +const passed = (attempts) => attempts.some((e) => e === null); +const skipped = (attempts) => attempts.every((e) => e?.startsWith("skip:")); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "worknot-render-")); +let wrangler; +let browser; +let failed = false; +try { + await generateWorker(dir); + wrangler = await startWrangler(dir); + browser = await chromium.launch({ args: ["--disable-blink-features=AutomationControlled"] }); + const worker = await runChecks(browser, TARGETS.worker); + const notion = await runChecks(browser, TARGETS.notion); + for (const name of Object.keys(CHECKS)) { + let status = "PASS"; + if (skipped(worker[name])) status = "SKIP"; + else if (!passed(worker[name])) status = passed(notion[name]) ? "FAIL" : "NOTION"; + if (status === "FAIL") failed = true; + const detail = status === "PASS" ? "" : ` ${worker[name].at(-1)}`; + console.log(`${status.padEnd(6)} ${name}${detail}`); + } + console.log("\nFAIL = broken through Worknot only. NOTION = also broken on notion.site."); +} finally { + await browser?.close(); + wrangler?.kill(); + fs.rmSync(dir, { recursive: true, force: true }); +} +process.exit(failed ? 1 : 0); From 4d0c93bbf23d01bb80db2f083031f669dbf7b34d Mon Sep 17 00:00:00 2001 From: shigahi Date: Fri, 25 Sep 2026 19:40:23 +0200 Subject: [PATCH 2/3] Run render check on Node 24 and separate setup failures wrangler dev requires Node.js 22+. Exit code 2 now means the check could not run, which fails the job without opening a regression issue. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/notion-monitor.yml | 17 +++++++++++++---- scripts/render-check.mjs | 9 +++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/notion-monitor.yml b/.github/workflows/notion-monitor.yml index deaaf3e..86ab8f7 100644 --- a/.github/workflows/notion-monitor.yml +++ b/.github/workflows/notion-monitor.yml @@ -93,7 +93,8 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: ".nvmrc" + # wrangler dev requires Node.js 22+ + node-version: 24 cache: "npm" - name: Install dependencies @@ -101,15 +102,19 @@ jobs: npm ci npx playwright install --with-deps chromium + # Exit code 1 is a rendering regression; 2 means the check could not run - name: Run render check id: render run: | + set +e npm run check:render 2>&1 | tee render-output.txt - exit "${PIPESTATUS[0]}" - continue-on-error: true + code="${PIPESTATUS[0]}" + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + [ "$code" = "1" ] && exit 0 + exit "$code" - name: Create or update issue if rendering broke - if: steps.render.outcome == 'failure' + if: steps.render.outputs.exit_code == '1' uses: actions/github-script@v7 with: script: | @@ -152,3 +157,7 @@ jobs: labels: ['render-regression'] }); } + + - name: Fail on rendering regression + if: steps.render.outputs.exit_code == '1' + run: exit 1 diff --git a/scripts/render-check.mjs b/scripts/render-check.mjs index 1031da8..0672a4f 100644 --- a/scripts/render-check.mjs +++ b/scripts/render-check.mjs @@ -5,6 +5,7 @@ // // Usage: npm run check:render (RUNS=3 to retry flaky Notion responses more) // Requires Chromium for Playwright: npx playwright install chromium +// Exit codes: 0 = pass, 1 = rendering regression, 2 = the check could not run import { spawn } from "node:child_process"; import fs from "node:fs"; @@ -263,13 +264,14 @@ const skipped = (attempts) => attempts.every((e) => e?.startsWith("skip:")); const dir = fs.mkdtempSync(path.join(os.tmpdir(), "worknot-render-")); let wrangler; let browser; -let failed = false; +let exitCode = 2; try { await generateWorker(dir); wrangler = await startWrangler(dir); browser = await chromium.launch({ args: ["--disable-blink-features=AutomationControlled"] }); const worker = await runChecks(browser, TARGETS.worker); const notion = await runChecks(browser, TARGETS.notion); + let failed = false; for (const name of Object.keys(CHECKS)) { let status = "PASS"; if (skipped(worker[name])) status = "SKIP"; @@ -279,9 +281,12 @@ try { console.log(`${status.padEnd(6)} ${name}${detail}`); } console.log("\nFAIL = broken through Worknot only. NOTION = also broken on notion.site."); + exitCode = failed ? 1 : 0; +} catch (error) { + console.error(`Render check could not run: ${error.message}`); } finally { await browser?.close(); wrangler?.kill(); fs.rmSync(dir, { recursive: true, force: true }); } -process.exit(failed ? 1 : 0); +process.exit(exitCode); From 5017a9417f5a0d792ff6c3c53acafecf65fa86ed Mon Sep 17 00:00:00 2001 From: shigahi Date: Fri, 25 Sep 2026 19:51:21 +0200 Subject: [PATCH 3/3] Wait for lazily rendered Mermaid diagrams in render check Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/render-check.mjs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/render-check.mjs b/scripts/render-check.mjs index 0672a4f..e32508e 100644 --- a/scripts/render-check.mjs +++ b/scripts/render-check.mjs @@ -195,8 +195,12 @@ const CHECKS = { if (!(await page.$(".notion-file-block"))) return "file block missing"; return null; }, - "mermaid diagram": async ({ page }) => - (await page.$(".notion-code-block svg")) ? null : "mermaid diagram not rendered", + "mermaid diagram": async ({ page }) => { + // Code blocks render lazily once they scroll into view + await page.locator(".notion-code-block").first().scrollIntoViewIfNeeded(); + const svg = await page.waitForSelector(".notion-code-block svg", { timeout: 15000 }).catch(() => null); + return svg ? null : "mermaid diagram not rendered"; + }, "embeds and bookmarks": async ({ page }) => { const embeds = await page.$$(".notion-embed-block, .notion-bookmark-block, .notion-tweet-block"); return embeds.length ? null : "skip: no embed or bookmark blocks on the test page";