diff --git a/README.md b/README.md index 909f4f2..d1a42f4 100644 --- a/README.md +++ b/README.md @@ -351,6 +351,28 @@ export default defineConfig({ }); ``` +### Popups + +Popups are wrapped automatically. When a wrapped page opens one — an OAuth window, say — the popup gets the same plugin treatment, no wiring: + +```ts +const popupPromise = page.waitForEvent("popup"); +await page.getByRole("button", { name: "Sign in" }).click(); +const popup = await popupPromise; // already wrapped +await popup.getByRole("button", { name: "Approve" }).click(); +``` + +In video mode, the popup renders as an overlay **in the main page's video**: scaled to fit 90% of the frame over the dimmed page, faded in and out on open/close, with popup clicks pointer-annotated inside the overlay. One composed video per test, popups included. The popup's facts land in `video-mode.json` under `children`. + +Details and escape hatches: + +- Plugins can control what a popup gets via the `forPopup(ctx)` hook — return a plugin for the popup, or `null` to skip. Hookless plugins are re-registered as-is (fine for stateless ones). +- `addPlugins({ ..., popups: false })` turns auto-wrap off. You can then wrap the popup manually with **fresh plugin instances** — a fresh `videoMode()` gives the popup its own standalone video, with `-2`-suffixed artifacts (`video-rendered-2.webm`, `video-mode-2.json`, …). +- Wrapping an already-wrapped page throws, as does reusing an active `videoMode` instance on a second page — one instance per page. +- Popup dialogs (`alert`/`confirm`/`prompt` opened by the popup) aren't annotated in video mode yet. + +See [spec/popup.spec.ts](spec/popup.spec.ts) and [spec/popup-overlay-demo.spec.ts](spec/popup-overlay-demo.spec.ts). + ## Writing your own plugin **Writing your own plugins is the intended way to use this package.** The bundled five exist because they were useful for one particular app; your app has its own loading conventions, error surfaces, and flake patterns. Each bundled plugin is one small self-contained file — use them as inspiration: [spinner-waiter](./src/plugins/spinner-waiter.ts) (conditional waiting + error enrichment + runtime settings via `AsyncLocalStorage`), [hydration-waiter](./src/plugins/hydration-waiter.ts) (the simplest one — start here), [ui-error-reporter](./src/plugins/ui-error-reporter.ts) (catch/enrich/rethrow), [video-mode](./src/plugins/video-mode.ts) (video annotations/artifacts + lifecycle hooks), [llm-recover](./src/plugins/llm-recover.ts) (recovery loops, artifacts, soft assertions). The source also ships inside the npm package, so it's right there in `node_modules/middlewright/src`. diff --git a/spec/auth-demo-app.ts b/spec/auth-demo-app.ts new file mode 100644 index 0000000..596fc57 --- /dev/null +++ b/spec/auth-demo-app.ts @@ -0,0 +1,130 @@ +import type { BrowserContext } from "@playwright/test"; + +/** + * app.middlewright.test shows a Sign in button that opens an auth popup on + * auth.middlewright.test; approving there posts a message back to the opener, + * which then shows who signed in. Routed on the context so the popup page is + * covered too. + */ +const demoStyle = ` + +`; + +/** + * Demo-video variant of the auth flow: same app page, but the popup is a + * realistic sign-in form (inert username/password fields, a Sign in button + * that notifies the opener and closes) on a visibly different background so + * the popout reads clearly in the rendered overlay. + */ +export const routeSignInDemoApp = async (context: BrowserContext) => { + await routeAuthDemoApp(context); + // The app page gets a colored background so the dimmed page under the + // popup overlay reads clearly in the rendered video. + await context.route("https://app.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} + +
+

middlewright dashboard

+ + + +
+ `, + contentType: "text/html", + }); + }); + await context.route("https://auth.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} + +
+

Sign in to middlewright

+ + + + + + +
+ `, + contentType: "text/html", + }); + }); +}; + +export const routeAuthDemoApp = async (context: BrowserContext) => { + await context.route("https://app.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} +
+

middlewright dashboard

+ + + +
+ `, + contentType: "text/html", + }); + }); + await context.route("https://auth.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} +
+

Authorize middlewright?

+ + +
+ `, + contentType: "text/html", + }); + }); +}; diff --git a/spec/popup-overlay-demo.spec.ts b/spec/popup-overlay-demo.spec.ts new file mode 100644 index 0000000..c386301 --- /dev/null +++ b/spec/popup-overlay-demo.spec.ts @@ -0,0 +1,48 @@ +// Demo-grade popup flow with the full watchable treatment — pointer +// highlights, step captions, address bar, popup overlay composite. The +// rendered output doubles as the PR/README demo video. +import { stat } from "node:fs/promises"; +import { test, expect } from "@playwright/test"; +import { addPlugins, videoMode } from "../src/index.ts"; +import { routeSignInDemoApp } from "./auth-demo-app.ts"; + +test.use({ video: "on", viewport: { width: 960, height: 540 } }); + +test("auth popup demo", async ({ page: basePage, context }, testInfo) => { + await routeSignInDemoApp(context); + const video = videoMode(); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + + const popupPromise = basePage.waitForEvent("popup"); + await test.step("Open the sign-in popup", async () => { + await page.goto("https://app.middlewright.test/"); + // Real frames on each side of the popup span keep the composite honest + // (and the demo watchable) — an instant flow would land before the + // screencast's first frame. + await page.waitForTimeout(500); + await page.getByRole("button", { name: "Sign in" }).click(); + }); + + const popup = await popupPromise; + await test.step("Sign in as mmkal", async () => { + await popup.waitForTimeout(500); + await popup.getByLabel("Username").fill("mmkal"); + await popup.getByLabel("Password").fill("hunter2"); + await popup.getByRole("button", { name: "Sign in" }).click(); + }); + + await test.step("Back on the app, signed in", async () => { + await page.getByText("Signed in as mmkal").waitFor(); + await page.waitForTimeout(500); + }); + } + + const metadata = await video.metadata(); + expect(metadata).toMatchObject({ + children: [{ closedAt: expect.any(Number), openedAt: expect.any(Number) }], + outputs: { raw: "video-raw.webm", rendered: "video-rendered.webm" }, + }); + expect(metadata.children[0].closedAt!).toBeGreaterThan(metadata.children[0].openedAt); + expect((await stat(video.outputPaths().rendered)).size).toBeGreaterThan(0); +}); diff --git a/spec/popup-video.spec.ts b/spec/popup-video.spec.ts new file mode 100644 index 0000000..c40fbc6 --- /dev/null +++ b/spec/popup-video.spec.ts @@ -0,0 +1,176 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { test, expect } from "@playwright/test"; +import { addPlugins, videoMode } from "../src/index.ts"; +import { routeAuthDemoApp } from "./auth-demo-app.ts"; + +test.use({ video: "on" }); + +test("captures an auto-wrapped popup's raw screencast for the composite", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + } + + const metadata = await video.metadata(); + expect(metadata.children).toMatchObject([ + { + // The demo popup closes itself after Approve, like a real OAuth popup — + // closedAt comes from the close event, and there is no settled + // recordingEndedAt (the screencast start approximates the timeline). + closedAt: expect.any(Number), + highlights: [{ method: "click" }], + openedAt: expect.any(Number), + raw: "video-raw-popup-1.webm", + viewport: { height: expect.any(Number), width: expect.any(Number) }, + }, + ]); + const [child] = metadata.children; + expect(child.closedAt!).toBeGreaterThan(child.openedAt); + expect((await stat(join(testInfo.outputDir, child.raw!))).size).toBeGreaterThan(0); +}); + +test("renders the popup as a dimmed overlay in one composed video", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ + addressBar: false, + finalHold: 0, + highlight: { mode: "outline", duration: 500 }, + trimStart: "never", + }); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await page.goto("https://app.middlewright.test/"); + // Let the screencast capture real frames on each side of the popup span — + // an instant flow lands entirely before the recorder's first frame. + await page.waitForTimeout(500); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + await popup.waitForTimeout(500); + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + await page.waitForTimeout(500); + } + + await expect(video.metadata()).resolves.toMatchObject({ + outputs: { rendered: "video-rendered.webm" }, + }); + const frames = await videoFrameSamples(video.outputPaths().rendered); + // The demo app's background is a light gray (~245) throughout, so a + // darkened corner marks a frame where the popup backdrop dim is active. + // The downscale blends the thin dim border with its bright neighbors, so + // dimmed corners read ~211 (overlay up) down to ~147 (exit fade), against + // ~245 when lit. + const dimmedFrames = frames.filter((frame) => frame.corner < 235); + const litFrames = frames.filter((frame) => frame.corner >= 235); + expect(dimmedFrames.length).toBeGreaterThan(0); + expect(litFrames.length).toBeGreaterThan(0); + // While dimmed, the popup's white card sits centered above the backdrop. + const overlayFrames = dimmedFrames.filter((frame) => frame.centerPeak > 220); + expect(overlayFrames.length).toBeGreaterThan(0); +}); + +test("records separate videos for the main page and an auth popup", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + let popupVideo!: ReturnType; + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + popupVideo = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using popup = await addPlugins({ + page: await popupPromise, + testInfo, + plugins: [popupVideo], + }); + + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + } + + // Playwright screencasts each page separately, so each instance ends the + // test with its own raw recording and its own annotated render. + await expect(video.metadata()).resolves.toMatchObject({ + outputs: { raw: "video-raw.webm", rendered: "video-rendered.webm" }, + }); + await expect(popupVideo.metadata()).resolves.toMatchObject({ + outputs: { raw: "video-raw-2.webm", rendered: "video-rendered-2.webm" }, + }); + for (const path of [ + video.outputPaths().raw, + video.outputPaths().rendered, + popupVideo.outputPaths().raw, + popupVideo.outputPaths().rendered, + ]) { + expect((await stat(path)).size).toBeGreaterThan(0); + } +}); + +const execFile = promisify(execFileCallback); + +/** + * Decode the video to small grayscale frames and sample each one: a pixel + * near the bottom-left corner (page background), and the brightest pixel of + * the central quarter (the popup card when the overlay is up). 0-255. + */ +const videoFrameSamples = async (path: string) => { + const size = 64; + const { stdout } = await execFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + path, + "-vf", + `fps=10,scale=${size}:${size},format=gray`, + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "pipe:1", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + const frameSize = size * size; + const frames: { centerPeak: number; corner: number }[] = []; + + for (let offset = 0; offset + frameSize <= stdout.length; offset += frameSize) { + let centerPeak = 0; + for (let y = Math.floor(size * 0.375); y < Math.floor(size * 0.625); y += 1) { + for (let x = Math.floor(size * 0.375); x < Math.floor(size * 0.625); x += 1) { + centerPeak = Math.max(centerPeak, stdout[offset + y * size + x]); + } + } + frames.push({ + centerPeak, + corner: stdout[offset + (size - 4) * size + 3], + }); + } + + return frames; +}; diff --git a/spec/popup.spec.ts b/spec/popup.spec.ts new file mode 100644 index 0000000..11c1202 --- /dev/null +++ b/spec/popup.spec.ts @@ -0,0 +1,233 @@ +import { test, expect } from "@playwright/test"; +import { addPlugins, videoMode } from "../src/index.ts"; +import type { Plugin } from "../src/index.ts"; +import { routeAuthDemoApp } from "./auth-demo-app.ts"; + +test("popups are wrapped automatically", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + // No forPopup hook: the plugin is re-registered as-is on the popup. + const recorder: Plugin = { + name: "action-recorder", + middleware: async (ctx, next) => { + actions.push(`${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }, + }; + await using page = await addPlugins({ page: basePage, testInfo, plugins: [recorder] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + + // No addPlugins call for the popup — it's already wrapped. + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "click on app.middlewright.test", + "click on auth.middlewright.test", + "waitFor on app.middlewright.test", + ]); +}); + +test("forPopup controls what popups get", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + const record = (label: string) => { + const middleware: Plugin["middleware"] = async (ctx, next) => { + actions.push(`${label}: ${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }; + return middleware; + }; + const inherited: Plugin = { + name: "inherited", + middleware: record("parent"), + forPopup: () => ({ name: "inherited-child", middleware: record("child") }), + }; + const skipped: Plugin = { + name: "skipped-on-popups", + middleware: record("skipped"), + forPopup: () => null, + }; + await using page = await addPlugins({ page: basePage, testInfo, plugins: [inherited, skipped] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "parent: click on app.middlewright.test", + "skipped: click on app.middlewright.test", + "child: click on auth.middlewright.test", + "parent: waitFor on app.middlewright.test", + "skipped: waitFor on app.middlewright.test", + ]); +}); + +test("wrapping an already-wrapped page throws", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [] }); + + await expect(addPlugins({ page: basePage, testInfo, plugins: [] })).rejects.toThrow( + "already has plugins", + ); + + // Popups are auto-wrapped, so wrapping one manually is also a double wrap. + await page.goto("https://app.middlewright.test/"); + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + await expect(addPlugins({ page: popup, testInfo, plugins: [] })).rejects.toThrow( + "popups: false", + ); +}); + +test("popups: false leaves popups unwrapped", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + const recorder: Plugin = { + name: "action-recorder", + middleware: async (ctx, next) => { + actions.push(`${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }, + }; + await using page = await addPlugins({ + page: basePage, + testInfo, + plugins: [recorder], + popups: false, + }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "click on app.middlewright.test", + "waitFor on app.middlewright.test", + ]); +}); + +test("videoMode records popup actions as a child timeline", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + const metadata = await video.metadata(); + expect(metadata).toMatchObject({ + // The main timeline has only the main page's actions... + highlights: [{ method: "click" }, { method: "waitFor" }], + // ...and the popup's actions land on a child timeline of the same clock. + children: [ + { + openedAt: expect.any(Number), + highlights: [{ method: "click" }], + }, + ], + }); + const [child] = metadata.children; + expect(child.openedAt).toBeGreaterThanOrEqual(metadata.highlights[0].start); + expect(child.openedAt).toBeLessThanOrEqual(child.highlights[0].start); +}); + +test("a manually wrapped popup runs actions through its own plugins", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popupVideo = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using popup = await addPlugins({ + page: await popupPromise, + testInfo, + plugins: [popupVideo], + }); + + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + // The popup's click went through the popup's own video-mode middleware... + await expect(popupVideo.metadata()).resolves.toMatchObject({ + highlights: [{ method: "click" }], + }); + // ...and the main page's timeline has only the main page's actions. + await expect(video.metadata()).resolves.toMatchObject({ + highlights: [{ method: "click" }, { method: "waitFor" }], + }); +}); + +test("each videoMode instance still owns its artifacts after the test", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + let popupVideo!: ReturnType; + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + popupVideo = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using popup = await addPlugins({ + page: await popupPromise, + testInfo, + plugins: [popupVideo], + }); + + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + } + + // Both pages have finalized. Each instance must write its own artifacts and + // read back its own timeline — not whichever page finalized last. + expect(popupVideo.outputPaths().metadata).not.toBe(video.outputPaths().metadata); + await expect(video.metadata()).resolves.toMatchObject({ + highlights: [{ method: "click" }, { method: "waitFor" }], + }); + await expect(popupVideo.metadata()).resolves.toMatchObject({ + highlights: [{ method: "click" }], + }); +}); + +test("reusing one videoMode instance on a popup fails with a clear error", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + + // Wiring the same instance to a second page would wipe the main page's + // timeline, so it must fail loudly instead. + await expect( + addPlugins({ page: await popupPromise, testInfo, plugins: [video] }), + ).rejects.toThrow("create a fresh videoMode() instance for each page"); +}); diff --git a/spec/video-mode.spec.ts b/spec/video-mode.spec.ts index ba28fa1..b3c1977 100644 --- a/spec/video-mode.spec.ts +++ b/spec/video-mode.spec.ts @@ -927,7 +927,7 @@ test("deadAir runs actions without video highlighting and records metadata", asy await basePage.waitForSelector('#result:has-text("(no style)")'); await expect(page.videoMode.metadata()).resolves.toMatchObject({ outputs: {}, - schemaVersion: 1, + schemaVersion: 2, timebase: "ms", }); expect((await page.videoMode.metadata()).deadAir).toContainEqual( @@ -947,7 +947,7 @@ test("deadAir runs actions without video highlighting and records metadata", asy expect(metadata).toMatchObject({ highlights: [], outputs: {}, - schemaVersion: 1, + schemaVersion: 2, timebase: "ms", }); expect(metadata.deadAir).toContainEqual( diff --git a/src/plugin-system.ts b/src/plugin-system.ts index 2f14da7..fb05a2a 100644 --- a/src/plugin-system.ts +++ b/src/plugin-system.ts @@ -104,6 +104,14 @@ export type PageExtensionContext = { testInfo: TestInfo; }; +export type PopupPluginContext = { + /** The newly opened popup page, not yet wrapped. */ + page: Page; + /** The wrapped page that opened the popup. */ + parentPage: Page; + testInfo: TestInfo; +}; + export type Plugin = { name: string; /** Middleware to wrap locator actions. Called in registration order. */ @@ -112,6 +120,13 @@ export type Plugin = { testLifecycle?: (emitter: Emittery) => void | (() => void); /** Add explicit test controls to the page returned from addPlugins. */ pageExtension?: (ctx: PageExtensionContext) => PageExtension; + /** + * Called when a page wrapped with this plugin opens a popup, to produce the + * plugin registered on the popup — often a fresh instance tied to this one. + * Return null to skip this plugin on popups. Plugins without this hook are + * re-registered as-is (fine for stateless plugins). + */ + forPopup?: (ctx: PopupPluginContext) => Plugin | false | null | undefined; }; const PLUGIN_STATE = Symbol("playwrightPluginState"); @@ -184,9 +199,24 @@ export const addPlugins = async (p page: Page; testInfo: TestInfo; plugins: Plugins; + /** + * Automatically add plugins to popups this page opens (and to their popups, + * recursively). Plugins may define `forPopup` to control or skip what gets + * registered on the popup. Default: true. Pass false to leave popups + * unwrapped — they fall through to original Playwright behavior and can be + * wrapped manually with fresh plugin instances. + */ + popups?: boolean; boxedStackPrefixes?: (defaults: string[]) => string[]; }): Promise>> => { const { page, testInfo, plugins, boxedStackPrefixes } = params; + if (getPluginState(page)) { + throw new Error( + "this page already has plugins added. Popups are auto-wrapped by default - " + + "pass popups: false to the parent addPlugins call for manual control, " + + "and use fresh plugin instances for each page", + ); + } // Patch Locator prototype once globally patchLocatorPrototype(page, boxedStackPrefixes); @@ -228,11 +258,51 @@ export const addPlugins = async (p // Emit beforeTest await state.lifecycleEmitter.emitSerial("beforeTest", { page, testInfo }); + // Auto-wrap popups (default on). The child addPlugins call attaches plugin + // state synchronously in the tick the popup event fires — before test code + // awaiting waitForEvent("popup") gets to act on the popup — because this + // listener is registered ahead of the test's own. + const childWraps: Promise[] = []; + let onPopup: ((popup: Page) => void) | undefined; + if (params.popups !== false) { + onPopup = (popupPage) => { + const childPlugins = plugins + .filter((plugin): plugin is Plugin => !!plugin) + .map((plugin) => + plugin.forPopup + ? plugin.forPopup({ page: popupPage, parentPage: page, testInfo }) + : plugin, + ); + const wrap = addPlugins({ page: popupPage, testInfo, plugins: childPlugins }); + childWraps.push(wrap); + // Failures surface at dispose; avoid unhandled-rejection noise meanwhile. + wrap.catch(() => {}); + }; + page.on("popup", onPopup); + } + // Add async dispose pageWithPlugins[Symbol.asyncDispose] = async () => { + if (onPopup) { + page.off("popup", onPopup); + } + // Children dispose first (newest first) so their plugins can finalize -- + // and, later, feed facts to parent plugins -- before the parent's own + // lifecycle events run. A failed child wrap must not stop the parent + // finalizing; it rethrows below once cleanup is done. + const settledChildren = await Promise.allSettled(childWraps); + for (const result of [...settledChildren].reverse()) { + if (result.status === "fulfilled") { + await result.value[Symbol.asyncDispose](); + } + } await state.lifecycleEmitter.emitSerial("afterTest", { page, testInfo }); await state.lifecycleEmitter.emitSerial("afterTestFinalize", { page, testInfo }); state.lifecycleCleanups.forEach((cleanup) => cleanup()); + const failedChildWrap = settledChildren.find((result) => result.status === "rejected"); + if (failedChildWrap) { + throw failedChildWrap.reason; + } }; return pageWithPlugins; diff --git a/src/plugins/video-mode.ts b/src/plugins/video-mode.ts index bc0e566..cee6302 100644 --- a/src/plugins/video-mode.ts +++ b/src/plugins/video-mode.ts @@ -18,6 +18,7 @@ import { extname, join } from "node:path"; import { promisify } from "node:util"; import type { Dialog, Locator, Page, TestInfo } from "@playwright/test"; import type { + ActionMiddleware, ActionTiming, LocatorWithOriginal, Plugin, @@ -141,6 +142,17 @@ export type VideoModeHighlight = VideoModeSpan & { fillReveal?: VideoModeFillReveal; image?: string; method?: OverrideableMethod; + /** + * Set at render time on projected popup highlights: maps child-frame + * coordinates (which `fillReveal` and its screenshots use) into the + * composite frame — scale plus the overlay's top-left corner. + */ + overlayTransform?: { + scale: number; + viewport: VideoModeViewport; + x: number; + y: number; + }; pan?: VideoModePan; rect: VideoModeRect; sourceFrameAt?: number; @@ -160,11 +172,34 @@ export type VideoModeAddressBar = VideoModeSpan & { url: string; }; +/** + * Recorded facts for a popup opened by the recorded page. Timestamps share + * the parent timeline (ms since the parent instance's timebase); the child's + * raw screencast has its own clock, mapped via `recordingEndedAt`. + */ +export type VideoModeChild = { + /** Parent-timeline ms when the popup closed. Missing: open at render end. */ + closedAt?: number; + highlights: VideoModeHighlight[]; + /** Parent-timeline ms when the popup opened. */ + openedAt: number; + /** Raw screencast artifact for the popup, when video was recorded. */ + raw?: string; + /** + * Parent-timeline ms of the popup recorder's settled endpoint (the raw + * video's last frame). Missing when the popup closed itself — the raw + * video's own end approximates `closedAt` then. + */ + recordingEndedAt?: number; + viewport?: VideoModeViewport; +}; + export type VideoModeMetadata = { - schemaVersion: 1; + schemaVersion: 2; timebase: "ms"; addressBars: VideoModeAddressBar[]; captions: VideoModeCaption[]; + children: VideoModeChild[]; deadAir: VideoModeSpan[]; highlights: VideoModeHighlight[]; outputs: VideoModeOutputs; @@ -265,7 +300,10 @@ export type VideoModeTrimStart = "auto" | "detect-blank" | "never" | ["selector" type VideoModeState = { addressBars: VideoModeAddressBar[]; + /** Distinguishes artifact filenames when a test has several instances. */ + artifactSuffix: string; captions: VideoModeCaption[]; + children: VideoModeChild[]; deadAirDepth: number; deadAirSpans: VideoModeSpan[]; highlights: VideoModeHighlight[]; @@ -686,7 +724,7 @@ const translateVideoTimeline = (options: { text: caption.text, })), deadAir: options.deadAir.map((span) => translateVideoSpan(span, options.offsetMs)), - highlights: options.highlights.map((highlight) => { + highlights: options.highlights.map((highlight): VideoModeHighlight => { const start = Math.max(0, Math.round(highlight.start + options.offsetMs)); return { ...highlight, @@ -729,10 +767,14 @@ const metadataFor = (state: VideoModeState): VideoModeMetadata => { .filter((addressBar) => addressBar.end > addressBar.start) .sort((left, right) => left.start - right.start || left.end - right.end), captions: normalizeVideoCaptions(state.captions), + children: state.children.map((child) => ({ + ...child, + highlights: normalizeVideoHighlights(child.highlights), + })), deadAir: mergeVideoSpans(state.deadAirSpans), highlights: normalizeVideoHighlights(state.highlights), outputs: state.outputs, - schemaVersion: 1, + schemaVersion: 2, sourceRange: normalizeSourceRange(state.sourceRange), timebase: "ms", }; @@ -862,13 +904,35 @@ const recordCaption = async ( } }; -const videoModeOutputPaths = (testInfo: TestInfo): VideoModeOutputPaths => { +/** + * Insert a per-instance suffix before the extension: `video-mode.json` → + * `video-mode-2.json`. The first instance in a test keeps the unsuffixed + * names, so single-page tests are unaffected. + */ +const suffixArtifactFileName = (fileName: string, artifactSuffix: string) => { + if (!artifactSuffix) return fileName; + const extension = extname(fileName); + return `${fileName.slice(0, fileName.length - extension.length)}${artifactSuffix}${extension}`; +}; + +/** + * Registrations per testInfo.outputDir. Each videoMode instance added within + * one test (e.g. a fresh instance for a popup) gets its own artifact + * namespace so it can't clobber the main page's files. + */ +const videoModeRegistrationCounts = new Map(); + +const videoModeOutputPaths = ( + testInfo: TestInfo, + artifactSuffix: string, +): VideoModeOutputPaths => { + const name = (fileName: string) => suffixArtifactFileName(fileName, artifactSuffix); return { - metadata: join(testInfo.outputDir, VIDEO_MODE_METADATA_FILE), - player: join(testInfo.outputDir, VIDEO_MODE_PLAYER_FILE), - raw: join(testInfo.outputDir, VIDEO_MODE_RAW_FILE), - rendered: join(testInfo.outputDir, VIDEO_MODE_RENDERED_FILE), - reportPlayer: join(testInfo.outputDir, VIDEO_MODE_REPORT_PLAYER_FILE), + metadata: join(testInfo.outputDir, name(VIDEO_MODE_METADATA_FILE)), + player: join(testInfo.outputDir, name(VIDEO_MODE_PLAYER_FILE)), + raw: join(testInfo.outputDir, name(VIDEO_MODE_RAW_FILE)), + rendered: join(testInfo.outputDir, name(VIDEO_MODE_RENDERED_FILE)), + reportPlayer: join(testInfo.outputDir, name(VIDEO_MODE_REPORT_PLAYER_FILE)), }; }; @@ -1258,8 +1322,8 @@ const recordHighlight = async (options: { : undefined; const image = pan - ? `video-mode-pan-${options.state.highlightImageIndex}.png` - : `video-mode-highlight-${options.state.highlightImageIndex}.png`; + ? `video-mode-pan${options.state.artifactSuffix}-${options.state.highlightImageIndex}.png` + : `video-mode-highlight${options.state.artifactSuffix}-${options.state.highlightImageIndex}.png`; options.state.highlightImageIndex += 1; const imagePath = join(options.testInfo.outputDir, image); await mkdir(options.testInfo.outputDir, { recursive: true }); @@ -1402,8 +1466,7 @@ const recordFillReveal = async (options: { value.length === 0 || value.length > captureOptions.maxCharacters || style.direction === "rtl" || - !["left", "start"].includes(style.textAlign) || - (element instanceof HTMLInputElement && element.type === "password") + !["left", "start"].includes(style.textAlign) ) { return { ...geometry, kind: "fallback" as const }; } @@ -1510,10 +1573,16 @@ const recordFillReveal = async (options: { return { ...geometry, kind: "fallback" as const }; } context.font = style.font; - const graphemes = Array.from( - new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(value), - ({ segment }) => segment, - ); + // A password input renders one bullet per character, so measure those + // glyphs — the reveal only ever shows the screenshot's dots, never the + // value. + const masked = element instanceof HTMLInputElement && element.type === "password"; + const graphemes = masked + ? Array.from(value, () => "•") + : Array.from( + new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(value), + ({ segment }) => segment, + ); const letterSpacing = pixels(style.letterSpacing); const textIndent = pixels(style.textIndent); const revealStops = graphemes.map((_, index) => { @@ -1546,7 +1615,7 @@ const recordFillReveal = async (options: { return; } - const image = `video-mode-fill-${options.state.highlightImageIndex}.png`; + const image = `video-mode-fill${options.state.artifactSuffix}-${options.state.highlightImageIndex}.png`; options.state.highlightImageIndex += 1; await mkdir(options.testInfo.outputDir, { recursive: true }); await options.locator.page().screenshot({ @@ -2262,6 +2331,12 @@ const videoPieces = (options: { addressBars: VideoModeAddressBar[]; frameDurationMs: number; highlights: VideoModeHighlight[]; + /** + * Source spans that must reach the output (popup enter/exit animations). + * Overlapping-hold skips normally jump the footage between two highlights; + * a skip is cancelled when it would leap across one of these. + */ + keepSpans: VideoModeSpan[]; preActionStabilizationMs: number; segments: RenderVideoSegment[]; }): VideoPiece[] => { @@ -2309,8 +2384,10 @@ const videoPieces = (options: { const nextHighlight = highlights[highlightIndex + 1]; if (highlight.start > cursor) { - const postAction = previousHighlight?.fillReveal ? previousHighlight : undefined; - const preAction = highlight.fillReveal ? highlight : undefined; + const stabilizable = (candidate: VideoModeHighlight | undefined) => + candidate?.fillReveal && !candidate.overlayTransform ? candidate : undefined; + const postAction = stabilizable(previousHighlight); + const preAction = stabilizable(highlight); // `trim` chooses whole source frames. Round down so the boundary frame // belongs to the stabilized piece instead of leaking from the raw gap. const preActionStart = preAction @@ -2364,7 +2441,13 @@ const videoPieces = (options: { let nextCursor = actionEnd; if (nextHighlight && highlight.end > nextHighlight.start) { - nextCursor = Math.max(nextCursor, nextHighlight.start); + const skipTo = Math.max(nextCursor, nextHighlight.start); + const skipCrossesKeptSpan = options.keepSpans.some( + (span) => span.start < skipTo && span.end > nextCursor, + ); + if (!skipCrossesKeptSpan) { + nextCursor = skipTo; + } } cursor = Math.min(segment.end, nextCursor); @@ -2374,7 +2457,10 @@ const videoPieces = (options: { if (segment.end > cursor) { pieces.push({ end: segment.end, - postAction: previousHighlight?.fillReveal ? previousHighlight : undefined, + postAction: + previousHighlight?.fillReveal && !previousHighlight.overlayTransform + ? previousHighlight + : undefined, speed: segment.speed, start: cursor, }); @@ -2603,8 +2689,20 @@ const highlightCursorPoint = ( highlight: VideoModeHighlight, video: { width: number; height: number }, ) => { - const rect = highlight.fillReveal - ? scaleVideoModeRect(highlight.fillReveal.initialRect, highlight.viewport, video) + // fillReveal rects live in child-frame coordinates on projected popup + // highlights — map them through the overlay transform first. + const transform = highlight.overlayTransform; + const projectedInitialRect = + highlight.fillReveal && transform + ? { + height: highlight.fillReveal.initialRect.height * transform.scale, + width: highlight.fillReveal.initialRect.width * transform.scale, + x: transform.x + highlight.fillReveal.initialRect.x * transform.scale, + y: transform.y + highlight.fillReveal.initialRect.y * transform.scale, + } + : highlight.fillReveal?.initialRect; + const rect = projectedInitialRect + ? scaleVideoModeRect(projectedInitialRect, highlight.viewport, video) : scaleHighlight(highlight, video); return { @@ -2906,6 +3004,7 @@ const renderedVideoFilter = (options: { highlightMode: "outline" | "pointer"; highlightInputs: HighlightInput[]; highlights: VideoModeHighlight[]; + keepSpans: VideoModeSpan[]; preActionStabilizationMs: number; segments: RenderVideoSegment[]; textPointerInput?: PointerInput; @@ -2918,10 +3017,23 @@ const renderedVideoFilter = (options: { addressBars: options.addressBars, frameDurationMs: options.video.frameDurationMs, highlights: options.highlights, + keepSpans: options.keepSpans, preActionStabilizationMs: options.preActionStabilizationMs, segments: options.segments, }); const renderedPieces = renderedVideoPieces(pieces); + if (process.env.MIDDLEWRIGHT_DEBUG_PIECES) { + for (const piece of renderedPieces) { + console.log( + `piece src[${piece.start}-${piece.end}] out[${Math.round(piece.outputStart)}-${Math.round(piece.outputEnd)}] speed=${piece.speed}` + + (piece.highlight ? ` highlight=${piece.highlight.method} hstart=${piece.highlight.start}` : "") + + (piece.addressBar ? " addressBar" : "") + + (piece.highlight?.overlayTransform ? " overlay" : "") + + (piece.highlight?.fillReveal ? " fillReveal" : "") + + (piece.highlight?.image ? ` image=${piece.highlight.image}` : ""), + ); + } + } const targets = cursorTargets({ highlights: options.highlights, pieces: renderedPieces, @@ -3073,6 +3185,188 @@ const renderedVideoFilter = (options: { continue; } + // Fill reveal inside a popup overlay: the base is a frozen composite + // frame from just before the fill (popup risen, field empty, dim and + // parent intact), and the typed reveal is the child screenshot's content + // rect scaled and positioned through the overlay transform. + if (piece.highlight && fillReveal && postFillInput && piece.highlight.overlayTransform) { + const transform = piece.highlight.overlayTransform; + const projectLength = (value: number) => Math.round(value * transform.scale); + const scaledImage = { + height: Math.max(2, projectLength(transform.viewport.height)), + width: Math.max(2, projectLength(transform.viewport.width)), + }; + const contentLocal = { + height: Math.max(1, Math.min(scaledImage.height, projectLength(fillReveal.contentRect.height))), + width: Math.max(1, Math.min(scaledImage.width, projectLength(fillReveal.contentRect.width))), + x: Math.max(0, projectLength(fillReveal.contentRect.x)), + y: Math.max(0, projectLength(fillReveal.contentRect.y)), + }; + const contentAbsolute = { + x: Math.round(transform.x + fillReveal.contentRect.x * transform.scale), + y: Math.round(transform.y + fillReveal.contentRect.y * transform.scale), + }; + const duration = renderedPieceDuration(piece); + const durationSeconds = formatSeconds(duration); + const revealStops = fillReveal.revealStops + .map((stop) => Math.max(1, Math.min(contentLocal.width, projectLength(stop)))) + .filter((stop, stopIndex, stops) => stopIndex === 0 || stop !== stops[stopIndex - 1]); + const revealSteps = fillReveal.revealBands.flatMap((band) => { + const y = Math.max(0, Math.min(contentLocal.height - 1, projectLength(band.y))); + const height = Math.max(1, Math.min(contentLocal.height - y, projectLength(band.height))); + return revealStops.map((width) => ({ height, width, y })); + }); + const target = plan.targets.find( + (candidate) => candidate.highlight === piece.highlight, + ); + const revealEnd = + options.highlightMode === "pointer" + ? Math.max(0, duration - TEXT_CURSOR_POINTER_TAIL_MS) + : duration; + const pointerArrival = target + ? Math.max(0, target.arriveAt - renderedPiece.outputStart) + : 0; + const availableAfterArrival = Math.max(0, revealEnd - pointerArrival); + const preRevealHold = Math.min( + TEXT_CURSOR_HOLD_IDEAL_MS, + availableAfterArrival / 2, + ); + const revealStart = Math.max( + 0, + Math.min(revealEnd, pointerArrival + preRevealHold), + ); + // The base predates the fill, so it shows the field unfocused. The + // post-fill screenshot has the focus ring: overlay the field's ring + // region from it at reveal start, immediately cover its text with the + // pre-fill screenshot's empty content box, and let the reveal bands + // type over that — the ring appears when the cursor lands and the + // letters arrive inside it, continuous with the live footage after. + const ringPaddingPx = 4; + const ringSource = { + height: fillReveal.initialRect.height + 2 * ringPaddingPx, + width: fillReveal.initialRect.width + 2 * ringPaddingPx, + x: fillReveal.initialRect.x - ringPaddingPx, + y: fillReveal.initialRect.y - ringPaddingPx, + }; + const ringLocal = { + height: Math.max(1, Math.min(scaledImage.height, projectLength(ringSource.height))), + width: Math.max(1, Math.min(scaledImage.width, projectLength(ringSource.width))), + x: Math.max(0, projectLength(ringSource.x)), + y: Math.max(0, projectLength(ringSource.y)), + }; + const ringAbsolute = { + x: Math.round(transform.x + ringSource.x * transform.scale), + y: Math.round(transform.y + ringSource.y * transform.scale), + }; + const baseLabel = `fillbase${index}`; + // One frame back only: rewinding further can cross the previous fill's + // completion (wiping its value from the frozen base). The content box + // is covered with the pre-fill empty state from t=0 below, so anchor + // imprecision inside this field can't leak early-typed text either. + const freezeStart = Math.max(0, piece.start - options.video.frameDurationMs); + filters.push( + [ + `[0:v]trim=start=${formatSeconds(freezeStart)}:end=${formatSeconds( + freezeStart + options.video.frameDurationMs, + )}`, + "setpts=PTS-STARTPTS", + `tpad=stop_mode=clone:stop_duration=${formatSeconds( + Math.max(0, duration - options.video.frameDurationMs), + )}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${baseLabel}]`, + ].join(","), + ); + + if (revealSteps.length === 0) { + filters.push(`[${baseLabel}]null[${label}]`); + continue; + } + + const splitLabels = revealSteps.map((_, stepIndex) => `fillpost${index}x${stepIndex}`); + filters.push( + [ + `[${postFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${contentLocal.width}:h=${contentLocal.height}:x=${contentLocal.x}:y=${contentLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + "setpts=PTS-STARTPTS", + `split=${revealSteps.length}${splitLabels.map((splitLabel) => `[${splitLabel}]`).join("")}`, + ].join(","), + ); + + let composedLabel = baseLabel; + if (preFillInput) { + const ringLabel = `fillring${index}`; + const emptyLabel = `fillempty${index}`; + const revealStartEnable = `enable='gte(t\\,${formatSeconds(revealStart)})'`; + filters.push( + [ + `[${postFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${ringLocal.width}:h=${ringLocal.height}:x=${ringLocal.x}:y=${ringLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${ringLabel}]`, + ].join(","), + ); + filters.push( + [ + `[${preFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${contentLocal.width}:h=${contentLocal.height}:x=${contentLocal.x}:y=${contentLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${emptyLabel}]`, + ].join(","), + ); + filters.push( + [ + `[${composedLabel}][${ringLabel}]overlay=x=${ringAbsolute.x}`, + `y=${ringAbsolute.y}`, + revealStartEnable, + `shortest=1[fillringcomposed${index}]`, + ].join(":"), + ); + filters.push( + [ + `[fillringcomposed${index}][${emptyLabel}]overlay=x=${contentAbsolute.x}`, + `y=${contentAbsolute.y}`, + `shortest=1[fillemptycomposed${index}]`, + ].join(":"), + ); + composedLabel = `fillemptycomposed${index}`; + } + for (let stepIndex = 0; stepIndex < revealSteps.length; stepIndex += 1) { + const step = revealSteps[stepIndex]; + const cropLabel = `fillcrop${index}x${stepIndex}`; + const nextLabel = `fillcomposed${index}x${stepIndex}`; + const showAt = + revealStart + + ((revealEnd - revealStart) * (stepIndex + 1)) / + (revealSteps.length + 1); + filters.push( + `${[ + `[${splitLabels[stepIndex]}]crop=w=${step.width}`, + `h=${step.height}`, + "x=0", + `y=${step.y}`, + ].join(":")}[${cropLabel}]`, + ); + filters.push( + [ + `[${composedLabel}][${cropLabel}]overlay=x=${contentAbsolute.x}`, + `y=${contentAbsolute.y + step.y}`, + `enable='gte(t\\,${formatSeconds(showAt)})'`, + `shortest=1[${nextLabel}]`, + ].join(":"), + ); + composedLabel = nextLabel; + } + + filters.push( + options.highlightMode === "outline" + ? `[${composedLabel}]${drawboxFilter(piece.highlight, options.video)}[${label}]` + : `[${composedLabel}]null[${label}]`, + ); + continue; + } + if (piece.highlight && fillReveal && preFillInput && postFillInput) { const scaledViewport = scaledViewportSize(piece.highlight.viewport, options.video); const contentRect = scaleVideoModeRect( @@ -3580,7 +3874,7 @@ const playwrightReportAttachmentName = async (path: string) => { return `${createHash("sha1").update(data).digest("hex")}${extname(path)}`; }; -const videoModePlayerHtml = (options: { raw: string; rendered?: string }) => { +const videoModePlayerHtml = (options: { metadata: string; raw: string; rendered?: string }) => { const primary = options.rendered || options.raw; const primaryLabel = options.rendered ? "Rendered video" : "Raw video"; const primaryActiveKey = options.rendered ? "rendered" : "raw"; @@ -3724,7 +4018,7 @@ const videoModePlayerHtml = (options: { raw: string; rendered?: string }) => {
frame: 0
duration: ?s
Left/right steps one frame. Shift+left/right steps ten. Space toggles play.
-
${VIDEO_MODE_METADATA_FILE}
+
${options.metadata}