+
+ ${videoElementHtml({ activeKey: primaryActiveKey, label: primaryLabel, source: primary })}
+ ${rawDetails}
+
+
+
+
+
+
+`;
+};
+
+const renderVideo = async (options: {
+ finalHoldMs: number;
+ highlightMode: "outline" | "pointer";
+ highlights: VideoModeHighlight[];
+ inputPath: string;
+ outputDir: string;
+ outputPath: string;
+ deadAir: VideoModeSpan[];
+ sourceRange: VideoModeSourceRange;
+ thresholdMs: number | undefined;
+}) => {
+ const info = await videoInfo(options.inputPath);
+ const sourceRangeStart = options.sourceRange.start === undefined ? 0 : options.sourceRange.start;
+ const sourceRangeEnd =
+ options.sourceRange.end === undefined ? info.durationMs : options.sourceRange.end;
+ const rangeStart = Math.max(0, Math.min(Math.round(sourceRangeStart), info.durationMs));
+ const rangeEnd = Math.max(
+ 0,
+ Math.min(Math.round(sourceRangeEnd), info.durationMs),
+ );
+
+ if (rangeEnd <= rangeStart) {
+ console.warn(
+ `videoMode source range is empty: start ${rangeStart}ms must be before end ${rangeEnd}ms`,
+ );
+ return false;
+ }
+
+ const highlightInputs = options.highlights
+ .filter((highlight) => highlight.image)
+ .map((highlight, index) => ({
+ durationMs: highlight.end - highlight.start,
+ image: highlight.image!,
+ inputIndex: index + 1,
+ path: join(options.outputDir, highlight.image!),
+ }));
+ const shouldRenderPointer = options.highlightMode === "pointer" && options.highlights.length > 0;
+ const cursorPointerInput: PointerInput | undefined = shouldRenderPointer
+ ? {
+ hotspot: VIDEO_MODE_POINTER_HOTSPOT,
+ inputIndex: highlightInputs.length + 1,
+ path: join(options.outputDir, VIDEO_MODE_POINTER_FILE),
+ size: VIDEO_MODE_POINTER_SIZE,
+ sourceSize: VIDEO_MODE_POINTER_SOURCE_SIZE,
+ }
+ : undefined;
+ const clickPointerInput: PointerInput | undefined = shouldRenderPointer
+ ? {
+ hotspot: VIDEO_MODE_CLICK_POINTER_HOTSPOT,
+ inputIndex: highlightInputs.length + 2,
+ path: join(options.outputDir, VIDEO_MODE_CLICK_POINTER_FILE),
+ size: VIDEO_MODE_CLICK_POINTER_SIZE,
+ sourceSize: VIDEO_MODE_CLICK_POINTER_SOURCE_SIZE,
+ }
+ : undefined;
+ const textPointerInput: PointerInput | undefined = shouldRenderPointer
+ ? {
+ hotspot: VIDEO_MODE_TEXT_POINTER_HOTSPOT,
+ inputIndex: highlightInputs.length + 3,
+ path: join(options.outputDir, VIDEO_MODE_TEXT_POINTER_FILE),
+ size: VIDEO_MODE_TEXT_POINTER_SIZE,
+ sourceSize: VIDEO_MODE_TEXT_POINTER_SOURCE_SIZE,
+ }
+ : undefined;
+ if (cursorPointerInput) {
+ await writeFile(cursorPointerInput.path, Buffer.from(VIDEO_MODE_POINTER_PNG, "base64"));
+ }
+ if (clickPointerInput) {
+ await writeFile(clickPointerInput.path, Buffer.from(VIDEO_MODE_CLICK_POINTER_PNG, "base64"));
+ }
+ if (textPointerInput) {
+ await writeFile(textPointerInput.path, Buffer.from(VIDEO_MODE_TEXT_POINTER_PNG, "base64"));
+ }
+ const segments = renderVideoSegments({
+ deadAir: options.deadAir,
+ finalEnd: rangeEnd,
+ start: rangeStart,
+ thresholdMs: options.thresholdMs,
+ });
+ const filter = renderedVideoFilter({
+ clickPointerInput,
+ cursorPointerInput,
+ finalHoldMs: options.finalHoldMs,
+ highlightMode: options.highlightMode,
+ highlightInputs,
+ highlights: options.highlights,
+ segments,
+ textPointerInput,
+ video: info,
+ });
+
+ if (!filter) {
+ return false;
+ }
+
+ await execFile(
+ "ffmpeg",
+ [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-i",
+ options.inputPath,
+ ...highlightInputs.flatMap((input) => [
+ "-loop",
+ "1",
+ "-t",
+ formatSeconds(input.durationMs),
+ "-i",
+ input.path,
+ ]),
+ ...(cursorPointerInput ? ["-loop", "1", "-i", cursorPointerInput.path] : []),
+ ...(clickPointerInput ? ["-loop", "1", "-i", clickPointerInput.path] : []),
+ ...(textPointerInput ? ["-loop", "1", "-i", textPointerInput.path] : []),
+ "-filter_complex",
+ filter.value,
+ "-map",
+ `[${filter.outputLabel}]`,
+ "-an",
+ options.outputPath,
+ ],
+ { maxBuffer: 10 * 1024 * 1024 },
+ );
+
+ return true;
+};
+
+/** Records video-mode facts and renders annotations into the recorded video. */
+export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => {
+ if (process.env.PWDEBUG) {
+ let testInfoForOutputPaths: TestInfo | undefined;
+ const controls: VideoModeControls = {
+ deadAir: async (action) => {
+ return await action();
+ },
+ getVideoTimestamp: () => 0,
+ metadata: async () => ({
+ deadAir: [],
+ highlights: [],
+ outputs: {},
+ schemaVersion: 1,
+ sourceRange: {},
+ timebase: "ms",
+ }),
+ outputPaths: () => {
+ if (!testInfoForOutputPaths) {
+ throw new Error("videoMode.outputPaths() is only available after addPlugins registers videoMode");
+ }
+
+ return videoModeOutputPaths(testInfoForOutputPaths);
+ },
+ setEndTime: () => {},
+ setStartTime: () => {},
+ };
+
+ return {
+ ...controls,
+ name: "video-mode",
+ pageExtension: ({ testInfo }) => {
+ testInfoForOutputPaths = testInfo;
+ return { videoMode: controls };
+ },
+ };
+ }
+
+ const finalHold = resolveNonNegativeNumber({
+ defaultValue: 3000,
+ name: "videoMode finalHold",
+ value: options.finalHold,
+ });
+ const highlight = resolveVideoModeHighlight(options);
const skipMethods = options.skipMethods || ["waitFor"];
const skipStackFrames = options.skipStackFrames || [];
+ const deadAirThreshold = resolveDeadAirThreshold(options.deadAirThreshold);
+ const state: VideoModeState = {
+ deadAirDepth: 0,
+ deadAirSpans: [],
+ highlightImageIndex: 0,
+ highlights: [],
+ outputs: {},
+ sourceRange: {},
+ startedAt: performance.now(),
+ };
+ let testInfoForOutputPaths: TestInfo | undefined;
+ const getVideoTimestamp = () => {
+ const now = performance.now();
+ return Math.round(now - (state.startedAt || now));
+ };
+ const controls: VideoModeControls = {
+ deadAir: async (action) => {
+ return await recordDeadAir(state, action);
+ },
+ getVideoTimestamp,
+ metadata: async () => {
+ if (!testInfoForOutputPaths) {
+ return metadataFor(state);
+ }
+
+ return await readVideoModeMetadata(videoModeOutputPaths(testInfoForOutputPaths).metadata, () =>
+ metadataFor(state),
+ );
+ },
+ outputPaths: () => {
+ if (!testInfoForOutputPaths) {
+ throw new Error("videoMode.outputPaths() is only available after addPlugins registers videoMode");
+ }
+
+ return videoModeOutputPaths(testInfoForOutputPaths);
+ },
+ setEndTime: (ms = getVideoTimestamp()) => {
+ state.sourceRange.end = resolveVideoTimestamp("setEndTime", ms);
+ },
+ setStartTime: (ms = getVideoTimestamp()) => {
+ state.sourceRange.start = resolveVideoTimestamp("setStartTime", ms);
+ },
+ };
return {
+ ...controls,
name: "video-mode",
+ pageExtension: ({ testInfo }) => {
+ testInfoForOutputPaths = testInfo;
+ return { videoMode: controls };
+ },
- middleware: async ({ locator, method }, next) => {
- if (skipMethods.includes(method)) return next();
+ middleware: async ({ locator, method, testInfo, timing }, next) => {
+ if (state.deadAirDepth > 0) return next();
// Skip if called from internal helpers (navigation, login flows etc)
if (skipStackFrames.length > 0) {
@@ -83,15 +1895,176 @@ export const videoMode = (options: VideoModeOptions = {}): Plugin => {
if (skipStackFrames.some((frame) => stack.includes(frame))) return next();
}
- using _ = await setupHighlight(locator, highlightStyle, pauseBefore);
- return await next();
+ if (method === "waitFor") {
+ try {
+ return await next();
+ } finally {
+ recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 0 });
+ }
+ }
+
+ recordMiddlewareWaitBeforeVideoMode(state, timing);
+
+ if (skipMethods.includes(method)) {
+ try {
+ return await next();
+ } finally {
+ if (timing.attachedAtStart) {
+ recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 });
+ }
+ recordAttachedWaitFromTiming(state, timing);
+ }
+ }
+
+ const recordedHighlight =
+ highlight.mode === "off"
+ ? undefined
+ : await recordHighlight({
+ color: highlight.color,
+ durationMs: highlight.durationMs,
+ locator,
+ method,
+ state,
+ testInfo,
+ thickness: highlight.thickness,
+ });
+
+ try {
+ return await next();
+ } finally {
+ if (recordedHighlight && state.startedAt !== undefined) {
+ recordedHighlight.actionEnd = Math.max(
+ recordedHighlight.start,
+ Math.round(performance.now() - state.startedAt),
+ );
+ }
+ if (
+ !recordedHighlight &&
+ (timing.attachedAtStart || timing.attachedAt === undefined)
+ ) {
+ recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 });
+ }
+ recordAttachedWaitFromTiming(state, timing);
+ }
},
testLifecycle: (emitter) => {
- return emitter.on("afterTest", async ({ testInfo }) => {
- await new Promise((resolve) => setTimeout(resolve, pauseAfterTest));
- console.log(`video will be written to ${testInfo.outputDir}/video.webm`);
+ const offBeforeTest = emitter.on("beforeTest", () => {
+ state.deadAirDepth = 0;
+ state.deadAirSpans = [];
+ state.highlightImageIndex = 0;
+ state.highlights = [];
+ state.outputs = {};
+ state.sourceRange = {};
+ if (state.startedAt) {
+ recordDeadAirSpan(state, {
+ end: Math.round(performance.now() - state.startedAt),
+ start: 0,
+ });
+ } else {
+ state.startedAt = performance.now();
+ }
});
+
+ const offAfterTestFinalize = emitter.on("afterTestFinalize", async ({ page, testInfo }) => {
+ const metadataBeforeVideo = metadataFor(state);
+ const deadAir = metadataBeforeVideo.deadAir;
+ const highlights = metadataBeforeVideo.highlights;
+ const sourceRange = metadataBeforeVideo.sourceRange;
+ const video = page.video();
+
+ if (video) {
+ const paths = videoModeOutputPaths(testInfo);
+ await mkdir(testInfo.outputDir, { recursive: true });
+
+ if (!page.isClosed()) {
+ await page.close({ runBeforeUnload: false });
+ }
+
+ const recordedVideoPath = await video.path();
+ await waitForNonEmptyFile(recordedVideoPath);
+ await copyFile(recordedVideoPath, paths.raw);
+ state.outputs.raw = VIDEO_MODE_RAW_FILE;
+ await testInfo.attach("video-raw", {
+ contentType: "video/webm",
+ path: paths.raw,
+ });
+
+ if (
+ highlights.length > 0 ||
+ deadAirThreshold !== undefined ||
+ finalHold > 0 ||
+ sourceRangeIsSet(sourceRange)
+ ) {
+ const wroteRenderedVideo = await renderVideo({
+ deadAir,
+ finalHoldMs: finalHold,
+ highlightMode: highlight.mode === "pointer" ? "pointer" : "outline",
+ highlights,
+ inputPath: paths.raw,
+ outputDir: testInfo.outputDir,
+ outputPath: paths.rendered,
+ sourceRange,
+ thresholdMs: deadAirThreshold,
+ });
+
+ if (wroteRenderedVideo) {
+ state.outputs.rendered = VIDEO_MODE_RENDERED_FILE;
+ await testInfo.attach("video-rendered", {
+ contentType: "video/webm",
+ path: paths.rendered,
+ });
+ }
+ }
+
+ state.outputs.player = VIDEO_MODE_PLAYER_FILE;
+ await writeFile(
+ paths.player,
+ videoModePlayerHtml({
+ raw: state.outputs.raw,
+ rendered: state.outputs.rendered,
+ }),
+ );
+
+ const reportPlayerHtml = videoModePlayerHtml({
+ raw: await playwrightReportAttachmentName(paths.raw),
+ rendered: state.outputs.rendered
+ ? await playwrightReportAttachmentName(paths.rendered)
+ : undefined,
+ });
+ await writeFile(paths.reportPlayer, reportPlayerHtml);
+ await testInfo.attach("video-mode-player", {
+ contentType: "text/html",
+ path: paths.reportPlayer,
+ });
+ }
+
+ const metadata = metadataFor(state);
+ if (
+ metadata.deadAir.length > 0 ||
+ metadata.highlights.length > 0 ||
+ metadata.outputs.player ||
+ metadata.outputs.raw ||
+ metadata.outputs.rendered ||
+ sourceRangeIsSet(metadata.sourceRange)
+ ) {
+ const path = videoModeOutputPaths(testInfo).metadata;
+ await mkdir(testInfo.outputDir, { recursive: true });
+ await writeFile(path, `${JSON.stringify(metadata, null, 2)}\n`);
+ await testInfo.attach("video-mode", {
+ contentType: "application/json",
+ path,
+ });
+ }
+
+ state.startedAt = undefined;
+ console.log(`video-mode metadata written to ${videoModeOutputPaths(testInfo).metadata}`);
+ });
+
+ return () => {
+ offBeforeTest();
+ offAfterTestFinalize();
+ };
},
};
};
diff --git a/writing-middlewright-tests.md b/writing-middlewright-tests.md
new file mode 100644
index 0000000..0f540bd
--- /dev/null
+++ b/writing-middlewright-tests.md
@@ -0,0 +1,16 @@
+## Timeouts
+
+The default `actionTimeout` in your playwright config should be *very aggressive* and short. The `spinner-waiter` plugin allows this. If and when test fail because of this, there are two recommended courses of action, neither of which involves just bumping an assertion timeout. The first is of course to just figure out why the UI is sometimes slow and fix it. But if that's not possible, or beyond the scope of the work you're doing, the second recommended fix is to add a loading spinner to the product UI - we've identified a slow part of your app, so real users should also see a loading spinner, or some text like "Loading..."/"Pending..."/"Creating foobar..." etc.
+
+## Locators over `expect`
+
+Avoid using the expect-based API for asserting that UI is visible/in a particular state. For example, don't bother with `await expect(page.getByRole("button", { name: "Run" })).toBeEnabled()` before clicking a button. Just call `await page.getByRole("button", { name: "Run" }).click()` directly. The `.click` implementation already waits for the button to exist, be visible, and to be enabled. Similarly if you want to assert that something is present on the page you can just do `await page.getByText("Welcome").waitFor()`. No need for any `await expect(...).toBeVisible()` rubbish. Similarly, no stupid assertions like `await expect(plugged.getByText("Receipt ready")).toContainText("Receipt ready");`. Just use `await page.getbyText("Receipt ready").waitFor()`.
+
+Avoid using `timeout` for actions like `click`, `waitFor` etc. Read the [middlewright docs](https://github.com/iterate/middlewright) for why (TL;DR: we should have progress UI in our app rather than bumping test timeouts): `.waitFor({ timeout: 5_000 })`
+
+Avoid doing `await myButton.waitFor()` and then `await runButton.click()`. It's another code-smell. `.click()` should _already_ wait for the button to be clickable so the `.waitFor()` is doing nothing other than give you another chance to run the test and hope for the flake gods to smile on you this time.
+
+
+## Error UI
+
+For common developer pitfalls, instead of littering your test code with defensive try/catch statements and custom selectors for app error UI, just add the `data-type="error"` attribute to relevant UI elements. Then, the `ui-error-reporter` plugin will pick up any errors on screen automatically (including toasts rendered using the `sonner` library). The plugin will find elements annotated in this way and include their text content in error reports, so agents and humans will quickly be able to get an indication of what went wrong.