feat(list-reporter): add printFailuresInline option (#40875)

Author: pavelfeldman

docs/src/test-reporters-js.md

@@ -70,7 +70,7 @@ export default defineConfig({
 });
 ```
 
-Here is an example output in the middle of a test run. Failures will be listed at the end.
+Here is an example output in the middle of a test run. Failures will be listed at the end by default.
 ```bash
 npx playwright test --reporter=list
 Running 124 tests using 6 workers
@@ -97,11 +97,22 @@ export default defineConfig({
 });
 ```
 
+You can print failures inline as soon as they are available instead of waiting until the end of the run:
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: [['list', { printFailuresInline: true }]],
+});
+```
+
 List report supports the following configuration options and environment variables:
 
 | Environment Variable Name | Reporter Config Option| Description | Default
 |---|---|---|---|
 | `PLAYWRIGHT_LIST_PRINT_STEPS` | `printSteps` | Whether to print each step on its own line. | `false`
+| `PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE` | `printFailuresInline` | Whether to print failure details immediately after a failed test instead of at the end. | `false`
 | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise.
 | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise.
 | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset

packages/playwright/src/reporters/list.ts

@@ -38,11 +38,14 @@ class ListReporter extends TerminalReporter {
   private _stepIndex = new Map<TestStep, string>();
   private _needNewLine = false;
   private _printSteps: boolean;
+  private _printFailuresInline: boolean;
+  private _failureIndex = 0;
   private _paused = new Set<TestResult>();
 
   constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {
     super(options);
     this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);
+    this._printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline);
   }
 
   override onBegin(suite: Suite) {
@@ -191,6 +194,15 @@ class ListReporter extends TerminalReporter {
     const wasPaused = this._paused.delete(result);
     if (!wasPaused)
       this._updateTestLine(test, result);
+    if (!wasPaused && this._printFailuresInline && !this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted'))
+      this._printFailure(test);
+  }
+
+  private _printFailure(test: TestCase) {
+    this._maybeWriteNewLine();
+    const message = '\n' + this.formatFailure(test, ++this._failureIndex) + '\n';
+    this._updateLineCountAndNewLineFlagForOutput(message);
+    this.screen.stdout.write(message);
   }
 
   private _updateTestLine(test: TestCase, result: TestResult) {
@@ -290,7 +302,7 @@ class ListReporter extends TerminalReporter {
   override async onEnd(result: FullResult) {
     await super.onEnd(result);
     this.screen.stdout.write('\n');
-    this.epilogue(true);
+    this.epilogue(!this._printFailuresInline);
   }
 }
 

packages/playwright/types/test.d.ts

@@ -19,7 +19,7 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions,
 export * from 'playwright-core';
 
 export type BlobReporterOptions = { outputDir?: string, fileName?: string };
-export type ListReporterOptions = { printSteps?: boolean };
+export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean };
 export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
 export type JsonReporterOptions = { outputFile?: string };
 export type HtmlReporterOptions = {

tests/playwright-test/reporter-list.spec.ts

@@ -259,6 +259,35 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       expect(result.exitCode).toBe(1);
     });
 
+    test('print failures inline with option', async ({ runInlineTest }) => {
+      const result = await runInlineTest({
+        'playwright.config.ts': `
+          module.exports = {
+            reporter: [['list', { printFailuresInline: true }]],
+            workers: 1,
+          };
+        `,
+        'a.test.ts': `
+          import { test, expect } from '@playwright/test';
+          test('fails early', async ({}) => {
+            expect(1).toBe(2);
+          });
+          test('runs later', async ({}) => {
+          });
+        `,
+      });
+      const text = result.output;
+      const failureHeader = '1) a.test.ts:3:15 › fails early';
+      const laterTestStatus = `${POSITIVE_STATUS_MARK} 2 a.test.ts:6:15 › runs later`;
+      const failureIndex = text.indexOf(failureHeader);
+      const laterTestIndex = text.indexOf(laterTestStatus);
+      expect(failureIndex).toBeGreaterThan(-1);
+      expect(laterTestIndex).toBeGreaterThan(failureIndex);
+      expect(text.indexOf('Error: expect(received).toBe(expected)', failureIndex)).toBeGreaterThan(failureIndex);
+      expect(text.indexOf(failureHeader, failureIndex + 1)).toBe(-1);
+      expect(result.exitCode).toBe(1);
+    });
+
     test('print stdio', async ({ runInlineTest }) => {
       const result = await runInlineTest({
         'a.test.ts': `

utils/generate_types/overrides-test.d.ts

@@ -18,7 +18,7 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions,
 export * from 'playwright-core';
 
 export type BlobReporterOptions = { outputDir?: string, fileName?: string };
-export type ListReporterOptions = { printSteps?: boolean };
+export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean };
 export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
 export type JsonReporterOptions = { outputFile?: string };
 export type HtmlReporterOptions = {