[JS] Add ability to assert against visual results, Add upload retry

visual-js/.changeset/swift-rings-sip.md

@@ -0,0 +1,6 @@
+---
+"@saucelabs/visual-playwright": minor
+"@saucelabs/visual-storybook": patch
+---
+
+Add ability to assert on visual results in Playwright

visual-js/visual-playwright/src/api.ts

@@ -1,6 +1,7 @@
 import type { Page } from 'playwright-core';
 import {
   BuildStatus,
+  DiffStatus,
   downloadDomScript,
   getApi as getVisualApi,
   getDomScript,
@@ -16,6 +17,7 @@ const clientVersion = 'PKG_VERSION';
 
 export class VisualPlaywright {
   constructor(public client: string = `visual-playwright/${clientVersion}`) {}
+  uploadedDiffIds: Record<string, string[]> = {};
 
   public get api() {
     let api = globalThis.visualApi;
@@ -135,11 +137,12 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       testName: string | undefined;
       suiteName: string | undefined;
       deviceName: string | undefined;
+      testId: string;
     },
     name: string,
     options?: Partial<SauceVisualParams>,
   ) {
-    const { testName, suiteName, deviceName } = info;
+    const { testName, suiteName, deviceName, testId } = info;
     const { buildId } = getOpts();
 
     if (!buildId) {
@@ -156,7 +159,13 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       ignoreRegions: userIgnoreRegions,
       diffingMethod,
     } = options ?? {};
-    const { animations = 'disabled', caret } = screenshotOptions;
+    const {
+      animations = 'disabled',
+      caret,
+      fullPage = true,
+      style,
+      timeout,
+    } = screenshotOptions;
     let ignoreRegions: RegionIn[] = [];
 
     const promises: Promise<unknown>[] = [
@@ -270,7 +279,9 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
     const devicePixelRatio = await page.evaluate(() => window.devicePixelRatio);
 
     const screenshotBuffer = await page.screenshot({
-      fullPage: true,
+      fullPage,
+      style,
+      timeout,
       animations,
       caret,
       clip,
@@ -318,12 +329,64 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       diffingMethod,
     });
 
-    await this.api.createSnapshot({
+    const { diffs } = await this.api.createSnapshot({
       ...meta,
       testName,
       suiteName,
       uploadUuid: uploadId,
     });
+
+    if (!this.uploadedDiffIds[testId]) this.uploadedDiffIds[testId] = [];
+
+    this.uploadedDiffIds[testId].push(...diffs.nodes.map((diff) => diff.id));
+  }
+
+  public async visualResults({ testId }: { testId: string }) {
+    const initialStatusSummary: Record<DiffStatus, number> = {
+      [DiffStatus.Approved]: 0,
+      [DiffStatus.Equal]: 0,
+      [DiffStatus.Unapproved]: 0,
+      [DiffStatus.Rejected]: 0,
+      [DiffStatus.Queued]: 0,
+      [DiffStatus.Errored]: 0,
+    };
+    const { buildId } = getOpts();
+    if (!buildId) {
+      throw new Error(
+        'No Sauce Visual build present, cannot determine visual results.',
+      );
+    }
+
+    // Bypass all API requests if we have no uploaded diff IDs for this test.
+    if (
+      !this.uploadedDiffIds[testId] ||
+      this.uploadedDiffIds[testId].length === 0
+    ) {
+      return initialStatusSummary;
+    }
+
+    return backOff(
+      async () => {
+        const summary = { ...initialStatusSummary };
+        const diffsForTestResult = await this.api.diffsForTestResult(buildId);
+        if (!diffsForTestResult) {
+          return summary;
+        }
+        const filterDiffsById = (diff: { id: string; status: DiffStatus }) =>
+          this.uploadedDiffIds[testId].includes(diff.id);
+
+        return diffsForTestResult.nodes
+          .filter(filterDiffsById)
+          .reduce((statusSummary, diff) => {
+            statusSummary[diff.status]++;
+            return statusSummary;
+          }, summary);
+      },
+      {
+        maxDelay: 10000,
+        numOfAttempts: 10,
+      },
+    );
   }
 
   /**

visual-js/visual-playwright/src/fixtures.ts

@@ -1,11 +1,23 @@
 import { TestFixture, TestInfo } from '@playwright/test';
 import { Page } from 'playwright-core';
 import { SauceVisualParams } from './types';
-import { sauceVisualCheck } from './playwright';
+import { sauceVisualCheck, sauceVisualResults } from './playwright';
+import { DiffStatus } from '@saucelabs/visual';
 
 export type SauceVisualFixtures = {
   sauceVisual: {
+    /**
+     * Takes a snapshot of the current page and uploads it to Sauce Labs for visual diffing.
+     * @param name
+     * @param options
+     */
     visualCheck: (name: string, options?: SauceVisualParams) => Promise<void>;
+    /**
+     * Returns the visual results for the active test. Can only be used inside a
+     * `test` or `afterEach` block since it uses the current test context for finding matching
+     * visual results.
+     */
+    sauceVisualResults: () => Promise<Record<DiffStatus, number>>;
   };
 };
 
@@ -20,6 +32,7 @@ export const sauceVisualFixtures: (defaultOptions?: SauceVisualParams) => {
           ...options,
         });
       },
+      sauceVisualResults: async () => await sauceVisualResults(testInfo),
     });
   },
 });

visual-js/visual-playwright/src/index.ts

@@ -2,6 +2,7 @@ export {
   sauceVisualSetup,
   sauceVisualTeardown,
   sauceVisualCheck,
+  sauceVisualResults,
 } from './playwright';
 export { sauceVisualFixtures, SauceVisualFixtures } from './fixtures';
 export { SauceVisualParams } from './types';

visual-js/visual-playwright/src/playwright.ts

@@ -10,6 +10,10 @@ export const sauceVisualSetup = async (opts?: Partial<VisualEnvOpts>) => {
 };
 export const sauceVisualTeardown =
   VisualPlaywright.teardown.bind(VisualPlaywright);
+
+/**
+ * Takes a snapshot of the current page and uploads it to Sauce Labs for visual diffing.
+ */
 export const sauceVisualCheck = async (
   page: Page,
   testInfo: TestInfo,
@@ -22,7 +26,18 @@ export const sauceVisualCheck = async (
       deviceName: testInfo.project.name,
       testName: testInfo.title,
       suiteName: testInfo.titlePath.slice(0, -1).join('/'),
+      testId: testInfo.testId,
     },
     name,
     options,
   );
+
+/**
+ * Returns the visual results for the active test. Can only be used inside a
+ * `test` or `afterEach` block since it uses the current test context for finding matching
+ * visual results.
+ */
+export const sauceVisualResults = async (testInfo: TestInfo) =>
+  await VisualPlaywright.visualResults({
+    testId: testInfo.testId,
+  });

visual-js/visual-playwright/src/types.ts

@@ -2,7 +2,10 @@ import { DiffingMethod, RegionIn, VisualEnvOpts } from '@saucelabs/visual';
 import { PageScreenshotOptions } from 'playwright-core';
 
 export interface SauceVisualParams {
-  screenshotOptions?: Pick<PageScreenshotOptions, 'animations' | 'caret'>;
+  screenshotOptions?: Pick<
+    PageScreenshotOptions,
+    'animations' | 'caret' | 'fullPage' | 'style' | 'timeout'
+  >;
   /**
    * Whether we should capture a dom snapshot.
    */

visual-js/visual-storybook/src/api.ts

@@ -33,6 +33,7 @@ export const postVisit = async (page: Page, context: TestContext) => {
       testName: undefined,
       suiteName: undefined,
       deviceName: deviceName || undefined,
+      testId: context.id,
     },
     `${context.title}/${context.name}`,
     {

visual-js/yarn.lock

@@ -3577,7 +3577,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@saucelabs/visual-playwright@^0.1.4, @saucelabs/visual-playwright@workspace:visual-playwright":
+"@saucelabs/visual-playwright@^0.1.5, @saucelabs/visual-playwright@workspace:visual-playwright":
   version: 0.0.0-use.local
   resolution: "@saucelabs/visual-playwright@workspace:visual-playwright"
   dependencies:
@@ -3613,7 +3613,7 @@ __metadata:
   dependencies:
     "@jest/globals": ^28.0.0 || ^29.0.0
     "@saucelabs/visual": ^0.9.0
-    "@saucelabs/visual-playwright": ^0.1.4
+    "@saucelabs/visual-playwright": ^0.1.5
     "@storybook/test-runner": ">=0.13.0"
     "@storybook/types": ^8.0.2
     "@tsconfig/node18": ^2.0.0