Add replay for cached asset on attribute changes

packages/rrweb-snapshot/src/index.ts

@@ -8,6 +8,7 @@ import snapshot, {
   classMatchesRegex,
   IGNORED_NODE,
   genId,
+  getSourcesFromSrcset,
 } from './snapshot';
 import rebuild, {
   buildNodeWithSN,
@@ -32,4 +33,5 @@ export {
   classMatchesRegex,
   IGNORED_NODE,
   genId,
+  getSourcesFromSrcset,
 };

packages/rrweb-snapshot/src/rebuild.ts

@@ -261,29 +261,12 @@ function buildNode(
               n.attributes.srcset as string,
             );
             continue;
-          } else if (
-            tagName === 'img' &&
-            name === 'src' &&
-            options.assetManager
-          ) {
-            const originalValue = value.toString();
-            node.setAttribute(name, originalValue);
-            void options.assetManager
-              .whenReady(value.toString())
-              .then((status) => {
-                if (
-                  status.status === 'loaded' &&
-                  node.getAttribute('src') === originalValue
-                ) {
-                  node.setAttribute(name, status.url);
-                } else {
-                  console.log(
-                    `failed to load asset: ${originalValue}, ${status.status}`,
-                  );
-                }
-              });
           } else {
             node.setAttribute(name, value.toString());
+
+            if (options.assetManager?.isAttributeCacheable(node, name)) {
+              options.assetManager.manageAttribute(node, name);
+            }
           }
         } catch (error) {
           // skip invalid attribute

packages/rrweb-snapshot/src/snapshot.ts

@@ -122,7 +122,11 @@ export function absoluteToStylesheet(
 const SRCSET_NOT_SPACES = /^[^ \t\n\r\u000c]+/; // Don't use \s, to avoid matching non-breaking space
 // eslint-disable-next-line no-control-regex
 const SRCSET_COMMAS_OR_SPACES = /^[, \t\n\r\u000c]+/;
-function getAbsoluteSrcsetString(doc: Document, attributeValue: string) {
+function parseSrcsetString(
+  doc: Document,
+  attributeValue: string,
+  urlCallback: (doc: Document, url: string) => string,
+) {
   /*
     run absoluteToDoc over every url in the srcset
 
@@ -159,13 +163,13 @@ function getAbsoluteSrcsetString(doc: Document, attributeValue: string) {
     let url = collectCharacters(SRCSET_NOT_SPACES);
     if (url.slice(-1) === ',') {
       // aside: according to spec more than one comma at the end is a parse error, but we ignore that
-      url = absoluteToDoc(doc, url.substring(0, url.length - 1));
+      url = urlCallback(doc, url.substring(0, url.length - 1));
       // the trailing comma splits the srcset, so the interpretion is that
       // another url will follow, and the descriptor is empty
       output.push(url);
     } else {
       let descriptorsStr = '';
-      url = absoluteToDoc(doc, url);
+      url = urlCallback(doc, url);
       let inParens = false;
       // eslint-disable-next-line no-constant-condition
       while (true) {
@@ -196,6 +200,21 @@ function getAbsoluteSrcsetString(doc: Document, attributeValue: string) {
   return output.join(', ');
 }
 
+function getAbsoluteSrcsetString(doc: Document, attributeValue: string) {
+  return parseSrcsetString(doc, attributeValue, (doc, url) =>
+    absoluteToDoc(doc, url),
+  );
+}
+
+export function getSourcesFromSrcset(attributeValue: string): string[] {
+  const urls = new Set<string>();
+  parseSrcsetString(document, attributeValue, (_, url) => {
+    urls.add(url);
+    return url;
+  });
+  return Array.from(urls);
+}
+
 export function absoluteToDoc(doc: Document, attributeValue: string): string {
   if (!attributeValue || attributeValue.trim() === '') {
     return attributeValue;

packages/rrweb/src/record/mutation.ts

@@ -10,6 +10,7 @@ import {
   isNativeShadowDom,
   getInputType,
   toLowerCase,
+  getSourcesFromSrcset,
 } from 'rrweb-snapshot';
 import type { observerParam, MutationBufferParam } from '../types';
 import type {
@@ -569,12 +570,6 @@ export default class MutationBuffer {
           } else {
             return;
           }
-        } else if (
-          target.tagName === 'IMG' &&
-          (attributeName === 'src' || attributeName === 'srcset') &&
-          value
-        ) {
-          this.assetManager.capture(value);
         }
         if (!item) {
           item = {
@@ -599,12 +594,26 @@ export default class MutationBuffer {
 
         if (!ignoreAttribute(target.tagName, attributeName, value)) {
           // overwrite attribute if the mutations was triggered in same time
-          item.attributes[attributeName] = transformAttribute(
-            this.doc,
-            toLowerCase(target.tagName),
-            toLowerCase(attributeName),
-            value,
-          );
+          const transformedValue = (item.attributes[attributeName] =
+            transformAttribute(
+              this.doc,
+              toLowerCase(target.tagName),
+              toLowerCase(attributeName),
+              value,
+            ));
+          if (
+            transformedValue &&
+            this.assetManager.isAttributeCacheable(target, attributeName)
+          ) {
+            if (attributeName === 'srcset') {
+              getSourcesFromSrcset(transformedValue).forEach((url) => {
+                this.assetManager.capture(url);
+              });
+            } else {
+              this.assetManager.capture(transformedValue);
+            }
+          }
+
           if (attributeName === 'style') {
             if (!this.unattachedDoc) {
               try {

packages/rrweb/src/record/observers/asset-manager.ts

@@ -7,7 +7,7 @@ import type {
 import type { assetCallback } from '@rrweb/types';
 import { encode } from 'base64-arraybuffer';
 
-import { patch } from '../../utils';
+import { isAttributeCacheable, patch } from '../../utils';
 import type { recordOptions } from '../../types';
 
 export default class AssetManager {
@@ -191,4 +191,8 @@ export default class AssetManager {
 
     return { status: 'capturing' };
   }
+
+  public isAttributeCacheable(n: Element, attribute: string): boolean {
+    return isAttributeCacheable(n, attribute);
+  }
 }

packages/rrweb/src/replay/assets/index.ts

@@ -5,6 +5,9 @@ import type {
   assetEvent,
 } from '@rrweb/types';
 import { deserializeArg } from '../canvas/deserialize-args';
+import { isAttributeCacheable } from '../../utils';
+import { getSourcesFromSrcset } from 'rrweb-snapshot';
+import type { RRElement } from 'rrdom';
 
 export default class AssetManager implements RebuildAssetManagerInterface {
   private originalToObjectURLMap: Map<string, string> = new Map();
@@ -55,6 +58,7 @@ export default class AssetManager implements RebuildAssetManagerInterface {
     }
   }
 
+  // TODO: turn this into a true promise that throws if the asset fails to load
   public async whenReady(url: string): Promise<RebuildAssetManagerFinalStatus> {
     const currentStatus = this.get(url);
     if (
@@ -103,6 +107,43 @@ export default class AssetManager implements RebuildAssetManagerInterface {
     };
   }
 
+  public isAttributeCacheable(
+    n: RRElement | Element,
+    attribute: string,
+  ): boolean {
+    return isAttributeCacheable(n as Element, attribute);
+  }
+
+  public async manageAttribute(
+    node: RRElement | Element,
+    attribute: string,
+  ): Promise<unknown> {
+    const originalValue = node.getAttribute(attribute);
+    if (!originalValue) return false;
+
+    const promises = [];
+
+    const values =
+      attribute === 'srcset'
+        ? getSourcesFromSrcset(originalValue)
+        : [originalValue];
+    for (const value of values) {
+      promises.push(
+        this.whenReady(value).then((status) => {
+          if (
+            status.status === 'loaded' &&
+            node.getAttribute(attribute) === originalValue
+          ) {
+            node.setAttribute(attribute, status.url);
+          } else {
+            // failed to load asset, or the attribute was changed
+          }
+        }),
+      );
+    }
+    return Promise.all(promises);
+  }
+
   public reset(): void {
     this.originalToObjectURLMap.forEach((objectURL) => {
       URL.revokeObjectURL(objectURL);

packages/rrweb/src/replay/index.ts

@@ -921,6 +921,7 @@ export class Replayer {
       skipChild: false,
       afterAppend,
       cache: this.cache,
+      assetManager: this.assetManager,
     });
     afterAppend(iframeEl.contentDocument! as Document, mutation.node.id);
 
@@ -1552,6 +1553,7 @@ export class Replayer {
         skipChild: true,
         hackCss: true,
         cache: this.cache,
+        assetManager: this.assetManager,
         /**
          * caveat: `afterAppend` only gets called on child nodes of target
          * we have to call it again below when this target was added to the DOM
@@ -1764,6 +1766,7 @@ export class Replayer {
                     skipChild: true,
                     hackCss: true,
                     cache: this.cache,
+                    assetManager: this.assetManager,
                   });
                   const siblingNode = target.nextSibling;
                   const parentNode = target.parentNode;
@@ -1780,10 +1783,15 @@ export class Replayer {
                   // for safe
                 }
               }
-              (target as Element | RRElement).setAttribute(
+              const targetEl = target as Element | RRElement;
                 attributeName,
                 value,
               );
+              if (
+                this.assetManager.isAttributeCacheable(targetEl, attributeName)
+              ) {
+                void this.assetManager.manageAttribute(targetEl, attributeName);
+              }
             } catch (error) {
               this.warn(
                 'An error occurred may due to the checkout feature.',

packages/rrweb/src/utils.ts

@@ -590,3 +590,25 @@ export function inDom(n: Node): boolean {
   if (!doc) return false;
   return doc.contains(n) || shadowHostInDom(n);
 }
+
+export const CACHEABLE_ELEMENT_ATTRIBUTE_COMBINATIONS = new Map([
+  ['IMG', new Set(['src', 'srcset'])],
+  ['VIDEO', new Set(['src'])],
+  ['AUDIO', new Set(['src'])],
+  ['EMBED', new Set(['src'])],
+  ['SOURCE', new Set(['src'])],
+  ['TRACK', new Set(['src'])],
+  ['INPUT', new Set(['src'])],
+  ['IFRAME', new Set(['src'])],
+  ['OBJECT', new Set(['src'])],
+]);
+
+export function isAttributeCacheable(n: Element, attribute: string): boolean {
+  const acceptedAttributesSet = CACHEABLE_ELEMENT_ATTRIBUTE_COMBINATIONS.get(
+    n.nodeName,
+  );
+  if (!acceptedAttributesSet) {
+    return false;
+  }
+  return acceptedAttributesSet.has(attribute);
+}

packages/rrweb/test/events/assets-mutation.ts

@@ -134,7 +134,7 @@ const events: eventWithTime[] = [
         ],
       },
     },
-    timestamp: 1636379532355,
+    timestamp: 1636379531391,
   },
 ];
 

packages/rrweb/test/replay/asset-integration.test.ts

@@ -38,7 +38,9 @@ describe('replayer', function () {
     });
     await page.evaluate(code);
     await page.evaluate(`let events = ${JSON.stringify(events)}`);
-    await page.evaluate(`let events2 = ${JSON.stringify(mutationEvents)}`);
+    await page.evaluate(
+      `let mutationEvents = ${JSON.stringify(mutationEvents)}`,
+    );
 
     page.on('console', (msg) => console.log('PAGE LOG:', msg.text()));
   });
@@ -93,16 +95,16 @@ describe('replayer', function () {
       window.replayer = new Replayer([], {
         liveMode: true,
       });
-      replayer.startLive();
-      window.replayer.addEvent(events2[0]);
-      window.replayer.addEvent(events2[1]);
-      window.replayer.addEvent(events2[2]);
+      replayer.startLive(mutationEvents[0].timestamp);
+      window.replayer.addEvent(mutationEvents[0]);
+      window.replayer.addEvent(mutationEvents[1]);
+      window.replayer.addEvent(mutationEvents[2]);
     `);
 
       await waitForRAF(page);
 
       await page.evaluate(`
-        window.replayer.addEvent(events2[3]);
+        window.replayer.addEvent(mutationEvents[3]);
       `);
 
       await waitForRAF(page);
@@ -111,6 +113,7 @@ describe('replayer', function () {
       expect(image).toMatchImageSnapshot();
     });
 
+    test.todo("should support work through rrelement's too");
     test.todo('should support video elements');
     test.todo('should support audio elements');
     test.todo('should support embed elements');