feat: safer rrweb events and regular full snapshots (#973)

## regular full snapshots

we currently only take a full snapshot at the start of a recording or on return from idle

so if someone has an active and complex 20 minute recording we accrue 20 minutes of incremental snapshots. this has performance impact on playback and on seeking

instead now, when we start recording we set an interval to take another full snapshot in 5 minutes

if we see a fullsnapshot before that we reset the timer, so that we would take a full snapshot 5 minutes after that

## safer rrweb events

we believe we are missing full snapshots and custom events - maybe on return from idle

this adds a simple in-memory queue so that if we cannot take one of these triggered events, we will retry the next time rrweb emits an event

it also adds a custom event to report this happened so that we can track how often and by how much this is happening (cos it might not help 🤷)

src/__tests__/extensions/replay/sessionrecording.test.ts

@@ -1445,4 +1445,64 @@ describe('SessionRecording', () => {
             expect(sessionRecording['buffer']?.data.length).toBe(undefined)
         })
     })
+
+    describe('when rrweb is not available', () => {
+        beforeEach(() => {
+            sessionRecording.afterDecideResponse(makeDecideResponse({ sessionRecording: { endpoint: '/s/' } }))
+            sessionRecording.startRecordingIfEnabled()
+            expect(loadScript).toHaveBeenCalled()
+
+            // fake that rrweb is not available
+            sessionRecording['rrwebRecord'] = undefined
+
+            expect(sessionRecording['queuedRRWebEvents']).toHaveLength(0)
+
+            sessionRecording['_tryAddCustomEvent']('test', { test: 'test' })
+        })
+
+        it('queues events', () => {
+            expect(sessionRecording['queuedRRWebEvents']).toHaveLength(1)
+        })
+
+        it('limits the queue of events', () => {
+            expect(sessionRecording['queuedRRWebEvents']).toHaveLength(1)
+
+            for (let i = 0; i < 100; i++) {
+                sessionRecording['_tryAddCustomEvent']('test', { test: 'test' })
+            }
+
+            expect(sessionRecording['queuedRRWebEvents']).toHaveLength(10)
+        })
+
+        it('processes the queue when rrweb is available again', () => {
+            // fake that rrweb is available again
+            sessionRecording['rrwebRecord'] = assignableWindow.rrwebRecord
+
+            _emit(createIncrementalSnapshot({ data: { source: 1 } }))
+
+            expect(sessionRecording['queuedRRWebEvents']).toHaveLength(0)
+            expect(sessionRecording['rrwebRecord']).not.toBeUndefined()
+        })
+    })
+
+    describe('scheduled full snapshots', () => {
+        it('starts out unscheduled', () => {
+            expect(sessionRecording['_fullSnapshotTimer']).toBe(undefined)
+        })
+
+        it('schedules a snapshot on start', () => {
+            sessionRecording.startRecordingIfEnabled()
+            expect(sessionRecording['_fullSnapshotTimer']).not.toBe(undefined)
+        })
+
+        it('reschedules a snapshot, when we take a full snapshot', () => {
+            sessionRecording.startRecordingIfEnabled()
+            const startTimer = sessionRecording['_fullSnapshotTimer']
+
+            _emit(createFullSnapshot())
+
+            expect(sessionRecording['_fullSnapshotTimer']).not.toBe(undefined)
+            expect(sessionRecording['_fullSnapshotTimer']).not.toBe(startTimer)
+        })
+    })
 })

src/extensions/replay/sessionrecording.ts

@@ -29,7 +29,9 @@ import { userOptedOut } from '../../gdpr-utils'
 
 const BASE_ENDPOINT = '/s/'
 
-export const RECORDING_IDLE_ACTIVITY_TIMEOUT_MS = 5 * 60 * 1000 // 5 minutes
+const FIVE_MINUTES = 1000 * 60 * 5
+const TWO_SECONDS = 2000
+export const RECORDING_IDLE_ACTIVITY_TIMEOUT_MS = FIVE_MINUTES
 export const RECORDING_MAX_EVENT_SIZE = 1024 * 1024 * 0.9 // ~1mb (with some wiggle room)
 export const RECORDING_BUFFER_TIMEOUT = 2000 // 2 seconds
 export const SESSION_RECORDING_BATCH_KEY = 'recordings'
@@ -84,11 +86,29 @@ interface SnapshotBuffer {
     windowId: string | null
 }
 
+interface QueuedRRWebEvent {
+    rrwebMethod: () => void
+    attempt: number
+    // the timestamp this was first put into this queue
+    enqueuedAt: number
+}
+
+const newQueuedEvent = (rrwebMethod: () => void): QueuedRRWebEvent => ({
+    rrwebMethod,
+    enqueuedAt: Date.now(),
+    attempt: 1,
+})
+
 export class SessionRecording {
     private instance: PostHog
     private _endpoint: string
     private flushBufferTimer?: any
+
+    // we have a buffer - that contains PostHog snapshot events ready to be sent to the server
     private buffer?: SnapshotBuffer
+    // and a queue - that contains rrweb events that we want to send to rrweb, but rrweb wasn't able to accept them yet
+    private queuedRRWebEvents: QueuedRRWebEvent[] = []
+
     private mutationRateLimiter?: MutationRateLimiter
     private _captureStarted: boolean
     private stopRrweb: listenerHandler | undefined
@@ -107,6 +127,8 @@ export class SessionRecording {
     private _canvasFps: number | null = null
     private _canvasQuality: number | null = null
 
+    private _fullSnapshotTimer?: number
+
     // Util to help developers working on this feature manually override
     _forceAllowLocalhostNetworkCapture = false
 
@@ -464,26 +486,29 @@ export class SessionRecording {
         }
     }
 
-    private _tryRRwebMethod(rrwebMethod: () => void): boolean {
-        if (!this._captureStarted) {
-            return false
-        }
+    private _tryRRWebMethod(queuedRRWebEvent: QueuedRRWebEvent): boolean {
         try {
-            rrwebMethod()
+            queuedRRWebEvent.rrwebMethod()
             return true
         } catch (e) {
             // Sometimes a race can occur where the recorder is not fully started yet
-            logger.error('[Session-Recording] using rrweb when not started.', e)
+            logger.warn('[Session-Recording] could not emit queued rrweb event.', e)
+            this.queuedRRWebEvents.length < 10 &&
+                this.queuedRRWebEvents.push({
+                    enqueuedAt: queuedRRWebEvent.enqueuedAt || Date.now(),
+                    attempt: queuedRRWebEvent.attempt++,
+                    rrwebMethod: queuedRRWebEvent.rrwebMethod,
+                })
             return false
         }
     }
 
     private _tryAddCustomEvent(tag: string, payload: any): boolean {
-        return this._tryRRwebMethod(() => this.rrwebRecord?.addCustomEvent(tag, payload))
+        return this._tryRRWebMethod(newQueuedEvent(() => this.rrwebRecord!.addCustomEvent(tag, payload)))
     }
 
     private _tryTakeFullSnapshot(): boolean {
-        return this._tryRRwebMethod(() => this.rrwebRecord?.takeFullSnapshot())
+        return this._tryRRWebMethod(newQueuedEvent(() => this.rrwebRecord!.takeFullSnapshot()))
     }
 
     private _onScriptLoaded() {
@@ -547,6 +572,11 @@ export class SessionRecording {
                 },
             })
 
+        // rrweb takes a snapshot on initialization,
+        // we want to take one in five minutes
+        // if nothing else happens to reset the timer
+        this._scheduleFullSnapshot()
+
         const activePlugins = this._gatherRRWebPlugins()
         this.stopRrweb = this.rrwebRecord({
             emit: (event) => {
@@ -584,6 +614,16 @@ export class SessionRecording {
         })
     }
 
+    private _scheduleFullSnapshot(): void {
+        if (this._fullSnapshotTimer) {
+            clearInterval(this._fullSnapshotTimer)
+        }
+
+        this._fullSnapshotTimer = setInterval(() => {
+            this._tryTakeFullSnapshot()
+        }, FIVE_MINUTES) // 5 minutes
+    }
+
     private _gatherRRWebPlugins() {
         const plugins: RecordPlugin<unknown>[] = []
 
@@ -609,6 +649,8 @@ export class SessionRecording {
     }
 
     onRRwebEmit(rawEvent: eventWithTime) {
+        this._processQueuedEvents()
+
         if (!rawEvent || !_isObject(rawEvent)) {
             return
         }
@@ -621,6 +663,11 @@ export class SessionRecording {
             rawEvent.data.href = href
         }
 
+        if (rawEvent.type === EventType.FullSnapshot) {
+            // we're processing a full snapshot, so we should reset the timer
+            this._scheduleFullSnapshot()
+        }
+
         const throttledEvent = this.mutationRateLimiter
             ? this.mutationRateLimiter.throttleMutations(rawEvent)
             : rawEvent
@@ -654,6 +701,40 @@ export class SessionRecording {
         }
     }
 
+    private _processQueuedEvents() {
+        if (this.queuedRRWebEvents.length) {
+            // if rrweb isn't ready to accept events earlier then we queued them up
+            // now that emit has been called rrweb should be ready to accept them
+            // so, before we process this event, we try our queued events _once_ each
+            // we don't want to risk queuing more things and never exiting this loop!
+            // if they fail here, they'll be pushed into a new queue,
+            // and tried on the next loop.
+            // there is a risk of this queue growing in an uncontrolled manner,
+            // so its length is limited elsewhere
+            // for now this is to help us ensure we can capture events that happen
+            // and try to identify more about when it is failing
+            const itemsToProcess = [...this.queuedRRWebEvents]
+            this.queuedRRWebEvents = []
+            itemsToProcess.forEach((queuedRRWebEvent) => {
+                if (Date.now() - queuedRRWebEvent.enqueuedAt > TWO_SECONDS) {
+                    this._tryAddCustomEvent('rrwebQueueTimeout', {
+                        enqueuedAt: queuedRRWebEvent.enqueuedAt,
+                        attempt: queuedRRWebEvent.attempt,
+                        queueLength: itemsToProcess.length,
+                    })
+                } else {
+                    if (this._tryRRWebMethod(queuedRRWebEvent)) {
+                        this._tryAddCustomEvent('rrwebQueueSuccess', {
+                            enqueuedAt: queuedRRWebEvent.enqueuedAt,
+                            attempt: queuedRRWebEvent.attempt,
+                            queueLength: itemsToProcess.length,
+                        })
+                    }
+                }
+            })
+        }
+    }
+
     private _maskUrl(url: string): string | undefined {
         const userSessionRecordingOptions = this.instance.config.session_recording