[No QA] feat: Use OnyxUpdateManager to fetch pending updates from server

src/libs/Middleware/SaveResponseInOnyx.ts

@@ -25,13 +25,16 @@ const SaveResponseInOnyx: Middleware = (requestResponse, request) =>
 
         const responseToApply = {
             type: CONST.ONYX_UPDATE_TYPES.HTTPS,
-            lastUpdateID: Number(response?.lastUpdateID ?? 0),
-            previousUpdateID: Number(response?.previousUpdateID ?? 0),
+            lastUpdateID: Number(response?.lastUpdateID ?? CONST.DEFAULT_NUMBER_ID),
+            previousUpdateID: Number(response?.previousUpdateID ?? CONST.DEFAULT_NUMBER_ID),
             request,
             response: response ?? {},
         };
 
-        if (requestsToIgnoreLastUpdateID.includes(request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(response?.previousUpdateID ?? 0))) {
+        if (
+            requestsToIgnoreLastUpdateID.includes(request.command) ||
+            !OnyxUpdates.doesClientNeedToBeUpdated({previousUpdateID: Number(response?.previousUpdateID ?? CONST.DEFAULT_NUMBER_ID)})
+        ) {
             return OnyxUpdates.apply(responseToApply);
         }
 

src/libs/Notification/PushNotification/NotificationType.ts

@@ -19,6 +19,7 @@ type BasePushNotificationData = {
     onyxData?: OnyxServerUpdate[];
     lastUpdateID?: number;
     previousUpdateID?: number;
+    hasPendingOnyxUpdates?: boolean;
 };
 
 type ReportActionPushNotificationData = BasePushNotificationData & {

src/libs/Notification/PushNotification/subscribePushNotification/index.ts

@@ -43,43 +43,69 @@ function getLastUpdateIDAppliedToClient(): Promise<number> {
     return new Promise((resolve) => {
         Onyx.connect({
             key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
-            callback: (value) => resolve(value ?? 0),
+            callback: (value) => resolve(value ?? CONST.DEFAULT_NUMBER_ID),
         });
     });
 }
 
-function applyOnyxData({reportID, reportActionID, onyxData, lastUpdateID, previousUpdateID}: ReportActionPushNotificationData): Promise<void> {
+function applyOnyxData({reportID, reportActionID, onyxData, lastUpdateID, previousUpdateID, hasPendingOnyxUpdates = false}: ReportActionPushNotificationData): Promise<void> {
     Log.info(`[PushNotification] Applying onyx data in the ${Visibility.isVisible() ? 'foreground' : 'background'}`, false, {reportID, reportActionID});
 
     if (!ActiveClientManager.isClientTheLeader()) {
         Log.info('[PushNotification] received report comment notification, but ignoring it since this is not the active client');
         return Promise.resolve();
     }
 
-    if (!onyxData || !lastUpdateID || !previousUpdateID) {
-        Log.hmmm("[PushNotification] didn't apply onyx updates because some data is missing", {lastUpdateID, previousUpdateID, onyxDataCount: onyxData?.length ?? 0});
-        return Promise.resolve();
-    }
+    const logMissingOnyxDataInfo = (isDataMissing: boolean): boolean => {
+        if (isDataMissing) {
+            Log.hmmm("[PushNotification] didn't apply onyx updates because some data is missing", {lastUpdateID, previousUpdateID, onyxDataCount: onyxData?.length ?? 0});
+            return false;
+        }
 
-    Log.info('[PushNotification] reliable onyx update received', false, {lastUpdateID, previousUpdateID, onyxDataCount: onyxData?.length ?? 0});
-    const updates: OnyxUpdatesFromServer = {
-        type: CONST.ONYX_UPDATE_TYPES.AIRSHIP,
-        lastUpdateID,
-        previousUpdateID,
-        updates: [
-            {
-                eventType: '', // This is only needed for Pusher events
-                data: onyxData,
-            },
-        ],
+        Log.info('[PushNotification] reliable onyx update received', false, {lastUpdateID, previousUpdateID, onyxDataCount: onyxData?.length ?? 0});
+        return true;
     };
 
+    let updates: OnyxUpdatesFromServer;
+    if (hasPendingOnyxUpdates) {
+        const isDataMissing = !lastUpdateID;
+        logMissingOnyxDataInfo(isDataMissing);
+        if (isDataMissing) {
+            return Promise.resolve();
+        }
+
+        updates = {
+            type: CONST.ONYX_UPDATE_TYPES.AIRSHIP,
+            lastUpdateID,
+            shouldFetchPendingUpdates: true,
+            updates: [],
+        };
+    } else {
+        const isDataMissing = !lastUpdateID || !onyxData || !previousUpdateID;
+        logMissingOnyxDataInfo(isDataMissing);
+        if (isDataMissing) {
+            return Promise.resolve();
+        }
+
+        updates = {
+            type: CONST.ONYX_UPDATE_TYPES.AIRSHIP,
+            lastUpdateID,
+            previousUpdateID,
+            updates: [
+                {
+                    eventType: '', // This is only needed for Pusher events
+                    data: onyxData,
+                },
+            ],
+        };
+    }
+
     /**
      * When this callback runs in the background on Android (via Headless JS), no other Onyx.connect callbacks will run. This means that
      * lastUpdateIDAppliedToClient will NOT be populated in other libs. To workaround this, we manually read the value here
      * and pass it as a param
      */
-    return getLastUpdateIDAppliedToClient().then((lastUpdateIDAppliedToClient) => applyOnyxUpdatesReliably(updates, true, lastUpdateIDAppliedToClient));
+    return getLastUpdateIDAppliedToClient().then((lastUpdateIDAppliedToClient) => applyOnyxUpdatesReliably(updates, {shouldRunSync: true, clientLastUpdateID: lastUpdateIDAppliedToClient}));
 }
 
 function navigateToReport({reportID, reportActionID}: ReportActionPushNotificationData): Promise<void> {

src/libs/actions/OnyxUpdateManager/index.ts

@@ -6,6 +6,7 @@ import * as NetworkStore from '@libs/Network/NetworkStore';
 import * as SequentialQueue from '@libs/Network/SequentialQueue';
 import * as App from '@userActions/App';
 import updateSessionAuthTokens from '@userActions/Session/updateSessionAuthTokens';
+import CONST from '@src/CONST';
 import ONYXKEYS from '@src/ONYXKEYS';
 import type {OnyxUpdatesFromServer, Session} from '@src/types/onyx';
 import {isValidOnyxUpdateFromServer} from '@src/types/onyx/OnyxUpdatesFromServer';
@@ -27,10 +28,10 @@ import * as DeferredOnyxUpdates from './utils/DeferredOnyxUpdates';
 // The circular dependency happens because this file calls API.GetMissingOnyxUpdates() which uses the SaveResponseInOnyx.js file (as a middleware).
 // Therefore, SaveResponseInOnyx.js can't import and use this file directly.
 
-let lastUpdateIDAppliedToClient = 0;
+let lastUpdateIDAppliedToClient: number = CONST.DEFAULT_NUMBER_ID;
 Onyx.connect({
     key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
-    callback: (value) => (lastUpdateIDAppliedToClient = value ?? 0),
+    callback: (value) => (lastUpdateIDAppliedToClient = value ?? CONST.DEFAULT_NUMBER_ID),
 });
 
 let isLoadingApp = false;
@@ -48,6 +49,7 @@ const createQueryPromiseWrapper = () =>
     });
 // eslint-disable-next-line import/no-mutable-exports
 let queryPromiseWrapper = createQueryPromiseWrapper();
+let isFetchingForPendingUpdates = false;
 
 const resetDeferralLogicVariables = () => {
     DeferredOnyxUpdates.clear({shouldUnpauseSequentialQueue: false});
@@ -61,18 +63,19 @@ function finalizeUpdatesAndResumeQueue() {
     queryPromiseWrapper = createQueryPromiseWrapper();
 
     DeferredOnyxUpdates.clear();
+    isFetchingForPendingUpdates = false;
 }
 
 /**
- *
- * @param onyxUpdatesFromServer
+ * Triggers the fetching process of either pending or missing updates.
+ * @param onyxUpdatesFromServer the current update that is supposed to be applied
  * @param clientLastUpdateID an optional override for the lastUpdateIDAppliedToClient
  * @returns
  */
-function handleOnyxUpdateGap(onyxUpdatesFromServer: OnyxEntry<OnyxUpdatesFromServer>, clientLastUpdateID?: number) {
+function handleMissingOnyxUpdates(onyxUpdatesFromServer: OnyxEntry<OnyxUpdatesFromServer>, clientLastUpdateID?: number) {
     // If isLoadingApp is positive it means that OpenApp command hasn't finished yet, and in that case
-    // we don't have base state of the app (reports, policies, etc) setup. If we apply this update,
-    // we'll only have them overriten by the openApp response. So let's skip it and return.
+    // we don't have base state of the app (reports, policies, etc.) setup. If we apply this update,
+    // we'll only have them overwritten by the openApp response. So let's skip it and return.
     if (isLoadingApp) {
         // When ONYX_UPDATES_FROM_SERVER is set, we pause the queue. Let's unpause
         // it so the app is not stuck forever without processing requests.
@@ -96,58 +99,96 @@ function handleOnyxUpdateGap(onyxUpdatesFromServer: OnyxEntry<OnyxUpdatesFromSer
     // current authToken is probably invalid.
     updateAuthTokenIfNecessary(onyxUpdatesFromServer);
 
-    const updateParams = onyxUpdatesFromServer;
+    const shouldFetchPendingUpdates = onyxUpdatesFromServer?.shouldFetchPendingUpdates ?? false;
     const lastUpdateIDFromServer = onyxUpdatesFromServer.lastUpdateID;
     const previousUpdateIDFromServer = onyxUpdatesFromServer.previousUpdateID;
-    const lastUpdateIDFromClient = clientLastUpdateID ?? lastUpdateIDAppliedToClient ?? 0;
-
-    // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient
-    // we need to perform one of the 2 possible cases:
-    //
-    // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before
-    // fully migrating to the reliable updates mode.
-    // 2. This client already has the reliable updates mode enabled, but it's missing some updates and it
-    // needs to fetch those.
-
-    // The flow below is setting the promise to a reconnect app to address flow (1) explained above.
-    if (!lastUpdateIDFromClient) {
-        // If there is a ReconnectApp query in progress, we should not start another one.
-        if (DeferredOnyxUpdates.getMissingOnyxUpdatesQueryPromise()) {
-            return;
+    const lastUpdateIDFromClient = clientLastUpdateID ?? lastUpdateIDAppliedToClient ?? CONST.DEFAULT_NUMBER_ID;
+
+    // Check if the client needs to send a backend request to fetch missing or pending updates and/or queue deferred updates.
+    // Returns a boolean indicating whether we should execute the finally block after the promise is done,
+    // in which the OnyxUpdateManager finishes its work and the SequentialQueue will is unpaused.
+    const checkIfClientNeedsToBeUpdated = (): boolean => {
+        // The OnyxUpdateManager can handle different types of re-fetch processes. Either there are pending updates,
+        // that we need to fetch manually, or we detected gaps in the previously fetched updates.
+        // Each of the flows below sets a promise through `DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise`, which we further process.
+        if (shouldFetchPendingUpdates) {
+            // This flow handles the case where the server didn't send updates because the payload was too big.
+            // We need to call the GetMissingOnyxUpdates query to fetch the missing updates up to the pendingLastUpdateID.
+            const pendingUpdateID = Number(lastUpdateIDFromServer);
+
+            isFetchingForPendingUpdates = true;
+
+            // If the pendingUpdateID is not newer than the last locally applied update, we don't need to fetch the missing updates.
+            if (pendingUpdateID <= lastUpdateIDFromClient) {
+                DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise(Promise.resolve());
+                return true;
+            }
+
+            console.debug(`[OnyxUpdateManager] Client is fetching pending updates from the server, from updates ${lastUpdateIDFromClient} to ${Number(pendingUpdateID)}`);
+            Log.info('There are pending updates from the server, so fetching incremental updates', true, {
+                pendingUpdateID,
+                lastUpdateIDFromClient,
+            });
+
+            // Get the missing Onyx updates from the server and afterward validate and apply the deferred updates.
+            // This will trigger recursive calls to "validateAndApplyDeferredUpdates" if there are gaps in the deferred updates.
+            DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise(
+                App.getMissingOnyxUpdates(lastUpdateIDFromClient, lastUpdateIDFromServer).then(() => OnyxUpdateManagerUtils.validateAndApplyDeferredUpdates(clientLastUpdateID)),
+            );
+
+            return true;
         }
 
-        Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process');
+        if (!lastUpdateIDFromClient) {
+            // This is the first time we're receiving an lastUpdateID, so we need to do a final ReconnectApp query before
+            // This flow is setting the promise to a ReconnectApp query.
+
+            // If there is a ReconnectApp query in progress, we should not start another one.
+            if (DeferredOnyxUpdates.getMissingOnyxUpdatesQueryPromise()) {
+                return false;
+            }
+
+            Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process');
+
+            // Since this is a full reconnectApp, we'll not apply the updates we received - those will come in the reconnect app request.
+            DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise(App.finalReconnectAppAfterActivatingReliableUpdates());
+
+            return true;
+        }
 
-        // Since this is a full reconnectApp, we'll not apply the updates we received - those will come in the reconnect app request.
-        DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise(App.finalReconnectAppAfterActivatingReliableUpdates());
-    } else {
-        // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above.
+        // This client already has the reliable updates mode enabled, but it's missing some updates and it needs to fetch those.
+        // Therefore, we are calling the GetMissingOnyxUpdates query, to fetch the missing updates.
 
         const areDeferredUpdatesQueued = !DeferredOnyxUpdates.isEmpty();
 
         // Add the new update to the deferred updates
-        DeferredOnyxUpdates.enqueue(updateParams, {shouldPauseSequentialQueue: false});
+        DeferredOnyxUpdates.enqueue(onyxUpdatesFromServer, {shouldPauseSequentialQueue: false});
 
         // If there are deferred updates already, we don't need to fetch the missing updates again.
-        if (areDeferredUpdatesQueued) {
-            return;
+        if (areDeferredUpdatesQueued || isFetchingForPendingUpdates) {
+            return false;
         }
 
-        console.debug(`[OnyxUpdateManager] Client is behind the server by ${Number(previousUpdateIDFromServer) - lastUpdateIDFromClient} so fetching incremental updates`);
-        Log.info('Gap detected in update IDs from server so fetching incremental updates', true, {
+        console.debug(`[OnyxUpdateManager] Client is fetching missing updates from the server, from updates ${lastUpdateIDFromClient} to ${Number(previousUpdateIDFromServer)}`);
+        Log.info('Gap detected in update IDs from the server so fetching incremental updates', true, {
+            lastUpdateIDFromClient,
             lastUpdateIDFromServer,
             previousUpdateIDFromServer,
-            lastUpdateIDFromClient,
         });
 
         // Get the missing Onyx updates from the server and afterwards validate and apply the deferred updates.
         // This will trigger recursive calls to "validateAndApplyDeferredUpdates" if there are gaps in the deferred updates.
         DeferredOnyxUpdates.setMissingOnyxUpdatesQueryPromise(
             App.getMissingOnyxUpdates(lastUpdateIDFromClient, previousUpdateIDFromServer).then(() => OnyxUpdateManagerUtils.validateAndApplyDeferredUpdates(clientLastUpdateID)),
         );
-    }
 
-    DeferredOnyxUpdates.getMissingOnyxUpdatesQueryPromise()?.finally(finalizeUpdatesAndResumeQueue);
+        return true;
+    };
+    const shouldFinalizeAndResume = checkIfClientNeedsToBeUpdated();
+
+    if (shouldFinalizeAndResume) {
+        DeferredOnyxUpdates.getMissingOnyxUpdatesQueryPromise()?.finally(finalizeUpdatesAndResumeQueue);
+    }
 }
 
 function updateAuthTokenIfNecessary(onyxUpdatesFromServer: OnyxEntry<OnyxUpdatesFromServer>): void {
@@ -177,8 +218,8 @@ export default () => {
     console.debug('[OnyxUpdateManager] Listening for updates from the server');
     Onyx.connect({
         key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER,
-        callback: (value) => handleOnyxUpdateGap(value),
+        callback: (value) => handleMissingOnyxUpdates(value),
     });
 };
 
-export {handleOnyxUpdateGap, queryPromiseWrapper as queryPromise, resetDeferralLogicVariables};
+export {handleMissingOnyxUpdates, queryPromiseWrapper as queryPromise, resetDeferralLogicVariables};

src/libs/actions/OnyxUpdateManager/utils/DeferredOnyxUpdates.ts

@@ -1,6 +1,7 @@
 import Onyx from 'react-native-onyx';
 import type {DeferredUpdatesDictionary} from '@libs/actions/OnyxUpdateManager/types';
 import * as SequentialQueue from '@libs/Network/SequentialQueue';
+import CONST from '@src/CONST';
 import ONYXKEYS from '@src/ONYXKEYS';
 import type {OnyxUpdatesFromServer, Response} from '@src/types/onyx';
 import {isValidOnyxUpdateFromServer} from '@src/types/onyx/OnyxUpdatesFromServer';
@@ -40,7 +41,7 @@ function getUpdates(options?: GetDeferredOnyxUpdatesOptiosn) {
     }
 
     return Object.entries(deferredUpdates).reduce<DeferredUpdatesDictionary>((acc, [lastUpdateID, update]) => {
-        if (Number(lastUpdateID) > (options.minUpdateID ?? 0)) {
+        if (Number(lastUpdateID) > (options.minUpdateID ?? CONST.DEFAULT_NUMBER_ID)) {
             acc[Number(lastUpdateID)] = update;
         }
         return acc;

src/libs/actions/OnyxUpdateManager/utils/__mocks__/applyUpdates.ts

@@ -1,16 +1,11 @@
-import Onyx from 'react-native-onyx';
 import type {DeferredUpdatesDictionary} from '@libs/actions/OnyxUpdateManager/types';
-import ONYXKEYS from '@src/ONYXKEYS';
+import * as OnyxUpdates from '@userActions/OnyxUpdates';
 import createProxyForObject from '@src/utils/createProxyForObject';
 
-let lastUpdateIDAppliedToClient = 0;
-Onyx.connect({
-    key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
-    callback: (value) => (lastUpdateIDAppliedToClient = value ?? 0),
-});
+jest.mock('@userActions/OnyxUpdates');
 
 type ApplyUpdatesMockValues = {
-    onApplyUpdates: ((updates: DeferredUpdatesDictionary) => Promise<void>) | undefined;
+    beforeApplyUpdates: ((updates: DeferredUpdatesDictionary) => Promise<void>) | undefined;
 };
 
 type ApplyUpdatesMock = {
@@ -19,15 +14,27 @@ type ApplyUpdatesMock = {
 };
 
 const mockValues: ApplyUpdatesMockValues = {
-    onApplyUpdates: undefined,
+    beforeApplyUpdates: undefined,
 };
 const mockValuesProxy = createProxyForObject(mockValues);
 
 const applyUpdates = jest.fn((updates: DeferredUpdatesDictionary) => {
-    const lastUpdateIdFromUpdates = Math.max(...Object.keys(updates).map(Number));
-    return (mockValuesProxy.onApplyUpdates === undefined ? Promise.resolve() : mockValuesProxy.onApplyUpdates(updates)).then(() =>
-        Onyx.set(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, Math.max(lastUpdateIDAppliedToClient, lastUpdateIdFromUpdates)),
-    );
+    const createChain = () => {
+        let chain = Promise.resolve();
+        Object.values(updates).forEach((update) => {
+            chain = chain.then(() => {
+                return OnyxUpdates.apply(update).then(() => undefined);
+            });
+        });
+
+        return chain;
+    };
+
+    if (mockValuesProxy.beforeApplyUpdates === undefined) {
+        return createChain();
+    }
+
+    return mockValuesProxy.beforeApplyUpdates(updates).then(() => createChain());
 });
 
 export {applyUpdates, mockValuesProxy as mockValues};