Merge pull request #42991 from margelo/fix/e2e-tests-loading-screen-v2

[NoQA] fix: e2e long loading

Author: mountiny

tests/e2e/compare/compare.ts

@@ -50,7 +50,7 @@ function buildCompareEntry(name: string, compare: Stats, baseline: Stats): Entry
 /**
  * Compare results between baseline and current entries and categorize.
  */
-function compareResults(compareEntries: Metric | string, baselineEntries: Metric | string) {
+function compareResults(baselineEntries: Metric | string, compareEntries: Metric | string = baselineEntries) {
     // Unique test scenario names
     const baselineKeys = Object.keys(baselineEntries ?? {});
     const names = Array.from(new Set([...baselineKeys]));
@@ -81,8 +81,8 @@ function compareResults(compareEntries: Metric | string, baselineEntries: Metric
 }
 
 export default (main: Metric | string, delta: Metric | string, outputFile: string, outputFormat = 'all') => {
-    // IMPORTANT NOTE: make sure you are passing the delta/compare results first, then the main/baseline results:
-    const outputData = compareResults(delta, main);
+    // IMPORTANT NOTE: make sure you are passing the main/baseline results first, then the delta/compare results:
+    const outputData = compareResults(main, delta);
 
     if (outputFormat === 'console' || outputFormat === 'all') {
         printToConsole(outputData);

tests/e2e/config.ts

@@ -68,7 +68,7 @@ export default {
     TESTS_CONFIG: {
         [TEST_NAMES.AppStartTime]: {
             name: TEST_NAMES.AppStartTime,
-
+            warmupRuns: 1,
             // ... any additional config you might need
         },
         [TEST_NAMES.OpenChatFinderPage]: {

tests/e2e/testRunner.ts

@@ -156,6 +156,10 @@ const runTests = async (): Promise<void> => {
     for (let testIndex = 0; testIndex < tests.length; testIndex++) {
         const test = Object.values(config.TESTS_CONFIG)[testIndex];
 
+        // re-instal app for each new test suite
+        await installApp(config.MAIN_APP_PACKAGE, mainAppPath);
+        await installApp(config.DELTA_APP_PACKAGE, deltaAppPath);
+
         // check if we want to skip the test
         if (args.includes('--includes')) {
             const includes = args[args.indexOf('--includes') + 1];
@@ -177,11 +181,17 @@ const runTests = async (): Promise<void> => {
 
         const warmupText = `Warmup for test '${test.name}' [${testIndex + 1}/${tests.length}]`;
 
-        // Warmup the main app:
-        await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}`);
+        // by default we do 2 warmups:
+        // - first warmup to pass a login flow
+        // - second warmup to pass an actual flow and cache network requests
+        const iterations = test.warmupRuns ?? 2;
+        for (let i = 0; i < iterations; i++) {
+            // Warmup the main app:
+            await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`);
 
-        // Warmup the delta app:
-        await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}`);
+            // Warmup the delta app:
+            await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`);
+        }
 
         // For each test case we allow the test to fail three times before we stop the test run:
         const errorCountRef = {