[metadata] set bypass ua regex string for ppr routes (#75977)

packages/next/src/build/index.ts

@@ -80,7 +80,6 @@ import {
   UNDERSCORE_NOT_FOUND_ROUTE_ENTRY,
   UNDERSCORE_NOT_FOUND_ROUTE,
   DYNAMIC_CSS_MANIFEST,
-  RESPONSE_CONFIG_MANIFEST,
 } from '../shared/lib/constants'
 import {
   getSortedRoutes,
@@ -1341,24 +1340,6 @@ export default async function build(
         NextBuildContext.clientRouterFilters = clientRouterFilters
       }
 
-      if (config.experimental.streamingMetadata) {
-        // Write html limited bots config to response-config-manifest
-        const responseConfigManifestPath = path.join(
-          distDir,
-          RESPONSE_CONFIG_MANIFEST
-        )
-        const responseConfigManifest: {
-          version: number
-          htmlLimitedBots: string
-        } = {
-          version: 0,
-          htmlLimitedBots:
-            config.experimental.htmlLimitedBots ||
-            HTML_LIMITED_BOT_UA_RE_STRING,
-        }
-        await writeManifest(responseConfigManifestPath, responseConfigManifest)
-      }
-
       // Ensure commonjs handling is used for files in the distDir (generally .next)
       // Files outside of the distDir can be "type": "module"
       await writeFileUtf8(
@@ -2269,9 +2250,6 @@ export default async function build(
               path.join(SERVER_DIRECTORY, FUNCTIONS_CONFIG_MANIFEST),
               path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
               path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'),
-              ...(config.experimental.streamingMetadata
-                ? [RESPONSE_CONFIG_MANIFEST]
-                : []),
               ...(!process.env.TURBOPACK
                 ? [
                     path.join(
@@ -2778,6 +2756,10 @@ export default async function build(
                 ? true
                 : undefined
 
+            const htmlBotsRegexString =
+              config.experimental.htmlLimitedBots ||
+              HTML_LIMITED_BOT_UA_RE_STRING
+
             // this flag is used to selectively bypass the static cache and invoke the lambda directly
             // to enable server actions on static routes
             const bypassFor: RouteHas[] = [
@@ -2787,6 +2769,17 @@ export default async function build(
                 key: 'content-type',
                 value: 'multipart/form-data;.*',
               },
+              // If it's PPR rendered non-static page, bypass the PPR cache when streaming metadata is enabled.
+              // This will skip the postpone data for those bots requests and instead produce a dynamic render.
+              ...(isRoutePPREnabled && config.experimental.streamingMetadata
+                ? [
+                    {
+                      type: 'header',
+                      key: 'user-agent',
+                      value: htmlBotsRegexString,
+                    },
+                  ]
+                : []),
             ]
 
             // We should collect all the dynamic routes into a single array for

packages/next/src/shared/lib/constants.ts

@@ -42,7 +42,6 @@ export const EXPORT_DETAIL = 'export-detail.json'
 export const PRERENDER_MANIFEST = 'prerender-manifest.json'
 export const ROUTES_MANIFEST = 'routes-manifest.json'
 export const IMAGES_MANIFEST = 'images-manifest.json'
-export const RESPONSE_CONFIG_MANIFEST = 'response-config-manifest.json'
 export const SERVER_FILES_MANIFEST = 'required-server-files.json'
 export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json'
 export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json'

test/production/app-dir/metadata-streaming-config/app/dynamic/page.tsx

@@ -0,0 +1,6 @@
+import { connection } from 'next/server'
+
+export default async function Page() {
+  await connection()
+  return <p>dynamic</p>
+}

test/production/app-dir/metadata-streaming-config/app/ppr/layout.tsx

@@ -0,0 +1,5 @@
+import { Suspense } from 'react'
+
+export default function Root({ children }) {
+  return <Suspense fallback={<p>Loading...</p>}>{children}</Suspense>
+}

test/production/app-dir/metadata-streaming-config/app/ppr/page.tsx

@@ -0,0 +1,8 @@
+import { connection } from 'next/server'
+
+export default async function Page() {
+  await connection()
+  return <p>ppr</p>
+}
+
+export const experimental_ppr = true

test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts

@@ -7,6 +7,7 @@ describe('app-dir - metadata-streaming-config-customized', () => {
       'next.config.js': `
         module.exports = {
           experimental: {
+            ppr: 'incremental',
             streamingMetadata: true,
             htmlLimitedBots: /MyBot/i,
           }
@@ -15,25 +16,27 @@ describe('app-dir - metadata-streaming-config-customized', () => {
     },
   })
 
-  it('should have the default streaming metadata config output in routes-manifest.json', async () => {
-    const requiredServerFiles = JSON.parse(
-      await next.readFile('.next/required-server-files.json')
+  it('should have the customized streaming metadata config output in routes-manifest.json', async () => {
+    const prerenderManifest = JSON.parse(
+      await next.readFile('.next/prerender-manifest.json')
     )
-    expect(requiredServerFiles.files).toContain(
-      '.next/response-config-manifest.json'
-    )
-    expect(
-      requiredServerFiles.config.experimental.htmlLimitedBots
-    ).toMatchInlineSnapshot(`"MyBot"`)
+    const { routes } = prerenderManifest
 
-    const responseConfigManifest = JSON.parse(
-      await next.readFile('.next/response-config-manifest.json')
-    )
+    const bypassConfigs = Object.keys(routes)
+      .map((route) => [route, routes[route].experimentalBypassFor?.[2]])
+      .filter(([, bypassConfig]) => Boolean(bypassConfig))
+      .reduce((acc, [route, bypassConfig]) => {
+        acc[route] = bypassConfig
+        return acc
+      }, {})
 
-    expect(responseConfigManifest).toMatchInlineSnapshot(`
+    expect(bypassConfigs).toMatchInlineSnapshot(`
      {
-       "htmlLimitedBots": "MyBot",
-       "version": 0,
+       "/ppr": {
+         "key": "user-agent",
+         "type": "header",
+         "value": "MyBot",
+       },
      }
     `)
   })

test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts

@@ -9,23 +9,39 @@ describe('app-dir - metadata-streaming-config', () => {
     const requiredServerFiles = JSON.parse(
       await next.readFile('.next/required-server-files.json')
     )
-    expect(requiredServerFiles.files).toContain(
-      '.next/response-config-manifest.json'
-    )
+
     expect(
       requiredServerFiles.config.experimental.htmlLimitedBots
     ).toMatchInlineSnapshot(
       `"Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview"`
     )
 
-    const responseConfigManifest = JSON.parse(
-      await next.readFile('.next/response-config-manifest.json')
+    const prerenderManifest = JSON.parse(
+      await next.readFile('.next/prerender-manifest.json')
     )
+    const { routes } = prerenderManifest
+
+    const bypassConfigs = Object.keys(routes)
+      // Pick the user-agent bypass config of each route
+      .map((route) => [
+        route,
+        routes[route].experimentalBypassFor?.find(
+          (bypassConfig) => bypassConfig.key === 'user-agent'
+        ),
+      ])
+      .filter(([, bypassConfig]) => Boolean(bypassConfig))
+      .reduce((acc, [route, bypassConfig]) => {
+        acc[route] = bypassConfig
+        return acc
+      }, {})
 
-    expect(responseConfigManifest).toMatchInlineSnapshot(`
+    expect(bypassConfigs).toMatchInlineSnapshot(`
      {
-       "htmlLimitedBots": "Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview",
-       "version": 0,
+       "/ppr": {
+         "key": "user-agent",
+         "type": "header",
+         "value": "Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview",
+       },
      }
     `)
   })

test/production/app-dir/metadata-streaming-config/next.config.js

@@ -3,6 +3,7 @@
  */
 const nextConfig = {
   experimental: {
+    ppr: 'incremental',
     streamingMetadata: true,
   },
 }