diff --git a/packages/next/src/build/babel/loader/get-config.ts b/packages/next/src/build/babel/loader/get-config.ts
index 186ce06f42d17..c4a8566ac5da9 100644
--- a/packages/next/src/build/babel/loader/get-config.ts
+++ b/packages/next/src/build/babel/loader/get-config.ts
@@ -273,6 +273,10 @@ function getFreshConfig(
       return false
     }
 
+    if (/[/\\]node_modules[/\\]/.test(filename)) {
+      return false
+    }
+
     if (
       loaderOptions.reactCompilerExclude &&
       loaderOptions.reactCompilerExclude(filename)
@@ -325,6 +329,10 @@ function getFreshConfig(
   }
 
   if (loaderOptions.transformMode === 'standalone') {
+    if (!reactCompilerPluginsIfEnabled.length) {
+      return null
+    }
+
     options.plugins = [jsx, ...reactCompilerPluginsIfEnabled]
     options.presets = [
       [
@@ -458,6 +466,9 @@ export default function getConfig(
   const cacheKey = getCacheKey(cacheCharacteristics)
   if (configCache.has(cacheKey)) {
     const cachedConfig = configCache.get(cacheKey)
+    if (!cachedConfig) {
+      return null
+    }
 
     return {
       ...cachedConfig,
diff --git a/packages/next/src/build/babel/loader/transform.ts b/packages/next/src/build/babel/loader/transform.ts
index 8357313bb943c..f76debc32b296 100644
--- a/packages/next/src/build/babel/loader/transform.ts
+++ b/packages/next/src/build/babel/loader/transform.ts
@@ -86,6 +86,9 @@ export default function transform(
     target,
     filename,
   })
+  if (!babelConfig) {
+    return { code: source, map: inputSourceMap }
+  }
   getConfigSpan.stop()
 
   const normalizeSpan = parentSpan.traceChild('babel-turbo-normalize-file')