fix: More docs fixes (#3919)

* fix: Cleaning up overview

* feat: Adding global flags and resolving a bit more of the gaps in the docs before fleshing out content for CLI

* fix: Fixing links

* fix: Removing hanging thought

* fix: Adding URL validation for link in flags

* fix: Making examples mandatory

* fix: Nits from CodeRabbit

* fix: Adding some whitespace after headings

* fix: Adding migration guide for cli-redesign

* fix: Nit from CodeRabbit

* fix: Adding mechanism for flag data lookup

* fix: Addressing nits

* feat: Add flags as data for all flags

* fix: Fixing env var in legacy docs

* fix: Fixing some typos in the old docs

* fix: Reworking all flags so that they offer rich information

* fix: Fixing all broken links for `cli-options`

* fix: Adding redirects for old routes

* fix: Adding docs for the `unit` block

* Delete astro.config.mjs

Author: yhakbar

docs-starlight/TODO.md

@@ -5,27 +5,34 @@ The Starlight rewrite of the Terragrunt website is a work in progress.
 Here are some of the tasks that need to be completed:
 
 ## Infrastructure
+
 - [x] **Docker compose local dev setup**
 - [x] **Vercel deployment**
   - [x] **Vercel preview deployments**
   - [x] **Custom domain setup** - [terragrunt-v1.gruntwork.io](https://terragrunt-v1.gruntwork.io)
+- [ ] **Remove the no-index robots.txt**
+- [ ] **Add a sitemap.xml**
+  - [x] **Add sitemap for terragrunt-v1.gruntwork.io**
+  - [ ] **Update sitemap to use terragrunt.gruntwork.io as the URL**
 
 ## Content
-- [ ] **Content parity with current docs site**
+
+- [x] **Content parity with current docs site**
   - [x] **Parity for all docs except CLI reference**
-  - [ ] **Parity for CLI reference**
-- [ ] **Redesign reference to use cards**
-- [ ] **Setup redirects for all old URLs**
+  - [x] **Parity for CLI reference**
+- [x] **Redesign reference to use cards**
+- [x] **Setup redirects for all old URLs**
 - [x] **Automate keeping versions updated**
   - [x] **Automate Terragrunt version lookup in docs**
   - [x] **Automate IaC Engine version lookup in docs**
 
 ## User Experience
+
 - [x] **User feedback collection**
-- [ ] **Broken link checking**
+- [x] **Broken link checking**
   - [x] **Automate broken link checking**
-  - [ ] **Fix broken links**
-    - [ ] **Fix broken links in CLI reference**
+  - [x] **Fix broken links**
+    - [x] **Fix broken links in CLI reference**
     - [x] **Fix broken links in all other docs**
   - [x] **Require link checking**
 - [ ] **Jekyll site banner indicating new site**

docs-starlight/astro.config.mjs

@@ -33,6 +33,7 @@ export const sidebar = [
 			{ label: 'Experiments', slug: 'docs/reference/experiments' },
 			{ label: 'Supported Versions', slug: 'docs/reference/supported-versions' },
 			{ label: 'Lock Files', slug: 'docs/reference/lock-files' },
+			{ label: 'Logging', autogenerate: { directory: '04-reference/07-logging', collapsed: true } },
 			{ label: 'Terragrunt Cache', slug: 'docs/reference/terragrunt-cache' },
 		],
 	},
@@ -48,6 +49,7 @@ export const sidebar = [
 
 // https://astro.build/config
 export default defineConfig({
+	site: 'https://terragrunt-v1.gruntwork.io',
 	output: 'server',
 	adapter: vercel({
 		isr: {
@@ -75,11 +77,60 @@ export default defineConfig({
 					'http://localhost:16686/',
 					'http://localhost:9090/',
 
-					// TODO: Remove these once the CLI redesign is done
-					'/docs/reference/cli**/*',
-					'/docs/reference/cli*',
+					// Unfortunately, these have to be ignored, as they're
+					// referencing links that exist outside the file.
+					'/docs/reference/cli/commands/run#*',
 				],
 			})],
 		}),
 	],
+	redirects: {
+		// Pages that have been rehomed.
+		"/docs/features/debugging/": "/docs/troubleshooting/debugging/",
+		"/docs/upgrade/upgrading_to_terragrunt_0.19.x/": "/docs/migrate/upgrading_to_terragrunt_0.19.x/",
+
+		// Redirects to external sites.
+		"/contact/": "https://gruntwork.io/contact",
+		"/commercial-support/": "https://gruntwork.io/support",
+		"/cookie-policy/": "https://gruntwork.io/legal/cookie-policy/",
+
+		// Restructured docs
+		"/docs/reference/configuration/": "/docs/reference/hcl/",
+		"/docs/reference/cli-options/": "/docs/reference/cli/",
+		"/docs/reference/built-in-functions/": "/docs/reference/hcl/functions/",
+		"/docs/reference/config-blocks-and-attributes/": "/docs/reference/hcl/blocks/",
+		"/docs/reference/strict-mode/": "/docs/reference/strict-controls/",
+		"/docs/reference/log-formatting/": "/docs/reference/logging/formatting/",
+		"/docs/features/aws-authentication/": "/docs/features/authentication/",
+
+		// Support old doc structure paths
+		"/docs/": "/docs/getting-started/quick-start/",
+		"/docs/getting-started/": "/docs/getting-started/quick-start/",
+		"/docs/features/": "/docs/features/units/",
+		"/docs/reference/": "/docs/reference/hcl/",
+		"/docs/troubleshooting/": "/docs/troubleshooting/debugging/",
+		"/docs/migrate/": "/docs/migrate/migrating-from-root-terragrunt-hcl/",
+
+		// Support old community paths
+		"/docs/community/": "/docs/community/contributing/",
+		"/support/": "/docs/community/support/",
+
+		// Support old reference paths
+		"/docs/reference/cli/commands/": "/docs/reference/cli/",
+		"/docs/reference/hcl/functions/": "/docs/reference/hcl/functions/",
+
+		// Support old feature paths
+		"/docs/features/inputs/": "/docs/features/units/",
+		"/docs/features/locals/": "/docs/features/units/",
+		"/docs/features/keep-your-terraform-code-dry/": "/docs/features/units/",
+		"/docs/features/execute-terraform-commands-on-multiple-units-at-once/": "/docs/features/stacks/",
+		"/docs/features/keep-your-terragrunt-architecture-dry/": "/docs/features/includes/",
+		"/docs/features/keep-your-remote-state-configuration-dry/": "/docs/features/state-backend/",
+		"/docs/features/keep-your-cli-flags-dry/": "/docs/features/extra-arguments/",
+		"/docs/features/aws-auth/": "/docs/features/aws-authentication/",
+		"/docs/features/work-with-multiple-aws-accounts/": "/docs/features/aws-authentication/",
+		"/docs/features/auto-retry/": "/docs/features/runtime-control/",
+		"/docs/features/provider-cache/": "/docs/features/provider-cache-server/",
+		"/docs/features/provider-caching/": "/docs/features/provider-cache-server/",
+	},
 });

docs-starlight/mise.toml

@@ -1,2 +1,4 @@
 [tools]
 bun = "1.2.2"
+poetry = "2.1.1"
+python = "3.13.2"

docs-starlight/package.json

@@ -11,8 +11,8 @@
   },
   "dependencies": {
     "@astrojs/starlight": "^0.31.1",
-    "@zachleat/table-saw": "^1.0.6",
     "@astrojs/vercel": "^8.0.7",
+    "@zachleat/table-saw": "^1.0.6",
     "astro": "^5.2.1",
     "gray-matter": "^4.0.3",
     "sharp": "^0.32.5",

docs-starlight/public/robots.txt

@@ -0,0 +1 @@
+User-agent: * Disallow: /

docs-starlight/src/assets/img/collections/documentation/dependency-graph.png

@@ -1,7 +1,8 @@
 ---
-import { Aside, Badge, Code, Card } from '@astrojs/starlight/components';
+import { Aside, Code } from '@astrojs/starlight/components';
 import { getEntry, render } from 'astro:content';
 import type { CollectionEntry } from 'astro:content';
+import Flag from './Flag.astro';
 
 const { path } = Astro.props;
 
@@ -49,23 +50,9 @@ const { Content } = await render(command);
   data?.flags ? (
   <h2 id="flags">Flags</h2>
   <div>
-  {data?.flags.map((flag) => (
-    flag &&
-    <section id={flag?.name} >
-    <Card title={"--" + flag?.name} icon="custom:terragrunt">
-      <p id={flag?.name}>{flag?.description}</p>
-        <div>
-        <span>Type: </span><Badge text={flag?.type} variant="note" style={{ fontWeight: 'bold' }}/>
-        </div>
-        <p>Environment Variables:</p>
-        <ul>
-          {flag?.env.map((env) => (
-            <li><code dir="auto">{env}</code></li>
-          ))}
-        </ul>
-      </Card>
-    </section>
+    {data?.flags.map((flagName) => (
+      flagName && <Flag name={flagName} />
     ))}
-</div>
+  </div>
   ) : null
 }

docs-starlight/src/components/Command.astro

@@ -0,0 +1,35 @@
+---
+import { Badge, Card } from '@astrojs/starlight/components';
+import type { CollectionEntry } from 'astro:content';
+import { render, getEntry } from 'astro:content';
+
+interface Props {
+  name: string;
+}
+
+const { name } = Astro.props;
+
+const flagEntry = await getEntry('flags', name) as CollectionEntry<'flags'>;
+const { Content } = await render(flagEntry);
+const { description, type, env } = flagEntry.data;
+---
+
+<section id={name}>
+  <Card title={"--" + name} icon="custom:terragrunt">
+    <p id={name}>{description}</p>
+    <Content />
+    <div>
+      <span>Type: </span><Badge text={type} variant="note" style={{ fontWeight: 'bold' }}/>
+    </div>
+    {env && (
+      <>
+        <p>Environment Variables:</p>
+        <ul>
+          {env.map((envVar: string) => (
+            <li><code dir="auto">{envVar}</code></li>
+          ))}
+        </ul>
+      </>
+    )}
+  </Card>
+</section>

docs-starlight/src/components/Flag.astro

@@ -12,7 +12,6 @@ const docs = defineCollection({
 				}),
 			}),
 		},
-
 	)
 });
 
@@ -30,18 +29,23 @@ const commands = defineCollection({
 		examples: z.array(z.object({
 			code: z.string(),
 			description: z.string().optional(),
-		})).optional(),
-		flags: z.array(z.object({
-			name: z.string(),
-			description: z.string(),
-			env: z.array(z.string()),
-			type: z.string(),
-		})).optional(),
+		})),
+		flags: z.array(z.string()).optional(),
 		experiment: z.object({
 			control: z.string(),
 			name: z.string(),
 		}).optional(),
 	}),
 });
 
-export const collections = { docs, commands };
+const flags = defineCollection({
+	loader: glob({ pattern: "**/*.mdx", base: "src/data/flags" }),
+	schema: z.object({
+		name: z.string(),
+		description: z.string(),
+		type: z.string(),
+		env: z.array(z.string()).optional(),
+	}),
+});
+
+export const collections = { docs, commands, flags };

docs-starlight/src/content.config.ts

@@ -46,7 +46,7 @@ This tutorial will not assume the following:
 2. You have any experience with Terragrunt.
 3. You have any existing Terragrunt, OpenTofu or Terraform projects.
 
-\* Note that if you have _both_ OpenTofu and Terraform installed, you'll want to read the [tf-path](/docs/reference/cli-options/#tf-path) docs to understand how Terragrunt determines which binary to use.
+\* Note that if you have _both_ OpenTofu and Terraform installed, you'll want to read the [tf-path](/docs/reference/cli/commands/run#tf-path) docs to understand how Terragrunt determines which binary to use.
 
 If you would like a less gentle introduction geared towards users with an active AWS account, familiarity with OpenTofu/Terraform, and potentially a team actively using Terragrunt, consider starting with the [Overview](/docs/getting-started/overview/).
 

docs-starlight/src/content/docs/01-getting-started/01-quick-start.mdx

@@ -48,7 +48,7 @@ While not a requirement, a general tendency experienced when working with Terrag
 
 A common pattern used in the repository structure for Terragrunt projects is to have a single `terragrunt.hcl` file located at the root of the repository, and multiple subdirectories each containing their own `terragrunt.hcl` file. This is typically done to promote code-reuse, as it allows for any configuration common to all units to be defined in the root `terragrunt.hcl` file, and for unit-specific configuration to be defined in child directories. In this pattern, the root `terragrunt.hcl` file is not considered a unit, while all the child directories containing `terragrunt.hcl` files are.
 
-Note that units don't technically need to call their configuration files `terragrunt.hcl` (that's configurable via the [--config](/docs/reference/cli-options/#config)), and users don't technically need to use a root `terragrunt.hcl` file or to name it that. This is the most common pattern followed by the community, however, and deviation from this pattern should be justified in the context of the project. It can help others with Terragrunt experience understand the project more easily if industry standard patterns are followed.
+Note that units don't technically need to call their configuration files `terragrunt.hcl` (that's configurable via the [--config](/docs/reference/cli/commands/run#config)), and users don't technically need to use a root `terragrunt.hcl` file or to name it that. This is the most common pattern followed by the community, however, and deviation from this pattern should be justified in the context of the project. It can help others with Terragrunt experience understand the project more easily if industry standard patterns are followed.
 
 ### Stack
 
@@ -154,17 +154,17 @@ These utilities are part of what makes Terragrunt so powerful, as they allow use
 
 The Run Queue is the queue of all units that Terragrunt will do work on over one or more runs.
 
-Certain commands like [run-all](/docs/reference/cli-options/#run-all) populate the Run Queue with all units in a stack, while other commands like `plan` or `apply` will only populate the Run Queue with the unit that the command was run in.
+Certain commands like [run-all](/docs/reference/cli/commands/run#all) populate the Run Queue with all units in a stack, while other commands like `plan` or `apply` will only populate the Run Queue with the unit that the command was run in.
 
-Certain flags like [--include-dir](/docs/reference/cli-options/#include-dir) can be used to adjust the Run Queue to include additional units. Conversely, there are flags like [--exclude-dir](/docs/reference/cli-options/#exclude-dir) that can be used to adjust the Run Queue to exclude units.
+Certain flags like [--include-dir](/docs/reference/cli/commands/run#include-dir) can be used to adjust the Run Queue to include additional units. Conversely, there are flags like [--exclude-dir](/docs/reference/cli/commands/run#exclude-dir) that can be used to adjust the Run Queue to exclude units.
 
 Terragrunt will always attempt to run until the Run Queue is empty.
 
 ### Runner Pool
 
 The Runner Pool is the pool of available resources that Terragrunt can use to execute runs.
 
-Units are dequeued from the Runner Pool into the Runner Pool depending on factors like [parallelism](/docs/reference/cli-options/#parallelism) and the DAG.
+Units are dequeued from the Runner Pool into the Runner Pool depending on factors like [parallelism](/docs/reference/cli/commands/run#parallelism) and the DAG.
 
 Units are only considered "running" when they are in the Runner Pool.
 
@@ -207,15 +207,15 @@ By default, Terragrunt will interact with OpenTofu/Terraform in order to retriev
 
 Terragrunt does have the ability to mock outputs, which is useful when dependencies do not yet have outputs to be consumed (e.g. during the run of a unit with a dependency that has not been applied).
 
-Terragrunt also has the ability to fetch outputs without interacting with OpenTofu/Terraform via [--fetch-dependency-output-from-state](/docs/reference/cli-options/#fetch-dependency-output-from-state) for dependencies where state is stored in AWS. This is an experimental feature, and more tooling is planned to make this easier to use.
+Terragrunt also has the ability to fetch outputs without interacting with OpenTofu/Terraform via [--fetch-dependency-output-from-state](/docs/reference/cli/commands/run#fetch-dependency-output-from-state) for dependencies where state is stored in AWS. This is an experimental feature, and more tooling is planned to make this easier to use.
 
 ### Feature
 
-A [feature](/docs/reference/cli-options/#feature) is a configuration that can be dynamically controlled in Terragrunt configurations.
+A [feature](/docs/reference/cli/commands/run#feature) is a configuration that can be dynamically controlled in Terragrunt configurations.
 
 They operate very similarly to variables, but are designed to be used to dynamically adjust the behavior of Terragrunt configurations, rather than OpenTofu/Terraform configurations.
 
-Features can be adjusted using feature flags, which are set in Terragrunt configurations using the [feature block](/docs/reference/hcl/blocks#feature) and the [feature flag](/docs/reference/cli-options#feature) attribute.
+Features can be adjusted using feature flags, which are set in Terragrunt configurations using the [feature block](/docs/reference/hcl/blocks#feature) and the [feature flag](/docs/reference/cli/commands/run#feature) attribute.
 
 Like all good feature flags, you are encouraged to use them with good judgement and to avoid using them as a crutch to avoid making decisions about permanent adjustments to your infrastructure.
 

docs-starlight/src/content/docs/01-getting-started/04-terminology.md

@@ -346,4 +346,4 @@ specified via the `path` parameter. It behaves exactly as if you had copy/pasted
 included file `generate` configuration into the child, but this approach is much easier to maintain\!
 ## Further Reading
 Note that if you're considering this solution because you're struggling with dynamic provider authentication in AWS,
-you may be interested in the dedicated documentation on [working with multiple AWS accounts](/docs/features/aws-authentication).
+you may be interested in the dedicated documentation on [working with multiple AWS accounts](/docs/features/authentication).

docs-starlight/src/content/docs/02-features/01-units.mdx

@@ -43,7 +43,7 @@ Right now, there is no special syntax for defining a stack as a single file, but
 Regardless of whether you are using the new syntax or not, the core functionality provided by Stacks are the same.
 The new `terragrunt.stack.hcl` file is merely a shorthand for defining a stack, just like you could by placing units directly in a folder.
 
-## The `run-all` command
+## The `run --all` command
 
 To solve the problem above, first convert the OpenTofu/Terraform modules into units. This is done simply by adding an empty `terragrunt.hcl` file within each root module folder.
 
@@ -95,10 +95,10 @@ terragrunt run-all output
 
 Finally, if you make some changes to your project, you could evaluate the impact by using `run-all` command with `plan`:
 
-Note: It is important to realize that you could get errors running `run-all plan` if you have dependencies between your
+Note: It is important to realize that you could get errors running `run --all plan` if you have dependencies between your
 projects and some of those dependencies haven’t been applied yet.
 
-_Ex: If unit A depends on unit B and unit B hasn’t been applied yet, then run-all plan will show the plan for B, but exit with an error when trying to show the plan for A._
+_Ex: If unit A depends on unit B and unit B hasn’t been applied yet, then `run --all plan` will show the plan for B, but exit with an error when trying to show the plan for A._
 
 ```bash
 cd root
@@ -109,7 +109,7 @@ terragrunt run-all plan
 
 If your units have dependencies between them, for example, you can’t deploy the backend-app until MySQL and redis are deployed. You’ll need to express those dependencies in your Terragrunt configuration as explained in the next section.
 
-Additional note: If your units have dependencies between them, and you run a `terragrunt run-all destroy` command, Terragrunt will destroy all the units under the current working directory, _as well as each of the unit dependencies_ (that is, units you depend on via `dependencies` and `dependency` blocks)! If you wish to use exclude dependencies from being destroyed, add the `--ignore-external-dependencies` flag, or use the `--exclude-dir` once for each directory you wish to exclude.
+Additional note: If your units have dependencies between them, and you run a `terragrunt run --all destroy` command, Terragrunt will destroy all the units under the current working directory, _as well as each of the unit dependencies_ (that is, units you depend on via `dependencies` and `dependency` blocks)! If you wish to prevent external dependencies from being destroyed, add the `--queue-exclude-external` flag, or use the `--exclude-dir` once for each directory you wish to exclude.
 
 ## Cutting down the file count
 
@@ -549,10 +549,10 @@ cd root/us-east-1
 terragrunt run-all apply
 ```
 
-Terragrunt will only include the units in the `us-east-1` stack and its children in the queue of units to run (unless external dependencies are pulled in, as discussed in the [run-all command](#the-run-all-command) section).
+Terragrunt will only include the units in the `us-east-1` stack and its children in the queue of units to run (unless external dependencies are pulled in, as discussed in the [run --all command](#the-run---all-command) section).
 
 Generally speaking, this is the primary tool Terragrunt users use to control the blast radius of their changes. For the most part, it is the current working directory that determines the blast radius of a `run-all` command.
 
-In addition to using your working directory to control what's included in a [run queue](/docs/getting-started/terminology/#run-queue), you can also use flags like [--include-dir](/docs/reference/cli-options/#include-dir) and [--exclude-dir](/docs/reference/cli-options/#exclude-dir) to explicitly control what's included in a run queue within a stack, or outside of it.
+In addition to using your working directory to control what's included in a [run queue](/docs/getting-started/terminology/#run-queue), you can also use flags like [--include-dir](/docs/reference/cli/commands/run#include-dir) and [--exclude-dir](/docs/reference/cli/commands/run#exclude-dir) to explicitly control what's included in a run queue within a stack, or outside of it.
 
-There are more flags that control the behavior of the `run-all` command, which you can find in the [CLI Options](/docs/reference/cli-options) section.
+There are more flags that control the behavior of the `run` command, which you can find in the [`run` docs](/docs/reference/cli/commands/run).