Skip to content

Commit be02d45

Browse files
authored
Stacks: values generation (#3914)
* Pre allocate checked paths * Stack value generation * Add loading of stack values * add stack values generation * Add generation of terragrunt values * Output fix * Add stack values generaiton * add experiment flag * Config update * add stack tests * Add auto generated token * Updated values header * Stack lint issues * improved error handling * code rabbit comment * Permissions simplification * Permissions constants * Added errors printing * Update documentation
1 parent 045fed2 commit be02d45

File tree

12 files changed

+168
-117
lines changed

12 files changed

+168
-117
lines changed

cli/commands/stack/action.go

-36
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@ import (
55
"os"
66
"path/filepath"
77

8-
"github.com/gruntwork-io/terragrunt/config"
9-
"github.com/zclconf/go-cty/cty"
10-
118
runall "github.com/gruntwork-io/terragrunt/cli/commands/run-all"
129
"github.com/gruntwork-io/terragrunt/internal/cli"
1310

@@ -20,7 +17,6 @@ import (
2017
const (
2118
stackDir = ".terragrunt-stack"
2219
defaultStackFile = "terragrunt.stack.hcl"
23-
dirPerm = 0755
2420
)
2521

2622
// RunGenerate runs the stack command.
@@ -29,10 +25,6 @@ func RunGenerate(ctx context.Context, opts *options.TerragruntOptions) error {
2925
return err
3026
}
3127

32-
if err := populateStackValues(ctx, opts); err != nil {
33-
return errors.New(err)
34-
}
35-
3628
return generateStack(ctx, opts)
3729
}
3830

@@ -42,10 +34,6 @@ func Run(ctx context.Context, opts *options.TerragruntOptions) error {
4234
return err
4335
}
4436

45-
if err := populateStackValues(ctx, opts); err != nil {
46-
return errors.New(err)
47-
}
48-
4937
if err := RunGenerate(ctx, opts); err != nil {
5038
return err
5139
}
@@ -61,10 +49,6 @@ func RunOutput(ctx context.Context, opts *options.TerragruntOptions, index strin
6149
return err
6250
}
6351

64-
if err := populateStackValues(ctx, opts); err != nil {
65-
return errors.New(err)
66-
}
67-
6852
// collect outputs
6953
outputs, err := generateOutput(ctx, opts)
7054
if err != nil {
@@ -118,23 +102,3 @@ func checkStackExperiment(opts *options.TerragruntOptions) error {
118102

119103
return nil
120104
}
121-
122-
func populateStackValues(ctx context.Context, opts *options.TerragruntOptions) error {
123-
opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
124-
125-
stackFile, err := config.ReadStackConfigFile(ctx, opts)
126-
if err != nil {
127-
return errors.Errorf("Failed to read stack file from %s %v", opts.WorkingDir, err)
128-
}
129-
130-
unitValues := map[string]*cty.Value{}
131-
132-
for _, unit := range stackFile.Units {
133-
path := filepath.Join(opts.WorkingDir, stackDir, unit.Path)
134-
unitValues[path] = unit.Values
135-
}
136-
137-
opts.StackValues = options.NewStackValues(&cty.NilVal, unitValues)
138-
139-
return nil
140-
}

cli/commands/stack/generate.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const (
2222

2323
func generateStack(ctx context.Context, opts *options.TerragruntOptions) error {
2424
opts.Logger.Infof("Generating stack from %s", opts.TerragruntStackConfigPath)
25+
opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
2526
stackFile, err := config.ReadStackConfigFile(ctx, opts)
2627

2728
if err != nil {
@@ -37,7 +38,7 @@ func generateStack(ctx context.Context, opts *options.TerragruntOptions) error {
3738

3839
func processStackFile(ctx context.Context, opts *options.TerragruntOptions, stackFile *config.StackConfigFile) error {
3940
baseDir := filepath.Join(opts.WorkingDir, stackDir)
40-
if err := os.MkdirAll(baseDir, dirPerm); err != nil {
41+
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
4142
return errors.New(fmt.Errorf("failed to create base directory: %w", err))
4243
}
4344

@@ -69,14 +70,19 @@ func processStackFile(ctx context.Context, opts *options.TerragruntOptions, stac
6970
return errors.New(err)
7071
}
7172
} else {
72-
if err := os.MkdirAll(dest, dirPerm); err != nil {
73+
if err := os.MkdirAll(dest, os.ModePerm); err != nil {
7374
return errors.New(err)
7475
}
7576

7677
if _, err := getter.GetAny(ctx, dest, src); err != nil {
7778
return errors.New(err)
7879
}
7980
}
81+
82+
// generate unit values file
83+
if err := config.WriteUnitValues(opts, unit, dest); err != nil {
84+
return errors.New(err)
85+
}
8086
}
8187

8288
return nil

cli/commands/stack/output.go

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"context"
66
"encoding/json"
77
"io"
8+
"path/filepath"
89
"strings"
910

1011
"github.com/hashicorp/hcl/v2/hclwrite"
@@ -20,6 +21,7 @@ import (
2021

2122
func generateOutput(ctx context.Context, opts *options.TerragruntOptions) (map[string]map[string]cty.Value, error) {
2223
opts.Logger.Debugf("Generating output from %s", opts.TerragruntStackConfigPath)
24+
opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
2325
stackFile, err := config.ReadStackConfigFile(ctx, opts)
2426

2527
if err != nil {

config/config.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const (
4242
DefaultTerragruntConfigPath = "terragrunt.hcl"
4343
DefaultTerragruntJSONConfigPath = "terragrunt.hcl.json"
4444
RecommendedParentConfigName = "root.hcl"
45+
StackValuesFile = "terragrunt.values.hcl"
4546

4647
FoundInFile = "found_in_file"
4748

@@ -78,7 +79,6 @@ const (
7879
MetadataErrors = "errors"
7980
MetadataRetry = "retry"
8081
MetadataIgnore = "ignore"
81-
MetadataUnit = "unit"
8282
MetadataValues = "values"
8383
)
8484

@@ -895,6 +895,16 @@ func ParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *Inc
895895
return nil, err
896896
}
897897

898+
// read unit files and add to context
899+
if ctx.TerragruntOptions.Experiments.Evaluate(experiment.Stacks) {
900+
unitValues, err := ReadUnitValues(ctx.Context, ctx.TerragruntOptions, filepath.Dir(file.ConfigPath))
901+
if err != nil {
902+
return nil, err
903+
}
904+
905+
ctx = ctx.WithValues(unitValues)
906+
}
907+
898908
// Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are.
899909
baseBlocks, err := DecodeBaseBlocks(ctx, file, includeFromChild)
900910
if err != nil {

config/config_helpers.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,10 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E
210210
evalCtx.Variables[MetadataFeatureFlag] = *ctx.Features
211211
}
212212

213+
if ctx.Values != nil {
214+
evalCtx.Variables[MetadataValues] = *ctx.Values
215+
}
216+
213217
if ctx.DecodedDependencies != nil {
214218
evalCtx.Variables[MetadataDependency] = *ctx.DecodedDependencies
215219
}
@@ -225,16 +229,6 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E
225229
evalCtx.Variables[MetadataInclude] = exposedInclude
226230
}
227231

228-
// Add to context unit values
229-
path := filepath.Dir(configPath)
230-
unitValues := ctx.TerragruntOptions.StackValues.UnitValues(path)
231-
232-
if unitValues != nil {
233-
evalCtx.Variables[MetadataUnit] = cty.ObjectVal(map[string]cty.Value{
234-
MetadataValues: *unitValues,
235-
})
236-
}
237-
238232
return evalCtx, nil
239233
}
240234

config/config_partial.go

+12
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"os"
66
"path/filepath"
77

8+
"github.com/gruntwork-io/terragrunt/internal/experiment"
9+
810
"github.com/gruntwork-io/terragrunt/pkg/log"
911
"github.com/huandu/go-clone"
1012

@@ -346,6 +348,16 @@ func PartialParseConfigString(ctx *ParsingContext, configPath, configString stri
346348
func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
347349
ctx = ctx.WithTrackInclude(nil)
348350

351+
// read unit files and add to context
352+
if ctx.TerragruntOptions.Experiments.Evaluate(experiment.Stacks) {
353+
unitValues, err := ReadUnitValues(ctx.Context, ctx.TerragruntOptions, filepath.Dir(file.ConfigPath))
354+
if err != nil {
355+
return nil, err
356+
}
357+
358+
ctx = ctx.WithValues(unitValues)
359+
}
360+
349361
// Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are.
350362
// Initialize evaluation ctx extensions from base blocks.
351363
baseBlocks, err := DecodeBaseBlocks(ctx, file, includeFromChild)

config/locals.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,8 @@ func canEvaluateLocals(expression hcl.Expression, evaluatedLocals map[string]cty
182182
case rootName == MetadataFeatureFlag:
183183
// If the variable is `feature`
184184

185-
case rootName == MetadataUnit:
186-
// If the variable is `unit`
185+
case rootName == MetadataValues:
186+
// If the variable is `values`
187187

188188
case rootName != "local":
189189
// We can't evaluate any variable other than `local`

config/parsing_context.go

+8
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ type ParsingContext struct {
2727
// Features are the feature flags that are enabled for the current terragrunt config.
2828
Features *cty.Value
2929

30+
// Values of the unit
31+
Values *cty.Value
32+
3033
// DecodedDependencies are references of other terragrunt config. This contains the following attributes that map to
3134
// various fields related to that config:
3235
// - outputs: The map of outputs from the terraform state obtained by running `terragrunt output` on that target config.
@@ -72,6 +75,11 @@ func (ctx ParsingContext) WithLocals(locals *cty.Value) *ParsingContext {
7275
return &ctx
7376
}
7477

78+
func (ctx ParsingContext) WithValues(values *cty.Value) *ParsingContext {
79+
ctx.Values = values
80+
return &ctx
81+
}
82+
7583
// WithFeatures sets the feature flags to be used in evaluation context.
7684
func (ctx ParsingContext) WithFeatures(features *cty.Value) *ParsingContext {
7785
ctx.Features = features

config/stack.go

+95-3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,15 @@ package config
33
import (
44
"context"
55
"fmt"
6+
"os"
67
"path/filepath"
78
"strings"
89

10+
"github.com/hashicorp/hcl/v2/hclsyntax"
11+
12+
"github.com/gruntwork-io/terragrunt/util"
13+
"github.com/hashicorp/hcl/v2/hclwrite"
14+
915
"github.com/zclconf/go-cty/cty"
1016

1117
"github.com/gruntwork-io/terragrunt/config/hclparse"
@@ -14,7 +20,10 @@ import (
1420
)
1521

1622
const (
17-
stackDir = ".terragrunt-stack"
23+
stackDir = ".terragrunt-stack"
24+
unitValuesFile = "terragrunt.values.hcl"
25+
unitDirPerm = 0755
26+
valueFilePerm = 0644
1827
)
1928

2029
// StackConfigFile represents the structure of terragrunt.stack.hcl stack file.
@@ -87,6 +96,88 @@ func ReadStackConfigFile(ctx context.Context, opts *options.TerragruntOptions) (
8796
return config, nil
8897
}
8998

99+
// WriteUnitValues generates and writes unit values to a terragrunt.values.hcl file in the specified unit directory.
100+
// If the unit has no values (Values is nil), the function logs a debug message and returns.
101+
// Parameters:
102+
// - opts: TerragruntOptions containing logger and other configuration
103+
// - unit: Unit containing the values to write
104+
// - unitDirectory: Target directory where the values file will be created
105+
//
106+
// Returns an error if the directory creation or file writing fails.
107+
func WriteUnitValues(opts *options.TerragruntOptions, unit *Unit, unitDirectory string) error {
108+
if unitDirectory == "" {
109+
return errors.New("WriteUnitValues: unit directory path cannot be empty")
110+
}
111+
112+
if err := os.MkdirAll(unitDirectory, unitDirPerm); err != nil {
113+
return errors.Errorf("failed to create directory %s: %w", unitDirectory, err)
114+
}
115+
116+
filePath := filepath.Join(unitDirectory, unitValuesFile)
117+
if unit.Values == nil {
118+
opts.Logger.Debugf("No values to write for unit %s in %s", unit.Name, filePath)
119+
return nil
120+
}
121+
122+
opts.Logger.Debugf("Writing values for unit %s in %s", unit.Name, filePath)
123+
124+
file := hclwrite.NewEmptyFile()
125+
body := file.Body()
126+
body.AppendUnstructuredTokens([]*hclwrite.Token{
127+
{
128+
Type: hclsyntax.TokenComment,
129+
Bytes: []byte("# Auto-generated by the terragrunt.stack.hcl file by Terragrunt. Do not edit manually\n"),
130+
},
131+
})
132+
133+
for key, val := range unit.Values.AsValueMap() {
134+
body.SetAttributeValue(key, val)
135+
}
136+
137+
if err := os.WriteFile(filePath, file.Bytes(), valueFilePerm); err != nil {
138+
return errors.Errorf("failed to write values file %s: %w", filePath, err)
139+
}
140+
141+
return nil
142+
}
143+
144+
// ReadUnitValues reads the unit values from the terragrunt.values.hcl file.
145+
func ReadUnitValues(ctx context.Context, opts *options.TerragruntOptions, unitDirectory string) (*cty.Value, error) {
146+
if unitDirectory == "" {
147+
return nil, errors.New("ReadUnitValues: unit directory path cannot be empty")
148+
}
149+
150+
filePath := filepath.Join(unitDirectory, unitValuesFile)
151+
152+
if util.FileNotExists(filePath) {
153+
return nil, nil
154+
}
155+
156+
opts.Logger.Debugf("Reading Terragrunt stack values file at %s", filePath)
157+
parser := NewParsingContext(ctx, opts)
158+
file, err := hclparse.NewParser(parser.ParserOptions...).ParseFromFile(filePath)
159+
160+
if err != nil {
161+
return nil, errors.New(err)
162+
}
163+
//nolint:contextcheck
164+
evalParsingContext, err := createTerragruntEvalContext(parser, file.ConfigPath)
165+
166+
if err != nil {
167+
return nil, errors.New(err)
168+
}
169+
170+
values := map[string]cty.Value{}
171+
172+
if err := file.Decode(&values, evalParsingContext); err != nil {
173+
return nil, errors.New(err)
174+
}
175+
176+
result := cty.ObjectVal(values)
177+
178+
return &result, nil
179+
}
180+
90181
// ValidateStackConfig validates a StackConfigFile instance according to the rules:
91182
// - Unit name, source, and path shouldn't be empty
92183
// - Unit names should be unique
@@ -98,8 +189,9 @@ func ValidateStackConfig(config *StackConfigFile) error {
98189

99190
validationErrors := &errors.MultiError{}
100191

101-
names := make(map[string]bool)
102-
paths := make(map[string]bool)
192+
// Pre-allocate maps with known capacity to avoid resizing
193+
names := make(map[string]bool, len(config.Units))
194+
paths := make(map[string]bool, len(config.Units))
103195

104196
for i, unit := range config.Units {
105197
name := strings.TrimSpace(unit.Name)

0 commit comments

Comments
 (0)