From 2b2b977c1b567490b2af5683dda4d76b35a70549 Mon Sep 17 00:00:00 2001 From: James Pogran Date: Tue, 27 Sep 2022 12:20:50 -0400 Subject: [PATCH] semantic tokens --- decoder/semantic_tokens.go | 96 ++++++++--- decoder/semantic_tokens_expr_test.go | 16 +- decoder/semantic_tokens_test.go | 233 ++++++++++++++++++++++++++- 3 files changed, 310 insertions(+), 35 deletions(-) diff --git a/decoder/semantic_tokens.go b/decoder/semantic_tokens.go index c1364810..6a557494 100644 --- a/decoder/semantic_tokens.go +++ b/decoder/semantic_tokens.go @@ -1,19 +1,22 @@ package decoder import ( + "context" "sort" + "github.com/zclconf/go-cty/cty" + + icontext "github.com/hashicorp/hcl-lang/context" "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/hcl-lang/reference" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" ) // SemanticTokensInFile returns a sequence of semantic tokens // within the config file. -func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToken, error) { +func (d *PathDecoder) SemanticTokensInFile(ctx context.Context, filename string) ([]lang.SemanticToken, error) { f, err := d.fileByName(filename) if err != nil { return nil, err @@ -28,7 +31,7 @@ func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToke return []lang.SemanticToken{}, nil } - tokens := d.tokensForBody(body, d.pathCtx.Schema, []lang.SemanticTokenModifier{}) + tokens := d.tokensForBody(ctx, body, d.pathCtx.Schema, []lang.SemanticTokenModifier{}) sort.Slice(tokens, func(i, j int) bool { return tokens[i].Range.Start.Byte < tokens[j].Range.Start.Byte @@ -37,21 +40,43 @@ func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToke return tokens, nil } -func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.BodySchema, parentModifiers []lang.SemanticTokenModifier) []lang.SemanticToken { +func (d *PathDecoder) tokensForBody(ctx context.Context, body *hclsyntax.Body, bodySchema *schema.BodySchema, parentModifiers []lang.SemanticTokenModifier) []lang.SemanticToken { tokens := make([]lang.SemanticToken, 0) if bodySchema == nil { return tokens } + if bodySchema.Extensions != nil { + ctx = icontext.WithExtensions(ctx, bodySchema.Extensions) + if bodySchema.Extensions.Count { + if _, ok := body.Attributes["count"]; ok { + // append to context we need count provided + ctx = icontext.WithActiveCount(ctx) + } + } + } + for name, attr := range body.Attributes { + attrSchema, ok := bodySchema.Attributes[name] if !ok { - if bodySchema.AnyAttribute == nil { - // unknown attribute - continue + if bodySchema.Extensions != nil && name == "count" && bodySchema.Extensions.Count { + attrSchema = &schema.AttributeSchema{ + Description: lang.MarkupContent{ + Kind: lang.MarkdownKind, + Value: "**count** _optional, number_\n\nThe distinct index number (starting with 0) corresponding to the instance", + }, + IsOptional: true, + Expr: schema.LiteralTypeOnly(cty.Number), + } + } else { + if bodySchema.AnyAttribute == nil { + // unknown attribute + continue + } + attrSchema = bodySchema.AnyAttribute } - attrSchema = bodySchema.AnyAttribute } attrModifiers := make([]lang.SemanticTokenModifier, 0) @@ -65,7 +90,7 @@ func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.Bod }) ec := ExprConstraints(attrSchema.Expr) - tokens = append(tokens, d.tokensForExpression(attr.Expr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, attr.Expr, ec)...) } for _, block := range body.Blocks { @@ -106,19 +131,19 @@ func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.Bod } if block.Body != nil { - tokens = append(tokens, d.tokensForBody(block.Body, blockSchema.Body, blockModifiers)...) + tokens = append(tokens, d.tokensForBody(ctx, block.Body, blockSchema.Body, blockModifiers)...) } depSchema, _, ok := NewBlockSchema(blockSchema).DependentBodySchema(block.AsHCLBlock()) if ok { - tokens = append(tokens, d.tokensForBody(block.Body, depSchema, []lang.SemanticTokenModifier{})...) + tokens = append(tokens, d.tokensForBody(ctx, block.Body, depSchema, []lang.SemanticTokenModifier{})...) } } return tokens } -func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints ExprConstraints) []lang.SemanticToken { +func (d *PathDecoder) tokensForExpression(ctx context.Context, expr hclsyntax.Expression, constraints ExprConstraints) []lang.SemanticToken { tokens := make([]lang.SemanticToken, 0) switch eType := expr.(type) { @@ -135,6 +160,31 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints } } + address, _ := lang.TraversalToAddress(eType.AsTraversal()) + countIndexAttr := lang.Address{ + lang.RootStep{ + Name: "count", + }, + lang.AttrStep{ + Name: "index", + }, + } + countAvailable := icontext.ActiveCountFromContext(ctx) + if address.Equals(countIndexAttr) && countAvailable { + traversal := eType.AsTraversal() + tokens = append(tokens, lang.SemanticToken{ + Type: lang.TokenTraversalStep, + Modifiers: []lang.SemanticTokenModifier{}, + Range: traversal[0].SourceRange(), + }) + tokens = append(tokens, lang.SemanticToken{ + Type: lang.TokenTraversalStep, + Modifiers: []lang.SemanticTokenModifier{}, + Range: traversal[1].SourceRange(), + }) + return tokens + } + tes, ok := constraints.TraversalExprs() if ok && d.pathCtx.ReferenceTargets != nil { traversal := eType.AsTraversal() @@ -231,7 +281,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints Range: eType.NameRange, }) for _, arg := range eType.Args { - tokens = append(tokens, d.tokensForExpression(arg, constraints)...) + tokens = append(tokens, d.tokensForExpression(ctx, arg, constraints)...) } return tokens } @@ -249,13 +299,13 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints return tokenForTypedExpression(eType, cty.String) } case *hclsyntax.TemplateWrapExpr: - return d.tokensForExpression(eType.Wrapped, constraints) + return d.tokensForExpression(ctx, eType.Wrapped, constraints) case *hclsyntax.TupleConsExpr: tc, ok := constraints.TupleConsExpr() if ok { ec := ExprConstraints(tc.AnyElem) for _, expr := range eType.Exprs { - tokens = append(tokens, d.tokensForExpression(expr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...) } return tokens } @@ -263,7 +313,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints if ok { ec := ExprConstraints(se.Elem) for _, expr := range eType.Exprs { - tokens = append(tokens, d.tokensForExpression(expr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...) } return tokens } @@ -271,7 +321,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints if ok { ec := ExprConstraints(le.Elem) for _, expr := range eType.Exprs { - tokens = append(tokens, d.tokensForExpression(expr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...) } return tokens } @@ -282,7 +332,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints break } ec := ExprConstraints(te.Elems[i]) - tokens = append(tokens, d.tokensForExpression(expr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...) } return tokens } @@ -316,7 +366,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints }) ec := ExprConstraints(attr.Expr) - tokens = append(tokens, d.tokensForExpression(item.ValueExpr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, ec)...) } return tokens } @@ -329,7 +379,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints Range: item.KeyExpr.Range(), }) ec := ExprConstraints(me.Elem) - tokens = append(tokens, d.tokensForExpression(item.ValueExpr, ec)...) + tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, ec)...) } return tokens } @@ -343,7 +393,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints } _, ok = constraints.TypeDeclarationExpr() if ok { - return d.tokensForObjectConsTypeDeclarationExpr(eType, constraints) + return d.tokensForObjectConsTypeDeclarationExpr(ctx, eType, constraints) } case *hclsyntax.LiteralValueExpr: valType := eType.Val.Type() @@ -389,7 +439,7 @@ func isComplexTypeDeclaration(funcName string) bool { return false } -func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(expr *hclsyntax.ObjectConsExpr, constraints ExprConstraints) []lang.SemanticToken { +func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(ctx context.Context, expr *hclsyntax.ObjectConsExpr, constraints ExprConstraints) []lang.SemanticToken { tokens := make([]lang.SemanticToken, 0) for _, item := range expr.Items { key, _ := item.KeyExpr.Value(nil) @@ -405,7 +455,7 @@ func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(expr *hclsyntax.Obj Range: item.KeyExpr.Range(), }) - tokens = append(tokens, d.tokensForExpression(item.ValueExpr, constraints)...) + tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, constraints)...) } return tokens } diff --git a/decoder/semantic_tokens_expr_test.go b/decoder/semantic_tokens_expr_test.go index 3c7f09da..66bcb67d 100644 --- a/decoder/semantic_tokens_expr_test.go +++ b/decoder/semantic_tokens_expr_test.go @@ -1,16 +1,18 @@ package decoder import ( + "context" "fmt" "testing" "github.com/google/go-cmp/cmp" + "github.com/zclconf/go-cty/cty" + "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/hcl-lang/reference" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" ) func TestDecoder_SemanticTokensInFile_expressions(t *testing.T) { @@ -1433,6 +1435,8 @@ EOT }, } + ctx := context.Background() + for i, tc := range testCases { t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { bodySchema := &schema.BodySchema{ @@ -1451,7 +1455,7 @@ EOT }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -1912,6 +1916,8 @@ func TestDecoder_SemanticTokensInFile_traversalExpression(t *testing.T) { }, } + ctx := context.Background() + for i, tc := range testCases { t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { bodySchema := &schema.BodySchema{ @@ -1931,7 +1937,7 @@ func TestDecoder_SemanticTokensInFile_traversalExpression(t *testing.T) { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -2227,6 +2233,8 @@ func TestDecoder_SemanticTokensInFile_typeDeclaration(t *testing.T) { }, } + ctx := context.Background() + for i, tc := range testCases { t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { bodySchema := &schema.BodySchema{ @@ -2245,7 +2253,7 @@ func TestDecoder_SemanticTokensInFile_typeDeclaration(t *testing.T) { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } diff --git a/decoder/semantic_tokens_test.go b/decoder/semantic_tokens_test.go index aa308858..3b098bd0 100644 --- a/decoder/semantic_tokens_test.go +++ b/decoder/semantic_tokens_test.go @@ -1,16 +1,18 @@ package decoder import ( + "context" "errors" "testing" "github.com/google/go-cmp/cmp" + "github.com/zclconf/go-cty/cty" + "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/hcl-lang/schema" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/json" - "github.com/zclconf/go-cty/cty" ) func TestDecoder_SemanticTokensInFile_emptyBody(t *testing.T) { @@ -23,7 +25,9 @@ func TestDecoder_SemanticTokensInFile_emptyBody(t *testing.T) { }, }) - _, err := d.SemanticTokensInFile("test.tf") + ctx := context.Background() + + _, err := d.SemanticTokensInFile(ctx, "test.tf") unknownFormatErr := &UnknownFileFormatError{} if !errors.As(err, &unknownFormatErr) { t.Fatal("expected UnknownFileFormatError for empty body") @@ -46,7 +50,9 @@ func TestDecoder_SemanticTokensInFile_json(t *testing.T) { }, }) - _, err := d.SemanticTokensInFile("test.tf.json") + ctx := context.Background() + + _, err := d.SemanticTokensInFile(ctx, "test.tf.json") unknownFormatErr := &UnknownFileFormatError{} if !errors.As(err, &unknownFormatErr) { t.Fatal("expected UnknownFileFormatError for JSON body") @@ -65,7 +71,9 @@ func TestDecoder_SemanticTokensInFile_zeroByteContent(t *testing.T) { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + ctx := context.Background() + + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -87,7 +95,9 @@ func TestDecoder_SemanticTokensInFile_fileNotFound(t *testing.T) { }, }) - _, err := d.SemanticTokensInFile("foobar.tf") + ctx := context.Background() + + _, err := d.SemanticTokensInFile(ctx, "foobar.tf") notFoundErr := &FileNotFoundError{} if !errors.As(err, ¬FoundErr) { t.Fatal("expected FileNotFoundError for non-existent file") @@ -149,7 +159,9 @@ resource "vault_auth_backend" "blah" { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + ctx := context.Background() + + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -361,7 +373,9 @@ resource "aws_instance" "beta" { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + ctx := context.Background() + + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -626,7 +640,9 @@ resource "vault_auth_backend" "blah" { }, }) - tokens, err := d.SemanticTokensInFile("test.tf") + ctx := context.Background() + + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") if err != nil { t.Fatal(err) } @@ -836,3 +852,204 @@ resource "vault_auth_backend" "blah" { t.Fatalf("unexpected tokens: %s", diff) } } + +func TestDecoder_SemanticTokensInFile_expression_extensions(t *testing.T) { + bodySchema := &schema.BodySchema{ + Blocks: map[string]*schema.BlockSchema{ + "resource": { + Body: &schema.BodySchema{ + Extensions: &schema.BodyExtensions{ + Count: true, + }, + Attributes: map[string]*schema.AttributeSchema{ + "cpu_count": { + Expr: schema.LiteralTypeOnly(cty.Number), + }, + }, + }, + Labels: []*schema.LabelSchema{ + { + Name: "type", + IsDepKey: true, + SemanticTokenModifiers: lang.SemanticTokenModifiers{ + lang.TokenModifierDependent, + }, + }, + {Name: "name"}, + }, + }, + }, + } + + testCfg := []byte(` +resource "vault_auth_backend" "blah" { + count = 1 + cpu_count = count.index +} +`) + + f, pDiags := hclsyntax.ParseConfig(testCfg, "test.tf", hcl.InitialPos) + if len(pDiags) > 0 { + t.Fatal(pDiags) + } + + d := testPathDecoder(t, &PathContext{ + Schema: bodySchema, + Files: map[string]*hcl.File{ + "test.tf": f, + }, + }) + + ctx := context.Background() + + tokens, err := d.SemanticTokensInFile(ctx, "test.tf") + if err != nil { + t.Fatal(err) + } + + expectedTokens := []lang.SemanticToken{ + { // resource + Type: lang.TokenBlockType, + Modifiers: []lang.SemanticTokenModifier{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 2, + Column: 1, + Byte: 1, + }, + End: hcl.Pos{ + Line: 2, + Column: 9, + Byte: 9, + }, + }, + }, + { // vault_auth_backend + Type: lang.TokenBlockLabel, + Modifiers: []lang.SemanticTokenModifier{ + lang.TokenModifierDependent, + }, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 2, + Column: 10, + Byte: 10, + }, + End: hcl.Pos{ + Line: 2, + Column: 30, + Byte: 30, + }, + }, + }, + { // blah + Type: lang.TokenBlockLabel, + Modifiers: []lang.SemanticTokenModifier{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 2, + Column: 31, + Byte: 31, + }, + End: hcl.Pos{ + Line: 2, + Column: 37, + Byte: 37, + }, + }, + }, + { // count + Type: lang.TokenAttrName, + Modifiers: lang.SemanticTokenModifiers{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 3, + Column: 2, + Byte: 41, + }, + End: hcl.Pos{ + Line: 3, + Column: 7, + Byte: 46, + }, + }, + }, + { // 1 + Type: lang.TokenNumber, + Modifiers: lang.SemanticTokenModifiers{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 3, + Column: 10, + Byte: 49, + }, + End: hcl.Pos{ + Line: 3, + Column: 11, + Byte: 50, + }, + }, + }, + { // cpu_count + Type: lang.TokenAttrName, + Modifiers: lang.SemanticTokenModifiers{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 4, + Column: 3, + Byte: 53, + }, + End: hcl.Pos{ + Line: 4, + Column: 12, + Byte: 62, + }, + }, + }, + { // count. + Type: lang.TokenTraversalStep, + Modifiers: lang.SemanticTokenModifiers{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 4, + Column: 15, + Byte: 65, + }, + End: hcl.Pos{ + Line: 4, + Column: 20, + Byte: 70, + }, + }, + }, + { // .index + Type: lang.TokenTraversalStep, + Modifiers: lang.SemanticTokenModifiers{}, + Range: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{ + Line: 4, + Column: 20, + Byte: 70, + }, + End: hcl.Pos{ + Line: 4, + Column: 26, + Byte: 76, + }, + }, + }, + } + + diff := cmp.Diff(expectedTokens, tokens) + if diff != "" { + t.Fatalf("unexpected tokens: %s", diff) + } +}