Skip to content

Commit

Permalink
Refactor ISyntaxInputNodes (#58800)
Browse files Browse the repository at this point in the history 
Refactor syntax handling in generator driver
  • Loading branch information
@chsienki
chsienki authored Feb 2, 2022
1 parent d5763e9 commit a7e3c09
Show file tree
Hide file tree
Showing 14 changed files with 461 additions and 278 deletions.
6 changes: 2 additions & 4 deletions src/Compilers/CSharp/Test/Semantic/SourceGeneration/StateTableTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,9 +325,6 @@ public void Driver_Table_Compacts_State_Tables_And_Drops_Steps_When_Made_Immutab
DriverStateTable.Builder builder2 = GetBuilder(driverStateTable, trackIncrementalGeneratorSteps: true);
builder2.GetLatestStateTableForNode(callbackNode);

// table persisted in driverStateTable does not have any steps
Assert.False(driverStateTable.GetStateTableOrEmpty<int>(callbackNode).HasTrackedSteps);

// table returned from the first instance was compacted by the builder
Assert.NotNull(passedIn);
AssertTableEntries(passedIn!, new[] { (1, EntryState.Cached, 0), (2, EntryState.Cached, 1), (3, EntryState.Cached, 2), (5, EntryState.Cached, 0), (6, EntryState.Cached, 1) });
Expand Down Expand Up @@ -955,11 +952,12 @@ private DriverStateTable.Builder GetBuilder(DriverStateTable previous, bool trac
ImmutableArray<AdditionalText>.Empty,
ImmutableArray<GeneratorState>.Empty,
previous,
SyntaxStore.Empty,
disabledOutputs: IncrementalGeneratorOutputKind.None,
runtime: TimeSpan.Zero,
trackIncrementalGeneratorSteps: trackIncrementalGeneratorSteps);

return new DriverStateTable.Builder(c, state, ImmutableArray<ISyntaxInputNode>.Empty);
return new DriverStateTable.Builder(c, state, SyntaxStore.Empty.ToBuilder(c, ImmutableArray<SyntaxInputNode>.Empty, trackIncrementalGeneratorSteps, cancellationToken: default));
}

private class CallbackNode<T> : IIncrementalGeneratorNode<T>
Expand Down
83 changes: 81 additions & 2 deletions src/Compilers/CSharp/Test/Semantic/SourceGeneration/SyntaxAwareGeneratorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1287,6 +1287,79 @@ class F
Assert.Equal("fieldD", Assert.Single(fieldsCalledFor));
}

[Fact]
public void IncrementalGenerator_With_Syntax_Filter_Doesnt_Run_When_Compilation_Is_Unchanged()
{
var source1 = @"
#pragma warning disable CS0414
class C
{
string fieldA = null;
}";
var source2 = @"
#pragma warning disable CS0414
class D
{
string fieldB = null;
}";
var source3 = @"
#pragma warning disable CS0414
class E
{
string fieldC = null;
}";
var parseOptions = TestOptions.RegularPreview;
Compilation compilation = CreateCompilation(new[] { source1, source2, source3 }, options: TestOptions.DebugDll, parseOptions: parseOptions);
compilation.VerifyDiagnostics();

List<string> fieldsCalledFor = new List<string>();
var testGenerator = new PipelineCallbackGenerator(context =>
{
var source = context.SyntaxProvider.CreateSyntaxProvider((c, _) => c is FieldDeclarationSyntax fds, (c, _) => ((FieldDeclarationSyntax)c.Node).Declaration.Variables[0].Identifier.ValueText).WithTrackingName("Fields");
context.RegisterSourceOutput(source, (spc, fieldName) =>
{
spc.AddSource(fieldName, "");
fieldsCalledFor.Add(fieldName);
});
});

GeneratorDriver driver = CSharpGeneratorDriver.Create(new[] { new IncrementalGeneratorWrapper(testGenerator) }, parseOptions: parseOptions, driverOptions: new GeneratorDriverOptions(IncrementalGeneratorOutputKind.None, trackIncrementalGeneratorSteps: false));
driver = driver.RunGenerators(compilation);

var results = driver.GetRunResult();
Assert.Empty(results.Diagnostics);
Assert.EndsWith("fieldA.cs", results.GeneratedTrees[0].FilePath);
Assert.EndsWith("fieldB.cs", results.GeneratedTrees[1].FilePath);
Assert.EndsWith("fieldC.cs", results.GeneratedTrees[2].FilePath);
Assert.Equal(new[] { "fieldA", "fieldB", "fieldC" }, fieldsCalledFor);

// now re-run the drivers with the same compilation
fieldsCalledFor.Clear();
driver = driver.RunGenerators(compilation);
results = driver.GetRunResult();
Assert.Empty(results.Diagnostics);
Assert.Equal(3, results.GeneratedTrees.Length);

// we produced the expected modified sources, but didn't call for any of the trees
Assert.EndsWith("fieldA.cs", results.GeneratedTrees[0].FilePath);
Assert.EndsWith("fieldB.cs", results.GeneratedTrees[1].FilePath);
Assert.EndsWith("fieldC.cs", results.GeneratedTrees[2].FilePath);
Assert.Empty(fieldsCalledFor);

// now re-run the drivers with the same compilation again to ensure we cached the trees correctly
fieldsCalledFor.Clear();
driver = driver.RunGenerators(compilation);
results = driver.GetRunResult();
Assert.Empty(results.Diagnostics);
Assert.Equal(3, results.GeneratedTrees.Length);

// we produced the expected modified sources, but didn't call for any of the trees
Assert.EndsWith("fieldA.cs", results.GeneratedTrees[0].FilePath);
Assert.EndsWith("fieldB.cs", results.GeneratedTrees[1].FilePath);
Assert.EndsWith("fieldC.cs", results.GeneratedTrees[2].FilePath);
Assert.Empty(fieldsCalledFor);
}

[Fact]
public void IncrementalGenerator_With_Syntax_Filter_And_Changed_Tree_Order()
{
Expand Down Expand Up @@ -1334,7 +1407,10 @@ class E
});
});

GeneratorDriver driver = CSharpGeneratorDriver.Create(new[] { new IncrementalGeneratorWrapper(testGenerator) }, parseOptions: parseOptions, driverOptions: new GeneratorDriverOptions(IncrementalGeneratorOutputKind.None, trackIncrementalGeneratorSteps: true));
GeneratorDriver driver = CSharpGeneratorDriver.Create(
new[] { new IncrementalGeneratorWrapper(testGenerator) },
parseOptions: parseOptions,
driverOptions: new GeneratorDriverOptions(IncrementalGeneratorOutputKind.None, trackIncrementalGeneratorSteps: true));
driver = driver.RunGenerators(compilation);

var results = driver.GetRunResult();
Expand Down Expand Up @@ -1452,7 +1528,10 @@ class C
});
});

GeneratorDriver driver = CSharpGeneratorDriver.Create(new[] { new IncrementalGeneratorWrapper(testGenerator) }, parseOptions: parseOptions, driverOptions: new GeneratorDriverOptions(IncrementalGeneratorOutputKind.None, trackIncrementalGeneratorSteps: true));
GeneratorDriver driver = CSharpGeneratorDriver.Create(
new[] { new IncrementalGeneratorWrapper(testGenerator) },
parseOptions: parseOptions,
driverOptions: new GeneratorDriverOptions(IncrementalGeneratorOutputKind.None, trackIncrementalGeneratorSteps: true));
driver = driver.RunGenerators(compilation);
var results = driver.GetRunResult();
Assert.Collection(results.Results[0].TrackedSteps["Fields"],
Expand Down
12 changes: 7 additions & 5 deletions src/Compilers/Core/Portable/SourceGeneration/GeneratorDriver.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ internal GeneratorDriver(GeneratorDriverState state)
internal GeneratorDriver(ParseOptions parseOptions, ImmutableArray<ISourceGenerator> generators, AnalyzerConfigOptionsProvider optionsProvider, ImmutableArray<AdditionalText> additionalTexts, GeneratorDriverOptions driverOptions)
{
(var filteredGenerators, var incrementalGenerators) = GetIncrementalGenerators(generators, SourceExtension);
_state = new GeneratorDriverState(parseOptions, optionsProvider, filteredGenerators, incrementalGenerators, additionalTexts, ImmutableArray.Create(new GeneratorState[filteredGenerators.Length]), DriverStateTable.Empty, driverOptions.DisabledOutputs, runtime: TimeSpan.Zero, driverOptions.TrackIncrementalGeneratorSteps);
_state = new GeneratorDriverState(parseOptions, optionsProvider, filteredGenerators, incrementalGenerators, additionalTexts, ImmutableArray.Create(new GeneratorState[filteredGenerators.Length]), DriverStateTable.Empty, SyntaxStore.Empty, driverOptions.DisabledOutputs, runtime: TimeSpan.Zero, driverOptions.TrackIncrementalGeneratorSteps);
}

public GeneratorDriver RunGenerators(Compilation compilation, CancellationToken cancellationToken = default)
Expand Down Expand Up @@ -171,7 +171,7 @@ internal GeneratorDriverState RunGeneratorsCore(Compilation compilation, Diagnos
var state = _state;
var stateBuilder = ArrayBuilder<GeneratorState>.GetInstance(state.Generators.Length);
var constantSourcesBuilder = ArrayBuilder<SyntaxTree>.GetInstance();
var syntaxInputNodes = ArrayBuilder<ISyntaxInputNode>.GetInstance();
var syntaxInputNodes = ArrayBuilder<SyntaxInputNode>.GetInstance();

for (int i = 0; i < state.IncrementalGenerators.Length; i++)
{
Expand All @@ -183,7 +183,7 @@ internal GeneratorDriverState RunGeneratorsCore(Compilation compilation, Diagnos
if (!generatorState.Initialized)
{
var outputBuilder = ArrayBuilder<IIncrementalGeneratorOutputNode>.GetInstance();
var inputBuilder = ArrayBuilder<ISyntaxInputNode>.GetInstance();
var inputBuilder = ArrayBuilder<SyntaxInputNode>.GetInstance();
var postInitSources = ImmutableArray<GeneratedSyntaxTree>.Empty;
var pipelineContext = new IncrementalGeneratorInitializationContext(inputBuilder, outputBuilder, SourceExtension);

Expand Down Expand Up @@ -241,7 +241,9 @@ internal GeneratorDriverState RunGeneratorsCore(Compilation compilation, Diagnos
}
constantSourcesBuilder.Free();

var driverStateBuilder = new DriverStateTable.Builder(compilation, _state, syntaxInputNodes.ToImmutableAndFree(), cancellationToken);
var syntaxStoreBuilder = _state.SyntaxStore.ToBuilder(compilation, syntaxInputNodes.ToImmutableAndFree(), _state.TrackIncrementalSteps, cancellationToken);

var driverStateBuilder = new DriverStateTable.Builder(compilation, _state, syntaxStoreBuilder, cancellationToken);
for (int i = 0; i < state.IncrementalGenerators.Length; i++)
{
var generatorState = stateBuilder[i];
Expand All @@ -266,7 +268,7 @@ internal GeneratorDriverState RunGeneratorsCore(Compilation compilation, Diagnos
}
}

state = state.With(stateTable: driverStateBuilder.ToImmutable(), generatorStates: stateBuilder.ToImmutableAndFree(), runTime: timer.Elapsed);
state = state.With(stateTable: driverStateBuilder.ToImmutable(), syntaxStore: syntaxStoreBuilder.ToImmutable(), generatorStates: stateBuilder.ToImmutableAndFree(), runTime: timer.Elapsed);
return state;
}

Expand Down
6 changes: 6 additions & 0 deletions src/Compilers/Core/Portable/SourceGeneration/GeneratorDriverState.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ internal GeneratorDriverState(ParseOptions parseOptions,
ImmutableArray<AdditionalText> additionalTexts,
ImmutableArray<GeneratorState> generatorStates,
DriverStateTable stateTable,
SyntaxStore syntaxStore,
IncrementalGeneratorOutputKind disabledOutputs,
TimeSpan runtime,
bool trackIncrementalGeneratorSteps)
Expand All @@ -29,6 +30,7 @@ internal GeneratorDriverState(ParseOptions parseOptions,
ParseOptions = parseOptions;
OptionsProvider = optionsProvider;
StateTable = stateTable;
SyntaxStore = syntaxStore;
DisabledOutputs = disabledOutputs;
RunTime = runtime;
TrackIncrementalSteps = trackIncrementalGeneratorSteps;
Expand Down Expand Up @@ -80,6 +82,8 @@ internal GeneratorDriverState(ParseOptions parseOptions,

internal readonly DriverStateTable StateTable;

internal readonly SyntaxStore SyntaxStore;

/// <summary>
/// A bit field containing the output kinds that should not be produced by this generator driver.
/// </summary>
Expand All @@ -95,6 +99,7 @@ internal GeneratorDriverState With(
ImmutableArray<GeneratorState>? generatorStates = null,
ImmutableArray<AdditionalText>? additionalTexts = null,
DriverStateTable? stateTable = null,
SyntaxStore? syntaxStore = null,
ParseOptions? parseOptions = null,
AnalyzerConfigOptionsProvider? optionsProvider = null,
IncrementalGeneratorOutputKind? disabledOutputs = null,
Expand All @@ -108,6 +113,7 @@ internal GeneratorDriverState With(
additionalTexts ?? this.AdditionalTexts,
generatorStates ?? this.GeneratorStates,
stateTable ?? this.StateTable,
syntaxStore ?? this.SyntaxStore,
disabledOutputs ?? this.DisabledOutputs,
runTime ?? this.RunTime,
this.TrackIncrementalSteps
Expand Down
12 changes: 6 additions & 6 deletions src/Compilers/Core/Portable/SourceGeneration/GeneratorState.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ public GeneratorState(GeneratorInfo info)
/// Creates a new generator state that contains information and constant trees
/// </summary>
public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees)
: this(info, postInitTrees, ImmutableArray<ISyntaxInputNode>.Empty, ImmutableArray<IIncrementalGeneratorOutputNode>.Empty)
: this(info, postInitTrees, ImmutableArray<SyntaxInputNode>.Empty, ImmutableArray<IIncrementalGeneratorOutputNode>.Empty)
{
}

/// <summary>
/// Creates a new generator state that contains information, constant trees and an execution pipeline
/// </summary>
public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<ISyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes)
public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<SyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes)
: this(info, postInitTrees, inputNodes, outputNodes, ImmutableArray<GeneratedSyntaxTree>.Empty, ImmutableArray<Diagnostic>.Empty, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, exception: null, elapsedTime: TimeSpan.Zero)
{
}
Expand All @@ -40,19 +40,19 @@ public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> po
/// Creates a new generator state that contains an exception and the associated diagnostic
/// </summary>
public GeneratorState(GeneratorInfo info, Exception e, Diagnostic error, TimeSpan elapsedTime)
: this(info, ImmutableArray<GeneratedSyntaxTree>.Empty, ImmutableArray<ISyntaxInputNode>.Empty, ImmutableArray<IIncrementalGeneratorOutputNode>.Empty, ImmutableArray<GeneratedSyntaxTree>.Empty, ImmutableArray.Create(error), ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, exception: e, elapsedTime: elapsedTime)
: this(info, ImmutableArray<GeneratedSyntaxTree>.Empty, ImmutableArray<SyntaxInputNode>.Empty, ImmutableArray<IIncrementalGeneratorOutputNode>.Empty, ImmutableArray<GeneratedSyntaxTree>.Empty, ImmutableArray.Create(error), ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>>.Empty, exception: e, elapsedTime: elapsedTime)
{
}

/// <summary>
/// Creates a generator state that contains results
/// </summary>
public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<ISyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes, ImmutableArray<GeneratedSyntaxTree> generatedTrees, ImmutableArray<Diagnostic> diagnostics, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> executedSteps, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> outputSteps, TimeSpan elapsedTime)
public GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<SyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes, ImmutableArray<GeneratedSyntaxTree> generatedTrees, ImmutableArray<Diagnostic> diagnostics, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> executedSteps, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> outputSteps, TimeSpan elapsedTime)
: this(info, postInitTrees, inputNodes, outputNodes, generatedTrees, diagnostics, executedSteps, outputSteps, exception: null, elapsedTime)
{
}

private GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<ISyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes, ImmutableArray<GeneratedSyntaxTree> generatedTrees, ImmutableArray<Diagnostic> diagnostics, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> executedSteps, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> outputSteps, Exception? exception, TimeSpan elapsedTime)
private GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> postInitTrees, ImmutableArray<SyntaxInputNode> inputNodes, ImmutableArray<IIncrementalGeneratorOutputNode> outputNodes, ImmutableArray<GeneratedSyntaxTree> generatedTrees, ImmutableArray<Diagnostic> diagnostics, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> executedSteps, ImmutableDictionary<string, ImmutableArray<IncrementalGeneratorRunStep>> outputSteps, Exception? exception, TimeSpan elapsedTime)
{
this.Initialized = true;
this.PostInitTrees = postInitTrees;
Expand All @@ -71,7 +71,7 @@ private GeneratorState(GeneratorInfo info, ImmutableArray<GeneratedSyntaxTree> p

internal ImmutableArray<GeneratedSyntaxTree> PostInitTrees { get; }

internal ImmutableArray<ISyntaxInputNode> InputNodes { get; }
internal ImmutableArray<SyntaxInputNode> InputNodes { get; }

internal ImmutableArray<IIncrementalGeneratorOutputNode> OutputNodes { get; }

Expand Down
4 changes: 2 additions & 2 deletions src/Compilers/Core/Portable/SourceGeneration/IncrementalContexts.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ namespace Microsoft.CodeAnalysis
/// </summary>
public readonly struct IncrementalGeneratorInitializationContext
{
private readonly ArrayBuilder<ISyntaxInputNode> _syntaxInputBuilder;
private readonly ArrayBuilder<SyntaxInputNode> _syntaxInputBuilder;
private readonly ArrayBuilder<IIncrementalGeneratorOutputNode> _outputNodes;
private readonly string _sourceExtension;

internal IncrementalGeneratorInitializationContext(ArrayBuilder<ISyntaxInputNode> syntaxInputBuilder, ArrayBuilder<IIncrementalGeneratorOutputNode> outputNodes, string sourceExtension)
internal IncrementalGeneratorInitializationContext(ArrayBuilder<SyntaxInputNode> syntaxInputBuilder, ArrayBuilder<IIncrementalGeneratorOutputNode> outputNodes, string sourceExtension)
{
_syntaxInputBuilder = syntaxInputBuilder;
_outputNodes = outputNodes;
Expand Down
Loading

0 comments on commit a7e3c09

Please sign in to comment.