Skip to content

Commit ed04790

Browse files
sanoursasanoursa
committed
Perf improvements and bug fixes:
- Clean up tracing to clearly indicate errors - Added an unattended mode for use in labs - Replace most ESENT databases with our own FileBasedCollection - Updated always_exclude to track individual files, not folders - Added new gvfs commands for managing cache servers - Integrated GvFlt perf improvements, including a negative path cache - Added ability to prefetch files - Miscellaneous other bug fixes
1 parent 153006d commit ed04790

File tree

243 files changed

+11479
-4724
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

243 files changed

+11479
-4724
lines changed

‎GVFS.sln

+29-1
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GVFS", "GVFS", "{2EF2EC94-3
2222
EndProject
2323
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.GVFlt", "GVFS\GVFS.GVFlt\GVFS.GVFlt.csproj", "{1118B427-7063-422F-83B9-5023C8EC5A7A}"
2424
EndProject
25-
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GvFlt", "GVFS\GVFS.GvFltWrapper\GvFlt.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}"
25+
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GvLib", "GVFS\GVFS.GvFltWrapper\GvLib.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}"
2626
ProjectSection(ProjectDependencies) = postProject
2727
{5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258}
2828
{374BF1E5-0B2D-4D4A-BD5E-4212299DEF09} = {374BF1E5-0B2D-4D4A-BD5E-4212299DEF09}
2929
EndProjectSection
3030
EndProject
3131
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Common", "GVFS\GVFS.Common\GVFS.Common.csproj", "{374BF1E5-0B2D-4D4A-BD5E-4212299DEF09}"
32+
ProjectSection(ProjectDependencies) = postProject
33+
{A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574}
34+
EndProjectSection
3235
EndProject
3336
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS", "GVFS\GVFS\GVFS.csproj", "{32220664-594C-4425-B9A0-88E0BE2F3D2A}"
3437
ProjectSection(ProjectDependencies) = postProject
@@ -59,6 +62,9 @@ EndProject
5962
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.NativeTests", "GVFS\GVFS.NativeTests\GVFS.NativeTests.vcxproj", "{3771C555-B5C1-45E2-B8B7-2CEF1619CDC5}"
6063
EndProject
6164
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Hooks", "GVFS\GVFS.Hooks\GVFS.Hooks.csproj", "{BDA91EE5-C684-4FC5-A90A-B7D677421917}"
65+
ProjectSection(ProjectDependencies) = postProject
66+
{A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574}
67+
EndProjectSection
6268
EndProject
6369
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Service", "GVFS\GVFS.Service\GVFS.Service.csproj", "{B8C1DFBA-CAFD-4F7E-A1A3-E11907B5467B}"
6470
ProjectSection(ProjectDependencies) = postProject
@@ -73,6 +79,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Mount", "GVFS\GVFS.Mou
7379
EndProjectSection
7480
EndProject
7581
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.ReadObjectHook", "GVFS\GVFS.ReadObjectHook\GVFS.ReadObjectHook.vcxproj", "{5A6656D5-81C7-472C-9DC8-32D071CB2258}"
82+
ProjectSection(ProjectDependencies) = postProject
83+
{A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574}
84+
EndProjectSection
7685
EndProject
7786
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{28674A4B-1223-4633-A460-C8CC39B09318}"
7887
ProjectSection(SolutionItems) = preProject
@@ -83,9 +92,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{2867
8392
Scripts\RunUnitTests.bat = Scripts\RunUnitTests.bat
8493
EndProjectSection
8594
EndProject
95+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.PerfProfiling", "GVFS\GVFS.PerfProfiling\GVFS.PerfProfiling.csproj", "{C5D3CA26-562F-4CA4-A378-B93E97A730E3}"
96+
EndProject
8697
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Service.UI", "GVFS\GVFS.Service.UI\GVFS.Service.UI.csproj", "{93B403FD-DAFB-46C5-9636-B122792A548A}"
8798
EndProject
99+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.PreBuild", "GVFS\GVFS.Build\GVFS.PreBuild.csproj", "{A4984251-840E-4622-AD0C-66DFCE2B2574}"
100+
EndProject
101+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{AB0D9230-3893-4486-8899-F9C871FB5D5F}"
102+
EndProject
88103
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GitHooksLoader", "GitHooksLoader\GitHooksLoader.vcxproj", "{798DE293-6EDA-4DC4-9395-BE7A71C563E3}"
104+
ProjectSection(ProjectDependencies) = postProject
105+
{A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574}
106+
EndProjectSection
89107
EndProject
90108
Global
91109
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -145,10 +163,18 @@ Global
145163
{5A6656D5-81C7-472C-9DC8-32D071CB2258}.Debug|x64.Build.0 = Debug|x64
146164
{5A6656D5-81C7-472C-9DC8-32D071CB2258}.Release|x64.ActiveCfg = Release|x64
147165
{5A6656D5-81C7-472C-9DC8-32D071CB2258}.Release|x64.Build.0 = Release|x64
166+
{C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Debug|x64.ActiveCfg = Debug|x64
167+
{C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Debug|x64.Build.0 = Debug|x64
168+
{C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Release|x64.ActiveCfg = Release|x64
169+
{C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Release|x64.Build.0 = Release|x64
148170
{93B403FD-DAFB-46C5-9636-B122792A548A}.Debug|x64.ActiveCfg = Debug|x64
149171
{93B403FD-DAFB-46C5-9636-B122792A548A}.Debug|x64.Build.0 = Debug|x64
150172
{93B403FD-DAFB-46C5-9636-B122792A548A}.Release|x64.ActiveCfg = Release|x64
151173
{93B403FD-DAFB-46C5-9636-B122792A548A}.Release|x64.Build.0 = Release|x64
174+
{A4984251-840E-4622-AD0C-66DFCE2B2574}.Debug|x64.ActiveCfg = Debug|x64
175+
{A4984251-840E-4622-AD0C-66DFCE2B2574}.Debug|x64.Build.0 = Debug|x64
176+
{A4984251-840E-4622-AD0C-66DFCE2B2574}.Release|x64.ActiveCfg = Release|x64
177+
{A4984251-840E-4622-AD0C-66DFCE2B2574}.Release|x64.Build.0 = Release|x64
152178
{798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Debug|x64.ActiveCfg = Debug|x64
153179
{798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Debug|x64.Build.0 = Debug|x64
154180
{798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Release|x64.ActiveCfg = Release|x64
@@ -172,6 +198,8 @@ Global
172198
{17498502-AEFF-4E70-90CC-1D0B56A8ADF5} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C}
173199
{5A6656D5-81C7-472C-9DC8-32D071CB2258} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C}
174200
{28674A4B-1223-4633-A460-C8CC39B09318} = {DCE11095-DA5F-4878-B58D-2702765560F5}
201+
{C5D3CA26-562F-4CA4-A378-B93E97A730E3} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA}
175202
{93B403FD-DAFB-46C5-9636-B122792A548A} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C}
203+
{A4984251-840E-4622-AD0C-66DFCE2B2574} = {AB0D9230-3893-4486-8899-F9C871FB5D5F}
176204
EndGlobalSection
177205
EndGlobal

‎GVFS/FastFetch/CheckoutFetchHelper.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public override void FastFetch(string branchOrCommit, bool isBranch)
6666
// Configure pipeline
6767
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
6868
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
69-
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
69+
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment);
7070
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
7171
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
7272
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);

‎GVFS/FastFetch/FastFetch.csproj

+5-2
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@
4343
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
4444
</PropertyGroup>
4545
<ItemGroup>
46-
<Reference Include="CommandLine">
47-
<HintPath>..\..\..\packages\CommandLineParser.2.0.275-beta\lib\net45\CommandLine.dll</HintPath>
46+
<Reference Include="CommandLine, Version=2.0.275.0, Culture=neutral, PublicKeyToken=de6f01bd326f8c32, processorArchitecture=MSIL">
47+
<SpecificVersion>False</SpecificVersion>
48+
<HintPath>..\..\..\packages\CommandLineParser.2.1.1-beta\lib\net45\CommandLine.dll</HintPath>
4849
<Private>True</Private>
4950
</Reference>
5051
<Reference Include="Microsoft.Diagnostics.Tracing.EventSource">
@@ -69,6 +70,8 @@
6970
<Compile Include="FetchHelper.cs" />
7071
<Compile Include="Git\BigEndianReader.cs" />
7172
<Compile Include="Git\EndianHelper.cs" />
73+
<Compile Include="Git\FastFetchGitObjects.cs" />
74+
<Compile Include="Git\FastFetchLibGit2Repo.cs" />
7275
<Compile Include="Git\GitIndexGenerator.cs" />
7376
<Compile Include="HashingStream.cs" />
7477
<Compile Include="WorkingTree.cs" />

‎GVFS/FastFetch/FastFetchVerb.cs

+29-17
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
using GVFS.Common.Tracing;
66
using Microsoft.Diagnostics.Tracing;
77
using System;
8-
using System.Diagnostics;
98

109
namespace FastFetch
1110
{
@@ -103,15 +102,15 @@ public class FastFetchVerb
103102
"folders",
104103
Required = false,
105104
Default = "",
106-
HelpText = "A semicolon-delimited list of paths to fetch")]
107-
public string PathWhitelist { get; set; }
105+
HelpText = "A semicolon-delimited list of folders to fetch")]
106+
public string FolderList { get; set; }
108107

109108
[Option(
110109
"folders-list",
111110
Required = false,
112111
Default = "",
113-
HelpText = "A file containing line-delimited list of paths to fetch")]
114-
public string PathWhitelistFile { get; set; }
112+
HelpText = "A file containing line-delimited list of folders to fetch")]
113+
public string FolderListFile { get; set; }
115114

116115
[Option(
117116
"Allow-index-metadata-update-from-working-tree",
@@ -192,7 +191,7 @@ private int ExecuteWithExitCode()
192191
Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID.");
193192
}
194193

195-
using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch"))
194+
using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", useCriticalTelemetryFlag: false))
196195
{
197196
if (this.Verbose)
198197
{
@@ -206,29 +205,26 @@ private int ExecuteWithExitCode()
206205
string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch");
207206
tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any);
208207

209-
RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
210-
211-
string error;
212-
CacheServerInfo cacheServer;
213-
if (!CacheServerInfo.TryDetermineCacheServer(this.CacheServerUrl, tracer, enlistment, retryConfig, out cacheServer, out error))
214-
{
215-
tracer.RelatedError(error);
216-
return ExitFailure;
217-
}
208+
CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null);
218209

219210
tracer.WriteStartEvent(
220211
enlistment.EnlistmentRoot,
221212
enlistment.RepoUrl,
222213
cacheServer.Url,
214+
enlistment.GitObjectsRoot,
223215
new EventMetadata
224216
{
225217
{ "TargetCommitish", commitish },
226218
{ "Checkout", this.Checkout },
227219
});
228220

221+
RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
229222
FetchHelper fetchHelper = this.GetFetchHelper(tracer, enlistment, cacheServer, retryConfig);
230-
if (!FetchHelper.TryLoadPathWhitelist(tracer, this.PathWhitelist, this.PathWhitelistFile, enlistment, fetchHelper.PathWhitelist))
223+
string error;
224+
if (!FetchHelper.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, fetchHelper.FolderList, out error))
231225
{
226+
tracer.RelatedError(error);
227+
Console.WriteLine(error);
232228
return ExitFailure;
233229
}
234230

@@ -291,7 +287,23 @@ private int ExecuteWithExitCode()
291287
return isSuccess ? ExitSuccess : ExitFailure;
292288
}
293289
}
294-
290+
291+
private string GetRemoteUrl(Enlistment enlistment)
292+
{
293+
if (!string.IsNullOrWhiteSpace(this.CacheServerUrl))
294+
{
295+
return this.CacheServerUrl;
296+
}
297+
298+
string configuredUrl = CacheServerResolver.GetUrlFromConfig(enlistment);
299+
if (!string.IsNullOrWhiteSpace(configuredUrl))
300+
{
301+
return configuredUrl;
302+
}
303+
304+
return enlistment.RepoUrl;
305+
}
306+
295307
private FetchHelper GetFetchHelper(ITracer tracer, Enlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig)
296308
{
297309
GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig);

0 commit comments

Comments
 (0)