diff --git a/integrationtests/Paket.IntegrationTests/InfoSpecs.fs b/integrationtests/Paket.IntegrationTests/InfoSpecs.fs index 7145482b03..8475709fb8 100644 --- a/integrationtests/Paket.IntegrationTests/InfoSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/InfoSpecs.fs @@ -17,7 +17,7 @@ let ``#3200 info should locate paket.dependencies``() = let ``paket info --paket-dependencies-dir`` workingDir = directPaketInPathEx "info --paket-dependencies-dir" workingDir - |> Seq.map PaketMsg.getMessage + |> Seq.map OutputMsg.getMessage |> List.ofSeq // paket.dependencies not exists diff --git a/integrationtests/Paket.IntegrationTests/InitSpecs.fs b/integrationtests/Paket.IntegrationTests/InitSpecs.fs index 80ad830680..cf6bf199ff 100644 --- a/integrationtests/Paket.IntegrationTests/InitSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/InitSpecs.fs @@ -29,7 +29,9 @@ let ``#1743 empty log file``() = use __ = paket "init --log-file" "i001040-init-downloads-bootstrapper" |> fst failwith "expected error" with - | exn when exn.Message.Split('\n').[0].Contains "--log-file" -> () + | ProcessFailedWithExitCode(_, _, msgs) -> + (msgs.Errors |> Seq.head).Contains "--log-file" + |> shouldEqual true [] #if PAKET_NETCORE diff --git a/integrationtests/Paket.IntegrationTests/LoadingScriptGenerationTests.fs b/integrationtests/Paket.IntegrationTests/LoadingScriptGenerationTests.fs index b692edbbcb..43be8b5c40 100644 --- a/integrationtests/Paket.IntegrationTests/LoadingScriptGenerationTests.fs +++ b/integrationtests/Paket.IntegrationTests/LoadingScriptGenerationTests.fs @@ -144,7 +144,7 @@ let ``fails on wrong framework given`` () = use __ = paket "install" scenario |> fst - let failure = Assert.Throws (fun () -> + let failure = Assert.Throws (fun () -> let result = directPaket "generate-load-scripts framework foo framework bar framework net45" scenario printf "%s" result ) @@ -161,7 +161,7 @@ let ``fails on wrong scripttype given`` () = use __ = paket "install" scenario |> fst - let failure = Assert.Throws (fun () -> + let failure = Assert.Throws (fun () -> let result = directPaket (sprintf "generate-load-scripts type foo type bar framework net45") scenario printf "%s" result ) diff --git a/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs b/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs index 68699ef719..2ed21bf8db 100644 --- a/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs +++ b/integrationtests/Paket.IntegrationTests/RestoreSpecs.fs @@ -8,6 +8,33 @@ open FsUnit open Paket open Paket.Utils +[] +let ``#3608 dotnet build should work with unparsable cache``() = + let project = "console" + let scenario = "i003608-invalid-cache" + use __ = prepareSdk scenario + let wd = (scenarioTempPath scenario) @@ project + // Build should work immediately (and call 'paket restore') + directDotnet true (sprintf "build %s.fsproj" project) wd + |> ignore + +[] +let ``#2684 Paket should not be called the second time in msbuild (Restore Performance)``() = + // NOTE: This test also ensure that FAKE can be used without paket on the CI server, see https://github.com/fsharp/FAKE/issues/2348 + let project = "console" + let scenario = "i002684-fast-restore" + use __ = prepareSdk scenario + + let wd = (scenarioTempPath scenario) @@ project + // first call paket restore (to restore and to extract the targets file as well as emulate a "full" restore) + directPaket "restore" scenario |> ignore + // second time no more paket calls should be required (as we already did a full restore) + directDotnetEx [ "PAKET_ERROR_ON_MSBUILD_EXEC", "true" ] true (sprintf "restore %s.fsproj" project) wd + |> ignore + // make sure it builds as well (checks if restore-targets contains syntax errors) + directDotnet true (sprintf "build %s.fsproj" project) wd + |> ignore + [] let ``#2496 Paket fails on projects that target multiple frameworks``() = let project = "EmptyTarget" diff --git a/integrationtests/Paket.IntegrationTests/TestHelper.fs b/integrationtests/Paket.IntegrationTests/TestHelper.fs index b411db2b2e..e0544d2b32 100644 --- a/integrationtests/Paket.IntegrationTests/TestHelper.fs +++ b/integrationtests/Paket.IntegrationTests/TestHelper.fs @@ -92,12 +92,23 @@ let prepareSdk scenario = FileHelper.CopyFile tmpPaketFolder targetsFile cleanup -type PaketMsg = +type OutputMsg = { IsError : bool; Message : string } - static member isError ({ IsError = e}:PaketMsg) = e - static member getMessage ({ Message = msg }:PaketMsg) = msg + static member isError ({ IsError = e}:OutputMsg) = e + static member getMessage ({ Message = msg }:OutputMsg) = msg +type OutputData = + internal { _Messages : ResizeArray } + member x.Messages : seq = x._Messages :> _ + member x.Errors = x._Messages |> Seq.filter OutputMsg.isError |> Seq.map OutputMsg.getMessage + member x.Outputs = x._Messages |> Seq.filter (not << OutputMsg.isError) |> Seq.map OutputMsg.getMessage + +exception ProcessFailedWithExitCode of exitCode:int * fileName:string * msgs:OutputData + with + override x.Message = + let output = String.Join(Environment.NewLine,x.msgs.Messages |> Seq.map (fun m -> (if m.IsError then "ERR:" else "OUT:") + m.Message )) + sprintf "The process '%s' exited with code %i, output: \n%s" x.fileName x.exitCode output -let directToolEx isPaket toolInfo commands workingDir = +let directToolEx env isPaket toolInfo commands workingDir = let processFilename, processArgs = match fst toolInfo, snd toolInfo with | "", path -> @@ -123,7 +134,7 @@ let directToolEx isPaket toolInfo commands workingDir = Environment.SetEnvironmentVariable("PAKET_DETAILED_WARNINGS", "true") printfn "%s> %s %s" workingDir (if isPaket then "paket" else processFilename) processArgs let perfMessages = ResizeArray() - let msgs = ResizeArray() + let msgs = ResizeArray() let mutable perfMessagesStarted = false let addAndPrint isError msg = if not isError then @@ -138,6 +149,8 @@ let directToolEx isPaket toolInfo commands workingDir = try ExecProcessWithLambdas (fun info -> info.FileName <- processFilename + for (key, value) in env do + info.EnvironmentVariables.[key] <- value info.WorkingDirectory <- workingDir info.CreateNoWindow <- true info.Arguments <- processArgs) @@ -163,38 +176,38 @@ let directToolEx isPaket toolInfo commands workingDir = if isPaket then // Only throw after the result <> 0 check because the current test might check the argument parsing // this is the only case where no performance is printed - let isUsageError = result <> 0 && msgs |> Seq.filter PaketMsg.isError |> Seq.map PaketMsg.getMessage |> Seq.exists (fun msg -> msg.Contains "USAGE:") + let isUsageError = result <> 0 && msgs |> Seq.filter OutputMsg.isError |> Seq.map OutputMsg.getMessage |> Seq.exists (fun msg -> msg.Contains "USAGE:") if not isUsageError then printfn "Performance:" for msg in perfMessages do printfn "%s" msg - if result <> 0 then - let errors = String.Join(Environment.NewLine,msgs |> Seq.filter PaketMsg.isError |> Seq.map PaketMsg.getMessage) - if String.IsNullOrWhiteSpace errors then - failwithf "The process exited with code %i" result - else - failwith errors + if result <> 0 then + raise <| ProcessFailedWithExitCode(result, processFilename, { _Messages = msgs }) msgs #endif let directPaketInPathEx command scenarioPath = - directToolEx true paketToolPath command scenarioPath + directToolEx [] true paketToolPath command scenarioPath let checkResults msgs = msgs - |> Seq.filter PaketMsg.isError + |> Seq.filter OutputMsg.isError |> Seq.toList |> shouldEqual [] -let directDotnet checkZeroWarn command workingDir = - let msgs = directToolEx false ("", dotnetToolPath) command workingDir +let directDotnetEx env checkZeroWarn command workingDir = + let msgs = directToolEx env false ("", dotnetToolPath) command workingDir if checkZeroWarn then checkResults msgs msgs + +let directDotnet checkZeroWarn command workingDir = + directDotnetEx [] checkZeroWarn command workingDir + let private fromMessages msgs = - String.Join(Environment.NewLine,msgs |> Seq.map PaketMsg.getMessage) + String.Join(Environment.NewLine,msgs |> Seq.map OutputMsg.getMessage) let directPaketInPath command scenarioPath = directPaketInPathEx command scenarioPath |> fromMessages diff --git a/integrationtests/scenarios/i002684-fast-restore/before/console/Program.fs b/integrationtests/scenarios/i002684-fast-restore/before/console/Program.fs new file mode 100644 index 0000000000..a7458f5220 --- /dev/null +++ b/integrationtests/scenarios/i002684-fast-restore/before/console/Program.fs @@ -0,0 +1,8 @@ +// Learn more about F# at http://fsharp.org + +open System + +[] +let main argv = + printfn "Hello World from F#!" + 0 // return an integer exit code diff --git a/integrationtests/scenarios/i002684-fast-restore/before/console/console.fsproj b/integrationtests/scenarios/i002684-fast-restore/before/console/console.fsproj new file mode 100644 index 0000000000..c3c40c2431 --- /dev/null +++ b/integrationtests/scenarios/i002684-fast-restore/before/console/console.fsproj @@ -0,0 +1,13 @@ + + + + Exe + netcoreapp2.1 + + + + + + + + \ No newline at end of file diff --git a/integrationtests/scenarios/i002684-fast-restore/before/console/paket.references b/integrationtests/scenarios/i002684-fast-restore/before/console/paket.references new file mode 100644 index 0000000000..640cf9145d --- /dev/null +++ b/integrationtests/scenarios/i002684-fast-restore/before/console/paket.references @@ -0,0 +1 @@ +FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i002684-fast-restore/before/paket.dependencies b/integrationtests/scenarios/i002684-fast-restore/before/paket.dependencies new file mode 100644 index 0000000000..2eb0f82c30 --- /dev/null +++ b/integrationtests/scenarios/i002684-fast-restore/before/paket.dependencies @@ -0,0 +1,4 @@ +source https://api.nuget.org/v3/index.json +restriction: || (==netstandard20) (== netcoreapp21) +storage: none +nuget FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i002684-fast-restore/before/paket.lock b/integrationtests/scenarios/i002684-fast-restore/before/paket.lock new file mode 100644 index 0000000000..2efe862347 --- /dev/null +++ b/integrationtests/scenarios/i002684-fast-restore/before/paket.lock @@ -0,0 +1,5 @@ +STORAGE: NONE +RESTRICTION: || (== netcoreapp2.1) (== netstandard2.0) +NUGET + remote: https://api.nuget.org/v3/index.json + FSharp.Core (4.6.2) diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/console/Program.fs b/integrationtests/scenarios/i003608-invalid-cache/before/console/Program.fs new file mode 100644 index 0000000000..a7458f5220 --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/console/Program.fs @@ -0,0 +1,8 @@ +// Learn more about F# at http://fsharp.org + +open System + +[] +let main argv = + printfn "Hello World from F#!" + 0 // return an integer exit code diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/console/console.fsproj b/integrationtests/scenarios/i003608-invalid-cache/before/console/console.fsproj new file mode 100644 index 0000000000..c3c40c2431 --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/console/console.fsproj @@ -0,0 +1,13 @@ + + + + Exe + netcoreapp2.1 + + + + + + + + \ No newline at end of file diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/console/paket.references b/integrationtests/scenarios/i003608-invalid-cache/before/console/paket.references new file mode 100644 index 0000000000..640cf9145d --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/console/paket.references @@ -0,0 +1 @@ +FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/paket-files/paket.restore.cached b/integrationtests/scenarios/i003608-invalid-cache/before/paket-files/paket.restore.cached new file mode 100644 index 0000000000..6f5dbc7aaf --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/paket-files/paket.restore.cached @@ -0,0 +1 @@ +Some not parsable File \ No newline at end of file diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/paket.dependencies b/integrationtests/scenarios/i003608-invalid-cache/before/paket.dependencies new file mode 100644 index 0000000000..2eb0f82c30 --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/paket.dependencies @@ -0,0 +1,4 @@ +source https://api.nuget.org/v3/index.json +restriction: || (==netstandard20) (== netcoreapp21) +storage: none +nuget FSharp.Core \ No newline at end of file diff --git a/integrationtests/scenarios/i003608-invalid-cache/before/paket.lock b/integrationtests/scenarios/i003608-invalid-cache/before/paket.lock new file mode 100644 index 0000000000..2efe862347 --- /dev/null +++ b/integrationtests/scenarios/i003608-invalid-cache/before/paket.lock @@ -0,0 +1,5 @@ +STORAGE: NONE +RESTRICTION: || (== netcoreapp2.1) (== netstandard2.0) +NUGET + remote: https://api.nuget.org/v3/index.json + FSharp.Core (4.6.2) diff --git a/src/Paket.Core/Installation/InstallProcess.fs b/src/Paket.Core/Installation/InstallProcess.fs index e6db765187..3fd63bffe9 100644 --- a/src/Paket.Core/Installation/InstallProcess.fs +++ b/src/Paket.Core/Installation/InstallProcess.fs @@ -585,7 +585,10 @@ let InstallIntoProjects(options : InstallerOptions, forceTouch, dependenciesFile first := false let restoreCacheFile = Path.Combine(root, Constants.PaketRestoreHashFilePath) - Paket.RestoreProcess.saveToFile (lockFile.ToString()) (FileInfo restoreCacheFile) |> ignore + let hash = Paket.RestoreProcess.getLockFileHashFromContent (lockFile.ToString()) + // NIT: We probably need to check if we have really fully restored (could be partial install) + // Lets see if users report issues + Paket.RestoreProcess.writeRestoreCache restoreCacheFile { PackagesDownloadedHash = hash; ProjectsRestoredHash = hash } Paket.RestoreProcess.WriteGitignore restoreCacheFile for project, _ in projectsAndReferences do diff --git a/src/Paket.Core/Installation/RestoreProcess.fs b/src/Paket.Core/Installation/RestoreProcess.fs index cebfe6828d..d01696d649 100644 --- a/src/Paket.Core/Installation/RestoreProcess.fs +++ b/src/Paket.Core/Installation/RestoreProcess.fs @@ -182,7 +182,7 @@ let copiedElements = ref false type private MyAssemblyFinder () = class end -let saveToFile newContent (targetFile:FileInfo) = +let private saveToFile newContent (targetFile:FileInfo) = let rec loop trials = try if not targetFile.Directory.Exists then @@ -567,6 +567,7 @@ let internal getStringHash (s:string) = |> BitConverter.ToString |> fun s -> s.Replace("-", "") + type internal Hash = | Hash of string member x.HashString = @@ -576,7 +577,11 @@ type internal Hash = match x with | Hash s -> String.IsNullOrEmpty s static member OfString s = Hash (getStringHash s) - + +let internal getLockFileHashFromContent (content:string) = + Hash.OfString content +let internal getLockFileHash (f:string) = + getLockFileHashFromContent (File.ReadAllText f) type internal RestoreCache = { PackagesDownloadedHash : Hash @@ -591,7 +596,7 @@ type internal RestoreCache = { PackagesDownloadedHash = Hash "" ProjectsRestoredHash = Hash "" } -let private writeRestoreCache (file:string) { PackagesDownloadedHash = Hash packagesDownloadedHash; ProjectsRestoredHash = Hash projectsRestoredHash} = +let internal writeRestoreCache (file:string) { PackagesDownloadedHash = Hash packagesDownloadedHash; ProjectsRestoredHash = Hash projectsRestoredHash} = let jobj = new Newtonsoft.Json.Linq.JObject() jobj.["packagesDownloadedHash"] <- Newtonsoft.Json.Linq.JToken.op_Implicit packagesDownloadedHash jobj.["projectsRestoredHash"] <- Newtonsoft.Json.Linq.JToken.op_Implicit projectsRestoredHash @@ -668,7 +673,7 @@ let Restore(dependenciesFileName,projectFile:RestoreProjectOptions,force,group,i None, RestoreCache.Empty, Hash "", false else let cache = readRestoreCache(lockFileName) - let lockFileHash = Hash.OfString (File.ReadAllText lockFileName.FullName) + let lockFileHash = getLockFileHash lockFileName.FullName let updatedCache = { PackagesDownloadedHash = lockFileHash // we always download all packages in that situation ProjectsRestoredHash = if projectFile = AllProjects then lockFileHash else cache.ProjectsRestoredHash } diff --git a/src/Paket.Core/embedded/Paket.Restore.targets b/src/Paket.Core/embedded/Paket.Restore.targets index 047a1aa2f5..c13fccb1be 100644 --- a/src/Paket.Core/embedded/Paket.Restore.targets +++ b/src/Paket.Core/embedded/Paket.Restore.targets @@ -73,37 +73,47 @@ + + + - + true $(NoWarn);NU1603;NU1604;NU1605;NU1608 + false + true - - - /usr/bin/shasum "$(PaketRestoreCacheFile)" | /usr/bin/awk '{ print $1 }' - /usr/bin/shasum "$(PaketLockFilePath)" | /usr/bin/awk '{ print $1 }' + + + + + + + $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) + + + + + + + $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[0].Replace(`"`, ``).Replace(` `, ``)) + $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[1].Replace(`"`, ``).Replace(` `, ``)) + + + + + %(PaketRestoreCachedKeyValue.Value) + %(PaketRestoreCachedKeyValue.Value) - - - - - - - - - - - - - - $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) - $([System.IO.File]::ReadAllText('$(PaketLockFilePath)')) + + true - false + false true @@ -116,7 +126,9 @@ + + @@ -126,7 +138,7 @@ - + $(PaketIntermediateOutputPath)\$(MSBuildProjectFile).paket.references.cached @@ -163,6 +175,7 @@ + @@ -255,6 +268,7 @@ <_NuspecFiles Include="$(AdjustedNuspecOutputPath)\*.$(PackageVersion.Split(`+`)[0]).nuspec"/> +