From e73eb5d03ddaf6c5ea681002f54dec3db38e3ee2 Mon Sep 17 00:00:00 2001 From: George Hoffman Date: Tue, 4 Dec 2018 11:44:02 -0800 Subject: [PATCH] Avoid unnecessary dependency downloading by providing fetches per cache policy; add ticker logging when they are downloading --- main/core/src/mill/eval/Evaluator.scala | 12 +++- main/core/src/mill/util/Logger.scala | 15 ++++ main/src/mill/modules/Jvm.scala | 72 ++++++++++++++++++-- scalalib/src/mill/scalalib/GenIdeaImpl.scala | 8 ++- scalalib/src/mill/scalalib/JavaModule.scala | 3 +- scalalib/src/mill/scalalib/Lib.scala | 12 ++-- scalalib/src/mill/scalalib/ScalaModule.scala | 4 +- 7 files changed, 110 insertions(+), 16 deletions(-) diff --git a/main/core/src/mill/eval/Evaluator.scala b/main/core/src/mill/eval/Evaluator.scala index 2aafdb7a13be..743916d1f2b2 100644 --- a/main/core/src/mill/eval/Evaluator.scala +++ b/main/core/src/mill/eval/Evaluator.scala @@ -251,7 +251,7 @@ case class Evaluator(home: os.Path, val nonEvaluatedTargets = group.indexed.filterNot(results.contains) - maybeTargetLabel.foreach { targetLabel => + val tickerPrefix = maybeTargetLabel.map { targetLabel => val inputResults = for { target <- nonEvaluatedTargets item <- target.inputs.filterNot(group.contains) @@ -259,10 +259,16 @@ case class Evaluator(home: os.Path, val logRun = inputResults.forall(_.isInstanceOf[Result.Success[_]]) - if(logRun) { log.ticker(s"[$counterMsg] $targetLabel ") } + val prefix = s"[$counterMsg] $targetLabel " + if(logRun) log.ticker(prefix) + prefix + "| " } - val multiLogger = resolveLogger(paths.map(_.log)) + val multiLogger = new ProxyLogger(resolveLogger(paths.map(_.log))) { + override def ticker(s: String): Unit = { + super.ticker(tickerPrefix.getOrElse("")+s) + } + } var usedDest = Option.empty[(Task[_], Array[StackTraceElement])] for (task <- nonEvaluatedTargets) { newEvaluated.append(task) diff --git a/main/core/src/mill/util/Logger.scala b/main/core/src/mill/util/Logger.scala index 4857953d3c95..5a833bfdf346 100644 --- a/main/core/src/mill/util/Logger.scala +++ b/main/core/src/mill/util/Logger.scala @@ -227,3 +227,18 @@ case class MultiLogger(colored: Boolean, logger1: Logger, logger2: Logger) exten logger2.close() } } + + +case class ProxyLogger(logger: Logger) extends Logger { + def colored = logger.colored + + lazy val outputStream = logger.outputStream + lazy val errorStream = logger.errorStream + lazy val inStream = logger.inStream + + def info(s: String) = logger.info(s) + def error(s: String) = logger.error(s) + def ticker(s: String) = logger.ticker(s) + def debug(s: String) = logger.debug(s) + override def close() = logger.close() +} diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 8d2c4de4fdda..c546062561a1 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -8,7 +8,7 @@ import java.nio.file.attribute.PosixFilePermission import java.util.Collections import java.util.jar.{JarEntry, JarFile, JarOutputStream} -import coursier.{Cache, Dependency, Fetch, Repository, Resolution} +import coursier.{Cache, Dependency, Fetch, Repository, Resolution, CachePolicy} import coursier.util.{Gather, Task} import geny.Generator import mill.main.client.InputPumper @@ -401,10 +401,11 @@ object Jvm { deps: TraversableOnce[coursier.Dependency], force: TraversableOnce[coursier.Dependency], sources: Boolean = false, - mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = { val (_, resolution) = resolveDependenciesMetadata( - repositories, deps, force, mapDependencies + repositories, deps, force, mapDependencies, ctx ) val errs = resolution.metadataErrors if(errs.nonEmpty) { @@ -458,7 +459,10 @@ object Jvm { def resolveDependenciesMetadata(repositories: Seq[Repository], deps: TraversableOnce[coursier.Dependency], force: TraversableOnce[coursier.Dependency], - mapDependencies: Option[Dependency => Dependency] = None) = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None) = { + + val cachePolicies = CachePolicy.default val forceVersions = force .map(mapDependencies.getOrElse(identity[Dependency](_))) @@ -471,10 +475,68 @@ object Jvm { mapDependencies = mapDependencies ) - val fetch = Fetch.from(repositories, Cache.fetch[Task]()) + val resolutionLogger = ctx.map(c => new TickerResolutionLogger(c)) + val fetches = cachePolicies.map { p => + Cache.fetch[Task]( + logger = resolutionLogger, + cachePolicy = p + ) + } + + val fetch = Fetch.from(repositories, fetches.head, fetches.tail: _*) import scala.concurrent.ExecutionContext.Implicits.global val resolution = start.process.run(fetch).unsafeRun() (deps.toSeq, resolution) } + + class TickerResolutionLogger(ctx: mill.util.Ctx.Log) extends Cache.Logger { + case class DownloadState(var current: Long, var total: Long) + var downloads = new mutable.TreeMap[String,DownloadState]() + var totalDownloadCount = 0 + var finishedCount = 0 + var finishedState = DownloadState(0,0) + + def updateTicker(): Unit = { + val sums = downloads.values + .fold(DownloadState(0,0)) { + (s1, s2) => DownloadState( + s1.current + s2.current, + Math.max(s1.current,s1.total) + Math.max(s2.current,s2.total) + ) + } + sums.current += finishedState.current + sums.total += finishedState.total + ctx.log.ticker(s"Downloading [${downloads.size + finishedCount}/$totalDownloadCount] artifacts (~${sums.current}/${sums.total} bytes)") + } + + override def downloadingArtifact(url: String, file: File): Unit = synchronized { + totalDownloadCount += 1 + downloads += url -> DownloadState(0,0) + updateTicker() + } + + override def downloadLength(url: String, totalLength: Long, alreadyDownloaded: Long, watching: Boolean): Unit = synchronized { + val state = downloads(url) + state.current = alreadyDownloaded + state.total = totalLength + updateTicker() + } + + override def downloadProgress(url: String, downloaded: Long): Unit = synchronized { + val state = downloads(url) + state.current = downloaded + updateTicker() + } + + override def downloadedArtifact(url: String, success: Boolean): Unit = synchronized { + val state = downloads(url) + finishedState.current += state.current + finishedState.total += Math.max(state.current, state.total) + finishedCount += 1 + downloads -= url + updateTicker() + } + } + } diff --git a/scalalib/src/mill/scalalib/GenIdeaImpl.scala b/scalalib/src/mill/scalalib/GenIdeaImpl.scala index 87b039dfaaa7..ae166ff02f1c 100644 --- a/scalalib/src/mill/scalalib/GenIdeaImpl.scala +++ b/scalalib/src/mill/scalalib/GenIdeaImpl.scala @@ -42,7 +42,7 @@ object GenIdeaImpl { val evaluator = new Evaluator(ctx.home, os.pwd / 'out, os.pwd / 'out, rootModule, ctx.log) - for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo)){ + for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo, ctx)){ os.write.over(os.pwd/relPath, pp.format(xml)) } } @@ -61,6 +61,7 @@ object GenIdeaImpl { def xmlFileLayout(evaluator: Evaluator, rootModule: mill.Module, jdkInfo: (String,String), + ctx: Log, fetchMillModules: Boolean = true): Seq[(os.RelPath, scala.xml.Node)] = { val modules = rootModule.millInternal.segmentsToModules.values @@ -78,7 +79,10 @@ object GenIdeaImpl { repos.toList, Lib.depToDependency(_, "2.12.4", ""), for(name <- artifactNames) - yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}" + yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}", + false, + None, + Some(ctx) ) res.items.toList.map(_.path) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 5cd9b60018c9..991256299157 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -138,7 +138,8 @@ trait JavaModule extends mill.Module with TaskModule { outer => resolveCoursierDependency().apply(_), deps(), sources, - mapDependencies = Some(mapDependencies()) + mapDependencies = Some(mapDependencies()), + Some(implicitly[mill.util.Ctx.Log]) ) } diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 807236375357..a5ef28c391f9 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -67,13 +67,15 @@ object Lib{ def resolveDependenciesMetadata(repositories: Seq[Repository], depToDependency: Dep => coursier.Dependency, deps: TraversableOnce[Dep], - mapDependencies: Option[Dependency => Dependency] = None) = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None) = { val depSeq = deps.toSeq mill.modules.Jvm.resolveDependenciesMetadata( repositories, depSeq.map(depToDependency), depSeq.filter(_.force).map(depToDependency), - mapDependencies + mapDependencies, + ctx ) } /** @@ -87,14 +89,16 @@ object Lib{ depToDependency: Dep => coursier.Dependency, deps: TraversableOnce[Dep], sources: Boolean = false, - mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = { val depSeq = deps.toSeq mill.modules.Jvm.resolveDependencies( repositories, depSeq.map(depToDependency), depSeq.filter(_.force).map(depToDependency), sources, - mapDependencies + mapDependencies, + ctx ) } def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String) = diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 38a5a8ea7804..0db64d1e1be8 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -104,7 +104,9 @@ trait ScalaModule extends JavaModule { outer => repositories, Lib.depToDependency(_, scalaVersion0, platformSuffix()), Seq(bridgeDep), - sources = true + sources = true, + mapDependencies = None, + Some(implicitly[mill.util.Ctx.Log]) ).map(deps => grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) )