Provide option to directly zip the output files

CHANGELOG.md

@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add gradle application plugin for command line execution with gradle run [#890](https://github.com/ie3-institute/simona/issues/890)
 - Additional tests to check flexibility options of thermal house and storage [#729](https://github.com/ie3-institute/simona/issues/729)
 - EmAgents should be able to handle initialization [#945](https://github.com/ie3-institute/simona/issues/945)
+- Added option to directly zip the output files [#793](https://github.com/ie3-institute/simona/issues/793)
 
 ### Changed
 - Adapted to changed data source in PSDM [#435](https://github.com/ie3-institute/simona/issues/435)

docs/readthedocs/config.md

@@ -94,9 +94,13 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 ```
 
+While using a csv sink, the raw data output files can be zipped directly when `zipFiles = true` is used.
+
+
 #### Output configuration of the grid
 
 The grid output configuration defines for which grid components simulation values are to be output.

input/samples/vn_simona/vn_simona.conf

@@ -51,6 +51,7 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 
 simona.output.grid = {

src/main/resources/config/config-template.conf

@@ -269,6 +269,7 @@ simona.output.sink.csv {
   isHierarchic = Boolean | false
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = "Boolean" | false
 }
 #@optional
 simona.output.sink.influxDb1x {

src/main/scala/edu/ie3/simona/config/SimonaConfig.scala

@@ -2054,6 +2054,7 @@ object SimonaConfig {
             filePrefix: java.lang.String,
             fileSuffix: java.lang.String,
             isHierarchic: scala.Boolean,
+            zipFiles: scala.Boolean,
         )
         object Csv {
           def apply(
@@ -2073,6 +2074,7 @@ object SimonaConfig {
                 else "",
               isHierarchic =
                 c.hasPathOrNull("isHierarchic") && c.getBoolean("isHierarchic"),
+              zipFiles = c.hasPathOrNull("zipFiles") && c.getBoolean("zipFiles"),
             )
           }
         }

src/main/scala/edu/ie3/simona/io/result/ResultSinkType.scala

@@ -21,6 +21,7 @@ object ResultSinkType {
       fileFormat: String = ".csv",
       filePrefix: String = "",
       fileSuffix: String = "",
+      zipFiles: Boolean = false,
   ) extends ResultSinkType
 
   final case class InfluxDb1x(url: String, database: String, scenario: String)
@@ -48,7 +49,12 @@ object ResultSinkType {
 
     sink.headOption match {
       case Some(params: SimonaConfig.Simona.Output.Sink.Csv) =>
-        Csv(params.fileFormat, params.filePrefix, params.fileSuffix)
+        Csv(
+          params.fileFormat,
+          params.filePrefix,
+          params.fileSuffix,
+          params.zipFiles,
+        )
       case Some(params: SimonaConfig.Simona.Output.Sink.InfluxDb1x) =>
         InfluxDb1x(buildInfluxDb1xUrl(params), params.database, runName)
       case Some(params: SimonaConfig.ResultKafkaParams) =>

src/main/scala/edu/ie3/simona/main/RunSimonaStandalone.scala

@@ -10,12 +10,17 @@ import edu.ie3.simona.config.{ArgsParser, ConfigFailFast, SimonaConfig}
 import edu.ie3.simona.main.RunSimona._
 import edu.ie3.simona.sim.SimonaSim
 import edu.ie3.simona.sim.setup.SimonaStandaloneSetup
+import edu.ie3.util.io.FileIOUtils
 import org.apache.pekko.actor.typed.scaladsl.AskPattern._
 import org.apache.pekko.actor.typed.{ActorSystem, Scheduler}
 import org.apache.pekko.util.Timeout
 
+import java.nio.file.Path
 import scala.concurrent.Await
-import scala.concurrent.duration.DurationInt
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration.{Duration, DurationInt}
+import scala.jdk.FutureConverters.CompletionStageOps
+import scala.util.{Failure, Success}
 
 /** Run a standalone simulation of simona
   *
@@ -24,6 +29,7 @@ import scala.concurrent.duration.DurationInt
 object RunSimonaStandalone extends RunSimona[SimonaStandaloneSetup] {
 
   override implicit val timeout: Timeout = Timeout(12.hours)
+  implicit val compressTimeoutDuration: Duration = 15.minutes
 
   override def setup(args: Array[String]): SimonaStandaloneSetup = {
     // get the config and prepare it with the provided args
@@ -56,9 +62,38 @@ object RunSimonaStandalone extends RunSimona[SimonaStandaloneSetup] {
       case SimonaEnded(successful) =>
         simonaSim.terminate()
 
+        val config = SimonaConfig(simonaSetup.typeSafeConfig).simona.output
+
+        config.sink.csv.map(_.zipFiles).foreach { zipFiles =>
+          if (zipFiles) {
+            val rawOutputPath =
+              Path.of(simonaSetup.resultFileHierarchy.rawOutputDataDir)
+
+            rawOutputPath.toFile.listFiles().foreach { file =>
+              val fileName = file.getName
+              val archiveName = fileName.replace(".csv", "")
+              val filePath = rawOutputPath.resolve(fileName)
+
+              val compressFuture =
+                FileIOUtils
+                  .compressFile(filePath, rawOutputPath.resolve(archiveName))
+                  .asScala
+              compressFuture.onComplete {
+                case Success(_) =>
+                  FileIOUtils.deleteRecursively(filePath)
+                case Failure(exception) =>
+                  logger.error(
+                    s"Compression of output file to '$archiveName' has failed. Keep raw data.",
+                    exception,
+                  )
+              }
+              Await.ready(compressFuture, compressTimeoutDuration)
+            }
+          }
+        }
+
         successful
     }
-
   }
 
 }

src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala

@@ -63,7 +63,7 @@ import scala.jdk.CollectionConverters._
 class SimonaStandaloneSetup(
     val typeSafeConfig: Config,
     simonaConfig: SimonaConfig,
-    resultFileHierarchy: ResultFileHierarchy,
+    val resultFileHierarchy: ResultFileHierarchy,
     runtimeEventQueue: Option[LinkedBlockingQueue[RuntimeEvent]] = None,
     override val args: Array[String],
 ) extends SimonaSetup {

src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala

@@ -856,7 +856,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData {
           intercept[InvalidConfigParameterException] {
             ConfigFailFast invokePrivate checkDataSink(
               Sink(
-                Some(Csv("", "", "", isHierarchic = false)),
+                Some(Csv("", "", "", isHierarchic = false, zipFiles = false)),
                 Some(InfluxDb1x("", 0, "")),
                 None,
               )

src/test/scala/edu/ie3/simona/io/result/ResultSinkTypeSpec.scala

@@ -23,17 +23,19 @@ class ResultSinkTypeSpec extends UnitSpec {
             filePrefix = "",
             fileSuffix = "",
             isHierarchic = false,
+            zipFiles = false,
           )
         ),
         influxDb1x = None,
         kafka = None,
       )
 
       inside(ResultSinkType(conf, "testRun")) {
-        case Csv(fileFormat, filePrefix, fileSuffix) =>
+        case Csv(fileFormat, filePrefix, fileSuffix, zipFiles) =>
           fileFormat shouldBe conf.csv.value.fileFormat
           filePrefix shouldBe conf.csv.value.filePrefix
           fileSuffix shouldBe conf.csv.value.fileSuffix
+          zipFiles shouldBe conf.csv.value.zipFiles
         case _ =>
           fail("Wrong ResultSinkType got instantiated.")
       }
@@ -105,6 +107,7 @@ class ResultSinkTypeSpec extends UnitSpec {
             filePrefix = "",
             fileSuffix = "",
             isHierarchic = false,
+            zipFiles = false,
           )
         ),
         influxDb1x = Some(