Provide option to directly zip the output files

CHANGELOG.md

@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Implementation of StorageAgent [#309](https://github.com/ie3-institute/simona/issues/309)
 - Enhanced Newton-Raphson-PowerFlow failures with more information [#815](https://github.com/ie3-institute/simona/issues/815)
 - Update RTD references and bibliography [#868](https://github.com/ie3-institute/simona/issues/868)
+- Added option to directly zip the output files [#793](https://github.com/ie3-institute/simona/issues/793)
 
 ### Changed
 - Adapted to changed data source in PSDM [#435](https://github.com/ie3-institute/simona/issues/435)

docs/readthedocs/config.md

@@ -94,9 +94,13 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 ```
 
+While using a csv sink, the raw data output files can be zipped directly when `zipFiles = true` is used.
+
+
 #### Output configuration of the grid
 
 The grid output configuration defines for which grid components simulation values are to be output.

input/samples/vn_simona/vn_simona.conf

@@ -51,6 +51,7 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 
 simona.output.grid = {

src/main/resources/config/config-template.conf

@@ -269,6 +269,7 @@ simona.output.sink.csv {
   isHierarchic = Boolean | false
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = "Boolean" | false
 }
 #@optional
 simona.output.sink.influxDb1x {

src/main/scala/edu/ie3/simona/config/SimonaConfig.scala

@@ -2054,6 +2054,7 @@ object SimonaConfig {
             filePrefix: java.lang.String,
             fileSuffix: java.lang.String,
             isHierarchic: scala.Boolean,
+            zipFiles: scala.Boolean,
         )
         object Csv {
           def apply(
@@ -2073,6 +2074,7 @@ object SimonaConfig {
                 else "",
               isHierarchic =
                 c.hasPathOrNull("isHierarchic") && c.getBoolean("isHierarchic"),
+              zipFiles = c.hasPathOrNull("zipFiles") && c.getBoolean("zipFiles"),
             )
           }
         }

src/main/scala/edu/ie3/simona/io/result/ResultSinkType.scala

@@ -21,6 +21,7 @@ object ResultSinkType {
       fileFormat: String = ".csv",
       filePrefix: String = "",
       fileSuffix: String = "",
+      zipFiles: Boolean = false,
   ) extends ResultSinkType
 
   final case class InfluxDb1x(url: String, database: String, scenario: String)
@@ -48,7 +49,12 @@ object ResultSinkType {
 
     sink.headOption match {
       case Some(params: SimonaConfig.Simona.Output.Sink.Csv) =>
-        Csv(params.fileFormat, params.filePrefix, params.fileSuffix)
+        Csv(
+          params.fileFormat,
+          params.filePrefix,
+          params.fileSuffix,
+          params.zipFiles,
+        )
       case Some(params: SimonaConfig.Simona.Output.Sink.InfluxDb1x) =>
         InfluxDb1x(buildInfluxDb1xUrl(params), params.database, runName)
       case Some(params: SimonaConfig.ResultKafkaParams) =>

src/main/scala/edu/ie3/simona/main/RunSimonaStandalone.scala

@@ -10,12 +10,17 @@ import edu.ie3.simona.config.{ArgsParser, ConfigFailFast, SimonaConfig}
 import edu.ie3.simona.main.RunSimona._
 import edu.ie3.simona.sim.SimonaSim
 import edu.ie3.simona.sim.setup.SimonaStandaloneSetup
+import edu.ie3.util.io.FileIOUtils
 import org.apache.pekko.actor.typed.scaladsl.AskPattern._
 import org.apache.pekko.actor.typed.{ActorSystem, Scheduler}
 import org.apache.pekko.util.Timeout
 
+import java.nio.file.Path
 import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.DurationInt
+import scala.jdk.FutureConverters.CompletionStageOps
+import scala.util.{Failure, Success}
 
 /** Run a standalone simulation of simona
   *
@@ -56,9 +61,34 @@ object RunSimonaStandalone extends RunSimona[SimonaStandaloneSetup] {
       case SimonaEnded(successful) =>
         simonaSim.terminate()
 
+        val config = SimonaConfig(simonaSetup.typeSafeConfig).simona.output
+
+        config.sink.csv.map(_.zipFiles).map { zipFiles =>
+          if (zipFiles) {
+            val rawOutputPath =
+              Path.of(simonaSetup.resultFileHierarchy.rawOutputDataDir)
+            val archiveName = "rawOutputData.tar.gz"
+            val archivePath = rawOutputPath.getParent.resolve(archiveName)
+
+            logger.info(s"Compressing raw output data to: `$archiveName`.")
+
+            val compressFuture =
+              FileIOUtils.compressDir(rawOutputPath, archivePath).asScala
+            compressFuture.onComplete {
+              case Success(_) =>
+                FileIOUtils.deleteRecursively(rawOutputPath)
+              case Failure(exception) =>
+                logger.error(
+                  s"Compression of output files to '$archivePath' has failed. Keep raw data.",
+                  exception,
+                )
+            }
+            Await.ready(compressFuture, 5.minutes)
+          }
+        }
+
         successful
     }
-
   }
 
 }

src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala

@@ -63,7 +63,7 @@ import scala.jdk.CollectionConverters._
 class SimonaStandaloneSetup(
     val typeSafeConfig: Config,
     simonaConfig: SimonaConfig,
-    resultFileHierarchy: ResultFileHierarchy,
+    val resultFileHierarchy: ResultFileHierarchy,
     runtimeEventQueue: Option[LinkedBlockingQueue[RuntimeEvent]] = None,
     override val args: Array[String],
 ) extends SimonaSetup {

src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala

@@ -856,7 +856,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData {
           intercept[InvalidConfigParameterException] {
             ConfigFailFast invokePrivate checkDataSink(
               Sink(
-                Some(Csv("", "", "", isHierarchic = false)),
+                Some(Csv("", "", "", isHierarchic = false, zipFiles = false)),
                 Some(InfluxDb1x("", 0, "")),
                 None,
               )

src/test/scala/edu/ie3/simona/io/result/ResultSinkTypeSpec.scala

@@ -23,17 +23,19 @@ class ResultSinkTypeSpec extends UnitSpec {
             filePrefix = "",
             fileSuffix = "",
             isHierarchic = false,
+            zipFiles = false,
           )
         ),
         influxDb1x = None,
         kafka = None,
       )
 
       inside(ResultSinkType(conf, "testRun")) {
-        case Csv(fileFormat, filePrefix, fileSuffix) =>
+        case Csv(fileFormat, filePrefix, fileSuffix, zipFiles) =>
           fileFormat shouldBe conf.csv.value.fileFormat
           filePrefix shouldBe conf.csv.value.filePrefix
           fileSuffix shouldBe conf.csv.value.fileSuffix
+          zipFiles shouldBe conf.csv.value.zipFiles
         case _ =>
           fail("Wrong ResultSinkType got instantiated.")
       }
@@ -105,6 +107,7 @@ class ResultSinkTypeSpec extends UnitSpec {
             filePrefix = "",
             fileSuffix = "",
             isHierarchic = false,
+            zipFiles = false,
           )
         ),
         influxDb1x = Some(