Merge remote-tracking branch 'origin/dev' into df/#98-transformer_sus…

…ceptance
ie3-institute · Mar 24, 2022 · 4330597 · 4330597
2 parents efd2775 + d80c00b
commit 4330597
Show file tree

Hide file tree

Showing 20 changed files with 693 additions and 326 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Added
+- Implement SQL source for primary data [#34](https://github.com/ie3-institute/simona/issues/34)
+
 ### Changed
 - Improving code readability in EvcsAgent by moving FreeLotsRequest to separate methods
 - Re-organizing test resources into their respective packages [#105](https://github.com/ie3-institute/simona/issues/105)

diff --git a/build.gradle b/build.gradle
@@ -10,7 +10,7 @@ plugins {
   id 'com.diffplug.spotless' version '6.3.0'// code format
   id 'com.github.onslip.gradle-one-jar' version '1.0.6' // pack a self contained jar
   id "com.github.ben-manes.versions" version '0.42.0'
-  id "de.undercouch.download" version "5.0.1" // downloads plugin
+  id "de.undercouch.download" version "5.0.2" // downloads plugin
   id "kr.motd.sphinx" version "2.10.1" // documentation generation
   id "com.github.johnrengelman.shadow" version "7.1.2" // fat jar
   id "org.sonarqube" version "3.3" // sonarqube
@@ -30,6 +30,8 @@ ext {
   tscfgVersion = '0.9.997'
   scapegoatVersion = '1.4.12'
 
+  testContainerVersion = '0.40.3'
+
   scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator // location of script plugins
 }
 
@@ -99,14 +101,18 @@ dependencies {
   implementation "ch.qos.logback:logback-classic:1.2.11"
 
   /* testing */
-  testImplementation 'org.spockframework:spock-core:2.1-M2-groovy-3.0'
+  testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0'
   testImplementation 'org.scalatestplus:mockito-3-4_2.13:3.2.10.0'
-  implementation 'org.mockito:mockito-core:4.3.1' // mocking framework
+  implementation 'org.mockito:mockito-core:4.4.0' // mocking framework
   testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.11"
   testRuntimeClasspath 'com.vladsch.flexmark:flexmark-all:0.64.0'
   testImplementation group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
   testImplementation "com.typesafe.akka:akka-testkit_${scalaVersion}:${akkaVersion}" // akka testkit
 
+  // testcontainers
+  testImplementation "com.dimafeng:testcontainers-scala-scalatest_${scalaVersion}:${testContainerVersion}"
+  testImplementation "com.dimafeng:testcontainers-scala-postgresql_${scalaVersion}:${testContainerVersion}"
+
   /* --- Scala libs --- */
   /* CORE Scala */
   implementation "org.scala-lang:scala-library:${scalaBinaryVersion}"

diff --git a/docs/readthedocs/usersguide.rst b/docs/readthedocs/usersguide.rst
@@ -201,3 +201,24 @@ These steps have to be performed each time updates to the external simulation ne
 
 Now, when a simulation with SIMONA is started (see `above <#running-a-standalone-simulation>`_), the external simulation is triggered at each tick that it requested.
 
+Troubleshooting
+===============
+
+My power flow calculation isn't converging - why is that?
+---------------------------------------------------------
+
+When your power flow is not converging it means that the load situation in the grid during the time of the power flow calculation is not physically feasible.
+
+This can have basically one of the following two reasons:
+
+#.
+    There is more load in the grid than it can physically handle.
+
+#.
+    There is more generation in the grid than it can physically handle.
+
+One of the main reasons is a misconfiguration of the grid and its assets.
+Assess the power of the load and generation units and check if the values make sense.
+Keep in mind the metric prefixes that are assumed for the models, which are listed in the `PSDM docs <https://powersystemdatamodel.readthedocs.io/en/latest/index.html>`_.
+If everything seems to be configured correctly it could also be the case that the grid itself is incorrectly configured.
+Do a similar sanity check for the grids assets.
diff --git a/gradle/scripts/tscfg.gradle b/gradle/scripts/tscfg.gradle
@@ -15,7 +15,7 @@ task genConfigClass {
       args = [
         "build/tscfg-${tscfgVersion}.jar",
         "--spec",
-        "src/main/resources/config/simona-config-template.conf",
+        "src/main/resources/config/config-template.conf",
         "--scala",
         "--durations",
         "--pn",

diff --git a/src/main/resources/config/config-template.conf b/src/main/resources/config/config-template.conf
@@ -101,9 +101,7 @@ simona.input.primary = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
       schemaName: string | "public"
-      timeColumnName: string
       timePattern: string | "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'" # default pattern from PSDM:TimeBasedSimpleValueFactory
     }
     #@optional
@@ -150,9 +148,9 @@ simona.input.weather.datasource = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
+      tableName: string
       schemaName: string | "public"
-      timeColumnName: string
+      timePattern: string | "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'" # default pattern from PSDM:TimeBasedSimpleValueFactory
     }
     #@optional
     couchbaseParams = {

diff --git a/src/main/scala/edu/ie3/simona/config/SimonaConfig.scala b/src/main/scala/edu/ie3/simona/config/SimonaConfig.scala
@@ -1,5 +1,5 @@
 /*
- * © 2021. TU Dortmund University,
+ * © 2022. TU Dortmund University,
  * Institute of Energy Systems, Energy Efficiency and Energy Economics,
  * Research group Distribution grid planning and operation
  */
@@ -908,10 +908,8 @@ object SimonaConfig {
             jdbcUrl: java.lang.String,
             password: java.lang.String,
             schemaName: java.lang.String,
-            timeColumnName: java.lang.String,
             timePattern: java.lang.String,
-            userName: java.lang.String,
-            weatherTableName: java.lang.String
+            userName: java.lang.String
         )
         object SqlParams {
           def apply(
@@ -925,14 +923,10 @@ object SimonaConfig {
               schemaName =
                 if (c.hasPathOrNull("schemaName")) c.getString("schemaName")
                 else "public",
-              timeColumnName =
-                $_reqStr(parentPath, c, "timeColumnName", $tsCfgValidator),
               timePattern =
                 if (c.hasPathOrNull("timePattern")) c.getString("timePattern")
                 else "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'",
-              userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator),
-              weatherTableName =
-                $_reqStr(parentPath, c, "weatherTableName", $tsCfgValidator)
+              userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator)
             )
           }
           private def $_reqStr(
@@ -1277,9 +1271,9 @@ object SimonaConfig {
               jdbcUrl: java.lang.String,
               password: java.lang.String,
               schemaName: java.lang.String,
-              timeColumnName: java.lang.String,
-              userName: java.lang.String,
-              weatherTableName: java.lang.String
+              tableName: java.lang.String,
+              timePattern: java.lang.String,
+              userName: java.lang.String
           )
           object SqlParams {
             def apply(
@@ -1293,11 +1287,12 @@ object SimonaConfig {
                 schemaName =
                   if (c.hasPathOrNull("schemaName")) c.getString("schemaName")
                   else "public",
-                timeColumnName =
-                  $_reqStr(parentPath, c, "timeColumnName", $tsCfgValidator),
-                userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator),
-                weatherTableName =
-                  $_reqStr(parentPath, c, "weatherTableName", $tsCfgValidator)
+                tableName =
+                  $_reqStr(parentPath, c, "tableName", $tsCfgValidator),
+                timePattern =
+                  if (c.hasPathOrNull("timePattern")) c.getString("timePattern")
+                  else "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'",
+                userName = $_reqStr(parentPath, c, "userName", $tsCfgValidator)
               )
             }
             private def $_reqStr(

diff --git a/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala b/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala
@@ -6,7 +6,8 @@
 
 package edu.ie3.simona.event.listener
 
-import akka.actor.{ActorRef, FSM, PoisonPill, Props, Stash}
+import akka.actor.{ActorRef, FSM, Props, Stash, Status}
+import akka.pattern.pipe
 import akka.stream.Materializer
 import edu.ie3.datamodel.io.processor.result.ResultEntityProcessor
 import edu.ie3.datamodel.models.result.ResultEntity
@@ -23,6 +24,7 @@ import edu.ie3.simona.event.listener.ResultEventListener.{
   BaseData,
   Init,
   ResultEventListenerData,
+  SinkResponse,
   Transformer3wKey,
   UninitializedData
 }
@@ -59,6 +61,10 @@ object ResultEventListener extends Transformer3wResultSupport {
 
   private final case object Init
 
+  private final case class SinkResponse(
+      response: Map[Class[_], ResultEntitySink]
+  )
+
   /** [[ResultEventListener]] base data containing all information the listener
     * needs
     *
@@ -106,29 +112,33 @@ object ResultEventListener extends Transformer3wResultSupport {
       case _: ResultSinkType.Csv =>
         eventClassesToConsider
           .map(resultClass => {
-            val fileName =
-              resultFileHierarchy.rawOutputDataFilePaths.getOrElse(
-                resultClass,
-                throw new FileHierarchyException(
-                  s"Unable to get file path for result class '${resultClass.getSimpleName}' from output file hierarchy! " +
-                    s"Available file result file paths: ${resultFileHierarchy.rawOutputDataFilePaths}"
-                )
-              )
-            if (fileName.endsWith(".csv") || fileName.endsWith(".csv.gz")) {
-              val sink =
-                ResultEntityCsvSink(
-                  fileName.replace(".gz", ""),
-                  new ResultEntityProcessor(resultClass),
-                  fileName.endsWith(".gz")
+            resultFileHierarchy.rawOutputDataFilePaths
+              .get(resultClass)
+              .map(Future.successful)
+              .getOrElse(
+                Future.failed(
+                  new FileHierarchyException(
+                    s"Unable to get file path for result class '${resultClass.getSimpleName}' from output file hierarchy! " +
+                      s"Available file result file paths: ${resultFileHierarchy.rawOutputDataFilePaths}"
+                  )
                 )
-              sink.map((resultClass, _))
-            } else {
-              throw new ProcessResultEventException(
-                s"Invalid output file format for file $fileName provided. Currently only '.csv' or '.csv.gz' is supported!"
               )
-            }
+              .flatMap { fileName =>
+                if (fileName.endsWith(".csv") || fileName.endsWith(".csv.gz")) {
+                  ResultEntityCsvSink(
+                    fileName.replace(".gz", ""),
+                    new ResultEntityProcessor(resultClass),
+                    fileName.endsWith(".gz")
+                  ).map((resultClass, _))
+                } else {
+                  Future(
+                    throw new ProcessResultEventException(
+                      s"Invalid output file format for file $fileName provided. Currently only '.csv' or '.csv.gz' is supported!"
+                    )
+                  )
+                }
+              }
           })
-
       case ResultSinkType.InfluxDb1x(url, database, scenario) =>
         // creates one connection per result entity that should be processed
         eventClassesToConsider
@@ -287,10 +297,6 @@ class ResultEventListener(
       stash()
       stay()
 
-    case Event(baseData: BaseData, UninitializedData) =>
-      unstashAll()
-      goto(Idle) using baseData
-
     case Event(Init, _) =>
       Future
         .sequence(
@@ -299,18 +305,20 @@ class ResultEventListener(
             resultFileHierarchy
           )
         )
-        .onComplete {
-          case Failure(exception) =>
-            throw new InitializationException(
-              "Cannot initialize result sinks!"
-            ).initCause(exception)
-            self ! PoisonPill
-          case Success(classToSink) =>
-            log.debug("Initialization complete!")
-            supervisor ! ServiceInitComplete
-            self ! BaseData(classToSink.toMap)
-        }
+        .map(result => SinkResponse(result.toMap))
+        .pipeTo(self)
       stay()
+
+    case Event(SinkResponse(classToSink), _) =>
+      // Sink Initialization succeeded
+      log.debug("Initialization complete!")
+      supervisor ! ServiceInitComplete
+
+      unstashAll()
+      goto(Idle) using BaseData(classToSink)
+
+    case Event(Status.Failure(ex), _) =>
+      throw new InitializationException("Unable to setup SimonaSim.", ex)
   }
 
   when(Idle) {