Add utility for GSheets logger to stream output to Sheets (#1072)

Author: khyatimahendru

.gencode_hash.txt

@@ -23,7 +23,7 @@ eeafb6e89a6c43816cfcb26244a95615a4ee75966e1c015cae5a13b6cfcb0078  gencode/docs/r
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-0f16008af7116a182322736046b05c2718ac8b1f2cd755a60d28c9cfaeb6a8a9  gencode/docs/site_metadata.html
+0da2fca987af8f37f82e93b8b3fd8b954e4fe185bfe1e1f55adee5fe7f8024a9  gencode/docs/site_metadata.html
 1bbcbf2d1ec1bf4554846b1daf09fa09c08dd967fa1f91ebec93027662529a31  gencode/docs/state.html
 b2398e7f9e352c47867e38f97168f3eb88e23280c7dd576f13e038c8c3f714c9  gencode/docs/state_mapping.html
 b8ff4cdb4b79449166b1c6e09ac0e58273ae62865b47bf943fdd365739c62a4c  gencode/docs/state_validation.html
@@ -121,7 +121,7 @@ fadf083200e73d46cfdec2f710d40e3d2bc4dbf05196e4d0900186e72b62d95e  gencode/java/u
 ad70168de2e1801e26f2c1a8c33a6169e236682de59fd3f6038a77ac264aca14  gencode/java/udmi/schema/SequenceValidationState.java
 72e3a07ce890c73fccd3d0cf757d073746b60b722c94edd5425dc066951a048f  gencode/java/udmi/schema/SetupUdmiConfig.java
 e5ce202bcf889287faa7d2d262d2f53f2241a10c659a83687de478803bdab127  gencode/java/udmi/schema/SetupUdmiState.java
-8fcb31df1070010da6d3ae61d741a77d2ce8b3757226bbe35ff9218a0581b1ec  gencode/java/udmi/schema/SiteLinks.java
+2f802c738dd4c19211af8b6dec71b7659c51177c2774cfd34fc07e617bbe5b3e  gencode/java/udmi/schema/SiteLinks.java
 a3dc7f14d4270ea370db2015785ad12d4bd61825213ab7ac4ee03dda21dde4f2  gencode/java/udmi/schema/SiteLocation.java
 4e51c21fcdb25e729258c428f55be4ac4909f526fdc3384734b6f5a87f1ef5cf  gencode/java/udmi/schema/SiteMetadata.java
 dc2fd582fffefbdeaf3bf7724ca34e94475c3cc820891ea4cc829847eb9ba79d  gencode/java/udmi/schema/State.java
@@ -213,7 +213,7 @@ d59671d0bc2ac184e820b70c277cdf91af0a936a60cdffd9b3af0ea314f08adc  gencode/python
 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77  gencode/python/udmi/schema/persistent_device.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
 a84e00db471b2038a473c2c83d72ea59b02c3d2bab56d38bc22435309f01e763  gencode/python/udmi/schema/query_cloud.py
-d42bae60bd667cda893d3670896ed087f20df629635949cf418ee9be2e975347  gencode/python/udmi/schema/site_metadata.py
+77bcd0622552871b9b1543f26c4fe961955813f64a8702f4efe894fba91ce116  gencode/python/udmi/schema/site_metadata.py
 e604cf0280fe772de5f4e5ecf10dc6c564b6177eeff9cd9fb8b385af8fe10a95  gencode/python/udmi/schema/state.py
 4a908cee3fb8afb559bcbfa594e57dbc515a35e4468e02600751b2fcce05a238  gencode/python/udmi/schema/state_blobset.py
 182e07b534403dcc121d980672e41b0fa2ee55c4da1f5c56f0dad5d599450c80  gencode/python/udmi/schema/state_blobset_blob.py

bin/toolrun

@@ -28,4 +28,22 @@ fi
 
 cmd="java -cp $UDMI_JAR $JDWP $JAVA_CLASS $util_name $@"
 echo $cmd
-$cmd 2>&1 | tee $OUT_DIR/$util_name.log
+
+site_path=$(realpath "$1")
+if [[ ! -d $site_path ]]; then
+  site_path=$(dirname "$site_path")
+fi
+
+if [[ -e "$site_path/site_metadata.json" ]]; then
+  spreadsheet=$(jq -r '.sheet' "$site_path/site_metadata.json")
+else
+  spreadsheet=
+fi
+
+if [[ $spreadsheet != "null" && -n "$spreadsheet" ]]; then
+  spreadsheet_id=$(echo "$spreadsheet" | grep -oP '(?<=/d/)[^/]+')
+  echo "Streaming logs to gsheet id $spreadsheet_id"
+  $cmd 2>&1 | tee $OUT_DIR/$util_name.log | stream_to_gsheets "$util_name" "$spreadsheet_id"
+else
+  $cmd 2>&1 | tee $OUT_DIR/$util_name.log
+fi

etc/shell_common.sh

@@ -104,3 +104,11 @@ export UDMI_VERSION=$udmi_version
 export UDMI_COMMIT=$udmi_commit
 export UDMI_TIMEVER=$udmi_timever
 export UDMI_REV=$udmi_rev
+
+function stream_to_gsheets {
+    local tool_name=$1
+    local sheet_id=$2
+    timestamp=$(date +%Y%m%d_%H%M%S)
+    java -cp "$UDMI_JAR" "com.google.udmi.util.SheetsOutputStream" "$tool_name" \
+        "$sheet_id" "$tool_name.$timestamp.log"
+}

gencode/docs/site_metadata.html

@@ -92,6 +92,11 @@
           "description": "Source repository where the UDMI site model is stored",
           "type": "string",
           "examples": ["https://github.com/faucetsdn/udmi_site_model", "git@github.com:faucetsdn/udmi_site_model.git"]
+        },
+        "sheet": {
+          "description": "Link to a spreadsheet to observe real-time output from any tool",
+          "type": "string",
+          "examples": ["https://docs.google.com/spreadsheets/d/<spreadsheet_id>"]
         }
       }
     },

gencode/java/udmi/schema/SiteLinks.java

@@ -111,4 +111,7 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
     testImplementation 'org.mockito:mockito-core:5.3.1'
+
+    implementation 'com.google.oauth-client:google-oauth-client-jetty:1.20.0'
+    implementation 'com.google.apis:google-api-services-sheets:v4-rev484-1.20.0'
 }

gencode/python/udmi/schema/site_metadata.py

@@ -104,4 +104,12 @@ dependencies {
     implementation 'com.google.cloud:google-cloud-firestore:0.84.0-beta'
     implementation group: 'junit', name: 'junit', version: '4.13.2'
     implementation 'org.jetbrains:annotations:20.1.0'
+
+    implementation 'com.google.oauth-client:google-oauth-client-jetty:1.20.0'
+    implementation 'com.google.apis:google-api-services-sheets:v4-rev484-1.20.0'
+
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2'
+    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
+    testImplementation 'org.mockito:mockito-core:5.3.1'
+    testImplementation 'org.mockito:mockito-junit-jupiter:5.6.0'
 }

schema/site_metadata.json

@@ -0,0 +1,50 @@
+package com.google.udmi.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * An OutputStream that duplicates output to two output streams, useful for scenarios such as
+ * logging to two destinations simultaneously.
+ */
+public class DualOutputStream extends OutputStream {
+
+  private final OutputStream primary;
+  private final OutputStream secondary;
+
+  public DualOutputStream(OutputStream primary, OutputStream secondary) {
+    this.primary = primary;
+    this.secondary = secondary;
+  }
+
+  @Override
+  public void write(int i) throws IOException {
+    primary.write(i);
+    secondary.write(i);
+  }
+
+  @Override
+  public void write(byte @NotNull [] b) throws IOException {
+    primary.write(b);
+    secondary.write(b);
+  }
+
+  @Override
+  public void write(byte @NotNull [] b, int off, int len) throws IOException {
+    primary.write(b, off, len);
+    secondary.write(b, off, len);
+  }
+
+  @Override
+  public void flush() throws IOException {
+    primary.flush();
+    secondary.flush();
+  }
+
+  @Override
+  public void close() throws IOException {
+    primary.close();
+    secondary.close();
+  }
+}