Data Source Adapter for Google Sheets

dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019-2023 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.adapter;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gson.Gson;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.polypheny.db.TestHelper;
+import org.polypheny.db.TestHelper.JdbcConnection;
+
+
+@Ignore // for debug purpose
+public class GoogleSheetSourceTest {
+
+    @BeforeClass
+    public static void start() throws SQLException {
+        TestHelper.getInstance();
+
+        try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) {
+            Connection connection = polyphenyDbConnection.getConnection();
+            try ( Statement statement = connection.createStatement() ) {
+                Map<String, String> settings = new HashMap<>();
+                settings.put( "maxStringLength", "255" );
+                settings.put( "querySize", "1000" );
+                settings.put( "sheetsURL", "-----ADD-----" );
+                settings.put( "mode", "remote" );
+                settings.put( "resetRefreshToken", "No" );
+                Gson gson = new Gson();
+                statement.executeUpdate( "ALTER ADAPTERS ADD \"googlesheetunit\" USING 'GoogleSheets' AS SOURCE WITH '" + gson.toJson( settings ) + "'" );
+            }
+        }
+    }
+
+
+    @AfterClass
+    public static void end() throws SQLException {
+        try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) {
+            Connection connection = polyphenyDbConnection.getConnection();
+            try ( Statement statement = connection.createStatement() ) {
+                statement.executeUpdate( "ALTER ADAPTERS DROP googlesheetunit" );
+            }
+        }
+
+    }
+
+
+    @Test
+    public void existsSelect() throws SQLException {
+        try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) {
+            Connection connection = polyphenyDbConnection.getConnection();
+            try ( Statement statement = connection.createStatement() ) {
+                List<Object[]> expectedResult = ImmutableList.of(
+                        new Object[]{ "Oracle", "Product Manager", 127000 },
+                        new Object[]{ "eBay", "Software Engineer", 100000 },
+                        new Object[]{ "Amazon", "Product Manager", 310000 },
+                        new Object[]{ "Apple", "Software Engineering Manager", 372000 },
+                        new Object[]{ "Microsoft", "Software Engineer", 157000 }
+
+                );
+                TestHelper.checkResultSet(
+                        statement.executeQuery( "SELECT \"Company\", \"Title\", \"Yearly Compensation\" from salaries LIMIT 5" ),
+                        expectedResult
+                );
+            }
+        }
+    }
+
+
+    @Test
+    public void existsSecondSelect() throws SQLException {
+        try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) {
+            Connection connection = polyphenyDbConnection.getConnection();
+            try ( Statement statement = connection.createStatement() ) {
+                List<Object[]> expectedResult = ImmutableList.of(
+                        new Object[]{ "GP", "F", 18, "U", "GT3", "A", 4, 4, "at_home", "teacher", "course", "mother", 2, 2, 0, "yes", "no", "no", "no", "yes", "yes", "no", "no", 4, 3, 4, 3, 6, 5, 6, 6 },
+                        new Object[]{ "GP", "F", 17, "U", "GT3", "T", 1, 1, "at_home", "other", "course", "father", 1, 2, 0, "no", "yes", "no", "no", "no", "yes", "yes", "no", 5, 3, 3, 3, 4, 5, 5, 6 },
+                        new Object[]{ "GP", "F", 15, "U", "LE3", "T", 1, 1, "at_home", "other", "other", "mother", 1, 2, 3, "yes", "no", "yes", "no", "yes", "yes", "yes", "no", 4, 3, 2, 3, 10, 7, 8, 10 }
+                );
+                TestHelper.checkResultSet(
+                        statement.executeQuery( "SELECT * from student_data LIMIT 3" ),
+                        expectedResult
+                );
+            }
+        }
+    }
+
+
+    @Test
+    public void existsThirdSelect() throws SQLException {
+        try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) {
+            Connection connection = polyphenyDbConnection.getConnection();
+            try ( Statement statement = connection.createStatement() ) {
+                List<Object[]> expectedResult = ImmutableList.of(
+                        new Object[]{ "Alexandra", "Female" },
+                        new Object[]{ "Andrew", "Male" },
+                        new Object[]{ "Anna", "Female" }
+                );
+                TestHelper.checkResultSet(
+                        statement.executeQuery( "SELECT \"Student Name\", \"Gender\" from student_formatting" ),
+                        expectedResult
+                );
+            }
+        }
+    }
+
+}

gradle.properties

@@ -59,6 +59,8 @@ foodmart_data_hsqldb_version = 0.3
 foodmart_data_json_version = 0.4
 foodmart_queries_version = 0.4.1
 geode_core_version = 1.6.0
+google_api_client_version = 1.33.0
+google_api_sheets_version = v4-rev20210629-1.32.1
 gradle_test_logger_version = 2.1.1
 gradle_lint_plugin = 17.7.1
 gson_version = 2.8.9
@@ -105,6 +107,7 @@ natty_version = 0.13
 okhttp_version = 4.2.2
 opencsv_version = 2.3
 oshi_core_version = 5.7.4
+oauth_client_version = 1.34.1
 pigunit_version = 0.16.0
 pig_version = 0.16.0
 poi_version = 5.2.3

monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2022 The Polypheny Project
+ * Copyright 2019-2023 The Polypheny Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -51,7 +53,8 @@ public class MonitoringQueueImpl implements MonitoringQueue {
     private final int MAXIMUM_POOL_SIZE;
     private final int KEEP_ALIVE_TIME;
 
-    private final boolean backgroundProcessingActive;
+    private boolean backgroundProcessingActive;
+    private int threadCount;
 
 
     /**
@@ -84,6 +87,24 @@ public MonitoringQueueImpl(
                     TimeUnit.SECONDS,
                     eventQueue );
         }
+
+        // Instantiated thread count
+        this.threadCount = threadPoolWorkers.getPoolSize();
+
+        // Create a scheduled, separate thread which gets new thread count every 500 milliseconds
+        Timer timer = new Timer();
+        timer.scheduleAtFixedRate( new TimerTask() {
+            @Override
+            public synchronized void run() {
+                int newThreadCount = threadPoolWorkers.getPoolSize();
+                if ( newThreadCount != threadCount ) {
+                    threadCount = newThreadCount;
+                    if ( log.isDebugEnabled() ) {
+                        log.debug( "Thread count is now: {}", threadCount );
+                    }
+                }
+            }
+        }, 500, 500 );
     }
 
 
@@ -98,7 +119,7 @@ public MonitoringQueueImpl(
 
 
     @Override
-    public void queueEvent( @NonNull MonitoringEvent event ) {
+    public synchronized void queueEvent( @NonNull MonitoringEvent event ) {
         if ( backgroundProcessingActive ) {
             threadPoolWorkers.execute( new MonitoringWorker( event ) );
         }
@@ -111,13 +132,13 @@ public void queueEvent( @NonNull MonitoringEvent event ) {
      * @return Current number of elements in Queue
      */
     @Override
-    public long getNumberOfElementsInQueue() {
+    public synchronized long getNumberOfElementsInQueue() {
         return threadPoolWorkers.getQueue().size();
     }
 
 
     @Override
-    public List<HashMap<String, String>> getInformationOnElementsInQueue() {
+    public synchronized List<HashMap<String, String>> getInformationOnElementsInQueue() {
         List<HashMap<String, String>> infoList = new ArrayList<>();
         List<MonitoringEvent> queueElements = new ArrayList<>();
 

plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java

@@ -33,20 +33,27 @@
 
 package org.polypheny.db.adapter.csv;
 
+import java.util.List;
 import org.apache.calcite.linq4j.tree.Blocks;
 import org.apache.calcite.linq4j.tree.Expressions;
 import org.apache.calcite.linq4j.tree.Primitive;
-import org.polypheny.db.adapter.enumerable.*;
+import org.polypheny.db.adapter.enumerable.EnumerableAlg;
+import org.polypheny.db.adapter.enumerable.EnumerableAlgImplementor;
+import org.polypheny.db.adapter.enumerable.EnumerableConvention;
+import org.polypheny.db.adapter.enumerable.PhysType;
+import org.polypheny.db.adapter.enumerable.PhysTypeImpl;
 import org.polypheny.db.algebra.AlgNode;
 import org.polypheny.db.algebra.AlgWriter;
 import org.polypheny.db.algebra.core.Scan;
 import org.polypheny.db.algebra.metadata.AlgMetadataQuery;
 import org.polypheny.db.algebra.type.AlgDataType;
 import org.polypheny.db.algebra.type.AlgDataTypeFactory;
 import org.polypheny.db.algebra.type.AlgDataTypeField;
-import org.polypheny.db.plan.*;
-
-import java.util.List;
+import org.polypheny.db.plan.AlgOptCluster;
+import org.polypheny.db.plan.AlgOptCost;
+import org.polypheny.db.plan.AlgOptPlanner;
+import org.polypheny.db.plan.AlgOptTable;
+import org.polypheny.db.plan.AlgTraitSet;
 
 
 /**

plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java

@@ -34,25 +34,36 @@
 package org.polypheny.db.test;
 
 
-import com.google.common.collect.Ordering;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.polypheny.db.sql.sql2alg.SqlToAlgConverter;
-import org.polypheny.db.util.Sources;
-import org.polypheny.db.util.Util;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.isA;
+import static org.junit.Assert.assertThat;
 
+import com.google.common.collect.Ordering;
 import java.io.PrintStream;
 import java.io.PrintWriter;
-import java.sql.*;
-import java.util.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.function.Consumer;
-
-import static org.hamcrest.CoreMatchers.*;
-import static org.junit.Assert.assertThat;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.polypheny.db.sql.sql2alg.SqlToAlgConverter;
+import org.polypheny.db.util.Sources;
+import org.polypheny.db.util.Util;
 
 
 /**

plugins/google-sheet-adapter/build.gradle

@@ -0,0 +1,48 @@
+group "org.polypheny"
+
+
+dependencies {
+    compileOnly project(":core")
+    compileOnly project(":plugins:sql-language")
+    compileOnly project(":dbms")
+
+    implementation group: "com.google.api-client", name: "google-api-client", version: google_api_client_version
+    implementation group: "com.google.oauth-client", name: "google-oauth-client-jetty", version: oauth_client_version
+    implementation group: "com.google.apis", name: "google-api-services-sheets", version: google_api_sheets_version
+
+
+    // --- Test Compile ---
+    testImplementation project(path: ":core", configuration: "tests")
+
+    testImplementation group: "junit", name: "junit", version: junit_version
+}
+
+
+compileJava {
+    dependsOn(":plugins:sql-language:processResources")
+}
+
+delombok {
+    dependsOn(":plugins:sql-language:processResources")
+}
+
+
+/**
+ * JARs
+ */
+jar {
+    manifest {
+        attributes "Manifest-Version": "1.0"
+        attributes "Copyright": "The Polypheny Project (polypheny.org)"
+        attributes "Version": "$project.version"
+    }
+}
+java {
+    withJavadocJar()
+    withSourcesJar()
+}
+
+licensee {
+    allow('Apache-2.0')
+    allow('MIT')
+}

plugins/google-sheet-adapter/gradle.properties

@@ -0,0 +1,27 @@
+#
+# Copyright 2019-2023 The Polypheny Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pluginVersion = 0.0.1
+
+pluginId = google-sheets-adapter
+pluginClass = org.polypheny.db.adapter.googlesheet.GoogleSheetPlugin
+pluginProvider = The Polypheny Project
+pluginDependencies = sql-language
+pluginUrlPath =
+pluginCategories = source
+pluginPolyDependencies =
+pluginIsSystemComponent = false
+pluginIsUiVisible = true