From e0d9fae4eef4419b9cd8026d796d0b3217fc515e Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Wed, 13 Apr 2022 17:57:40 -0400 Subject: [PATCH 01/19] added thread monitoring --- .../monitoring/core/MonitoringQueueImpl.java | 73 +++++++++++++++++-- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java index fc9d7523d9..d961b8aa2e 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java @@ -16,9 +16,7 @@ package org.polypheny.db.monitoring.core; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -51,7 +49,8 @@ public class MonitoringQueueImpl implements MonitoringQueue { private final int MAXIMUM_POOL_SIZE; private final int KEEP_ALIVE_TIME; - private final boolean backgroundProcessingActive; + private boolean backgroundProcessingActive; + private int threadCount; /** @@ -84,6 +83,23 @@ public MonitoringQueueImpl( TimeUnit.SECONDS, eventQueue ); } + + // instantiated thread count + this.threadCount = threadPoolWorkers.getPoolSize(); + + // create a scheduled, separate thread which gets new thread count every 500 milliseconds + Timer timer = new Timer(); + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public synchronized void run() { + int newThreadCount = threadPoolWorkers.getPoolSize(); + + if (newThreadCount != threadCount) { + threadCount = newThreadCount; + log.info("Thread count is now: {}", threadCount); + } + } + }, 500, 500); } @@ -98,7 +114,7 @@ public MonitoringQueueImpl( @Override - public void queueEvent( @NonNull MonitoringEvent event ) { + public synchronized void queueEvent( @NonNull MonitoringEvent event ) { if ( backgroundProcessingActive ) { threadPoolWorkers.execute( new MonitoringWorker( event ) ); } @@ -111,13 +127,13 @@ public void queueEvent( @NonNull MonitoringEvent event ) { * @return Current number of elements in Queue */ @Override - public long getNumberOfElementsInQueue() { + public synchronized long getNumberOfElementsInQueue() { return threadPoolWorkers.getQueue().size(); } @Override - public List> getInformationOnElementsInQueue() { + public synchronized List> getInformationOnElementsInQueue() { List> infoList = new ArrayList<>(); List queueElements = new ArrayList<>(); @@ -235,4 +251,47 @@ private void processQueue() { } + /* + class ThreadMonitor implements Runnable{ + private int corePoolThreadsCount; + private int threadCount; + private ThreadPoolExecutor monitoredThread; + + public ThreadMonitor(ThreadPoolExecutor t) { + super(); + this.monitoredThread = t; + this.corePoolThreadsCount = t.getCorePoolSize(); + this.threadCount = t.getPoolSize(); + } + + public void run() { + while (true) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + log.error("Thread Monitor wasn't able to sleep"); + } + + log.info("The guard is monitoring the threats."); + + int newCorePoolCount = monitoredThread.getCorePoolSize(); + int newThreadCount = monitoredThread.getPoolSize(); + + if (newCorePoolCount != this.corePoolThreadsCount) { + this.corePoolThreadsCount = newCorePoolCount; + log.info("new core pool thread count is = {}", this.corePoolThreadsCount); + } + + if (newThreadCount != this.threadCount) { + this.threadCount = newThreadCount; + log.info("new thread count is = {}", this.threadCount); + } + } + } + + + + } + */ + } From 357f5ac61064ba28bda5ea62a6af15a0376fa49a Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Mon, 18 Jul 2022 03:02:36 -0400 Subject: [PATCH 02/19] first semi-working version --- google-sheet-adapter/build.gradle | 81 +++++ google-sheet-adapter/lombok.config | 4 + .../googlesheet/GoogleSheetEnumerator.java | 326 ++++++++++++++++++ .../googlesheet/GoogleSheetFieldType.java | 108 ++++++ .../googlesheet/GoogleSheetReader.java | 136 ++++++++ .../googlesheet/GoogleSheetSchema.java | 153 ++++++++ .../googlesheet/GoogleSheetSource.java | 240 +++++++++++++ .../adapter/googlesheet/GoogleSheetTable.java | 116 +++++++ .../adapter/googlesheet/SheetsQuickstart.java | 116 +++++++ .../src/main/resources/credentials.json | 11 + .../main/resources/tokens/StoredCredential | Bin 0 -> 1132 bytes settings.gradle | 1 + 12 files changed, 1292 insertions(+) create mode 100644 google-sheet-adapter/build.gradle create mode 100644 google-sheet-adapter/lombok.config create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java create mode 100644 google-sheet-adapter/src/main/resources/credentials.json create mode 100644 google-sheet-adapter/src/main/resources/tokens/StoredCredential diff --git a/google-sheet-adapter/build.gradle b/google-sheet-adapter/build.gradle new file mode 100644 index 0000000000..6339677852 --- /dev/null +++ b/google-sheet-adapter/build.gradle @@ -0,0 +1,81 @@ +plugins { + id 'java' +} + +version '0.7.1-SNAPSHOT' + +repositories { + mavenCentral() +} + +dependencies { +// testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' +// testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' +} + +test { + useJUnitPlatform() +} + +group "org.polypheny" + + +dependencies { + implementation project(":core") + implementation project(":sql-language") + + implementation 'com.google.api-client:google-api-client:1.33.0' + implementation 'com.google.oauth-client:google-oauth-client-jetty:1.32.1' + implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' + +// implementation group: "net.sf.opencsv", name: "opencsv", version: opencsv_version // Apache 2.0 +// implementation group: "commons-io", name: "commons-io", version: commons_io_version // Apache 2.0 + + + // --- Test Compile --- +// testImplementation project(path: ":core", configuration: "tests") + +// testImplementation group: "junit", name: "junit", version: junit_version +// testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version // BSD 3-clause +} + + + +sourceSets { + main { + java { + srcDirs = ["src/main/java"] + outputDir = file(project.buildDir.absolutePath + "/classes") + } + resources { + srcDirs = ["src/main/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/classes") + } + test { + java { + srcDirs = ["src/test/java"] + outputDir = file(project.buildDir.absolutePath + "/test-classes") + } + resources { + srcDirs = ["src/test/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") + } +} + + +/** + * JARs + */ +jar { + manifest { + attributes "Manifest-Version": "1.0" + attributes "Copyright": "The Polypheny Project (polypheny.org)" + attributes "Version": "$project.version" + } +} +java { + withJavadocJar() + withSourcesJar() +} diff --git a/google-sheet-adapter/lombok.config b/google-sheet-adapter/lombok.config new file mode 100644 index 0000000000..b034fcb60e --- /dev/null +++ b/google-sheet-adapter/lombok.config @@ -0,0 +1,4 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.val.flagUsage = error +lombok.var.flagUsage = error diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java new file mode 100644 index 0000000000..a1cb764cf4 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -0,0 +1,326 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +import org.apache.calcite.avatica.util.DateTimeUtils; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.commons.lang3.time.FastDateFormat; + +import java.math.BigInteger; +import java.net.URL; +import java.text.ParseException; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * Questions: + * - What does array row converter do, single column row converter too... + * - Identity list?? + * - How does row converter work? And all the other abstract classes where I literally could not understand what it does? + */ + +/** + * VARIABLES: + * timezone stuff - copy paste + * (optional, seen in CSV): extra data of read-in time. + * source, + * reader (blockReader in ETH, or just import) + * (optional) filter values + * cancelFlag, + * rowConverter, + * E current - the current row type + */ + +/** + * FUNCTIONS + * - MAIN: setting up variables, standard. + * + * - converter: depends on SingleColumnRowConverter and ArrayRowConverter (same everywhere), converter is the same too. + * + * - (optional) Identity list - copy paste for now + * + * - public E current() - returns the current row type. + * + * - moveNext: moves onto the next row. If there is something, convert row. If not, return False. + * all wrapped in try catch. Add filter if necessary. + * + * - Row converter: copy it + * + * - reset, close: the same thing + * + */ + +/** + * @param Row Type + */ + +public class GoogleSheetEnumerator implements Enumerator { + + private static final FastDateFormat TIME_FORMAT_DATE; + private static final FastDateFormat TIME_FORMAT_TIME; + private static final FastDateFormat TIME_FORMAT_TIMESTAMP; + + + static { + final TimeZone gmt = TimeZone.getTimeZone( "GMT" ); + TIME_FORMAT_DATE = FastDateFormat.getInstance( "yyyy-MM-dd", gmt ); + TIME_FORMAT_TIME = FastDateFormat.getInstance( "HH:mm:ss", gmt ); + TIME_FORMAT_TIMESTAMP = FastDateFormat.getInstance( "yyyy-MM-dd HH:mm:ss", gmt ); + } + + private final URL sheetsUrl; + private final GoogleSheetReader reader; + private final AtomicBoolean cancelFlag; + private final RowConverter rowConverter; + private E current; + + GoogleSheetEnumerator(URL sheetsUrl, AtomicBoolean cancelFlag, boolean stream, RowConverter rowConverter) { + this.sheetsUrl = sheetsUrl; + this.cancelFlag = cancelFlag; + this.rowConverter = rowConverter; + this.reader = new GoogleSheetReader(sheetsUrl); + reader.readNext(); + } + + static RowConverter converter(List fieldTypes, int[] fields ) { + if ( fields.length == 1 ) { + final int field = fields[0]; + return new SingleColumnRowConverter( fieldTypes.get( field ), field ); + } else { + return new ArrayRowConverter( fieldTypes, fields ); + } + } + + static int[] identityList( int n ) { + int[] integers = new int[n]; + for ( int i = 0; i < n; i++ ) { + integers[i] = i; + } + return integers; + } + + @Override + public E current() { + return current; + } + + // TODO: implement MOVENEXT based on reader + @Override + public boolean moveNext() { + outer: + for ( ; ; ) { + if ( cancelFlag.get() ) { + return false; + } + final String[] strings = reader.readNext(); + if ( strings == null ) { + return false; + } + + current = rowConverter.convertRow( strings ); + return true; + } + + } + + @Override + public void reset() { + throw new UnsupportedOperationException(); + } + + + @Override + public void close() { +// try { +// reader.close(); +// } catch ( IOException e ) { +// throw new RuntimeException( "Error closing Blockchain reader", e ); +// } + } + + + /** + * Row converter. + * + * @param element type + */ + abstract static class RowConverter { + + abstract E convertRow( String[] rows ); + + + protected Object convert( GoogleSheetFieldType fieldType, String string ) { + if ( fieldType == null ) { + return string; + } + switch ( fieldType ) { + case BOOLEAN: + if ( string.length() == 0 ) { + return null; + } + return Boolean.parseBoolean( string ); + case BYTE: + if ( string.length() == 0 ) { + return null; + } + return Byte.parseByte( string ); + case SHORT: + if ( string.length() == 0 ) { + return null; + } + return Short.parseShort( string ); + case INT: + if ( string.length() == 0 ) { + return null; + } + return Integer.parseInt( string ); + case LONG: + if ( string.length() == 0 ) { + return null; + } + + return new BigInteger( string ); + case FLOAT: + if ( string.length() == 0 ) { + return null; + } + return Float.parseFloat( string ); + case DOUBLE: + if ( string.length() == 0 ) { + return null; + } + return Double.parseDouble( string ); + case DATE: + if ( string.length() == 0 ) { + return null; + } + try { + Date date = TIME_FORMAT_DATE.parse( string ); + return (int) (date.getTime() / DateTimeUtils.MILLIS_PER_DAY); + } catch ( ParseException e ) { + return null; + } + case TIME: + if ( string.length() == 0 ) { + return null; + } + try { + Date date = TIME_FORMAT_TIME.parse( string ); + return (int) date.getTime(); + } catch ( ParseException e ) { + return null; + } + case TIMESTAMP: + if ( string.length() == 0 ) { + return null; + } + try { + Date date = new Date( Long.parseLong( string ) * 1000 ); + return date.getTime(); + } catch ( Exception e ) { + return null; + } + case STRING: + default: + return string; + } + } + } + + /** + * Array row converter. + */ + static class ArrayRowConverter extends RowConverter { + + private final GoogleSheetFieldType[] fieldTypes; + private final int[] fields; + // whether the row to convert is from a stream + private final boolean stream; + + + ArrayRowConverter( List fieldTypes, int[] fields ) { + this.fieldTypes = fieldTypes.toArray( new GoogleSheetFieldType[0] ); + this.fields = fields; + this.stream = false; + } + + + ArrayRowConverter( List fieldTypes, int[] fields, boolean stream ) { + this.fieldTypes = fieldTypes.toArray( new GoogleSheetFieldType[0] ); + this.fields = fields; + this.stream = stream; + } + + + @Override + public Object[] convertRow( String[] strings ) { + if ( stream ) { + return convertStreamRow( strings ); + } else { + return convertNormalRow( strings ); + } + } + + + public Object[] convertNormalRow( String[] strings ) { + final Object[] objects = new Object[fields.length]; + for ( int i = 0; i < fields.length; i++ ) { + int field = fields[i]; + objects[i] = convert( fieldTypes[i], strings[field] ); + } + return objects; + } + + + public Object[] convertStreamRow( String[] strings ) { + final Object[] objects = new Object[fields.length]; + objects[0] = System.currentTimeMillis(); + for ( int i = 0; i < fields.length; i++ ) { + int field = fields[i]; + objects[i] = convert( fieldTypes[i], strings[field] ); + } + return objects; + } + + } + + /** + * Single column row converter. + */ + private static class SingleColumnRowConverter extends RowConverter { + + private final GoogleSheetFieldType fieldType; + private final int fieldIndex; + + + private SingleColumnRowConverter( GoogleSheetFieldType fieldType, int fieldIndex ) { + this.fieldType = fieldType; + this.fieldIndex = fieldIndex; + } + + + @Override + public Object convertRow( String[] strings ) { + return convert( fieldType, strings[fieldIndex] ); + } + + } + +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java new file mode 100644 index 0000000000..b0f129d805 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java @@ -0,0 +1,108 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +import org.apache.calcite.linq4j.tree.Primitive; +import org.polypheny.db.adapter.java.JavaTypeFactory; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.type.PolyType; + +import java.util.HashMap; +import java.util.Map; + +/** + * Both of the files are the same, so not much to say here. Copy-paste + */ + +public enum GoogleSheetFieldType { + STRING( String.class, "string" ), + BOOLEAN( Primitive.BOOLEAN ), + BYTE( Primitive.BYTE ), + CHAR( Primitive.CHAR ), + SHORT( Primitive.SHORT ), + INT( Primitive.INT ), + LONG( Primitive.LONG ), + FLOAT( Primitive.FLOAT ), + DOUBLE( Primitive.DOUBLE ), + DATE( java.sql.Date.class, "date" ), + TIME( java.sql.Time.class, "time" ), + TIMESTAMP( java.sql.Timestamp.class, "timestamp" ); + + private static final Map MAP = new HashMap<>(); + + + static { + for ( GoogleSheetFieldType value : values() ) { + MAP.put( value.simpleName, value ); + } + } + + + private final Class clazz; + private final String simpleName; + + + GoogleSheetFieldType( Primitive primitive ) { + this( primitive.boxClass, primitive.primitiveClass.getSimpleName() ); + } + + + GoogleSheetFieldType( Class clazz, String simpleName ) { + this.clazz = clazz; + this.simpleName = simpleName; + } + + + public static GoogleSheetFieldType getGoogleSheetFieldType( PolyType type ) { + switch ( type ) { + case BOOLEAN: + return GoogleSheetFieldType.BOOLEAN; + case VARBINARY: + return GoogleSheetFieldType.BYTE; + case INTEGER: + return GoogleSheetFieldType.INT; + case BIGINT: + return GoogleSheetFieldType.LONG; + case REAL: + return GoogleSheetFieldType.FLOAT; + case DOUBLE: + return GoogleSheetFieldType.DOUBLE; + case VARCHAR: + return GoogleSheetFieldType.STRING; + case DATE: + return GoogleSheetFieldType.DATE; + case TIME: + return GoogleSheetFieldType.TIME; + case TIMESTAMP: + return GoogleSheetFieldType.TIMESTAMP; + default: + throw new RuntimeException( "Unsupported datatype: " + type.name() ); + } + } + + + public static GoogleSheetFieldType of( String typeString ) { + return MAP.get( typeString ); + } + + + public AlgDataType toType(JavaTypeFactory typeFactory ) { + AlgDataType javaType = typeFactory.createJavaType( clazz ); + AlgDataType sqlType = typeFactory.createPolyType( javaType.getPolyType() ); + return typeFactory.createTypeWithNullability( sqlType, true ); + } +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java new file mode 100644 index 0000000000..868f37ad79 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -0,0 +1,136 @@ +package org.polypheny.db.adapter.googlesheet; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; +import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; +import com.google.api.services.sheets.v4.model.Sheet; +import com.google.api.services.sheets.v4.model.Spreadsheet; +import com.google.api.services.sheets.v4.model.ValueRange; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + + +/** + * SOURCE: sheets quick start by google. + */ + +/** + * How to optimize hen we've already read into the table? + */ + +public class GoogleSheetReader { + private final String APPLICATION_NAME = "POLYPHENY GOOGLE SHEET READER"; + private final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + private final String TOKENS_DIRECTORY_PATH = "google-sheet-adapter/src/main/resources/tokens"; + private final List SCOPES = Collections.singletonList(SheetsScopes.SPREADSHEETS_READONLY); + private final String CREDENTIALS_FILE_PATH = "/credentials.json"; + + + + private HashMap>> tableData = new HashMap<>(); + private String firstTableName; + private List> firstTableData; + private int currBlock; // ptr to the current row to read from in the first table. + + public GoogleSheetReader(URL url) { + readAllTables(); + setFirstTable(); + currBlock = 0; + } + + + private Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException { + // Load client secrets. + InputStream in = GoogleSheetReader.class.getResourceAsStream(CREDENTIALS_FILE_PATH); + if (in == null) { + throw new FileNotFoundException("Resource not found: " + CREDENTIALS_FILE_PATH); + } + GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in)); + + // Build flow and trigger user authorization request. + GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( + HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES) + .setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH))) + .setAccessType("offline") + .build(); + LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8080).build(); + return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); + } + + + private void readAllTables() { + if (!tableData.isEmpty()){ + return; + } + try { + final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; + + Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) + .setApplicationName(APPLICATION_NAME) + .build(); + + // get the properties of all the sheets + Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); + + for (Sheet sheet: s.getSheets()) { + String sheetName = sheet.getProperties().getTitle(); + ValueRange response = service.spreadsheets().values() + .get(spreadsheetId, sheetName) + .execute(); + + List> values = response.getValues(); + tableData.put(sheetName, values); + } + } catch (IOException | GeneralSecurityException e ) { + throw new RuntimeException(e); + } + } + + private void setFirstTable() { + ArrayList tableNames = new ArrayList(tableData.keySet()); + Collections.sort(tableNames); + firstTableName = tableNames.get(0); + firstTableData = tableData.get(firstTableName); + } + + + public String[] readNext() { + if (currBlock >= firstTableData.size()) { + return null; + } + + List results = firstTableData.get(currBlock); + List resultsStr = new ArrayList<>(); + currBlock += 1; + + for (Object val: results) { + resultsStr.add(val.toString()); + } + + return resultsStr.toArray(new String[0]); + } + + + public HashMap>> getTableData() { + return tableData; + } +} \ No newline at end of file diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java new file mode 100644 index 0000000000..75ceac7e52 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -0,0 +1,153 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + + +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; +import org.polypheny.db.algebra.type.AlgDataTypeImpl; +import org.polypheny.db.algebra.type.AlgDataTypeSystem; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.entity.CatalogColumn; +import org.polypheny.db.catalog.entity.CatalogColumnPlacement; +import org.polypheny.db.catalog.entity.CatalogTable; +import org.polypheny.db.schema.Table; +import org.polypheny.db.schema.impl.AbstractSchema; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.PolyTypeFactoryImpl; +import org.polypheny.db.util.Util; + +import java.net.URL; +import java.util.*; + +/** + * Questions: How do all of these things work? + */ + +/** + * VARIABLES: sheets URL, tableMap + * + * - Main: set the sheetsURL + * + * - createSheetsTable: you're using CatalogTable, List of Catalog Column placements, and DataSource to return Table + * No idea what all the functions are doing - but not needed. Copy-paste, build Table, put into map, done. + * Refer to GoogleSheetsTable + * + * - getTableMap: map version of table + * + * - private sqlType, parseTypeString: same thing, don't understand, copy-paste. + * + * + */ + +public class GoogleSheetSchema extends AbstractSchema { + private final URL sheetsURL; + private Map tableMap = new HashMap<>(); + + + public GoogleSheetSchema( URL sheetsURL ) { + this.sheetsURL = sheetsURL; + } + + public Table createGoogleSheetTable(CatalogTable catalogTable, List columnPlacementsOnStore, GoogleSheetSource googleSheetSource) { + final AlgDataTypeFactory typeFactory = new PolyTypeFactoryImpl( AlgDataTypeSystem.DEFAULT ); + final AlgDataTypeFactory.Builder fieldInfo = typeFactory.builder(); + List fieldTypes = new LinkedList<>(); + List fieldIds = new ArrayList<>( columnPlacementsOnStore.size() ); + + for ( CatalogColumnPlacement placement : columnPlacementsOnStore ) { + CatalogColumn catalogColumn = Catalog.getInstance().getColumn( placement.columnId ); + AlgDataType sqlType = sqlType( typeFactory, catalogColumn.type, catalogColumn.length, catalogColumn.scale, null ); + fieldInfo.add( catalogColumn.name, placement.physicalColumnName, sqlType ).nullable( catalogColumn.nullable ); + fieldTypes.add( GoogleSheetFieldType.getGoogleSheetFieldType( catalogColumn.type ) ); + fieldIds.add( (int) placement.physicalPosition ); + } + + int[] fields = fieldIds.stream().mapToInt( i -> i ).toArray(); + // build table and return later based on what you need for the table + GoogleSheetTable table = new GoogleSheetTable(sheetsURL, AlgDataTypeImpl.proto( fieldInfo.build() ), fields, googleSheetSource, fieldTypes); + tableMap.put( catalogTable.name, table ); + return table; + } + + @Override + public Map getTableMap() { + return new HashMap<>( tableMap ); + } + + private AlgDataType sqlType(AlgDataTypeFactory typeFactory, PolyType dataTypeName, Integer length, Integer scale, String typeString ) { + // Fall back to ANY if type is unknown + final PolyType polyType = Util.first( dataTypeName, PolyType.ANY ); + switch ( polyType ) { + case ARRAY: + AlgDataType component = null; + if ( typeString != null && typeString.endsWith( " ARRAY" ) ) { + // E.g. hsqldb gives "INTEGER ARRAY", so we deduce the component type "INTEGER". + final String remaining = typeString.substring( 0, typeString.length() - " ARRAY".length() ); + component = parseTypeString( typeFactory, remaining ); + } + if ( component == null ) { + component = typeFactory.createTypeWithNullability( typeFactory.createPolyType( PolyType.ANY ), true ); + } + return typeFactory.createArrayType( component, -1 ); + } + if ( scale != null && length != null && length >= 0 && scale >= 0 && polyType.allowsPrecScale( true, true ) ) { + return typeFactory.createPolyType( polyType, length, scale ); + } else if ( length != null && length >= 0 && polyType.allowsPrecNoScale() ) { + return typeFactory.createPolyType( polyType, length ); + } else { + assert polyType.allowsNoPrecNoScale(); + return typeFactory.createPolyType( polyType ); + } + } + + + /** + * Given "INTEGER", returns BasicSqlType(INTEGER). + * Given "VARCHAR(10)", returns BasicSqlType(VARCHAR, 10). + * Given "NUMERIC(10, 2)", returns BasicSqlType(NUMERIC, 10, 2). + */ + private AlgDataType parseTypeString( AlgDataTypeFactory typeFactory, String typeString ) { + int precision = -1; + int scale = -1; + int open = typeString.indexOf( "(" ); + if ( open >= 0 ) { + int close = typeString.indexOf( ")", open ); + if ( close >= 0 ) { + String rest = typeString.substring( open + 1, close ); + typeString = typeString.substring( 0, open ); + int comma = rest.indexOf( "," ); + if ( comma >= 0 ) { + precision = Integer.parseInt( rest.substring( 0, comma ) ); + scale = Integer.parseInt( rest.substring( comma ) ); + } else { + precision = Integer.parseInt( rest ); + } + } + } + try { + final PolyType typeName = PolyType.valueOf( typeString ); + return typeName.allowsPrecScale( true, true ) + ? typeFactory.createPolyType( typeName, precision, scale ) + : typeName.allowsPrecScale( true, false ) + ? typeFactory.createPolyType( typeName, precision ) + : typeFactory.createPolyType( typeName ); + } catch ( IllegalArgumentException e ) { + return typeFactory.createTypeWithNullability( typeFactory.createPolyType( PolyType.ANY ), true ); + } + } +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java new file mode 100644 index 0000000000..2bc1d02626 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -0,0 +1,240 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.adapter.Adapter; +import org.polypheny.db.adapter.DataSource; +import org.polypheny.db.adapter.DeployMode; +import org.polypheny.db.catalog.entity.CatalogColumnPlacement; +import org.polypheny.db.catalog.entity.CatalogPartitionPlacement; +import org.polypheny.db.catalog.entity.CatalogTable; +import org.polypheny.db.information.InformationGroup; +import org.polypheny.db.information.InformationTable; +import org.polypheny.db.prepare.Context; +import org.polypheny.db.schema.Schema; +import org.polypheny.db.schema.SchemaPlus; +import org.polypheny.db.schema.Table; +import org.polypheny.db.transaction.PolyXid; +import org.polypheny.db.type.PolyType; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; + + +/** + * A LIST of QUESTIONS: + * - Should this be Embedded or Remote deploy? + * - What is length - scale - dimension - cardinality in the ExportedColumn? - Physical name vs name in Exported Column + * - Should I implement truncate? prepare? If so, what is Context? CatalogTable? PolyXID? + */ + + +/** + * Functions that need to be implemented (based on other data sources) + * - Main: sets the URL, and other settings if needed, create information page, enable information page. + * + * - set URL: (ETH) + * + * - create information page: + * + * + * - getExportedColumns: + * this is where you return a map of {table_name: existing columns}. + * existing columns: each existing column is an ExportedColumn data type, with: column name, PolyType, collectionsType (can be null), length - scale - dimension - cardinality can all be null, schema_name, table_name, + * physical_name (name again), position (index), when to be primary key (== 0 is good enough). + * + * - EnableInformationPage: call from Adapter package, do nothing. + * + * - shutdown(): call from adapter package to shutdown. + * + * - reloadSettings: just update current settings (sheetsURL, maybe extra modes in the future). + * + * - createNewSchema: refer to GoogleSheetSchema file + * + * - createTableSchema: calls createTable in GoogleSheetSchema + * + * - getCurrentSchema: return currentSchema + * + * - truncate: (removes all rows from table) throw exception. As + * + */ + +@Slf4j +@Adapter.AdapterProperties( + name = "Google Sheets", + description = "An adapter for querying online Google Sheets, using the Google Sheets Java API. Currently, this adapter only supports read operations.", + usedModes = DeployMode.EMBEDDED) +@Adapter.AdapterSettingString(name = "SheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", position = 1) +@Adapter.AdapterSettingInteger(name = "maxStringLength", defaultValue = 255, position = 2, + description = "Which length (number of characters including whitespace) should be used for the varchar columns. Make sure this is equal or larger than the longest string in any of the columns.") +public class GoogleSheetSource extends DataSource { + private URL sheetsUrl; + private GoogleSheetSchema currentSchema; + private final int maxStringLength; + private Map> exportedColumnCache; + + public GoogleSheetSource(final int storeId, final String uniqueName, final Map settings){ + super(storeId, uniqueName, settings, true); + + // Validate maxStringLength setting + maxStringLength = Integer.parseInt( settings.get( "maxStringLength" ) ); + if ( maxStringLength < 1 ) { + throw new RuntimeException( "Invalid value for maxStringLength: " + maxStringLength ); + } + setSheetsUrl(settings); + + createInformationPage(); + enableInformationPage(); + } + + private void setSheetsUrl (Map settings) { + try { + sheetsUrl = new URL(settings.get("SheetsURL")); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + + /** + * manipulates getExportedColumns to add information to 2 (black box) objects: InformationGroup, InformationTable. + * * informationGroup is about the tables + * * InformationTable is about the columns. + * * Currently using CSV's version (more information), could use ETH for simplicity + */ + protected void createInformationPage() { + for ( Map.Entry> entry : getExportedColumns().entrySet() ) { + InformationGroup group = new InformationGroup( + informationPage, + entry.getValue().get( 0 ).physicalSchemaName + "." + entry.getValue().get( 0 ).physicalTableName ); + + InformationTable table = new InformationTable( + group, + // Arrays.asList( "Position", "Column Name", "Type", "Primary" ) ); + Arrays.asList( "Position", "Column Name", "Type", "Nullable", "Filename", "Primary" ) ); + for ( ExportedColumn exportedColumn : entry.getValue() ) { + table.addRow( + exportedColumn.physicalPosition, + exportedColumn.name, + exportedColumn.getDisplayType(), + exportedColumn.nullable ? "✔" : "", + exportedColumn.physicalSchemaName, + exportedColumn.primary ? "✔" : "" + ); + } + informationElements.add( table ); + informationGroups.add( group ); + } + } + + /** + * Do once you have reader + * @return a map of {table_name: existing columns}. + * * existing columns: each existing column is an ExportedColumn data type, with: column name, PolyType, collectionsType (can be null), length - scale - dimension - cardinality can all be null, schema_name, table_name, + * * physical_name (name again), position (index), when to be primary key (== 0 is good enough). + */ + @Override + public Map> getExportedColumns(){ + if (exportedColumnCache != null) { + return exportedColumnCache; + } + + Map> exportedColumnCache = new HashMap<>(); + GoogleSheetReader reader = new GoogleSheetReader(sheetsUrl); + HashMap>> tablesData = reader.getTableData(); + + for (Map.Entry>> entry: tablesData.entrySet()) { + String tableName = entry.getKey(); + List tableColumns = entry.getValue().get(0); + List exportedCols = new ArrayList<>(); + int position = 0; + for (Object col: tableColumns) { + exportedCols.add(new ExportedColumn( + col.toString(), + PolyType.VARCHAR, // defaulting to VARCHAR currently + null, + null, + null, + null, + null, + false, + "public", + tableName, + col.toString(), + position, + position == 0 + )); + position++; + } + + exportedColumnCache.put(tableName, exportedCols); + } + + return exportedColumnCache; + } + + @Override + public void shutdown() { + removeInformationPage(); + } + + @Override + protected void reloadSettings(List updatedSettings) { + if (updatedSettings.contains("SheetsURL")) { + setSheetsUrl(settings); + } + } + + @Override + public void createNewSchema(SchemaPlus rootSchema, String name) { + currentSchema = new GoogleSheetSchema(this.sheetsUrl); + } + + @Override + public Table createTableSchema(CatalogTable combinedTable, List columnPlacementsOnStore, CatalogPartitionPlacement partitionPlacement) { + return currentSchema.createGoogleSheetTable( combinedTable, columnPlacementsOnStore, this ); + } + + @Override + public Schema getCurrentSchema() { + return currentSchema; + } + + @Override + public void truncate(Context context, CatalogTable table ) { + throw new RuntimeException( "CSV adapter does not support truncate" ); + } + + @Override + public boolean prepare( PolyXid xid ) { + log.debug( "CSV Store does not support prepare()." ); + return true; + } + + @Override + public void commit( PolyXid xid ) { + log.debug( "CSV Store does not support commit()." ); + } + + + @Override + public void rollback( PolyXid xid ) { + log.debug( "CSV Store does not support rollback()." ); + } +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java new file mode 100644 index 0000000000..a45bc83638 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +/** + * Questions: + * - What is protoRowType? + */ + +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.polypheny.db.adapter.DataContext; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; +import org.polypheny.db.algebra.type.AlgDataTypeField; +import org.polypheny.db.algebra.type.AlgProtoDataType; +import org.polypheny.db.rex.RexNode; +import org.polypheny.db.schema.FilterableTable; +import org.polypheny.db.schema.impl.AbstractTable; +import org.polypheny.db.util.Pair; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Variables: sheets url, protoRowType, []int fields, List of SheetFieldTypes, SheetDataSource, (?) Mapper + * + * - String: return "Google Sheets table" + * + * - getRowType: there are generally two different structures that we can see here. + * (ETH): you create a list of types, a list of names, stitch them together, call typeFactory to handle it. Does not handle case where protoRowType is none. + * (CSV): do the above in one line (not sure they're the same), and if protoRowType is null, we call deduceRowType from Enum... what does it to, well: + * it goes into the file, looks at the names and associated, puts them into names[] and types[] list, calls typeFactory.createStructType on them. + * + * - scan: optional, but not really - implement for Sheets. + * (ETH): registerInvolvedAdapter for Context, then create a bunch of options to create the ETH Enumerator. + * For Google Sheets, you need: cancelFlag, RowConverter (which is the CSV.RowConverter) - copy this + * + * + */ + +public class GoogleSheetTable extends AbstractTable implements FilterableTable { + protected final URL sheetsUrl; + protected final AlgProtoDataType protoRowType; + protected final int[] fields; + protected final GoogleSheetSource googleSheetSource; + protected List fieldTypes; + + public GoogleSheetTable ( + URL sheetsUrl, + AlgProtoDataType protoRowType, + int[] fields, + GoogleSheetSource googleSheetSource, + List fieldTypes ) { + this.sheetsUrl = sheetsUrl; + this.protoRowType = protoRowType; + this.fields = fields; + this.googleSheetSource = googleSheetSource; + this.fieldTypes = fieldTypes; + } + + public String toString() { + return "GoogleSheetTable"; + } + + /** + * Could change to more detailed implementation such as with CSV later? + */ + @Override + public AlgDataType getRowType(AlgDataTypeFactory typeFactory ) { + final List types = new ArrayList<>(); + final List names = new ArrayList<>(); + for ( AlgDataTypeField field : this.protoRowType.apply( typeFactory ).getFieldList() ) { + types.add( field.getType() ); + names.add( field.getName() ); + } + return typeFactory.createStructType( Pair.zip( names, types ) ); + } + + /** + * Not really filtering, but can't tell if it's necessary or not for any scan. Assuming it is. + */ + @Override + public Enumerable scan(DataContext dataContext, List filters ) { + dataContext.getStatement().getTransaction().registerInvolvedAdapter( googleSheetSource ); + final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get( dataContext ); + + // TODO: check this. + return new AbstractEnumerable() { + @Override + public Enumerator enumerator() { + return new GoogleSheetEnumerator<>(sheetsUrl, cancelFlag, false, new GoogleSheetEnumerator.ArrayRowConverter( fieldTypes, fields )); + } + }; + } + + + +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java new file mode 100644 index 0000000000..2b9c3eef77 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java @@ -0,0 +1,116 @@ +package org.polypheny.db.adapter.googlesheet; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; +import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; +import com.google.api.services.sheets.v4.model.Sheet; +import com.google.api.services.sheets.v4.model.Spreadsheet; +import com.google.api.services.sheets.v4.model.ValueRange; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.security.GeneralSecurityException; +import java.util.Collections; +import java.util.List; + +public class SheetsQuickstart { + private static final String APPLICATION_NAME = "Google Sheets API Java Quickstart"; + private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + private static final String TOKENS_DIRECTORY_PATH = "google-sheet-adapter/src/main/resources/tokens"; + + /** + * Global instance of the scopes required by this quickstart. + * If modifying these scopes, delete your previously saved tokens/ folder. + */ + private static final List SCOPES = Collections.singletonList(SheetsScopes.SPREADSHEETS_READONLY); + private static final String CREDENTIALS_FILE_PATH = "/credentials.json"; + + /** + * Creates an authorized Credential object. + * @param HTTP_TRANSPORT The network HTTP Transport. + * @return An authorized Credential object. + * @throws IOException If the credentials.json file cannot be found. + */ + private static Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException { + // Load client secrets. + InputStream in = SheetsQuickstart.class.getResourceAsStream(CREDENTIALS_FILE_PATH); + if (in == null) { + throw new FileNotFoundException("Resource not found: " + CREDENTIALS_FILE_PATH); + } + GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in)); + + // Build flow and trigger user authorization request. + GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( + HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES) + .setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH))) + .setAccessType("offline") + .build(); + LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8080).build(); + return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); + } + + /** + * Prints the names and majors of students in a sample spreadsheet: + * https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit + */ + public static void main(String... args) throws IOException, GeneralSecurityException { + // Build a new authorized API client service. +// final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); +// final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; +// final String range = "Class Data"; +// Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) +// .setApplicationName(APPLICATION_NAME) +// .build(); +// Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); +// for (Sheet sheet: s.getSheets()) { +// System.out.println(sheet.getProperties().getTitle()); +// } +// ValueRange response = service.spreadsheets().values() +// .get(spreadsheetId, range) +// .execute(); +// List> values = response.getValues(); +// if (values == null || values.isEmpty()) { +// System.out.println("No data found."); +// } else { +// System.out.println("Name, Major"); +// for (List row : values) { +// // Print columns A and E, which correspond to indices 0 and 4. +// System.out.println(values.get(0).get(0).getClass()); +// System.out.printf("%s, %s\n", row.get(0), row.get(4)); +// } +// } + final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; + + Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) + .setApplicationName(APPLICATION_NAME) + .build(); + + // get the properties of all the sheets + Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); + + for (Sheet sheet: s.getSheets()) { + String sheetName = sheet.getProperties().getTitle(); + System.out.println(sheetName); + ValueRange response = service.spreadsheets().values() + .get(spreadsheetId, sheetName) + .execute(); + + List> values = response.getValues(); + for (List row : values) { + System.out.printf("%s\n", row.get(0)); + } + } + } +} \ No newline at end of file diff --git a/google-sheet-adapter/src/main/resources/credentials.json b/google-sheet-adapter/src/main/resources/credentials.json new file mode 100644 index 0000000000..e0d0f1c099 --- /dev/null +++ b/google-sheet-adapter/src/main/resources/credentials.json @@ -0,0 +1,11 @@ +{ + "web": { + "client_id": "849988809941-kul5bmk9pks6qu1t57cdlfdum2jala3c.apps.googleusercontent.com", + "project_id": "precise-space-354614", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "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", + "redirect_uris": ["https://polypheny.org/"] + } +} diff --git a/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/google-sheet-adapter/src/main/resources/tokens/StoredCredential new file mode 100644 index 0000000000000000000000000000000000000000..256cea441951bb53127f8e95f8bceb87bda7c542 GIT binary patch literal 1132 zcma)5zi-<{6h0|-o7O?$bVz^zDNr;+hlrvio3W4~p=47(OfeBD$`U&`C-V5iq&9#CGY|FmiNBzd+)n< z2QR^LLcr-gxDUCMdI8sliQ9wG@`r!@_WPL|PfmcM1U?Qhv?OR!Ogi8r>JpT=IIyzl z%2kk`Q?rvSuCiO8U@1)yNeL)^)i`){_P-BK4xSYO$RYqvygoRTtD1PqIT$+u;$Y-) zX5gWaaxkSXhx1tFG^T~DCc7--!627mr%(h-GFXA8iIQZ1eU=2D%6a$Y0ESLkqr?lH z%Q85JvdANldN>?-Q`GZ zEHdg-1lhR*3;7s)2MSq4!QXROt#cwT4#oPsvX6(ermpm3qo&wmMQhE3UXON5!N$ml z=4x2kj`T6oX7Tl#p}RXbnseP(U0(31>>Af4s4H{1v8IdbgqPyUMy3mK(^*e2x#pE% z#jNdi<$YB&c~gpgMQZmuyW5SLV7Nl-l8H1s6DNa~MX}JGU>v)|?9#1uM;Yr%xTDj? za7!BWhiXIT4OJbCD@MDsrfsS=tkSTzgQ->7BdKA{_=-?td-BO*PjaWZ zg>Sc>-~RKbFP;`bp$nGxVUVJXfYqZSC7n(X`QhO1`IS4*ZoOH9!uPAnw@(<`mq%N7 zacd%<*Mu$0V0lY;X5Ug{n61&skNXAx6eJFemd1&22D zNDf6mR{zz${&gj9pMcV_DRS(i&+grMxM_TIh+TgVR-7bkUX%7ylp-s~x&(*bO+?uA z&Gd@;51YQjek+AEcUd?F*R@#)Cjok!toHPm$3IW;gEwyh00nlXT;}T=9iuyt{dR*| zO=sK@RZ(n-zNR=*Nv?OurXWtl%-^nrp^|#;aF6i(?$&f0mZYX!H3n_T-rDJwrs7`W XkC1E|{ZW0iF&oN+Pn>bwj-Bj(Ist_N literal 0 HcmV?d00001 diff --git a/settings.gradle b/settings.gradle index 643a62d9f4..e86ba4affb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -43,6 +43,7 @@ include 'plugins:postgres-adapter' include 'plugins:monetdb-adapter' include 'plugins:mongodb-adapter' include 'plugins:file-adapter' +include 'plugins:google-sheet-adapter' include 'plugins:excel-adapter' // other plugins From 5f667a1b56a00b6cf37c814d4b8488ebe2747d93 Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Fri, 22 Jul 2022 19:04:11 -0400 Subject: [PATCH 03/19] works with two sheets --- .../googlesheet/GoogleSheetEnumerator.java | 12 ++++---- .../googlesheet/GoogleSheetFieldType.java | 3 +- .../googlesheet/GoogleSheetReader.java | 28 ++++++++++-------- .../googlesheet/GoogleSheetSchema.java | 11 +++++-- .../adapter/googlesheet/GoogleSheetTable.java | 7 +++-- .../adapter/googlesheet/SheetsQuickstart.java | 3 ++ .../main/resources/tokens/StoredCredential | Bin 1132 -> 1133 bytes 7 files changed, 41 insertions(+), 23 deletions(-) diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index a1cb764cf4..3d19be3cf2 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -54,14 +54,14 @@ * * - converter: depends on SingleColumnRowConverter and ArrayRowConverter (same everywhere), converter is the same too. * - * - (optional) Identity list - copy paste for now + * - (optional) Identity list * * - public E current() - returns the current row type. * * - moveNext: moves onto the next row. If there is something, convert row. If not, return False. * all wrapped in try catch. Add filter if necessary. * - * - Row converter: copy it + * - Row converter: self-explanatory * * - reset, close: the same thing * @@ -86,16 +86,18 @@ public class GoogleSheetEnumerator implements Enumerator { } private final URL sheetsUrl; + private final String tableName; private final GoogleSheetReader reader; private final AtomicBoolean cancelFlag; private final RowConverter rowConverter; private E current; - GoogleSheetEnumerator(URL sheetsUrl, AtomicBoolean cancelFlag, boolean stream, RowConverter rowConverter) { + GoogleSheetEnumerator(URL sheetsUrl, String tableName, AtomicBoolean cancelFlag, boolean stream, RowConverter rowConverter) { this.sheetsUrl = sheetsUrl; + this.tableName = tableName; this.cancelFlag = cancelFlag; this.rowConverter = rowConverter; - this.reader = new GoogleSheetReader(sheetsUrl); + this.reader = new GoogleSheetReader(sheetsUrl, tableName); reader.readNext(); } @@ -121,7 +123,7 @@ public E current() { return current; } - // TODO: implement MOVENEXT based on reader + @Override public boolean moveNext() { outer: diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java index b0f129d805..19d234a029 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java @@ -25,7 +25,8 @@ import java.util.Map; /** - * Both of the files are the same, so not much to say here. Copy-paste + * Both of the files are the same, so not much to say here. Types for Google Sheet (not yet used since we're + * currently only dealing with strings) */ public enum GoogleSheetFieldType { diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index 868f37ad79..b243730977 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -33,7 +33,7 @@ */ /** - * How to optimize hen we've already read into the table? + * How to optimize when we've already read into the table? */ public class GoogleSheetReader { @@ -46,14 +46,20 @@ public class GoogleSheetReader { private HashMap>> tableData = new HashMap<>(); - private String firstTableName; - private List> firstTableData; + private String targetTableName; + private List> targetTableData; private int currBlock; // ptr to the current row to read from in the first table. public GoogleSheetReader(URL url) { readAllTables(); - setFirstTable(); + } + + public GoogleSheetReader(URL url, String tableName) { + readAllTables(); + setTargetTable(tableName); currBlock = 0; + System.out.println(targetTableName); + System.out.println(targetTableData.size()); } @@ -71,7 +77,7 @@ private Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws .setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH))) .setAccessType("offline") .build(); - LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8080).build(); + LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8888).build(); return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); } @@ -105,20 +111,18 @@ private void readAllTables() { } } - private void setFirstTable() { - ArrayList tableNames = new ArrayList(tableData.keySet()); - Collections.sort(tableNames); - firstTableName = tableNames.get(0); - firstTableData = tableData.get(firstTableName); + private void setTargetTable(String tableName) { + targetTableName = tableName; + targetTableData = tableData.get(tableName); } public String[] readNext() { - if (currBlock >= firstTableData.size()) { + if (currBlock >= targetTableData.size()) { return null; } - List results = firstTableData.get(currBlock); + List results = targetTableData.get(currBlock); List resultsStr = new ArrayList<>(); currBlock += 1; diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java index 75ceac7e52..ce962415dc 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -44,12 +44,12 @@ * - Main: set the sheetsURL * * - createSheetsTable: you're using CatalogTable, List of Catalog Column placements, and DataSource to return Table - * No idea what all the functions are doing - but not needed. Copy-paste, build Table, put into map, done. + * No idea what all the functions are doing - but not needed. CP: build Table, put into map. * Refer to GoogleSheetsTable * * - getTableMap: map version of table * - * - private sqlType, parseTypeString: same thing, don't understand, copy-paste. + * - private sqlType, parseTypeString: pretty similarly defined in the other directories. * * */ @@ -77,9 +77,14 @@ public Table createGoogleSheetTable(CatalogTable catalogTable, List i ).toArray(); // build table and return later based on what you need for the table - GoogleSheetTable table = new GoogleSheetTable(sheetsURL, AlgDataTypeImpl.proto( fieldInfo.build() ), fields, googleSheetSource, fieldTypes); + GoogleSheetTable table = new GoogleSheetTable(sheetsURL, tableName, AlgDataTypeImpl.proto( fieldInfo.build() ), fields, googleSheetSource, fieldTypes); tableMap.put( catalogTable.name, table ); return table; } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index a45bc83638..c008a118eb 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -51,13 +51,14 @@ * * - scan: optional, but not really - implement for Sheets. * (ETH): registerInvolvedAdapter for Context, then create a bunch of options to create the ETH Enumerator. - * For Google Sheets, you need: cancelFlag, RowConverter (which is the CSV.RowConverter) - copy this + * For Google Sheets, you need: cancelFlag, RowConverter (which is the CSV.RowConverter) - CP * * */ public class GoogleSheetTable extends AbstractTable implements FilterableTable { protected final URL sheetsUrl; + protected final String tableName; protected final AlgProtoDataType protoRowType; protected final int[] fields; protected final GoogleSheetSource googleSheetSource; @@ -65,11 +66,13 @@ public class GoogleSheetTable extends AbstractTable implements FilterableTable { public GoogleSheetTable ( URL sheetsUrl, + String tableName, AlgProtoDataType protoRowType, int[] fields, GoogleSheetSource googleSheetSource, List fieldTypes ) { this.sheetsUrl = sheetsUrl; + this.tableName = tableName; this.protoRowType = protoRowType; this.fields = fields; this.googleSheetSource = googleSheetSource; @@ -106,7 +109,7 @@ public Enumerable scan(DataContext dataContext, List filters ) { return new AbstractEnumerable() { @Override public Enumerator enumerator() { - return new GoogleSheetEnumerator<>(sheetsUrl, cancelFlag, false, new GoogleSheetEnumerator.ArrayRowConverter( fieldTypes, fields )); + return new GoogleSheetEnumerator<>(sheetsUrl, tableName, cancelFlag, false, new GoogleSheetEnumerator.ArrayRowConverter( fieldTypes, fields )); } }; } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java index 2b9c3eef77..50a680f8a4 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java @@ -24,6 +24,9 @@ import java.util.Collections; import java.util.List; +/** + * Base file for Java API calling - currently not used. + */ public class SheetsQuickstart { private static final String APPLICATION_NAME = "Google Sheets API Java Quickstart"; private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); diff --git a/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/google-sheet-adapter/src/main/resources/tokens/StoredCredential index 256cea441951bb53127f8e95f8bceb87bda7c542..93706115f1e69fe5cf7299b6c84f24badff04a26 100644 GIT binary patch delta 326 zcmWN_%~FCu007|R;Db0q;SWM!ikdCCmL$aX9#)Ne(De@`?Q=3{LD*!io;+TDkxbC3C;eq@nt?p?Zv+}`oyqYoA5eLx;K6E{3c;qRw5lAE zyyk*r5`_i@YH*d#_HyQza57~B-N?rF+!D-{asF;y-{>UamLo*eF_>&P+9%vRF-FnM zwyncLKX(|leVH<^#r~pm{$V=rZ_Gn$8}_^19snjQC(Uw;eN7G>4CfFkkQ!=**eSq! zTmx=l9V%p%5kj8jBi_ut#VL0p6e(_c7vB!HUg=pEwfQ>wYytWv3a-ob~~$QbQltv6;(=8sw8 zwpeVf)JSM9t@H%Xp=Y617p^TZDriW2zOgtW#-lPXZnu=j1gxv-M*{?O07;6VyiiTL7~FPyQ%*S22JWxOUbiJN~< Cb8esj From 9e1d86c1ad4bb6042dae167aeb505934e26d1fe5 Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Fri, 22 Jul 2022 19:30:13 -0400 Subject: [PATCH 04/19] works I guess --- .../googlesheet/GoogleSheetReader.java | 12 +++++++++++- .../main/resources/tokens/StoredCredential | Bin 1133 -> 1135 bytes 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index b243730977..b11cf398d5 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -45,16 +45,20 @@ public class GoogleSheetReader { + private final URL url; private HashMap>> tableData = new HashMap<>(); private String targetTableName; private List> targetTableData; private int currBlock; // ptr to the current row to read from in the first table. public GoogleSheetReader(URL url) { + this.url = url; readAllTables(); + } public GoogleSheetReader(URL url, String tableName) { + this.url = url; readAllTables(); setTargetTable(tableName); currBlock = 0; @@ -81,6 +85,12 @@ private Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); } + private String parseUrlToString(URL url) { + String content = url.getPath(); + String[] contentArr = content.split("/"); + return contentArr[3]; + } + private void readAllTables() { if (!tableData.isEmpty()){ @@ -88,7 +98,7 @@ private void readAllTables() { } try { final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; + final String spreadsheetId = parseUrlToString(url); Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) .setApplicationName(APPLICATION_NAME) diff --git a/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/google-sheet-adapter/src/main/resources/tokens/StoredCredential index 93706115f1e69fe5cf7299b6c84f24badff04a26..5f687a3aa2672f85d0d58e033070a9f4eacd9739 100644 GIT binary patch delta 240 zcmV~$IZ}gA002;-;t*CgQiuaG#83iZF(l!$2I(L~zHDI&Tazp3P2vUJ*lUjD9F85Rw(`rKQmM4{K%%UQB;;Mv8Y(#EY3(KEVRtLk zPPdmMZqb-D#=qz zrdDU3S!^$>7BVq#377?$awju2L4q`#^K-c!J2p_3x#g^-X!=IZFDzPbLRz{vB?+wU mO|2DM`M2vIRwf!*>Ny%XhB;bR8Wu;Ihq+g#WIKDB=Vav+Ta-prdOAgf zW|vlmI960xxH~6iS^5MQo0~*91!fgg0YyDaiwjc|y_4gM(#s;fJdM*T@^#(9!wW4U z!wuZbQY^BQTr4XClJlIi3JXlKoeSMPO%kJW&C*he%LB^GoKzzUDvZ){yfe~+vn~D7 za(#=O^HR$ky?u1k0)z8h!-K-iLK1@~e_)hR3XI4xigIy_@N>2FPBF?hj4%qy33kcw q4G#}12@NzbG&7#8$K=H!l*_cVxQIbxav`(y Date: Mon, 5 Sep 2022 19:29:43 -0400 Subject: [PATCH 05/19] Google Sheet Adapter - complete --- .../main/resources/tokens/StoredCredential | Bin 0 -> 1119 bytes .../db/adapter/GoogleSheetSourceTest.java | 138 ++++++++ .../polypheny/db/adapter/MysqlSourceTest.java | 4 +- google-sheet-adapter/build.gradle | 33 +- .../googlesheet/GoogleSheetEnumerator.java | 88 ++---- .../googlesheet/GoogleSheetFieldType.java | 10 +- .../GoogleSheetProjectTableScanRule.java | 73 +++++ .../googlesheet/GoogleSheetReader.java | 299 ++++++++++++++---- .../googlesheet/GoogleSheetSchema.java | 52 +-- .../googlesheet/GoogleSheetSource.java | 149 ++++----- .../adapter/googlesheet/GoogleSheetTable.java | 86 +++-- .../GoogleSheetTableScanProject.java | 105 ++++++ .../db/adapter/googlesheet/README.md | 39 +++ .../adapter/googlesheet/SheetsQuickstart.java | 119 ------- .../src/main/resources/credentials.json | 12 +- .../main/resources/tokens/StoredCredential | Bin 1135 -> 0 bytes .../java/org/polypheny/db/test/CsvTest.java | 8 +- 17 files changed, 785 insertions(+), 430 deletions(-) create mode 100644 dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential create mode 100644 dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java create mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md delete mode 100644 google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java delete mode 100644 google-sheet-adapter/src/main/resources/tokens/StoredCredential diff --git a/dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential new file mode 100644 index 0000000000000000000000000000000000000000..555ada4e91dcbaf5fd1ace5553825b3a77f5f3c4 GIT binary patch literal 1119 zcma)5zi-n(6uzV_zgnTf01^Tu1Or3kByN%>VgM&?(zH$>v6H4@C>Q%;JF$Jv`J6fd zu^=I&4luw!zyzp>6()pO7+8>)SXv1rgb-q2VdR`vDlDic-CN%KzVF`m-fjK>6D|SA zI0=mEp)eR1!6Z}^c>s2bSXHulL?%NvSP&}gh@n$q#G4kWYkjxwF3u6 z@;z`HgnWmBj|0eOI2aS^LS_*6>a}!P?JhK1`Fh)3SWEWgJRfg03u#AA#=2B!54`?d zZ@H$$(#hFEV_lSrzO^C`hkUl_FXdI*5Dhh=wX8v{Cg&n%RgN3IL{qNWVySa&(X=Dl zqP3chxs{IMq=&s;isw^nVlyHNp3@~hDQ6nO)_(Ck6O0v~BI$DjuTyA95#+L{0@1NfagU}L~ z=tI*(J^|BvMP<)wA#!)~(V5AI+Z(%+58atQJ^78XUEEu=J$t#4y<8PBs|mDe;B&k7 zWM<}F=R4C8!MHoH6$+yN8_qxI3lald@*Sp_*stP+GPc{$AORHJe=9{;6u*v^mg_g_uRcOUS9#R9WPOxnZ5n02;Qd!GK#7vP? z%hE!L7NC1gT(cFG6r5%3uJ}ux`dq)(O}Cd;t0_w){f3^1B-PR^8CI3`<-ve=6EP{C IlJtuI2mP{u4FCWD literal 0 HcmV?d00001 diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java new file mode 100644 index 0000000000..ec8cc68d48 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter; + +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; + + +public class GoogleSheetSourceTest { + + @BeforeClass + public static void start() throws SQLException { + TestHelper.getInstance(); + + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + Map settings = new HashMap<>(); + settings.put( "maxStringLength", "255" ); + settings.put( "querySize", "1000" ); + settings.put( "sheetsURL", "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0" ); + settings.put( "mode", "remote" ); + settings.put( "resetRefreshToken", "No" ); + Gson gson = new Gson(); + statement.executeUpdate( "ALTER ADAPTERS ADD googlesheetunit USING 'org.polypheny.db.adapter.googlesheet.GoogleSheetSource' WITH '" + gson.toJson( settings ) + "'" ); + } + } + } + + + @AfterClass + public static void end() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "ALTER ADAPTERS DROP googlesheetunit" ); + } + } + + } + + + @Test + public void testTest() { + assertTrue( true ); + } + + + @Test + public void existsSelect() throws SQLException { + try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expectedResult = ImmutableList.of( + new Object[]{ "Oracle", "Product Manager", "127000" }, + new Object[]{ "eBay", "Software Engineer", "100000" }, + new Object[]{ "Amazon", "Product Manager", "310000" }, + new Object[]{ "Apple", "Software Engineering Manager", "372000" }, + new Object[]{ "Microsoft", "Software Engineer", "157000" } + + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT \"Company\", \"Title\", \"Yearly Compensation\" from salaries LIMIT 5" ), + expectedResult + ); + } + } + } + + + @Test + public void existsSecondSelect() throws SQLException { + try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expectedResult = ImmutableList.of( + new Object[]{ "GP", "F", "18", "U", "GT3", "A", "4", "4", "at_home", "teacher", "course", "mother", "2", "2", "0", "yes", "no", "no", "no", "yes", "yes", "no", "no", "4", "3", "4", "3", "6", "5", "6", "6" }, + new Object[]{ "GP", "F", "17", "U", "GT3", "T", "1", "1", "at_home", "other", "course", "father", "1", "2", "0", "no", "yes", "no", "no", "no", "yes", "yes", "no", "5", "3", "3", "3", "4", "5", "5", "6" }, + new Object[]{ "GP", "F", "15", "U", "LE3", "T", "1", "1", "at_home", "other", "other", "mother", "1", "2", "3", "yes", "no", "yes", "no", "yes", "yes", "yes", "no", "4", "3", "2", "3", "10", "7", "8", "10" } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT * from student_data LIMIT 3" ), + expectedResult + ); + } + } + } + + + @Test + public void existsThirdSelect() throws SQLException { + try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expectedResult = ImmutableList.of( + new Object[]{ "Alexandra", "Female" }, + new Object[]{ "Andrew", "Male" }, + new Object[]{ "Anna", "Female" } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT \"Student Name\", \"Gender\" from student_formatting" ), + expectedResult + ); + } + } + } + +} + + + diff --git a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java index b348501ba5..1df3ec6066 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java @@ -27,12 +27,11 @@ import java.util.Map; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.polypheny.db.TestHelper; import org.polypheny.db.TestHelper.JdbcConnection; @SuppressWarnings("SqlDialectInspection") -@Ignore +//@Ignore public class MysqlSourceTest extends AbstractSourceTest { private static EmbeddedMysql server; @@ -66,6 +65,7 @@ public static void start() throws SQLException { statement.executeUpdate( "ALTER ADAPTERS ADD mariadbunit USING 'Mysql' AS 'Source' WITH '" + gson.toJson( settings ) + "'" ); } } + System.out.println("RAN HERE"); } diff --git a/google-sheet-adapter/build.gradle b/google-sheet-adapter/build.gradle index 6339677852..787fb607ca 100644 --- a/google-sheet-adapter/build.gradle +++ b/google-sheet-adapter/build.gradle @@ -1,22 +1,3 @@ -plugins { - id 'java' -} - -version '0.7.1-SNAPSHOT' - -repositories { - mavenCentral() -} - -dependencies { -// testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' -// testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' -} - -test { - useJUnitPlatform() -} - group "org.polypheny" @@ -28,19 +9,15 @@ dependencies { implementation 'com.google.oauth-client:google-oauth-client-jetty:1.32.1' implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' -// implementation group: "net.sf.opencsv", name: "opencsv", version: opencsv_version // Apache 2.0 -// implementation group: "commons-io", name: "commons-io", version: commons_io_version // Apache 2.0 +// --- Test Compile --- + testImplementation project(path: ":core", configuration: "tests") - - // --- Test Compile --- -// testImplementation project(path: ":core", configuration: "tests") - -// testImplementation group: "junit", name: "junit", version: junit_version -// testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version // BSD 3-clause + testImplementation group: "junit", name: "junit", version: junit_version + testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version // BSD 3-clause + testImplementation group: "com.konghq", name: "unirest-java", version: unirest_version // MIT } - sourceSets { main { java { diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index 3d19be3cf2..a1d719948e 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -16,10 +16,6 @@ package org.polypheny.db.adapter.googlesheet; -import org.apache.calcite.avatica.util.DateTimeUtils; -import org.apache.calcite.linq4j.Enumerator; -import org.apache.commons.lang3.time.FastDateFormat; - import java.math.BigInteger; import java.net.URL; import java.text.ParseException; @@ -27,48 +23,15 @@ import java.util.List; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.calcite.avatica.util.DateTimeUtils; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.commons.lang3.time.FastDateFormat; /** - * Questions: - * - What does array row converter do, single column row converter too... - * - Identity list?? - * - How does row converter work? And all the other abstract classes where I literally could not understand what it does? - */ - -/** - * VARIABLES: - * timezone stuff - copy paste - * (optional, seen in CSV): extra data of read-in time. - * source, - * reader (blockReader in ETH, or just import) - * (optional) filter values - * cancelFlag, - * rowConverter, - * E current - the current row type - */ - -/** - * FUNCTIONS - * - MAIN: setting up variables, standard. - * - * - converter: depends on SingleColumnRowConverter and ArrayRowConverter (same everywhere), converter is the same too. - * - * - (optional) Identity list - * - * - public E current() - returns the current row type. - * - * - moveNext: moves onto the next row. If there is something, convert row. If not, return False. - * all wrapped in try catch. Add filter if necessary. - * - * - Row converter: self-explanatory - * - * - reset, close: the same thing + * Enumerator that reads from the blockchain * - */ - -/** - * @param Row Type + * @param Row type */ public class GoogleSheetEnumerator implements Enumerator { @@ -85,23 +48,29 @@ public class GoogleSheetEnumerator implements Enumerator { TIME_FORMAT_TIMESTAMP = FastDateFormat.getInstance( "yyyy-MM-dd HH:mm:ss", gmt ); } - private final URL sheetsUrl; + private final String tableName; private final GoogleSheetReader reader; private final AtomicBoolean cancelFlag; private final RowConverter rowConverter; private E current; - GoogleSheetEnumerator(URL sheetsUrl, String tableName, AtomicBoolean cancelFlag, boolean stream, RowConverter rowConverter) { - this.sheetsUrl = sheetsUrl; + + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, RowConverter rowConverter ) { this.tableName = tableName; this.cancelFlag = cancelFlag; this.rowConverter = rowConverter; - this.reader = new GoogleSheetReader(sheetsUrl, tableName); - reader.readNext(); + this.reader = new GoogleSheetReader( sheetsUrl, querySize ); + } - static RowConverter converter(List fieldTypes, int[] fields ) { + + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, List fieldTypes, int[] fields ) { + this( sheetsUrl, querySize, tableName, cancelFlag, (RowConverter) converter( fieldTypes, fields ) ); + } + + + static RowConverter converter( List fieldTypes, int[] fields ) { if ( fields.length == 1 ) { final int field = fields[0]; return new SingleColumnRowConverter( fieldTypes.get( field ), field ); @@ -110,13 +79,6 @@ static RowConverter converter(List fieldTypes, int[] fi } } - static int[] identityList( int n ) { - int[] integers = new int[n]; - for ( int i = 0; i < n; i++ ) { - integers[i] = i; - } - return integers; - } @Override public E current() { @@ -126,12 +88,11 @@ public E current() { @Override public boolean moveNext() { - outer: for ( ; ; ) { if ( cancelFlag.get() ) { return false; } - final String[] strings = reader.readNext(); + final String[] strings = reader.readNext( tableName ); if ( strings == null ) { return false; } @@ -139,22 +100,20 @@ public boolean moveNext() { current = rowConverter.convertRow( strings ); return true; } - } + @Override public void reset() { throw new UnsupportedOperationException(); } + /** + * Doesn't need to do any thing to close + */ @Override public void close() { -// try { -// reader.close(); -// } catch ( IOException e ) { -// throw new RuntimeException( "Error closing Blockchain reader", e ); -// } } @@ -244,8 +203,10 @@ protected Object convert( GoogleSheetFieldType fieldType, String string ) { return string; } } + } + /** * Array row converter. */ @@ -303,6 +264,7 @@ public Object[] convertStreamRow( String[] strings ) { } + /** * Single column row converter. */ diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java index 19d234a029..ea927f165c 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java @@ -16,19 +16,17 @@ package org.polypheny.db.adapter.googlesheet; +import java.util.HashMap; +import java.util.Map; import org.apache.calcite.linq4j.tree.Primitive; import org.polypheny.db.adapter.java.JavaTypeFactory; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.type.PolyType; -import java.util.HashMap; -import java.util.Map; /** - * Both of the files are the same, so not much to say here. Types for Google Sheet (not yet used since we're - * currently only dealing with strings) + * Type of field in a Google Sheet. A field is always of type String. */ - public enum GoogleSheetFieldType { STRING( String.class, "string" ), BOOLEAN( Primitive.BOOLEAN ), @@ -101,7 +99,7 @@ public static GoogleSheetFieldType of( String typeString ) { } - public AlgDataType toType(JavaTypeFactory typeFactory ) { + public AlgDataType toType( JavaTypeFactory typeFactory ) { AlgDataType javaType = typeFactory.createJavaType( clazz ); AlgDataType sqlType = typeFactory.createPolyType( javaType.getPolyType() ); return typeFactory.createTypeWithNullability( sqlType, true ); diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java new file mode 100644 index 0000000000..d4bfe289c6 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +import java.util.List; +import org.polypheny.db.algebra.core.AlgFactories; +import org.polypheny.db.algebra.logical.LogicalProject; +import org.polypheny.db.plan.AlgOptRule; +import org.polypheny.db.plan.AlgOptRuleCall; +import org.polypheny.db.rex.RexInputRef; +import org.polypheny.db.rex.RexNode; +import org.polypheny.db.tools.AlgBuilderFactory; + + +/** + * Planner rule that projects from a {@GoogleTable} scan just the columns needed to satisfy a projection. If the projection's expressions are trivial, the projection is removed. + */ +public class GoogleSheetProjectTableScanRule extends AlgOptRule { + + public static final GoogleSheetProjectTableScanRule INSTANCE = new GoogleSheetProjectTableScanRule( AlgFactories.LOGICAL_BUILDER ); + + + public GoogleSheetProjectTableScanRule( AlgBuilderFactory algBuilderFactory ) { + super( + operand( LogicalProject.class, operand( GoogleSheetTableScanProject.class, none() ) ), + algBuilderFactory, + "GoogleSheetProjectTableScanRule" + ); + } + + + @Override + public void onMatch( AlgOptRuleCall call ) { + final LogicalProject project = call.alg( 0 ); + final GoogleSheetTableScanProject scan = call.alg( 1 ); + int[] fields = getProjectFields( project.getProjects() ); + if ( fields == null ) { + // Project contains expressions more complex than just field references. + return; + } + call.transformTo( + new GoogleSheetTableScanProject( scan.getCluster(), scan.getTable(), scan.googleSheetTable, fields ) ); + } + + + private int[] getProjectFields( List exps ) { + final int[] fields = new int[exps.size()]; + for ( int i = 0; i < exps.size(); i++ ) { + final RexNode exp = exps.get( i ); + if ( exp instanceof RexInputRef ) { + fields[i] = ((RexInputRef) exp).getIndex(); + } else { + return null; // not a simple projection + } + } + return fields; + } + +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index b11cf398d5..b0c836a138 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -12,10 +12,11 @@ import com.google.api.client.util.store.FileDataStoreFactory; import com.google.api.services.sheets.v4.Sheets; import com.google.api.services.sheets.v4.SheetsScopes; +import com.google.api.services.sheets.v4.model.GridProperties; import com.google.api.services.sheets.v4.model.Sheet; import com.google.api.services.sheets.v4.model.Spreadsheet; import com.google.api.services.sheets.v4.model.ValueRange; - +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -26,125 +27,303 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Objects; /** - * SOURCE: sheets quick start by google. - */ - -/** - * How to optimize when we've already read into the table? + * Class that scans the Google Sheet directly using the GoogleSheetApi. */ - public class GoogleSheetReader { + private final String APPLICATION_NAME = "POLYPHENY GOOGLE SHEET READER"; private final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); private final String TOKENS_DIRECTORY_PATH = "google-sheet-adapter/src/main/resources/tokens"; - private final List SCOPES = Collections.singletonList(SheetsScopes.SPREADSHEETS_READONLY); + private final List SCOPES = Collections.singletonList( SheetsScopes.SPREADSHEETS_READONLY ); private final String CREDENTIALS_FILE_PATH = "/credentials.json"; - private final URL url; + private final int querySize; + private HashMap> tableSurfaceData = new HashMap<>(); private HashMap>> tableData = new HashMap<>(); - private String targetTableName; - private List> targetTableData; - private int currBlock; // ptr to the current row to read from in the first table. + private HashMap tableStart = new HashMap<>(); + private HashMap tableLeftOffset = new HashMap<>(); + private HashMap enumPointer = new HashMap<>(); + - public GoogleSheetReader(URL url) { + /** + * @param url - url of the Google Sheet to source. + * @param querySize - size of the query (in case of large files) + */ + public GoogleSheetReader( URL url, int querySize ) { this.url = url; - readAllTables(); + this.querySize = querySize; + } + + + /** + * Finds first row for field names + */ + private void readTableSurfaceData() { + if ( !tableSurfaceData.isEmpty() ) { + return; + } + try { + final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + final String spreadsheetId = parseUrlToString( url ); + Sheets service = new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( HTTP_TRANSPORT ) ) + .setApplicationName( APPLICATION_NAME ) + .build(); + + // get the properties of all the sheets + Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); + + for ( Sheet sheet : s.getSheets() ) { + String sheetName = sheet.getProperties().getTitle(); + GridProperties gp = sheet.getProperties().getGridProperties(); + int queryStartRow = 1; + while ( true ) { + if ( queryStartRow > gp.getRowCount() ) { // nothing in the sheet + break; + } + int queryEndRow = queryStartRow + querySize - 1; + String sheetRange = sheetName + "!" + queryStartRow + ":" + queryEndRow; + ValueRange response = service.spreadsheets().values() + .get( spreadsheetId, sheetRange ) + .execute(); + + List> values = response.getValues(); + if ( values == null ) { // no rows had values + queryStartRow += querySize; + continue; + } + for ( List row : values ) { // found at least 1 row + if ( row.size() != 0 ) { + for ( int i = 0; i < row.size(); i++ ) { + if ( !Objects.equals( row.get( i ).toString(), "" ) ) { + row = row.subList( i, row.size() ); + break; + } + } + tableSurfaceData.put( sheetName, row ); + break; + } + } + break; + } + } + } catch ( IOException | GeneralSecurityException e ) { + throw new RuntimeException( e ); + } } - public GoogleSheetReader(URL url, String tableName) { - this.url = url; - readAllTables(); - setTargetTable(tableName); - currBlock = 0; - System.out.println(targetTableName); - System.out.println(targetTableData.size()); + + public HashMap> getTableSurfaceData() { + if ( tableSurfaceData.isEmpty() ) { + readTableSurfaceData(); + } + return tableSurfaceData; } - private Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException { + private Credential getCredentials( final NetHttpTransport HTTP_TRANSPORT ) throws IOException { // Load client secrets. - InputStream in = GoogleSheetReader.class.getResourceAsStream(CREDENTIALS_FILE_PATH); - if (in == null) { - throw new FileNotFoundException("Resource not found: " + CREDENTIALS_FILE_PATH); + InputStream in = GoogleSheetReader.class.getResourceAsStream( CREDENTIALS_FILE_PATH ); + if ( in == null ) { + throw new FileNotFoundException( "Resource not found: " + CREDENTIALS_FILE_PATH ); } - GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in)); + GoogleClientSecrets clientSecrets = GoogleClientSecrets.load( JSON_FACTORY, new InputStreamReader( in ) ); // Build flow and trigger user authorization request. GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( - HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES) - .setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH))) - .setAccessType("offline") + HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES ) + .setDataStoreFactory( new FileDataStoreFactory( new java.io.File( TOKENS_DIRECTORY_PATH ) ) ) + .setAccessType( "offline" ) .build(); - LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8888).build(); - return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); + LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort( 8888 ).build(); + return new AuthorizationCodeInstalledApp( flow, receiver ).authorize( "user" ); + } + + + public void deleteToken() { + File file = new File( TOKENS_DIRECTORY_PATH + "/StoredCredential" ); + if ( file.exists() ) { + file.delete(); + } + } - private String parseUrlToString(URL url) { + + private String parseUrlToString( URL url ) { String content = url.getPath(); - String[] contentArr = content.split("/"); + String[] contentArr = content.split( "/" ); return contentArr[3]; } - private void readAllTables() { - if (!tableData.isEmpty()){ - return; - } + /** + * Reads sheet in Google Sheet URL block by block + * + * @param tableName - name of the sheet to read in the URL. + */ + private void readTable( String tableName ) { try { final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - final String spreadsheetId = parseUrlToString(url); + final String spreadsheetId = parseUrlToString( url ); - Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) - .setApplicationName(APPLICATION_NAME) + Sheets service = new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( HTTP_TRANSPORT ) ) + .setApplicationName( APPLICATION_NAME ) .build(); - // get the properties of all the sheets - Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); + // first query! let's start searching till we find first row. + if ( !tableStart.containsKey( tableName ) ) { + Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); // get all the sheets + Sheet chosen_sheet = new Sheet(); + for ( Sheet sheet : s.getSheets() ) { + if ( Objects.equals( sheet.getProperties().getTitle(), tableName ) ) { + chosen_sheet = sheet; + break; + } + } - for (Sheet sheet: s.getSheets()) { - String sheetName = sheet.getProperties().getTitle(); + int queryStartRow = 1; + while ( true ) { + // nothing in sheet + if ( queryStartRow > chosen_sheet.getProperties().getGridProperties().getRowCount() ) { + tableStart.put( tableName, -1 ); + return; + } + + int queryEndRow = queryStartRow + querySize - 1; + + String sheetRange = tableName + "!" + queryStartRow + ":" + queryEndRow; + ValueRange response = service.spreadsheets().values() + .get( spreadsheetId, sheetRange ) + .execute(); + List> values = response.getValues(); + + if ( values == null ) { + queryStartRow += querySize; + continue; + } + + int firstRowIndex = -1; + int indexCounter = 0; + for ( List row : values ) { // found at least 1 row, set the left shift + if ( row.size() != 0 ) { + for ( int i = 0; i < row.size(); i++ ) { + if ( !Objects.equals( row.get( i ).toString(), "" ) ) { + tableLeftOffset.put( tableName, i ); + break; + } + } + firstRowIndex = indexCounter + 1; + break; + } + indexCounter++; + } + + // TODO: optimize this + // add the remaining data to tableData + for ( int i = firstRowIndex; i < values.size(); i++ ) { + List fullRow = values.get( i ); + List trueRow = fullRow.subList( tableLeftOffset.get( tableName ), fullRow.size() ); + values.set( i, trueRow ); + } + tableData.put( tableName, values.subList( firstRowIndex, values.size() ) ); + + if ( values.size() < querySize ) { // we already queried everything + tableStart.put( tableName, -1 ); + } else { // we'll start at that value later? + tableStart.put( tableName, firstRowIndex + 1 ); + } + + break; + + } + } else if ( tableStart.get( tableName ) == -1 ) { // end of document already + return; + } else { // begin getting from + + Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); // get all the sheets + Sheet chosen_sheet = new Sheet(); + for ( Sheet sheet : s.getSheets() ) { + if ( Objects.equals( sheet.getProperties().getTitle(), tableName ) ) { + chosen_sheet = sheet; + break; + } + } + + int queryStartRow = tableStart.get( tableName ); + + int queryEndRow = queryStartRow + querySize - 1; + + if ( queryStartRow > chosen_sheet.getProperties().getGridProperties().getRowCount() ) { + tableStart.put( tableName, -1 ); + return; + } + + String sheetRange = tableName + "!" + queryStartRow + ":" + queryEndRow; ValueRange response = service.spreadsheets().values() - .get(spreadsheetId, sheetName) + .get( spreadsheetId, sheetRange ) .execute(); - List> values = response.getValues(); - tableData.put(sheetName, values); + + if ( values == null ) { // we've reached an empty part of the document, so all rows have been queried + tableStart.put( tableName, -1 ); + return; + } + + List> currData = tableData.getOrDefault( tableName, new ArrayList<>() ); + for ( List row : values ) { + List trueRow = row.subList( tableLeftOffset.get( tableName ), row.size() ); + currData.add( trueRow ); + } + tableData.put( tableName, currData ); + + // final check: if the current data that we have is smaller than our query, we reached + // the end, so we set our start to -1. + if ( values.size() < querySize ) { + tableStart.put( tableName, -1 ); + } else { + tableStart.put( tableName, queryEndRow + 1 ); + } } - } catch (IOException | GeneralSecurityException e ) { - throw new RuntimeException(e); + } catch ( IOException | GeneralSecurityException e ) { + throw new RuntimeException( e ); } } - private void setTargetTable(String tableName) { - targetTableName = tableName; - targetTableData = tableData.get(tableName); - } + public String[] readNext( String tableName ) { + if ( !enumPointer.containsKey( tableName ) ) { + enumPointer.put( tableName, 0 ); + } - public String[] readNext() { - if (currBlock >= targetTableData.size()) { + if ( !tableData.containsKey( tableName ) || tableData.get( tableName ).size() <= enumPointer.get( tableName ) ) { + readTable( tableName ); + } + + // still true, so we've reached the end of the google sheet + if ( tableData.get( tableName ).size() <= enumPointer.get( tableName ) ) { return null; } - List results = targetTableData.get(currBlock); + List results = tableData.get( tableName ).get( enumPointer.get( tableName ) ); List resultsStr = new ArrayList<>(); - currBlock += 1; + enumPointer.put( tableName, enumPointer.get( tableName ) + 1 ); - for (Object val: results) { - resultsStr.add(val.toString()); + for ( Object val : results ) { + resultsStr.add( val.toString() ); } - return resultsStr.toArray(new String[0]); + return resultsStr.toArray( new String[0] ); } public HashMap>> getTableData() { return tableData; } + } \ No newline at end of file diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java index ce962415dc..94ec02c380 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -17,6 +17,12 @@ package org.polypheny.db.adapter.googlesheet; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.algebra.type.AlgDataTypeImpl; @@ -31,39 +37,30 @@ import org.polypheny.db.type.PolyTypeFactoryImpl; import org.polypheny.db.util.Util; -import java.net.URL; -import java.util.*; /** - * Questions: How do all of these things work? + * Schema mapped onto a Google Sheet URL. Each table in the schema is a sheet in the URL. */ - -/** - * VARIABLES: sheets URL, tableMap - * - * - Main: set the sheetsURL - * - * - createSheetsTable: you're using CatalogTable, List of Catalog Column placements, and DataSource to return Table - * No idea what all the functions are doing - but not needed. CP: build Table, put into map. - * Refer to GoogleSheetsTable - * - * - getTableMap: map version of table - * - * - private sqlType, parseTypeString: pretty similarly defined in the other directories. - * - * - */ - public class GoogleSheetSchema extends AbstractSchema { + private final URL sheetsURL; + private final int querySize; private Map tableMap = new HashMap<>(); - public GoogleSheetSchema( URL sheetsURL ) { + /** + * Creates a GoogleSheet Schema + * + * @param sheetsURL - the url of the Google Sheet + * @param querySize - the size of each query while scanning + */ + public GoogleSheetSchema( URL sheetsURL, int querySize ) { this.sheetsURL = sheetsURL; + this.querySize = querySize; } - public Table createGoogleSheetTable(CatalogTable catalogTable, List columnPlacementsOnStore, GoogleSheetSource googleSheetSource) { + + public Table createGoogleSheetTable( CatalogTable catalogTable, List columnPlacementsOnStore, GoogleSheetSource googleSheetSource ) { final AlgDataTypeFactory typeFactory = new PolyTypeFactoryImpl( AlgDataTypeSystem.DEFAULT ); final AlgDataTypeFactory.Builder fieldInfo = typeFactory.builder(); List fieldTypes = new LinkedList<>(); @@ -84,17 +81,19 @@ public Table createGoogleSheetTable(CatalogTable catalogTable, List i ).toArray(); // build table and return later based on what you need for the table - GoogleSheetTable table = new GoogleSheetTable(sheetsURL, tableName, AlgDataTypeImpl.proto( fieldInfo.build() ), fields, googleSheetSource, fieldTypes); + GoogleSheetTable table = new GoogleSheetTable( sheetsURL, querySize, tableName, AlgDataTypeImpl.proto( fieldInfo.build() ), fields, googleSheetSource, fieldTypes ); tableMap.put( catalogTable.name, table ); return table; } + @Override public Map getTableMap() { return new HashMap<>( tableMap ); } - private AlgDataType sqlType(AlgDataTypeFactory typeFactory, PolyType dataTypeName, Integer length, Integer scale, String typeString ) { + + private AlgDataType sqlType( AlgDataTypeFactory typeFactory, PolyType dataTypeName, Integer length, Integer scale, String typeString ) { // Fall back to ANY if type is unknown final PolyType polyType = Util.first( dataTypeName, PolyType.ANY ); switch ( polyType ) { @@ -149,10 +148,11 @@ private AlgDataType parseTypeString( AlgDataTypeFactory typeFactory, String type return typeName.allowsPrecScale( true, true ) ? typeFactory.createPolyType( typeName, precision, scale ) : typeName.allowsPrecScale( true, false ) - ? typeFactory.createPolyType( typeName, precision ) - : typeFactory.createPolyType( typeName ); + ? typeFactory.createPolyType( typeName, precision ) + : typeFactory.createPolyType( typeName ); } catch ( IllegalArgumentException e ) { return typeFactory.createTypeWithNullability( typeFactory.createPolyType( PolyType.ANY ), true ); } } + } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index 2bc1d02626..9fbbf70048 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -16,6 +16,13 @@ package org.polypheny.db.adapter.googlesheet; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.Adapter; import org.polypheny.db.adapter.DataSource; @@ -32,92 +39,58 @@ import org.polypheny.db.transaction.PolyXid; import org.polypheny.db.type.PolyType; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; - - -/** - * A LIST of QUESTIONS: - * - Should this be Embedded or Remote deploy? - * - What is length - scale - dimension - cardinality in the ExportedColumn? - Physical name vs name in Exported Column - * - Should I implement truncate? prepare? If so, what is Context? CatalogTable? PolyXID? - */ - - -/** - * Functions that need to be implemented (based on other data sources) - * - Main: sets the URL, and other settings if needed, create information page, enable information page. - * - * - set URL: (ETH) - * - * - create information page: - * - * - * - getExportedColumns: - * this is where you return a map of {table_name: existing columns}. - * existing columns: each existing column is an ExportedColumn data type, with: column name, PolyType, collectionsType (can be null), length - scale - dimension - cardinality can all be null, schema_name, table_name, - * physical_name (name again), position (index), when to be primary key (== 0 is good enough). - * - * - EnableInformationPage: call from Adapter package, do nothing. - * - * - shutdown(): call from adapter package to shutdown. - * - * - reloadSettings: just update current settings (sheetsURL, maybe extra modes in the future). - * - * - createNewSchema: refer to GoogleSheetSchema file - * - * - createTableSchema: calls createTable in GoogleSheetSchema - * - * - getCurrentSchema: return currentSchema - * - * - truncate: (removes all rows from table) throw exception. As - * - */ @Slf4j @Adapter.AdapterProperties( - name = "Google Sheets", + name = "GoogleSheets", description = "An adapter for querying online Google Sheets, using the Google Sheets Java API. Currently, this adapter only supports read operations.", - usedModes = DeployMode.EMBEDDED) -@Adapter.AdapterSettingString(name = "SheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", position = 1) + usedModes = DeployMode.REMOTE) +@Adapter.AdapterSettingString(name = "sheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", position = 1) @Adapter.AdapterSettingInteger(name = "maxStringLength", defaultValue = 255, position = 2, description = "Which length (number of characters including whitespace) should be used for the varchar columns. Make sure this is equal or larger than the longest string in any of the columns.") +@Adapter.AdapterSettingInteger(name = "querySize", defaultValue = 1000, position = 3, + description = "How many rows should be queried per network call. Can be larger than number of rows.") +@Adapter.AdapterSettingString(name = "resetRefreshToken", defaultValue = "No", position = 4, description = "If you want to change the current email used to access the Google Sheet, input \"YES\".") public class GoogleSheetSource extends DataSource { + private URL sheetsUrl; + private int querySize; private GoogleSheetSchema currentSchema; private final int maxStringLength; private Map> exportedColumnCache; - public GoogleSheetSource(final int storeId, final String uniqueName, final Map settings){ - super(storeId, uniqueName, settings, true); - // Validate maxStringLength setting + public GoogleSheetSource( final int storeId, final String uniqueName, final Map settings ) { + super( storeId, uniqueName, settings, true ); + maxStringLength = Integer.parseInt( settings.get( "maxStringLength" ) ); if ( maxStringLength < 1 ) { throw new RuntimeException( "Invalid value for maxStringLength: " + maxStringLength ); } - setSheetsUrl(settings); + querySize = Integer.parseInt( settings.get( "querySize" ) ); + if ( querySize < 1 ) { + throw new RuntimeException( "Invalid value for querySize: " + querySize ); + } + setSheetsUrl( settings ); createInformationPage(); enableInformationPage(); + if ( settings.get( "resetRefreshToken" ).equalsIgnoreCase( "yes" ) ) { + GoogleSheetReader r = new GoogleSheetReader( sheetsUrl, querySize ); + r.deleteToken(); + } } - private void setSheetsUrl (Map settings) { + + private void setSheetsUrl( Map settings ) { try { - sheetsUrl = new URL(settings.get("SheetsURL")); - } catch (MalformedURLException e) { - throw new RuntimeException(e); + sheetsUrl = new URL( settings.get( "sheetsURL" ) ); + } catch ( MalformedURLException e ) { + throw new RuntimeException( e ); } } - /** - * manipulates getExportedColumns to add information to 2 (black box) objects: InformationGroup, InformationTable. - * * informationGroup is about the tables - * * InformationTable is about the columns. - * * Currently using CSV's version (more information), could use ETH for simplicity - */ protected void createInformationPage() { for ( Map.Entry> entry : getExportedColumns().entrySet() ) { InformationGroup group = new InformationGroup( @@ -143,29 +116,24 @@ protected void createInformationPage() { } } - /** - * Do once you have reader - * @return a map of {table_name: existing columns}. - * * existing columns: each existing column is an ExportedColumn data type, with: column name, PolyType, collectionsType (can be null), length - scale - dimension - cardinality can all be null, schema_name, table_name, - * * physical_name (name again), position (index), when to be primary key (== 0 is good enough). - */ + @Override - public Map> getExportedColumns(){ - if (exportedColumnCache != null) { + public Map> getExportedColumns() { + if ( exportedColumnCache != null ) { return exportedColumnCache; } Map> exportedColumnCache = new HashMap<>(); - GoogleSheetReader reader = new GoogleSheetReader(sheetsUrl); - HashMap>> tablesData = reader.getTableData(); + GoogleSheetReader reader = new GoogleSheetReader( sheetsUrl, querySize ); + HashMap> tablesData = reader.getTableSurfaceData(); - for (Map.Entry>> entry: tablesData.entrySet()) { + for ( Map.Entry> entry : tablesData.entrySet() ) { String tableName = entry.getKey(); - List tableColumns = entry.getValue().get(0); + List tableColumns = entry.getValue(); List exportedCols = new ArrayList<>(); int position = 0; - for (Object col: tableColumns) { - exportedCols.add(new ExportedColumn( + for ( Object col : tableColumns ) { + exportedCols.add( new ExportedColumn( col.toString(), PolyType.VARCHAR, // defaulting to VARCHAR currently null, @@ -179,62 +147,71 @@ public Map> getExportedColumns(){ col.toString(), position, position == 0 - )); + ) ); position++; } - exportedColumnCache.put(tableName, exportedCols); + exportedColumnCache.put( tableName, exportedCols ); } return exportedColumnCache; } + @Override public void shutdown() { removeInformationPage(); } + @Override - protected void reloadSettings(List updatedSettings) { - if (updatedSettings.contains("SheetsURL")) { - setSheetsUrl(settings); + protected void reloadSettings( List updatedSettings ) { + if ( updatedSettings.contains( "sheetsURL" ) ) { + setSheetsUrl( settings ); } } + @Override - public void createNewSchema(SchemaPlus rootSchema, String name) { - currentSchema = new GoogleSheetSchema(this.sheetsUrl); + public void createNewSchema( SchemaPlus rootSchema, String name ) { + currentSchema = new GoogleSheetSchema( this.sheetsUrl, this.querySize ); } + @Override - public Table createTableSchema(CatalogTable combinedTable, List columnPlacementsOnStore, CatalogPartitionPlacement partitionPlacement) { + public Table createTableSchema( CatalogTable combinedTable, List columnPlacementsOnStore, CatalogPartitionPlacement partitionPlacement ) { return currentSchema.createGoogleSheetTable( combinedTable, columnPlacementsOnStore, this ); } + @Override public Schema getCurrentSchema() { return currentSchema; } + @Override - public void truncate(Context context, CatalogTable table ) { - throw new RuntimeException( "CSV adapter does not support truncate" ); + public void truncate( Context context, CatalogTable table ) { + throw new RuntimeException( "Google Sheet adapter does not support truncate" ); } + @Override public boolean prepare( PolyXid xid ) { - log.debug( "CSV Store does not support prepare()." ); + log.debug( "Google Sheet adapter does not support prepare()." ); return true; } + @Override public void commit( PolyXid xid ) { - log.debug( "CSV Store does not support commit()." ); + log.debug( "Google Sheet adapter does not support commit()." ); } @Override public void rollback( PolyXid xid ) { - log.debug( "CSV Store does not support rollback()." ); + log.debug( "Google Sheet adapter does not support rollback()." ); } + } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index c008a118eb..d0413fe2dc 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -21,57 +21,55 @@ * - What is protoRowType? */ +import java.lang.reflect.Type; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.calcite.linq4j.AbstractEnumerable; import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.Queryable; +import org.apache.calcite.linq4j.tree.Expression; import org.polypheny.db.adapter.DataContext; +import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.algebra.type.AlgProtoDataType; -import org.polypheny.db.rex.RexNode; -import org.polypheny.db.schema.FilterableTable; +import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.schema.QueryableTable; +import org.polypheny.db.schema.SchemaPlus; +import org.polypheny.db.schema.Schemas; +import org.polypheny.db.schema.TranslatableTable; import org.polypheny.db.schema.impl.AbstractTable; import org.polypheny.db.util.Pair; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; /** - * Variables: sheets url, protoRowType, []int fields, List of SheetFieldTypes, SheetDataSource, (?) Mapper - * - * - String: return "Google Sheets table" - * - * - getRowType: there are generally two different structures that we can see here. - * (ETH): you create a list of types, a list of names, stitch them together, call typeFactory to handle it. Does not handle case where protoRowType is none. - * (CSV): do the above in one line (not sure they're the same), and if protoRowType is null, we call deduceRowType from Enum... what does it to, well: - * it goes into the file, looks at the names and associated, puts them into names[] and types[] list, calls typeFactory.createStructType on them. - * - * - scan: optional, but not really - implement for Sheets. - * (ETH): registerInvolvedAdapter for Context, then create a bunch of options to create the ETH Enumerator. - * For Google Sheets, you need: cancelFlag, RowConverter (which is the CSV.RowConverter) - CP - * - * + * Base table class based on individual Google Sheets. */ +public class GoogleSheetTable extends AbstractTable implements QueryableTable, TranslatableTable { -public class GoogleSheetTable extends AbstractTable implements FilterableTable { protected final URL sheetsUrl; + protected final int querySize; protected final String tableName; protected final AlgProtoDataType protoRowType; protected final int[] fields; protected final GoogleSheetSource googleSheetSource; protected List fieldTypes; - public GoogleSheetTable ( + + public GoogleSheetTable( URL sheetsUrl, + int querySize, String tableName, AlgProtoDataType protoRowType, int[] fields, GoogleSheetSource googleSheetSource, List fieldTypes ) { this.sheetsUrl = sheetsUrl; + this.querySize = querySize; this.tableName = tableName; this.protoRowType = protoRowType; this.fields = fields; @@ -79,15 +77,14 @@ public GoogleSheetTable ( this.fieldTypes = fieldTypes; } + public String toString() { return "GoogleSheetTable"; } - /** - * Could change to more detailed implementation such as with CSV later? - */ + @Override - public AlgDataType getRowType(AlgDataTypeFactory typeFactory ) { + public AlgDataType getRowType( AlgDataTypeFactory typeFactory ) { final List types = new ArrayList<>(); final List names = new ArrayList<>(); for ( AlgDataTypeField field : this.protoRowType.apply( typeFactory ).getFieldList() ) { @@ -97,23 +94,44 @@ public AlgDataType getRowType(AlgDataTypeFactory typeFactory ) { return typeFactory.createStructType( Pair.zip( names, types ) ); } + /** - * Not really filtering, but can't tell if it's necessary or not for any scan. Assuming it is. + * Returns an enumerator over a given projection */ - @Override - public Enumerable scan(DataContext dataContext, List filters ) { + public Enumerable project( final DataContext dataContext, final int[] fields ) { dataContext.getStatement().getTransaction().registerInvolvedAdapter( googleSheetSource ); final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get( dataContext ); - - // TODO: check this. - return new AbstractEnumerable() { + return new AbstractEnumerable() { @Override - public Enumerator enumerator() { - return new GoogleSheetEnumerator<>(sheetsUrl, tableName, cancelFlag, false, new GoogleSheetEnumerator.ArrayRowConverter( fieldTypes, fields )); + public Enumerator enumerator() { + return new GoogleSheetEnumerator<>( sheetsUrl, querySize, tableName, cancelFlag, fieldTypes, fields ); } }; } + @Override + public Queryable asQueryable( DataContext dataContext, SchemaPlus schema, String tableName ) { + throw new UnsupportedOperationException(); + } + + + @Override + public Type getElementType() { + return Object[].class; + } + + + @Override + public Expression getExpression( SchemaPlus schema, String tableName, Class clazz ) { + return Schemas.tableExpression( schema, getElementType(), tableName, clazz ); + } + + + @Override + public AlgNode toAlg( AlgOptTable.ToAlgContext context, AlgOptTable algOptTable ) { + // Request all fields. + return new GoogleSheetTableScanProject( context.getCluster(), algOptTable, this, fields ); + } } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java new file mode 100644 index 0000000000..41c9ec3b2d --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019-2022 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + +import java.util.List; +import org.apache.calcite.linq4j.tree.Blocks; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Primitive; +import org.polypheny.db.adapter.enumerable.EnumerableAlg; +import org.polypheny.db.adapter.enumerable.EnumerableAlgImplementor; +import org.polypheny.db.adapter.enumerable.EnumerableConvention; +import org.polypheny.db.adapter.enumerable.PhysType; +import org.polypheny.db.adapter.enumerable.PhysTypeImpl; +import org.polypheny.db.algebra.AlgNode; +import org.polypheny.db.algebra.AlgWriter; +import org.polypheny.db.algebra.core.TableScan; +import org.polypheny.db.algebra.metadata.AlgMetadataQuery; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; +import org.polypheny.db.algebra.type.AlgDataTypeField; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgOptCost; +import org.polypheny.db.plan.AlgOptPlanner; +import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.plan.AlgTraitSet; + + +/** + * Relational expression representing a scan of a Google Sheet. + * + * Like any table scan, it serves as a leaf node of a query tree. + */ +public class GoogleSheetTableScanProject extends TableScan implements EnumerableAlg { + + final GoogleSheetTable googleSheetTable; + final int[] fields; + + + protected GoogleSheetTableScanProject( AlgOptCluster cluster, AlgOptTable table, GoogleSheetTable googleSheetTable, int[] fields ) { + super( cluster, cluster.traitSetOf( EnumerableConvention.INSTANCE ), table ); + this.googleSheetTable = googleSheetTable; + this.fields = fields; + + assert googleSheetTable != null; + } + + + @Override + public AlgNode copy( AlgTraitSet traitSet, List inputs ) { + assert inputs.isEmpty(); + return new GoogleSheetTableScanProject( getCluster(), table, googleSheetTable, fields ); + } + + + @Override + public AlgWriter explainTerms( AlgWriter pw ) { + return super.explainTerms( pw ).item( "fields", Primitive.asList( fields ) ); + } + + + @Override + public AlgDataType deriveRowType() { + final List fieldList = table.getRowType().getFieldList(); + final AlgDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder(); + for ( int field : fields ) { + builder.add( fieldList.get( field ) ); + } + return builder.build(); + } + + + @Override + public AlgOptCost computeSelfCost( AlgOptPlanner planner, AlgMetadataQuery mq ) { + return super.computeSelfCost( planner, mq ).multiplyBy( ((double) fields.length + 2D) / ((double) table.getRowType().getFieldCount() + 2D) ); + } + + + @Override + public void register( AlgOptPlanner planner ) { + planner.addRule( GoogleSheetProjectTableScanRule.INSTANCE ); + } + + + @Override + public Result implement( EnumerableAlgImplementor implementor, Prefer pref ) { + PhysType physType = PhysTypeImpl.of( implementor.getTypeFactory(), getRowType(), pref.preferArray() ); + + return implementor.result( physType, Blocks.toBlock( Expressions.call( table.getExpression( GoogleSheetTable.class ), "project", implementor.getRootExpression(), Expressions.constant( fields ) ) ) ); + } + +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md new file mode 100644 index 0000000000..6589ed1688 --- /dev/null +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md @@ -0,0 +1,39 @@ +### For users + +The Google Sheet Adapter is a data-source adapter that allows data to be sourced from any Google Sheet URL into Polypheny. To do so, please go into the +Adapters section, choose data source, then Google Sheets. Here, you will be presented with multiple values to fill out: + +- Name: name of the schema +- sheetsURL: the Google Sheet URL to source from. +- MaxStringLength: max length of a string in all sheets in the sheetsURL +- querySize: the number of rows scanned per query on the GoogleSheet. +- resetRefreshToken: type "YES" to delete the current refresh token and use another account for the sheet adapter. + + The adapter adjusts itself to different formats of the sheet, but keep in mind there may be issues in the following cases: + +- There are two tables nested within the same sheet in the URL +- There is a gap between the table rows (the adapter won't read after the gap) + +Otherwise, as long as the email used has access to the sheet, there should be no issues. + + +### For future development + +**Token management** + +- The application currently runs using the Google Cloud Console under a private email's project. Please create a new project +under the Polypheny email, enable the Google Sheet API, and add an OAUTH credential under the "Credentials" section. The credential +should be configured with origins as needed, with a http://localhost:8888/Callback and http://localhost:8888 redirect URL. + (http://localhost:8888 is used because this is the port in which the GoogleSheetReader opens up to receive the access tokens +from OAUTH. This can be changed as needed.) Then, download the credentials and add it to `google-sheet-adapter/src/main/resources/credentials.json`. + +- Finally, there is the unhandled issue of expiring refresh tokens roughly after 1 week (which can cause errors to the users). +Users can fix this manually with the resetRefreshToken option in the settings. However, to improve the experience and +unlimit the number of API calls to Google Sheets servers, it is necessary to go to OAuth Consent Screen, and "Publish App" +which then extends refresh tokens for infinite use along with removes the API call limit. + +**Improvements which can be made** + +- Splitting GoogleSheetTableScanProject -> Scan and Project classes separately. + +- Moving the GoogleSheetSourceTest from `dbms/src/test/java/org.polypheny.db/adapter/` to google-sheet-adapter package. \ No newline at end of file diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java deleted file mode 100644 index 50a680f8a4..0000000000 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/SheetsQuickstart.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.polypheny.db.adapter.googlesheet; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.store.FileDataStoreFactory; -import com.google.api.services.sheets.v4.Sheets; -import com.google.api.services.sheets.v4.SheetsScopes; -import com.google.api.services.sheets.v4.model.Sheet; -import com.google.api.services.sheets.v4.model.Spreadsheet; -import com.google.api.services.sheets.v4.model.ValueRange; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.security.GeneralSecurityException; -import java.util.Collections; -import java.util.List; - -/** - * Base file for Java API calling - currently not used. - */ -public class SheetsQuickstart { - private static final String APPLICATION_NAME = "Google Sheets API Java Quickstart"; - private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - private static final String TOKENS_DIRECTORY_PATH = "google-sheet-adapter/src/main/resources/tokens"; - - /** - * Global instance of the scopes required by this quickstart. - * If modifying these scopes, delete your previously saved tokens/ folder. - */ - private static final List SCOPES = Collections.singletonList(SheetsScopes.SPREADSHEETS_READONLY); - private static final String CREDENTIALS_FILE_PATH = "/credentials.json"; - - /** - * Creates an authorized Credential object. - * @param HTTP_TRANSPORT The network HTTP Transport. - * @return An authorized Credential object. - * @throws IOException If the credentials.json file cannot be found. - */ - private static Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException { - // Load client secrets. - InputStream in = SheetsQuickstart.class.getResourceAsStream(CREDENTIALS_FILE_PATH); - if (in == null) { - throw new FileNotFoundException("Resource not found: " + CREDENTIALS_FILE_PATH); - } - GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, new InputStreamReader(in)); - - // Build flow and trigger user authorization request. - GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( - HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES) - .setDataStoreFactory(new FileDataStoreFactory(new java.io.File(TOKENS_DIRECTORY_PATH))) - .setAccessType("offline") - .build(); - LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort(8080).build(); - return new AuthorizationCodeInstalledApp(flow, receiver).authorize("user"); - } - - /** - * Prints the names and majors of students in a sample spreadsheet: - * https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit - */ - public static void main(String... args) throws IOException, GeneralSecurityException { - // Build a new authorized API client service. -// final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); -// final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; -// final String range = "Class Data"; -// Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) -// .setApplicationName(APPLICATION_NAME) -// .build(); -// Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); -// for (Sheet sheet: s.getSheets()) { -// System.out.println(sheet.getProperties().getTitle()); -// } -// ValueRange response = service.spreadsheets().values() -// .get(spreadsheetId, range) -// .execute(); -// List> values = response.getValues(); -// if (values == null || values.isEmpty()) { -// System.out.println("No data found."); -// } else { -// System.out.println("Name, Major"); -// for (List row : values) { -// // Print columns A and E, which correspond to indices 0 and 4. -// System.out.println(values.get(0).get(0).getClass()); -// System.out.printf("%s, %s\n", row.get(0), row.get(4)); -// } -// } - final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - final String spreadsheetId = "1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q"; - - Sheets service = new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT)) - .setApplicationName(APPLICATION_NAME) - .build(); - - // get the properties of all the sheets - Spreadsheet s = service.spreadsheets().get(spreadsheetId).execute(); - - for (Sheet sheet: s.getSheets()) { - String sheetName = sheet.getProperties().getTitle(); - System.out.println(sheetName); - ValueRange response = service.spreadsheets().values() - .get(spreadsheetId, sheetName) - .execute(); - - List> values = response.getValues(); - for (List row : values) { - System.out.printf("%s\n", row.get(0)); - } - } - } -} \ No newline at end of file diff --git a/google-sheet-adapter/src/main/resources/credentials.json b/google-sheet-adapter/src/main/resources/credentials.json index e0d0f1c099..b79243655e 100644 --- a/google-sheet-adapter/src/main/resources/credentials.json +++ b/google-sheet-adapter/src/main/resources/credentials.json @@ -6,6 +6,14 @@ "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "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", - "redirect_uris": ["https://polypheny.org/"] + "redirect_uris": [ + "http://localhost:8080", + "http://localhost:8888/Callback", + "http://localhost:8888", + "http://localhost:8080/Callback" + ], + "javascript_origins": [ + "http://localhost" + ] } -} +} \ No newline at end of file diff --git a/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/google-sheet-adapter/src/main/resources/tokens/StoredCredential deleted file mode 100644 index 5f687a3aa2672f85d0d58e033070a9f4eacd9739..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1135 zcma)5J&YSg6dpTw5V!~(6d*++5=x2`@!D&jFK4Al);_PjxjlQme`hbnF}t4IU3>pL zJG)*xQ3?u#lmdz%da48+qJ;_}8VVX(nnX&Z6j3S~I+*oEy3j;7(!A!q@B7~S=FRa- zupVRZ{sg9wNr>w)T^Kw2Fj{~6uU~)r@UuV9fg%sy^+Gu2p+!P0fOm+4QS5}?IE!xH z0{OWxKTqQ(y#)f+k{ICxgW{*nZj7uj7{>%`zJ?IPRsEV5T2lf(ImzTslxeY*U`!>-c``1G zG*apd7#-yftmJ3l9w=lH0Z$jO+F-Z}r*n-3n_@W2*ge)u4QbjHqv^Ogtf&j@ECVk# zrb=jCZ^il?b!<*drO9MIY{|o2ZMWTC+C4Wj*r?PuW-aHS*4N-9^9M&wzprS-k)g>- z9XUxN3G>F@tRC-e8%lJ5Q>Q+Wh9$>0{jSid$EAwojdz+mopd>GnBCifs_*mmNb3a$ z8fofnULELiQ>z%Vtd3ZtD{QH?0U6i$k;3!RzSfKTU1uP-iI3Wqj=%YM0Hrs(7Ur z21n4vITU?aeXiX3Wg~ANgVLEPa_mo+zI*Wfu5tGSTYm$VGh;fhMFt5<&^X6BI1Jn+ z!gTspa#Mavrx)mdC6MGUZ=ZpyKM$bkq1VZ3kAM962S0rD>MZ~uz-G(kN~58cTHeBN z1thiXk-*6u*Wsp0Z_by*hJb~vQPp%Z=k?U#c~eJxO^r-#8I2s?=^XmqNqx&TQ&Zou Z1;W*&gQ3vKwhwF7sh}L%*I8n7*?&75hadm| diff --git a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java index c7d306ad57..08b3430550 100644 --- a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java +++ b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java @@ -36,7 +36,6 @@ import com.google.common.collect.Ordering; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.polypheny.db.sql.sql2alg.SqlToAlgConverter; import org.polypheny.db.util.Sources; @@ -58,7 +57,6 @@ /** * Unit test of the Polypheny-DB CSV adapter. */ -@Ignore public class CsvTest { /** @@ -68,6 +66,10 @@ private static String escapeString( String s ) { return escapeString( new StringBuilder(), s ).toString(); } + @Test + public void what() { + assertTrue( true ); + } /** * Quotes a string for Java or JSON, into a builder. @@ -176,7 +178,6 @@ private void close( Connection connection, Statement statement ) { /** * Tests the vanity driver. */ - @Ignore @Test public void testVanityDriver() throws SQLException { Properties info = new Properties(); @@ -188,7 +189,6 @@ public void testVanityDriver() throws SQLException { /** * Tests the vanity driver with properties in the URL. */ - @Ignore @Test public void testVanityDriverArgsInUrl() throws SQLException { Connection connection = DriverManager.getConnection( "jdbc:csv:" + "directory='foo'" ); From d4c45d778d6f5cfac921e31365ae97215c028c87 Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:36:35 -0400 Subject: [PATCH 06/19] reverted small changes --- .../java/org/polypheny/db/adapter/csv/CsvScan.java | 14 ++++++++++---- .../test/java/org/polypheny/db/test/CsvTest.java | 8 +++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java index 5fa20003f2..e446d470ec 100644 --- a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java +++ b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java @@ -36,7 +36,11 @@ import org.apache.calcite.linq4j.tree.Blocks; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Primitive; -import org.polypheny.db.adapter.enumerable.*; +import org.polypheny.db.adapter.enumerable.EnumerableAlg; +import org.polypheny.db.adapter.enumerable.EnumerableAlgImplementor; +import org.polypheny.db.adapter.enumerable.EnumerableConvention; +import org.polypheny.db.adapter.enumerable.PhysType; +import org.polypheny.db.adapter.enumerable.PhysTypeImpl; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgWriter; import org.polypheny.db.algebra.core.Scan; @@ -44,9 +48,11 @@ import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.algebra.type.AlgDataTypeField; -import org.polypheny.db.plan.*; - -import java.util.List; +import org.polypheny.db.plan.AlgOptCluster; +import org.polypheny.db.plan.AlgOptCost; +import org.polypheny.db.plan.AlgOptPlanner; +import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.plan.AlgTraitSet; /** diff --git a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java index 08b3430550..86a11a9f22 100644 --- a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java +++ b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java @@ -57,6 +57,7 @@ /** * Unit test of the Polypheny-DB CSV adapter. */ +@Ignore public class CsvTest { /** @@ -66,11 +67,6 @@ private static String escapeString( String s ) { return escapeString( new StringBuilder(), s ).toString(); } - @Test - public void what() { - assertTrue( true ); - } - /** * Quotes a string for Java or JSON, into a builder. */ @@ -178,6 +174,7 @@ private void close( Connection connection, Statement statement ) { /** * Tests the vanity driver. */ + @Ignore @Test public void testVanityDriver() throws SQLException { Properties info = new Properties(); @@ -189,6 +186,7 @@ public void testVanityDriver() throws SQLException { /** * Tests the vanity driver with properties in the URL. */ + @Ignore @Test public void testVanityDriverArgsInUrl() throws SQLException { Connection connection = DriverManager.getConnection( "jdbc:csv:" + "directory='foo'" ); From 285b32c63a5ce9849633d4c274572f8b0d0da67b Mon Sep 17 00:00:00 2001 From: Slayzur02 <44528372+Slayzur02@users.noreply.github.com> Date: Tue, 6 Sep 2022 19:15:25 -0400 Subject: [PATCH 07/19] added typing system, updated README, changed tests --- .../main/resources/tokens/StoredCredential | Bin 1119 -> 0 bytes .../db/adapter/GoogleSheetSourceTest.java | 24 ++---- .../googlesheet/GoogleSheetSource.java | 57 +++++++++++++- .../db/adapter/googlesheet/README.md | 70 ++++++++++++++---- 4 files changed, 119 insertions(+), 32 deletions(-) delete mode 100644 dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential diff --git a/dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential b/dbms/google-sheet-adapter/src/main/resources/tokens/StoredCredential deleted file mode 100644 index 555ada4e91dcbaf5fd1ace5553825b3a77f5f3c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1119 zcma)5zi-n(6uzV_zgnTf01^Tu1Or3kByN%>VgM&?(zH$>v6H4@C>Q%;JF$Jv`J6fd zu^=I&4luw!zyzp>6()pO7+8>)SXv1rgb-q2VdR`vDlDic-CN%KzVF`m-fjK>6D|SA zI0=mEp)eR1!6Z}^c>s2bSXHulL?%NvSP&}gh@n$q#G4kWYkjxwF3u6 z@;z`HgnWmBj|0eOI2aS^LS_*6>a}!P?JhK1`Fh)3SWEWgJRfg03u#AA#=2B!54`?d zZ@H$$(#hFEV_lSrzO^C`hkUl_FXdI*5Dhh=wX8v{Cg&n%RgN3IL{qNWVySa&(X=Dl zqP3chxs{IMq=&s;isw^nVlyHNp3@~hDQ6nO)_(Ck6O0v~BI$DjuTyA95#+L{0@1NfagU}L~ z=tI*(J^|BvMP<)wA#!)~(V5AI+Z(%+58atQJ^78XUEEu=J$t#4y<8PBs|mDe;B&k7 zWM<}F=R4C8!MHoH6$+yN8_qxI3lald@*Sp_*stP+GPc{$AORHJe=9{;6u*v^mg_g_uRcOUS9#R9WPOxnZ5n02;Qd!GK#7vP? z%hE!L7NC1gT(cFG6r5%3uJ}ux`dq)(O}Cd;t0_w){f3^1B-PR^8CI3`<-ve=6EP{C IlJtuI2mP{u4FCWD diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index ec8cc68d48..23ab7da9e7 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -16,8 +16,6 @@ package org.polypheny.db.adapter; -import static org.junit.Assert.assertTrue; - import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.sql.Connection; @@ -67,23 +65,17 @@ public static void end() throws SQLException { } - @Test - public void testTest() { - assertTrue( true ); - } - - @Test public void existsSelect() throws SQLException { try ( TestHelper.JdbcConnection polyphenyDbConnection = new TestHelper.JdbcConnection( true ) ) { Connection connection = polyphenyDbConnection.getConnection(); try ( Statement statement = connection.createStatement() ) { List expectedResult = ImmutableList.of( - new Object[]{ "Oracle", "Product Manager", "127000" }, - new Object[]{ "eBay", "Software Engineer", "100000" }, - new Object[]{ "Amazon", "Product Manager", "310000" }, - new Object[]{ "Apple", "Software Engineering Manager", "372000" }, - new Object[]{ "Microsoft", "Software Engineer", "157000" } + new Object[]{ "Oracle", "Product Manager", 127000 }, + new Object[]{ "eBay", "Software Engineer", 100000 }, + new Object[]{ "Amazon", "Product Manager", 310000 }, + new Object[]{ "Apple", "Software Engineering Manager", 372000 }, + new Object[]{ "Microsoft", "Software Engineer", 157000 } ); TestHelper.checkResultSet( @@ -101,9 +93,9 @@ public void existsSecondSelect() throws SQLException { Connection connection = polyphenyDbConnection.getConnection(); try ( Statement statement = connection.createStatement() ) { List expectedResult = ImmutableList.of( - new Object[]{ "GP", "F", "18", "U", "GT3", "A", "4", "4", "at_home", "teacher", "course", "mother", "2", "2", "0", "yes", "no", "no", "no", "yes", "yes", "no", "no", "4", "3", "4", "3", "6", "5", "6", "6" }, - new Object[]{ "GP", "F", "17", "U", "GT3", "T", "1", "1", "at_home", "other", "course", "father", "1", "2", "0", "no", "yes", "no", "no", "no", "yes", "yes", "no", "5", "3", "3", "3", "4", "5", "5", "6" }, - new Object[]{ "GP", "F", "15", "U", "LE3", "T", "1", "1", "at_home", "other", "other", "mother", "1", "2", "3", "yes", "no", "yes", "no", "yes", "yes", "yes", "no", "4", "3", "2", "3", "10", "7", "8", "10" } + new Object[]{ "GP", "F", 18, "U", "GT3", "A", 4, 4, "at_home", "teacher", "course", "mother", 2, 2, 0, "yes", "no", "no", "no", "yes", "yes", "no", "no", 4, 3, 4, 3, 6, 5, 6, 6 }, + new Object[]{ "GP", "F", 17, "U", "GT3", "T", 1, 1, "at_home", "other", "course", "father", 1, 2, 0, "no", "yes", "no", "no", "no", "yes", "yes", "no", 5, 3, 3, 3, 4, 5, 5, 6 }, + new Object[]{ "GP", "F", 15, "U", "LE3", "T", 1, 1, "at_home", "other", "other", "mother", 1, 2, 3, "yes", "no", "yes", "no", "yes", "yes", "yes", "no", 4, 3, 2, 3, 10, 7, 8, 10 } ); TestHelper.checkResultSet( statement.executeQuery( "SELECT * from student_data LIMIT 3" ), diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index 9fbbf70048..af1ddc6986 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.Adapter; @@ -133,11 +134,61 @@ public Map> getExportedColumns() { List exportedCols = new ArrayList<>(); int position = 0; for ( Object col : tableColumns ) { + String colStr = col.toString(); + int colon_index = colStr.indexOf( ':' ); + + String colName = colStr; + PolyType type = PolyType.VARCHAR; + Integer length = maxStringLength; + + if (colon_index != -1) { + colName = colStr.substring( 0, colon_index ).trim(); + String colType = colStr.substring( colon_index+1 ).toLowerCase( Locale.ROOT ).trim(); + switch (colType) { + case "int": + type = PolyType.INTEGER; + length = null; + break; + case "string": + break; + case "boolean": + type = PolyType.BOOLEAN; + length = null; + break; + case "long": + type = PolyType.BIGINT; + length = null; + break; + case "float": + type = PolyType.REAL; + length = null; + break; + case "double": + type = PolyType.DOUBLE; + length = null; + break; + case "date": + type = PolyType.DATE; + length = null; + break; + case "time": + type = PolyType.TIME; + length = 0; + break; + case "timestamp": + type = PolyType.TIMESTAMP; + length = 0; + break; + default: + throw new RuntimeException( "Unknown type: " + colType.toLowerCase() ); + } + } + exportedCols.add( new ExportedColumn( - col.toString(), - PolyType.VARCHAR, // defaulting to VARCHAR currently - null, + colName, + type, null, + length, null, null, null, diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md index 6589ed1688..559d2f5ace 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md +++ b/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md @@ -1,23 +1,67 @@ -### For users +# For user website -The Google Sheet Adapter is a data-source adapter that allows data to be sourced from any Google Sheet URL into Polypheny. To do so, please go into the -Adapters section, choose data source, then Google Sheets. Here, you will be presented with multiple values to fill out: +--- +layout: plain +title: CSV Adapter +--- -- Name: name of the schema -- sheetsURL: the Google Sheet URL to source from. -- MaxStringLength: max length of a string in all sheets in the sheetsURL -- querySize: the number of rows scanned per query on the GoogleSheet. -- resetRefreshToken: type "YES" to delete the current refresh token and use another account for the sheet adapter. +The Google Sheet Adapter is a data-source adapter that allows data to be sourced from any Google Sheet URL into Polypheny +as relational tables. The adapter is read-only, with DML queries not supported. +The content of the google sheets can be changed in the background as long +as the names of the files and the columns itself don't change. - The adapter adjusts itself to different formats of the sheet, but keep in mind there may be issues in the following cases: +The first column will always be the primary key. The Adapter also does not +support null values -- There are two tables nested within the same sheet in the URL +For formatting, the adapter itself adjusts to different formatting, but keep in mind there may be issues in the following cases: + +- There are two tables nested within the same sheet in the URL (will cause error) - There is a gap between the table rows (the adapter won't read after the gap) -Otherwise, as long as the email used has access to the sheet, there should be no issues. +Otherwise, as long as the email used has access to the sheet, there should be no issues. + +## Adapter settings + +The Google Sheet Source Adapter has the following settings: + +| Name | Description | +|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Name | Unique name for Google Sheet URL tables | +| sheetsURL | the Google Sheet URL to source from | +| maxStringLength | the maximum size of the strings in all tables | +| querySize | the number of rows scanned per network-call with the GoogleSheet API | +| resetRefreshToken | type "Yes" to delete the current refresh token associated to the current email used to read from Google Sheets and use another account for the Google Sheet Adapter. "No" is default. | + + + +## Supported Data Types + +It should be noted that all columns in the GoogleSheetAdapters are of type string, unless the columns of the sheet +are specially formatted. More specifically, all columns will have type string unless the column name is formatted as +`columnName:type`, in which case the type of column will be the specified type. Types can be one of the following: `int, string, boolean, long, float, double, date, time, timestamp`. + +For example: +- If the Google Sheet has `user_id:int` as the column name, the imported column name would be `user_id` and the column type would be `int` +- If the Google Sheet has `birthday:date` as the column, then the imported column name would be `birthday` and the column type would be `date` +- If the Google Sheet has `age` as the column name, then even if all values in the column are of type `int`, it would still be a `string` when imported +into Polypheny + +## Deployment + +The adapter can be deployed using Polypheny-UI (Adapters -> Sources), or using the following SQL statement: + + +{% highlight sql %} ALTER ADAPTERS ADD unique_name USING 'org.polypheny.db.adapter.googlesheet.GoogleSheetSource' WITH +'{maxStringLength: "255", querySize: "1000", sheetsURL: "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", +mode: "remote", resetRefreshToken: "No"}' {% highlight sql %} + +Please refer to the settings above and alter the maxStringLength, querySize, ... as needed. After successful deployment, all +Google Sheets are mapped as a table in the public schema. + +--- -### For future development +# For future development **Token management** @@ -36,4 +80,4 @@ which then extends refresh tokens for infinite use along with removes the API ca - Splitting GoogleSheetTableScanProject -> Scan and Project classes separately. -- Moving the GoogleSheetSourceTest from `dbms/src/test/java/org.polypheny.db/adapter/` to google-sheet-adapter package. \ No newline at end of file +- Moving the GoogleSheetSourceTest from `dbms/src/test/java/org.polypheny.db/adapter/` to google-sheet-adapter package. From 770a1d48e369cd069b01158f8b6c98521e2a012d Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 9 Feb 2023 16:42:36 +0100 Subject: [PATCH 08/19] reformatting code, restructuring to Plugin, adjusted licensee, monitoring to debug, rfm --- google-sheet-adapter/build.gradle | 58 ---------- .../monitoring/core/MonitoringQueueImpl.java | 50 +------- .../org/polypheny/db/adapter/csv/CsvScan.java | 1 + plugins/google-sheet-adapter/build.gradle | 107 ++++++++++++++++++ .../google-sheet-adapter/gradle.properties | 27 +++++ .../googlesheet/GoogleSheetEnumerator.java | 2 +- .../googlesheet/GoogleSheetFieldType.java | 2 +- .../googlesheet/GoogleSheetPlugin.java | 50 ++++++++ .../GoogleSheetProjectTableScanRule.java | 4 +- .../googlesheet/GoogleSheetReader.java | 16 +++ .../googlesheet/GoogleSheetSchema.java | 2 +- .../googlesheet/GoogleSheetSource.java | 10 +- .../adapter/googlesheet/GoogleSheetTable.java | 5 +- .../GoogleSheetTableScanProject.java | 6 +- .../db/adapter/googlesheet/README.md | 0 .../src/main/resources/credentials.json | 0 .../db/http/HttpInterfacePlugin.java | 2 +- 17 files changed, 224 insertions(+), 118 deletions(-) delete mode 100644 google-sheet-adapter/build.gradle create mode 100644 plugins/google-sheet-adapter/build.gradle create mode 100644 plugins/google-sheet-adapter/gradle.properties rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java (99%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java (98%) create mode 100644 plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java (95%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java (95%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java (99%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java (97%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java (97%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java (95%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/java/org/polypheny/db/adapter/googlesheet/README.md (100%) rename {google-sheet-adapter => plugins/google-sheet-adapter}/src/main/resources/credentials.json (100%) diff --git a/google-sheet-adapter/build.gradle b/google-sheet-adapter/build.gradle deleted file mode 100644 index 787fb607ca..0000000000 --- a/google-sheet-adapter/build.gradle +++ /dev/null @@ -1,58 +0,0 @@ -group "org.polypheny" - - -dependencies { - implementation project(":core") - implementation project(":sql-language") - - implementation 'com.google.api-client:google-api-client:1.33.0' - implementation 'com.google.oauth-client:google-oauth-client-jetty:1.32.1' - implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' - -// --- Test Compile --- - testImplementation project(path: ":core", configuration: "tests") - - testImplementation group: "junit", name: "junit", version: junit_version - testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version // BSD 3-clause - testImplementation group: "com.konghq", name: "unirest-java", version: unirest_version // MIT -} - - -sourceSets { - main { - java { - srcDirs = ["src/main/java"] - outputDir = file(project.buildDir.absolutePath + "/classes") - } - resources { - srcDirs = ["src/main/resources"] - } - output.resourcesDir = file(project.buildDir.absolutePath + "/classes") - } - test { - java { - srcDirs = ["src/test/java"] - outputDir = file(project.buildDir.absolutePath + "/test-classes") - } - resources { - srcDirs = ["src/test/resources"] - } - output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") - } -} - - -/** - * JARs - */ -jar { - manifest { - attributes "Manifest-Version": "1.0" - attributes "Copyright": "The Polypheny Project (polypheny.org)" - attributes "Version": "$project.version" - } -} -java { - withJavadocJar() - withSourcesJar() -} diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java index d961b8aa2e..e0c7d6574a 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java @@ -16,7 +16,11 @@ package org.polypheny.db.monitoring.core; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -96,7 +100,7 @@ public synchronized void run() { if (newThreadCount != threadCount) { threadCount = newThreadCount; - log.info("Thread count is now: {}", threadCount); + log.debug( "Thread count is now: {}", threadCount ); } } }, 500, 500); @@ -251,47 +255,5 @@ private void processQueue() { } - /* - class ThreadMonitor implements Runnable{ - private int corePoolThreadsCount; - private int threadCount; - private ThreadPoolExecutor monitoredThread; - - public ThreadMonitor(ThreadPoolExecutor t) { - super(); - this.monitoredThread = t; - this.corePoolThreadsCount = t.getCorePoolSize(); - this.threadCount = t.getPoolSize(); - } - - public void run() { - while (true) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - log.error("Thread Monitor wasn't able to sleep"); - } - - log.info("The guard is monitoring the threats."); - - int newCorePoolCount = monitoredThread.getCorePoolSize(); - int newThreadCount = monitoredThread.getPoolSize(); - - if (newCorePoolCount != this.corePoolThreadsCount) { - this.corePoolThreadsCount = newCorePoolCount; - log.info("new core pool thread count is = {}", this.corePoolThreadsCount); - } - - if (newThreadCount != this.threadCount) { - this.threadCount = newThreadCount; - log.info("new thread count is = {}", this.threadCount); - } - } - } - - - - } - */ } diff --git a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java index e446d470ec..7001483ee8 100644 --- a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java +++ b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvScan.java @@ -33,6 +33,7 @@ package org.polypheny.db.adapter.csv; +import java.util.List; import org.apache.calcite.linq4j.tree.Blocks; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Primitive; diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle new file mode 100644 index 0000000000..0b4f02438e --- /dev/null +++ b/plugins/google-sheet-adapter/build.gradle @@ -0,0 +1,107 @@ +group "org.polypheny" + + +dependencies { + implementation project(":core") + implementation project(":plugins:sql-language") + + implementation 'com.google.api-client:google-api-client:1.33.0' + implementation 'com.google.oauth-client:google-oauth-client-jetty:1.32.1' + implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' + +// --- Test Compile --- + testImplementation project(path: ":core", configuration: "tests") + + testImplementation group: "junit", name: "junit", version: junit_version +} + + +sourceSets { + main { + java { + srcDirs = ["src/main/java"] + outputDir = file(project.buildDir.absolutePath + "/classes") + } + resources { + srcDirs = ["src/main/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/classes") + } + test { + java { + srcDirs = ["src/test/java"] + outputDir = file(project.buildDir.absolutePath + "/test-classes") + } + resources { + srcDirs = ["src/test/resources"] + } + output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") + } +} + + +/** + * JARs + */ +jar { + manifest { + attributes "Manifest-Version": "1.0" + attributes "Copyright": "The Polypheny Project (polypheny.org)" + attributes "Version": "$project.version" + } +} +java { + withJavadocJar() + withSourcesJar() +} + +licensee { + allow('Apache-2.0') + allow('MIT') + allow('BSD-3-Clause') + allow('BSD-2-Clause') + allow('MPL-1.1') + allow('CC0-1.0') // Public Domain Dedication + + allowUrl('http://www.eclipse.org/legal/epl-2.0') // EPL 2.0 + allowUrl('http://www.opensource.org/licenses/Apache-2.0') // Apache 2.0 + allowUrl('http://www.jcabi.com/LICENSE.txt') // Own licensee but conforms + allowUrl('https://www.eclipse.org/org/documents/epl-v10.php') // EPL 1.0 + allowUrl('https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt') // EPL 2.0 + allowUrl('http://www.eclipse.org/org/documents/edl-v10.php') // EDL 1.0 + + allowUrl('http://asm.ow2.org/license.html') // because BSD 3-Clause + allowUrl('https://raw.githubusercontent.com/janino-compiler/janino/master/LICENSE') // open janino license + + + allowDependency('com.adobe.xmp', 'xmpcore', '6.0.6') { because 'BSD 3-Clause' } + allowDependency('com.github.jnr', 'jnr-posix', '3.0.50') { because 'Eclipse Public License v. 2.0' } + allowDependency('com.j256.simplemagic', 'simplemagic', '1.16') { because 'ISC license' } + allowDependency('jakarta.annotation', 'jakarta.annotation-api', '1.3.5') { because 'Eclipse Public License v. 2.0' } + allowDependency('jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.2') { because 'Eclipse Distribution License 1.0' } + allowDependency('jakarta.activation', 'jakarta.activation-api', '1.2.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('jakarta.ws.rs', 'jakarta.ws.rs-api', '2.1.6') { because 'Eclipse Public License v. 2.0' } + allowDependency('net.java.dev.jna', 'jna', '5.8.0') { because 'Apache 2.0 license' } + + allowDependency('org.bouncycastle', 'bcpkix-jdk15on', '1.64') { because 'MIT license' } + allowDependency('org.bouncycastle', 'bcprov-jdk15on', '1.64') { because 'MIT license' } + + allowDependency('org.glassfish.jersey.inject', 'jersey-hk2', '2.30.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2.external', 'jakarta.inject', '2.6.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2.external', 'aopalliance-repackaged', '2.6.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2', 'osgi-resource-locator', '1.0.3') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2', 'hk2-api', '2.6.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2', 'hk2-locator', '2.6.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.hk2', 'hk2-utils', '2.6.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.jersey.core', 'jersey-common', '2.30.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.jersey.core', 'jersey-client', '2.30.1') { because 'Eclipse Public License v. 2.0' } + allowDependency('org.glassfish.jersey.connectors', 'jersey-apache-connector', '2.30.1') { because 'Eclipse Public License v. 2.0' } + + allowDependency('org.javassist', 'javassist', '3.28.0-GA') { because 'Apache 2.0 license' } + allowDependency('org.reflections', 'reflections', '0.10.2') { because 'Apache 2.0 license' } + allowDependency('org.yaml', 'snakeyaml', '1.15') { because 'Apache 2.0 license' } + + allowDependency('javax.servlet', 'javax.servlet-api', '3.1.0') { + because 'Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original source code for this can be found at http://www.eclipse.org/jetty/downloads.php' + } +} diff --git a/plugins/google-sheet-adapter/gradle.properties b/plugins/google-sheet-adapter/gradle.properties new file mode 100644 index 0000000000..202c77eacb --- /dev/null +++ b/plugins/google-sheet-adapter/gradle.properties @@ -0,0 +1,27 @@ +# +# Copyright 2019-2023 The Polypheny Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pluginVersion = 0.0.1 + +pluginId = google-sheets-adapter +pluginClass = org.polypheny.db.adapter.googlesheet.GoogleSheetPlugin +pluginProvider = The Polypheny Project +pluginDependencies = sql-language +pluginUrlPath = +pluginCategories = source +pluginPolyDependencies = +pluginIsSystemComponent = false +pluginIsUiVisible = true \ No newline at end of file diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java similarity index 99% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index a1d719948e..9326fa029c 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java similarity index 98% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java index ea927f165c..38b03a16d2 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetFieldType.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java new file mode 100644 index 0000000000..738571e57b --- /dev/null +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet; + + +import java.util.HashMap; +import java.util.Map; +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.polypheny.db.catalog.Adapter; + +public class GoogleSheetPlugin extends Plugin { + + /** + * Constructor to be used by plugin manager for plugin instantiation. + * Your plugins have to provide constructor with this exact signature to be successfully loaded by manager. + */ + public GoogleSheetPlugin( PluginWrapper wrapper ) { + super( wrapper ); + } + + + @Override + public void start() { + Map settings = new HashMap<>(); + settings.put( "maxStringLength", "255" ); + settings.put( "querySize", "1000" ); + settings.put( "sheetsURL", "" ); + settings.put( "mode", "remote" ); + settings.put( "resetRefreshToken", "No" ); + + Adapter.addAdapter( GoogleSheetSource.class, "GOOGLE", settings ); + } + + +} diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java similarity index 95% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java index d4bfe289c6..2f487f4acb 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import java.util.List; import org.polypheny.db.algebra.core.AlgFactories; -import org.polypheny.db.algebra.logical.LogicalProject; +import org.polypheny.db.algebra.logical.relational.LogicalProject; import org.polypheny.db.plan.AlgOptRule; import org.polypheny.db.plan.AlgOptRuleCall; import org.polypheny.db.rex.RexInputRef; diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java similarity index 95% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index b0c836a138..cb52016654 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -1,3 +1,19 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.polypheny.db.adapter.googlesheet; import com.google.api.client.auth.oauth2.Credential; diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java similarity index 99% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java index 94ec02c380..c0210c6f70 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java similarity index 97% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index af1ddc6986..eea33be87a 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -135,16 +135,16 @@ public Map> getExportedColumns() { int position = 0; for ( Object col : tableColumns ) { String colStr = col.toString(); - int colon_index = colStr.indexOf( ':' ); + int colon_index = colStr.indexOf( ':' ); String colName = colStr; PolyType type = PolyType.VARCHAR; Integer length = maxStringLength; - if (colon_index != -1) { + if ( colon_index != -1 ) { colName = colStr.substring( 0, colon_index ).trim(); - String colType = colStr.substring( colon_index+1 ).toLowerCase( Locale.ROOT ).trim(); - switch (colType) { + String colType = colStr.substring( colon_index + 1 ).toLowerCase( Locale.ROOT ).trim(); + switch ( colType ) { case "int": type = PolyType.INTEGER; length = null; diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java similarity index 97% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index d0413fe2dc..d6117a8373 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.algebra.type.AlgProtoDataType; import org.polypheny.db.plan.AlgOptTable; +import org.polypheny.db.plan.AlgTraitSet; import org.polypheny.db.schema.QueryableTable; import org.polypheny.db.schema.SchemaPlus; import org.polypheny.db.schema.Schemas; @@ -129,7 +130,7 @@ public Expression getExpression( SchemaPlus schema, String tableName, Class claz @Override - public AlgNode toAlg( AlgOptTable.ToAlgContext context, AlgOptTable algOptTable ) { + public AlgNode toAlg( AlgOptTable.ToAlgContext context, AlgOptTable algOptTable, AlgTraitSet traitSet ) { // Request all fields. return new GoogleSheetTableScanProject( context.getCluster(), algOptTable, this, fields ); } diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java similarity index 95% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java index 41c9ec3b2d..48eebe3c4a 100644 --- a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTableScanProject.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ import org.polypheny.db.adapter.enumerable.PhysTypeImpl; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgWriter; -import org.polypheny.db.algebra.core.TableScan; +import org.polypheny.db.algebra.core.Scan; import org.polypheny.db.algebra.metadata.AlgMetadataQuery; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeFactory; @@ -44,7 +44,7 @@ * * Like any table scan, it serves as a leaf node of a query tree. */ -public class GoogleSheetTableScanProject extends TableScan implements EnumerableAlg { +public class GoogleSheetTableScanProject extends Scan implements EnumerableAlg { final GoogleSheetTable googleSheetTable; final int[] fields; diff --git a/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md similarity index 100% rename from google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md rename to plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md diff --git a/google-sheet-adapter/src/main/resources/credentials.json b/plugins/google-sheet-adapter/src/main/resources/credentials.json similarity index 100% rename from google-sheet-adapter/src/main/resources/credentials.json rename to plugins/google-sheet-adapter/src/main/resources/credentials.json diff --git a/plugins/http-interface/src/main/java/org/polypheny/db/http/HttpInterfacePlugin.java b/plugins/http-interface/src/main/java/org/polypheny/db/http/HttpInterfacePlugin.java index c3d118a872..1ac9f13c8c 100644 --- a/plugins/http-interface/src/main/java/org/polypheny/db/http/HttpInterfacePlugin.java +++ b/plugins/http-interface/src/main/java/org/polypheny/db/http/HttpInterfacePlugin.java @@ -164,7 +164,7 @@ public String toJsonString( @NotNull Object obj ) { public void addRoute( QueryLanguage language ) { for ( String route : language.getOtherNames() ) { - log.info( "added http route: {}", route ); + log.info( "Added HTTP route: /{}", route ); server.post( route, ctx -> anyQuery( language, ctx ) ); } } From 29663b8e9d3a464dbdc7944b1963e3cfd10953f3 Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 9 Feb 2023 17:36:01 +0100 Subject: [PATCH 09/19] minor adjustment to fix sourcefiles and missing header adjustment --- .../db/adapter/GoogleSheetSourceTest.java | 2 +- .../db/adapter/googlesheet => }/README.md | 0 plugins/google-sheet-adapter/build.gradle | 27 +++++-------------- 3 files changed, 7 insertions(+), 22 deletions(-) rename plugins/google-sheet-adapter/{src/main/java/org/polypheny/db/adapter/googlesheet => }/README.md (100%) diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index 23ab7da9e7..b989b686ba 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md b/plugins/google-sheet-adapter/README.md similarity index 100% rename from plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/README.md rename to plugins/google-sheet-adapter/README.md diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle index 0b4f02438e..df03d1121d 100644 --- a/plugins/google-sheet-adapter/build.gradle +++ b/plugins/google-sheet-adapter/build.gradle @@ -16,27 +16,12 @@ dependencies { } -sourceSets { - main { - java { - srcDirs = ["src/main/java"] - outputDir = file(project.buildDir.absolutePath + "/classes") - } - resources { - srcDirs = ["src/main/resources"] - } - output.resourcesDir = file(project.buildDir.absolutePath + "/classes") - } - test { - java { - srcDirs = ["src/test/java"] - outputDir = file(project.buildDir.absolutePath + "/test-classes") - } - resources { - srcDirs = ["src/test/resources"] - } - output.resourcesDir = file(project.buildDir.absolutePath + "/test-classes") - } +compileJava { + dependsOn(":plugins:sql-language:processResources") +} + +delombok { + dependsOn(":plugins:sql-language:processResources") } From 2ee35d76f08464bdb82d3be28a97290dd985ba60 Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 9 Feb 2023 18:04:26 +0100 Subject: [PATCH 10/19] added missing import --- .../java/org/polypheny/db/test/CsvTest.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java index 86a11a9f22..553bea4c40 100644 --- a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java +++ b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java @@ -34,24 +34,36 @@ package org.polypheny.db.test; -import com.google.common.collect.Ordering; -import org.junit.Assert; -import org.junit.Test; -import org.polypheny.db.sql.sql2alg.SqlToAlgConverter; -import org.polypheny.db.util.Sources; -import org.polypheny.db.util.Util; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.isA; +import static org.junit.Assert.assertThat; +import com.google.common.collect.Ordering; import java.io.PrintStream; import java.io.PrintWriter; -import java.sql.*; -import java.util.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.function.Consumer; - -import static org.hamcrest.CoreMatchers.*; -import static org.junit.Assert.assertThat; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.polypheny.db.sql.sql2alg.SqlToAlgConverter; +import org.polypheny.db.util.Sources; +import org.polypheny.db.util.Util; /** From 6dc96b592c687452107148de8e4360f92418953a Mon Sep 17 00:00:00 2001 From: datomo Date: Thu, 9 Feb 2023 21:00:23 +0100 Subject: [PATCH 11/19] fixing test, but ignore due to setup required, ignored mysql as on master --- .../java/org/polypheny/db/adapter/GoogleSheetSourceTest.java | 5 +++-- .../test/java/org/polypheny/db/adapter/MysqlSourceTest.java | 3 ++- .../polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index b989b686ba..b60aa3c512 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -26,11 +26,12 @@ import java.util.Map; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.polypheny.db.TestHelper; import org.polypheny.db.TestHelper.JdbcConnection; - +@Ignore // for debug purpose public class GoogleSheetSourceTest { @BeforeClass @@ -47,7 +48,7 @@ public static void start() throws SQLException { settings.put( "mode", "remote" ); settings.put( "resetRefreshToken", "No" ); Gson gson = new Gson(); - statement.executeUpdate( "ALTER ADAPTERS ADD googlesheetunit USING 'org.polypheny.db.adapter.googlesheet.GoogleSheetSource' WITH '" + gson.toJson( settings ) + "'" ); + statement.executeUpdate( "ALTER ADAPTERS ADD \"googlesheetunit\" USING 'GoogleSheets' AS 'Source' WITH '" + gson.toJson( settings ) + "'" ); } } } diff --git a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java index 1df3ec6066..b62effd146 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java @@ -27,11 +27,12 @@ import java.util.Map; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.polypheny.db.TestHelper; import org.polypheny.db.TestHelper.JdbcConnection; @SuppressWarnings("SqlDialectInspection") -//@Ignore +@Ignore public class MysqlSourceTest extends AbstractSourceTest { private static EmbeddedMysql server; diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java index 738571e57b..ee1b43e47b 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java @@ -43,7 +43,7 @@ public void start() { settings.put( "mode", "remote" ); settings.put( "resetRefreshToken", "No" ); - Adapter.addAdapter( GoogleSheetSource.class, "GOOGLE", settings ); + Adapter.addAdapter( GoogleSheetSource.class, "GOOGLESHEETS", settings ); } From c8e7e6269a4318dd2847c4e66a8abe34cf92d711 Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 27 Feb 2023 17:05:44 +0100 Subject: [PATCH 12/19] replaced hardcoded files with parameters, fixed empty row problem, fixed tableName inconsistency --- dbms/build.gradle | 2 + gradle.properties | 1 + plugins/google-sheet-adapter/build.gradle | 7 +- .../googlesheet/GoogleSheetEnumerator.java | 17 ++- .../googlesheet/GoogleSheetReader.java | 73 +++--------- .../googlesheet/GoogleSheetSchema.java | 5 +- .../googlesheet/GoogleSheetSource.java | 108 +++++++++++++++--- .../adapter/googlesheet/GoogleSheetTable.java | 4 +- .../src/main/resources/credentials.json | 19 --- 9 files changed, 134 insertions(+), 102 deletions(-) delete mode 100644 plugins/google-sheet-adapter/src/main/resources/credentials.json diff --git a/dbms/build.gradle b/dbms/build.gradle index 72bfa7c84f..6ceb7529be 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -51,6 +51,8 @@ dependencies { // workaround to fix cottontail todo remove: when grpc-all is same in cottontail and for plugin implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version + // workaround for oauth, uses IOUtils... + implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") diff --git a/gradle.properties b/gradle.properties index fe4d774477..532b574c84 100644 --- a/gradle.properties +++ b/gradle.properties @@ -105,6 +105,7 @@ natty_version = 0.13 okhttp_version = 4.2.2 opencsv_version = 2.3 oshi_core_version = 5.7.4 +oauth_client_version = 1.32.1 pigunit_version = 0.16.0 pig_version = 0.16.0 poi_version = 5.2.3 diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle index df03d1121d..032686491a 100644 --- a/plugins/google-sheet-adapter/build.gradle +++ b/plugins/google-sheet-adapter/build.gradle @@ -2,11 +2,12 @@ group "org.polypheny" dependencies { - implementation project(":core") - implementation project(":plugins:sql-language") + compileOnly project(":core") + compileOnly project(":plugins:sql-language") + compileOnly project(":dbms") implementation 'com.google.api-client:google-api-client:1.33.0' - implementation 'com.google.oauth-client:google-oauth-client-jetty:1.32.1' + implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' // --- Test Compile --- diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index 9326fa029c..f0e0d8f7b0 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -26,6 +26,7 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.linq4j.Enumerator; import org.apache.commons.lang3.time.FastDateFormat; +import org.polypheny.db.util.Pair; /** @@ -56,17 +57,16 @@ public class GoogleSheetEnumerator implements Enumerator { private E current; - GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, RowConverter rowConverter ) { + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, RowConverter rowConverter, Pair oAuthIdKey ) { this.tableName = tableName; this.cancelFlag = cancelFlag; this.rowConverter = rowConverter; - this.reader = new GoogleSheetReader( sheetsUrl, querySize ); - + this.reader = new GoogleSheetReader( sheetsUrl, querySize, oAuthIdKey ); } - GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, List fieldTypes, int[] fields ) { - this( sheetsUrl, querySize, tableName, cancelFlag, (RowConverter) converter( fieldTypes, fields ) ); + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, List fieldTypes, int[] fields, Pair oAuthIdKey ) { + this( sheetsUrl, querySize, tableName, cancelFlag, (RowConverter) converter( fieldTypes, fields ), oAuthIdKey ); } @@ -246,7 +246,12 @@ public Object[] convertNormalRow( String[] strings ) { final Object[] objects = new Object[fields.length]; for ( int i = 0; i < fields.length; i++ ) { int field = fields[i]; - objects[i] = convert( fieldTypes[i], strings[field] ); + if ( fields.length == strings.length || i < strings.length ) { // if last value is null the returned strings is one less + objects[i] = convert( fieldTypes[i], strings[field] ); + } else { + objects[i] = convert( fieldTypes[i], "" ); + } + } return objects; } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index cb52016654..cfa5bdfd14 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -16,34 +16,22 @@ package org.polypheny.db.adapter.googlesheet; -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.store.FileDataStoreFactory; import com.google.api.services.sheets.v4.Sheets; -import com.google.api.services.sheets.v4.SheetsScopes; import com.google.api.services.sheets.v4.model.GridProperties; import com.google.api.services.sheets.v4.model.Sheet; import com.google.api.services.sheets.v4.model.Spreadsheet; import com.google.api.services.sheets.v4.model.ValueRange; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URL; import java.security.GeneralSecurityException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Objects; +import org.polypheny.db.util.Pair; /** @@ -51,29 +39,25 @@ */ public class GoogleSheetReader { - private final String APPLICATION_NAME = "POLYPHENY GOOGLE SHEET READER"; - private final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - private final String TOKENS_DIRECTORY_PATH = "google-sheet-adapter/src/main/resources/tokens"; - private final List SCOPES = Collections.singletonList( SheetsScopes.SPREADSHEETS_READONLY ); - private final String CREDENTIALS_FILE_PATH = "/credentials.json"; - private final URL url; private final int querySize; - private HashMap> tableSurfaceData = new HashMap<>(); - private HashMap>> tableData = new HashMap<>(); - private HashMap tableStart = new HashMap<>(); - private HashMap tableLeftOffset = new HashMap<>(); - private HashMap enumPointer = new HashMap<>(); + private final Pair oAuthIdKey; + private final HashMap> tableSurfaceData = new HashMap<>(); + private final HashMap>> tableData = new HashMap<>(); + private final HashMap tableStart = new HashMap<>(); + private final HashMap tableLeftOffset = new HashMap<>(); + private final HashMap enumPointer = new HashMap<>(); /** * @param url - url of the Google Sheet to source. * @param querySize - size of the query (in case of large files) */ - public GoogleSheetReader( URL url, int querySize ) { + public GoogleSheetReader( URL url, int querySize, Pair oAuthIdKey ) { this.url = url; this.querySize = querySize; + this.oAuthIdKey = oAuthIdKey; } @@ -85,12 +69,9 @@ private void readTableSurfaceData() { return; } try { - final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - final String spreadsheetId = parseUrlToString( url ); - Sheets service = new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( HTTP_TRANSPORT ) ) - .setApplicationName( APPLICATION_NAME ) - .build(); + final String spreadsheetId = parseUrlToString( url ); + Sheets service = GoogleSheetSource.getSheets( oAuthIdKey ); // get the properties of all the sheets Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); @@ -129,7 +110,7 @@ private void readTableSurfaceData() { break; } } - } catch ( IOException | GeneralSecurityException e ) { + } catch ( IOException e ) { throw new RuntimeException( e ); } } @@ -143,27 +124,8 @@ public HashMap> getTableSurfaceData() { } - private Credential getCredentials( final NetHttpTransport HTTP_TRANSPORT ) throws IOException { - // Load client secrets. - InputStream in = GoogleSheetReader.class.getResourceAsStream( CREDENTIALS_FILE_PATH ); - if ( in == null ) { - throw new FileNotFoundException( "Resource not found: " + CREDENTIALS_FILE_PATH ); - } - GoogleClientSecrets clientSecrets = GoogleClientSecrets.load( JSON_FACTORY, new InputStreamReader( in ) ); - - // Build flow and trigger user authorization request. - GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( - HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES ) - .setDataStoreFactory( new FileDataStoreFactory( new java.io.File( TOKENS_DIRECTORY_PATH ) ) ) - .setAccessType( "offline" ) - .build(); - LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort( 8888 ).build(); - return new AuthorizationCodeInstalledApp( flow, receiver ).authorize( "user" ); - } - - public void deleteToken() { - File file = new File( TOKENS_DIRECTORY_PATH + "/StoredCredential" ); + File file = new File( GoogleSheetSource.TOKENS_PATH, "/StoredCredential" ); if ( file.exists() ) { file.delete(); } @@ -188,8 +150,8 @@ private void readTable( String tableName ) { final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); final String spreadsheetId = parseUrlToString( url ); - Sheets service = new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( HTTP_TRANSPORT ) ) - .setApplicationName( APPLICATION_NAME ) + Sheets service = new Sheets.Builder( HTTP_TRANSPORT, GoogleSheetSource.JSON_FACTORY, GoogleSheetSource.getCredentials( oAuthIdKey, HTTP_TRANSPORT ) ) + .setApplicationName( GoogleSheetSource.APPLICATION_NAME ) .build(); // first query! let's start searching till we find first row. @@ -197,11 +159,14 @@ private void readTable( String tableName ) { Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); // get all the sheets Sheet chosen_sheet = new Sheet(); for ( Sheet sheet : s.getSheets() ) { - if ( Objects.equals( sheet.getProperties().getTitle(), tableName ) ) { + if ( tableName.equalsIgnoreCase( sheet.getProperties().getTitle() ) ) { chosen_sheet = sheet; break; } } + if ( chosen_sheet.isEmpty() ) { + chosen_sheet = s.getSheets().get( 0 ); + } int queryStartRow = 1; while ( true ) { diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java index c0210c6f70..1b1e3ca967 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -74,10 +74,7 @@ public Table createGoogleSheetTable( CatalogTable catalogTable, List i ).toArray(); // build table and return later based on what you need for the table diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index eea33be87a..bea72fd051 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -16,21 +16,42 @@ package org.polypheny.db.adapter.googlesheet; +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; +import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.Details; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; +import java.io.File; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.polypheny.db.adapter.Adapter; +import org.polypheny.db.adapter.Adapter.AdapterProperties; +import org.polypheny.db.adapter.Adapter.AdapterSettingBoolean; +import org.polypheny.db.adapter.Adapter.AdapterSettingInteger; +import org.polypheny.db.adapter.Adapter.AdapterSettingString; import org.polypheny.db.adapter.DataSource; import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.catalog.entity.CatalogColumnPlacement; import org.polypheny.db.catalog.entity.CatalogPartitionPlacement; import org.polypheny.db.catalog.entity.CatalogTable; +import org.polypheny.db.config.RuntimeConfig; import org.polypheny.db.information.InformationGroup; import org.polypheny.db.information.InformationTable; import org.polypheny.db.prepare.Context; @@ -39,31 +60,46 @@ import org.polypheny.db.schema.Table; import org.polypheny.db.transaction.PolyXid; import org.polypheny.db.type.PolyType; +import org.polypheny.db.util.Pair; +import org.polypheny.db.util.PolyphenyHomeDirManager; @Slf4j -@Adapter.AdapterProperties( +@AdapterProperties( name = "GoogleSheets", description = "An adapter for querying online Google Sheets, using the Google Sheets Java API. Currently, this adapter only supports read operations.", usedModes = DeployMode.REMOTE) -@Adapter.AdapterSettingString(name = "sheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", position = 1) -@Adapter.AdapterSettingInteger(name = "maxStringLength", defaultValue = 255, position = 2, +@AdapterSettingString(name = "sheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1oDSHJkGIvgCfmBoi6oDr6EpjL5d9ntWwRApXCRlXxII/edit#gid=0", position = 1) +@AdapterSettingInteger(name = "maxStringLength", defaultValue = 255, position = 2, description = "Which length (number of characters including whitespace) should be used for the varchar columns. Make sure this is equal or larger than the longest string in any of the columns.") -@Adapter.AdapterSettingInteger(name = "querySize", defaultValue = 1000, position = 3, +@AdapterSettingInteger(name = "querySize", defaultValue = 1000, position = 3, description = "How many rows should be queried per network call. Can be larger than number of rows.") -@Adapter.AdapterSettingString(name = "resetRefreshToken", defaultValue = "No", position = 4, description = "If you want to change the current email used to access the Google Sheet, input \"YES\".") +@AdapterSettingBoolean(name = "resetRefreshToken", defaultValue = false, position = 4, description = "If you want to change the current email used to access the Google Sheet, input \"YES\".") +@AdapterSettingString(name = "oAuth-Client-ID", description = "Authentication credentials used for GoogleSheets API. Not the account credentials.", defaultValue = "", position = 5) +@AdapterSettingString(name = "oAuth-Client-Key", description = "Authentication credentials used for GoogleSheets API. Not the account credentials.", defaultValue = "") +@AdapterSettingString(name = "sheetName", description = "Name of sheet to use.", defaultValue = "") public class GoogleSheetSource extends DataSource { + static final String APPLICATION_NAME = "POLYPHENY GOOGLE SHEET"; + static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); + static final List SCOPES = Collections.singletonList( SheetsScopes.SPREADSHEETS_READONLY ); + public final String clientId; + public final String clientKey; + + public static final File TOKENS_PATH = PolyphenyHomeDirManager.getInstance().registerNewFolder( "tokens" ); + public final String sheet; private URL sheetsUrl; - private int querySize; + private final int querySize; private GoogleSheetSchema currentSchema; private final int maxStringLength; - private Map> exportedColumnCache; public GoogleSheetSource( final int storeId, final String uniqueName, final Map settings ) { super( storeId, uniqueName, settings, true ); + this.clientId = getSettingOrFail( "oAuth-Client-ID", settings ); + this.clientKey = getSettingOrFail( "oAuth-Client-Key", settings ); + this.sheet = getSettingOrFail( "sheetName", settings ); maxStringLength = Integer.parseInt( settings.get( "maxStringLength" ) ); if ( maxStringLength < 1 ) { throw new RuntimeException( "Invalid value for maxStringLength: " + maxStringLength ); @@ -77,12 +113,60 @@ public GoogleSheetSource( final int storeId, final String uniqueName, final Map< createInformationPage(); enableInformationPage(); if ( settings.get( "resetRefreshToken" ).equalsIgnoreCase( "yes" ) ) { - GoogleSheetReader r = new GoogleSheetReader( sheetsUrl, querySize ); + GoogleSheetReader r = new GoogleSheetReader( sheetsUrl, querySize, Pair.of( clientId, clientKey ) ); r.deleteToken(); } } + static Credential getCredentials( Pair oAuthIdKey, final NetHttpTransport HTTP_TRANSPORT ) throws IOException { + // Load client secrets. + GoogleClientSecrets clientSecrets = new GoogleClientSecrets(); + + Details details = new Details(); + details.set( "client_id", oAuthIdKey.getKey() ); + details.set( "client_secret", oAuthIdKey.getValue() ); + details.setRedirectUris( List.of( + "http://localhost:" + RuntimeConfig.WEBUI_SERVER_PORT, + "http://localhost:" + RuntimeConfig.WEBUI_SERVER_PORT + "/Callback", + "http://localhost:8888/Callback", + "http://localhost:8888" ) ); + details.set( "auth_provider_x509_cert_url", "https://www.googleapis.com/oauth2/v1/certs" ); + clientSecrets.setInstalled( details ); + + // Build flow and trigger user authorization request. + GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( + HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES ) + .setDataStoreFactory( new FileDataStoreFactory( TOKENS_PATH ) ) + .setAccessType( "offline" ) + .build(); + LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort( 8888 ).build(); + AuthorizationCodeInstalledApp auth = new AuthorizationCodeInstalledApp( flow, receiver ); + return auth.authorize( "user" ); + } + + + static Sheets getSheets( Pair oAuthIdKey ) { + final NetHttpTransport HTTP_TRANSPORT; + try { + HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + return new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( oAuthIdKey, HTTP_TRANSPORT ) ) + .setApplicationName( APPLICATION_NAME ) + .build(); + } catch ( GeneralSecurityException | IOException e ) { + throw new RuntimeException( e ); + } + } + + + private String getSettingOrFail( String key, Map settings ) { + if ( !settings.containsKey( key ) ) { + throw new RuntimeException( "Settings do not contain required key: " + key ); + } + return settings.get( key ); + } + + private void setSheetsUrl( Map settings ) { try { sheetsUrl = new URL( settings.get( "sheetsURL" ) ); @@ -100,7 +184,6 @@ protected void createInformationPage() { InformationTable table = new InformationTable( group, - // Arrays.asList( "Position", "Column Name", "Type", "Primary" ) ); Arrays.asList( "Position", "Column Name", "Type", "Nullable", "Filename", "Primary" ) ); for ( ExportedColumn exportedColumn : entry.getValue() ) { table.addRow( @@ -120,12 +203,9 @@ protected void createInformationPage() { @Override public Map> getExportedColumns() { - if ( exportedColumnCache != null ) { - return exportedColumnCache; - } Map> exportedColumnCache = new HashMap<>(); - GoogleSheetReader reader = new GoogleSheetReader( sheetsUrl, querySize ); + GoogleSheetReader reader = new GoogleSheetReader( sheetsUrl, querySize, Pair.of( clientId, clientKey ) ); HashMap> tablesData = reader.getTableSurfaceData(); for ( Map.Entry> entry : tablesData.entrySet() ) { diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index d6117a8373..a7f5407900 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -102,10 +102,10 @@ public AlgDataType getRowType( AlgDataTypeFactory typeFactory ) { public Enumerable project( final DataContext dataContext, final int[] fields ) { dataContext.getStatement().getTransaction().registerInvolvedAdapter( googleSheetSource ); final AtomicBoolean cancelFlag = DataContext.Variable.CANCEL_FLAG.get( dataContext ); - return new AbstractEnumerable() { + return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new GoogleSheetEnumerator<>( sheetsUrl, querySize, tableName, cancelFlag, fieldTypes, fields ); + return new GoogleSheetEnumerator<>( sheetsUrl, querySize, tableName, cancelFlag, fieldTypes, fields, Pair.of( googleSheetSource.clientId, googleSheetSource.clientKey ) ); } }; } diff --git a/plugins/google-sheet-adapter/src/main/resources/credentials.json b/plugins/google-sheet-adapter/src/main/resources/credentials.json deleted file mode 100644 index b79243655e..0000000000 --- a/plugins/google-sheet-adapter/src/main/resources/credentials.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "web": { - "client_id": "849988809941-kul5bmk9pks6qu1t57cdlfdum2jala3c.apps.googleusercontent.com", - "project_id": "precise-space-354614", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "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", - "redirect_uris": [ - "http://localhost:8080", - "http://localhost:8888/Callback", - "http://localhost:8888", - "http://localhost:8080/Callback" - ], - "javascript_origins": [ - "http://localhost" - ] - } -} \ No newline at end of file From faf1bd080970de5f877ff1b063d87fa8bf796a4a Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 27 Feb 2023 19:51:55 +0100 Subject: [PATCH 13/19] checked failing dependencies --- dbms/build.gradle | 3 ++- plugins/google-sheet-adapter/build.gradle | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/build.gradle b/dbms/build.gradle index 6ceb7529be..e591caa53d 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -52,7 +52,7 @@ dependencies { implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version // workaround for oauth, uses IOUtils... - implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version + implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version // Apache 2.0 // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") @@ -159,6 +159,7 @@ licensee { // transitive allowDependency('org.yaml', 'snakeyaml', '1.15') { because 'Apache 2.0 license' } + allowDependency('org.checkerframework', 'checker-compat-qual', '2.5.5') { because 'Dual License MIT ' } } compileJava { diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle index 032686491a..f1122c488b 100644 --- a/plugins/google-sheet-adapter/build.gradle +++ b/plugins/google-sheet-adapter/build.gradle @@ -86,6 +86,7 @@ licensee { allowDependency('org.javassist', 'javassist', '3.28.0-GA') { because 'Apache 2.0 license' } allowDependency('org.reflections', 'reflections', '0.10.2') { because 'Apache 2.0 license' } allowDependency('org.yaml', 'snakeyaml', '1.15') { because 'Apache 2.0 license' } + allowDependency('org.checkerframework', 'checker-compat-qual', '2.5.5') { because 'Dual License MIT ' } allowDependency('javax.servlet', 'javax.servlet-api', '3.1.0') { because 'Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original source code for this can be found at http://www.eclipse.org/jetty/downloads.php' From be0a26d35468292d2108ec90e92de99b6013be0c Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 27 Feb 2023 21:57:47 +0100 Subject: [PATCH 14/19] fixed error of IOUtils from google in googleSheets plugin, adjusted methods to be accessible within the plugin --- dbms/build.gradle | 2 +- gradle.properties | 2 +- .../googlesheet/GoogleSheetEnumerator.java | 9 +- .../googlesheet/GoogleSheetReader.java | 10 +- .../googlesheet/GoogleSheetSource.java | 26 +++-- .../adapter/googlesheet/GoogleSheetTable.java | 2 +- .../util/PolyphenyTokenStoreFactory.java | 106 ++++++++++++++++++ 7 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java diff --git a/dbms/build.gradle b/dbms/build.gradle index e591caa53d..27dff9088b 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -52,7 +52,7 @@ dependencies { implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version // workaround for oauth, uses IOUtils... - implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version // Apache 2.0 + //implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version // Apache 2.0 // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") diff --git a/gradle.properties b/gradle.properties index 532b574c84..694f365fdb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -105,7 +105,7 @@ natty_version = 0.13 okhttp_version = 4.2.2 opencsv_version = 2.3 oshi_core_version = 5.7.4 -oauth_client_version = 1.32.1 +oauth_client_version = 1.34.1 pigunit_version = 0.16.0 pig_version = 0.16.0 poi_version = 5.2.3 diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index f0e0d8f7b0..47da6db9d5 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -26,7 +26,6 @@ import org.apache.calcite.avatica.util.DateTimeUtils; import org.apache.calcite.linq4j.Enumerator; import org.apache.commons.lang3.time.FastDateFormat; -import org.polypheny.db.util.Pair; /** @@ -57,16 +56,16 @@ public class GoogleSheetEnumerator implements Enumerator { private E current; - GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, RowConverter rowConverter, Pair oAuthIdKey ) { + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, RowConverter rowConverter, GoogleSheetSource googleSheetSource ) { this.tableName = tableName; this.cancelFlag = cancelFlag; this.rowConverter = rowConverter; - this.reader = new GoogleSheetReader( sheetsUrl, querySize, oAuthIdKey ); + this.reader = new GoogleSheetReader( sheetsUrl, querySize, googleSheetSource ); } - GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, List fieldTypes, int[] fields, Pair oAuthIdKey ) { - this( sheetsUrl, querySize, tableName, cancelFlag, (RowConverter) converter( fieldTypes, fields ), oAuthIdKey ); + GoogleSheetEnumerator( URL sheetsUrl, int querySize, String tableName, AtomicBoolean cancelFlag, List fieldTypes, int[] fields, GoogleSheetSource googleSheetSource ) { + this( sheetsUrl, querySize, tableName, cancelFlag, (RowConverter) converter( fieldTypes, fields ), googleSheetSource ); } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index cfa5bdfd14..9613542255 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -48,16 +48,18 @@ public class GoogleSheetReader { private final HashMap tableStart = new HashMap<>(); private final HashMap tableLeftOffset = new HashMap<>(); private final HashMap enumPointer = new HashMap<>(); + private final GoogleSheetSource googleSheetSource; /** * @param url - url of the Google Sheet to source. * @param querySize - size of the query (in case of large files) */ - public GoogleSheetReader( URL url, int querySize, Pair oAuthIdKey ) { + public GoogleSheetReader( URL url, int querySize, GoogleSheetSource googleSheetSource ) { this.url = url; this.querySize = querySize; - this.oAuthIdKey = oAuthIdKey; + this.googleSheetSource = googleSheetSource; + this.oAuthIdKey = Pair.of( googleSheetSource.clientId, googleSheetSource.clientKey ); } @@ -71,7 +73,7 @@ private void readTableSurfaceData() { try { final String spreadsheetId = parseUrlToString( url ); - Sheets service = GoogleSheetSource.getSheets( oAuthIdKey ); + Sheets service = GoogleSheetSource.getSheets( oAuthIdKey, googleSheetSource ); // get the properties of all the sheets Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); @@ -150,7 +152,7 @@ private void readTable( String tableName ) { final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); final String spreadsheetId = parseUrlToString( url ); - Sheets service = new Sheets.Builder( HTTP_TRANSPORT, GoogleSheetSource.JSON_FACTORY, GoogleSheetSource.getCredentials( oAuthIdKey, HTTP_TRANSPORT ) ) + Sheets service = new Sheets.Builder( HTTP_TRANSPORT, GoogleSheetSource.JSON_FACTORY, GoogleSheetSource.getCredentials( oAuthIdKey, HTTP_TRANSPORT, googleSheetSource ) ) .setApplicationName( GoogleSheetSource.APPLICATION_NAME ) .build(); diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index bea72fd051..5a18cd3aff 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -26,7 +26,6 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.store.FileDataStoreFactory; import com.google.api.services.sheets.v4.Sheets; import com.google.api.services.sheets.v4.SheetsScopes; import java.io.File; @@ -41,6 +40,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.Adapter.AdapterProperties; import org.polypheny.db.adapter.Adapter.AdapterSettingBoolean; @@ -48,6 +49,7 @@ import org.polypheny.db.adapter.Adapter.AdapterSettingString; import org.polypheny.db.adapter.DataSource; import org.polypheny.db.adapter.DeployMode; +import org.polypheny.db.adapter.googlesheet.util.PolyphenyTokenStoreFactory; import org.polypheny.db.catalog.entity.CatalogColumnPlacement; import org.polypheny.db.catalog.entity.CatalogPartitionPlacement; import org.polypheny.db.catalog.entity.CatalogTable; @@ -93,6 +95,10 @@ public class GoogleSheetSource extends DataSource { private GoogleSheetSchema currentSchema; private final int maxStringLength; + @Getter + @Setter + Credential credentials; + public GoogleSheetSource( final int storeId, final String uniqueName, final Map settings ) { super( storeId, uniqueName, settings, true ); @@ -113,13 +119,16 @@ public GoogleSheetSource( final int storeId, final String uniqueName, final Map< createInformationPage(); enableInformationPage(); if ( settings.get( "resetRefreshToken" ).equalsIgnoreCase( "yes" ) ) { - GoogleSheetReader r = new GoogleSheetReader( sheetsUrl, querySize, Pair.of( clientId, clientKey ) ); + GoogleSheetReader r = new GoogleSheetReader( sheetsUrl, querySize, this ); r.deleteToken(); } } - static Credential getCredentials( Pair oAuthIdKey, final NetHttpTransport HTTP_TRANSPORT ) throws IOException { + static Credential getCredentials( Pair oAuthIdKey, final NetHttpTransport HTTP_TRANSPORT, GoogleSheetSource googleSheetSource ) throws IOException { + if ( googleSheetSource.getCredentials() != null ) { + return googleSheetSource.getCredentials(); + } // Load client secrets. GoogleClientSecrets clientSecrets = new GoogleClientSecrets(); @@ -137,20 +146,21 @@ static Credential getCredentials( Pair oAuthIdKey, final NetHttp // Build flow and trigger user authorization request. GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( HTTP_TRANSPORT, JSON_FACTORY, clientSecrets, SCOPES ) - .setDataStoreFactory( new FileDataStoreFactory( TOKENS_PATH ) ) + .setDataStoreFactory( new PolyphenyTokenStoreFactory( TOKENS_PATH ) )// own store, due to plugin system .setAccessType( "offline" ) .build(); LocalServerReceiver receiver = new LocalServerReceiver.Builder().setPort( 8888 ).build(); AuthorizationCodeInstalledApp auth = new AuthorizationCodeInstalledApp( flow, receiver ); - return auth.authorize( "user" ); + googleSheetSource.setCredentials( auth.authorize( "user" ) ); + return googleSheetSource.getCredentials(); } - static Sheets getSheets( Pair oAuthIdKey ) { + static Sheets getSheets( Pair oAuthIdKey, GoogleSheetSource googleSheetSource ) { final NetHttpTransport HTTP_TRANSPORT; try { HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - return new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( oAuthIdKey, HTTP_TRANSPORT ) ) + return new Sheets.Builder( HTTP_TRANSPORT, JSON_FACTORY, getCredentials( oAuthIdKey, HTTP_TRANSPORT, googleSheetSource ) ) .setApplicationName( APPLICATION_NAME ) .build(); } catch ( GeneralSecurityException | IOException e ) { @@ -205,7 +215,7 @@ protected void createInformationPage() { public Map> getExportedColumns() { Map> exportedColumnCache = new HashMap<>(); - GoogleSheetReader reader = new GoogleSheetReader( sheetsUrl, querySize, Pair.of( clientId, clientKey ) ); + GoogleSheetReader reader = new GoogleSheetReader( sheetsUrl, querySize, this ); HashMap> tablesData = reader.getTableSurfaceData(); for ( Map.Entry> entry : tablesData.entrySet() ) { diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index a7f5407900..7a053ff062 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -105,7 +105,7 @@ public Enumerable project( final DataContext dataContext, final int[] fi return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new GoogleSheetEnumerator<>( sheetsUrl, querySize, tableName, cancelFlag, fieldTypes, fields, Pair.of( googleSheetSource.clientId, googleSheetSource.clientKey ) ); + return new GoogleSheetEnumerator<>( sheetsUrl, querySize, tableName, cancelFlag, fieldTypes, fields, googleSheetSource ); } }; } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java new file mode 100644 index 0000000000..7e86bfa2b3 --- /dev/null +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.googlesheet.util; + +import com.google.api.client.util.store.AbstractDataStore; +import com.google.api.client.util.store.DataStore; +import com.google.api.client.util.store.DataStoreFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Collection; +import java.util.HashMap; +import java.util.Set; + +/** + * This class serves as a temporary store for the token of the GoogleSheet API. + * Could be turned into a file store later on. + * Default store leads to errors as plugin, as it uses IOUtils of Google, which forces default Classloader, which cannot find this plugin. + */ +public class PolyphenyTokenStoreFactory extends FileDataStoreFactory { + + /** + * @param dataDirectory data directory + */ + public PolyphenyTokenStoreFactory( File dataDirectory ) throws IOException { + super( dataDirectory ); + } + + + @Override + protected DataStore createDataStore( String id ) throws IOException { + return new PolyTokenStore<>( this, id ); + } + + + public static class PolyTokenStore extends AbstractDataStore { + + private final HashMap map; + + + /** + * @param dataStoreFactory data store factory + * @param id data store ID + */ + protected PolyTokenStore( DataStoreFactory dataStoreFactory, String id ) { + super( dataStoreFactory, id ); + this.map = new HashMap<>(); + } + + + @Override + public Set keySet() { + return map.keySet(); + } + + + @Override + public Collection values() { + return map.values(); + } + + + @Override + public V get( String key ) { + return map.get( key ); + } + + + @Override + public DataStore set( String key, V value ) { + map.put( key, value ); + return this; + } + + + @Override + public DataStore clear() { + map.clear(); + return this; + } + + + @Override + public DataStore delete( String key ) { + map.remove( key ); + return this; + } + + } + +} From 89da05231d2b33f90af199673876f38a694703f9 Mon Sep 17 00:00:00 2001 From: datomo Date: Mon, 27 Feb 2023 22:02:11 +0100 Subject: [PATCH 15/19] removed unnecessary workaround from dbms --- dbms/build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbms/build.gradle b/dbms/build.gradle index 27dff9088b..62bda4ab91 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -51,9 +51,6 @@ dependencies { // workaround to fix cottontail todo remove: when grpc-all is same in cottontail and for plugin implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version - // workaround for oauth, uses IOUtils... - //implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version // Apache 2.0 - // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") @@ -159,7 +156,6 @@ licensee { // transitive allowDependency('org.yaml', 'snakeyaml', '1.15') { because 'Apache 2.0 license' } - allowDependency('org.checkerframework', 'checker-compat-qual', '2.5.5') { because 'Dual License MIT ' } } compileJava { From 972d6a3b0d60190060105bd08ba5733c09f269aa Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Tue, 28 Feb 2023 13:43:13 +0100 Subject: [PATCH 16/19] Minor adjustments and clean-up --- dbms/build.gradle | 1 + .../db/adapter/GoogleSheetSourceTest.java | 4 +- .../polypheny/db/adapter/MysqlSourceTest.java | 1 - google-sheet-adapter/lombok.config | 4 - .../monitoring/core/MonitoringQueueImpl.java | 18 ++-- .../java/org/polypheny/db/test/CsvTest.java | 1 + plugins/google-sheet-adapter/README.md | 83 ------------------- plugins/google-sheet-adapter/build.gradle | 3 +- .../googlesheet/GoogleSheetPlugin.java | 1 - .../GoogleSheetProjectTableScanRule.java | 6 +- .../googlesheet/GoogleSheetSchema.java | 1 - .../util/PolyphenyTokenStoreFactory.java | 1 + 12 files changed, 18 insertions(+), 106 deletions(-) delete mode 100644 google-sheet-adapter/lombok.config delete mode 100644 plugins/google-sheet-adapter/README.md diff --git a/dbms/build.gradle b/dbms/build.gradle index 62bda4ab91..72bfa7c84f 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -51,6 +51,7 @@ dependencies { // workaround to fix cottontail todo remove: when grpc-all is same in cottontail and for plugin implementation group: "io.grpc", name: "grpc-all", version: cottontaildb_grpc_version + // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index b60aa3c512..3755026c68 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -31,6 +31,7 @@ import org.polypheny.db.TestHelper; import org.polypheny.db.TestHelper.JdbcConnection; + @Ignore // for debug purpose public class GoogleSheetSourceTest { @@ -126,6 +127,3 @@ public void existsThirdSelect() throws SQLException { } } - - - diff --git a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java index b62effd146..b348501ba5 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/MysqlSourceTest.java @@ -66,7 +66,6 @@ public static void start() throws SQLException { statement.executeUpdate( "ALTER ADAPTERS ADD mariadbunit USING 'Mysql' AS 'Source' WITH '" + gson.toJson( settings ) + "'" ); } } - System.out.println("RAN HERE"); } diff --git a/google-sheet-adapter/lombok.config b/google-sheet-adapter/lombok.config deleted file mode 100644 index b034fcb60e..0000000000 --- a/google-sheet-adapter/lombok.config +++ /dev/null @@ -1,4 +0,0 @@ -# This file is generated by the 'io.freefair.lombok' Gradle plugin -config.stopBubbling = true -lombok.val.flagUsage = error -lombok.var.flagUsage = error diff --git a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java index e0c7d6574a..83232228f1 100644 --- a/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java +++ b/monitoring/src/main/java/org/polypheny/db/monitoring/core/MonitoringQueueImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The Polypheny Project + * Copyright 2019-2023 The Polypheny Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -88,22 +88,23 @@ public MonitoringQueueImpl( eventQueue ); } - // instantiated thread count + // Instantiated thread count this.threadCount = threadPoolWorkers.getPoolSize(); - // create a scheduled, separate thread which gets new thread count every 500 milliseconds + // Create a scheduled, separate thread which gets new thread count every 500 milliseconds Timer timer = new Timer(); - timer.scheduleAtFixedRate(new TimerTask() { + timer.scheduleAtFixedRate( new TimerTask() { @Override public synchronized void run() { int newThreadCount = threadPoolWorkers.getPoolSize(); - - if (newThreadCount != threadCount) { + if ( newThreadCount != threadCount ) { threadCount = newThreadCount; - log.debug( "Thread count is now: {}", threadCount ); + if ( log.isDebugEnabled() ) { + log.debug( "Thread count is now: {}", threadCount ); + } } } - }, 500, 500); + }, 500, 500 ); } @@ -255,5 +256,4 @@ private void processQueue() { } - } diff --git a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java index 553bea4c40..86f4734caa 100644 --- a/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java +++ b/plugins/csv-adapter/src/test/java/org/polypheny/db/test/CsvTest.java @@ -79,6 +79,7 @@ private static String escapeString( String s ) { return escapeString( new StringBuilder(), s ).toString(); } + /** * Quotes a string for Java or JSON, into a builder. */ diff --git a/plugins/google-sheet-adapter/README.md b/plugins/google-sheet-adapter/README.md deleted file mode 100644 index 559d2f5ace..0000000000 --- a/plugins/google-sheet-adapter/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# For user website - ---- -layout: plain -title: CSV Adapter ---- - -The Google Sheet Adapter is a data-source adapter that allows data to be sourced from any Google Sheet URL into Polypheny -as relational tables. The adapter is read-only, with DML queries not supported. -The content of the google sheets can be changed in the background as long -as the names of the files and the columns itself don't change. - -The first column will always be the primary key. The Adapter also does not -support null values - -For formatting, the adapter itself adjusts to different formatting, but keep in mind there may be issues in the following cases: - -- There are two tables nested within the same sheet in the URL (will cause error) -- There is a gap between the table rows (the adapter won't read after the gap) - -Otherwise, as long as the email used has access to the sheet, there should be no issues. - -## Adapter settings - -The Google Sheet Source Adapter has the following settings: - -| Name | Description | -|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Name | Unique name for Google Sheet URL tables | -| sheetsURL | the Google Sheet URL to source from | -| maxStringLength | the maximum size of the strings in all tables | -| querySize | the number of rows scanned per network-call with the GoogleSheet API | -| resetRefreshToken | type "Yes" to delete the current refresh token associated to the current email used to read from Google Sheets and use another account for the Google Sheet Adapter. "No" is default. | - - - -## Supported Data Types - -It should be noted that all columns in the GoogleSheetAdapters are of type string, unless the columns of the sheet -are specially formatted. More specifically, all columns will have type string unless the column name is formatted as -`columnName:type`, in which case the type of column will be the specified type. Types can be one of the following: `int, string, boolean, long, float, double, date, time, timestamp`. - -For example: -- If the Google Sheet has `user_id:int` as the column name, the imported column name would be `user_id` and the column type would be `int` -- If the Google Sheet has `birthday:date` as the column, then the imported column name would be `birthday` and the column type would be `date` -- If the Google Sheet has `age` as the column name, then even if all values in the column are of type `int`, it would still be a `string` when imported -into Polypheny - -## Deployment - -The adapter can be deployed using Polypheny-UI (Adapters -> Sources), or using the following SQL statement: - - -{% highlight sql %} ALTER ADAPTERS ADD unique_name USING 'org.polypheny.db.adapter.googlesheet.GoogleSheetSource' WITH -'{maxStringLength: "255", querySize: "1000", sheetsURL: "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0", -mode: "remote", resetRefreshToken: "No"}' {% highlight sql %} - -Please refer to the settings above and alter the maxStringLength, querySize, ... as needed. After successful deployment, all -Google Sheets are mapped as a table in the public schema. - - ---- - -# For future development - -**Token management** - -- The application currently runs using the Google Cloud Console under a private email's project. Please create a new project -under the Polypheny email, enable the Google Sheet API, and add an OAUTH credential under the "Credentials" section. The credential -should be configured with origins as needed, with a http://localhost:8888/Callback and http://localhost:8888 redirect URL. - (http://localhost:8888 is used because this is the port in which the GoogleSheetReader opens up to receive the access tokens -from OAUTH. This can be changed as needed.) Then, download the credentials and add it to `google-sheet-adapter/src/main/resources/credentials.json`. - -- Finally, there is the unhandled issue of expiring refresh tokens roughly after 1 week (which can cause errors to the users). -Users can fix this manually with the resetRefreshToken option in the settings. However, to improve the experience and -unlimit the number of API calls to Google Sheets servers, it is necessary to go to OAuth Consent Screen, and "Publish App" -which then extends refresh tokens for infinite use along with removes the API call limit. - -**Improvements which can be made** - -- Splitting GoogleSheetTableScanProject -> Scan and Project classes separately. - -- Moving the GoogleSheetSourceTest from `dbms/src/test/java/org.polypheny.db/adapter/` to google-sheet-adapter package. diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle index f1122c488b..c7f4fe4be5 100644 --- a/plugins/google-sheet-adapter/build.gradle +++ b/plugins/google-sheet-adapter/build.gradle @@ -10,7 +10,8 @@ dependencies { implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' -// --- Test Compile --- + + // --- Test Compile --- testImplementation project(path: ":core", configuration: "tests") testImplementation group: "junit", name: "junit", version: junit_version diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java index ee1b43e47b..98a88e09db 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java @@ -46,5 +46,4 @@ public void start() { Adapter.addAdapter( GoogleSheetSource.class, "GOOGLESHEETS", settings ); } - } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java index 2f487f4acb..20d316b360 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetProjectTableScanRule.java @@ -27,7 +27,8 @@ /** - * Planner rule that projects from a {@GoogleTable} scan just the columns needed to satisfy a projection. If the projection's expressions are trivial, the projection is removed. + * Planner rule that projects from a {@GoogleTable} scan just the columns needed to satisfy a projection. If the + * projection's expressions are trivial, the projection is removed. */ public class GoogleSheetProjectTableScanRule extends AlgOptRule { @@ -52,8 +53,7 @@ public void onMatch( AlgOptRuleCall call ) { // Project contains expressions more complex than just field references. return; } - call.transformTo( - new GoogleSheetTableScanProject( scan.getCluster(), scan.getTable(), scan.googleSheetTable, fields ) ); + call.transformTo( new GoogleSheetTableScanProject( scan.getCluster(), scan.getTable(), scan.googleSheetTable, fields ) ); } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java index 1b1e3ca967..fcf7e37278 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSchema.java @@ -16,7 +16,6 @@ package org.polypheny.db.adapter.googlesheet; - import java.net.URL; import java.util.ArrayList; import java.util.HashMap; diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java index 7e86bfa2b3..826822286a 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/util/PolyphenyTokenStoreFactory.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Set; + /** * This class serves as a temporary store for the token of the GoogleSheet API. * Could be turned into a file store later on. From 1e2fd663298c713f587208897a3520e5359dfd94 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Tue, 28 Feb 2023 13:59:48 +0100 Subject: [PATCH 17/19] Improve dependency handling, clean-up license check task and remove URL of test sheet --- .../db/adapter/GoogleSheetSourceTest.java | 2 +- gradle.properties | 2 + plugins/google-sheet-adapter/build.gradle | 53 ++----------------- .../googlesheet/GoogleSheetSource.java | 2 +- 4 files changed, 7 insertions(+), 52 deletions(-) diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index 3755026c68..229d11f899 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -45,7 +45,7 @@ public static void start() throws SQLException { Map settings = new HashMap<>(); settings.put( "maxStringLength", "255" ); settings.put( "querySize", "1000" ); - settings.put( "sheetsURL", "https://docs.google.com/spreadsheets/d/1-int7xwx0UyyigB4FLGMOxaCiuHXSNhi09fYSuAIX2Q/edit#gid=0" ); + settings.put( "sheetsURL", "-----ADD-----" ); settings.put( "mode", "remote" ); settings.put( "resetRefreshToken", "No" ); Gson gson = new Gson(); diff --git a/gradle.properties b/gradle.properties index 694f365fdb..8625dbc538 100644 --- a/gradle.properties +++ b/gradle.properties @@ -59,6 +59,8 @@ foodmart_data_hsqldb_version = 0.3 foodmart_data_json_version = 0.4 foodmart_queries_version = 0.4.1 geode_core_version = 1.6.0 +google_api_client_version = 1.33.0 +google_api_sheets_version = v4-rev20210629-1.32.1 gradle_test_logger_version = 2.1.1 gradle_lint_plugin = 17.7.1 gson_version = 2.8.9 diff --git a/plugins/google-sheet-adapter/build.gradle b/plugins/google-sheet-adapter/build.gradle index c7f4fe4be5..106c901a78 100644 --- a/plugins/google-sheet-adapter/build.gradle +++ b/plugins/google-sheet-adapter/build.gradle @@ -6,9 +6,9 @@ dependencies { compileOnly project(":plugins:sql-language") compileOnly project(":dbms") - implementation 'com.google.api-client:google-api-client:1.33.0' - implementation group: 'com.google.oauth-client', name: 'google-oauth-client-jetty', version: oauth_client_version - implementation 'com.google.apis:google-api-services-sheets:v4-rev20210629-1.32.1' + implementation group: "com.google.api-client", name: "google-api-client", version: google_api_client_version + implementation group: "com.google.oauth-client", name: "google-oauth-client-jetty", version: oauth_client_version + implementation group: "com.google.apis", name: "google-api-services-sheets", version: google_api_sheets_version // --- Test Compile --- @@ -45,51 +45,4 @@ java { licensee { allow('Apache-2.0') allow('MIT') - allow('BSD-3-Clause') - allow('BSD-2-Clause') - allow('MPL-1.1') - allow('CC0-1.0') // Public Domain Dedication - - allowUrl('http://www.eclipse.org/legal/epl-2.0') // EPL 2.0 - allowUrl('http://www.opensource.org/licenses/Apache-2.0') // Apache 2.0 - allowUrl('http://www.jcabi.com/LICENSE.txt') // Own licensee but conforms - allowUrl('https://www.eclipse.org/org/documents/epl-v10.php') // EPL 1.0 - allowUrl('https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt') // EPL 2.0 - allowUrl('http://www.eclipse.org/org/documents/edl-v10.php') // EDL 1.0 - - allowUrl('http://asm.ow2.org/license.html') // because BSD 3-Clause - allowUrl('https://raw.githubusercontent.com/janino-compiler/janino/master/LICENSE') // open janino license - - - allowDependency('com.adobe.xmp', 'xmpcore', '6.0.6') { because 'BSD 3-Clause' } - allowDependency('com.github.jnr', 'jnr-posix', '3.0.50') { because 'Eclipse Public License v. 2.0' } - allowDependency('com.j256.simplemagic', 'simplemagic', '1.16') { because 'ISC license' } - allowDependency('jakarta.annotation', 'jakarta.annotation-api', '1.3.5') { because 'Eclipse Public License v. 2.0' } - allowDependency('jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.2') { because 'Eclipse Distribution License 1.0' } - allowDependency('jakarta.activation', 'jakarta.activation-api', '1.2.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('jakarta.ws.rs', 'jakarta.ws.rs-api', '2.1.6') { because 'Eclipse Public License v. 2.0' } - allowDependency('net.java.dev.jna', 'jna', '5.8.0') { because 'Apache 2.0 license' } - - allowDependency('org.bouncycastle', 'bcpkix-jdk15on', '1.64') { because 'MIT license' } - allowDependency('org.bouncycastle', 'bcprov-jdk15on', '1.64') { because 'MIT license' } - - allowDependency('org.glassfish.jersey.inject', 'jersey-hk2', '2.30.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2.external', 'jakarta.inject', '2.6.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2.external', 'aopalliance-repackaged', '2.6.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2', 'osgi-resource-locator', '1.0.3') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2', 'hk2-api', '2.6.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2', 'hk2-locator', '2.6.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.hk2', 'hk2-utils', '2.6.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.jersey.core', 'jersey-common', '2.30.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.jersey.core', 'jersey-client', '2.30.1') { because 'Eclipse Public License v. 2.0' } - allowDependency('org.glassfish.jersey.connectors', 'jersey-apache-connector', '2.30.1') { because 'Eclipse Public License v. 2.0' } - - allowDependency('org.javassist', 'javassist', '3.28.0-GA') { because 'Apache 2.0 license' } - allowDependency('org.reflections', 'reflections', '0.10.2') { because 'Apache 2.0 license' } - allowDependency('org.yaml', 'snakeyaml', '1.15') { because 'Apache 2.0 license' } - allowDependency('org.checkerframework', 'checker-compat-qual', '2.5.5') { because 'Dual License MIT ' } - - allowDependency('javax.servlet', 'javax.servlet-api', '3.1.0') { - because 'Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original source code for this can be found at http://www.eclipse.org/jetty/downloads.php' - } } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index 5a18cd3aff..9c5e689a47 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -71,7 +71,7 @@ name = "GoogleSheets", description = "An adapter for querying online Google Sheets, using the Google Sheets Java API. Currently, this adapter only supports read operations.", usedModes = DeployMode.REMOTE) -@AdapterSettingString(name = "sheetsURL", description = "The URL of the Google Sheets to query", defaultValue = "https://docs.google.com/spreadsheets/d/1oDSHJkGIvgCfmBoi6oDr6EpjL5d9ntWwRApXCRlXxII/edit#gid=0", position = 1) +@AdapterSettingString(name = "sheetsURL", description = "The URL of the Google Sheet to query.", defaultValue = "", position = 1) @AdapterSettingInteger(name = "maxStringLength", defaultValue = 255, position = 2, description = "Which length (number of characters including whitespace) should be used for the varchar columns. Make sure this is equal or larger than the longest string in any of the columns.") @AdapterSettingInteger(name = "querySize", defaultValue = 1000, position = 3, From 446c793b265e7b47bb483c9031d603f61fd44b22 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Tue, 28 Feb 2023 14:02:20 +0100 Subject: [PATCH 18/19] Further clean-up --- .../polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java | 1 - .../polypheny/db/adapter/googlesheet/GoogleSheetTable.java | 5 ----- 2 files changed, 6 deletions(-) diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java index 98a88e09db..8bb93840d1 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java @@ -16,7 +16,6 @@ package org.polypheny.db.adapter.googlesheet; - import java.util.HashMap; import java.util.Map; import org.pf4j.Plugin; diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java index 7a053ff062..bdc4158785 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetTable.java @@ -16,11 +16,6 @@ package org.polypheny.db.adapter.googlesheet; -/** - * Questions: - * - What is protoRowType? - */ - import java.lang.reflect.Type; import java.net.URL; import java.util.ArrayList; From f70ac0c96c9db1e751b75bc5845df8c53fe4f1f0 Mon Sep 17 00:00:00 2001 From: Marco Vogt Date: Tue, 28 Feb 2023 14:33:28 +0100 Subject: [PATCH 19/19] Minor code style adjustments --- .../db/adapter/GoogleSheetSourceTest.java | 2 +- .../googlesheet/GoogleSheetEnumerator.java | 16 +++++++++---- .../googlesheet/GoogleSheetPlugin.java | 1 + .../googlesheet/GoogleSheetReader.java | 23 ++++++++----------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java index 229d11f899..2ece83abd4 100644 --- a/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java +++ b/dbms/src/test/java/org/polypheny/db/adapter/GoogleSheetSourceTest.java @@ -49,7 +49,7 @@ public static void start() throws SQLException { settings.put( "mode", "remote" ); settings.put( "resetRefreshToken", "No" ); Gson gson = new Gson(); - statement.executeUpdate( "ALTER ADAPTERS ADD \"googlesheetunit\" USING 'GoogleSheets' AS 'Source' WITH '" + gson.toJson( settings ) + "'" ); + statement.executeUpdate( "ALTER ADAPTERS ADD \"googlesheetunit\" USING 'GoogleSheets' AS SOURCE WITH '" + gson.toJson( settings ) + "'" ); } } } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java index 47da6db9d5..92f8c55228 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetEnumerator.java @@ -109,7 +109,7 @@ public void reset() { /** - * Doesn't need to do any thing to close + * Doesn't need to do anything to close */ @Override public void close() { @@ -136,37 +136,43 @@ protected Object convert( GoogleSheetFieldType fieldType, String string ) { return null; } return Boolean.parseBoolean( string ); + case BYTE: if ( string.length() == 0 ) { return null; } return Byte.parseByte( string ); + case SHORT: if ( string.length() == 0 ) { return null; } return Short.parseShort( string ); + case INT: if ( string.length() == 0 ) { return null; } return Integer.parseInt( string ); + case LONG: if ( string.length() == 0 ) { return null; } - return new BigInteger( string ); + case FLOAT: if ( string.length() == 0 ) { return null; } return Float.parseFloat( string ); + case DOUBLE: if ( string.length() == 0 ) { return null; } return Double.parseDouble( string ); + case DATE: if ( string.length() == 0 ) { return null; @@ -177,6 +183,7 @@ protected Object convert( GoogleSheetFieldType fieldType, String string ) { } catch ( ParseException e ) { return null; } + case TIME: if ( string.length() == 0 ) { return null; @@ -187,6 +194,7 @@ protected Object convert( GoogleSheetFieldType fieldType, String string ) { } catch ( ParseException e ) { return null; } + case TIMESTAMP: if ( string.length() == 0 ) { return null; @@ -197,6 +205,7 @@ protected Object convert( GoogleSheetFieldType fieldType, String string ) { } catch ( Exception e ) { return null; } + case STRING: default: return string; @@ -213,7 +222,7 @@ static class ArrayRowConverter extends RowConverter { private final GoogleSheetFieldType[] fieldTypes; private final int[] fields; - // whether the row to convert is from a stream + // Whether the row to convert is from a stream private final boolean stream; @@ -250,7 +259,6 @@ public Object[] convertNormalRow( String[] strings ) { } else { objects[i] = convert( fieldTypes[i], "" ); } - } return objects; } diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java index 8bb93840d1..30a1105e29 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetPlugin.java @@ -22,6 +22,7 @@ import org.pf4j.PluginWrapper; import org.polypheny.db.catalog.Adapter; + public class GoogleSheetPlugin extends Plugin { /** diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java index 9613542255..b0c582ec91 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetReader.java @@ -39,7 +39,6 @@ */ public class GoogleSheetReader { - private final URL url; private final int querySize; private final Pair oAuthIdKey; @@ -52,8 +51,8 @@ public class GoogleSheetReader { /** - * @param url - url of the Google Sheet to source. - * @param querySize - size of the query (in case of large files) + * @param url URL of the Google Sheet to source. + * @param querySize Size of the query (in case of large files) */ public GoogleSheetReader( URL url, int querySize, GoogleSheetSource googleSheetSource ) { this.url = url; @@ -71,11 +70,10 @@ private void readTableSurfaceData() { return; } try { - final String spreadsheetId = parseUrlToString( url ); Sheets service = GoogleSheetSource.getSheets( oAuthIdKey, googleSheetSource ); - // get the properties of all the sheets + // Get the properties of all the sheets Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); for ( Sheet sheet : s.getSheets() ) { @@ -131,7 +129,6 @@ public void deleteToken() { if ( file.exists() ) { file.delete(); } - } @@ -156,7 +153,7 @@ private void readTable( String tableName ) { .setApplicationName( GoogleSheetSource.APPLICATION_NAME ) .build(); - // first query! let's start searching till we find first row. + // First query! let's start searching till we find first row. if ( !tableStart.containsKey( tableName ) ) { Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); // get all the sheets Sheet chosen_sheet = new Sheet(); @@ -172,7 +169,7 @@ private void readTable( String tableName ) { int queryStartRow = 1; while ( true ) { - // nothing in sheet + // Nothing in sheet if ( queryStartRow > chosen_sheet.getProperties().getGridProperties().getRowCount() ) { tableStart.put( tableName, -1 ); return; @@ -208,7 +205,7 @@ private void readTable( String tableName ) { } // TODO: optimize this - // add the remaining data to tableData + // Add the remaining data to tableData for ( int i = firstRowIndex; i < values.size(); i++ ) { List fullRow = values.get( i ); List trueRow = fullRow.subList( tableLeftOffset.get( tableName ), fullRow.size() ); @@ -223,12 +220,10 @@ private void readTable( String tableName ) { } break; - } } else if ( tableStart.get( tableName ) == -1 ) { // end of document already return; } else { // begin getting from - Spreadsheet s = service.spreadsheets().get( spreadsheetId ).execute(); // get all the sheets Sheet chosen_sheet = new Sheet(); for ( Sheet sheet : s.getSheets() ) { @@ -265,7 +260,7 @@ private void readTable( String tableName ) { } tableData.put( tableName, currData ); - // final check: if the current data that we have is smaller than our query, we reached + // Final check: if the current data that we have is smaller than our query, we reached // the end, so we set our start to -1. if ( values.size() < querySize ) { tableStart.put( tableName, -1 ); @@ -288,7 +283,7 @@ public String[] readNext( String tableName ) { readTable( tableName ); } - // still true, so we've reached the end of the google sheet + // Still true, so we've reached the end of the google sheet if ( tableData.get( tableName ).size() <= enumPointer.get( tableName ) ) { return null; } @@ -309,4 +304,4 @@ public HashMap>> getTableData() { return tableData; } -} \ No newline at end of file +}