From 55f3f961e2a6301cdb2fd493af373f8fd2f3059d Mon Sep 17 00:00:00 2001 From: Martin Vanek Date: Thu, 7 Mar 2024 19:08:34 +0000 Subject: [PATCH 1/3] feat: Add short, integer, long, boolean conversions into string --- .../storage/v1/JsonToProtoMessage.java | 6 +++ .../storage/v1/JsonStreamWriterTest.java | 16 ++++---- .../storage/v1/JsonToProtoMessageTest.java | 38 ++++++++++++++----- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 6cde31081a..be785d0fad 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -650,6 +650,9 @@ private void fillField( if (val instanceof String) { protoMsg.setField(fieldDescriptor, val); return; + } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + protoMsg.setField(fieldDescriptor, String.valueOf(val)); + return; } break; case DOUBLE: @@ -910,6 +913,9 @@ private void fillRepeatedField( case STRING: if (val instanceof String) { protoMsg.addRepeatedField(fieldDescriptor, val); + } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + protoMsg.addRepeatedField(fieldDescriptor, String.valueOf(val)); + return; } else { throwWrongFieldType(fieldDescriptor, currentScope, index); } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index d3a25510f8..a9d9b8bd3c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -37,6 +37,7 @@ import com.google.cloud.bigquery.storage.v1.ConnectionWorkerPool.Settings; import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializationError; import com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode; +import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; @@ -1406,8 +1407,8 @@ public void testMultipleAppendSerializationErrors() // put a vaild value into the field foo1.put("foo", "allen"); JSONObject foo2 = new JSONObject(); - // put a number into a string field - foo2.put("foo", 666); + // put a field which is not part of the expected schema + foo2.put("not_bar", "woody"); JSONArray jsonArr = new JSONArray(); jsonArr.put(foo); jsonArr.put(foo1); @@ -1425,14 +1426,11 @@ public void testMultipleAppendSerializationErrors() } catch (AppendSerializationError appendSerializationError) { Map rowIndexToErrorMessage = appendSerializationError.getRowIndexToErrorMessage(); - assertEquals(2, rowIndexToErrorMessage.size()); - assertEquals( - "The source object has fields unknown to BigQuery: root.not_foo.", - rowIndexToErrorMessage.get(0)); assertEquals( - "Field root.foo failed to convert to STRING. Error: JSONObject does not have a string" - + " field at root.foo.", - rowIndexToErrorMessage.get(2)); + ImmutableMap.of( + 0, "The source object has fields unknown to BigQuery: root.not_foo.", + 2, "The source object has fields unknown to BigQuery: root.not_bar." + ), rowIndexToErrorMessage); } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index dd3a6dcfa1..3f1123833b 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -87,7 +87,12 @@ public class JsonToProtoMessageTest { }) .put( StringType.getDescriptor(), - new Message[] {StringType.newBuilder().setTestFieldType("test").build()}) + new Message[] { + StringType.newBuilder().setTestFieldType("9223372036854775807").build(), + StringType.newBuilder().setTestFieldType("2147483647").build(), + StringType.newBuilder().setTestFieldType("true").build(), + StringType.newBuilder().setTestFieldType("test").build() + }) .put( RepeatedType.getDescriptor(), new Message[] { @@ -147,6 +152,9 @@ public class JsonToProtoMessageTest { .put( RepeatedString.getDescriptor(), new Message[] { + RepeatedString.newBuilder().addTestRepeated("9223372036854775807").build(), + RepeatedString.newBuilder().addTestRepeated("2147483647").build(), + RepeatedString.newBuilder().addTestRepeated("true").build(), RepeatedString.newBuilder().addTestRepeated("hello").addTestRepeated("test").build() }) .put( @@ -925,6 +933,8 @@ public void testAllTypes() throws Exception { } else if (entry.getKey() == Int64Type.getDescriptor() || entry.getKey() == BytesType.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); + } else if(entry.getKey() == StringType.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 4, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); } @@ -962,6 +972,8 @@ public void testAllRepeatedTypesWithLimits() throws Exception { assertEquals(entry.getKey().getFullName(), 4, success); } else if (entry.getKey() == RepeatedInt64.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); + } else if (entry.getKey() == RepeatedString.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 4, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); } @@ -1007,16 +1019,22 @@ public void testRequired() throws Exception { } } - @Test - public void testStructSimple() throws Exception { + @Test + public void testStructSimple() throws Exception { + structSimple("test", "test"); + structSimple(true, "true"); + structSimple(1, "1"); + structSimple((short) 1, "1"); + structSimple((long) 1, "1"); + } + + private void structSimple(Object value, String expected) throws Exception { MessageType expectedProto = MessageType.newBuilder() - .setTestFieldType(StringType.newBuilder().setTestFieldType("test").build()) + .setTestFieldType(StringType.newBuilder().setTestFieldType(expected).build()) .build(); - JSONObject stringType = new JSONObject(); - stringType.put("test_field_type", "test"); - JSONObject json = new JSONObject(); - json.put("test_field_type", stringType); + JSONObject stringType = new JSONObject(ImmutableMap.of("test_field_type", value)); + JSONObject json = new JSONObject(ImmutableMap.of("test_field_type", stringType)); DynamicMessage protoMsg = JsonToProtoMessage.INSTANCE.convertToProtoMessage(MessageType.getDescriptor(), json); @@ -1026,7 +1044,7 @@ public void testStructSimple() throws Exception { @Test public void testStructSimpleFail() throws Exception { JSONObject stringType = new JSONObject(); - stringType.put("test_field_type", 1); + stringType.put("test_field_type", new boolean[0]); JSONObject json = new JSONObject(); json.put("test_field_type", stringType); try { @@ -1268,7 +1286,7 @@ public void testNestedRepeatedComplex() throws Exception { @Test public void testNestedRepeatedComplexFail() throws Exception { double[] doubleArr = {1.1, 2.2, 3.3, 4.4, 5.5}; - Boolean[] fakeStringArr = {true, false}; + Boolean[][] fakeStringArr = {new Boolean[0], new Boolean[0]}; int[] intArr = {1, 2, 3, 4, 5}; JSONObject json = new JSONObject(); From 6e873b03a43b81764c5caf9b402cfc1d41abcf35 Mon Sep 17 00:00:00 2001 From: Martin Vanek Date: Wed, 13 Mar 2024 22:25:21 +0000 Subject: [PATCH 2/3] chore: linting --- .../storage/v1/JsonToProtoMessage.java | 10 ++++++++-- .../storage/v1/JsonStreamWriterTest.java | 8 ++++---- .../storage/v1/JsonToProtoMessageTest.java | 18 +++++++++--------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index be785d0fad..c145b4ed25 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -650,7 +650,10 @@ private void fillField( if (val instanceof String) { protoMsg.setField(fieldDescriptor, val); return; - } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + } else if (val instanceof Short + || val instanceof Integer + || val instanceof Long + || val instanceof Boolean) { protoMsg.setField(fieldDescriptor, String.valueOf(val)); return; } @@ -913,7 +916,10 @@ private void fillRepeatedField( case STRING: if (val instanceof String) { protoMsg.addRepeatedField(fieldDescriptor, val); - } else if (val instanceof Short || val instanceof Integer || val instanceof Long || val instanceof Boolean) { + } else if (val instanceof Short + || val instanceof Integer + || val instanceof Long + || val instanceof Boolean) { protoMsg.addRepeatedField(fieldDescriptor, String.valueOf(val)); return; } else { diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index a9d9b8bd3c..6d3baea406 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -1427,10 +1427,10 @@ public void testMultipleAppendSerializationErrors() Map rowIndexToErrorMessage = appendSerializationError.getRowIndexToErrorMessage(); assertEquals( - ImmutableMap.of( - 0, "The source object has fields unknown to BigQuery: root.not_foo.", - 2, "The source object has fields unknown to BigQuery: root.not_bar." - ), rowIndexToErrorMessage); + ImmutableMap.of( + 0, "The source object has fields unknown to BigQuery: root.not_foo.", + 2, "The source object has fields unknown to BigQuery: root.not_bar."), + rowIndexToErrorMessage); } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index 3f1123833b..a347e65e79 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -933,7 +933,7 @@ public void testAllTypes() throws Exception { } else if (entry.getKey() == Int64Type.getDescriptor() || entry.getKey() == BytesType.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); - } else if(entry.getKey() == StringType.getDescriptor()) { + } else if (entry.getKey() == StringType.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 4, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); @@ -1019,14 +1019,14 @@ public void testRequired() throws Exception { } } - @Test - public void testStructSimple() throws Exception { - structSimple("test", "test"); - structSimple(true, "true"); - structSimple(1, "1"); - structSimple((short) 1, "1"); - structSimple((long) 1, "1"); - } + @Test + public void testStructSimple() throws Exception { + structSimple("test", "test"); + structSimple(true, "true"); + structSimple(1, "1"); + structSimple((short) 1, "1"); + structSimple((long) 1, "1"); + } private void structSimple(Object value, String expected) throws Exception { MessageType expectedProto = From 2f255dab21fc84dff367815d5b6c29578ba88d4e Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 14 Mar 2024 16:49:13 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3171e5e6f5..72c58d3d6a 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.33.0') +implementation platform('com.google.cloud:libraries-bom:26.34.0') implementation 'com.google.cloud:google-cloud-bigquerystorage' ```