From 518a388420305771d460d59483c8652bd5b65f5b Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 29 Jun 2023 15:34:26 -0400 Subject: [PATCH] feat: add experimental support for reverse scans public preview (#4060) * feat: add experimental support for reverse scans public preview Change-Id: I2381b26dbc428ad4e88206092c23ba85ab26a8b5 * upgrade bigtable version and exclude reverse scans from emulator Change-Id: I3010755174bbd767563ba286edfff01ac77914f8 * fix test to work with gax's url escaping fix Change-Id: I62bbe039d3d74418304af61975b616d962f98e62 * fix dep scope Change-Id: I90e8bced65a9455f83d53feab3131321b819350d * fix deps Change-Id: I6a91ef0e4600e9e2e24aa18157641b198dd3e9e7 --- .../pom.xml | 7 + .../google/cloud/bigtable/hbase/TestScan.java | 151 ++++++++++++++++++ .../hbase/adapters/read/ScanAdapter.java | 48 +++++- .../TestAbstractBigtableConnection.java | 12 +- .../pom.xml | 7 + .../pom.xml | 7 + pom.xml | 2 +- 7 files changed, 223 insertions(+), 11 deletions(-) diff --git a/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml b/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml index 1f6d34a91f..69b75ba231 100644 --- a/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml +++ b/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml @@ -124,6 +124,13 @@ limitations under the License. ${junit.version} test + + + com.google.truth + truth + 1.1.3 + test + diff --git a/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/TestScan.java b/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/TestScan.java index 2596244f22..3422f3bade 100644 --- a/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/TestScan.java +++ b/bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/TestScan.java @@ -17,11 +17,16 @@ import static com.google.cloud.bigtable.hbase.test_env.SharedTestEnvRule.COLUMN_FAMILY; import static com.google.cloud.bigtable.hbase.test_env.SharedTestEnvRule.COLUMN_FAMILY2; +import static com.google.common.truth.Truth.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.TableName; @@ -31,8 +36,11 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.MultiRowRangeFilter; +import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; import org.junit.Assert; import org.junit.Test; +import org.junit.experimental.categories.Category; public class TestScan extends AbstractTest { @@ -302,6 +310,149 @@ public void testStartEndEquals() throws IOException { } } + @Test + @Category(KnownEmulatorGap.class) + public void testBasicReverseScan() throws IOException { + String prefix = "reverse_basic"; + int rowsToWrite = 10; + + // Initialize variables + Table table = getDefaultTable(); + + byte[][] rowKeys = new byte[rowsToWrite][]; + rowKeys[0] = dataHelper.randomData(prefix); + for (int i = 1; i < rowsToWrite; i++) { + rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]); + } + + byte[] qualifier = dataHelper.randomData("qual-"); + byte[] value = dataHelper.randomData("value-"); + + ArrayList puts = new ArrayList<>(rowsToWrite); + + // Insert some columns + for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) { + Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value); + puts.add(put); + } + table.put(puts); + + Scan scan = new Scan().setReversed(true).withStartRow(rowKeys[6]).withStopRow(rowKeys[2]); + + List actualRowKeys = + StreamSupport.stream(table.getScanner(scan).spliterator(), false) + .map(Result::getRow) + .map(String::new) + .collect(Collectors.toList()); + + List expectedRowKeys = + ImmutableList.of( + new String(rowKeys[6]), + new String(rowKeys[5]), + new String(rowKeys[4]), + new String(rowKeys[3])); + + assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder(); + } + + @Test + @Category(KnownEmulatorGap.class) + public void testReverseScanWithFilter() throws IOException { + String prefix = "reverse_filter"; + int rowsToWrite = 10; + + // Initialize variables + Table table = getDefaultTable(); + + byte[][] rowKeys = new byte[rowsToWrite][]; + rowKeys[0] = dataHelper.randomData(prefix); + for (int i = 1; i < rowsToWrite; i++) { + rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]); + } + + byte[] qualifier = dataHelper.randomData("qual-"); + byte[] value = dataHelper.randomData("value-"); + + ArrayList puts = new ArrayList<>(rowsToWrite); + + // Insert some columns + for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) { + Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value); + puts.add(put); + } + table.put(puts); + + Scan scan = + new Scan() + .setReversed(true) + .setFilter( + new MultiRowRangeFilter( + Lists.newArrayList( + new RowRange(rowKeys[3], false, rowKeys[4], true), + new RowRange(rowKeys[6], true, rowKeys[8], false)))); + + List actualRowKeys = + StreamSupport.stream(table.getScanner(scan).spliterator(), false) + .map(Result::getRow) + .map(String::new) + .collect(Collectors.toList()); + + List expectedRowKeys = + ImmutableList.of(new String(rowKeys[7]), new String(rowKeys[6]), new String(rowKeys[4])); + + assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder(); + } + + @Test + @Category(KnownEmulatorGap.class) + public void testReverseScanWithFilterAndRange() throws IOException { + String prefix = "reverse_filter"; + int rowsToWrite = 10; + + // Initialize variables + Table table = getDefaultTable(); + + byte[][] rowKeys = new byte[rowsToWrite][]; + rowKeys[0] = dataHelper.randomData(prefix); + for (int i = 1; i < rowsToWrite; i++) { + rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]); + } + + byte[] qualifier = dataHelper.randomData("qual-"); + byte[] value = dataHelper.randomData("value-"); + + ArrayList puts = new ArrayList<>(rowsToWrite); + + // Insert some columns + for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) { + Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value); + puts.add(put); + } + table.put(puts); + + Scan scan = + new Scan() + .setReversed(true) + .withStartRow(rowKeys[6]) + .withStopRow(rowKeys[1]) + .setFilter( + new MultiRowRangeFilter( + Lists.newArrayList( + new RowRange(rowKeys[3], true, rowKeys[4], true), + new RowRange(rowKeys[6], true, rowKeys[8], false)))); + + List actualRowKeys = + StreamSupport.stream(table.getScanner(scan).spliterator(), false) + .map(Result::getRow) + .map(String::new) + .collect(Collectors.toList()); + + List expectedRowKeys = + ImmutableList.of(new String(rowKeys[6]), new String(rowKeys[4]), new String(rowKeys[3])); + + assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder(); + } + @Test public void testColFamilyTimeRange() throws IOException { TableName tableName = sharedTestEnv.newTestTableName(); diff --git a/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java b/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java index e7af008ad2..fb8fd573c7 100644 --- a/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java +++ b/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java @@ -55,6 +55,7 @@ public class ScanAdapter implements ReadOperationAdapter { private static final int UNSET_MAX_RESULTS_PER_COLUMN_FAMILY = -1; private static final boolean OPEN_CLOSED_AVAILABLE = isOpenClosedAvailable(); private static final boolean LIMIT_AVAILABLE = isLimitAvailable(); + private static final boolean REVERSED_AVAILABLE = isReversedAvailable(); /** * HBase supports include(Stop|Start)Row only at 1.4.0+, so check to make sure that the HBase @@ -78,6 +79,15 @@ private static boolean isLimitAvailable() { } } + private static boolean isReversedAvailable() { + try { + new Scan().setReversed(true); + return true; + } catch (NoSuchMethodError e) { + return false; + } + } + private final FilterAdapter filterAdapter; private final RowRangeAdapter rowRangeAdapter; @@ -161,6 +171,8 @@ public Query adapt(Scan scan, ReadHooks readHooks, Query query) { return Query.fromProto(((BigtableFixedProtoScan) scan).getRequest()); } else { throwIfUnsupportedScan(scan); + + query.reversed(scan.isReversed()); toByteStringRange(scan, query); query.filter(buildFilter(scan, readHooks)); @@ -182,19 +194,41 @@ private RangeSet getRangeSet(Scan scan) { return rowRangeAdapter.rowSetToRangeSet(rowSet); } else { RangeSet rangeSet = TreeRangeSet.create(); - final ByteString startRow = ByteString.copyFrom(scan.getStartRow()); - final ByteString stopRow = ByteString.copyFrom(scan.getStopRow()); if (scan.isGetScan()) { - rangeSet.add(Range.singleton(new RowKeyWrapper(startRow))); + rangeSet.add(Range.singleton(new RowKeyWrapper(ByteString.copyFrom(scan.getStartRow())))); + return rangeSet; + } + + ByteString startRow; + BoundType startBound; + ByteString stopRow; + BoundType stopBound; + + // For reverse scans, HBase wants the lexicographically greater key to be the start. But + // java-bigtable keeps the bounds the same as forward scans. So this will flip the ranges for + // reverse scans. Please note that prior to hbase 1.4 the only range bound that was available + // was [start, stop). + if (REVERSED_AVAILABLE && scan.isReversed()) { + startRow = ByteString.copyFrom(scan.getStopRow()); + startBound = + (!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED; + + stopRow = ByteString.copyFrom(scan.getStartRow()); + stopBound = + (!OPEN_CLOSED_AVAILABLE || scan.includeStartRow()) ? BoundType.CLOSED : BoundType.OPEN; } else { - final BoundType startBound = + + startRow = ByteString.copyFrom(scan.getStartRow()); + startBound = (!OPEN_CLOSED_AVAILABLE || scan.includeStartRow()) ? BoundType.CLOSED : BoundType.OPEN; - final BoundType endBound = - (!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED; - rangeSet.add(rowRangeAdapter.boundedRange(startBound, startRow, endBound, stopRow)); + stopRow = ByteString.copyFrom(scan.getStopRow()); + stopBound = + (!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED; } + + rangeSet.add(rowRangeAdapter.boundedRange(startBound, startRow, stopBound, stopRow)); return rangeSet; } } diff --git a/bigtable-client-core-parent/bigtable-hbase/src/test/java/org/apache/hadoop/hbase/client/TestAbstractBigtableConnection.java b/bigtable-client-core-parent/bigtable-hbase/src/test/java/org/apache/hadoop/hbase/client/TestAbstractBigtableConnection.java index aab4b76ab9..5b0b5f2af2 100644 --- a/bigtable-client-core-parent/bigtable-hbase/src/test/java/org/apache/hadoop/hbase/client/TestAbstractBigtableConnection.java +++ b/bigtable-client-core-parent/bigtable-hbase/src/test/java/org/apache/hadoop/hbase/client/TestAbstractBigtableConnection.java @@ -35,6 +35,7 @@ import com.google.cloud.bigtable.hbase.BigtableOptionsFactory; import com.google.cloud.bigtable.hbase.adapters.SampledRowKeysAdapter; import com.google.cloud.bigtable.test.helper.TestServerBuilder; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.Queues; import com.google.common.truth.Truth; @@ -212,9 +213,14 @@ public void testHeaders() throws IOException { Truth.assertThat(requestParams) .contains( "table_name=" - + String.format( - "projects/%s/instances/%s/tables/%s", - PROJECT_ID, INSTANCE_ID, TABLE_NAME.getNameAsString())); + + Joiner.on("%2F") + .join( + "projects", + PROJECT_ID, + "instances", + INSTANCE_ID, + "tables", + TABLE_NAME.getNameAsString())); Truth.assertThat(resourcePath).isNull(); } else { Truth.assertThat(resourcePath) diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml index 75a534845f..4928a91e98 100644 --- a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml +++ b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml @@ -355,6 +355,13 @@ limitations under the License. ${junit.version} test + + + com.google.truth + truth + 1.1.3 + test + diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml index 92e40a52ea..82aeacaad1 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml @@ -322,6 +322,13 @@ limitations under the License. ${junit.version} test + + + com.google.truth + truth + 1.1.3 + test + diff --git a/pom.xml b/pom.xml index 649b2cce2f..3dad7dae6b 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ limitations under the License. UTF-8 - 2.22.0 + 2.24.1 0.157.3 1.28.0 2.5.2