Skip to content

Commit

Permalink
feat: add experimental support for reverse scans public preview (#4060)
Browse files Browse the repository at this point in the history 
* feat: add experimental support for reverse scans public preview

Change-Id: I2381b26dbc428ad4e88206092c23ba85ab26a8b5

* upgrade bigtable version and exclude reverse scans from emulator

Change-Id: I3010755174bbd767563ba286edfff01ac77914f8

* fix test to work with gax's url escaping fix

Change-Id: I62bbe039d3d74418304af61975b616d962f98e62

* fix dep scope

Change-Id: I90e8bced65a9455f83d53feab3131321b819350d

* fix deps

Change-Id: I6a91ef0e4600e9e2e24aa18157641b198dd3e9e7
  • Loading branch information
@igorbernstein2
igorbernstein2 authored Jun 29, 2023
1 parent 0d75465 commit 518a388
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 11 deletions.
7 changes: 7 additions & 0 deletions bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,13 @@ limitations under the License.
<version>${junit.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
<version>1.1.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
151 changes: 151 additions & 0 deletions ...base-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/TestScan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,16 @@

import static com.google.cloud.bigtable.hbase.test_env.SharedTestEnvRule.COLUMN_FAMILY;
import static com.google.cloud.bigtable.hbase.test_env.SharedTestEnvRule.COLUMN_FAMILY2;
import static com.google.common.truth.Truth.*;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
Expand All @@ -31,8 +36,11 @@
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.MultiRowRangeFilter;
import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;

public class TestScan extends AbstractTest {

Expand Down Expand Up @@ -302,6 +310,149 @@ public void testStartEndEquals() throws IOException {
}
}

@Test
@Category(KnownEmulatorGap.class)
public void testBasicReverseScan() throws IOException {
String prefix = "reverse_basic";
int rowsToWrite = 10;

// Initialize variables
Table table = getDefaultTable();

byte[][] rowKeys = new byte[rowsToWrite][];
rowKeys[0] = dataHelper.randomData(prefix);
for (int i = 1; i < rowsToWrite; i++) {
rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]);
}

byte[] qualifier = dataHelper.randomData("qual-");
byte[] value = dataHelper.randomData("value-");

ArrayList<Put> puts = new ArrayList<>(rowsToWrite);

// Insert some columns
for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) {
Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value);
puts.add(put);
}
table.put(puts);

Scan scan = new Scan().setReversed(true).withStartRow(rowKeys[6]).withStopRow(rowKeys[2]);

List<String> actualRowKeys =
StreamSupport.stream(table.getScanner(scan).spliterator(), false)
.map(Result::getRow)
.map(String::new)
.collect(Collectors.toList());

List<String> expectedRowKeys =
ImmutableList.of(
new String(rowKeys[6]),
new String(rowKeys[5]),
new String(rowKeys[4]),
new String(rowKeys[3]));

assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder();
}

@Test
@Category(KnownEmulatorGap.class)
public void testReverseScanWithFilter() throws IOException {
String prefix = "reverse_filter";
int rowsToWrite = 10;

// Initialize variables
Table table = getDefaultTable();

byte[][] rowKeys = new byte[rowsToWrite][];
rowKeys[0] = dataHelper.randomData(prefix);
for (int i = 1; i < rowsToWrite; i++) {
rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]);
}

byte[] qualifier = dataHelper.randomData("qual-");
byte[] value = dataHelper.randomData("value-");

ArrayList<Put> puts = new ArrayList<>(rowsToWrite);

// Insert some columns
for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) {
Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value);
puts.add(put);
}
table.put(puts);

Scan scan =
new Scan()
.setReversed(true)
.setFilter(
new MultiRowRangeFilter(
Lists.newArrayList(
new RowRange(rowKeys[3], false, rowKeys[4], true),
new RowRange(rowKeys[6], true, rowKeys[8], false))));

List<String> actualRowKeys =
StreamSupport.stream(table.getScanner(scan).spliterator(), false)
.map(Result::getRow)
.map(String::new)
.collect(Collectors.toList());

List<String> expectedRowKeys =
ImmutableList.of(new String(rowKeys[7]), new String(rowKeys[6]), new String(rowKeys[4]));

assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder();
}

@Test
@Category(KnownEmulatorGap.class)
public void testReverseScanWithFilterAndRange() throws IOException {
String prefix = "reverse_filter";
int rowsToWrite = 10;

// Initialize variables
Table table = getDefaultTable();

byte[][] rowKeys = new byte[rowsToWrite][];
rowKeys[0] = dataHelper.randomData(prefix);
for (int i = 1; i < rowsToWrite; i++) {
rowKeys[i] = rowFollowingSameLength(rowKeys[i - 1]);
}

byte[] qualifier = dataHelper.randomData("qual-");
byte[] value = dataHelper.randomData("value-");

ArrayList<Put> puts = new ArrayList<>(rowsToWrite);

// Insert some columns
for (int rowIndex = 0; rowIndex < rowsToWrite; rowIndex++) {
Put put = new Put(rowKeys[rowIndex]).addColumn(COLUMN_FAMILY, qualifier, value);
puts.add(put);
}
table.put(puts);

Scan scan =
new Scan()
.setReversed(true)
.withStartRow(rowKeys[6])
.withStopRow(rowKeys[1])
.setFilter(
new MultiRowRangeFilter(
Lists.newArrayList(
new RowRange(rowKeys[3], true, rowKeys[4], true),
new RowRange(rowKeys[6], true, rowKeys[8], false))));

List<String> actualRowKeys =
StreamSupport.stream(table.getScanner(scan).spliterator(), false)
.map(Result::getRow)
.map(String::new)
.collect(Collectors.toList());

List<String> expectedRowKeys =
ImmutableList.of(new String(rowKeys[6]), new String(rowKeys[4]), new String(rowKeys[3]));

assertThat(actualRowKeys).containsExactlyElementsIn(expectedRowKeys).inOrder();
}

@Test
public void testColFamilyTimeRange() throws IOException {
TableName tableName = sharedTestEnv.newTestTableName();
Expand Down
48 changes: 41 additions & 7 deletions ...gtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public class ScanAdapter implements ReadOperationAdapter<Scan> {
private static final int UNSET_MAX_RESULTS_PER_COLUMN_FAMILY = -1;
private static final boolean OPEN_CLOSED_AVAILABLE = isOpenClosedAvailable();
private static final boolean LIMIT_AVAILABLE = isLimitAvailable();
private static final boolean REVERSED_AVAILABLE = isReversedAvailable();

/**
* HBase supports include(Stop|Start)Row only at 1.4.0+, so check to make sure that the HBase
Expand All @@ -78,6 +79,15 @@ private static boolean isLimitAvailable() {
}
}

private static boolean isReversedAvailable() {
try {
new Scan().setReversed(true);
return true;
} catch (NoSuchMethodError e) {
return false;
}
}

private final FilterAdapter filterAdapter;
private final RowRangeAdapter rowRangeAdapter;

Expand Down Expand Up @@ -161,6 +171,8 @@ public Query adapt(Scan scan, ReadHooks readHooks, Query query) {
return Query.fromProto(((BigtableFixedProtoScan) scan).getRequest());
} else {
throwIfUnsupportedScan(scan);

query.reversed(scan.isReversed());
toByteStringRange(scan, query);
query.filter(buildFilter(scan, readHooks));

Expand All @@ -182,19 +194,41 @@ private RangeSet<RowKeyWrapper> getRangeSet(Scan scan) {
return rowRangeAdapter.rowSetToRangeSet(rowSet);
} else {
RangeSet<RowKeyWrapper> rangeSet = TreeRangeSet.create();
final ByteString startRow = ByteString.copyFrom(scan.getStartRow());
final ByteString stopRow = ByteString.copyFrom(scan.getStopRow());

if (scan.isGetScan()) {
rangeSet.add(Range.singleton(new RowKeyWrapper(startRow)));
rangeSet.add(Range.singleton(new RowKeyWrapper(ByteString.copyFrom(scan.getStartRow()))));
return rangeSet;
}

ByteString startRow;
BoundType startBound;
ByteString stopRow;
BoundType stopBound;

// For reverse scans, HBase wants the lexicographically greater key to be the start. But
// java-bigtable keeps the bounds the same as forward scans. So this will flip the ranges for
// reverse scans. Please note that prior to hbase 1.4 the only range bound that was available
// was [start, stop).
if (REVERSED_AVAILABLE && scan.isReversed()) {
startRow = ByteString.copyFrom(scan.getStopRow());
startBound =
(!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED;

stopRow = ByteString.copyFrom(scan.getStartRow());
stopBound =
(!OPEN_CLOSED_AVAILABLE || scan.includeStartRow()) ? BoundType.CLOSED : BoundType.OPEN;
} else {
final BoundType startBound =

startRow = ByteString.copyFrom(scan.getStartRow());
startBound =
(!OPEN_CLOSED_AVAILABLE || scan.includeStartRow()) ? BoundType.CLOSED : BoundType.OPEN;
final BoundType endBound =
(!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED;

rangeSet.add(rowRangeAdapter.boundedRange(startBound, startRow, endBound, stopRow));
stopRow = ByteString.copyFrom(scan.getStopRow());
stopBound =
(!OPEN_CLOSED_AVAILABLE || !scan.includeStopRow()) ? BoundType.OPEN : BoundType.CLOSED;
}

rangeSet.add(rowRangeAdapter.boundedRange(startBound, startRow, stopBound, stopRow));
return rangeSet;
}
}
Expand Down
12 changes: 9 additions & 3 deletions ...le-hbase/src/test/java/org/apache/hadoop/hbase/client/TestAbstractBigtableConnection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.google.cloud.bigtable.hbase.BigtableOptionsFactory;
import com.google.cloud.bigtable.hbase.adapters.SampledRowKeysAdapter;
import com.google.cloud.bigtable.test.helper.TestServerBuilder;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Queues;
import com.google.common.truth.Truth;
Expand Down Expand Up @@ -212,9 +213,14 @@ public void testHeaders() throws IOException {
Truth.assertThat(requestParams)
.contains(
"table_name="
+ String.format(
"projects/%s/instances/%s/tables/%s",
PROJECT_ID, INSTANCE_ID, TABLE_NAME.getNameAsString()));
+ Joiner.on("%2F")
.join(
"projects",
PROJECT_ID,
"instances",
INSTANCE_ID,
"tables",
TABLE_NAME.getNameAsString()));
Truth.assertThat(resourcePath).isNull();
} else {
Truth.assertThat(resourcePath)
Expand Down
7 changes: 7 additions & 0 deletions bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,13 @@ limitations under the License.
<version>${junit.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
<version>1.1.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
7 changes: 7 additions & 0 deletions bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,13 @@ limitations under the License.
<version>${junit.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
<version>1.1.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ limitations under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

<!-- core dependency versions -->
<bigtable.version>2.22.0</bigtable.version>
<bigtable.version>2.24.1</bigtable.version>
<google-cloud-bigtable-emulator.version>0.157.3</google-cloud-bigtable-emulator.version>
<bigtable-client-core.version>1.28.0</bigtable-client-core.version>
<grpc-conscrypt.version>2.5.2</grpc-conscrypt.version>
Expand Down

0 comments on commit 518a388

Please sign in to comment.