fix: dynamically load BigtableAdmin and BigtableAsyncAdmin (#3341)

* fix: load BigtableAdmin with ProxyFactory

* update hbase2_x

* fix connection class

* reformat

* use bytebuddy instead

* refactor

* fix constructor

* update javadoc

* update error handling

* udpate

bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java

@@ -45,6 +45,8 @@
 import com.google.protobuf.ByteString;
 import io.grpc.Status;
 import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -1337,4 +1339,12 @@ public int getMasterInfoPort() throws IOException {
   public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException {
     throw new UnsupportedOperationException("rollWALWriter"); // TODO
   }
+
+  /** Handler for unsupported operations for generating Admin class at runtime. */
+  public static class UnsupportedOperationsHandler implements InvocationHandler {
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+      throw new UnsupportedOperationException(method.getName());
+    }
+  }
 }

bigtable-hbase-1.x-parent/bigtable-hbase-1.x-shaded/pom.xml

@@ -159,6 +159,7 @@ limitations under the License.
                     <exclude>META-INF/NOTICE</exclude>
                     <exclude>META-INF/DEPENDENCIES</exclude>
                     <exclude>META-INF/NOTICE.txt</exclude>
+                    <exclude>META-INF/versions/9/module-info.class</exclude>
                   </excludes>
                 </filter>
               </filters>
@@ -331,6 +332,10 @@ limitations under the License.
                   <pattern>javax.annotation</pattern>
                   <shadedPattern>com.google.bigtable.repackaged.javax.annotation</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>net.bytebuddy</pattern>
+                  <shadedPattern>com.google.bigtable.repackaged.net.bytebuddy</shadedPattern>
+                </relocation>
               </relocations>
             </configuration>
           </execution>

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/pom.xml

@@ -146,6 +146,12 @@ limitations under the License.
       <artifactId>grpc-google-cloud-bigtable-v2</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>net.bytebuddy</groupId>
+      <artifactId>byte-buddy</artifactId>
+      <version>1.12.2</version>
+    </dependency>
+
 
     <!-- Test dependencies-->
     <dependency>

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java

@@ -16,39 +16,31 @@
 package com.google.cloud.bigtable.hbase1_x;
 
 import com.google.api.core.InternalApi;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.Futures;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.implementation.InvocationHandlerAdapter;
+import net.bytebuddy.matcher.ElementMatchers;
 import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.AbstractBigtableAdmin;
+import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.CommonConnection;
-import org.apache.hadoop.hbase.client.LogEntry;
-import org.apache.hadoop.hbase.client.ServerType;
-import org.apache.hadoop.hbase.client.security.SecurityCapability;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
-import org.apache.hadoop.hbase.quotas.QuotaFilter;
-import org.apache.hadoop.hbase.quotas.QuotaRetriever;
-import org.apache.hadoop.hbase.quotas.QuotaSettings;
-import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
-import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 
 /**
  * This is an hbase 1.x implementation of {@link AbstractBigtableAdmin}. Most methods in this class
@@ -58,7 +50,7 @@
  */
 @InternalApi("For internal usage only")
 @SuppressWarnings("deprecation")
-public class BigtableAdmin extends AbstractBigtableAdmin {
+public abstract class BigtableAdmin extends AbstractBigtableAdmin {
 
   public BigtableAdmin(CommonConnection connection) throws IOException {
     super(connection);
@@ -83,32 +75,12 @@ public void disableTableAsync(TableName tableName) throws IOException {
     disableTable(tableName);
   }
 
-  @Override
-  public Map<ServerName, Boolean> compactionSwitch(boolean b, List<String> list)
-      throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
   /** {@inheritDoc} */
   @Override
   public void enableTableAsync(TableName tableName) throws IOException {
     enableTable(tableName);
   }
 
-  /** {@inheritDoc} */
-  @Override
-  public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(TableName tableName)
-      throws IOException {
-    throw new UnsupportedOperationException("getCompactionState");
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion(byte[] bytes)
-      throws IOException {
-    throw new UnsupportedOperationException("getCompactionStateForRegion");
-  }
-
   /**
    * {@inheritDoc}
    *
@@ -123,33 +95,13 @@ public void snapshot(
     snapshot(snapshotId, tableName);
   }
 
-  @Override
-  public void snapshot(
-      String s, TableName tableName, SnapshotDescription.Type type, Map<String, Object> map)
-      throws IOException, SnapshotCreationException, IllegalArgumentException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
   /** {@inheritDoc} */
   @Override
   public void snapshot(HBaseProtos.SnapshotDescription snapshot)
       throws IOException, SnapshotCreationException, IllegalArgumentException {
     snapshot(snapshot.getName(), TableName.valueOf(snapshot.getTable()));
   }
 
-  /** {@inheritDoc} */
-  @Override
-  public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshot)
-      throws IOException, HBaseSnapshotException, UnknownSnapshotException {
-    throw new UnsupportedOperationException("isSnapshotFinished"); // TODO
-  }
-
-  @Override
-  public void restoreSnapshot(String s, boolean b, boolean b1)
-      throws IOException, RestoreSnapshotException {
-    throw new UnsupportedOperationException("restoreSnapshot"); // TODO
-  }
-
   @Override
   public void cloneSnapshot(String s, TableName tableName, boolean b)
       throws IOException, TableExistsException, RestoreSnapshotException {
@@ -196,152 +148,6 @@ public List<HBaseProtos.SnapshotDescription> listSnapshots(Pattern pattern) thro
     return response;
   }
 
-  @Override
-  public List<SnapshotDescription> listTableSnapshots(
-      String tableNameRegex, String snapshotNameRegex) throws IOException {
-    throw new UnsupportedOperationException("Unsupported - please use listSnapshots");
-  }
-
-  @Override
-  public List<SnapshotDescription> listTableSnapshots(
-      Pattern tableNamePattern, Pattern snapshotNamePattern) throws IOException {
-    throw new UnsupportedOperationException("Unsupported - please use listSnapshots");
-  }
-
-  @Override
-  public boolean isBalancerEnabled() throws IOException {
-    throw new UnsupportedOperationException("isBalancerEnabled"); // TODO
-  }
-
-  @Override
-  public long getLastMajorCompactionTimestamp(TableName tableName) throws IOException {
-    throw new UnsupportedOperationException("getLastMajorCompactionTimestamp"); // TODO
-  }
-
-  @Override
-  public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
-    throw new UnsupportedOperationException("getLastMajorCompactionTimestampForRegion"); // TODO
-  }
-
-  @Override
-  public void setQuota(QuotaSettings quota) throws IOException {
-    throw new UnsupportedOperationException("setQuota"); // TODO
-  }
-
-  @Override
-  public QuotaRetriever getQuotaRetriever(QuotaFilter filter) throws IOException {
-    throw new UnsupportedOperationException("getQuotaRetriever"); // TODO
-  }
-
-  @Override
-  public ServerName getMaster() throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
-  @Override
-  public boolean normalize() throws IOException {
-    throw new UnsupportedOperationException("normalize"); // TODO
-  }
-
-  @Override
-  public boolean isNormalizerEnabled() throws IOException {
-    throw new UnsupportedOperationException("isNormalizerEnabled"); // TODO
-  }
-
-  @Override
-  public boolean setNormalizerRunning(boolean on) throws IOException {
-    throw new UnsupportedOperationException("setNormalizerRunning"); // TODO
-  }
-
-  @Override
-  public boolean setCleanerChoreRunning(boolean b) throws IOException {
-    throw new UnsupportedOperationException("setCleanerChoreRunning"); // TODO
-  }
-
-  @Override
-  public boolean runCleanerChore() throws IOException {
-    throw new UnsupportedOperationException("runCleanerChore"); // TODO
-  }
-
-  @Override
-  public boolean isCleanerChoreEnabled() throws IOException {
-    throw new UnsupportedOperationException("isCleanerChoreEnabled"); // TODO
-  }
-
-  @Override
-  public boolean isMasterInMaintenanceMode() throws IOException {
-    throw new UnsupportedOperationException("isMasterInMaintenanceMode"); // TODO
-  }
-
-  @Override
-  public boolean abortProcedure(long procId, boolean mayInterruptIfRunning) throws IOException {
-    throw new UnsupportedOperationException("abortProcedure"); // TODO
-  }
-
-  @Override
-  public ProcedureInfo[] listProcedures() throws IOException {
-    throw new UnsupportedOperationException("listProcedures"); // TODO
-  }
-
-  @Override
-  public Future<Boolean> abortProcedureAsync(long procId, boolean mayInterruptIfRunning)
-      throws IOException {
-    throw new UnsupportedOperationException("abortProcedureAsync"); // TODO
-  }
-
-  @Override
-  public List<SecurityCapability> getSecurityCapabilities() throws IOException {
-    throw new UnsupportedOperationException("getSecurityCapabilities"); // TODO
-  }
-
-  @Override
-  public boolean balancer(boolean arg0) throws IOException {
-    throw new UnsupportedOperationException("balancer"); // TODO
-  }
-
-  @Override
-  public boolean isSplitOrMergeEnabled(MasterSwitchType arg0) throws IOException {
-    throw new UnsupportedOperationException("isSplitOrMergeEnabled"); // TODO
-  }
-
-  @Override
-  public List<ServerName> listDeadServers() throws IOException {
-    throw new UnsupportedOperationException("listDeadServers"); // TODO
-  }
-
-  @Override
-  public List<ServerName> clearDeadServers(List<ServerName> list) throws IOException {
-    throw new UnsupportedOperationException("clearDeadServers"); // TODO
-  }
-
-  @Override
-  public boolean snapshotCleanupSwitch(boolean b, boolean b1) throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
-  @Override
-  public boolean isSnapshotCleanupEnabled() throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
-  @Override
-  public List<Boolean> clearSlowLogResponses(Set<ServerName> set) throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
-  @Override
-  public List<LogEntry> getLogEntries(
-      Set<ServerName> set, String s, ServerType serverType, int i, Map<String, Object> map)
-      throws IOException {
-    throw new UnsupportedOperationException("not implemented");
-  }
-
-  @Override
-  public boolean[] setSplitOrMergeEnabled(boolean arg0, boolean arg1, MasterSwitchType... arg2)
-      throws IOException {
-    throw new UnsupportedOperationException("setSplitOrMergeEnabled"); // TODO
-  }
-
   @Override
   public ClusterStatus getClusterStatus() throws IOException {
     return new ClusterStatus() {
@@ -352,8 +158,43 @@ public Collection<ServerName> getServers() {
     };
   }
 
-  @Override
-  public String[] listNamespaces() throws IOException {
-    throw new UnsupportedOperationException("not implemented");
+  private static Class<? extends BigtableAdmin> adminClass = null;
+
+  /**
+   * This is a workaround for incompatible changes in hbase minor versions. Dynamically generates a
+   * class that extends BigtableAdmin so incompatible methods won't be accessed unless the methods
+   * are called. If a method is implemented by BigtableAdmin, the generated class will invoke the
+   * implementation in BigtableAdmin. Otherwise it'll throw {@link UnsupportedOperationException}.
+   */
+  private static synchronized Class<? extends BigtableAdmin> getSubclass() {
+    if (adminClass == null) {
+      adminClass =
+          new ByteBuddy()
+              .subclass(BigtableAdmin.class)
+              .method(ElementMatchers.isAbstract())
+              .intercept(
+                  InvocationHandlerAdapter.of(
+                      new AbstractBigtableAdmin.UnsupportedOperationsHandler()))
+              .make()
+              .load(BigtableAdmin.class.getClassLoader())
+              .getLoaded();
+    }
+    return adminClass;
+  }
+
+  public static Admin createInstance(CommonConnection connection) throws IOException {
+    try {
+      return getSubclass().getDeclaredConstructor(CommonConnection.class).newInstance(connection);
+    } catch (InvocationTargetException e) {
+      // Unwrap and throw IOException or RuntimeException as is, and convert all other exceptions to
+      // IOException because
+      // org.apache.hadoop.hbase.client.Connection#getAdmin() only throws
+      // IOException
+      Throwables.throwIfInstanceOf(e.getTargetException(), IOException.class);
+      Throwables.throwIfInstanceOf(e.getTargetException(), RuntimeException.class);
+      throw new IOException(e);
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
   }
 }

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableConnection.java

@@ -58,7 +58,7 @@ public BigtableConnection(Configuration conf) throws IOException {
   /** {@inheritDoc} */
   @Override
   public Admin getAdmin() throws IOException {
-    return new BigtableAdmin(this);
+    return BigtableAdmin.createInstance(this);
   }
 
   @Override