[Extensions] Moving ExtensionsRequest to use Protobuf (opensearch-project#6960)

Author: saratvemulapalli

CHANGELOG.md

@@ -80,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
 
 ### Dependencies
 

buildSrc/version.properties

@@ -22,6 +22,7 @@ woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1
 guava             = 31.1-jre
+protobuf          = 3.22.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0

modules/transport-netty4/build.gradle

@@ -132,12 +132,6 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
     // classes are missing
 
-    // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty)
-    'com.google.protobuf.ExtensionRegistry',
-    'com.google.protobuf.MessageLite$Builder',
-    'com.google.protobuf.MessageLite',
-    'com.google.protobuf.Parser',
-
     // from io.netty.logging.CommonsLoggerFactory (netty)
     'org.apache.commons.logging.Log',
     'org.apache.commons.logging.LogFactory',
@@ -192,8 +186,6 @@ thirdPartyAudit {
     'org.slf4j.spi.LocationAwareLogger',
 
     'com.github.luben.zstd.Zstd',
-    'com.google.protobuf.ExtensionRegistryLite',
-    'com.google.protobuf.MessageLiteOrBuilder',
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.jcraft.jzlib.Deflater',

plugins/repository-gcs/build.gradle

@@ -65,8 +65,6 @@ dependencies {
   api 'com.google.api:api-common:1.8.1'
   api 'com.google.api:gax:2.17.0'
   api 'org.threeten:threetenbp:1.4.4'
-  api 'com.google.protobuf:protobuf-java-util:3.22.2'
-  api 'com.google.protobuf:protobuf-java:3.22.2'
   api 'com.google.code.gson:gson:2.9.0'
   api 'com.google.api.grpc:proto-google-common-protos:2.10.0'
   api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
@@ -105,13 +103,6 @@ tasks.named("dependencyLicenses").configure {
 thirdPartyAudit {
   ignoreViolations(
     // uses internal java api: sun.misc.Unsafe
-    'com.google.protobuf.UnsafeUtil',
-    'com.google.protobuf.UnsafeUtil$1',
-    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
-    'com.google.protobuf.MessageSchema',
-    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
     'com.google.common.cache.Striped64',
     'com.google.common.cache.Striped64$1',
     'com.google.common.cache.Striped64$Cell',
@@ -151,6 +142,10 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
     'com.oracle.svm.core.configure.ResourcesRegistry',
+    'com.google.protobuf.util.JsonFormat',
+    'com.google.protobuf.util.JsonFormat$Parser',
+    'com.google.protobuf.util.JsonFormat$Printer',
+    'com.google.protobuf.util.Timestamps',
     // commons-logging optional dependencies
     'org.apache.avalon.framework.logger.Logger',
     'org.apache.log.Hierarchy',

plugins/repository-gcs/licenses/protobuf-java-util-3.22.2.jar.sha1

@@ -69,7 +69,6 @@ dependencies {
   api 'org.apache.avro:avro:1.11.1'
   api 'com.google.code.gson:gson:2.10.1'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
-  api 'com.google.protobuf:protobuf-java:3.22.3'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api 'commons-cli:commons-cli:1.5.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -114,19 +113,6 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /hadoop-.*/, to: 'hadoop'
 }
 
-thirdPartyAudit {
-  ignoreViolations(
-    // uses internal java api: sun.misc.Unsafe
-    'com.google.protobuf.MessageSchema',
-    'com.google.protobuf.UnsafeUtil',
-    'com.google.protobuf.UnsafeUtil$1',
-    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
-  )
-}
-
 tasks.named("integTest").configure {
   it.dependsOn(project.tasks.named("bundlePlugin"))
 }

plugins/repository-hdfs/build.gradle

@@ -66,12 +66,6 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 
-    // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty)
-    'com.google.protobuf.ExtensionRegistry',
-    'com.google.protobuf.MessageLite$Builder',
-    'com.google.protobuf.MessageLite',
-    'com.google.protobuf.Parser',
-
     // from io.netty.logging.CommonsLoggerFactory (netty)
     'org.apache.commons.logging.Log',
     'org.apache.commons.logging.LogFactory',
@@ -118,8 +112,6 @@ thirdPartyAudit {
     'org.slf4j.spi.LocationAwareLogger',
 
     'com.github.luben.zstd.Zstd',
-    'com.google.protobuf.ExtensionRegistryLite',
-    'com.google.protobuf.MessageLiteOrBuilder',
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.jcraft.jzlib.Deflater',

plugins/repository-hdfs/licenses/protobuf-java-3.22.3.jar.sha1

@@ -30,10 +30,13 @@
 
 import org.opensearch.gradle.info.BuildParams
 
-apply plugin: 'opensearch.build'
-apply plugin: 'com.netflix.nebula.optional-base'
-apply plugin: 'opensearch.publish'
-apply plugin: 'opensearch.internal-cluster-test'
+plugins {
+  id('com.google.protobuf') version 'latest.release'
+  id('opensearch.build')
+  id('com.netflix.nebula.optional-base')
+  id('opensearch.publish')
+  id('opensearch.internal-cluster-test')
+}
 
 publishing {
   publications {
@@ -45,6 +48,13 @@ publishing {
 
 archivesBaseName = 'opensearch'
 
+sourceSets {
+  main {
+    java {
+      srcDir "${buildDir}/generated/source/proto/main/java"
+    }
+  }
+}
 // we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 11 so we do not include this source set in our IDEs
 if (!isEclipse) {
   sourceSets {
@@ -136,6 +146,9 @@ dependencies {
   // jna
   api "net.java.dev.jna:jna:${versions.jna}"
 
+  // protobuf
+  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
@@ -157,6 +170,7 @@ tasks.named("internalClusterTest").configure {
 }
 
 tasks.named("forbiddenPatterns").configure {
+    dependsOn("generateProto")
     exclude '**/*.json'
     exclude '**/*.jmx'
     exclude '**/*.dic'
@@ -203,6 +217,12 @@ def generatePluginsList = tasks.register("generatePluginsList") {
     }
 }
 
+protobuf {
+  protoc {
+    artifact = "com.google.protobuf:protoc:${versions.protobuf}"
+  }
+}
+
 tasks.named("processResources").configure {
     dependsOn generateModulesList, generatePluginsList
 }
@@ -303,6 +323,15 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.common.geometry.S2$Metric',
             'com.google.common.geometry.S2LatLng'
     )
+    ignoreViolations(
+            'com.google.protobuf.MessageSchema',
+            'com.google.protobuf.UnsafeUtil',
+            'com.google.protobuf.UnsafeUtil$1',
+            'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+    )
 }
 
 tasks.named("dependencyLicenses").configure {
@@ -315,17 +344,46 @@ tasks.named("dependencyLicenses").configure {
     }
 }
 
+tasks.named("missingJavadoc").configure {
+  /*
+   * Generated code doesn't follow javadocs formats.
+   * Unfortunately the missingJavadoc task doesnt take *.
+   * TODO: Add support to missingJavadoc task to ignore all generated source code
+   *  https://github.com/opensearch-project/OpenSearch/issues/7264
+   */
+  dependsOn("generateProto")
+  javadocMissingIgnore = [
+    "org.opensearch.extensions.proto.ExtensionRequestProto",
+    "org.opensearch.extensions.proto.ExtensionRequestProto.ExtensionRequestOrBuilder",
+    "org.opensearch.extensions.proto"
+  ]
+}
+
+tasks.named("filepermissions").configure {
+  mustRunAfter("generateProto")
+}
+
 tasks.named("licenseHeaders").configure {
+    dependsOn("generateProto")
     // Ignore our vendored version of Google Guice
     excludes << 'org/opensearch/common/inject/**/*'
     // Ignore temporary copies of impending 8.7 Lucene classes
     excludes << 'org/apache/lucene/search/RegExp87*'
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
     excludes << 'org/opensearch/client/documentation/placeholder.txt'
+    // Ignore for protobuf generated code
+    excludes << 'org/opensearch/extensions/proto/*'
 }
 
 tasks.test {
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
         jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"]
     }
 }
+
+tasks.named("sourcesJar").configure {
+  // Ignore duplicates for protobuf generated code (main and generatedSources).
+  filesMatching("**/proto/*") {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  }
+}

plugins/repository-hdfs/licenses/protobuf-java-LICENSE.txt

@@ -13,11 +13,11 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.extensions.proto.ExtensionRequestProto;
 import org.opensearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Objects;
-import java.util.Optional;
 
 /**
  * CLusterService Request for Extensibility
@@ -26,54 +26,54 @@
  */
 public class ExtensionRequest extends TransportRequest {
     private static final Logger logger = LogManager.getLogger(ExtensionRequest.class);
-    private final ExtensionsManager.RequestType requestType;
-    private final Optional<String> uniqueId;
+    private final ExtensionRequestProto.ExtensionRequest request;
 
-    public ExtensionRequest(ExtensionsManager.RequestType requestType) {
+    public ExtensionRequest(ExtensionRequestProto.RequestType requestType) {
         this(requestType, null);
     }
 
-    public ExtensionRequest(ExtensionsManager.RequestType requestType, @Nullable String uniqueId) {
-        this.requestType = requestType;
-        this.uniqueId = uniqueId == null ? Optional.empty() : Optional.of(uniqueId);
+    public ExtensionRequest(ExtensionRequestProto.RequestType requestType, @Nullable String uniqueId) {
+        ExtensionRequestProto.ExtensionRequest.Builder builder = ExtensionRequestProto.ExtensionRequest.newBuilder();
+        if (uniqueId != null) {
+            builder.setUniqueId(uniqueId);
+        }
+        this.request = builder.setRequestType(requestType).build();
     }
 
     public ExtensionRequest(StreamInput in) throws IOException {
         super(in);
-        this.requestType = in.readEnum(ExtensionsManager.RequestType.class);
-        String id = in.readOptionalString();
-        this.uniqueId = id == null ? Optional.empty() : Optional.of(id);
+        this.request = ExtensionRequestProto.ExtensionRequest.parseFrom(in.readByteArray());
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeEnum(requestType);
-        out.writeOptionalString(uniqueId.orElse(null));
+        out.writeByteArray(request.toByteArray());
     }
 
-    public ExtensionsManager.RequestType getRequestType() {
-        return this.requestType;
+    public ExtensionRequestProto.RequestType getRequestType() {
+        return this.request.getRequestType();
     }
 
-    public Optional<String> getUniqueId() {
-        return uniqueId;
+    public String getUniqueId() {
+        return request.getUniqueId();
     }
 
     public String toString() {
-        return "ExtensionRequest{" + "requestType=" + requestType + "uniqueId=" + uniqueId + '}';
+        return "ExtensionRequest{" + request.toString() + '}';
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         ExtensionRequest that = (ExtensionRequest) o;
-        return Objects.equals(requestType, that.requestType) && Objects.equals(uniqueId, that.uniqueId);
+        return Objects.equals(request.getRequestType(), that.request.getRequestType())
+            && Objects.equals(request.getUniqueId(), that.request.getUniqueId());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(requestType, uniqueId);
+        return Objects.hash(request.getRequestType(), request.getUniqueId());
     }
 }