Skip to content

Commit

Permalink
deps: clean up mirroring client deps (#4261)
Browse files Browse the repository at this point in the history 
* add IDE workaround for mirroring-client shaded deps to be resolved from ~/.m2
* drop shading from intermediate artifacts
* organize mirroring client poms so that env deps come first
* update mirroring client integration tests to avoid duplicated classes from hbase-shaded-client and hbase-shaded-testing-util
* use reload4j for integration test logging
* add google-cloud-bigtable as a dep to integration test modules to resolve conflict about guava
* update integration test modules to exclude transitive deps from each other to make easier to reason about the classpath

Change-Id: I0a9c57ac5eac609847cb28014c156436fb94abe2
  • Loading branch information
@igorbernstein2
igorbernstein2 authored Nov 29, 2023
1 parent 781d586 commit ce9e53b
Show file tree
Hide file tree
Showing 13 changed files with 356 additions and 339 deletions.
45 changes: 19 additions & 26 deletions ...able-hbase-mirroring-client-1.x-parent/bigtable-hbase-mirroring-client-1.x-hadoop/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,25 @@ limitations under the License.
</properties>

<dependencies>
<!-- Environment deps first -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase1.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
<version>${reload4j.version}</version>
<scope>runtime</scope>
</dependency>

<!-- Internal dependencies that will be shaded along with their transitive dependencies. -->
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
Expand All @@ -60,31 +79,8 @@ limitations under the License.
<groupId>${project.groupId}</groupId>
<artifactId>bigtable-hbase-mirroring-client-1.x</artifactId>
</exclusion>
<exclusion>
<groupId>io.opencensus</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- Manually promote public dependencies: This is necessary to avoid flattening hbase-shaded-client's dependency tree -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase1.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
<version>${reload4j.version}</version>
<scope>runtime</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -147,9 +143,6 @@ limitations under the License.
<include>
com.google.cloud.bigtable:bigtable-hbase-mirroring-client-1.x-shaded
</include>
<include>
com.google.cloud.bigtable:bigtable-hbase-mirroring-client-1.x
</include>
</includes>
</artifactSet>
<relocations>
Expand Down
107 changes: 46 additions & 61 deletions ...mirroring-client-1.x-parent/bigtable-hbase-mirroring-client-1.x-integration-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,29 +178,13 @@ limitations under the License.
</profiles>

<dependencies>
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-1.x</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<scope>test</scope>
</dependency>

<!-- Environment deps first -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<artifactId>hbase-shaded-testing-util</artifactId>
<version>${hbase1.version}</version>
<scope>test</scope>
<exclusions>
<!-- exclusion to avoid version conflict with existing artifact -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<!-- exclusion to avoid version conflict with existing artifact -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
Expand All @@ -211,48 +195,64 @@ limitations under the License.
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
<version>${reload4j.version}</version>
<scope>runtime</scope>
<scope>test</scope>
</dependency>

<!-- Project Modules -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>bigtable-hbase-1.x-hadoop</artifactId>
<version>2.12.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-1.x</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- HBase testing tools -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-testing-util</artifactId>
<version>${hbase1.version}</version>
<groupId>${project.groupId}</groupId>
<artifactId>bigtable-hbase-1.x</artifactId>
<version>2.12.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
<scope>test</scope>
<exclusions>
<!-- exclusion to avoid version conflict with existing artifact -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<!-- exclusion to avoid version conflict with existing artifact -->
<!-- already included in the hbase-shaded-testing-util -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- Misc -->
<!-- This gets pulled in by bigtable-hbase-1.x.
Add ths to resolve conflict with the guava version that opencensus-exporter-trace-zipkin pulls in -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>${bigtable.version}</version>
<scope>test</scope>
</dependency>


<!-- Internal testing utils -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.22.2</version>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-core</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<!-- Only want test helpers -->
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- Misc -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
Expand All @@ -264,22 +264,26 @@ limitations under the License.
<groupId>io.opencensus</groupId>
<artifactId>opencensus-impl</artifactId>
<version>${opencensus.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-trace-zipkin</artifactId>
<version>${opencensus.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-stats-prometheus</artifactId>
<version>${opencensus.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<!-- Should match prometheus version used by opencensus-exporter-stats-prometheus -->
<version>0.6.0</version>
<scope>test</scope>
</dependency>

<!-- Testing deps -->
Expand All @@ -302,25 +306,6 @@ limitations under the License.
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.20.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.20.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-core</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
31 changes: 15 additions & 16 deletions ...able-hbase-mirroring-client-1.x-parent/bigtable-hbase-mirroring-client-1.x-shaded/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,7 @@ limitations under the License.
</description>

<dependencies>
<!-- Internal dependencies that will be shaded along with their transitive dependencies. -->
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-1.x</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<scope>compile</scope>
</dependency>

<!-- Manually promote public dependencies: This is necessary to avoid flattening hbase-shaded-client's dependency tree -->
<!-- Environment deps first -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
Expand All @@ -60,6 +52,14 @@ limitations under the License.
<artifactId>reload4j</artifactId>
<version>${reload4j.version}</version>
</dependency>

<!-- Internal dependencies that will be shaded along with their transitive dependencies. -->
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-1.x</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<scope>compile</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -100,18 +100,17 @@ limitations under the License.
</transformers>
<artifactSet>
<excludes>
<!-- exclude user visible deps -->
<exclude>commons-logging:commons-logging</exclude>
<!-- exclude hbase-shaded-client & all of its dependencies -->
<exclude>org.apache.hbase:hbase-shaded-client</exclude>
<exclude>commons-logging:commons-logging</exclude>
<exclude>org.slf4j:slf4j-api</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<exclude>log4j:log4j</exclude>
<exclude>ch.qos.reload4j:reload4j</exclude>
<exclude>org.apache.htrace:htrace-core4</exclude>
<exclude>org.apache.htrace:htrace-core</exclude>
<exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
<exclude>org.apache.htrace:htrace-core4</exclude>
<exclude>log4j:log4j</exclude>
<exclude>org.apache.htrace:htrace-core</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<!-- Exclude reload4j -->
<exclude>ch.qos.reload4j:reload4j</exclude>
</excludes>
</artifactSet>
<relocations>
Expand Down
65 changes: 40 additions & 25 deletions ...nt/bigtable-hbase-mirroring-client-1.x-parent/bigtable-hbase-mirroring-client-1.x/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,46 @@ limitations under the License.
</description>

<dependencies>
<!-- Environment deps first -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase1.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
<version>${reload4j.version}</version>
<scope>runtime</scope>
</dependency>

<!-- Project deps -->
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-mirroring-client-core</artifactId>
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<scope>compile</scope>

<exclusions>
<!-- Exclude the environment deps, as they are declared explicitly above -->
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- Test deps -->
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand All @@ -66,6 +98,13 @@ limitations under the License.
<version>0.7.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-hbase-mirroring:current} -->
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<!-- We only want the test helpers -->
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

Expand Down Expand Up @@ -96,30 +135,6 @@ limitations under the License.
</execution>
</executions>
</plugin>

<!-- copy protobuf-java-format-shaded and core into jar -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<createDependencyReducedPom>true</createDependencyReducedPom>
<artifactSet>
<includes>
<include>com.google.cloud.bigtable:protobuf-java-format-shaded</include>
<include>com.google.cloud.bigtable}:bigtable-hbase-mirroring-client-core</include>
</includes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit ce9e53b

Please sign in to comment.