From 96f508084d23a88d10d30ea89129faee74a1d4b7 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Wed, 20 Dec 2023 17:27:20 +0000 Subject: [PATCH] Remove versioning from RocksDB, now in separate VERSION_DATADATA.json Signed-off-by: Matthew Whitehead --- plugins/rocksdb/build.gradle | 2 - .../RocksDBKeyValuePrivacyStorageFactory.java | 5 +- .../RocksDBKeyValueStorageFactory.java | 79 ++----------------- .../configuration/DatabaseMetadata.java | 52 ++---------- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 5 -- .../RocksDBKeyValueStorageFactoryTest.java | 3 +- .../configuration/DatabaseMetadataTest.java | 32 +------- 7 files changed, 15 insertions(+), 163 deletions(-) diff --git a/plugins/rocksdb/build.gradle b/plugins/rocksdb/build.gradle index 5007049c423..f76e8d80ae0 100644 --- a/plugins/rocksdb/build.gradle +++ b/plugins/rocksdb/build.gradle @@ -41,8 +41,6 @@ dependencies { implementation project(':util') implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' - implementation 'org.apache.maven:maven-artifact' implementation 'com.google.guava:guava' implementation 'info.picocli:picocli' implementation 'io.opentelemetry:opentelemetry-api' diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java index 7675ae5bd7a..355cc056924 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java @@ -130,10 +130,7 @@ private int readDatabaseVersion(final BesuConfiguration commonConfiguration) thr dataDir, privacyDatabaseVersion); Files.createDirectories(dataDir); - new DatabaseMetadata( - publicFactory.getDefaultVersion(), - privacyDatabaseVersion, - commonConfiguration.getBesuVersion()) + new DatabaseMetadata(publicFactory.getDefaultVersion(), privacyDatabaseVersion) .writeToDirectory(dataDir); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index fc553c788cb..0f08e3c0f9b 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -35,12 +35,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.maven.artifact.versioning.ComparableVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -235,10 +233,7 @@ protected Path storagePath(final BesuConfiguration commonConfiguration) { private void init(final BesuConfiguration commonConfiguration) { try { - // This call fails if Besu version doesn't match. If it doesn't fail, write the - // current version databaseVersion = readDatabaseVersion(commonConfiguration); - } catch (final IOException e) { final String message = "Failed to retrieve the RocksDB database meta version: " @@ -261,77 +256,19 @@ private int readDatabaseVersion(final BesuConfiguration commonConfiguration) thr final boolean databaseExists = commonConfiguration.getStoragePath().toFile().exists(); final boolean dataDirExists = dataDir.toFile().exists(); final int databaseVersion; - final String besuVersion; if (databaseExists) { - DatabaseMetadata dbMetaData = DatabaseMetadata.lookUpFrom(dataDir); - databaseVersion = dbMetaData.getVersion(); - besuVersion = dbMetaData.getBesuVersion(); + databaseVersion = DatabaseMetadata.lookUpFrom(dataDir).getVersion(); LOG.info( - "Existing database detected at {}. DB version {}. Besu version {}. Compacting database...", + "Existing database detected at {}. Version {}. Compacting database...", dataDir, - databaseVersion, - besuVersion); - - if (!besuVersion.equals(DatabaseMetadata.BESU_VERSION_UNKNOWN)) { - final String installedVersion = commonConfiguration.getBesuVersion().split("-", 2)[0]; - final String dbBesuVersion = besuVersion.split("-", 2)[0]; - final int versionComparison = - new ComparableVersion(installedVersion).compareTo(new ComparableVersion(dbBesuVersion)); - if (versionComparison == 0) { - // Versions match - no-op - } else if (versionComparison < 0) { - if (false) { - LOG.warn( - "Besu version {} is lower than version {} that last wrote to the database. Allowing startup because --allow-downgrade has been enabled.", - installedVersion, - dbBesuVersion); - // We've allowed startup at an older version of Besu. Since the version in the metadata - // file records the latest version of - // Besu to write to the database we'll update the metadata version to this - // downgraded-version. This avoids the need after a successful - // downgrade to keep specifying --allow-downgrade on every startup. - writeDatabaseMetadata( - databaseVersion, - Optional.ofNullable(commonConfiguration.getBesuVersion()), - dataDir); - } else { - final String message = - "Besu version " - + installedVersion - + " is lower than version " - + dbBesuVersion - + " that last updated the database." - + ". Specify --allow-downgrade to allow Besu to start at the lower version (warning - this may have unrecoverable effects on the database)."; - LOG.error(message); - throw new StorageException(message); - } - } else { - LOG.info( - "Besu version {} is higher than version {} that last updated the DB. Updating DB metadata.", - installedVersion, - dbBesuVersion); - writeDatabaseMetadata( - databaseVersion, Optional.ofNullable(commonConfiguration.getBesuVersion()), dataDir); - } - } else { - // No besu version information was found in the metadata file, so update it with the current - // version and carry on - LOG.info("Adding Besu version to metadata file."); - writeDatabaseMetadata( - databaseVersion, Optional.ofNullable(commonConfiguration.getBesuVersion()), dataDir); - } + databaseVersion); } else { databaseVersion = commonConfiguration.getDatabaseVersion(); - besuVersion = commonConfiguration.getBesuVersion(); - LOG.info( - "No existing database detected at {}. Using version {}. Besu Version {}.", - dataDir, - databaseVersion, - besuVersion); + LOG.info("No existing database detected at {}. Using version {}", dataDir, databaseVersion); if (!dataDirExists) { Files.createDirectories(dataDir); } - writeDatabaseMetadata(databaseVersion, Optional.ofNullable(besuVersion), dataDir); + new DatabaseMetadata(databaseVersion).writeToDirectory(dataDir); } if (!SUPPORTED_VERSIONS.contains(databaseVersion)) { @@ -343,12 +280,6 @@ private int readDatabaseVersion(final BesuConfiguration commonConfiguration) thr return databaseVersion; } - private void writeDatabaseMetadata( - final int databaseVersion, final Optional besuVersion, final Path dataDir) - throws IOException { - new DatabaseMetadata(databaseVersion, besuVersion).writeToDirectory(dataDir); - } - @Override public void close() throws IOException { if (segmentedStorage != null) { diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java index 6c1ac814a74..1faf610cdc9 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java @@ -27,7 +27,6 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,27 +35,19 @@ public class DatabaseMetadata { private static final Logger LOG = LoggerFactory.getLogger(DatabaseMetadata.class); private static final String METADATA_FILENAME = "DATABASE_METADATA.json"; - - /** Represents an unknown Besu version in the database metadata file */ - public static final String BESU_VERSION_UNKNOWN = "UNKNOWN"; - - private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new Jdk8Module()); + private static final ObjectMapper MAPPER = new ObjectMapper(); private final int version; - private Optional besuVersion; private Optional privacyVersion; /** * Instantiates a new Database metadata. * * @param version the version - * @param besuVersion the version of Besu */ @JsonCreator - public DatabaseMetadata( - @JsonProperty("version") final int version, - @JsonProperty("besuVersion") final Optional besuVersion) { - this(version, Optional.empty(), besuVersion); + public DatabaseMetadata(@JsonProperty("version") final int version) { + this(version, Optional.empty()); } /** @@ -64,15 +55,10 @@ public DatabaseMetadata( * * @param version the version * @param privacyVersion the privacy version - * @param besuVersion the Besu version */ - public DatabaseMetadata( - final int version, - final Optional privacyVersion, - final Optional besuVersion) { + public DatabaseMetadata(final int version, final Optional privacyVersion) { this.version = version; this.privacyVersion = privacyVersion; - this.besuVersion = besuVersion; } /** @@ -80,10 +66,9 @@ public DatabaseMetadata( * * @param version the version * @param privacyVersion the privacy version - * @param besuVersion the Besu version */ - public DatabaseMetadata(final int version, final int privacyVersion, final String besuVersion) { - this(version, Optional.of(privacyVersion), Optional.ofNullable(besuVersion)); + public DatabaseMetadata(final int version, final int privacyVersion) { + this(version, Optional.of(privacyVersion)); } /** @@ -95,29 +80,6 @@ public int getVersion() { return version; } - /** - * Sets Besu version. - * - * @param besuVersion the Besu version - */ - @JsonSetter("besuVersion") - public void setBesuVersion(final String besuVersion) { - this.besuVersion = Optional.of(besuVersion); - } - - /** - * Gets version of Besu. - * - * @return the version of Besu - */ - @JsonGetter("besuVersion") - public String getBesuVersion() { - if (besuVersion != null) { - return besuVersion.orElse(BESU_VERSION_UNKNOWN); - } - return BESU_VERSION_UNKNOWN; - } - /** * Sets privacy version. * @@ -189,7 +151,7 @@ private static DatabaseMetadata resolveDatabaseMetadata(final File metadataFile) try { databaseMetadata = MAPPER.readValue(metadataFile, DatabaseMetadata.class); } catch (FileNotFoundException fnfe) { - databaseMetadata = new DatabaseMetadata(1, 1, BESU_VERSION_UNKNOWN); + databaseMetadata = new DatabaseMetadata(1, 1); } catch (JsonProcessingException jpe) { throw new IllegalStateException( String.format("Invalid metadata file %s", metadataFile.getAbsolutePath()), jpe); diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index 0d600cec983..e0ed3f7b396 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -71,8 +71,6 @@ public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getBesuVersion()).isEqualTo("UNKNOWN"); - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion().get()) .isEqualTo(DEFAULT_VERSION); } @@ -84,7 +82,6 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); - when(commonConfiguration.getBesuVersion()).thenReturn("23.10.3"); final RocksDBKeyValuePrivacyStorageFactory storageFactory = new RocksDBKeyValuePrivacyStorageFactory( @@ -100,8 +97,6 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersion()).isEqualTo(DEFAULT_VERSION); - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getBesuVersion()).isEqualTo("23.10.3"); - assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).maybePrivacyVersion().get()) .isEqualTo(DEFAULT_PRIVACY_VERSION); } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index 6a4c3431492..c3a8c32eb27 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -32,7 +32,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Optional; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -138,7 +137,7 @@ public void shouldThrowExceptionWhenVersionNumberIsInvalid() throws Exception { Files.createDirectories(tempDataDir); when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - new DatabaseMetadata(-1, Optional.of("UNKNOWN")).writeToDirectory(tempDataDir); + new DatabaseMetadata(-1).writeToDirectory(tempDataDir); assertThatThrownBy( () -> new RocksDBKeyValueStorageFactory( diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java index 666a3c0c030..569819f8312 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadataTest.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -30,10 +29,9 @@ class DatabaseMetadataTest { @Test void getVersion() { - final DatabaseMetadata databaseMetadata = new DatabaseMetadata(42, Optional.of("23.10.3")); + final DatabaseMetadata databaseMetadata = new DatabaseMetadata(42); assertThat(databaseMetadata).isNotNull(); assertThat(databaseMetadata.getVersion()).isEqualTo(42); - assertThat(databaseMetadata.getBesuVersion()).isEqualTo("23.10.3"); } @Test @@ -49,40 +47,12 @@ void metaFileShouldMayContain() throws Exception { assertThat(databaseMetadata.maybePrivacyVersion().get()).isEqualTo(55); } - @Test - void metaFileWithAllValuesInAnyOrder() throws Exception { - final Path tempDataDir = - createAndWrite( - "data", - "DATABASE_METADATA.json", - "{\"besuVersion\":\"23.10.4\", \"version\":42 , \"privacyVersion\":55}"); - - final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); - assertThat(databaseMetadata).isNotNull(); - assertThat(databaseMetadata.getVersion()).isEqualTo(42); - assertThat(databaseMetadata.maybePrivacyVersion()).isNotEmpty(); - assertThat(databaseMetadata.maybePrivacyVersion().get()).isEqualTo(55); - assertThat(databaseMetadata.getBesuVersion()).isEqualTo("23.10.4"); - } - @Test void metaFileShouldBeSoughtIntoDataDirFirst() throws Exception { final Path tempDataDir = createAndWrite("data", "DATABASE_METADATA.json", "{\"version\":42}"); final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); assertThat(databaseMetadata).isNotNull(); assertThat(databaseMetadata.getVersion()).isEqualTo(42); - assertThat(databaseMetadata.getBesuVersion()).isEqualTo("UNKNOWN"); - } - - @Test - void metaFileShouldBeParsedForBesuVersion() throws Exception { - final Path tempDataDir = - createAndWrite( - "data", "DATABASE_METADATA.json", "{\"version\":42, \"besuVersion\":\"23.10.4\"}"); - final DatabaseMetadata databaseMetadata = DatabaseMetadata.lookUpFrom(tempDataDir); - assertThat(databaseMetadata).isNotNull(); - assertThat(databaseMetadata.getVersion()).isEqualTo(42); - assertThat(databaseMetadata.getBesuVersion()).isEqualTo("23.10.4"); } private Path createAndWrite(final String dir, final String file, final String content)