diff --git a/.build/build-rat.xml b/.build/build-rat.xml index 27e8f63ae43d..a4dd053def9a 100644 --- a/.build/build-rat.xml +++ b/.build/build-rat.xml @@ -59,6 +59,7 @@ + diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java index 841f7b305198..312017a62776 100644 --- a/src/java/org/apache/cassandra/db/SerializationHeader.java +++ b/src/java/org/apache/cassandra/db/SerializationHeader.java @@ -276,6 +276,18 @@ public static class Component extends MetadataComponent private final Map> regularColumns; private final EncodingStats stats; + /** + * Only exposed for {@link org.apache.cassandra.io.sstable.SSTableHeaderFix}. + */ + public static Component buildComponentForTools(AbstractType keyType, + List> clusteringTypes, + Map> staticColumns, + Map> regularColumns, + EncodingStats stats) + { + return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats); + } + private Component(AbstractType keyType, List> clusteringTypes, Map> staticColumns, diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java index 934a17b06494..10c591a87d5f 100644 --- a/src/java/org/apache/cassandra/service/StartupChecks.java +++ b/src/java/org/apache/cassandra/service/StartupChecks.java @@ -70,6 +70,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JavaUtils; import org.apache.cassandra.utils.NativeLibrary; @@ -133,6 +134,7 @@ public enum StartupCheckType // always want the system keyspace check run last, as this actually loads the schema for that // keyspace. All other checks should not require any schema initialization. private final List DEFAULT_TESTS = ImmutableList.of(checkKernelBug1057843, + checkPreviousVersion, checkJemalloc, checkLz4Native, checkValidLaunchDate, @@ -249,6 +251,18 @@ public void execute(StartupChecksOptions startupChecksOptions) throws StartupExc } }; + public static final StartupCheck checkPreviousVersion = new StartupCheck() + { + @Override + public void execute(StartupChecksOptions options) throws StartupException + { + String previousVersionString = FBUtilities.getPreviousReleaseVersionString(); + if (null != previousVersionString && new CassandraVersion(previousVersionString).major <= 3) + throw new StartupException(StartupException.ERR_WRONG_DISK_STATE, + String.format("Only upgrades from >=4.0 are supported, found %s", previousVersionString)); + } + }; + public static final StartupCheck checkJemalloc = new StartupCheck() { @Override diff --git a/src/java/org/apache/cassandra/tools/SSTableHeaderFix.java b/src/java/org/apache/cassandra/tools/SSTableHeaderFix.java new file mode 100644 index 000000000000..c8d3f77f50fd --- /dev/null +++ b/src/java/org/apache/cassandra/tools/SSTableHeaderFix.java @@ -0,0 +1,969 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cql3.CQL3Type; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.marshal.AbstractCompositeType; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.DynamicCompositeType; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.SetType; +import org.apache.cassandra.db.marshal.TupleType; +import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.DroppedColumn; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; + +import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_SKIP_AUTOMATIC_UDT_FIX; + +/** + * Validates and fixes type issues in the serialization-header of sstables. + */ +public abstract class SSTableHeaderFix +{ + // C* 3.0 upgrade code + private static final boolean SKIP_AUTOMATIC_FIX_ON_UPGRADE = CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getBoolean(); + + @VisibleForTesting + public static void fixDroppedNonFrozenUDTIfUpgradeFrom30() + { + String previousVersionString = FBUtilities.getPreviousReleaseVersionString(); + if (previousVersionString == null) + return; + CassandraVersion previousVersion = new CassandraVersion(previousVersionString); + if (previousVersion.major != 3 || previousVersion.minor > 0) + { + // Not an upgrade from 3.0, nothing to do here + return; + } + + boolean hasDroppedFrozenColumn = false; + for (String keyspace : Schema.instance.getUserKeyspaces()) + for (TableMetadata tableMetadata : Schema.instance.getTablesAndViews(keyspace)) + if (!tableMetadata.isIndex() && !tableMetadata.isView() && !tableMetadata.isVirtual()) + hasDroppedFrozenColumn |= hasDroppedFrozenColumn(tableMetadata.droppedColumns.values()); + + if (!hasDroppedFrozenColumn) + { + logger.info("No dropped frozen columns found - NOT fixing sstable metadata serialization-headers"); + return; + } + + if (SKIP_AUTOMATIC_FIX_ON_UPGRADE) + { + logger.warn("Detected upgrade from {} to {}, but -D{}=true, NOT fixing UDT type references in " + + "sstable metadata serialization-headers", + previousVersionString, + FBUtilities.getReleaseVersionString(), + CASSANDRA_SKIP_AUTOMATIC_UDT_FIX.getKey()); + return; + } + + logger.info("Detected upgrade from {} to {}, and dropped UDT column, fixing UDT type references in sstable metadata serialization-headers", + previousVersionString, + FBUtilities.getReleaseVersionString()); + + SSTableHeaderFix instance = SSTableHeaderFix.builder() + .schemaCallback(() -> Schema.instance::getTableMetadata) + .build(); + instance.execute(); + } + + // "regular" SSTableHeaderFix code, also used by StandaloneScrubber. + + private static final Logger logger = LoggerFactory.getLogger(SSTableHeaderFix.class); + + protected final Consumer info; + protected final Consumer warn; + protected final Consumer error; + protected final boolean dryRun; + protected final Function schemaCallback; + + private final List descriptors; + + private final List>> updates = new ArrayList<>(); + private boolean hasErrors; + + SSTableHeaderFix(Builder builder) + { + this.info = builder.info; + this.warn = builder.warn; + this.error = builder.error; + this.dryRun = builder.dryRun; + this.schemaCallback = builder.schemaCallback.get(); + this.descriptors = new ArrayList<>(builder.descriptors); + Objects.requireNonNull(this.info, "info is null"); + Objects.requireNonNull(this.warn, "warn is null"); + Objects.requireNonNull(this.error, "error is null"); + Objects.requireNonNull(this.schemaCallback, "schemaCallback is null"); + } + + public static Builder builder() + { + return new Builder(); + } + + /** + * Builder to configure and construct an instance of {@link SSTableHeaderFix}. + * Default settings: + *
    + *
  • log via the slf4j logger of {@link SSTableHeaderFix}
  • + *
  • no dry-run (i.e. validate and fix, if no serious errors are detected)
  • + *
  • no schema callback
  • + *
+ * If neither {@link #withDescriptor(Descriptor)} nor {@link #withPath(Path)} are used, + * all "live" sstables in all data directories will be scanned. + */ + public static class Builder + { + private final List paths = new ArrayList<>(); + private final List descriptors = new ArrayList<>(); + private Consumer info = (ln) -> logger.info("{}", ln); + private Consumer warn = (ln) -> logger.warn("{}", ln); + private Consumer error = (ln) -> logger.error("{}", ln); + private boolean dryRun; + private Supplier> schemaCallback = () -> null; + + private Builder() + {} + + /** + * Only validate and prepare fix, but do not write updated (fixed) sstable serialization-headers. + */ + public Builder dryRun() + { + dryRun = true; + return this; + } + + public Builder info(Consumer output) + { + this.info = output; + return this; + } + + public Builder warn(Consumer warn) + { + this.warn = warn; + return this; + } + + public Builder error(Consumer error) + { + this.error = error; + return this; + } + + /** + * Manually provide an individual sstable or directory containing sstables. + * + * Implementation note: procesing "live" sstables in their data directories as well as sstables + * in snapshots and backups in the data directories works. + * + * But processing sstables that reside somewhere else (i.e. verifying sstables before import) + * requires the use of {@link #withDescriptor(Descriptor)}. + */ + public Builder withPath(Path path) + { + this.paths.add(path); + return this; + } + + public Builder withDescriptor(Descriptor descriptor) + { + this.descriptors.add(descriptor); + return this; + } + + /** + * Schema callback to retrieve the schema of a table. Production code always delegates to the + * live schema ({@code Schema.instance}). Unit tests use this method to feed a custom schema. + */ + public Builder schemaCallback(Supplier> schemaCallback) + { + this.schemaCallback = schemaCallback; + return this; + } + + public SSTableHeaderFix build() + { + if (paths.isEmpty() && descriptors.isEmpty()) + return new AutomaticHeaderFix(this); + + return new ManualHeaderFix(this); + } + + public Builder logToList(List output) + { + return info(ln -> output.add("INFO " + ln)) + .warn(ln -> output.add("WARN " + ln)) + .error(ln -> output.add("ERROR " + ln)); + } + } + + public final void execute() + { + prepare(); + + logger.debug("Processing {} sstables:{}", + descriptors.size(), + descriptors.stream().map(Descriptor::toString).collect(Collectors.joining("\n ", "\n ", ""))); + + descriptors.forEach(this::processSSTable); + + if (updates.isEmpty()) + return; + + if (hasErrors) + { + info.accept("Stopping due to previous errors. Either fix the errors or specify the ignore-errors option."); + return; + } + + if (dryRun) + { + info.accept("Not fixing identified and fixable serialization-header issues."); + return; + } + + info.accept("Writing new metadata files"); + updates.forEach(descAndMeta -> writeNewMetadata(descAndMeta.left, descAndMeta.right)); + info.accept("Finished writing new metadata files"); + } + + /** + * Whether {@link #execute()} encountered an error. + */ + public boolean hasError() + { + return hasErrors; + } + + /** + * Whether {@link #execute()} found mismatches. + */ + public boolean hasChanges() + { + return !updates.isEmpty(); + } + + abstract void prepare(); + + private void error(String format, Object... args) + { + hasErrors = true; + error.accept(String.format(format, args)); + } + + void processFileOrDirectory(Path path) + { + Stream.of(path) + .flatMap(SSTableHeaderFix::maybeExpandDirectory) + .filter(p -> { + try + { + return Descriptor.fromFileWithComponent(new File(p)).right.type == Types.DATA; + } + catch (IllegalArgumentException t) + { + logger.info("Couldn't parse filename {}, ignoring", p); + return false; + } + }) + .map(File::new) + .map(file -> Descriptor.fromFileWithComponent(file, false).left) + .forEach(descriptors::add); + } + + private static Stream maybeExpandDirectory(Path path) + { + if (Files.isRegularFile(path)) + return Stream.of(path); + return LifecycleTransaction.getFiles(path, (file, fileType) -> fileType == Directories.FileType.FINAL, Directories.OnTxnErr.IGNORE) + .stream() + .map(File::toPath); + } + + private static boolean hasDroppedFrozenColumn(Collection droppedColumns) + { + boolean hasDroppedFrozenColumn = false; + for (DroppedColumn droppedColumn : droppedColumns) + hasDroppedFrozenColumn |= droppedColumn.column.type.equals(droppedColumn.column.type.freeze()) + && multiCellCapableType(droppedColumn.column.type); + + return hasDroppedFrozenColumn; + } + + private static boolean multiCellCapableType(AbstractType columnType) + { + return columnType instanceof CollectionType || columnType instanceof TupleType || columnType instanceof UserType; + } + + private void processSSTable(Descriptor desc) + { + if (desc.cfname.indexOf('.') != -1) + { + // secondary index not checked + + // partition-key is the indexed column type + // clustering-key is org.apache.cassandra.db.marshal.PartitionerDefinedOrder + // no static columns, no regular columns + return; + } + + TableMetadata tableMetadata = schemaCallback.apply(desc); + if (tableMetadata == null) + { + error("Table %s.%s not found in the schema - NOT checking sstable %s", desc.ksname, desc.cfname, desc); + return; + } + + if (!hasDroppedFrozenColumn(tableMetadata.droppedColumns.values())) + { + info.accept(String.format("Table %s.%s has no dropped frozen columns - NOT checking sstable %s", desc.ksname, desc.cfname, desc)); + return; + } + + Set components = desc.discoverComponents(); + if (components.stream().noneMatch(c -> c.type == Types.STATS)) + { + error("sstable %s has no -Statistics.db component.", desc); + return; + } + + Map metadata = readSSTableMetadata(desc); + if (metadata == null) + return; + + MetadataComponent component = metadata.get(MetadataType.HEADER); + if (!(component instanceof SerializationHeader.Component)) + { + error("sstable %s: Expected %s, but got %s from metadata.get(MetadataType.HEADER)", + desc, + SerializationHeader.Component.class.getName(), + component != null ? component.getClass().getName() : "'null'"); + return; + } + SerializationHeader.Component header = (SerializationHeader.Component) component; + + // check partition key type + AbstractType keyType = validatePartitionKey(desc, tableMetadata, header); + + // check clustering columns + List> clusteringTypes = validateClusteringColumns(desc, tableMetadata, header); + + // check static and regular columns + LinkedHashMap> staticColumns = validateColumns(desc, tableMetadata, header.getStaticColumns(), ColumnMetadata.Kind.STATIC); + LinkedHashMap> regularColumns = validateColumns(desc, tableMetadata, header.getRegularColumns(), ColumnMetadata.Kind.REGULAR); + + SerializationHeader.Component newHeader = SerializationHeader.Component.buildComponentForTools(keyType, + clusteringTypes, + staticColumns, + regularColumns, + header.getEncodingStats()); + + // SerializationHeader.Component has no equals(), but a "good" toString() + if (header.toString().equals(newHeader.toString())) + return; + + Map newMetadata = new LinkedHashMap<>(metadata); + newMetadata.put(MetadataType.HEADER, newHeader); + + updates.add(Pair.create(desc, newMetadata)); + } + + private AbstractType validatePartitionKey(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header) + { + boolean keyMismatch = false; + AbstractType headerKeyType = header.getKeyType(); + AbstractType schemaKeyType = tableMetadata.partitionKeyType; + boolean headerKeyComposite = headerKeyType instanceof CompositeType; + boolean schemaKeyComposite = schemaKeyType instanceof CompositeType; + if (headerKeyComposite != schemaKeyComposite) + { + // one is a composite partition key, the other is not - very suspicious + keyMismatch = true; + } + else if (headerKeyComposite) // && schemaKeyComposite + { + // Note, the logic is similar as just calling 'fixType()' using the composite partition key, + // but the log messages should use the composite partition key column names. + List> headerKeyComponents = ((CompositeType) headerKeyType).types; + List> schemaKeyComponents = ((CompositeType) schemaKeyType).types; + if (headerKeyComponents.size() != schemaKeyComponents.size()) + { + // different number of components in composite partition keys - very suspicious + keyMismatch = true; + // Just use the original type from the header. Since the number of partition key components + // don't match, there's nothing to meaningfully validate against. + } + else + { + // fix components in composite partition key, if necessary + List> newComponents = new ArrayList<>(schemaKeyComponents.size()); + for (int i = 0; i < schemaKeyComponents.size(); i++) + { + AbstractType headerKeyComponent = headerKeyComponents.get(i); + AbstractType schemaKeyComponent = schemaKeyComponents.get(i); + AbstractType fixedType = fixType(desc, + tableMetadata.partitionKeyColumns().get(i).name.bytes, + headerKeyComponent, + schemaKeyComponent, + false); + if (fixedType == null) + keyMismatch = true; + else + headerKeyComponent = fixedType; + newComponents.add(fixType(desc, + tableMetadata.partitionKeyColumns().get(i).name.bytes, + headerKeyComponent, + schemaKeyComponent, + false)); + } + headerKeyType = CompositeType.getInstance(newComponents); + } + } + else + { + // fix non-composite partition key, if necessary + AbstractType fixedType = fixType(desc, tableMetadata.partitionKeyColumns().get(0).name.bytes, headerKeyType, schemaKeyType, false); + if (fixedType == null) + // non-composite partition key doesn't match and cannot be fixed + keyMismatch = true; + else + headerKeyType = fixedType; + } + if (keyMismatch) + error("sstable %s: Mismatch in partition key type between sstable serialization-header and schema (%s vs %s)", + desc, + headerKeyType.asCQL3Type(), + schemaKeyType.asCQL3Type()); + return headerKeyType; + } + + private List> validateClusteringColumns(Descriptor desc, TableMetadata tableMetadata, SerializationHeader.Component header) + { + List> headerClusteringTypes = header.getClusteringTypes(); + List> clusteringTypes = new ArrayList<>(); + boolean clusteringMismatch = false; + List schemaClustering = tableMetadata.clusteringColumns(); + if (schemaClustering.size() != headerClusteringTypes.size()) + { + clusteringMismatch = true; + // Just use the original types. Since the number of clustering columns don't match, there's nothing to + // meaningfully validate against. + clusteringTypes.addAll(headerClusteringTypes); + } + else + { + for (int i = 0; i < headerClusteringTypes.size(); i++) + { + AbstractType headerType = headerClusteringTypes.get(i); + ColumnMetadata column = schemaClustering.get(i); + AbstractType schemaType = column.type; + AbstractType fixedType = fixType(desc, column.name.bytes, headerType, schemaType, false); + if (fixedType == null) + clusteringMismatch = true; + else + headerType = fixedType; + clusteringTypes.add(headerType); + } + } + if (clusteringMismatch) + error("sstable %s: mismatch in clustering columns between sstable serialization-header and schema (%s vs %s)", + desc, + headerClusteringTypes.stream().map(AbstractType::asCQL3Type).map(CQL3Type::toString).collect(Collectors.joining(",")), + schemaClustering.stream().map(cd -> cd.type.asCQL3Type().toString()).collect(Collectors.joining(","))); + return clusteringTypes; + } + + private LinkedHashMap> validateColumns(Descriptor desc, TableMetadata tableMetadata, Map> columns, ColumnMetadata.Kind kind) + { + LinkedHashMap> target = new LinkedHashMap<>(); + for (Map.Entry> nameAndType : columns.entrySet()) + { + ByteBuffer name = nameAndType.getKey(); + AbstractType type = nameAndType.getValue(); + + AbstractType fixedType = validateColumn(desc, tableMetadata, kind, name, type); + if (fixedType == null) + { + error("sstable %s: contains column '%s' of type '%s', which could not be validated", + desc, + type, + logColumnName(name)); + // don't use a "null" type instance + fixedType = type; + } + + target.put(name, fixedType); + } + return target; + } + + private AbstractType validateColumn(Descriptor desc, TableMetadata tableMetadata, ColumnMetadata.Kind kind, ByteBuffer name, AbstractType type) + { + ColumnMetadata cd = tableMetadata.getColumn(name); + if (cd == null) + { + // In case the column was dropped, there is not much that we can actually validate. + // The column could have been recreated using the same or a different kind or the same or + // a different type. Lottery... + + cd = tableMetadata.getDroppedColumn(name, kind == ColumnMetadata.Kind.STATIC); + if (cd == null) + { + for (IndexMetadata indexMetadata : tableMetadata.indexes) + { + String target = indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME); + if (target != null && ByteBufferUtil.bytes(target).equals(name)) + { + warn.accept(String.format("sstable %s: contains column '%s', which is not a column in the table '%s.%s', but a target for that table's index '%s'", + desc, + logColumnName(name), + tableMetadata.keyspace, + tableMetadata.name, + indexMetadata.name)); + return type; + } + } + + warn.accept(String.format("sstable %s: contains column '%s', which is not present in the schema", + desc, + logColumnName(name))); + } + else + { + // This is a best-effort approach to handle the case of a UDT column created *AND* dropped in + // C* 3.0. + if (type instanceof UserType && cd.type instanceof TupleType) + { + // At this point, we know that the type belongs to a dropped column, recorded with the + // dropped column type "TupleType" and using "UserType" in the sstable. So it is very + // likely, that this belongs to a dropped UDT. Fix that information to tuple-type. + return fixType(desc, name, type, cd.type, true); + } + } + + return type; + } + + // At this point, the column name is known to be a "non-dropped" column in the table. + if (cd.kind != kind) + error("sstable %s: contains column '%s' as a %s column, but is of kind %s in the schema", + desc, + logColumnName(name), + kind.name().toLowerCase(), + cd.kind.name().toLowerCase()); + else + type = fixType(desc, name, type, cd.type, false); + return type; + } + + private AbstractType fixType(Descriptor desc, ByteBuffer name, AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) + { + AbstractType fixedType = fixTypeInner(typeInHeader, typeInSchema, droppedColumnMode); + if (fixedType != null) + { + if (fixedType != typeInHeader) + info.accept(String.format("sstable %s: Column '%s' needs to be updated from type '%s' to '%s'", + desc, + logColumnName(name), + typeInHeader.asCQL3Type(), + fixedType.asCQL3Type())); + return fixedType; + } + + error("sstable %s: contains column '%s' as type '%s', but schema mentions '%s'", + desc, + logColumnName(name), + typeInHeader.asCQL3Type(), + typeInSchema.asCQL3Type()); + + return typeInHeader; + } + + private AbstractType fixTypeInner(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) + { + if (typeEquals(typeInHeader, typeInSchema)) + return typeInHeader; + + if (typeInHeader instanceof CollectionType) + return fixTypeInnerCollection(typeInHeader, typeInSchema, droppedColumnMode); + + if (typeInHeader instanceof AbstractCompositeType) + return fixTypeInnerAbstractComposite(typeInHeader, typeInSchema, droppedColumnMode); + + if (typeInHeader instanceof TupleType) + return fixTypeInnerAbstractTuple(typeInHeader, typeInSchema, droppedColumnMode); + + // all types, beside CollectionType + AbstractCompositeType + TupleType, should be ok (no nested types) - just check for compatibility + if (typeInHeader.isCompatibleWith(typeInSchema)) + return typeInHeader; + + return null; + } + + private AbstractType fixTypeInnerAbstractTuple(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) + { + // This first 'if' handles the case when a UDT has been dropped, as a dropped UDT is recorded as a tuple + // in dropped_columns. If a UDT is to be replaced with a tuple, then also do that for the inner UDTs. + if (droppedColumnMode && typeInHeader.getClass() == UserType.class && typeInSchema instanceof TupleType) + return fixTypeInnerUserTypeDropped((UserType) typeInHeader, (TupleType) typeInSchema); + + if (typeInHeader.getClass() != typeInSchema.getClass()) + return null; + + if (typeInHeader.getClass() == UserType.class) + return fixTypeInnerUserType((UserType) typeInHeader, (UserType) typeInSchema); + + if (typeInHeader.getClass() == TupleType.class) + return fixTypeInnerTuple((TupleType) typeInHeader, (TupleType) typeInSchema, droppedColumnMode); + + throw new IllegalArgumentException("Unknown tuple type class " + typeInHeader.getClass().getName()); + } + + private AbstractType fixTypeInnerCollection(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) + { + if (typeInHeader.getClass() != typeInSchema.getClass()) + return null; + + if (typeInHeader.getClass() == ListType.class) + return fixTypeInnerList((ListType) typeInHeader, (ListType) typeInSchema, droppedColumnMode); + + if (typeInHeader.getClass() == SetType.class) + return fixTypeInnerSet((SetType) typeInHeader, (SetType) typeInSchema, droppedColumnMode); + + if (typeInHeader.getClass() == MapType.class) + return fixTypeInnerMap((MapType) typeInHeader, (MapType) typeInSchema, droppedColumnMode); + + throw new IllegalArgumentException("Unknown collection type class " + typeInHeader.getClass().getName()); + } + + private AbstractType fixTypeInnerAbstractComposite(AbstractType typeInHeader, AbstractType typeInSchema, boolean droppedColumnMode) + { + if (typeInHeader.getClass() != typeInSchema.getClass()) + return null; + + if (typeInHeader.getClass() == CompositeType.class) + return fixTypeInnerComposite((CompositeType) typeInHeader, (CompositeType) typeInSchema, droppedColumnMode); + + if (typeInHeader.getClass() == DynamicCompositeType.class) + { + // Not sure if we should care about UDTs in DynamicCompositeType at all... + if (!typeInHeader.isCompatibleWith(typeInSchema)) + return null; + + return typeInHeader; + } + + throw new IllegalArgumentException("Unknown composite type class " + typeInHeader.getClass().getName()); + } + + private AbstractType fixTypeInnerUserType(UserType cHeader, UserType cSchema) + { + if (!cHeader.keyspace.equals(cSchema.keyspace) || !cHeader.name.equals(cSchema.name)) + // different UDT - bummer... + return null; + + if (cHeader.isMultiCell() != cSchema.isMultiCell()) + { + if (cHeader.isMultiCell() && !cSchema.isMultiCell()) + { + // C* 3.0 writes broken SerializationHeader.Component instances - i.e. broken UDT type + // definitions into the sstable -Stats.db file, because 3.0 does not enclose frozen UDTs + // (and all UDTs in 3.0 were frozen) with an '' bracket. Since CASSANDRA-7423 (support + // for non-frozen UDTs, committed to C* 3.6), that frozen-bracket is quite important. + // Non-frozen (= multi-cell) UDTs are serialized in a fundamentally different way than + // frozen UDTs in sstables - most importantly, the order of serialized columns depends on + // the type: fixed-width types first, then variable length types (like frozen types), + // multi-cell types last. If C* >= 3.6 reads an sstable with a UDT that's written by + // C* < 3.6, a variety of CorruptSSTableExceptions get logged and clients will encounter + // read errors. + // At this point, we know that the type belongs to a "live" (non-dropped) column, so it + // is safe to correct the information from the header. + return cSchema; + } + + // In all other cases, there's not much we can do. + return null; + } + + return cHeader; + } + + private AbstractType fixTypeInnerUserTypeDropped(UserType cHeader, TupleType cSchema) + { + // Do not mess around with the UserType in the serialization header, if the column has been dropped. + // Only fix the multi-cell status when the header contains it as a multicell (non-frozen) UserType, + // but the schema says "frozen". + if (cHeader.isMultiCell() && !cSchema.isMultiCell()) + { + return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), cHeader.fieldTypes(), cSchema.isMultiCell()); + } + + return cHeader; + } + + private AbstractType fixTypeInnerTuple(TupleType cHeader, TupleType cSchema, boolean droppedColumnMode) + { + if (cHeader.size() != cSchema.size()) + // different number of components - bummer... + return null; + List> cHeaderFixed = new ArrayList<>(cHeader.size()); + boolean anyChanged = false; + for (int i = 0; i < cHeader.size(); i++) + { + AbstractType cHeaderComp = cHeader.type(i); + AbstractType cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.type(i), droppedColumnMode); + if (cHeaderCompFixed == null) + // incompatible, bummer... + return null; + cHeaderFixed.add(cHeaderCompFixed); + anyChanged |= cHeaderComp != cHeaderCompFixed; + } + if (anyChanged || cSchema.isMultiCell() != cHeader.isMultiCell()) + // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT + return new TupleType(cHeaderFixed); + return cHeader; + } + + private AbstractType fixTypeInnerComposite(CompositeType cHeader, CompositeType cSchema, boolean droppedColumnMode) + { + if (cHeader.types.size() != cSchema.types.size()) + // different number of components - bummer... + return null; + List> cHeaderFixed = new ArrayList<>(cHeader.types.size()); + boolean anyChanged = false; + for (int i = 0; i < cHeader.types.size(); i++) + { + AbstractType cHeaderComp = cHeader.types.get(i); + AbstractType cHeaderCompFixed = fixTypeInner(cHeaderComp, cSchema.types.get(i), droppedColumnMode); + if (cHeaderCompFixed == null) + // incompatible, bummer... + return null; + cHeaderFixed.add(cHeaderCompFixed); + anyChanged |= cHeaderComp != cHeaderCompFixed; + } + if (anyChanged) + return CompositeType.getInstance(cHeaderFixed); + return cHeader; + } + + private AbstractType fixTypeInnerList(ListType cHeader, ListType cSchema, boolean droppedColumnMode) + { + AbstractType cHeaderElem = cHeader.getElementsType(); + AbstractType cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode); + if (cHeaderElemFixed == null) + // bummer... + return null; + if (cHeaderElem != cHeaderElemFixed) + // element type changed + return ListType.getInstance(cHeaderElemFixed, cHeader.isMultiCell()); + return cHeader; + } + + private AbstractType fixTypeInnerSet(SetType cHeader, SetType cSchema, boolean droppedColumnMode) + { + AbstractType cHeaderElem = cHeader.getElementsType(); + AbstractType cHeaderElemFixed = fixTypeInner(cHeaderElem, cSchema.getElementsType(), droppedColumnMode); + if (cHeaderElemFixed == null) + // bummer... + return null; + if (cHeaderElem != cHeaderElemFixed) + // element type changed + return SetType.getInstance(cHeaderElemFixed, cHeader.isMultiCell()); + return cHeader; + } + + private AbstractType fixTypeInnerMap(MapType cHeader, MapType cSchema, boolean droppedColumnMode) + { + AbstractType cHeaderKey = cHeader.getKeysType(); + AbstractType cHeaderVal = cHeader.getValuesType(); + AbstractType cHeaderKeyFixed = fixTypeInner(cHeaderKey, cSchema.getKeysType(), droppedColumnMode); + AbstractType cHeaderValFixed = fixTypeInner(cHeaderVal, cSchema.getValuesType(), droppedColumnMode); + if (cHeaderKeyFixed == null || cHeaderValFixed == null) + // bummer... + return null; + if (cHeaderKey != cHeaderKeyFixed || cHeaderVal != cHeaderValFixed) + // element type changed + return MapType.getInstance(cHeaderKeyFixed, cHeaderValFixed, cHeader.isMultiCell()); + return cHeader; + } + + private boolean typeEquals(AbstractType typeInHeader, AbstractType typeInSchema) + { + // Quite annoying, but the implementations of equals() on some implementation of AbstractType seems to be + // wrong, but toString() seems to work in such cases. + return typeInHeader.equals(typeInSchema) || typeInHeader.toString().equals(typeInSchema.toString()); + } + + private static String logColumnName(ByteBuffer columnName) + { + try + { + return ByteBufferUtil.string(columnName); + } + catch (CharacterCodingException e) + { + return "?? " + e; + } + } + + private Map readSSTableMetadata(Descriptor desc) + { + Map metadata; + try + { + metadata = desc.getMetadataSerializer().deserialize(desc, EnumSet.allOf(MetadataType.class)); + } + catch (IOException e) + { + error("Failed to deserialize metadata for sstable %s: %s", desc, e.toString()); + return null; + } + return metadata; + } + + private void writeNewMetadata(Descriptor desc, Map newMetadata) + { + File file = desc.fileFor(Components.STATS); + info.accept(String.format(" Writing new metadata file %s", file)); + try + { + desc.getMetadataSerializer().rewriteSSTableMetadata(desc, newMetadata); + } + catch (IOException e) + { + error("Failed to write metadata component for %s: %s", file, e.toString()); + throw new RuntimeException(e); + } + } + + /** + * Fix individually provided sstables or directories containing sstables. + */ + static class ManualHeaderFix extends SSTableHeaderFix + { + private final List paths; + + ManualHeaderFix(Builder builder) + { + super(builder); + this.paths = builder.paths; + } + + public void prepare() + { + paths.forEach(this::processFileOrDirectory); + } + } + + /** + * Fix all sstables in the configured data-directories. + */ + static class AutomaticHeaderFix extends SSTableHeaderFix + { + AutomaticHeaderFix(Builder builder) + { + super(builder); + } + + public void prepare() + { + info.accept("Scanning all data directories..."); + for (Directories.DataDirectory dataDirectory : Directories.dataDirectories) + scanDataDirectory(dataDirectory); + info.accept("Finished scanning all data directories..."); + } + + private void scanDataDirectory(Directories.DataDirectory dataDirectory) + { + info.accept(String.format("Scanning data directory %s", dataDirectory.location)); + File[] ksDirs = dataDirectory.location.tryList(); + if (ksDirs == null) + return; + for (File ksDir : ksDirs) + { + if (!ksDir.isDirectory() || !ksDir.isReadable()) + continue; + + String name = ksDir.name(); + + // silently ignore all system keyspaces + if (SchemaConstants.isLocalSystemKeyspace(name) || SchemaConstants.isReplicatedSystemKeyspace(name)) + continue; + + File[] tabDirs = ksDir.tryList(); + if (tabDirs == null) + continue; + for (File tabDir : tabDirs) + { + if (!tabDir.isDirectory() || !tabDir.isReadable()) + continue; + + processFileOrDirectory(tabDir.toPath()); + } + } + } + } +} diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 36fda102e976..96423bdbb7bc 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -74,9 +74,6 @@ public class StandaloneScrubber private static final String SKIP_CORRUPTED_OPTION = "skip-corrupted"; private static final String NO_VALIDATE_OPTION = "no-validate"; private static final String REINSERT_OVERFLOWED_TTL_OPTION = "reinsert-overflowed-ttl"; - /** - * This option was logically removed from the code, but to avoid breaking backwards compatability the option remains - */ private static final String HEADERFIX_OPTION = "header-fix"; public static void main(String args[]) @@ -135,6 +132,66 @@ public static void main(String args[]) } System.out.println(String.format("Pre-scrub sstables snapshotted into snapshot %s", snapshotName)); + if (options.headerFixMode != Options.HeaderFixMode.OFF) + { + // Run the frozen-UDT checks _before_ the sstables are opened + + List logOutput = new ArrayList<>(); + + SSTableHeaderFix.Builder headerFixBuilder = SSTableHeaderFix.builder() + .logToList(logOutput) + .schemaCallback(() -> Schema.instance::getTableMetadata); + if (options.headerFixMode == Options.HeaderFixMode.VALIDATE) + headerFixBuilder = headerFixBuilder.dryRun(); + + for (Pair> p : listResult) + headerFixBuilder.withPath(p.left.fileFor(Components.DATA).toPath()); + + SSTableHeaderFix headerFix = headerFixBuilder.build(); + try + { + headerFix.execute(); + } + catch (Exception e) + { + JVMStabilityInspector.inspectThrowable(e); + if (options.debug) + e.printStackTrace(System.err); + } + + if (headerFix.hasChanges() || headerFix.hasError()) + logOutput.forEach(System.out::println); + + if (headerFix.hasError()) + { + System.err.println("Errors in serialization-header detected, aborting."); + System.exit(1); + } + + switch (options.headerFixMode) + { + case VALIDATE_ONLY: + case FIX_ONLY: + System.out.printf("Not continuing with scrub, since '--%s %s' was specified.%n", + HEADERFIX_OPTION, + options.headerFixMode.asCommandLineOption()); + System.exit(0); + case VALIDATE: + if (headerFix.hasChanges()) + { + System.err.printf("Unfixed, but fixable errors in serialization-header detected, aborting. " + + "Use a non-validating mode ('-e %s' or '-e %s') for --%s%n", + Options.HeaderFixMode.FIX.asCommandLineOption(), + Options.HeaderFixMode.FIX_ONLY.asCommandLineOption(), + HEADERFIX_OPTION); + System.exit(2); + } + break; + case FIX: + break; + } + } + List sstables = new ArrayList<>(); // Open sstables @@ -292,7 +349,17 @@ public static Options parseArgs(String cmdArgs[]) opts.checkData(!cmd.hasOption(NO_VALIDATE_OPTION)); opts.reinsertOverflowedTTLRows(cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION)); if (cmd.hasOption(HEADERFIX_OPTION)) - System.err.println(String.format("Option %s is deprecated and no longer functional", HEADERFIX_OPTION)); + { + try + { + opts.headerFixMode = HeaderFixMode.fromCommandLine(cmd.getOptionValue(HEADERFIX_OPTION)); + } + catch (Exception e) + { + errorMsg(String.format("Invalid argument value '%s' for --%s", cmd.getOptionValue(HEADERFIX_OPTION), HEADERFIX_OPTION), options); + return null; + } + } return opts; } catch (ParseException e) diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-CompressionInfo.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-CompressionInfo.db new file mode 100644 index 000000000000..5a82349dd2e2 Binary files /dev/null and b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Data.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Data.db new file mode 100644 index 000000000000..d281b869c654 Binary files /dev/null and b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Data.db differ diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Digest.crc32 b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Digest.crc32 new file mode 100644 index 000000000000..0778d5baa4e5 --- /dev/null +++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Digest.crc32 @@ -0,0 +1 @@ +798454228 \ No newline at end of file diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Filter.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Filter.db new file mode 100644 index 000000000000..8868e5c18008 Binary files /dev/null and b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Filter.db differ diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Partitions.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Partitions.db new file mode 100644 index 000000000000..b096fdac1490 Binary files /dev/null and b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Rows.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Rows.db new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Statistics.db b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Statistics.db new file mode 100644 index 000000000000..fdc9bb3bd64a Binary files /dev/null and b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-TOC.txt b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-TOC.txt new file mode 100644 index 000000000000..298910cfdc58 --- /dev/null +++ b/test/data/legacy-sstables/da/legacy_tables/legacy_da_tuple/da-1-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-CompressionInfo.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-CompressionInfo.db new file mode 100644 index 000000000000..8e08a5dca913 Binary files /dev/null and b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Data.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Data.db new file mode 100644 index 000000000000..38aa3536628b Binary files /dev/null and b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Data.db differ diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Digest.crc32 b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Digest.crc32 new file mode 100644 index 000000000000..90049599e034 --- /dev/null +++ b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Digest.crc32 @@ -0,0 +1 @@ +2353277624 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Filter.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Filter.db rename to test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Filter.db diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Index.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Index.db new file mode 100644 index 000000000000..0398b1f8a0da Binary files /dev/null and b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Index.db differ diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Statistics.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Statistics.db new file mode 100644 index 000000000000..99471825afc6 Binary files /dev/null and b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Summary.db b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Summary.db rename to test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-Summary.db diff --git a/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-TOC.txt b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-TOC.txt new file mode 100644 index 000000000000..4ab645afba51 --- /dev/null +++ b/test/data/legacy-sstables/ma/legacy_tables/legacy_ma_tuple/ma-1-big-TOC.txt @@ -0,0 +1,8 @@ +Summary.db +Data.db +Index.db +Statistics.db +Filter.db +TOC.txt +CompressionInfo.db +Digest.crc32 diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-CompressionInfo.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-CompressionInfo.db new file mode 100644 index 000000000000..53c889d0e457 Binary files /dev/null and b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Data.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Data.db new file mode 100644 index 000000000000..ec7c382817d3 Binary files /dev/null and b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Data.db differ diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Digest.crc32 b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Digest.crc32 new file mode 100644 index 000000000000..e86fd57a347b --- /dev/null +++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Digest.crc32 @@ -0,0 +1 @@ +768627854 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Filter.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Filter.db rename to test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Filter.db diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Index.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Index.db new file mode 100644 index 000000000000..3e735baf9ae0 Binary files /dev/null and b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Index.db differ diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Statistics.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Statistics.db new file mode 100644 index 000000000000..8ebf44bc2229 Binary files /dev/null and b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Summary.db b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Summary.db rename to test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-Summary.db diff --git a/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-TOC.txt b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-TOC.txt new file mode 100644 index 000000000000..8ea9eedf3bba --- /dev/null +++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_tuple/mb-1-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Digest.crc32 +Statistics.db +TOC.txt +Data.db +Index.db +Filter.db +Summary.db diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-CompressionInfo.db new file mode 100644 index 000000000000..8e08a5dca913 Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Data.db new file mode 100644 index 000000000000..73694b2f46ea Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Data.db differ diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Digest.crc32 new file mode 100644 index 000000000000..b57ebd32f353 --- /dev/null +++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Digest.crc32 @@ -0,0 +1 @@ +527605309 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Filter.db rename to test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Filter.db diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Index.db new file mode 100644 index 000000000000..0398b1f8a0da Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Index.db differ diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Statistics.db new file mode 100644 index 000000000000..fd80793b5b09 Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Summary.db rename to test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-Summary.db diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-TOC.txt new file mode 100644 index 000000000000..9e670e31e346 --- /dev/null +++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_tuple/mc-1-big-TOC.txt @@ -0,0 +1,8 @@ +Statistics.db +Data.db +Filter.db +CompressionInfo.db +TOC.txt +Summary.db +Digest.crc32 +Index.db diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-CompressionInfo.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-CompressionInfo.db new file mode 100644 index 000000000000..8e08a5dca913 Binary files /dev/null and b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Data.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Data.db new file mode 100644 index 000000000000..0c874f65fab2 Binary files /dev/null and b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Data.db differ diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Digest.crc32 b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Digest.crc32 new file mode 100644 index 000000000000..2fc48a9ce7f6 --- /dev/null +++ b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Digest.crc32 @@ -0,0 +1 @@ +1520549740 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Filter.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Filter.db rename to test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Filter.db diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Index.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Index.db new file mode 100644 index 000000000000..0398b1f8a0da Binary files /dev/null and b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Index.db differ diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Statistics.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Statistics.db new file mode 100644 index 000000000000..f4aa1057e088 Binary files /dev/null and b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Summary.db b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Summary.db rename to test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-Summary.db diff --git a/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-TOC.txt b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-TOC.txt new file mode 100644 index 000000000000..f2df26ca5694 --- /dev/null +++ b/test/data/legacy-sstables/md/legacy_tables/legacy_md_tuple/md-1-big-TOC.txt @@ -0,0 +1,8 @@ +TOC.txt +Data.db +Statistics.db +Summary.db +Index.db +Filter.db +CompressionInfo.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-CompressionInfo.db new file mode 100644 index 000000000000..5f53ae065801 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Data.db new file mode 100644 index 000000000000..2c082def965b Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..f4803c470ff2 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +3346976800 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Filter.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Filter.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Index.db new file mode 100644 index 000000000000..8c62daca3bbb Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Statistics.db new file mode 100644 index 000000000000..78bfef8ec528 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Summary.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-Summary.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Filter.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Filter.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Summary.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-Summary.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-CompressionInfo.db new file mode 100644 index 000000000000..dfbc93bfef7f Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Data.db new file mode 100644 index 000000000000..4b3dc5714cae Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..afe9d4969179 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +2310485613 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Filter.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Filter.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Index.db new file mode 100644 index 000000000000..a4f797dfc9fc Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Statistics.db new file mode 100644 index 000000000000..a2aaf2853b6b Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Summary.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-Summary.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Filter.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Filter.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Filter.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Summary.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Summary.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-Summary.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_compact/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-CompressionInfo.db new file mode 100644 index 000000000000..c0a4633f440a Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Data.db new file mode 100644 index 000000000000..5f592c4e84d1 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..17149972ebc6 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +704145780 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Index.db new file mode 100644 index 000000000000..baadf5f36441 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Statistics.db new file mode 100644 index 000000000000..c481e489b673 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_counter_compact/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Data.db new file mode 100644 index 000000000000..a630c4ff9c88 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..a51df1bb9234 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +1415075435 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Statistics.db new file mode 100644 index 000000000000..5db2740efe37 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-CompressionInfo.db new file mode 100644 index 000000000000..0b7faea4a8a4 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Index.db new file mode 100644 index 000000000000..b3094bffbad9 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Data.db new file mode 100644 index 000000000000..4243803bc107 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..99ffe738abee --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +2721811463 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Statistics.db new file mode 100644 index 000000000000..ae651c4e084d Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-CompressionInfo.db new file mode 100644 index 000000000000..adb7fc457930 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Index.db new file mode 100644 index 000000000000..56f29df1cb9b Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_compact/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-CompressionInfo.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-CompressionInfo.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-CompressionInfo.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Data.db new file mode 100644 index 000000000000..b8a8f0e6a043 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Digest.crc32 new file mode 100644 index 000000000000..80dc60278f12 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Digest.crc32 @@ -0,0 +1 @@ +3723331170 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Index.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Index.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Index.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Statistics.db new file mode 100644 index 000000000000..8bdebb654758 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-3025-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-CompressionInfo.db new file mode 100644 index 000000000000..56c95a8a3633 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Data.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Data.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Data.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Digest.crc32 similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Digest.crc32 rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Index.db new file mode 100644 index 000000000000..d094f73e2135 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Statistics.db similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-Statistics.db rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Statistics.db diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-TOC.txt similarity index 100% rename from test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-1-big-TOC.txt rename to test/data/legacy-sstables/me/legacy_tables/legacy_me_simple_counter_compact/me-31111-big-TOC.txt diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-CompressionInfo.db new file mode 100644 index 000000000000..916b491e39ce Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Data.db new file mode 100644 index 000000000000..a05ba8371a7e Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Digest.crc32 new file mode 100644 index 000000000000..6a41d2d577f6 --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Digest.crc32 @@ -0,0 +1 @@ +416438660 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Index.db new file mode 100644 index 000000000000..7cb91dc9b381 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Statistics.db new file mode 100644 index 000000000000..4fd6530c8610 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple/me-31111-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-CompressionInfo.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-CompressionInfo.db new file mode 100644 index 000000000000..916b491e39ce Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Data.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Data.db new file mode 100644 index 000000000000..af0f76965a48 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Data.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Digest.crc32 b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Digest.crc32 new file mode 100644 index 000000000000..48b3a9b8234e --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Digest.crc32 @@ -0,0 +1 @@ +601622112 \ No newline at end of file diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Filter.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Filter.db new file mode 100644 index 000000000000..2e1d5d29ca06 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Filter.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Index.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Index.db new file mode 100644 index 000000000000..7cb91dc9b381 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Index.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Statistics.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Statistics.db new file mode 100644 index 000000000000..f4d51e712701 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Statistics.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Summary.db b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-Summary.db differ diff --git a/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-TOC.txt b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-TOC.txt new file mode 100644 index 000000000000..aaf69afa269f --- /dev/null +++ b/test/data/legacy-sstables/me/legacy_tables/legacy_me_tuple_compact/me-31111-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +TOC.txt +Filter.db +Statistics.db +Data.db +Summary.db +Index.db +Digest.crc32 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_tuple/.keep b/test/data/legacy-sstables/na/legacy_tables/legacy_na_tuple/.keep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-CompressionInfo.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-CompressionInfo.db new file mode 100644 index 000000000000..ef683177e8f6 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Data.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Data.db new file mode 100644 index 000000000000..a72b4ee38017 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Data.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Digest.crc32 b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Digest.crc32 new file mode 100644 index 000000000000..682a656d0108 --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Digest.crc32 @@ -0,0 +1 @@ +626505151 \ No newline at end of file diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Filter.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Filter.db new file mode 100644 index 000000000000..8868e5c18008 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Filter.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Index.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Index.db new file mode 100644 index 000000000000..b3094bffbad9 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Index.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Statistics.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Statistics.db new file mode 100644 index 000000000000..836b7091cc73 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Statistics.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Summary.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-Summary.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-TOC.txt b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-TOC.txt new file mode 100644 index 000000000000..8484db90983c --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-3gll_1kke_3qdyo2i4kzfpt3cqcy-big-TOC.txt @@ -0,0 +1,8 @@ +Filter.db +Statistics.db +TOC.txt +Index.db +Summary.db +Digest.crc32 +Data.db +CompressionInfo.db diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-CompressionInfo.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-CompressionInfo.db new file mode 100644 index 000000000000..f69c841bc292 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Data.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Data.db new file mode 100644 index 000000000000..eda05d21c56d Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Data.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Digest.crc32 b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Digest.crc32 new file mode 100644 index 000000000000..3b8ed43b90e9 --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Digest.crc32 @@ -0,0 +1 @@ +4271820172 \ No newline at end of file diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Filter.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Filter.db new file mode 100644 index 000000000000..8868e5c18008 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Index.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Index.db new file mode 100644 index 000000000000..f33dab8bba25 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Index.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Statistics.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Statistics.db new file mode 100644 index 000000000000..a6348d64d015 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Summary.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-TOC.txt b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-TOC.txt new file mode 100644 index 000000000000..576c5e0598c5 --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-1-big-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Index.db +Summary.db diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-CompressionInfo.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-CompressionInfo.db new file mode 100644 index 000000000000..361f57d7d4df Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Data.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Data.db new file mode 100644 index 000000000000..e97f414ff026 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Data.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Digest.crc32 b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Digest.crc32 new file mode 100644 index 000000000000..7b1035c7ce92 --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Digest.crc32 @@ -0,0 +1 @@ +3757037919 \ No newline at end of file diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Filter.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Filter.db new file mode 100644 index 000000000000..8868e5c18008 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Filter.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Index.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Index.db new file mode 100644 index 000000000000..0398b1f8a0da Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Index.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Statistics.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Statistics.db new file mode 100644 index 000000000000..6dc52a3c8b8f Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Statistics.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Summary.db b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-Summary.db differ diff --git a/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-TOC.txt b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-TOC.txt new file mode 100644 index 000000000000..8484db90983c --- /dev/null +++ b/test/data/legacy-sstables/nb/legacy_tables/legacy_nb_tuple/nb-3gll_1kke_4abk02i4kzfpt3cqcy-big-TOC.txt @@ -0,0 +1,8 @@ +Filter.db +Statistics.db +TOC.txt +Index.db +Summary.db +Digest.crc32 +Data.db +CompressionInfo.db diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_tuple/.keep b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_tuple/.keep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-CompressionInfo.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-CompressionInfo.db new file mode 100644 index 000000000000..5a82349dd2e2 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Data.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Data.db new file mode 100644 index 000000000000..f605927b22c1 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Data.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Digest.crc32 b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Digest.crc32 new file mode 100644 index 000000000000..c5c321ba1866 --- /dev/null +++ b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Digest.crc32 @@ -0,0 +1 @@ +3312043736 \ No newline at end of file diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Filter.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Filter.db new file mode 100644 index 000000000000..8868e5c18008 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Filter.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Index.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Index.db new file mode 100644 index 000000000000..8d7a18f061e8 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Index.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Statistics.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Statistics.db new file mode 100644 index 000000000000..979a9a888ce4 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Statistics.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Summary.db b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Summary.db new file mode 100644 index 000000000000..9b24e0450c73 Binary files /dev/null and b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-Summary.db differ diff --git a/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-TOC.txt b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-TOC.txt new file mode 100644 index 000000000000..576c5e0598c5 --- /dev/null +++ b/test/data/legacy-sstables/oa/legacy_tables/legacy_oa_tuple/oa-1-big-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Index.db +Summary.db diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 3a1bf38e0349..da156a8bff6c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -19,13 +19,14 @@ import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Random; -import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.junit.After; @@ -66,6 +67,7 @@ import org.apache.cassandra.streaming.OutgoingStream; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.tools.SSTableHeaderFix; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.OutputHandler; @@ -95,13 +97,20 @@ public class LegacySSTableTest * When adding a new sstable version, add that one here. * See {@link #testGenerateSstables()} to generate sstables. * Take care on commit as you need to add the sstable files using {@code git add -f} + * + * There are two me sstables, where the sequence number indicates the C* version they come from. + * For example: + * me-3025-big-* sstables are generated from 3.0.25 + * me-31111-big-* sstables are generated from 3.11.11 + * Both exist because of differences introduced in 3.6 (and 3.11) in how frozen multi-cell headers are serialised + * without the sstable format `me` being bumped, ref CASSANDRA-15035 */ public static String[] legacyVersions = null; // Get all versions up to the current one. Useful for testing in compatibility mode C18301 private static String[] getValidLegacyVersions() { - String[] versions = {"oa", "da", "nb", "na", "me", "md", "mc", "mb", "ma"}; + String[] versions = {"da", "oa", "nb", "na", "me", "md", "mc", "mb", "ma"}; return Arrays.stream(versions).filter((v) -> v.compareTo(BigFormat.getInstance().getLatestVersion().toString()) <= 0).toArray(String[]::new); } @@ -138,7 +147,6 @@ public static void defineSchema() throws ConfigurationException { createTables(legacyVersion); } - } @After @@ -155,9 +163,12 @@ public void tearDown() */ protected Descriptor getDescriptor(String legacyVersion, String table) throws IOException { - File[] files = getTableDir(legacyVersion, table).list(); - Preconditions.checkArgument(files.length > 0, "No files for version=%s and table=%s", legacyVersion, table); - return Descriptor.fromFileWithComponent(files[0]).left; + Path file = Files.list(getTableDir(legacyVersion, table).toPath()) + .findFirst() + .orElseThrow(() -> new RuntimeException(String.format("No files for verion=%s and table=%s", legacyVersion, table))); + + // ignore intentionally empty directory .keep files + return ".keep".equals(file.toFile().getName()) ? null : Descriptor.fromFile(new File(file)); } @Test @@ -317,7 +328,7 @@ public void testStreamLegacyCqlTables() throws Exception public void testInaccurateSSTableMinMax() throws Exception { QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))"); - loadLegacyTable("legacy_%s_inaccurate_min_max", "mc"); + loadLegacyTable("mc", "inaccurate_min_max"); /* sstable has the following mutations: @@ -335,12 +346,59 @@ public void testInaccurateSSTableMinMax() throws Exception } @Test - public void testVerifyOldSSTables() throws IOException + public void testVerifyOldSimpleSSTables() throws IOException + { + verifyOldSSTables("simple"); + } + + @Test + public void testVerifyOldTupleSSTables() throws IOException + { + verifyOldSSTables("tuple"); + } + + @Test + public void testVerifyOldDroppedTupleSSTables() throws IOException + { + try { + for (String legacyVersion : legacyVersions) { + QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple DROP val", legacyVersion)); + QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple DROP val2", legacyVersion)); + QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple DROP val3", legacyVersion)); + // dropping non-frozen UDTs disabled, see AlterTableStatement.DropColumns.dropColumn(..) + //QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple DROP val4", legacyVersion)); + } + + verifyOldSSTables("tuple"); + } + finally + { + for (String legacyVersion : legacyVersions) { + alterTableAddColumn(legacyVersion, "val frozen,set>>"); + alterTableAddColumn(legacyVersion, "val2 tuple,set>"); + alterTableAddColumn(legacyVersion, String.format("val3 frozen", legacyVersion)); + // dropping non-frozen UDTs disabled, see AlterTableStatement.DropColumns.dropColumn(..) + //alterTableAddColumn(legacyVersion, String.format("val4 legacy_%s_tuple_udt", legacyVersion)); + } + } + } + + private static void alterTableAddColumn(String legacyVersion, String column_definition) { + // main-5.0 can use "IF NOT EXISTS" (and can also do all columns in one ALTER TABLEā€¦ + //QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple ADD IF NOT EXISTS %s", legacyVersion, column_definition)); + try + { + QueryProcessor.executeInternal(String.format("ALTER TABLE legacy_tables.legacy_%s_tuple ADD %s", legacyVersion, column_definition)); + } + catch (Throwable e) {} + } + + private void verifyOldSSTables(String tableSuffix) throws IOException { for (String legacyVersion : legacyVersions) { - ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)); - loadLegacyTable("legacy_%s_simple", legacyVersion); + ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_%s", legacyVersion, tableSuffix)); + loadLegacyTable(legacyVersion, tableSuffix); for (SSTableReader sstable : cfs.getLiveSSTables()) { @@ -374,7 +432,7 @@ public void testPendingAntiCompactionOldSSTables() throws Exception for (String legacyVersion : legacyVersions) { ColumnFamilyStore cfs = Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple", legacyVersion)); - loadLegacyTable("legacy_%s_simple", legacyVersion); + loadLegacyTable(legacyVersion, "simple"); boolean shouldFail = !cfs.getLiveSSTables().stream().allMatch(sstable -> sstable.descriptor.version.hasPendingRepair()); IPartitioner p = Iterables.getFirst(cfs.getLiveSSTables(), null).getPartitioner(); @@ -422,27 +480,33 @@ public void testAutomaticUpgrade() throws Exception private void streamLegacyTables(String legacyVersion) throws Exception { - logger.info("Streaming legacy version {}", legacyVersion); - streamLegacyTable("legacy_%s_simple", legacyVersion); - streamLegacyTable("legacy_%s_simple_counter", legacyVersion); - streamLegacyTable("legacy_%s_clust", legacyVersion); - streamLegacyTable("legacy_%s_clust_counter", legacyVersion); + logger.info("Streaming legacy version {}", legacyVersion); + streamLegacyTable("legacy_%s_simple", legacyVersion); + streamLegacyTable("legacy_%s_simple_counter", legacyVersion); + streamLegacyTable("legacy_%s_clust", legacyVersion); + streamLegacyTable("legacy_%s_clust_counter", legacyVersion); + streamLegacyTable("legacy_%s_tuple", legacyVersion); } private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception { String table = String.format(tablePattern, legacyVersion); - SSTableReader sstable = SSTableReader.open(null, getDescriptor(legacyVersion, table)); - IPartitioner p = sstable.getPartitioner(); - List> ranges = new ArrayList<>(); - ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100")))); - ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("100")), p.getMinimumToken())); - List streams = Lists.newArrayList(new CassandraOutgoingFile(StreamOperation.OTHER, - sstable.ref(), - sstable.getPositionsForRanges(ranges), - ranges, - sstable.estimatedKeysForRanges(ranges))); - new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get(); + Descriptor descriptor = getDescriptor(legacyVersion, table); + if (null != descriptor) { + SSTableReader sstable = SSTableReader.open(null, getDescriptor(legacyVersion, table)); + IPartitioner p = sstable.getPartitioner(); + List> ranges = new ArrayList<>(); + ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100")))); + ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("100")), p.getMinimumToken())); + + List streams = Lists.newArrayList(new CassandraOutgoingFile(StreamOperation.OTHER, + sstable.ref(), + sstable.getPositionsForRanges(ranges), + ranges, + sstable.estimatedKeysForRanges(ranges))); + + new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get(); + } } public static void truncateLegacyTables(String legacyVersion) throws Exception @@ -452,6 +516,9 @@ public static void truncateLegacyTables(String legacyVersion) throws Exception Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple_counter", legacyVersion)).truncateBlocking(); Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).truncateBlocking(); Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).truncateBlocking(); + Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_tuple", legacyVersion)).truncateBlocking(); + CacheService.instance.invalidateCounterCache(); + CacheService.instance.invalidateKeyCache(); } private static void compactLegacyTables(String legacyVersion) throws Exception @@ -461,15 +528,17 @@ private static void compactLegacyTables(String legacyVersion) throws Exception Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_simple_counter", legacyVersion)).forceMajorCompaction(); Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).forceMajorCompaction(); Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).forceMajorCompaction(); + Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_tuple", legacyVersion)).forceMajorCompaction(); } public static void loadLegacyTables(String legacyVersion) throws Exception { - logger.info("Preparing legacy version {}", legacyVersion); - loadLegacyTable("legacy_%s_simple", legacyVersion); - loadLegacyTable("legacy_%s_simple_counter", legacyVersion); - loadLegacyTable("legacy_%s_clust", legacyVersion); - loadLegacyTable("legacy_%s_clust_counter", legacyVersion); + logger.info("Preparing legacy version {}", legacyVersion); + loadLegacyTable(legacyVersion, "simple"); + loadLegacyTable(legacyVersion, "simple_counter"); + loadLegacyTable(legacyVersion, "clust"); + loadLegacyTable(legacyVersion, "clust_counter"); + loadLegacyTable(legacyVersion, "tuple"); } private static void verifyCache(String legacyVersion, long startCount) throws InterruptedException, java.util.concurrent.ExecutionException @@ -567,6 +636,21 @@ private static void createTables(String legacyVersion) QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple_counter (pk text PRIMARY KEY, val counter)", legacyVersion)); QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust (pk text, ck text, val text, PRIMARY KEY (pk, ck))", legacyVersion)); QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_counter (pk text, ck text, val counter, PRIMARY KEY (pk, ck))", legacyVersion)); + + + QueryProcessor.executeInternal(String.format("CREATE TYPE legacy_tables.legacy_%s_tuple_udt (name tuple)", legacyVersion)); + + if (legacyVersion.startsWith("m") && legacyVersion.compareTo("me") <= 0) + { + // sstable formats possibly from 3.0.x would have had a schema with everything frozen + QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%1$s_tuple (pk text PRIMARY KEY, " + + "val frozen,set>>, val2 frozen,set>>, val3 frozen, val4 frozen, extra text)", legacyVersion)); + } + else + { + QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%1$s_tuple (pk text PRIMARY KEY, " + + "val frozen,set>>, val2 tuple,set>, val3 frozen, val4 legacy_%1$s_tuple_udt, extra text)", legacyVersion)); + } } private static void truncateTables(String legacyVersion) @@ -592,9 +676,13 @@ private static void assertLegacyClustRows(int count, UntypedResultSet rs) } } - private static void loadLegacyTable(String tablePattern, String legacyVersion) throws IOException + private static void loadLegacyTable(String legacyVersion, String tableSuffix) throws IOException { - String table = String.format(tablePattern, legacyVersion); + String table = String.format("legacy_%s_%s", legacyVersion, tableSuffix); + + // ignore if no sstables are in this legacyVersion directory + if (0 == getTableDir(legacyVersion, table).tryList(f -> f.name().endsWith(".db")).length) + return; logger.info("Loading legacy table {}", table); @@ -605,6 +693,14 @@ private static void loadLegacyTable(String tablePattern, String legacyVersion) t copySstablesToTestData(legacyVersion, table, cfDir); } + if (legacyVersion.startsWith("m") && legacyVersion.compareTo("me") <= 0) + { + // sstables <= me are potentially broken, pretend offline upgrade where the user ran the scrub's header fix + FBUtilities.setPreviousReleaseVersionString("3.0.25"); + SSTableHeaderFix.fixDroppedNonFrozenUDTIfUpgradeFrom30(); + } + + int s0 = cfs.getLiveSSTables().size(); cfs.loadNewSSTables(); } @@ -639,6 +735,11 @@ public void testGenerateSstables() throws Throwable QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_simple_counter SET val = val + 1 WHERE pk = '%s'", format.getLatestVersion(), valPk)); + QueryProcessor.executeInternal( + String.format("INSERT INTO legacy_tables.legacy_%s_tuple (pk, val, val2, val3, val4, extra)" + + " VALUES ('%s', ({1,2,3},{'a','b','c'}), ({1,2,3},{'a','b','c'}), {name: ('abc','def')}, {name: ('abc','def')}, '%s')", + format.getLatestVersion(), valPk, randomString)); + for (int ck = 0; ck < 50; ck++) { String valCk = Integer.toString(ck); @@ -655,10 +756,16 @@ public void testGenerateSstables() throws Throwable File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", format.getLatestVersion())); ksDir.tryCreateDirectories(); - copySstablesFromTestData(String.format("legacy_%s_simple", format.getLatestVersion()), ksDir, LEGACY_TABLES_KEYSPACE); - copySstablesFromTestData(String.format("legacy_%s_simple_counter", format.getLatestVersion()), ksDir, LEGACY_TABLES_KEYSPACE); - copySstablesFromTestData(String.format("legacy_%s_clust", format.getLatestVersion()), ksDir, LEGACY_TABLES_KEYSPACE); - copySstablesFromTestData(String.format("legacy_%s_clust_counter", format.getLatestVersion()), ksDir, LEGACY_TABLES_KEYSPACE); + copySstablesFromTestData(String.format("legacy_%s_simple", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_simple_counter", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust_counter", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_tuple", format.getLatestVersion()), ksDir); + } + + public static void copySstablesFromTestData(String table, File ksDir) throws IOException + { + copySstablesFromTestData(table, ksDir, LEGACY_TABLES_KEYSPACE); } public static void copySstablesFromTestData(String table, File ksDir, String ks) throws IOException diff --git a/test/unit/org/apache/cassandra/tools/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/tools/SSTableHeaderFixTest.java new file mode 100644 index 000000000000..e765c968c65a --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/SSTableHeaderFixTest.java @@ -0,0 +1,1013 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import com.google.common.collect.Sets; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.FieldIdentifier; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.marshal.AbstractCompositeType; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.FloatType; +import org.apache.cassandra.db.marshal.FrozenType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.SetType; +import org.apache.cassandra.db.marshal.TupleType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableId; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.MockSchema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test the functionality of {@link SSTableHeaderFix}. + * It writes an 'big-m' version sstable(s) and executes against these. + */ +@RunWith(Parameterized.class) +public class SSTableHeaderFixTest +{ + static + { + DatabaseDescriptor.toolInitialization(); + } + + private File temporaryFolder; + + @Parameterized.Parameter + public Supplier sstableIdGen; + + @Parameterized.Parameters + public static Collection parameters() + { + return MockSchema.sstableIdGenerators(); + } + + @BeforeClass + public static void beforeClass() + { + Assume.assumeTrue(BigFormat.isSelected()); + } + + @Before + public void setup() + { + MockSchema.sstableIds.clear(); + MockSchema.sstableIdGenerator = sstableIdGen; + File f = FileUtils.createTempFile("SSTableUDTFixTest", ""); + f.tryDelete(); + f.tryCreateDirectories(); + temporaryFolder = f; + } + + @After + public void teardown() + { + FileUtils.deleteRecursive(temporaryFolder); + } + + private static final AbstractType udtPK = makeUDT("udt_pk"); + private static final AbstractType udtCK = makeUDT("udt_ck"); + private static final AbstractType udtStatic = makeUDT("udt_static"); + private static final AbstractType udtRegular = makeUDT("udt_regular"); + private static final AbstractType udtInner = makeUDT("udt_inner"); + private static final AbstractType udtNested = new UserType("ks", + ByteBufferUtil.bytes("udt_nested"), + Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), + new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), + Arrays.asList(UTF8Type.instance, + udtInner), + true); + private static final AbstractType tupleInTuple = makeTuple(makeTuple()); + private static final AbstractType udtInTuple = makeTuple(udtInner); + private static final AbstractType tupleInComposite = CompositeType.getInstance(UTF8Type.instance, makeTuple()); + private static final AbstractType udtInComposite = CompositeType.getInstance(UTF8Type.instance, udtInner); + private static final AbstractType udtInList = ListType.getInstance(udtInner, true); + private static final AbstractType udtInSet = SetType.getInstance(udtInner, true); + private static final AbstractType udtInMap = MapType.getInstance(UTF8Type.instance, udtInner, true); + private static final AbstractType udtInFrozenList = ListType.getInstance(udtInner, false); + private static final AbstractType udtInFrozenSet = SetType.getInstance(udtInner, false); + private static final AbstractType udtInFrozenMap = MapType.getInstance(UTF8Type.instance, udtInner, false); + + private static AbstractType makeUDT2(String udtName, boolean multiCell) + { + return new UserType("ks", + ByteBufferUtil.bytes(udtName), + Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), + new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), + Arrays.asList(UTF8Type.instance, + udtInner), + multiCell); + } + + private static AbstractType makeUDT(String udtName) + { + return new UserType("ks", + ByteBufferUtil.bytes(udtName), + Collections.singletonList(new FieldIdentifier(ByteBufferUtil.bytes("a_field"))), + Collections.singletonList(UTF8Type.instance), + true); + } + + private static TupleType makeTuple() + { + return makeTuple(Int32Type.instance); + } + + private static TupleType makeTuple(AbstractType second) + { + return new TupleType(Arrays.asList(UTF8Type.instance, + second)); + } + + private static TupleType makeTupleSimple() + { + // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT + return new TupleType(Collections.singletonList(UTF8Type.instance)); + } + + private static final Version version = BigFormat.getInstance().getVersion("mc"); + + private TableMetadata tableMetadata; + private final Set updatedColumns = new HashSet<>(); + + private ColumnMetadata getColDef(String n) + { + return tableMetadata.getColumn(ByteBufferUtil.bytes(n)); + } + + /** + * Very basic test whether {@link SSTableHeaderFix} detect a type mismatch (regular_c 'int' vs 'float'). + */ + @Test + public void verifyTypeMismatchTest() throws Exception + { + File dir = temporaryFolder; + File sstable = generateFakeSSTable(dir, 1); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + ColumnMetadata cd = getColDef("regular_c"); + tableMetadata = tableMetadata.unbuild() + .removeRegularOrStaticColumn(cd.name) + .addRegularColumn("regular_c", FloatType.instance) + .build(); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertTrue(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + @Test + public void verifyTypeMatchTest() throws Exception + { + File dir = temporaryFolder; + + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + File sstable = buildFakeSSTable(dir, 1, cols, false); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertTrue(updatedColumns.isEmpty()); + assertFalse(headerFix.hasError()); + assertFalse(headerFix.hasChanges()); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + /** + * Simulates the case when an sstable contains a column not present in the schema, which can just be ignored. + */ + @Test + public void verifyWithUnknownColumnTest() throws Exception + { + File dir = temporaryFolder; + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + cols.addRegularColumn("solr_query", UTF8Type.instance); + File sstable = buildFakeSSTable(dir, 1, cols, true); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + ColumnMetadata cd = getColDef("solr_query"); + tableMetadata = tableMetadata.unbuild() + .removeRegularOrStaticColumn(cd.name) + .build(); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + + /** + * Simulates the case when an sstable contains a column not present in the table but as a target for an index. + * It can just be ignored. + */ + @Test + public void verifyWithIndexedUnknownColumnTest() throws Exception + { + File dir = temporaryFolder; + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + cols.addRegularColumn("solr_query", UTF8Type.instance); + File sstable = buildFakeSSTable(dir, 1, cols, true); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + ColumnMetadata cd = getColDef("solr_query"); + tableMetadata = tableMetadata.unbuild() + .indexes(tableMetadata.indexes.with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query")))) + .removeRegularOrStaticColumn(cd.name) + .build(); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + + @Test + public void complexTypeMatchTest() throws Exception + { + File dir = temporaryFolder; + + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + cols.addRegularColumn("tuple_in_tuple", tupleInTuple) + .addRegularColumn("udt_nested", udtNested) + .addRegularColumn("udt_in_tuple", udtInTuple) + .addRegularColumn("tuple_in_composite", tupleInComposite) + .addRegularColumn("udt_in_composite", udtInComposite) + .addRegularColumn("udt_in_list", udtInList) + .addRegularColumn("udt_in_set", udtInSet) + .addRegularColumn("udt_in_map", udtInMap) + .addRegularColumn("udt_in_frozen_list", udtInFrozenList) + .addRegularColumn("udt_in_frozen_set", udtInFrozenSet) + .addRegularColumn("udt_in_frozen_map", udtInFrozenMap); + File sstable = buildFakeSSTable(dir, 1, cols, true); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", + "udt_nested", "udt_in_composite"), updatedColumns); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + + @Test + public void complexTypeDroppedColumnsMatchTest() throws Exception + { + File dir = temporaryFolder; + + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + cols.addRegularColumn("tuple_in_tuple", tupleInTuple) + .addRegularColumn("udt_nested", udtNested) + .addRegularColumn("udt_in_tuple", udtInTuple) + .addRegularColumn("tuple_in_composite", tupleInComposite) + .addRegularColumn("udt_in_composite", udtInComposite) + .addRegularColumn("udt_in_list", udtInList) + .addRegularColumn("udt_in_set", udtInSet) + .addRegularColumn("udt_in_map", udtInMap) + .addRegularColumn("udt_in_frozen_list", udtInFrozenList) + .addRegularColumn("udt_in_frozen_set", udtInFrozenSet) + .addRegularColumn("udt_in_frozen_map", udtInFrozenMap); + File sstable = buildFakeSSTable(dir, 1, cols, true); + + cols = tableMetadata.unbuild(); + for (String col : new String[]{"tuple_in_tuple", "udt_nested", "udt_in_tuple", + "tuple_in_composite", "udt_in_composite", + "udt_in_list", "udt_in_set", "udt_in_map", + "udt_in_frozen_list", "udt_in_frozen_set", "udt_in_frozen_map"}) + { + ColumnIdentifier ci = new ColumnIdentifier(col, true); + ColumnMetadata cd = getColDef(col); + AbstractType dropType = cd.type.expandUserTypes(); + cols.removeRegularOrStaticColumn(ci) + .recordColumnDrop(new ColumnMetadata(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind, cd.getMask()), FBUtilities.timestampMicros()); + } + tableMetadata = cols.build(); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", "udt_nested"), updatedColumns); + + // must not have re-written the stats-component + header = readHeader(sstable); + // do not check the inner types, as the inner types were not fixed in the serialization-header (test thing) + assertFrozenUdt(header, true, false); + } + + @Test + public void variousDroppedUserTypes() throws Exception + { + File dir = temporaryFolder; + + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + + ColSpec[] colSpecs = new ColSpec[] + { + // 'frozen' / live + new ColSpec("frozen_udt_as_frozen_udt_live", + makeUDT2("frozen_udt_as_frozen_udt_live", false), + makeUDT2("frozen_udt_as_frozen_udt_live", false), + false, + false), + // 'frozen' / live / as 'udt' + new ColSpec("frozen_udt_as_unfrozen_udt_live", + makeUDT2("frozen_udt_as_unfrozen_udt_live", false), + makeUDT2("frozen_udt_as_unfrozen_udt_live", true), + false, + true), + // 'frozen' / dropped + new ColSpec("frozen_udt_as_frozen_udt_dropped", + makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(), + makeUDT2("frozen_udt_as_frozen_udt_dropped", false), + makeUDT2("frozen_udt_as_frozen_udt_dropped", false), + true, + false), + // 'frozen' / dropped / as 'udt' + new ColSpec("frozen_udt_as_unfrozen_udt_dropped", + makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().freeze().expandUserTypes(), + makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true), + makeUDT2("frozen_udt_as_unfrozen_udt_dropped", false), + true, + true), + // 'udt' / live + new ColSpec("unfrozen_udt_as_unfrozen_udt_live", + makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), + makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), + false, + false), + // 'udt' / dropped +// TODO unable to test dropping a non-frozen UDT, as that requires an unfrozen tuple as well +// new ColSpec("unfrozen_udt_as_unfrozen_udt_dropped", +// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().expandUserTypes(), +// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), +// makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), +// true, +// false), + // 'frozen' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') + new ColSpec("frozen_tuple_as_frozen_tuple_live", + makeTupleSimple(), + makeTupleSimple(), + false, + false), + // 'frozen' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') + new ColSpec("frozen_tuple_as_frozen_tuple_dropped", + makeTupleSimple(), + makeTupleSimple(), + true, + false) + }; + + Arrays.stream(colSpecs).forEach(c -> cols.addRegularColumn(c.name, + // use the initial column type for the serialization header header. + c.preFix)); + + Map colSpecMap = Arrays.stream(colSpecs).collect(Collectors.toMap(c -> c.name, c -> c)); + File sstable = buildFakeSSTable(dir, 1, cols, c -> { + ColSpec cs = colSpecMap.get(c.name.toString()); + if (cs == null) + return c; + // update the column type in the schema to the "correct" one. + return c.withNewType(cs.schema); + }); + + Arrays.stream(colSpecs) + .filter(c -> c.dropped) + .forEach(c -> { + ColumnMetadata cd = getColDef(c.name); + tableMetadata = tableMetadata.unbuild() + .removeRegularOrStaticColumn(cd.name) + .recordColumnDrop(cd, FBUtilities.timestampMicros()) + .build(); + }); + + SerializationHeader.Component header = readHeader(sstable); + for (ColSpec colSpec : colSpecs) + { + AbstractType hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); + assertEquals(colSpec.name, colSpec.preFix, hdrType); + assertEquals(colSpec.name, colSpec.preFix.isMultiCell(), hdrType.isMultiCell()); + } + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + // Verify that all columns to fix are in the updatedColumns set (paranoid, yet) + Arrays.stream(colSpecs) + .filter(c -> c.mustFix) + .forEach(c -> assertTrue("expect " + c.name + " to be updated, but was not (" + updatedColumns + ")", updatedColumns.contains(c.name))); + // Verify that the number of updated columns maches the expected number of columns to fix + assertEquals(Arrays.stream(colSpecs).filter(c -> c.mustFix).count(), updatedColumns.size()); + + header = readHeader(sstable); + for (ColSpec colSpec : colSpecs) + { + AbstractType hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); + assertEquals(colSpec.name, colSpec.expect, hdrType); + assertEquals(colSpec.name, colSpec.expect.isMultiCell(), hdrType.isMultiCell()); + } + } + + static class ColSpec + { + final String name; + final AbstractType schema; + final AbstractType preFix; + final AbstractType expect; + final boolean dropped; + final boolean mustFix; + + ColSpec(String name, AbstractType schema, AbstractType preFix, boolean dropped, boolean mustFix) + { + this(name, schema, preFix, schema, dropped, mustFix); + } + + ColSpec(String name, AbstractType schema, AbstractType preFix, AbstractType expect, boolean dropped, boolean mustFix) + { + this.name = name; + this.schema = schema; + this.preFix = preFix; + this.expect = expect; + this.dropped = dropped; + this.mustFix = mustFix; + } + } + + @Test + public void verifyTypeMatchCompositeKeyTest() throws Exception + { + File dir = temporaryFolder; + + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk1", UTF8Type.instance) + .addPartitionKeyColumn("pk2", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + File sstable = buildFakeSSTable(dir, 1, cols, false); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertFalse(headerFix.hasChanges()); + assertTrue(updatedColumns.isEmpty()); + + // must not have re-written the stats-component + header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + @Test + public void compositePartitionKey() throws Exception + { + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk1", UTF8Type.instance) + .addPartitionKeyColumn("pk2", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + + File dir = temporaryFolder; + File sstable = buildFakeSSTable(dir, 1, cols, true); + + SerializationHeader.Component header = readHeader(sstable); + assertTrue(header.getKeyType() instanceof CompositeType); + CompositeType keyType = (CompositeType) header.getKeyType(); + assertEquals(Arrays.asList(UTF8Type.instance, udtPK), keyType.getComponents()); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + assertEquals(Sets.newHashSet("pk2", "ck", "regular_b", "static_b"), updatedColumns); + + header = readHeader(sstable); + assertTrue(header.getKeyType() instanceof CompositeType); + keyType = (CompositeType) header.getKeyType(); + assertEquals(Arrays.asList(UTF8Type.instance, udtPK.freeze()), keyType.getComponents()); + } + + @Test + public void compositeClusteringKey() throws Exception + { + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck1", Int32Type.instance) + .addClusteringColumn("ck2", udtCK); + commonColumns(cols); + + File dir = temporaryFolder; + File sstable = buildFakeSSTable(dir, 1, cols, true); + + SerializationHeader.Component header = readHeader(sstable); + assertEquals(Arrays.asList(Int32Type.instance, udtCK), header.getClusteringTypes()); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertFalse(headerFix.hasError()); + assertTrue(headerFix.hasChanges()); + assertEquals(Sets.newHashSet("pk", "ck2", "regular_b", "static_b"), updatedColumns); + + header = readHeader(sstable); + assertEquals(Arrays.asList(Int32Type.instance, udtCK.freeze()), header.getClusteringTypes()); + } + + /** + * Check whether {@link SSTableHeaderFix} can operate on a single file. + */ + @Test + public void singleFileUDTFixTest() throws Exception + { + File dir = temporaryFolder; + File sstable = generateFakeSSTable(dir, 1); + + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + + SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) + .build(); + headerFix.execute(); + + assertTrue(headerFix.hasChanges()); + assertFalse(headerFix.hasError()); + + header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + + /** + * Check whether {@link SSTableHeaderFix} can operate on a file in a directory. + */ + @Test + public void singleDirectoryUDTFixTest() throws Exception + { + File dir = temporaryFolder; + List sstables = IntStream.range(1, 11) + .mapToObj(g -> generateFakeSSTable(dir, g)) + .collect(Collectors.toList()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) + .build(); + headerFix.execute(); + + assertTrue(headerFix.hasChanges()); + assertFalse(headerFix.hasError()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + } + + /** + * Check whether {@link SSTableHeaderFix} can operate multiple, single files. + */ + @Test + public void multipleFilesUDTFixTest() throws Exception + { + File dir = temporaryFolder; + List sstables = IntStream.range(1, 11) + .mapToObj(g -> generateFakeSSTable(dir, g)) + .collect(Collectors.toList()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + SSTableHeaderFix.Builder builder = builder(); + sstables.stream().map(File::toPath).forEach(builder::withPath); + SSTableHeaderFix headerFix = builder.build(); + headerFix.execute(); + + assertTrue(headerFix.hasChanges()); + assertFalse(headerFix.hasError()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + } + + /** + * Check whether {@link SSTableHeaderFix} can operate multiple files in a directory. + */ + @Test + public void multipleFilesInDirectoryUDTFixTest() throws Exception + { + File dir = temporaryFolder; + List sstables = IntStream.range(1, 11) + .mapToObj(g -> generateFakeSSTable(dir, g)) + .collect(Collectors.toList()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, false, true); + } + + SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) + .build(); + headerFix.execute(); + + assertTrue(headerFix.hasChanges()); + assertFalse(headerFix.hasError()); + + for (File sstable : sstables) + { + SerializationHeader.Component header = readHeader(sstable); + assertFrozenUdt(header, true, true); + } + } + + @Test + public void ignoresStaleFilesTest() throws Exception + { + File dir = temporaryFolder; + IntStream.range(1, 2).forEach(g -> generateFakeSSTable(dir, g)); + + File newFile = new File(dir.toAbsolute(), "something_something-something.something"); + Assert.assertTrue(newFile.createFileIfNotExists()); + + SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) + .build(); + headerFix.execute(); + } + + private static final Pattern p = Pattern.compile(".* Column '([^']+)' needs to be updated from type .*"); + + private SSTableHeaderFix.Builder builder() + { + updatedColumns.clear(); + return SSTableHeaderFix.builder() + .schemaCallback(() -> (desc) -> tableMetadata) + .info(ln -> { + System.out.println("INFO: " + ln); + Matcher m = p.matcher(ln); + if (m.matches()) + updatedColumns.add(m.group(1)); + }) + .warn(ln -> System.out.println("WARN: " + ln)) + .error(ln -> System.out.println("ERROR: " + ln)); + } + + private File generateFakeSSTable(File dir, int generation) + { + TableMetadata.Builder cols = TableMetadata.builder("ks", "cf") + .addPartitionKeyColumn("pk", udtPK) + .addClusteringColumn("ck", udtCK); + commonColumns(cols); + return buildFakeSSTable(dir, generation, cols, true); + } + + private void commonColumns(TableMetadata.Builder cols) + { + cols.addRegularColumn("regular_a", UTF8Type.instance) + .addRegularColumn("regular_b", udtRegular) + .addRegularColumn("regular_c", Int32Type.instance) + .addRegularColumn("to_be_dropped", udtRegular.freeze()) + .addStaticColumn("static_a", UTF8Type.instance) + .addStaticColumn("static_b", udtStatic) + .addStaticColumn("static_c", Int32Type.instance); + + // drop a column for SSTableHeaderFix to execute + ColumnMetadata dropped = cols.getColumn(ByteBufferUtil.bytes("to_be_dropped")); + cols.removeRegularOrStaticColumn(dropped.name).recordColumnDrop(dropped, FBUtilities.timestampMicros()); + } + + private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, boolean freezeInSchema) + { + return buildFakeSSTable(dir, generation, cols, freezeInSchema + ? c -> c.withNewType(freezeUdt(c.type)) + : c -> c); + } + + private File buildFakeSSTable(File dir, int generation, TableMetadata.Builder cols, Function freezer) + { + TableMetadata headerMetadata = cols.build(); + + TableMetadata.Builder schemaCols = TableMetadata.builder("ks", "cf"); + for (ColumnMetadata cm : cols.columns()) + schemaCols.addColumn(freezer.apply(cm)); + schemaCols.droppedColumns(headerMetadata.droppedColumns); + tableMetadata = schemaCols.build(); + + try + { + + Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation)); + + // Just create the component files - we don't really need those. + for (Component component : requiredComponents) + assertTrue(desc.fileFor(component).createFileIfNotExists()); + + AbstractType partitionKey = headerMetadata.partitionKeyType; + List> clusteringKey = headerMetadata.clusteringColumns() + .stream() + .map(cd -> cd.type) + .collect(Collectors.toList()); + Map> staticColumns = headerMetadata.columns() + .stream() + .filter(cd -> cd.kind == ColumnMetadata.Kind.STATIC) + .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); + Map> regularColumns = headerMetadata.columns() + .stream() + .filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR) + .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); + + File statsFile = desc.fileFor(Components.STATS); + SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey, + clusteringKey, + staticColumns, + regularColumns, + EncodingStats.NO_STATS); + + try (SequentialWriter out = new SequentialWriter(statsFile)) + { + desc.getMetadataSerializer().serialize(Collections.singletonMap(MetadataType.HEADER, header), out, version); + out.finish(); + } + + return desc.fileFor(Components.DATA); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private AbstractType freezeUdt(AbstractType type) + { + if (type instanceof CollectionType) + { + if (type.getClass() == ListType.class) + { + ListType cHeader = (ListType) type; + return ListType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); + } + else if (type.getClass() == SetType.class) + { + SetType cHeader = (SetType) type; + return SetType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); + } + else if (type.getClass() == MapType.class) + { + MapType cHeader = (MapType) type; + return MapType.getInstance(freezeUdt(cHeader.getKeysType()), freezeUdt(cHeader.getValuesType()), cHeader.isMultiCell()); + } + } + else if (type instanceof AbstractCompositeType) + { + if (type.getClass() == CompositeType.class) + { + CompositeType cHeader = (CompositeType) type; + return CompositeType.getInstance(cHeader.types.stream().map(this::freezeUdt).collect(Collectors.toList())); + } + } + else if (type instanceof TupleType) + { + if (type.getClass() == UserType.class) + { + UserType cHeader = (UserType) type; + cHeader = cHeader.freeze(); + return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), + cHeader.allTypes().stream().map(this::freezeUdt).collect(Collectors.toList()), + cHeader.isMultiCell()); + } + } + return type; + } + + private void assertFrozenUdt(SerializationHeader.Component header, boolean frozen, boolean checkInner) + { + AbstractType keyType = header.getKeyType(); + if (keyType instanceof CompositeType) + { + for (AbstractType component : ((CompositeType) keyType).types) + assertFrozenUdt("partition-key-component", component, frozen, checkInner); + } + assertFrozenUdt("partition-key", keyType, frozen, checkInner); + + for (AbstractType type : header.getClusteringTypes()) + assertFrozenUdt("clustering-part", type, frozen, checkInner); + for (Map.Entry> col : header.getStaticColumns().entrySet()) + assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); + for (Map.Entry> col : header.getRegularColumns().entrySet()) + assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); + } + + private void assertFrozenUdt(String name, AbstractType type, boolean frozen, boolean checkInner) + { + if (type instanceof CompositeType) + { + if (checkInner) + for (AbstractType component : ((CompositeType) type).types) + assertFrozenUdt(name, component, frozen, true); + } + else if (type instanceof CollectionType) + { + if (checkInner) + { + if (type instanceof MapType) + { + MapType map = (MapType) type; + // only descend for non-frozen types (checking frozen in frozen is just stupid) + if (map.isMultiCell()) + { + assertFrozenUdt(name + "", map.getKeysType(), frozen, true); + assertFrozenUdt(name + "", map.getValuesType(), true, true); + } + } + else if (type instanceof SetType) + { + SetType set = (SetType) type; + // only descend for non-frozen types (checking frozen in frozen is just stupid) + if (set.isMultiCell()) + assertFrozenUdt(name + "", set.getElementsType(), true, true); + } + else if (type instanceof ListType) + { + ListType list = (ListType) type; + // only descend for non-frozen types (checking frozen in frozen is just stupid) + if (list.isMultiCell()) + assertFrozenUdt(name + "", list.getElementsType(), true, true); + } + } + } + else if (type instanceof TupleType) + { + if (checkInner) + { + TupleType tuple = (TupleType) type; + // only descend for non-frozen types (checking frozen in frozen is just stupid) + if (tuple.isMultiCell()) + for (AbstractType component : tuple.allTypes()) + assertFrozenUdt(name + "", component, frozen, true); + } + } + + if (type instanceof UserType) + { + String typeString = type.toString(); + assertEquals(name + ": " + typeString, frozen, !type.isMultiCell()); + if (typeString.startsWith(UserType.class.getName() + '(')) + if (frozen) + fail(name + ": " + typeString); + if (typeString.startsWith(FrozenType.class.getName() + '(' + UserType.class.getName() + '(')) + if (!frozen) + fail(name + ": " + typeString); + } + } + + private SerializationHeader.Component readHeader(File sstable) throws Exception + { + Descriptor desc = Descriptor.fromFileWithComponent(sstable, false).left; + return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER); + } + + private static final Component[] requiredComponents = new Component[]{ Components.DATA, Components.FILTER, Components.PRIMARY_INDEX, Components.TOC }; +} diff --git a/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java b/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java index bf8e876a5f1b..a872af389429 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneScrubberTest.java @@ -21,13 +21,15 @@ import java.util.Arrays; import org.junit.Test; - -import org.apache.cassandra.tools.ToolRunner.ToolResult; +import org.apache.commons.lang3.tuple.Pair; import org.assertj.core.api.Assertions; import org.hamcrest.CoreMatchers; +import org.apache.cassandra.tools.ToolRunner.ToolResult; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /* * We are testing cmd line params here. @@ -131,14 +133,41 @@ public void testHelpArg() @Test public void testHeaderFixArg() { - Arrays.asList("-e", "--header-fix").forEach(arg -> { - ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, - arg, - "doesntmatter", - "system_schema", - "tables"); - Assertions.assertThat(tool.getCleanedStderr().trim()).isEqualTo("Option header-fix is deprecated and no longer functional"); - Assertions.assertThat(tool.getExitCode()).isEqualTo(0); - }); + Arrays.asList(Pair.of("-e", ""), + Pair.of("-e", "wrong"), + Pair.of("--header-fix", ""), + Pair.of("--header-fix", "wrong")) + .forEach(arg -> { + ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, + arg.getLeft(), + arg.getRight(), + "system_schema", + "tables"); + assertThat("Arg: [" + arg + "]", tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:")); + assertTrue("Arg: [" + arg + "]\n" + tool.getCleanedStderr(), tool.getCleanedStderr().contains("Invalid argument value")); + assertEquals(1, tool.getExitCode()); + }); + + Arrays.asList(Pair.of("-e", "validate-only"), + Pair.of("-e", "validate"), + Pair.of("-e", "fix-only"), + Pair.of("-e", "fix"), + Pair.of("-e", "off"), + Pair.of("--header-fix", "validate-only"), + Pair.of("--header-fix", "validate"), + Pair.of("--header-fix", "fix-only"), + Pair.of("--header-fix", "fix"), + Pair.of("--header-fix", "off")) + .forEach(arg -> { + ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, + arg.getLeft(), + arg.getRight(), + "system_schema", + "tables"); + assertThat("Arg: [" + arg + "]", tool.getStdout(), CoreMatchers.containsStringIgnoringCase("Pre-scrub sstables snapshotted into snapshot")); + Assertions.assertThat(tool.getCleanedStderr()).as("Arg: [%s]", arg).isEmpty(); + tool.assertOnExitCode(); + assertCorrectEnvPostTest(); + }); } }