Skip to content

Commit

Permalink
fixes and added more tests for new setting property UnmodifiableOnRes…
Browse files Browse the repository at this point in the history 
…tore

Signed-off-by: AnnTian Shao <anntians@amazon.com>
  • Loading branch information
AnnTian Shao committed Jan 17, 2025
1 parent c29a0db commit a24b673
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 39 deletions.
104 changes: 95 additions & 9 deletions server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.opensearch.remotestore;

import org.opensearch.Version;
import org.opensearch.action.DocWriteResponse;
import org.opensearch.action.LatchedActionListener;
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
Expand Down Expand Up @@ -494,6 +495,51 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
}

public void testIndexRestoredFromSnapshotWithUpdateSetting() throws IOException, ExecutionException, InterruptedException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNodes(2);

String indexName1 = "testindex1";
String snapshotRepoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
logger.info("Snapshot Path [{}]", absolutePath1);

createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));

Settings indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName1, indexSettings);

final int numDocsInIndex1 = randomIntBetween(20, 30);
indexDocuments(client(), indexName1, numDocsInIndex1);
flushAndRefresh(indexName1);
ensureGreen(indexName1);

logger.info("--> snapshot");
SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));

assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get());
assertFalse(indexExists(indexName1));

// try index restore with index.number_of_replicas setting modified. index.number_of_replicas can be modified on restore
Settings numberOfReplicasSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build();

RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(numberOfReplicasSettings)
.setIndices(indexName1)
.get();

assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED);
ensureGreen(indexName1);
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
}

protected IndexShard getIndexShard(String node, String indexName) {
final Index index = resolveIndex(indexName);
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
Expand Down Expand Up @@ -721,42 +767,82 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
);
assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore"));

// try index restore with remote store repository modified
Settings remoteStoreIndexSettings = Settings.builder()
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo)
.build();
// try index restore with index.number_of_shards setting modified
Settings numberOfShardsSettingsDiff = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(remoteStoreIndexSettings)
.setIndexSettings(numberOfShardsSettingsDiff)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore"));
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore"));

// try index restore with index.number_of_shards setting modified
Settings indexKnnSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, false).build();
// try index restore with index.number_of_shards setting same
Settings numberOfShardsSettingsSame = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(indexKnnSettings)
.setIndexSettings(numberOfShardsSettingsSame)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore"));

// try index restore with mix of modifiable and unmodifiable settings on restore
// index.version.created is unmodifiable, index.number_of_replicas is modifiable
Settings mixedSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_EMPTY)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
.build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(mixedSettings)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.version.created]" + " on restore"));

// try index restore with multiple UnmodifiableOnRestore settings on restore
Settings unmodifiableSettings = Settings.builder()
.put(IndexMetadata.SETTING_CREATION_DATE, -1L)
.put(IndexMetadata.SETTING_INDEX_UUID, IndexMetadata.INDEX_UUID_NA_VALUE)
.put(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE)
.build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(unmodifiableSettings)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.creation_date]" + " on restore"));

// try index restore with remote store repository and translog store repository disabled
exception = expectThrows(
SnapshotRestoreException.class,
Expand Down
104 changes: 95 additions & 9 deletions ...r/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.opensearch.remotestore;

import org.opensearch.Version;
import org.opensearch.action.DocWriteResponse;
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
Expand Down Expand Up @@ -561,6 +562,51 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
}

public void testIndexRestoredFromSnapshotWithUpdateSetting() throws IOException, ExecutionException, InterruptedException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNodes(2);

String indexName1 = "testindex1";
String snapshotRepoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
logger.info("Snapshot Path [{}]", absolutePath1);

createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));

Settings indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName1, indexSettings);

final int numDocsInIndex1 = randomIntBetween(20, 30);
indexDocuments(client(), indexName1, numDocsInIndex1);
flushAndRefresh(indexName1);
ensureGreen(indexName1);

logger.info("--> snapshot");
SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));

assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get());
assertFalse(indexExists(indexName1));

// try index restore with index.number_of_replicas setting modified. index.number_of_replicas can be modified on restore
Settings numberOfReplicasSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build();

RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(numberOfReplicasSettings)
.setIndices(indexName1)
.get();

assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED);
ensureGreen(indexName1);
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
}

private IndexShard getIndexShard(String node, String indexName) {
final Index index = resolveIndex(indexName);
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
Expand Down Expand Up @@ -788,42 +834,82 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
);
assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore"));

// try index restore with remote store repository modified
Settings remoteStoreIndexSettings = Settings.builder()
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo)
.build();
// try index restore with index.number_of_shards setting modified
Settings numberOfShardsSettingsDiff = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(remoteStoreIndexSettings)
.setIndexSettings(numberOfShardsSettingsDiff)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore"));
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore"));

// try index restore with index.number_of_shards setting modified
Settings indexKnnSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, false).build();
// try index restore with index.number_of_shards setting same
Settings numberOfShardsSettingsSame = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(indexKnnSettings)
.setIndexSettings(numberOfShardsSettingsSame)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore"));

// try index restore with mix of modifiable and unmodifiable settings on restore
// index.version.created is unmodifiable, index.number_of_replicas is modifiable
Settings mixedSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_EMPTY)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
.build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(mixedSettings)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.version.created]" + " on restore"));

// try index restore with multiple UnmodifiableOnRestore settings on restore
Settings unmodifiableSettings = Settings.builder()
.put(IndexMetadata.SETTING_CREATION_DATE, -1L)
.put(IndexMetadata.SETTING_INDEX_UUID, IndexMetadata.INDEX_UUID_NA_VALUE)
.put(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE)
.build();

exception = expectThrows(
SnapshotRestoreException.class,
() -> client().admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
.setWaitForCompletion(false)
.setIndexSettings(unmodifiableSettings)
.setIndices(index)
.setRenamePattern(index)
.setRenameReplacement(restoredIndex)
.get()
);
assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.creation_date]" + " on restore"));

// try index restore with remote store repository and translog store repository disabled
exception = expectThrows(
SnapshotRestoreException.class,
Expand Down
36 changes: 32 additions & 4 deletions server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,8 @@ public Iterator<Setting<?>> settings() {
},
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
Property.Dynamic,
Property.UnmodifiableOnRestore
);

/**
Expand Down Expand Up @@ -424,7 +425,8 @@ public Iterator<Setting<?>> settings() {
},
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
Property.Dynamic,
Property.UnmodifiableOnRestore
);

private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
Expand Down Expand Up @@ -475,7 +477,8 @@ public Iterator<Setting<?>> settings() {
},
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
Property.Dynamic,
Property.UnmodifiableOnRestore
);

public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas";
Expand Down Expand Up @@ -567,13 +570,21 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
SETTING_VERSION_CREATED,
Version.V_EMPTY,
Property.IndexScope,
Property.PrivateIndex
Property.PrivateIndex,
Property.UnmodifiableOnRestore
);

public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded";
public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
public static final String SETTING_CREATION_DATE = "index.creation_date";

public static final Setting<Long> SETTING_INDEX_CREATION_DATE = Setting.longSetting(
SETTING_CREATION_DATE,
-1L,
Property.IndexScope,
Property.UnmodifiableOnRestore
);
/**
* The user provided name for an index. This is the plain string provided by the user when the index was created.
* It might still contain date math expressions etc. (added in 5.0)
Expand All @@ -597,8 +608,25 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
Function.identity(),
Property.IndexScope
);

public static final String INDEX_UUID_NA_VALUE = Strings.UNKNOWN_UUID_VALUE;

public static final Setting<String> INDEX_UUID_SETTING = Setting.simpleString(
SETTING_INDEX_UUID,
INDEX_UUID_NA_VALUE,
Property.IndexScope,
Property.PrivateIndex,
Property.UnmodifiableOnRestore
);

public static final Setting<String> SETTING_INDEX_HISTORY_UUID = Setting.simpleString(
SETTING_HISTORY_UUID,
INDEX_UUID_NA_VALUE,
Property.IndexScope,
Property.PrivateIndex,
Property.UnmodifiableOnRestore
);

public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require";
public static final String INDEX_ROUTING_INCLUDE_GROUP_PREFIX = "index.routing.allocation.include";
public static final String INDEX_ROUTING_EXCLUDE_GROUP_PREFIX = "index.routing.allocation.exclude";
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING,
IndexMetadata.SETTING_INDEX_VERSION_CREATED,
IndexMetadata.INDEX_UUID_SETTING,
IndexMetadata.SETTING_INDEX_CREATION_DATE,
IndexMetadata.SETTING_INDEX_HISTORY_UUID,
IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING,
IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING,
IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
Expand Down
Loading

0 comments on commit a24b673

Please sign in to comment.