From d03b6a04c85b07ef9eb6f6e6f3639b2182a04aa8 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Sat, 18 Jan 2025 19:05:01 +0800 Subject: [PATCH] test --- .../action/bulk/TransportShardBulkAction.java | 3 +- .../opensearch/index/shard/IndexShard.java | 23 +++++++++----- .../RecoveryDuringReplicationTests.java | 3 +- .../index/shard/IndexShardTests.java | 30 ++++++++++++------- .../PeerRecoveryTargetServiceTests.java | 3 +- .../indices/recovery/RecoveryTests.java | 12 +++++--- .../index/shard/IndexShardTestCase.java | 3 +- 7 files changed, 52 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index fdba8a42c0170..da6219b7a3b4d 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -880,7 +880,8 @@ private static Engine.Result performOpOnReplica( primaryResponse.getVersion(), indexRequest.getAutoGeneratedTimestamp(), indexRequest.isRetry(), - sourceToParse + sourceToParse, + primaryResponse.indexingStrategy() ); break; case DELETE: diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 02f20504b07ba..d992fae6c948e 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -126,6 +126,7 @@ import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineException; import org.opensearch.index.engine.EngineFactory; +import org.opensearch.index.engine.InternalEngine; import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.ReadOnlyEngine; import org.opensearch.index.engine.RefreshFailedEngineException; @@ -1054,6 +1055,7 @@ public Engine.IndexResult applyIndexOperationOnPrimary( isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse, + null, null ); } @@ -1065,7 +1067,8 @@ public Engine.IndexResult applyIndexOperationOnReplica( long version, long autoGeneratedTimeStamp, boolean isRetry, - SourceToParse sourceToParse + SourceToParse sourceToParse, + InternalEngine.IndexingStrategy indexingStrategy ) throws IOException { return applyIndexOperation( getEngine(), @@ -1079,7 +1082,8 @@ public Engine.IndexResult applyIndexOperationOnReplica( isRetry, Engine.Operation.Origin.REPLICA, sourceToParse, - id + id, + indexingStrategy ); } @@ -1095,7 +1099,8 @@ private Engine.IndexResult applyIndexOperation( boolean isRetry, Engine.Operation.Origin origin, SourceToParse sourceToParse, - String id + String id, + InternalEngine.IndexingStrategy indexingStrategy ) throws IOException { // For Segment Replication enabled replica shards we can be skip parsing the documents as we directly copy segments from primary @@ -1136,7 +1141,8 @@ private Engine.IndexResult applyIndexOperation( autoGeneratedTimeStamp, isRetry, ifSeqNo, - ifPrimaryTerm + ifPrimaryTerm, + indexingStrategy ); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { @@ -1165,7 +1171,8 @@ public static Engine.Index prepareIndex( long autoGeneratedIdTimestamp, boolean isRetry, long ifSeqNo, - long ifPrimaryTerm + long ifPrimaryTerm, + InternalEngine.IndexingStrategy indexingStrategy ) { long startTime = System.nanoTime(); ParsedDocument doc = docMapper.getDocumentMapper().parse(source); @@ -1185,7 +1192,8 @@ public static Engine.Index prepareIndex( autoGeneratedIdTimestamp, isRetry, ifSeqNo, - ifPrimaryTerm + ifPrimaryTerm, + indexingStrategy ); } @@ -2417,7 +2425,8 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o MediaTypeRegistry.xContentType(index.source()), index.routing() ), - index.id() + index.id(), + null ); break; case DELETE: diff --git a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java index b891ac63378ac..45c9f4757941b 100644 --- a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java @@ -160,7 +160,8 @@ public void testRecoveryToReplicaThatReceivedExtraDocument() throws Exception { 1, randomNonNegativeLong(), false, - new SourceToParse("index", "replica", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse("index", "replica", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); shards.promoteReplicaToPrimary(promotedReplica).get(); oldPrimary.close("demoted", randomBoolean(), false); diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 535adfbff8dcc..a2ecc41f96f57 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2344,7 +2344,8 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId().getIndexName(), "id", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId().getIndexName(), "id", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); shard.applyIndexOperationOnReplica( UUID.randomUUID().toString(), @@ -2353,7 +2354,8 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { 3, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId().getIndexName(), "id-3", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId().getIndexName(), "id-3", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); // Flushing a new commit with local checkpoint=1 allows to skip the translog gen #1 in recovery. shard.flush(new FlushRequest().force(true).waitIfOngoing(true)); @@ -2364,7 +2366,8 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { 3, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId().getIndexName(), "id-2", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId().getIndexName(), "id-2", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); shard.applyIndexOperationOnReplica( UUID.randomUUID().toString(), @@ -2373,7 +2376,8 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId().getIndexName(), "id-5", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId().getIndexName(), "id-5", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); shard.sync(); // advance local checkpoint @@ -2521,7 +2525,8 @@ public void testRecoverFromStoreWithNoOps() throws IOException { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - sourceToParse + sourceToParse, + null ); final ShardRouting primaryShardRouting = shard.routingEntry(); @@ -2649,7 +2654,8 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "doc-0", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "doc-0", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); flushShard(shard); shard.updateGlobalCheckpointOnReplica(0, "test"); // stick the global checkpoint here. @@ -2660,7 +2666,8 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "doc-1", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "doc-1", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1")); @@ -2673,7 +2680,8 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "doc-2", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "doc-2", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1", "doc-2")); @@ -4155,7 +4163,8 @@ private Result indexOnReplicaWithGaps(final IndexShard indexShard, final int ope 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - sourceToParse + sourceToParse, + null ); if (!gap && i == localCheckpoint + 1) { localCheckpoint++; @@ -4780,7 +4789,8 @@ public void testDoNotTrimCommitsWhenOpenReadOnlyEngine() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId.getIndexName(), Long.toString(i), new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId.getIndexName(), Long.toString(i), new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); shard.updateGlobalCheckpointOnReplica(shard.getLocalCheckpoint(), "test"); if (randomInt(100) < 10) { diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java index a8e5a02011538..5962af7825370 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -190,7 +190,8 @@ private SeqNoStats populateRandomData(IndexShard shard) throws IOException { shard.getOperationPrimaryTerm(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(shard.shardId().getIndexName(), UUIDs.randomBase64UUID(), new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(shard.shardId().getIndexName(), UUIDs.randomBase64UUID(), new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); if (randomInt(100) < 5) { shard.flush(new FlushRequest().waitIfOngoing(true)); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java index 71d89e2856c6e..71b07ed192024 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java @@ -190,7 +190,8 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "id", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "id", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); // index #3 orgReplica.applyIndexOperationOnReplica( @@ -200,7 +201,8 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "id-3", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "id-3", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); // Flushing a new commit with local checkpoint=1 allows to delete the translog gen #1. orgReplica.flush(new FlushRequest().force(true).waitIfOngoing(true)); @@ -212,7 +214,8 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "id-2", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "id-2", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); orgReplica.sync(); // advance local checkpoint orgReplica.updateGlobalCheckpointOnReplica(3L, "test"); @@ -224,7 +227,8 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception { 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - new SourceToParse(indexName, "id-5", new BytesArray("{}"), MediaTypeRegistry.JSON) + new SourceToParse(indexName, "id-5", new BytesArray("{}"), MediaTypeRegistry.JSON), + null ); if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 655a9eb7d5d38..4214508916f93 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -1335,7 +1335,8 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, - sourceToParse + sourceToParse, + null ); shard.sync(); // advance local checkpoint if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {