From 9abade8eacff13e2cda9139eeaf249e2de030277 Mon Sep 17 00:00:00 2001 From: Rampreeth Ethiraj Date: Thu, 13 Feb 2025 22:37:13 +0530 Subject: [PATCH 1/4] Change ConcurrentSearchTasksIT to use custom slice count Signed-off-by: Rampreeth Ethiraj --- .../admin/cluster/node/tasks/ConcurrentSearchTasksIT.java | 4 +++- .../java/org/opensearch/cluster/metadata/IndexMetadata.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index e2a86b9358829..38ed11cbaa48e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -89,6 +89,7 @@ public void testConcurrentSearchTaskTracking() { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.MAX_SLICE_COUNT, 5) .build() ); ensureGreen(INDEX_NAME); // Make sure all shards are allocated to catch replication tasks @@ -110,7 +111,8 @@ public void testConcurrentSearchTaskTracking() { Map> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId()); // Concurrent search forks each slice of 5 segments to different thread (see please // https://github.com/apache/lucene/issues/12498) - assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size()); + Metadata metadata = client().admin().cluster().prepareState().get().getState().metadata(); + assertEquals(Integer.parseInt(metadata.index(INDEX_NAME).getSettings().get(IndexMetadata.MAX_SLICE_COUNT)) + 1, threadStats.size()); // assert that all task descriptions have non-zero length MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0)); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index d50192f106cfe..88ddecdb2de42 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -248,6 +248,7 @@ static Setting buildNumberOfShardsSetting() { static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards"; public static final Setting INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting(); public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; + public static final String MAX_SLICE_COUNT = "index.search.concurrent.max_slice_count"; public static final Setting INDEX_NUMBER_OF_REPLICAS_SETTING = Setting.intSetting( SETTING_NUMBER_OF_REPLICAS, 1, From c4d000ae42ad82f70bf0c1d92e05b90914c74e33 Mon Sep 17 00:00:00 2001 From: Rampreeth Ethiraj Date: Thu, 13 Feb 2025 22:50:38 +0530 Subject: [PATCH 2/4] Change ConcurrentSearchTasksIT to use custom slice count Signed-off-by: Rampreeth Ethiraj --- .../action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index 38ed11cbaa48e..a78b74eac1c07 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -11,6 +11,7 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.search.SearchAction; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.FeatureFlagSettings; import org.opensearch.common.settings.Setting; From fa04c657d74386f62b4c4ac8de89449ca5ff05ba Mon Sep 17 00:00:00 2001 From: Rampreeth Ethiraj Date: Thu, 13 Feb 2025 23:09:45 +0530 Subject: [PATCH 3/4] Change ConcurrentSearchTasksIT to use custom slice count Signed-off-by: Rampreeth Ethiraj --- .../admin/cluster/node/tasks/ConcurrentSearchTasksIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index a78b74eac1c07..46efd301d1bd0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -113,8 +113,10 @@ public void testConcurrentSearchTaskTracking() { // Concurrent search forks each slice of 5 segments to different thread (see please // https://github.com/apache/lucene/issues/12498) Metadata metadata = client().admin().cluster().prepareState().get().getState().metadata(); - assertEquals(Integer.parseInt(metadata.index(INDEX_NAME).getSettings().get(IndexMetadata.MAX_SLICE_COUNT)) + 1, threadStats.size()); - + assertEquals( + Integer.parseInt(metadata.index(INDEX_NAME).getSettings().get(IndexMetadata.MAX_SLICE_COUNT)) + 1, + threadStats.size() + ); // assert that all task descriptions have non-zero length MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0)); } From a519924e20e2c45bb77dc62918c67c6cac77b988 Mon Sep 17 00:00:00 2001 From: Rampreeth Ethiraj Date: Tue, 18 Feb 2025 21:05:26 +0530 Subject: [PATCH 4/4] Change ConcurrentSearchTasksIT to use custom slice count Signed-off-by: Rampreeth Ethiraj --- .../node/tasks/ConcurrentSearchTasksIT.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index 46efd301d1bd0..2c5f60805529e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -49,19 +49,6 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } - private int getSegmentCount(String indexName) { - return client().admin() - .indices() - .segments(new IndicesSegmentsRequest(indexName)) - .actionGet() - .getIndices() - .get(indexName) - .getShards() - .get(0) - .getShards()[0].getSegments() - .size(); - } - @Override protected Settings featureFlagSettings() { Settings.Builder featureSettings = Settings.builder(); @@ -82,6 +69,8 @@ public void testConcurrentSearchTaskTracking() { final String INDEX_NAME = "test"; final int NUM_SHARDS = 1; final int NUM_DOCS = 7; + int maxSliceCount = randomIntBetween(1, NUM_DOCS); + registerTaskManagerListeners(SearchAction.NAME); // coordinator task registerTaskManagerListeners(SearchAction.NAME + "[*]"); // shard task @@ -90,7 +79,7 @@ public void testConcurrentSearchTaskTracking() { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.MAX_SLICE_COUNT, 5) + .put(IndexMetadata.MAX_SLICE_COUNT, maxSliceCount) .build() ); ensureGreen(INDEX_NAME); // Make sure all shards are allocated to catch replication tasks @@ -114,7 +103,7 @@ public void testConcurrentSearchTaskTracking() { // https://github.com/apache/lucene/issues/12498) Metadata metadata = client().admin().cluster().prepareState().get().getState().metadata(); assertEquals( - Integer.parseInt(metadata.index(INDEX_NAME).getSettings().get(IndexMetadata.MAX_SLICE_COUNT)) + 1, + Integer.parseInt(metadata.index(INDEX_NAME).getSettings().get(IndexMetadata.MAX_SLICE_COUNT)), threadStats.size() ); // assert that all task descriptions have non-zero length