From ab05bdd9d1cf4150dfeaaf2353374e1210c1e010 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 21 Jul 2025 11:19:01 -0700 Subject: [PATCH 01/67] Start with test --- .../collections/AbstractInstallShardTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java index ecac1c494648..59fb5e0eaed4 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java @@ -174,6 +174,8 @@ public void testInstallToSingleShardCollection() throws Exception { CollectionAdminRequest.installDataToShard( collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) .process(cluster.getSolrClient()); + waitForState("The failed core-install (previous leader) should recover and become healthy", + collectionName, 30, TimeUnit.SECONDS, SolrCloudTestCase.activeClusterShape(1, replicasPerShard)); assertCollectionHasNumDocs(collectionName, singleShardNumDocs); } @@ -209,6 +211,31 @@ public void testInstallReportsErrorsAppropriately() throws Exception { } } + @Test + public void testInstallSucceedsOnASingleError() throws Exception { + final String collectionName = createAndAwaitEmptyCollection(1, replicasPerShard); + deleteAfterTest(collectionName); + enableReadOnly(collectionName); + final String nonExistentLocation = nonExistentLocationUri.toString(); + + { // Test synchronous request error reporting + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", nonExistentLocation, BACKUP_REPO_NAME) + .process(cluster.getSolrClient()); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + + { // Test asynchronous request error reporting + final var requestStatusState = + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", nonExistentLocation, BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 15); + + assertEquals(RequestStatusState.COMPLETED, requestStatusState); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + } + @Test public void testSerialInstallToMultiShardCollection() throws Exception { final String collectionName = From 8328c78e6cd3340af634ae5ac275fcb9c61508cf Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 21 Jul 2025 13:07:46 -0700 Subject: [PATCH 02/67] Solve issue, but need to clean up. Need to fix for Restore as well --- .../solr/cloud/RecoveringCoreTermWatcher.java | 19 +++ .../apache/solr/cloud/RecoveryStrategy.java | 9 +- .../org/apache/solr/cloud/ZkShardTerms.java | 5 + .../cloud/api/collections/AddReplicaCmd.java | 3 +- .../solr/cloud/api/collections/BackupCmd.java | 4 +- .../cloud/api/collections/CollApiCmds.java | 3 +- .../collections/CollectionHandlingUtils.java | 121 ++++++++++++++---- .../api/collections/CreateCollectionCmd.java | 2 +- .../api/collections/CreateSnapshotCmd.java | 2 +- .../api/collections/DeleteReplicaCmd.java | 2 +- .../api/collections/DeleteSnapshotCmd.java | 2 +- .../api/collections/InstallShardDataCmd.java | 95 +++++++++++++- .../cloud/api/collections/MigrateCmd.java | 12 +- .../cloud/api/collections/SplitShardCmd.java | 9 +- .../handler/admin/api/InstallCoreData.java | 10 +- .../solr/handler/component/ShardRequest.java | 3 + .../solr/update/DefaultSolrCoreState.java | 8 +- .../solr/client/solrj/cloud/ShardTerms.java | 22 ++++ .../collections/AbstractInstallShardTest.java | 14 +- 19 files changed, 279 insertions(+), 66 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index 6bba8e993196..a59760b6c195 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -18,8 +18,10 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.solr.client.solrj.cloud.ShardTerms; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; @@ -58,6 +60,23 @@ public boolean onTermChanged(ShardTerms terms) { log.info( "Start recovery on {} because core's term is less than leader's term", coreNodeName); lastTermDoRecovery.set(terms.getTerm(coreNodeName)); + if (coreDescriptor.getCloudDescriptor().isLeader()) { + log.warn( + "Removing {} leader as leader, since its term is no longer the highest", + coreNodeName); + coreContainer.getZkController().giveupLeadership(coreDescriptor); + coreContainer + .getZkController() + .getZkStateReader() + .waitForState( + coreDescriptor.getCollectionName(), + 20, + TimeUnit.SECONDS, + dc -> { + Replica leader = dc.getLeader(coreDescriptor.getCloudDescriptor().getShardId()); + return leader == null || !leader.getCoreName().equals(coreDescriptor.getName()); + }); + } solrCore .getUpdateHandler() .getSolrCoreState() diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 022ea8a9b345..8974f7d42a1f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -229,9 +229,14 @@ private void replicate(String nodeName, SolrCore core, ZkNodeProps leaderprops) log.info("Attempting to replicate from core [{}] on node [{}].", leaderCore, leaderBaseUrl); - // send commit if replica could be a leader + // send commit if replica could be a leader and this collection is not in a read-only state if (replicaType.leaderEligible) { - commitOnLeader(leaderBaseUrl, leaderCore); + if (!zkController + .getClusterState() + .getCollection(coreDescriptor.getCollectionName()) + .isReadOnly()) { + commitOnLeader(leaderBaseUrl, leaderCore); + } } // use rep handler directly, so we can do this sync rather than async diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java index 5c0f5186e916..f629b6c24227 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java @@ -121,6 +121,11 @@ public void ensureTermsIsHigher(String leader, Set replicasNeedingRecove mutate(terms -> terms.increaseTerms(leader, replicasNeedingRecovery)); } + public void ensureHighestTerms(Set mostUpToDateCores) { + if (mostUpToDateCores.isEmpty()) return; + mutate(terms -> terms.setHighestTerms(mostUpToDateCores)); + } + public ShardTerms getShardTerms() { return terms.get(); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index a71a515c56d8..a7755f5a3101 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -174,7 +174,8 @@ List addReplica( ModifiableSolrParams params = getReplicaParams( message, collectionName, coll, skipCreateReplicaInClusterState, createReplica); - shardRequestTracker.sendShardRequest(createReplica.node, params, shardHandler); + shardRequestTracker.sendShardRequest( + createReplica.node, createReplica.coreName, params, shardHandler); } Runnable runnable = diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 151cbeeee1a8..468856e71a48 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -340,7 +340,7 @@ private void incrementalCopyIndexFiles( slice.getName(), backupManager.getBackupId().getId()); params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString()); - shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(replica, params, shardHandler); log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); } log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); @@ -522,7 +522,7 @@ private void copyIndexFiles( params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName()); } - shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(replica, params, shardHandler); log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); } log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index d5bb2d245688..68ec101655c9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -306,7 +306,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results, boolean failur static void commit(NamedList results, String slice, Replica parentShardLeader) { log.debug("Calling soft commit to make sub shard updates visible"); - String coreUrl = parentShardLeader.getCoreUrl(); // HttpShardHandler is hard coded to send a QueryRequest hence we go direct // and we force open a searcher so that we have documents to show upon switching states UpdateResponse updateResponse = null; try { updateResponse = softCommit(parentShardLeader.getBaseUrl(), parentShardLeader.getCoreName()); CollectionHandlingUtils.processResponse( - results, null, coreUrl, updateResponse, slice, Collections.emptySet()); + results, + null, + parentShardLeader.getNodeName(), + parentShardLeader.getCoreName(), + updateResponse, + slice, + Collections.emptySet()); } catch (Exception e) { CollectionHandlingUtils.processResponse( - results, e, coreUrl, updateResponse, slice, Collections.emptySet()); + results, + e, + parentShardLeader.getNodeName(), + parentShardLeader.getCoreName(), + updateResponse, + slice, + Collections.emptySet()); throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Unable to call distrib softCommit on: " + coreUrl, + "Unable to call distrib softCommit on: " + parentShardLeader.getCoreUrl(), e); } } @@ -429,16 +440,19 @@ static void processResponse( NamedList results, ShardResponse srsp, Set okayExceptions) { Throwable e = srsp.getException(); String nodeName = srsp.getNodeName(); + // Use core or coreNodeName if given as a param, otherwise use nodeName + String coreName = srsp.getShardRequest().coreName; SolrResponse solrResponse = srsp.getSolrResponse(); String shard = srsp.getShard(); - processResponse(results, e, nodeName, solrResponse, shard, okayExceptions); + processResponse(results, e, nodeName, coreName, solrResponse, shard, okayExceptions); } static void processResponse( NamedList results, Throwable e, String nodeName, + String coreName, SolrResponse solrResponse, String shard, Set okayExceptions) { @@ -449,9 +463,9 @@ static void processResponse( if (e != null && (rootThrowable == null || !okayExceptions.contains(rootThrowable))) { log.error("Error from shard: {}", shard, e); - addFailure(results, nodeName, e.getClass().getName() + ":" + e.getMessage()); + addFailure(results, nodeName, coreName, e); } else { - addSuccess(results, nodeName, solrResponse.getResponse()); + addSuccess(results, nodeName, coreName, solrResponse.getResponse()); } } @@ -475,24 +489,38 @@ public static void addExceptionToNamedList( results.add("exception", nl); } - private static void addFailure(NamedList results, String key, Object value) { + public static String requestKey(Replica replica) { + return requestKey(replica.getNodeName(), replica.getCoreName()); + } + + public static String requestKey(String nodeName, String coreName) { + if (coreName == null) { + return nodeName; + } else { + return nodeName + "/" + coreName; + } + } + + private static void addFailure( + NamedList results, String nodeName, String coreName, Object value) { @SuppressWarnings("unchecked") SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure"); if (failure == null) { failure = new SimpleOrderedMap<>(); results.add("failure", failure); } - failure.add(key, value); + failure.add(requestKey(nodeName, coreName), value); } - private static void addSuccess(NamedList results, String key, Object value) { + private static void addSuccess( + NamedList results, String nodeName, String coreName, Object value) { @SuppressWarnings("unchecked") SimpleOrderedMap success = (SimpleOrderedMap) results.get("success"); if (success == null) { success = new SimpleOrderedMap<>(); results.add("success", success); } - success.add(key, value); + success.add(requestKey(nodeName, coreName), value); } private static NamedList waitForCoreAdminAsyncCallToComplete( @@ -500,6 +528,7 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( String adminPath, ZkStateReader zkStateReader, String nodeName, + String coreName, String requestId) { ShardHandler shardHandler = shardHandlerFactory.getShardHandler(); ModifiableSolrParams params = new ModifiableSolrParams(); @@ -515,6 +544,8 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( sreq.shards = new String[] {replica}; sreq.actualShards = sreq.shards; sreq.params = params; + sreq.nodeName = nodeName; + sreq.coreName = coreName; shardHandler.submit(sreq, replica, sreq.params); @@ -600,7 +631,7 @@ public static class ShardRequestTracker { private final String adminPath; private final ZkStateReader zkStateReader; private final ShardHandlerFactory shardHandlerFactory; - private final NamedList shardAsyncIdByNode = new NamedList(); + private final List shardAsyncCmds = new ArrayList<>(); public ShardRequestTracker( String asyncId, @@ -635,7 +666,10 @@ public List sliceCmd( cloneParams.set(CoreAdminParams.CORE, replica.getStr(ZkStateReader.CORE_NAME_PROP)); sendShardRequest( - replica.getStr(ZkStateReader.NODE_NAME_PROP), cloneParams, shardHandler); + replica.getStr(ZkStateReader.NODE_NAME_PROP), + replica.getStr(ZkStateReader.CORE_NAME_PROP), + cloneParams, + shardHandler); } else { notLiveReplicas.add(replica); } @@ -645,12 +679,24 @@ public List sliceCmd( } public void sendShardRequest( - String nodeName, ModifiableSolrParams params, ShardHandler shardHandler) { - sendShardRequest(nodeName, params, shardHandler, adminPath, zkStateReader); + Replica replica, ModifiableSolrParams params, ShardHandler shardHandler) { + sendShardRequest( + replica.getNodeName(), + replica.getCoreName(), + params, + shardHandler, + adminPath, + zkStateReader); + } + + public void sendShardRequest( + String nodeName, String coreName, ModifiableSolrParams params, ShardHandler shardHandler) { + sendShardRequest(nodeName, coreName, params, shardHandler, adminPath, zkStateReader); } public void sendShardRequest( String nodeName, + String coreName, ModifiableSolrParams params, ShardHandler shardHandler, String adminPath, @@ -658,7 +704,7 @@ public void sendShardRequest( if (asyncId != null) { String coreAdminAsyncId = asyncId + Math.abs(System.nanoTime()); params.set(ASYNC, coreAdminAsyncId); - track(nodeName, coreAdminAsyncId); + track(nodeName, coreName, coreAdminAsyncId); } ShardRequest sreq = new ShardRequest(); @@ -668,6 +714,7 @@ public void sendShardRequest( sreq.shards = new String[] {replica}; sreq.actualShards = sreq.shards; sreq.nodeName = nodeName; + sreq.coreName = coreName; sreq.params = params; shardHandler.submit(sreq, replica, sreq.params); @@ -690,7 +737,10 @@ void processResponses( // Processes all shard responses ShardResponse srsp; do { - srsp = shardHandler.takeCompletedOrError(); + srsp = + abortOnError + ? shardHandler.takeCompletedOrError() + : shardHandler.takeCompletedIncludingErrors(); if (srsp != null) { processResponse(results, srsp, okayExceptions); Throwable exception = srsp.getException(); @@ -708,26 +758,27 @@ void processResponses( if (asyncId != null) { // TODO: Shouldn't we abort with msgOnError exception when failure? waitForAsyncCallsToComplete(results); - shardAsyncIdByNode.clear(); + shardAsyncCmds.clear(); } } private void waitForAsyncCallsToComplete(NamedList results) { - for (Map.Entry nodeToAsync : shardAsyncIdByNode) { - final String node = nodeToAsync.getKey(); - final String shardAsyncId = nodeToAsync.getValue(); - log.debug("I am Waiting for :{}/{}", node, shardAsyncId); + for (AsyncCmdInfo asyncCmdInfo : shardAsyncCmds) { + final String node = asyncCmdInfo.nodeName; + final String coreName = asyncCmdInfo.coreName; + final String shardAsyncId = asyncCmdInfo.asyncId; + log.info("I am Waiting for :{}/{}/{}", node, coreName, shardAsyncId); NamedList reqResult = waitForCoreAdminAsyncCallToComplete( - shardHandlerFactory, adminPath, zkStateReader, node, shardAsyncId); + shardHandlerFactory, adminPath, zkStateReader, node, coreName, shardAsyncId); if (INCLUDE_TOP_LEVEL_RESPONSE) { results.add(shardAsyncId, reqResult); } if ("failed".equalsIgnoreCase(((String) reqResult.get("STATUS")))) { log.error("Error from shard {}: {}", node, reqResult); - addFailure(results, node, reqResult); + addFailure(results, node, coreName, reqResult); } else { - addSuccess(results, node, reqResult); + addSuccess(results, node, coreName, reqResult); } } } @@ -736,8 +787,24 @@ private void waitForAsyncCallsToComplete(NamedList results) { * @deprecated consider to make it private after {@link CreateCollectionCmd} refactoring */ @Deprecated - void track(String nodeName, String coreAdminAsyncId) { - shardAsyncIdByNode.add(nodeName, coreAdminAsyncId); + void track(String nodeName, String coreName, String coreAdminAsyncId) { + shardAsyncCmds.add(AsyncCmdInfo.from(nodeName, coreName, coreAdminAsyncId)); + } + } + + private static class AsyncCmdInfo { + protected final String nodeName; + protected final String coreName; + protected final String asyncId; + + public AsyncCmdInfo(String nodeName, String coreName, String asyncId) { + this.nodeName = nodeName; + this.coreName = coreName; + this.asyncId = asyncId; + } + + public static AsyncCmdInfo from(String nodeName, String coreName, String asyncId) { + return new AsyncCmdInfo(nodeName, coreName, asyncId); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index 2b8a5007bf1d..46fc96db3d5b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -361,7 +361,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList resu params.set(CORE_NAME_PROP, coreName); params.set(CoreAdminParams.COMMIT_NAME, commitName); - shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(replica, params, shardHandler); log.debug( "Sent createsnapshot request to core={} with commitName={}", coreName, commitName); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index 138dec2ef55d..8ac4fae4755f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -313,7 +313,7 @@ void deleteCore( final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); if (isLive) { - shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(replica, params, shardHandler); } Callable callable = diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index b3970210cb6b..35f526fc6fcc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -126,7 +126,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu log.info( "Sending deletesnapshot request to core={} with commitName={}", coreName, commitName); - shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(replica, params, shardHandler); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 18ce5aa40716..f0f326fc3e44 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -23,11 +23,19 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.invoke.MethodHandles; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.solr.cloud.ZkShardTerms; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -58,6 +66,7 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { } @Override + @SuppressWarnings("unchecked") public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { final RemoteMessage typedMessage = @@ -84,14 +93,94 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu // Send the core-admin request to each replica in the slice final ShardHandler shardHandler = ccc.newShardHandler(); - shardRequestTracker.sliceCmd(clusterState, coreApiParams, null, installSlice, shardHandler); + List notLiveReplicas = + shardRequestTracker.sliceCmd(clusterState, coreApiParams, null, installSlice, shardHandler); final String errorMessage = String.format( Locale.ROOT, - "Could not install data to collection [%s] and shard [%s]", + "Could not install data to collection [%s] and shard [%s] on any leader-eligible replicas", typedMessage.collection, typedMessage.shard); - shardRequestTracker.processResponses(results, shardHandler, true, errorMessage); + shardRequestTracker.processResponses(results, shardHandler, false, errorMessage); + Collection allReplicas = + clusterState + .getCollection(typedMessage.collection) + .getSlice(typedMessage.shard) + .getReplicas(); + + // Ensure that terms are correct for this shard after the execution is done + // We only care about leader eligible replicas, all others will eventually get updated. + List leaderEligibleReplicas = + allReplicas.stream().filter(r -> r.getType().leaderEligible).collect(Collectors.toList()); + + NamedList failures = (NamedList) results.get("failure"); + Set successfulReplicas = + leaderEligibleReplicas.stream() + .filter(replica -> !notLiveReplicas.contains(replica)) + // TODO: This does not work for nodes that have multiple cores of the shard + .filter( + replica -> + failures == null + || failures.get(CollectionHandlingUtils.requestKey(replica)) == null) + .collect(Collectors.toSet()); + + if (successfulReplicas.isEmpty()) { + // No leader-eligible replicas succeeded, return failure + if (failures == null) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + errorMessage + ". No leader-eligible replicas are live."); + } else { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, errorMessage, (Throwable) failures.getVal(0)); + } + } else if (successfulReplicas.size() < leaderEligibleReplicas.size()) { + // Some, but not all, leader-eligible replicas succeeded. + // Ensure the shard terms are correct so that the non-successful replicas go into recovery + ZkShardTerms shardTerms = + ccc.getCoreContainer() + .getZkController() + .getShardTerms(typedMessage.collection, typedMessage.shard); + shardTerms.ensureHighestTerms( + successfulReplicas.stream().map(Replica::getName).collect(Collectors.toSet())); + Set replicasToRecover = + leaderEligibleReplicas.stream() + .filter(r -> !successfulReplicas.contains(r)) + .map(Replica::getName) + .collect(Collectors.toSet()); + ccc.getZkStateReader() + .waitForState( + typedMessage.collection, + 10, + TimeUnit.SECONDS, + (liveNodes, collectionState) -> + collectionState.getSlice(typedMessage.shard).getReplicas().stream() + .filter(r -> replicasToRecover.contains(r.getName())) + .allMatch(r -> Replica.State.RECOVERING.equals(r.getState()))); + + // In order for the async request to succeed, we need to ensure that there is no failure + // message + NamedList successes = (NamedList) results.get("success"); + failures.forEach( + (replicaKey, value) -> { + successes.add( + replicaKey, + new NamedList<>( + Map.of( + "explanation", + "Core install failed, but is now recovering from the leader", + "failure", + value))); + }); + results.remove("failure"); + } else { + // other replicas to-be-created will know that they are out of date by + // looking at their term : 0 compare to term of this core : 1 + ccc.getCoreContainer() + .getZkController() + .getShardTerms(typedMessage.collection, typedMessage.shard) + .ensureHighestTermsAreNotZero(); + } } /** A value-type representing the message received by {@link InstallShardDataCmd} */ diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index b09f806535a6..97182ad1dc9c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -235,7 +235,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); - shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to request node to buffer updates"); @@ -352,7 +352,7 @@ private void migrateKey( CollectionHandlingUtils.syncRequestTracker(ccc); // we don't want this to happen asynchronously syncRequestTracker.sendShardRequest( - tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler); + tempSourceLeader, new ModifiableSolrParams(cmd.getParams()), shardHandler); syncRequestTracker.processResponses( results, @@ -375,7 +375,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); - shardRequestTracker.sendShardRequest(tempNodeName, params, shardHandler); + shardRequestTracker.sendShardRequest(sourceLeader, params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command"); } @@ -443,7 +443,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); - shardRequestTracker.sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(tempSourceLeader, params, shardHandler); shardRequestTracker.processResponses( results, @@ -464,7 +464,7 @@ private void migrateKey( final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); - shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler); String msg = "MIGRATE failed to merge " + tempCollectionReplica2 @@ -483,7 +483,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); - shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); + shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates"); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 12ac6a4d89a4..65d52ab86a73 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -279,8 +279,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList getRangesResults = new SimpleOrderedMap<>(); String msgOnError = "SPLITSHARD failed to invoke SPLIT.getRanges core admin command"; shardRequestTracker.processResponses(getRangesResults, shardHandler, true, msgOnError); @@ -457,7 +456,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList replicasNeedingRecove return replicasNeedingRecovery.contains(key); } + public ShardTerms setHighestTerms(Set highestTermKeys) { + long newMaxTerm = maxTerm + 1; + boolean keyFound = false; + HashMap newValues = new HashMap<>(values); + long nextHighestTerm = 0; + for (String key : values.keySet()) { + if (highestTermKeys.contains(key)) { + newValues.put(key, newMaxTerm); + keyFound = true; + } else { + nextHighestTerm = Math.max(nextHighestTerm, values.get(key)); + } + } + // We only want to update if increasing the maxTerm makes an impact. + // If the nextHighestTerm is already < maxTerm, then upping the maxTerm doesn't do anything. + if (nextHighestTerm == maxTerm && keyFound) { + return new ShardTerms(newValues, version); + } else { + return null; + } + } + /** * Return a new {@link ShardTerms} in which the highest terms are not zero * diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java index 59fb5e0eaed4..7ff6630f8560 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java @@ -174,8 +174,12 @@ public void testInstallToSingleShardCollection() throws Exception { CollectionAdminRequest.installDataToShard( collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) .process(cluster.getSolrClient()); - waitForState("The failed core-install (previous leader) should recover and become healthy", - collectionName, 30, TimeUnit.SECONDS, SolrCloudTestCase.activeClusterShape(1, replicasPerShard)); + waitForState( + "The failed core-install (previous leader) should recover and become healthy", + collectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(1, replicasPerShard)); assertCollectionHasNumDocs(collectionName, singleShardNumDocs); } @@ -216,11 +220,11 @@ public void testInstallSucceedsOnASingleError() throws Exception { final String collectionName = createAndAwaitEmptyCollection(1, replicasPerShard); deleteAfterTest(collectionName); enableReadOnly(collectionName); - final String nonExistentLocation = nonExistentLocationUri.toString(); + final String singleShardLocation = singleShard1Uri.toString(); { // Test synchronous request error reporting CollectionAdminRequest.installDataToShard( - collectionName, "shard1", nonExistentLocation, BACKUP_REPO_NAME) + collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) .process(cluster.getSolrClient()); assertCollectionHasNumDocs(collectionName, singleShardNumDocs); } @@ -228,7 +232,7 @@ public void testInstallSucceedsOnASingleError() throws Exception { { // Test asynchronous request error reporting final var requestStatusState = CollectionAdminRequest.installDataToShard( - collectionName, "shard1", nonExistentLocation, BACKUP_REPO_NAME) + collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) .processAndWait(cluster.getSolrClient(), 15); assertEquals(RequestStatusState.COMPLETED, requestStatusState); From d1d6bf51eb34786503906e751351eddce274551a Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 21 Jul 2025 13:31:30 -0700 Subject: [PATCH 03/67] Use new shardrequest constructs for SyncStrategy --- .../java/org/apache/solr/cloud/SyncStrategy.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index a686743f6b50..f072dbe1336f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -72,11 +72,6 @@ public SyncStrategy(CoreContainer cc) { updateExecutor = updateShardHandler.getUpdateExecutor(); } - private static class ShardCoreRequest extends ShardRequest { - String coreName; - public String baseUrl; - } - public PeerSync.PeerSyncResult sync( ZkController zkController, SolrCore core, ZkNodeProps leaderProps) { return sync(zkController, core, leaderProps, false); @@ -283,8 +278,8 @@ private void syncToMe( } else { RecoveryRequest rr = new RecoveryRequest(); rr.leaderProps = leaderProps; - rr.baseUrl = ((ShardCoreRequest) srsp.getShardRequest()).baseUrl; - rr.coreName = ((ShardCoreRequest) srsp.getShardRequest()).coreName; + rr.baseUrl = srsp.getShardRequest().nodeName; + rr.coreName = srsp.getShardRequest().coreName; recoveryRequests.add(rr); } } else { @@ -316,9 +311,9 @@ private boolean handleResponse(ShardResponse srsp) { private void requestSync( String baseUrl, String replica, String leaderUrl, String coreName, int nUpdates) { // TODO should we use peerSyncWithLeader instead? - ShardCoreRequest sreq = new ShardCoreRequest(); + ShardRequest sreq = new ShardRequest(); sreq.coreName = coreName; - sreq.baseUrl = baseUrl; + sreq.nodeName = baseUrl; sreq.purpose = ShardRequest.PURPOSE_PRIVATE; sreq.shards = new String[] {replica}; sreq.actualShards = sreq.shards; From f5ee23475d02fccb610f8fe8139ee768a6aa1312 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 21 Jul 2025 14:18:34 -0700 Subject: [PATCH 04/67] Response status should now be the number of replicas --- .../test/org/apache/solr/cloud/CollectionsAPISolrJTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index e3aa232f768e..8e701b773cca 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -125,7 +125,7 @@ public void testCreateWithDefaultConfigSet() throws Exception { assertEquals(0, response.getStatus()); assertTrue(response.isSuccess()); Map> nodesStatus = response.getCollectionNodesStatus(); - assertEquals(nodesStatus.toString(), nodesCreated, nodesStatus.size()); + assertEquals(nodesStatus.toString(), 4, nodesStatus.size()); waitForState( "Expected " + collectionName + " to disappear from cluster state", @@ -253,7 +253,7 @@ public void testCreateAndDeleteCollection() throws Exception { assertTrue(response.isSuccess()); Map> nodesStatus = response.getCollectionNodesStatus(); // Delete could have been sent before the collection was finished coming online - assertEquals(nodesStatus.toString(), nodesCreated, nodesStatus.size()); + assertEquals(nodesStatus.toString(), 4, nodesStatus.size()); waitForState( "Expected " + collectionName + " to disappear from cluster state", From 02d1e9085352de9cfb40f0886cf50b81ffd2a03a Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 22 Jul 2025 11:12:27 -0700 Subject: [PATCH 05/67] Cleanup unused parts of tests --- .../test/org/apache/solr/cloud/CollectionsAPISolrJTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index 8e701b773cca..083fcd7d824a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -112,8 +112,7 @@ public void testCreateWithDefaultConfigSet() throws Exception { assertEquals(0, (int) status.get("status")); assertTrue(status.get("QTime") > 0); } - // Sometimes multiple cores land on the same node so it's less than 4 - int nodesCreated = response.getCollectionNodesStatus().size(); + // Use of _default configset should generate a warning for data-driven functionality in // production use assertTrue( @@ -244,8 +243,6 @@ public void testCreateAndDeleteCollection() throws Exception { assertTrue(status.get("QTime") > 0); } - // Sometimes multiple cores land on the same node so it's less than 4 - int nodesCreated = response.getCollectionNodesStatus().size(); response = CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); From 1be4ed07ea553528c4f3533fcbafa10def66f6be Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 24 Jul 2025 11:10:53 -0700 Subject: [PATCH 06/67] Huge commit - restore uses installshard - big update in locking --- .../model/InstallShardDataRequestBody.java | 6 + .../java/org/apache/solr/cloud/LockTree.java | 87 ++++++++++++-- .../OverseerConfigSetMessageHandler.java | 20 +++- .../solr/cloud/OverseerMessageHandler.java | 7 +- .../solr/cloud/OverseerTaskProcessor.java | 2 +- .../cloud/api/collections/AddReplicaCmd.java | 3 +- .../solr/cloud/api/collections/AliasCmd.java | 3 +- .../solr/cloud/api/collections/BackupCmd.java | 3 +- .../api/collections/BalanceReplicasCmd.java | 3 +- .../cloud/api/collections/CollApiCmds.java | 31 +++-- .../collections/CollectionApiLockFactory.java | 5 + .../collections/CollectionHandlingUtils.java | 36 +++++- .../cloud/api/collections/CreateAliasCmd.java | 21 +++- .../api/collections/CreateCollectionCmd.java | 5 +- .../cloud/api/collections/CreateShardCmd.java | 5 +- .../api/collections/CreateSnapshotCmd.java | 3 +- .../cloud/api/collections/DeleteAliasCmd.java | 3 +- .../api/collections/DeleteBackupCmd.java | 3 +- .../api/collections/DeleteCollectionCmd.java | 3 +- .../cloud/api/collections/DeleteNodeCmd.java | 3 +- .../api/collections/DeleteReplicaCmd.java | 3 +- .../cloud/api/collections/DeleteShardCmd.java | 3 +- .../api/collections/DeleteSnapshotCmd.java | 3 +- ...butedCollectionConfigSetCommandRunner.java | 7 +- .../api/collections/InstallShardDataCmd.java | 111 +++++++++++++++++- .../collections/MaintainRoutedAliasCmd.java | 17 +-- .../cloud/api/collections/MigrateCmd.java | 15 ++- .../api/collections/MigrateReplicasCmd.java | 3 +- .../cloud/api/collections/MoveReplicaCmd.java | 3 +- .../OverseerCollectionMessageHandler.java | 8 +- .../api/collections/OverseerRoleCmd.java | 3 +- .../api/collections/OverseerStatusCmd.java | 3 +- .../api/collections/ReindexCollectionCmd.java | 25 ++-- .../solr/cloud/api/collections/RenameCmd.java | 3 +- .../cloud/api/collections/ReplaceNodeCmd.java | 3 +- .../cloud/api/collections/RestoreCmd.java | 81 +++++++++---- .../api/collections/SetAliasPropCmd.java | 3 +- .../cloud/api/collections/SplitShardCmd.java | 17 +-- .../handler/admin/CollectionsHandler.java | 5 + .../handler/admin/api/InstallShardData.java | 9 +- .../solr/handler/admin/api/RestoreCore.java | 6 - .../solr/s3/S3IncrementalBackupTest.java | 4 + .../common/params/CollectionAdminParams.java | 2 + .../solr/common/params/CollectionParams.java | 3 +- .../solrj/impl/CloudHttp2SolrClientTest.java | 5 +- .../AbstractIncrementalBackupTest.java | 13 ++ 46 files changed, 479 insertions(+), 131 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index 31bec8eb4346..a72b9c2bc920 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -24,5 +24,11 @@ public class InstallShardDataRequestBody { @JsonProperty public String repository; + @JsonProperty public String name; + + @JsonProperty public String shardBackupId; + @JsonProperty public String async; + + @JsonProperty public String callingLockId; } diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index e8d96d4f2cd5..20a5686093f7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.solr.cloud.OverseerMessageHandler.Lock; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.LockLevel; @@ -38,20 +39,35 @@ public class LockTree { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final Node root = new Node(null, LockLevel.CLUSTER, null); + public final Map allLocks = new HashMap<>(); + private class LockImpl implements Lock { final Node node; + final String id; LockImpl(Node node) { this.node = node; + this.id = UUID.randomUUID().toString(); } @Override public void unlock() { synchronized (LockTree.this) { node.unlock(this); + allLocks.remove(id); } } + @Override + public String id() { + return id; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return node.validateSubpath(lockLevel, path); + } + @Override public String toString() { return StrUtils.join(node.constructPath(new ArrayDeque<>()), '/'); @@ -71,12 +87,33 @@ public String toString() { public class Session { private SessionNode root = new SessionNode(LockLevel.CLUSTER); - public Lock lock(CollectionParams.CollectionAction action, List path) { + public Lock lock( + CollectionParams.CollectionAction action, List path, String callingLockId) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; + log.info("Calling lock level: {}", callingLockId); + Node startingNode = LockTree.this.root; + SessionNode startingSession = root; + + // If a callingLockId was passed in, validate it with the current lock path, and only start + // locking below the calling lock + Lock callingLock = callingLockId != null ? allLocks.get(callingLockId) : null; + boolean ignoreCallingLock = false; + if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { + startingNode = ((LockImpl) callingLock).node; + startingSession = startingSession.find(startingNode.level.getHeight(), path); + if (startingSession == null) { + startingSession = root; + } + ignoreCallingLock = true; + } synchronized (LockTree.this) { - if (root.isBusy(action.lockLevel, path)) return null; - Lock lockObject = LockTree.this.root.lock(action.lockLevel, path); - if (lockObject == null) root.markBusy(action.lockLevel, path); + if (startingSession.isBusy(action.lockLevel, path)) return null; + Lock lockObject = startingNode.lock(action.lockLevel, path, ignoreCallingLock); + if (lockObject == null) { + startingSession.markBusy(action.lockLevel, path); + } else { + allLocks.put(lockObject.id(), lockObject); + } return lockObject; } } @@ -125,6 +162,18 @@ boolean isBusy(LockLevel lockLevel, List path) { return false; } } + + SessionNode find(int lockLevel, List path) { + if (level.getHeight() == lockLevel) { + return this; + } else if (level.getHeight() < lockLevel + && kids != null + && kids.containsKey(path.get(level.getHeight()))) { + return kids.get(path.get(level.getHeight())).find(lockLevel, path); + } else { + return null; + } + } } public Session getSession() { @@ -158,8 +207,9 @@ void unlock(LockImpl lockObject) { } } - Lock lock(LockLevel lockLevel, List path) { - if (myLock != null) return null; // I'm already locked. no need to go any further + Lock lock(LockLevel lockLevel, List path, boolean ignoreCurrentLock) { + if (myLock != null && !ignoreCurrentLock) + return null; // I'm already locked. no need to go any further if (lockLevel == level) { // lock is supposed to be acquired at this level // If I am locked or any of my children or grandchildren are locked @@ -171,10 +221,16 @@ Lock lock(LockLevel lockLevel, List path) { Node child = children.get(childName); if (child == null) children.put(childName, child = new Node(childName, level.getChild(), this)); - return child.lock(lockLevel, path); + return child.lock(lockLevel, path, false); } } + boolean validateSubpath(int lockLevel, List path) { + return level.getHeight() < lockLevel + && (level.getHeight() == 0 || name.equals(path.get(level.getHeight() - 1))) + && (mom == null || mom.validateSubpath(lockLevel, path)); + } + ArrayDeque constructPath(ArrayDeque collect) { if (name != null) collect.addFirst(name); if (mom != null) mom.constructPath(collect); @@ -182,5 +238,20 @@ ArrayDeque constructPath(ArrayDeque collect) { } } - static final Lock FREELOCK = () -> {}; + static final String FREELOCK_ID = "-1"; + static final Lock FREELOCK = + new Lock() { + @Override + public void unlock() {} + + @Override + public String id() { + return FREELOCK_ID; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java index 0bf454a06421..849626a30b42 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java @@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -61,7 +62,7 @@ public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader, CoreContaine } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { NamedList results = new NamedList<>(); try { if (!operation.startsWith(CONFIGSETS_ACTION_PREFIX)) { @@ -117,7 +118,22 @@ public Lock lockTask(ZkNodeProps message, long ignored) { String configSetName = getTaskKey(message); if (canExecute(configSetName, message)) { markExclusiveTask(configSetName, message); - return () -> unmarkExclusiveTask(configSetName, message); + return new Lock() { + @Override + public void unlock() { + unmarkExclusiveTask(configSetName, message); + } + + @Override + public String id() { + return configSetName; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } return null; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java index 3e369b907316..4bc13e2fe6a6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.common.cloud.ZkNodeProps; /** Interface for processing messages received by an {@link OverseerTaskProcessor} */ @@ -26,7 +27,7 @@ public interface OverseerMessageHandler { * @param operation the operation to process * @return response */ - OverseerSolrResponse processMessage(ZkNodeProps message, String operation); + OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock); /** * @return the name of the OverseerMessageHandler @@ -41,6 +42,10 @@ public interface OverseerMessageHandler { interface Lock { void unlock(); + + String id(); + + boolean validateSubpath(int lockLevel, List path); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index 3751d850f09b..74a6fc642e0c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -545,7 +545,7 @@ public void run() { if (log.isDebugEnabled()) { log.debug("Runner processing {}", head.getId()); } - response = messageHandler.processMessage(message, operation); + response = messageHandler.processMessage(message, operation, lock); } finally { timerContext.stop(); updateStats(statsName); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index a7755f5a3101..941c873af65e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -76,7 +76,8 @@ public AddReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { addReplica(state, message, results, null); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 4815a76b8070..98d4f1b311aa 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -56,6 +56,7 @@ static NamedList createCollectionAndWait( String aliasName, Map aliasMetadata, String createCollName, + String lockId, CollectionCommandContext ccc) throws Exception { // Map alias metadata starting with a prefix to a create-collection API request @@ -83,7 +84,7 @@ static NamedList createCollectionAndWait( // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(clusterState, createMessage, results); + new CreateCollectionCmd(ccc).call(clusterState, createMessage, lockId, results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 468856e71a48..8ffaf5c47ce3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -71,7 +71,8 @@ public BackupCmd(CollectionCommandContext ccc) { @SuppressWarnings("unchecked") @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java index 7d84c7c6633b..aaa0e684f18c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java @@ -41,7 +41,8 @@ public BalanceReplicasCmd(CollectionCommandContext ccc) { @SuppressWarnings({"unchecked"}) @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { Set nodes; Object nodesRaw = message.get(CollectionParams.NODES); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index 68ec101655c9..e4fe808dbd54 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -114,7 +114,8 @@ public class CollApiCmds { * classes whose names ends in {@code Cmd}. */ protected interface CollectionApiCommand { - void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; + void call(ClusterState state, ZkNodeProps message, String lockId, NamedList results) + throws Exception; } /** @@ -206,7 +207,8 @@ public TraceAwareCommand(CollectionApiCommand command) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final Span localSpan; final Context localContext; @@ -226,7 +228,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu try (var scope = localContext.makeCurrent()) { assert scope != null; // prevent javac warning about scope being unused - command.call(state, message, results); + command.call(state, message, lockId, results); } finally { if (localSpan != null) { localSpan.end(); @@ -238,7 +240,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu public static class MockOperationCmd implements CollectionApiCommand { @Override @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws InterruptedException { // only for test purposes Thread.sleep(message.getInt("sleep", 1)); @@ -260,7 +263,8 @@ public ReloadCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) { + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); @@ -285,7 +289,8 @@ public RebalanceLeadersCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -327,7 +332,8 @@ public AddReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -361,7 +367,8 @@ public DeleteReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); @@ -390,7 +397,8 @@ public BalanceShardsUniqueCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (StrUtils.isBlank(message.getStr(COLLECTION_PROP)) || StrUtils.isBlank(message.getStr(PROPERTY_PROP))) { @@ -426,7 +434,8 @@ public ModifyCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); @@ -509,7 +518,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList locks = new ArrayList<>(iterationOrder.length); for (CollectionParams.LockLevel level : iterationOrder) { + // We do not want to re-lock from the parent lock level + // TODO: Fix + // if (calledFromLockLevel.isHigherOrEqual(level)) { + // continue; + // } // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 8f213f417023..d2bea0747813 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -326,7 +326,7 @@ static void cleanupCollection( Map props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collectionName); new DeleteCollectionCmd(ccc) - .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), results); + .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), null, results); } static Map waitToSeeReplicasInState( @@ -562,6 +562,16 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( } String r = (String) srsp.getSolrResponse().getResponse().get("STATUS"); + if (r == null) { + // For Collections API Calls + r = (String) srsp.getSolrResponse().getResponse()._get("status/state"); + } + if (r == null) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Could not find status of async command in response: " + + srsp.getSolrResponse().getResponse().toString()); + } if (r.equals("running")) { log.debug("The task is still RUNNING, continuing to wait."); try { @@ -571,6 +581,15 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( } continue; + } else if (r.equals("submitted")) { + log.debug("The task is still SUBMITTED, continuing to wait."); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + continue; + } else if (r.equals("completed")) { log.debug("The task is COMPLETED, returning"); return srsp.getSolrResponse().getResponse(); @@ -620,6 +639,12 @@ public static ShardRequestTracker asyncRequestTracker( ccc.newShardHandler().getShardHandlerFactory()); } + public static ShardRequestTracker asyncRequestTracker( + String asyncId, String adminPath, CollectionCommandContext ccc) { + return new ShardRequestTracker( + asyncId, adminPath, ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); + } + public static class ShardRequestTracker { /* * backward compatibility reasons, add the response with the async ID as top level. @@ -767,14 +792,19 @@ private void waitForAsyncCallsToComplete(NamedList results) { final String node = asyncCmdInfo.nodeName; final String coreName = asyncCmdInfo.coreName; final String shardAsyncId = asyncCmdInfo.asyncId; - log.info("I am Waiting for :{}/{}/{}", node, coreName, shardAsyncId); + log.info("I am Waiting for: {}/{}/{}", node, coreName, shardAsyncId); NamedList reqResult = waitForCoreAdminAsyncCallToComplete( shardHandlerFactory, adminPath, zkStateReader, node, coreName, shardAsyncId); if (INCLUDE_TOP_LEVEL_RESPONSE) { results.add(shardAsyncId, reqResult); } - if ("failed".equalsIgnoreCase(((String) reqResult.get("STATUS")))) { + String status = (String) reqResult.get("STATUS"); + if (status == null) { + // For Collections API Calls + status = (String) reqResult._get("status/state"); + } + if ("failed".equalsIgnoreCase(status)) { log.error("Error from shard {}: {}", node, reqResult); addFailure(results, node, coreName, reqResult); } else { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index f48a984aad16..ff5a533aea91 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -50,7 +50,8 @@ public CreateAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); @@ -63,7 +64,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu if (!anyRoutingParams(message)) { callCreatePlainAlias(message, aliasName, zkStateReader); } else { - callCreateRoutedAlias(message, aliasName, zkStateReader, state); + callCreateRoutedAlias(message, aliasName, zkStateReader, state, lockId); } // Sleep a bit to allow ZooKeeper state propagation. @@ -112,7 +113,11 @@ private List parseCollectionsParameter(Object collections) { } private void callCreateRoutedAlias( - ZkNodeProps message, String aliasName, ZkStateReader zkStateReader, ClusterState state) + ZkNodeProps message, + String aliasName, + ZkStateReader zkStateReader, + ClusterState state, + String lockId) throws Exception { // Validate we got a basic minimum if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) { @@ -150,7 +155,12 @@ private void callCreateRoutedAlias( // Create the first collection. Prior validation ensures that this is not a standard alias collectionListStr = routedAlias.computeInitialCollectionName(); ensureAliasCollection( - aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), collectionListStr); + aliasName, + zkStateReader, + state, + routedAlias.getAliasMetadata(), + collectionListStr, + lockId); } else { List collectionList = aliases.resolveAliases(aliasName); collectionListStr = String.join(",", collectionList); @@ -167,10 +177,11 @@ private void ensureAliasCollection( ZkStateReader zkStateReader, ClusterState state, Map aliasProperties, + String lockId, String initialCollectionName) throws Exception { // Create the collection - createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ccc); + createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, lockId, ccc); validateAllCollectionsExistAndNoDuplicates( Collections.singletonList(initialCollectionName), zkStateReader); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index 46fc96db3d5b..b7d8ee94c6a9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -104,7 +104,8 @@ public CreateCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader ccc.getZkStateReader().aliasesManager.update(); @@ -250,7 +251,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); String sliceName = message.getStr(SHARD_ID_PROP); @@ -149,7 +150,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java index 2c99dc36ef7c..2cfbdf7ffb60 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java @@ -32,7 +32,8 @@ public DeleteAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java index aa6c8f9ef5bf..eaf97accf3ea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -71,7 +71,8 @@ public DeleteBackupCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); String backupName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 1d8629c4fdcd..190193fbc486 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -64,7 +64,8 @@ public DeleteCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { Object o = message.get(MaintainRoutedAliasCmd.INVOKED_BY_ROUTED_ALIAS); if (o != null) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java index b93980ff7da0..948c1893fe1c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java @@ -36,7 +36,8 @@ public DeleteNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired(message, "node"); String node = message.getStr("node"); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index 8ac4fae4755f..7b9876f3c31f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -61,7 +61,8 @@ public DeleteReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { deleteReplica(clusterState, message, results, null); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java index d32b80d4b390..25b4fa45da65 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java @@ -58,7 +58,8 @@ public DeleteShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP); String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index 35f526fc6fcc..0d858d984547 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -60,7 +60,8 @@ public DeleteSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index bd7c8404886c..ee36cec69eb8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -429,7 +429,12 @@ public OverseerSolrResponse call() { CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(ccc.getSolrCloudManager().getClusterState(), message, results); + // TODO: FIX + command.call( + ccc.getSolrCloudManager().getClusterState(), + message, + lock.hashCode() + "", + results); } else { asyncTaskTracker.cancelAsyncId(asyncId); // Seeing this is a bug, not bad user data diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index f0f326fc3e44..348c9cdf69fa 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.invoke.MethodHandles; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -67,7 +66,8 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { @Override @SuppressWarnings("unchecked") - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final RemoteMessage typedMessage = new ObjectMapper().convertValue(message.getProperties(), RemoteMessage.class); @@ -88,8 +88,11 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu // Build the core-admin request final ModifiableSolrParams coreApiParams = new ModifiableSolrParams(); coreApiParams.set( - CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.INSTALLCOREDATA.toString()); - typedMessage.toMap(new HashMap<>()).forEach((k, v) -> coreApiParams.set(k, v.toString())); + CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); + coreApiParams.set(CoreAdminParams.BACKUP_LOCATION, typedMessage.location); + coreApiParams.set(CoreAdminParams.BACKUP_REPOSITORY, typedMessage.repository); + coreApiParams.set(CoreAdminParams.NAME, typedMessage.name); + coreApiParams.set(CoreAdminParams.SHARD_BACKUP_ID, typedMessage.shardBackupId); // Send the core-admin request to each replica in the slice final ShardHandler shardHandler = ccc.newShardHandler(); @@ -117,7 +120,6 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu Set successfulReplicas = leaderEligibleReplicas.stream() .filter(replica -> !notLiveReplicas.contains(replica)) - // TODO: This does not work for nodes that have multiple cores of the shard .filter( replica -> failures == null @@ -183,9 +185,104 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } } + /* + public static void handleCoreRestoreResponses( + CoreContainer coreContainer, + String collection, + List shards, + NamedList results, + List notLiveReplicas, + String errorMessage) { + DocCollection collectionState = coreContainer.getZkController().getZkStateReader().getCollectionLive(collection); + Collection allReplicas = + shards.stream() + .flatMap(shard -> collectionState.getSlice(shard).getReplicas().stream()) + .toList(); + + // Ensure that terms are correct for this shard after the execution is done + // We only care about leader eligible replicas, all others will eventually get updated. + List leaderEligibleReplicas = + allReplicas.stream() + .filter(r -> r.getType().leaderEligible) + .toList(); + + NamedList failures = (NamedList) results.get("failure"); + Set successfulReplicas = + leaderEligibleReplicas.stream() + .filter(replica -> !notLiveReplicas.contains(replica)) + .filter( + replica -> + failures == null + || failures.get(CollectionHandlingUtils.requestKey(replica)) == null) + .collect(Collectors.toSet()); + + if (successfulReplicas.isEmpty()) { + // No leader-eligible replicas succeeded, return failure + if (failures == null) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + errorMessage + ". No leader-eligible replicas are live."); + } else { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, errorMessage, (Throwable) failures.getVal(0)); + } + } else if (successfulReplicas.size() < leaderEligibleReplicas.size()) { + // Some, but not all, leader-eligible replicas succeeded. + // Ensure the shard terms are correct so that the non-successful replicas go into recovery + ZkShardTerms shardTerms = + coreContainer + .getZkController() + .getShardTerms(typedMessage.collection, typedMessage.shard); + shardTerms.ensureHighestTerms( + successfulReplicas.stream().map(Replica::getName).collect(Collectors.toSet())); + Set replicasToRecover = + leaderEligibleReplicas.stream() + .filter(r -> !successfulReplicas.contains(r)) + .map(Replica::getName) + .collect(Collectors.toSet()); + coreContainer + .getZkController() + .getZkStateReader() + .waitForState( + collection, + 10, + TimeUnit.SECONDS, + (liveNodes, colState) -> + colState.getSlice(typedMessage.shard).getReplicas().stream() + .filter(r -> replicasToRecover.contains(r.getName())) + .allMatch(r -> Replica.State.RECOVERING.equals(r.getState()))); + + // In order for the async request to succeed, we need to ensure that there is no failure + // message + NamedList successes = (NamedList) results.get("success"); + failures.forEach( + (replicaKey, value) -> { + successes.add( + replicaKey, + new NamedList<>( + Map.of( + "explanation", + "Core install failed, but is now recovering from the leader", + "failure", + value))); + }); + results.remove("failure"); + } else { + // other replicas to-be-created will know that they are out of date by + // looking at their term : 0 compare to term of this core : 1 + coreContainer + .getZkController() + .getShardTerms(collection, typedMessage.shard) + .ensureHighestTermsAreNotZero(); + } + } + */ + /** A value-type representing the message received by {@link InstallShardDataCmd} */ public static class RemoteMessage implements JacksonReflectMapWriter { + @JsonProperty public String callingLockId; + @JsonProperty(QUEUE_OPERATION) public String operation = CollectionParams.CollectionAction.INSTALLSHARDDATA.toLower(); @@ -197,6 +294,10 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String location; + @JsonProperty public String name = ""; + + @JsonProperty public String shardBackupId; + @JsonProperty(ASYNC) public String asyncId; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index 3320ea0b5b75..a89601e402b9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -109,7 +109,8 @@ private void removeCollectionFromAlias( } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { // ---- PARSE PRIMARY MESSAGE PARAMS // important that we use NAME for the alias as that is what the Overseer will get a lock on @@ -146,7 +147,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList { try { deleteTargetCollection( - clusterState, results, aliasName, aliasesManager, action); + clusterState, results, aliasName, aliasesManager, action, lockId); } catch (Exception e) { log.warn( "Deletion of {} by {} {} failed (this might be ok if two clients were", @@ -161,7 +162,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList aliasMetadata, - RoutedAlias.Action action) + RoutedAlias.Action action, + String lockId) throws Exception { NamedList createResults = createCollectionAndWait( - clusterState, aliasName, aliasMetadata, action.targetCollection, ccc); + clusterState, aliasName, aliasMetadata, action.targetCollection, lockId, ccc); if (createResults != null) { results.add("create", createResults); } @@ -210,7 +212,8 @@ public void deleteTargetCollection( NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, - RoutedAlias.Action action) + RoutedAlias.Action action, + String lockId) throws Exception { Map delProps = new HashMap<>(); delProps.put( @@ -219,6 +222,6 @@ public void deleteTargetCollection( () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection)); delProps.put(NAME, action.targetCollection); ZkNodeProps messageDelete = new ZkNodeProps(delProps); - new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, results); + new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, lockId, results); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index 97182ad1dc9c..b1c631a141bf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -65,7 +65,8 @@ public MigrateCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extSourceCollectionName = message.getStr("collection"); String splitKey = message.getStr("split.key"); @@ -156,7 +157,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results, String asyncId, - ZkNodeProps message) + ZkNodeProps message, + String lockId) throws Exception { String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); @@ -183,7 +186,7 @@ private void migrateKey( try { new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); clusterState = zkStateReader.getClusterState(); } catch (Exception e) { log.warn( @@ -318,7 +321,7 @@ private void migrateKey( } log.info("Creating temporary collection: {}", props); - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), results); + new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), lockId, results); // refresh cluster state clusterState = zkStateReader.getClusterState(); Slice tempSourceSlice = @@ -491,7 +494,7 @@ private void migrateKey( log.info("Deleting temporary collection: {}", tempSourceCollectionName); props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); } catch (Exception e) { log.error( "Unable to delete temporary collection: {}. Please remove it manually", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java index 3912e096cb67..c5080438a8a9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java @@ -48,7 +48,8 @@ public MigrateReplicasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); Set sourceNodes = getNodesFromParam(message, CollectionParams.SOURCE_NODES); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 48f065f537e6..6c0a6e489eb2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -60,7 +60,8 @@ public MoveReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { moveReplica(ccc.getZkStateReader().getClusterState(), message, results); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index df933b597fea..4b6a1adb708e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -41,6 +41,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; @@ -112,7 +113,7 @@ public OverseerCollectionMessageHandler( } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { // sometimes overseer messages have the collection name in 'name' field, not 'collection' MDCLoggingContext.setCollection( message.getStr(COLLECTION_PROP) != null @@ -127,7 +128,7 @@ public OverseerSolrResponse processMessage(ZkNodeProps message, String operation CollectionAction action = getCollectionAction(operation); CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(cloudManager.getClusterState(), message, results); + command.call(cloudManager.getClusterState(), message, lock.id(), results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); } @@ -193,7 +194,8 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId) { Arrays.asList( getTaskKey(message), message.getStr(ZkStateReader.SHARD_ID_PROP), - message.getStr(ZkStateReader.REPLICA_PROP))); + message.getStr(ZkStateReader.REPLICA_PROP)), + message.getStr(CollectionAdminParams.CALLING_LOCK_ID)); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java index d8e7a3dea644..5f1bcdbc555c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java @@ -55,7 +55,8 @@ public OverseerRoleCmd( } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (ccc.isDistributedCollectionAPI()) { // No Overseer (not accessible from Collection API command execution in any case) so this diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java index 0b841fe87902..c723e1e0d141 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java @@ -161,7 +161,8 @@ public OverseerStatusCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { // If Collection API execution is distributed, we're not running on the Overseer node so can't // return any Overseer stats. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index 217ce0228c7e..bcb8275e4b70 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -167,7 +167,8 @@ public ReindexCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { log.debug("*** called: {}", message); @@ -297,7 +298,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); createdTarget = true; CollectionHandlingUtils.checkResults( "creating target collection " + targetCollection, cmdResults, true); @@ -351,7 +352,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "creating checkpoint collection " + chkCollection, cmdResults, true); // wait for a while until we see both collections @@ -480,7 +481,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateAliasCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateAliasCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true); reindexingState.put("alias", extCollection + " -> " + targetCollection); @@ -505,7 +506,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "deleting checkpoint collection " + chkCollection, cmdResults, true); @@ -521,7 +522,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "deleting source collection " + collection, cmdResults, true); } else { @@ -580,7 +581,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java index 8677ecf72ef2..d76f74f3e653 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java @@ -41,7 +41,8 @@ public RenameCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(CoreAdminParams.NAME); String target = message.getStr(CollectionAdminParams.TARGET); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index 5134e9953bda..0067a6708ccc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -45,7 +45,8 @@ public ReplaceNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); String source = message.getStr(CollectionParams.SOURCE_NODE); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 55d1725e6ec8..25e4a8cf859d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -23,6 +23,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.INSTALLSHARDDATA; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -92,9 +93,10 @@ public RestoreCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { - try (RestoreContext restoreContext = new RestoreContext(message, ccc)) { + try (RestoreContext restoreContext = new RestoreContext(message, lockId, ccc)) { if (state.hasCollection(restoreContext.restoreCollectionName)) { RestoreOnExistingCollection restoreOnExistingCollection = new RestoreOnExistingCollection(restoreContext); @@ -108,21 +110,24 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu } } - private void requestReplicasToRestore( + private void requestShardsToRestore( NamedList results, DocCollection restoreCollection, - ClusterState clusterState, BackupProperties backupProperties, URI backupPath, String repo, ShardHandler shardHandler, - String asyncId) { + String asyncId, + String lockId) { ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(asyncId, "/admin/collections", ccc); // Copy data from backed up index to each replica for (Slice slice : restoreCollection.getSlices()) { ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); + params.set(CollectionAdminParams.CALLING_LOCK_ID, lockId); + params.set(CollectionAdminParams.COLLECTION, slice.getCollection()); + params.set(CollectionAdminParams.SHARD, slice.getName()); + params.set(CoreAdminParams.ACTION, INSTALLSHARDDATA.toString()); Optional shardBackupId = backupProperties.getShardBackupIdFor(slice.getName()); if (shardBackupId.isPresent()) { params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.get().getIdAsString()); @@ -131,10 +136,24 @@ private void requestReplicasToRestore( } params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); + Replica replica = slice.getLeader(); + if (replica == null) { + replica = + slice.getReplicas().stream() + .findFirst() + .orElseThrow( + () -> + new SolrException( + ErrorCode.INVALID_STATE, + String.format( + Locale.ROOT, + "No replicas for shard %s in collection %s. Cannot restore to a shard with no replicas", + slice.getName(), + slice.getCollection()))); + } + shardRequestTracker.sendShardRequest(replica, params, shardHandler); } - shardRequestTracker.processResponses( - new NamedList<>(), shardHandler, true, "Could not restore core"); + shardRequestTracker.processResponses(results, shardHandler, true, "Could not restore shard"); } /** Encapsulates the parsing and access for common parameters restore parameters and values */ @@ -151,6 +170,7 @@ private static class RestoreContext implements Closeable { final URI backupPath; final List nodeList; + final String lockId; final CoreContainer container; final BackupRepository repository; final ZkStateReader zkStateReader; @@ -159,13 +179,15 @@ private static class RestoreContext implements Closeable { final DocCollection backupCollectionState; final ShardHandler shardHandler; - private RestoreContext(ZkNodeProps message, CollectionCommandContext ccc) throws IOException { + private RestoreContext(ZkNodeProps message, String lockId, CollectionCommandContext ccc) + throws IOException { this.restoreCollectionName = message.getStr(COLLECTION_PROP); this.backupName = message.getStr(NAME); // of backup this.asyncId = message.getStr(ASYNC); this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); + this.lockId = lockId; this.container = ccc.getCoreContainer(); this.repository = this.container.newBackupRepository(repo); @@ -239,7 +261,8 @@ public void process(NamedList results, RestoreContext rc) throws Excepti rc.restoreCollectionName, rc.restoreConfigName, rc.backupCollectionState, - rc.zkStateReader.getClusterState()); + rc.zkStateReader.getClusterState(), + rc.lockId); // note: when createCollection() returns, the collection exists (no race) // Restore collection properties @@ -269,15 +292,15 @@ public void process(NamedList results, RestoreContext rc) throws Excepti // refresh the location copy of collection state restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName); - requestReplicasToRestore( + requestShardsToRestore( results, restoreCollection, - clusterState, rc.backupProperties, rc.backupPath, rc.repo, rc.shardHandler, - rc.asyncId); + rc.asyncId, + rc.lockId); markAllShardsAsActive(restoreCollection); addReplicasToShards(results, clusterState, restoreCollection, replicaPositions, rc.asyncId); restoringAlias(rc.backupProperties); @@ -313,7 +336,8 @@ private void createCoreLessCollection( String restoreCollectionName, String restoreConfigName, DocCollection backupCollectionState, - ClusterState clusterState) + ClusterState clusterState, + String lockId) throws Exception { Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); @@ -368,7 +392,8 @@ private void createCoreLessCollection( propMap.put(CollectionHandlingUtils.SHARDS_PROP, newSlices); } - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(propMap), new NamedList<>()); + new CreateCollectionCmd(ccc) + .call(clusterState, new ZkNodeProps(propMap), lockId, new NamedList<>()); // note: when createCollection() returns, the collection exists (no race) } @@ -616,23 +641,24 @@ public void process(RestoreContext rc, NamedList results) throws Excepti ClusterState clusterState = rc.zkStateReader.getClusterState(); DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); - enableReadOnly(clusterState, restoreCollection); + enableReadOnly(clusterState, restoreCollection, rc.lockId); try { - requestReplicasToRestore( + requestShardsToRestore( results, restoreCollection, - clusterState, rc.backupProperties, rc.backupPath, rc.repo, rc.shardHandler, - rc.asyncId); + rc.asyncId, + rc.lockId); } finally { - disableReadOnly(clusterState, restoreCollection); + disableReadOnly(clusterState, restoreCollection, rc.lockId); } } - private void disableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void disableReadOnly( + ClusterState clusterState, DocCollection restoreCollection, String lockId) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -640,10 +666,12 @@ private void disableReadOnly(ClusterState clusterState, DocCollection restoreCol CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, null); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call(clusterState, params, lockId, new NamedList<>()); } - private void enableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void enableReadOnly( + ClusterState clusterState, DocCollection restoreCollection, String lockId) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -651,7 +679,8 @@ private void enableReadOnly(ClusterState clusterState, DocCollection restoreColl CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, "true"); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call(clusterState, params, lockId, new NamedList<>()); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java index 671b71dddb67..fe20b9e30a54 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java @@ -46,7 +46,8 @@ public SetAliasPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 65d52ab86a73..0dbe5e94a447 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -101,9 +101,10 @@ public SplitShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { - split(state, message, results); + split(state, message, lockId, results); } /** @@ -131,7 +132,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu *

There is a shard split doc (dev-docs/shard-split/shard-split.adoc) on how shard split works; * illustrated with diagrams. */ - public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList results) + public boolean split( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String asyncId = message.getStr(ASYNC); @@ -340,7 +342,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList()); + new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList<>()); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -818,7 +820,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList subSlices, - Set offlineSlices) { + Set offlineSlices, + String lockId) { log.info("Cleaning up after a failed split of {}/{}", collectionName, parentShard); // get the latest state try { @@ -992,7 +995,7 @@ private void cleanupAfterFailure( props.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(props); try { - new DeleteShardCmd(ccc).call(clusterState, m, new NamedList()); + new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList()); } catch (Exception e) { log.warn( "Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 85cc036d9edc..c0d8b17b205e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -34,6 +34,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -100,6 +101,7 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; +import static org.apache.solr.common.params.CoreAdminParams.SHARD_BACKUP_ID; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -1077,6 +1079,9 @@ public Map execute( reqBody.async = req.getParams().get(ASYNC); reqBody.repository = req.getParams().get(BACKUP_REPOSITORY); reqBody.location = req.getParams().get(BACKUP_LOCATION); + reqBody.name = req.getParams().get(NAME); + reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); + reqBody.callingLockId = req.getParams().get(CALLING_LOCK_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index ac13ef759087..ae0b8919f440 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -81,13 +81,13 @@ public AsyncJerseyResponse installShardData( // Only install data to shards which belong to a collection in read-only mode final DocCollection dc = coreContainer.getZkController().getZkStateReader().getCollection(collName); - if (!dc.isReadOnly()) { + if (dc.getSlice(shardName).getReplicas().size() > 1 && !dc.isReadOnly()) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Collection must be in readOnly mode before installing data to shard"); + "Collection must be in readOnly mode before installing data to shard with more than 1 replica"); } - final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); + ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer, @@ -126,7 +126,10 @@ public static ZkNodeProps createRemoteMessage( if (requestBody != null) { messageTyped.location = requestBody.location; messageTyped.repository = requestBody.repository; + messageTyped.name = requestBody.name; + messageTyped.shardBackupId = requestBody.shardBackupId; messageTyped.asyncId = requestBody.async; + messageTyped.callingLockId = requestBody.callingLockId; } messageTyped.validate(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java index 3997b1971b4e..dcf1cfe85c1b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java @@ -132,12 +132,6 @@ private void doRestore(String coreName, RestoreCoreRequestBody requestBody) thro throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); } - // other replicas to-be-created will know that they are out of date by - // looking at their term : 0 compare to term of this core : 1 - coreContainer - .getZkController() - .getShardTerms(cd.getCollectionName(), cd.getShardId()) - .ensureHighestTermsAreNotZero(); // transitions state of update log to ACTIVE UpdateLog updateLog = core.getUpdateHandler().getUpdateLog(); diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index 17223f6deb26..6e959480db6f 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.slf4j.Logger; @@ -31,6 +32,9 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index a7c6043fb19a..3916d6ec5739 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -127,4 +127,6 @@ public interface CollectionAdminParams { String PROPERTY_PREFIX = "property."; String PER_REPLICA_STATE = CollectionStateProps.PER_REPLICA_STATE; + + String CALLING_LOCK_ID = "callingLockId"; } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 73e2e9518a1d..86dd98d8f24c 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -48,7 +48,8 @@ enum LockLevel { REPLICA(3, null), SHARD(2, REPLICA), COLLECTION(1, SHARD), - CLUSTER(0, COLLECTION); + CLUSTER(0, COLLECTION), + BASE(-1, CLUSTER); private final int height; private final LockLevel child; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index c74a9ccbc80f..73403943a88b 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -532,7 +532,10 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // And since all the nodes are hosting cores from all shards, the // distributed query formed by this node will select cores from the // local shards only - QueryResponse qResponse = cloudClient.query(collectionName, qRequest); + QueryResponse qResponse = null; + for (int i = 0; i < 100; i++) { + qResponse = cloudClient.query(collectionName, qRequest); + } Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); assertNotNull("Unable to obtain " + ShardParams.SHARDS_INFO, shardsInfo); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 1d7276a2e229..04e607f86613 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -71,6 +71,7 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -84,6 +85,9 @@ *

For a similar test harness for snapshot backup/restoration see {@link * AbstractCloudBackupRestoreTestCase} */ +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -127,6 +131,9 @@ public void setTestSuffix(String testSuffix) { public abstract String getBackupLocation(); @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSimple() throws Exception { setTestSuffix("testbackupincsimple"); final String backupCollectionName = getCollectionName(); @@ -193,6 +200,9 @@ public void testSimple() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testRestoreToOriginalCollection() throws Exception { setTestSuffix("testbackuprestoretooriginal"); final String backupCollectionName = getCollectionName(); @@ -355,6 +365,9 @@ public void testBackupIncremental() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSkipConfigset() throws Exception { setTestSuffix("testskipconfigset"); final String backupCollectionName = getCollectionName(); From 8365a239d6447863c743e8ea22157b263208ad42 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 1 Aug 2025 15:11:02 -0700 Subject: [PATCH 07/67] Implement callingLock mirroring for distributed API Manager locking --- .../DistributedCollectionLockFactory.java | 4 +- .../apache/solr/cloud/DistributedLock.java | 4 ++ .../solr/cloud/DistributedMultiLock.java | 5 ++ .../ZkDistributedCollectionLockFactory.java | 7 +- .../ZkDistributedConfigSetLockFactory.java | 3 +- .../apache/solr/cloud/ZkDistributedLock.java | 56 ++++++++++----- .../solr/cloud/ZkDistributedLockFactory.java | 11 ++- .../collections/CollectionApiLockFactory.java | 19 ++++- .../cloud/api/collections/CreateAliasCmd.java | 4 +- ...butedCollectionConfigSetCommandRunner.java | 23 +++--- .../handler/admin/CollectionsHandler.java | 2 +- .../org/apache/solr/cloud/TestLockTree.java | 17 +++-- .../solr/cloud/ZkDistributedLockTest.java | 71 ++++++++++++++++--- .../collections/CollectionApiLockingTest.java | 16 ++--- .../admin/api/AddReplicaPropertyAPITest.java | 6 +- .../admin/api/MigrateReplicasAPITest.java | 9 ++- .../handler/admin/api/ReplaceNodeAPITest.java | 12 ++-- 17 files changed, 200 insertions(+), 69 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java index e49aba6c3f51..ec6fa179aeb5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.CollectionApiLockFactory; import org.apache.solr.common.params.CollectionParams; @@ -62,5 +63,6 @@ DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName); + String replicaName, + List callingLockIds); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java index 1929766e86ef..c26c5d499f03 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java @@ -24,4 +24,8 @@ public interface DistributedLock { void release(); boolean isAcquired(); + + String getLockId(); + + boolean isMirroringLock(); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 9979c144e84b..1baea10051f5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.lang.invoke.MethodHandles; import java.util.List; +import java.util.stream.Collectors; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +71,10 @@ public boolean isAcquired() { return true; } + public String getLockId() { + return locks.stream().map(DistributedLock::getLockId).collect(Collectors.joining(",")); + } + @VisibleForTesting public int getCountInternalLocks() { return locks.size(); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java index fe77b18c3f2b..2e3e5c4b8d73 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import java.util.Objects; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; @@ -44,7 +45,8 @@ public DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName) { + String replicaName, + List callingLockIds) { Objects.requireNonNull(collName, "collName can't be null"); if (level != CollectionParams.LockLevel.COLLECTION) { Objects.requireNonNull( @@ -56,7 +58,8 @@ public DistributedLock createLock( } String lockPath = getLockPath(level, collName, shardId, replicaName); - return doCreateLock(isWriteLock, lockPath); + + return doCreateLock(isWriteLock, lockPath, callingLockIds); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index 884703a8829b..c21d8bdeb78c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.Objects; import org.apache.solr.common.cloud.SolrZkClient; @@ -40,7 +41,7 @@ public DistributedLock createLock(boolean isWriteLock, String configSetName) { Objects.requireNonNull(configSetName, "configSetName can't be null"); String lockPath = getLockPath(configSetName); - return doCreateLock(isWriteLock, lockPath); + return doCreateLock(isWriteLock, lockPath, Collections.emptyList()); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index b9fedc5e0763..505a05ef9e03 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -48,9 +48,9 @@ abstract class ZkDistributedLock implements DistributedLock { /** Read lock. */ static class Read extends ZkDistributedLock { - protected Read(SolrZkClient zkClient, String lockPath) + protected Read(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, READ_LOCK_PREFIX); + super(zkClient, lockPath, READ_LOCK_PREFIX, mirroredLockId); } @Override @@ -63,9 +63,9 @@ boolean isBlockedByNodeType(String otherLockName) { /** Write lock. */ static class Write extends ZkDistributedLock { - protected Write(SolrZkClient zkClient, String lockPath) + protected Write(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, WRITE_LOCK_PREFIX); + super(zkClient, lockPath, WRITE_LOCK_PREFIX, mirroredLockId); } @Override @@ -80,23 +80,31 @@ boolean isBlockedByNodeType(String otherLockName) { private final String lockNode; protected final long sequence; protected volatile boolean released = false; + protected final boolean mirrored; - protected ZkDistributedLock(SolrZkClient zkClient, String lockDir, String lockNodePrefix) + protected ZkDistributedLock( + SolrZkClient zkClient, String lockDir, String lockNodePrefix, String mirroredLockId) throws KeeperException, InterruptedException { this.zkClient = zkClient; this.lockDir = lockDir; // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - lockNode = - zkClient.create( - lockDir - + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR - + lockNodePrefix, - null, - CreateMode.EPHEMERAL_SEQUENTIAL, - false); + if (mirroredLockId == null || mirroredLockId.startsWith(lockDir)) { + lockNode = + zkClient.create( + lockDir + + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + + lockNodePrefix, + null, + CreateMode.EPHEMERAL_SEQUENTIAL, + false); + mirrored = false; + } else { + lockNode = mirroredLockId; + mirrored = true; + } sequence = getSequenceFromNodename(lockNode); } @@ -159,8 +167,10 @@ public void waitUntilAcquired() { @Override public void release() { try { - zkClient.delete(lockNode, -1, true); - released = true; + if (!mirrored) { + zkClient.delete(lockNode, -1, true); + released = true; + } } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { @@ -213,7 +223,11 @@ String nodeToWatch() throws KeeperException, InterruptedException { if (!foundSelf) { // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only // here. - throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + if (mirrored) { + throw new SolrException(SERVER_ERROR, "Missing mirrored lock node " + lockNode); + } else { + throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + } } // Didn't return early on any other blocking lock, means we own it @@ -241,6 +255,16 @@ static long getSequenceFromNodename(String lockNode) { return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH)); } + @Override + public String getLockId() { + return lockNode; + } + + @Override + public boolean isMirroringLock() { + return mirrored; + } + @Override public String toString() { return lockNode; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index 54434c8e6bcb..17b25c69edae 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; @@ -33,16 +34,20 @@ abstract class ZkDistributedLockFactory { this.rootPath = rootPath; } - protected DistributedLock doCreateLock(boolean isWriteLock, String lockPath) { + protected DistributedLock doCreateLock( + boolean isWriteLock, String lockPath, List callingLockIds) { try { // TODO optimize by first attempting to create the ZkDistributedLock without calling // makeLockPath() and only call it if the lock creation fails. This will be less costly on // high contention (and slightly more on low contention) makeLockPath(lockPath); + // If a callingLock has the same lockPath as this lock, we just want to mirror that lock + String mirroredLockId = + callingLockIds.stream().filter(p -> p.startsWith(lockPath)).findFirst().orElse(null); return isWriteLock - ? new ZkDistributedLock.Write(zkClient, lockPath) - : new ZkDistributedLock.Read(zkClient, lockPath); + ? new ZkDistributedLock.Write(zkClient, lockPath, mirroredLockId) + : new ZkDistributedLock.Read(zkClient, lockPath, mirroredLockId); } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index e1f6894f3011..d51c8ba5c174 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -17,19 +17,24 @@ package org.apache.solr.cloud.api.collections; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.solr.cloud.DistributedCollectionLockFactory; import org.apache.solr.cloud.DistributedLock; import org.apache.solr.cloud.DistributedMultiLock; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CollectionParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class implements a higher level locking abstraction for the Collection API using lower level * read and write locks. */ public class CollectionApiLockFactory { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final DistributedCollectionLockFactory lockFactory; @@ -54,7 +59,11 @@ public class CollectionApiLockFactory { * prevent other threads from locking. */ DistributedMultiLock createCollectionApiLock( - CollectionParams.LockLevel lockLevel, String collName, String shardId, String replicaName) { + CollectionParams.LockLevel lockLevel, + String collName, + String shardId, + String replicaName, + String callingLockId) { if (lockLevel == CollectionParams.LockLevel.NONE) { return new DistributedMultiLock(List.of()); } @@ -102,6 +111,9 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } + List callingLockIdList = + callingLockId == null ? Collections.emptyList() : List.of(callingLockId.split(",")); + // The first requested lock is a write one (on the target object for the action, depending on // lock level), then requesting read locks on "higher" levels (collection > shard > replica here // for the level. Note LockLevel "height" is other way around). @@ -121,7 +133,10 @@ DistributedMultiLock createCollectionApiLock( // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { - locks.add(lockFactory.createLock(requestWriteLock, level, collName, shardId, replicaName)); + DistributedLock lock = + lockFactory.createLock( + requestWriteLock, level, collName, shardId, replicaName, callingLockIdList); + locks.add(lock); requestWriteLock = false; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index ff5a533aea91..3c162fbea97a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -177,8 +177,8 @@ private void ensureAliasCollection( ZkStateReader zkStateReader, ClusterState state, Map aliasProperties, - String lockId, - String initialCollectionName) + String initialCollectionName, + String lockId) throws Exception { // Create the collection createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, lockId, ccc); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index ee36cec69eb8..6741fa450ecf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -257,7 +257,10 @@ public void runConfigSetCommand( * */ public OverseerSolrResponse runCollectionCommand( - ZkNodeProps message, CollectionParams.CollectionAction action, long timeoutMs) { + ZkNodeProps message, + CollectionParams.CollectionAction action, + long timeoutMs, + String callingLockId) { // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it // through this method earlier). See stopAndWaitForPendingTasksToComplete() below if (shuttingDown) { @@ -282,7 +285,8 @@ public OverseerSolrResponse runCollectionCommand( "Task with the same requestid already exists. (" + asyncId + ")"); } - CollectionCommandRunner commandRunner = new CollectionCommandRunner(message, action, asyncId); + CollectionCommandRunner commandRunner = + new CollectionCommandRunner(message, action, asyncId, callingLockId); final Future taskFuture; try { taskFuture = commandsExecutor.submit(commandRunner); @@ -366,12 +370,17 @@ private class CollectionCommandRunner implements Callable private final ZkNodeProps message; private final CollectionParams.CollectionAction action; private final String asyncId; + private final String callingLockId; private CollectionCommandRunner( - ZkNodeProps message, CollectionParams.CollectionAction action, String asyncId) { + ZkNodeProps message, + CollectionParams.CollectionAction action, + String asyncId, + String callingLockId) { this.message = message; this.action = action; this.asyncId = asyncId; + this.callingLockId = callingLockId; } /** @@ -404,7 +413,8 @@ public OverseerSolrResponse call() { new CollectionApiLockFactory( new ZkDistributedCollectionLockFactory( ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)) - .createCollectionApiLock(action.lockLevel, collName, shardId, replicaName); + .createCollectionApiLock( + action.lockLevel, collName, shardId, replicaName, callingLockId); try { log.debug( @@ -431,10 +441,7 @@ public OverseerSolrResponse call() { if (command != null) { // TODO: FIX command.call( - ccc.getSolrCloudManager().getClusterState(), - message, - lock.hashCode() + "", - results); + ccc.getSolrCloudManager().getClusterState(), message, lock.getLockId(), results); } else { asyncTaskTracker.cancelAsyncId(asyncId); // Seeing this is a bug, not bad user data diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index c0d8b17b205e..e861dd6ce655 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -372,7 +372,7 @@ public static SolrResponse submitCollectionApiCommand( if (distributedCollectionConfigSetCommandRunner.isPresent()) { return distributedCollectionConfigSetCommandRunner .get() - .runCollectionCommand(m, action, timeout); + .runCollectionCommand(m, action, timeout, m.getStr(CALLING_LOCK_ID)); } else { // Sending the Collection API message to Overseer via a Zookeeper queue String operation = m.getStr(QUEUE_OPERATION); if (operation == null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 33455cc2f0c7..795917e57acb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -40,25 +40,30 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); - Lock coll1Lock = lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1")); + Lock coll1Lock = + lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), null); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"))); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null)); assertNull( - lockTree.getSession().lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"))); + lockTree + .getSession() + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null)); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1")); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = - lockTree.getSession().lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2")); + lockTree + .getSession() + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -81,7 +86,7 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = session.lock(operation.first(), operation.second()); + final Lock lock = session.lock(operation.first(), operation.second(), null); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 2c767b91fef3..c6b132ccb94d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; @@ -75,17 +76,32 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Collection level locks DistributedLock collRL1 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); DistributedLock collRL2 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); DistributedLock collWL3 = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); @@ -100,7 +116,12 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock collRL4 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired()); @@ -110,14 +131,24 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // should see no impact. DistributedLock shardWL5 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertTrue( "shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired()); DistributedLock shardWL6 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired()); @@ -125,12 +156,22 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Get a lock on a Replica. Again this is independent of collection or shard level DistributedLock replicaRL7 = factory.createLock( - false, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + false, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); DistributedLock replicaWL8 = factory.createLock( - true, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + true, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired()); @@ -164,13 +205,23 @@ private void multithreadedCollectionTests(DistributedCollectionLockFactory facto // Acquiring right away a read lock DistributedLock readLock = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("readLock should have been acquired", readLock.isAcquired()); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); // Wait for acquisition of the write lock on another thread (and be notified via a latch) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index fb943f6a5e69..e3f38db7e2d3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -65,7 +65,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // hierarchy) DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); assertEquals( "Lock at collection level expected to need one distributed lock", @@ -76,7 +76,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // above DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); assertEquals( "Lock at shard level expected to need two distributed locks", @@ -87,7 +87,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // collection lock above DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null, null); assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); assertTrue("Collection should still be acquired", collLock.isAcquired()); @@ -104,7 +104,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); assertFalse( "replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired()); @@ -112,7 +112,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Now ask for a new lock on the collection collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertFalse( "Collection should not have been acquired, shard1 and shard2 locks preventing it", @@ -131,7 +131,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME, null); assertFalse( "replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired()); @@ -158,13 +158,13 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw // Lock on collection... DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); assertFalse( "replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 6de9c8235366..31f92e92476a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -20,6 +20,7 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -70,7 +71,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); @@ -103,7 +104,8 @@ public void testCreatesValidOverseerMessage() throws Exception { addReplicaPropApi.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index f6124c99703e..9a7dadc9bacc 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -65,7 +66,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -81,7 +82,8 @@ public void testCreatesValidOverseerMessage() throws Exception { new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -98,7 +100,8 @@ public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index e49eb56de0de..cc045c2ddce7 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -62,7 +63,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -76,7 +77,8 @@ public void setUp() throws Exception { public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -91,7 +93,8 @@ public void testCreatesValidOverseerMessage() throws Exception { @Test public void testRequestBodyCanBeOmittedAltogether() throws Exception { replaceNodeApi.replaceNode("demoSourceNode", null); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -104,7 +107,8 @@ public void testRequestBodyCanBeOmittedAltogether() throws Exception { public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); From 00a67b41def85302d571b3b6fd2de4c895d0f3ac Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 24 Jul 2025 11:10:53 -0700 Subject: [PATCH 08/67] Huge commit - restore uses installshard - big update in locking --- .../model/InstallShardDataRequestBody.java | 6 ++ .../java/org/apache/solr/cloud/LockTree.java | 87 +++++++++++++++++-- .../OverseerConfigSetMessageHandler.java | 20 ++++- .../solr/cloud/OverseerMessageHandler.java | 7 +- .../solr/cloud/OverseerTaskProcessor.java | 2 +- .../cloud/api/collections/AddReplicaCmd.java | 3 +- .../solr/cloud/api/collections/AliasCmd.java | 3 +- .../solr/cloud/api/collections/BackupCmd.java | 3 +- .../api/collections/BalanceReplicasCmd.java | 3 +- .../cloud/api/collections/CollApiCmds.java | 31 ++++--- .../collections/CollectionApiLockFactory.java | 5 ++ .../collections/CollectionHandlingUtils.java | 2 +- .../cloud/api/collections/CreateAliasCmd.java | 21 +++-- .../api/collections/CreateCollectionCmd.java | 5 +- .../cloud/api/collections/CreateShardCmd.java | 5 +- .../api/collections/CreateSnapshotCmd.java | 3 +- .../cloud/api/collections/DeleteAliasCmd.java | 3 +- .../api/collections/DeleteBackupCmd.java | 3 +- .../api/collections/DeleteCollectionCmd.java | 3 +- .../cloud/api/collections/DeleteNodeCmd.java | 3 +- .../api/collections/DeleteReplicaCmd.java | 3 +- .../cloud/api/collections/DeleteShardCmd.java | 3 +- .../api/collections/DeleteSnapshotCmd.java | 3 +- ...butedCollectionConfigSetCommandRunner.java | 7 +- .../api/collections/InstallShardDataCmd.java | 10 ++- .../collections/MaintainRoutedAliasCmd.java | 17 ++-- .../cloud/api/collections/MigrateCmd.java | 15 ++-- .../api/collections/MigrateReplicasCmd.java | 3 +- .../cloud/api/collections/MoveReplicaCmd.java | 3 +- .../OverseerCollectionMessageHandler.java | 8 +- .../api/collections/OverseerRoleCmd.java | 3 +- .../api/collections/OverseerStatusCmd.java | 3 +- .../api/collections/ReindexCollectionCmd.java | 25 +++--- .../solr/cloud/api/collections/RenameCmd.java | 3 +- .../cloud/api/collections/ReplaceNodeCmd.java | 3 +- .../cloud/api/collections/RestoreCmd.java | 35 +++++--- .../api/collections/SetAliasPropCmd.java | 3 +- .../cloud/api/collections/SplitShardCmd.java | 17 ++-- .../handler/admin/CollectionsHandler.java | 5 ++ .../handler/admin/api/InstallShardData.java | 9 +- .../solr/handler/admin/api/RestoreCore.java | 6 -- .../solr/s3/S3IncrementalBackupTest.java | 4 + .../common/params/CollectionAdminParams.java | 2 + .../solr/common/params/CollectionParams.java | 3 +- .../solrj/impl/CloudHttp2SolrClientTest.java | 5 +- .../AbstractIncrementalBackupTest.java | 13 +++ 46 files changed, 318 insertions(+), 111 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index 31bec8eb4346..a72b9c2bc920 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -24,5 +24,11 @@ public class InstallShardDataRequestBody { @JsonProperty public String repository; + @JsonProperty public String name; + + @JsonProperty public String shardBackupId; + @JsonProperty public String async; + + @JsonProperty public String callingLockId; } diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index e8d96d4f2cd5..20a5686093f7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.solr.cloud.OverseerMessageHandler.Lock; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.LockLevel; @@ -38,20 +39,35 @@ public class LockTree { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final Node root = new Node(null, LockLevel.CLUSTER, null); + public final Map allLocks = new HashMap<>(); + private class LockImpl implements Lock { final Node node; + final String id; LockImpl(Node node) { this.node = node; + this.id = UUID.randomUUID().toString(); } @Override public void unlock() { synchronized (LockTree.this) { node.unlock(this); + allLocks.remove(id); } } + @Override + public String id() { + return id; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return node.validateSubpath(lockLevel, path); + } + @Override public String toString() { return StrUtils.join(node.constructPath(new ArrayDeque<>()), '/'); @@ -71,12 +87,33 @@ public String toString() { public class Session { private SessionNode root = new SessionNode(LockLevel.CLUSTER); - public Lock lock(CollectionParams.CollectionAction action, List path) { + public Lock lock( + CollectionParams.CollectionAction action, List path, String callingLockId) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; + log.info("Calling lock level: {}", callingLockId); + Node startingNode = LockTree.this.root; + SessionNode startingSession = root; + + // If a callingLockId was passed in, validate it with the current lock path, and only start + // locking below the calling lock + Lock callingLock = callingLockId != null ? allLocks.get(callingLockId) : null; + boolean ignoreCallingLock = false; + if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { + startingNode = ((LockImpl) callingLock).node; + startingSession = startingSession.find(startingNode.level.getHeight(), path); + if (startingSession == null) { + startingSession = root; + } + ignoreCallingLock = true; + } synchronized (LockTree.this) { - if (root.isBusy(action.lockLevel, path)) return null; - Lock lockObject = LockTree.this.root.lock(action.lockLevel, path); - if (lockObject == null) root.markBusy(action.lockLevel, path); + if (startingSession.isBusy(action.lockLevel, path)) return null; + Lock lockObject = startingNode.lock(action.lockLevel, path, ignoreCallingLock); + if (lockObject == null) { + startingSession.markBusy(action.lockLevel, path); + } else { + allLocks.put(lockObject.id(), lockObject); + } return lockObject; } } @@ -125,6 +162,18 @@ boolean isBusy(LockLevel lockLevel, List path) { return false; } } + + SessionNode find(int lockLevel, List path) { + if (level.getHeight() == lockLevel) { + return this; + } else if (level.getHeight() < lockLevel + && kids != null + && kids.containsKey(path.get(level.getHeight()))) { + return kids.get(path.get(level.getHeight())).find(lockLevel, path); + } else { + return null; + } + } } public Session getSession() { @@ -158,8 +207,9 @@ void unlock(LockImpl lockObject) { } } - Lock lock(LockLevel lockLevel, List path) { - if (myLock != null) return null; // I'm already locked. no need to go any further + Lock lock(LockLevel lockLevel, List path, boolean ignoreCurrentLock) { + if (myLock != null && !ignoreCurrentLock) + return null; // I'm already locked. no need to go any further if (lockLevel == level) { // lock is supposed to be acquired at this level // If I am locked or any of my children or grandchildren are locked @@ -171,10 +221,16 @@ Lock lock(LockLevel lockLevel, List path) { Node child = children.get(childName); if (child == null) children.put(childName, child = new Node(childName, level.getChild(), this)); - return child.lock(lockLevel, path); + return child.lock(lockLevel, path, false); } } + boolean validateSubpath(int lockLevel, List path) { + return level.getHeight() < lockLevel + && (level.getHeight() == 0 || name.equals(path.get(level.getHeight() - 1))) + && (mom == null || mom.validateSubpath(lockLevel, path)); + } + ArrayDeque constructPath(ArrayDeque collect) { if (name != null) collect.addFirst(name); if (mom != null) mom.constructPath(collect); @@ -182,5 +238,20 @@ ArrayDeque constructPath(ArrayDeque collect) { } } - static final Lock FREELOCK = () -> {}; + static final String FREELOCK_ID = "-1"; + static final Lock FREELOCK = + new Lock() { + @Override + public void unlock() {} + + @Override + public String id() { + return FREELOCK_ID; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java index 0bf454a06421..849626a30b42 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java @@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -61,7 +62,7 @@ public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader, CoreContaine } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { NamedList results = new NamedList<>(); try { if (!operation.startsWith(CONFIGSETS_ACTION_PREFIX)) { @@ -117,7 +118,22 @@ public Lock lockTask(ZkNodeProps message, long ignored) { String configSetName = getTaskKey(message); if (canExecute(configSetName, message)) { markExclusiveTask(configSetName, message); - return () -> unmarkExclusiveTask(configSetName, message); + return new Lock() { + @Override + public void unlock() { + unmarkExclusiveTask(configSetName, message); + } + + @Override + public String id() { + return configSetName; + } + + @Override + public boolean validateSubpath(int lockLevel, List path) { + return false; + } + }; } return null; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java index 3e369b907316..4bc13e2fe6a6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.common.cloud.ZkNodeProps; /** Interface for processing messages received by an {@link OverseerTaskProcessor} */ @@ -26,7 +27,7 @@ public interface OverseerMessageHandler { * @param operation the operation to process * @return response */ - OverseerSolrResponse processMessage(ZkNodeProps message, String operation); + OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock); /** * @return the name of the OverseerMessageHandler @@ -41,6 +42,10 @@ public interface OverseerMessageHandler { interface Lock { void unlock(); + + String id(); + + boolean validateSubpath(int lockLevel, List path); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index 3751d850f09b..74a6fc642e0c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -545,7 +545,7 @@ public void run() { if (log.isDebugEnabled()) { log.debug("Runner processing {}", head.getId()); } - response = messageHandler.processMessage(message, operation); + response = messageHandler.processMessage(message, operation, lock); } finally { timerContext.stop(); updateStats(statsName); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index a71a515c56d8..1ce8194d564d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -76,7 +76,8 @@ public AddReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { addReplica(state, message, results, null); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 4815a76b8070..98d4f1b311aa 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -56,6 +56,7 @@ static NamedList createCollectionAndWait( String aliasName, Map aliasMetadata, String createCollName, + String lockId, CollectionCommandContext ccc) throws Exception { // Map alias metadata starting with a prefix to a create-collection API request @@ -83,7 +84,7 @@ static NamedList createCollectionAndWait( // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(clusterState, createMessage, results); + new CreateCollectionCmd(ccc).call(clusterState, createMessage, lockId, results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 151cbeeee1a8..87507cee57be 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -71,7 +71,8 @@ public BackupCmd(CollectionCommandContext ccc) { @SuppressWarnings("unchecked") @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java index 7d84c7c6633b..aaa0e684f18c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java @@ -41,7 +41,8 @@ public BalanceReplicasCmd(CollectionCommandContext ccc) { @SuppressWarnings({"unchecked"}) @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { Set nodes; Object nodesRaw = message.get(CollectionParams.NODES); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index d5bb2d245688..1f8550d769ab 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -114,7 +114,8 @@ public class CollApiCmds { * classes whose names ends in {@code Cmd}. */ protected interface CollectionApiCommand { - void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception; + void call(ClusterState state, ZkNodeProps message, String lockId, NamedList results) + throws Exception; } /** @@ -206,7 +207,8 @@ public TraceAwareCommand(CollectionApiCommand command) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final Span localSpan; final Context localContext; @@ -226,7 +228,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu try (var scope = localContext.makeCurrent()) { assert scope != null; // prevent javac warning about scope being unused - command.call(state, message, results); + command.call(state, message, lockId, results); } finally { if (localSpan != null) { localSpan.end(); @@ -238,7 +240,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu public static class MockOperationCmd implements CollectionApiCommand { @Override @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws InterruptedException { // only for test purposes Thread.sleep(message.getInt("sleep", 1)); @@ -260,7 +263,8 @@ public ReloadCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) { + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); @@ -285,7 +289,8 @@ public RebalanceLeadersCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -326,7 +331,8 @@ public AddReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -360,7 +366,8 @@ public DeleteReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); @@ -389,7 +396,8 @@ public BalanceShardsUniqueCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (StrUtils.isBlank(message.getStr(COLLECTION_PROP)) || StrUtils.isBlank(message.getStr(PROPERTY_PROP))) { @@ -425,7 +433,8 @@ public ModifyCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); @@ -508,7 +517,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList locks = new ArrayList<>(iterationOrder.length); for (CollectionParams.LockLevel level : iterationOrder) { + // We do not want to re-lock from the parent lock level + // TODO: Fix + // if (calledFromLockLevel.isHigherOrEqual(level)) { + // continue; + // } // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 738994a94d53..753a0c850e10 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -315,7 +315,7 @@ static void cleanupCollection( Map props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collectionName); new DeleteCollectionCmd(ccc) - .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), results); + .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), null, results); } static Map waitToSeeReplicasInState( diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index f48a984aad16..ff5a533aea91 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -50,7 +50,8 @@ public CreateAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); @@ -63,7 +64,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu if (!anyRoutingParams(message)) { callCreatePlainAlias(message, aliasName, zkStateReader); } else { - callCreateRoutedAlias(message, aliasName, zkStateReader, state); + callCreateRoutedAlias(message, aliasName, zkStateReader, state, lockId); } // Sleep a bit to allow ZooKeeper state propagation. @@ -112,7 +113,11 @@ private List parseCollectionsParameter(Object collections) { } private void callCreateRoutedAlias( - ZkNodeProps message, String aliasName, ZkStateReader zkStateReader, ClusterState state) + ZkNodeProps message, + String aliasName, + ZkStateReader zkStateReader, + ClusterState state, + String lockId) throws Exception { // Validate we got a basic minimum if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) { @@ -150,7 +155,12 @@ private void callCreateRoutedAlias( // Create the first collection. Prior validation ensures that this is not a standard alias collectionListStr = routedAlias.computeInitialCollectionName(); ensureAliasCollection( - aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), collectionListStr); + aliasName, + zkStateReader, + state, + routedAlias.getAliasMetadata(), + collectionListStr, + lockId); } else { List collectionList = aliases.resolveAliases(aliasName); collectionListStr = String.join(",", collectionList); @@ -167,10 +177,11 @@ private void ensureAliasCollection( ZkStateReader zkStateReader, ClusterState state, Map aliasProperties, + String lockId, String initialCollectionName) throws Exception { // Create the collection - createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ccc); + createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, lockId, ccc); validateAllCollectionsExistAndNoDuplicates( Collections.singletonList(initialCollectionName), zkStateReader); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index 2b8a5007bf1d..2a0c935e60aa 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -104,7 +104,8 @@ public CreateCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader ccc.getZkStateReader().aliasesManager.update(); @@ -250,7 +251,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); String sliceName = message.getStr(SHARD_ID_PROP); @@ -149,7 +150,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java index 2c99dc36ef7c..2cfbdf7ffb60 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java @@ -32,7 +32,8 @@ public DeleteAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java index aa6c8f9ef5bf..eaf97accf3ea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -71,7 +71,8 @@ public DeleteBackupCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); String backupName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 1d8629c4fdcd..190193fbc486 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -64,7 +64,8 @@ public DeleteCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { Object o = message.get(MaintainRoutedAliasCmd.INVOKED_BY_ROUTED_ALIAS); if (o != null) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java index b93980ff7da0..948c1893fe1c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java @@ -36,7 +36,8 @@ public DeleteNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired(message, "node"); String node = message.getStr("node"); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index 138dec2ef55d..1cf89549afd7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -61,7 +61,8 @@ public DeleteReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { deleteReplica(clusterState, message, results, null); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java index d32b80d4b390..25b4fa45da65 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java @@ -58,7 +58,8 @@ public DeleteShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP); String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index b3970210cb6b..71678f2eb4a2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -60,7 +60,8 @@ public DeleteSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index bd7c8404886c..ee36cec69eb8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -429,7 +429,12 @@ public OverseerSolrResponse call() { CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(ccc.getSolrCloudManager().getClusterState(), message, results); + // TODO: FIX + command.call( + ccc.getSolrCloudManager().getClusterState(), + message, + lock.hashCode() + "", + results); } else { asyncTaskTracker.cancelAsyncId(asyncId); // Seeing this is a bug, not bad user data diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 18ce5aa40716..061bed68025d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -58,7 +58,9 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + @SuppressWarnings("unchecked") + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { final RemoteMessage typedMessage = new ObjectMapper().convertValue(message.getProperties(), RemoteMessage.class); @@ -97,6 +99,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu /** A value-type representing the message received by {@link InstallShardDataCmd} */ public static class RemoteMessage implements JacksonReflectMapWriter { + @JsonProperty public String callingLockId; + @JsonProperty(QUEUE_OPERATION) public String operation = CollectionParams.CollectionAction.INSTALLSHARDDATA.toLower(); @@ -108,6 +112,10 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String location; + @JsonProperty public String name = ""; + + @JsonProperty public String shardBackupId; + @JsonProperty(ASYNC) public String asyncId; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index 3320ea0b5b75..a89601e402b9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -109,7 +109,8 @@ private void removeCollectionFromAlias( } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { // ---- PARSE PRIMARY MESSAGE PARAMS // important that we use NAME for the alias as that is what the Overseer will get a lock on @@ -146,7 +147,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList { try { deleteTargetCollection( - clusterState, results, aliasName, aliasesManager, action); + clusterState, results, aliasName, aliasesManager, action, lockId); } catch (Exception e) { log.warn( "Deletion of {} by {} {} failed (this might be ok if two clients were", @@ -161,7 +162,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList aliasMetadata, - RoutedAlias.Action action) + RoutedAlias.Action action, + String lockId) throws Exception { NamedList createResults = createCollectionAndWait( - clusterState, aliasName, aliasMetadata, action.targetCollection, ccc); + clusterState, aliasName, aliasMetadata, action.targetCollection, lockId, ccc); if (createResults != null) { results.add("create", createResults); } @@ -210,7 +212,8 @@ public void deleteTargetCollection( NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, - RoutedAlias.Action action) + RoutedAlias.Action action, + String lockId) throws Exception { Map delProps = new HashMap<>(); delProps.put( @@ -219,6 +222,6 @@ public void deleteTargetCollection( () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection)); delProps.put(NAME, action.targetCollection); ZkNodeProps messageDelete = new ZkNodeProps(delProps); - new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, results); + new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, lockId, results); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index b09f806535a6..cc54aa9aa3df 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -65,7 +65,8 @@ public MigrateCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extSourceCollectionName = message.getStr("collection"); String splitKey = message.getStr("split.key"); @@ -156,7 +157,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList results, String asyncId, - ZkNodeProps message) + ZkNodeProps message, + String lockId) throws Exception { String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); @@ -183,7 +186,7 @@ private void migrateKey( try { new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); clusterState = zkStateReader.getClusterState(); } catch (Exception e) { log.warn( @@ -318,7 +321,7 @@ private void migrateKey( } log.info("Creating temporary collection: {}", props); - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), results); + new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), lockId, results); // refresh cluster state clusterState = zkStateReader.getClusterState(); Slice tempSourceSlice = @@ -491,7 +494,7 @@ private void migrateKey( log.info("Deleting temporary collection: {}", tempSourceCollectionName); props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), results); + .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); } catch (Exception e) { log.error( "Unable to delete temporary collection: {}. Please remove it manually", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java index 3912e096cb67..c5080438a8a9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java @@ -48,7 +48,8 @@ public MigrateReplicasCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); Set sourceNodes = getNodesFromParam(message, CollectionParams.SOURCE_NODES); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 48f065f537e6..6c0a6e489eb2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -60,7 +60,8 @@ public MoveReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { moveReplica(ccc.getZkStateReader().getClusterState(), message, results); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index df933b597fea..4b6a1adb708e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -41,6 +41,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; @@ -112,7 +113,7 @@ public OverseerCollectionMessageHandler( } @Override - public OverseerSolrResponse processMessage(ZkNodeProps message, String operation) { + public OverseerSolrResponse processMessage(ZkNodeProps message, String operation, Lock lock) { // sometimes overseer messages have the collection name in 'name' field, not 'collection' MDCLoggingContext.setCollection( message.getStr(COLLECTION_PROP) != null @@ -127,7 +128,7 @@ public OverseerSolrResponse processMessage(ZkNodeProps message, String operation CollectionAction action = getCollectionAction(operation); CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(cloudManager.getClusterState(), message, results); + command.call(cloudManager.getClusterState(), message, lock.id(), results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); } @@ -193,7 +194,8 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId) { Arrays.asList( getTaskKey(message), message.getStr(ZkStateReader.SHARD_ID_PROP), - message.getStr(ZkStateReader.REPLICA_PROP))); + message.getStr(ZkStateReader.REPLICA_PROP)), + message.getStr(CollectionAdminParams.CALLING_LOCK_ID)); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java index d8e7a3dea644..5f1bcdbc555c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java @@ -55,7 +55,8 @@ public OverseerRoleCmd( } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { if (ccc.isDistributedCollectionAPI()) { // No Overseer (not accessible from Collection API command execution in any case) so this diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java index 0b841fe87902..c723e1e0d141 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java @@ -161,7 +161,8 @@ public OverseerStatusCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { // If Collection API execution is distributed, we're not running on the Overseer node so can't // return any Overseer stats. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index 217ce0228c7e..bcb8275e4b70 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -167,7 +167,8 @@ public ReindexCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) + public void call( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { log.debug("*** called: {}", message); @@ -297,7 +298,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); createdTarget = true; CollectionHandlingUtils.checkResults( "creating target collection " + targetCollection, cmdResults, true); @@ -351,7 +352,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "creating checkpoint collection " + chkCollection, cmdResults, true); // wait for a while until we see both collections @@ -480,7 +481,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new CreateAliasCmd(ccc).call(clusterState, cmd, cmdResults); + new CreateAliasCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true); reindexingState.put("alias", extCollection + " -> " + targetCollection); @@ -505,7 +506,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "deleting checkpoint collection " + chkCollection, cmdResults, true); @@ -521,7 +522,7 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "deleting source collection " + collection, cmdResults, true); } else { @@ -580,7 +581,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, cmdResults); + new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java index 8677ecf72ef2..d76f74f3e653 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java @@ -41,7 +41,8 @@ public RenameCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String extCollectionName = message.getStr(CoreAdminParams.NAME); String target = message.getStr(CollectionAdminParams.TARGET); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index 5134e9953bda..0067a6708ccc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -45,7 +45,8 @@ public ReplaceNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); String source = message.getStr(CollectionParams.SOURCE_NODE); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 55d1725e6ec8..11ab41495392 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -92,9 +92,10 @@ public RestoreCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { - try (RestoreContext restoreContext = new RestoreContext(message, ccc)) { + try (RestoreContext restoreContext = new RestoreContext(message, lockId, ccc)) { if (state.hasCollection(restoreContext.restoreCollectionName)) { RestoreOnExistingCollection restoreOnExistingCollection = new RestoreOnExistingCollection(restoreContext); @@ -151,6 +152,7 @@ private static class RestoreContext implements Closeable { final URI backupPath; final List nodeList; + final String lockId; final CoreContainer container; final BackupRepository repository; final ZkStateReader zkStateReader; @@ -159,13 +161,15 @@ private static class RestoreContext implements Closeable { final DocCollection backupCollectionState; final ShardHandler shardHandler; - private RestoreContext(ZkNodeProps message, CollectionCommandContext ccc) throws IOException { + private RestoreContext(ZkNodeProps message, String lockId, CollectionCommandContext ccc) + throws IOException { this.restoreCollectionName = message.getStr(COLLECTION_PROP); this.backupName = message.getStr(NAME); // of backup this.asyncId = message.getStr(ASYNC); this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); + this.lockId = lockId; this.container = ccc.getCoreContainer(); this.repository = this.container.newBackupRepository(repo); @@ -239,7 +243,8 @@ public void process(NamedList results, RestoreContext rc) throws Excepti rc.restoreCollectionName, rc.restoreConfigName, rc.backupCollectionState, - rc.zkStateReader.getClusterState()); + rc.zkStateReader.getClusterState(), + rc.lockId); // note: when createCollection() returns, the collection exists (no race) // Restore collection properties @@ -313,7 +318,8 @@ private void createCoreLessCollection( String restoreCollectionName, String restoreConfigName, DocCollection backupCollectionState, - ClusterState clusterState) + ClusterState clusterState, + String lockId) throws Exception { Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); @@ -368,7 +374,8 @@ private void createCoreLessCollection( propMap.put(CollectionHandlingUtils.SHARDS_PROP, newSlices); } - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(propMap), new NamedList<>()); + new CreateCollectionCmd(ccc) + .call(clusterState, new ZkNodeProps(propMap), lockId, new NamedList<>()); // note: when createCollection() returns, the collection exists (no race) } @@ -616,7 +623,7 @@ public void process(RestoreContext rc, NamedList results) throws Excepti ClusterState clusterState = rc.zkStateReader.getClusterState(); DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); - enableReadOnly(clusterState, restoreCollection); + enableReadOnly(clusterState, restoreCollection, rc.lockId); try { requestReplicasToRestore( results, @@ -628,11 +635,12 @@ public void process(RestoreContext rc, NamedList results) throws Excepti rc.shardHandler, rc.asyncId); } finally { - disableReadOnly(clusterState, restoreCollection); + disableReadOnly(clusterState, restoreCollection, rc.lockId); } } - private void disableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void disableReadOnly( + ClusterState clusterState, DocCollection restoreCollection, String lockId) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -640,10 +648,12 @@ private void disableReadOnly(ClusterState clusterState, DocCollection restoreCol CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, null); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call(clusterState, params, lockId, new NamedList<>()); } - private void enableReadOnly(ClusterState clusterState, DocCollection restoreCollection) + private void enableReadOnly( + ClusterState clusterState, DocCollection restoreCollection, String lockId) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -651,7 +661,8 @@ private void enableReadOnly(ClusterState clusterState, DocCollection restoreColl CollectionParams.CollectionAction.MODIFYCOLLECTION.toString(), ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, "true"); - new CollApiCmds.ModifyCollectionCmd(ccc).call(clusterState, params, new NamedList<>()); + new CollApiCmds.ModifyCollectionCmd(ccc) + .call(clusterState, params, lockId, new NamedList<>()); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java index 671b71dddb67..fe20b9e30a54 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java @@ -46,7 +46,8 @@ public SetAliasPropCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { String aliasName = message.getStr(NAME); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 12ac6a4d89a4..107c77a21a2d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -101,9 +101,10 @@ public SplitShardCmd(CollectionCommandContext ccc) { } @Override - public void call(ClusterState state, ZkNodeProps message, NamedList results) + public void call( + ClusterState state, ZkNodeProps message, String lockId, NamedList results) throws Exception { - split(state, message, results); + split(state, message, lockId, results); } /** @@ -131,7 +132,8 @@ public void call(ClusterState state, ZkNodeProps message, NamedList resu *

There is a shard split doc (dev-docs/shard-split/shard-split.adoc) on how shard split works; * illustrated with diagrams. */ - public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList results) + public boolean split( + ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) throws Exception { final String asyncId = message.getStr(ASYNC); @@ -341,7 +343,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList()); + new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList<>()); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -819,7 +821,7 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList subSlices, - Set offlineSlices) { + Set offlineSlices, + String lockId) { log.info("Cleaning up after a failed split of {}/{}", collectionName, parentShard); // get the latest state try { @@ -993,7 +996,7 @@ private void cleanupAfterFailure( props.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(props); try { - new DeleteShardCmd(ccc).call(clusterState, m, new NamedList()); + new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList()); } catch (Exception e) { log.warn( "Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 85cc036d9edc..c0d8b17b205e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -34,6 +34,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -100,6 +101,7 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; +import static org.apache.solr.common.params.CoreAdminParams.SHARD_BACKUP_ID; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -1077,6 +1079,9 @@ public Map execute( reqBody.async = req.getParams().get(ASYNC); reqBody.repository = req.getParams().get(BACKUP_REPOSITORY); reqBody.location = req.getParams().get(BACKUP_LOCATION); + reqBody.name = req.getParams().get(NAME); + reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); + reqBody.callingLockId = req.getParams().get(CALLING_LOCK_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index ac13ef759087..ae0b8919f440 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -81,13 +81,13 @@ public AsyncJerseyResponse installShardData( // Only install data to shards which belong to a collection in read-only mode final DocCollection dc = coreContainer.getZkController().getZkStateReader().getCollection(collName); - if (!dc.isReadOnly()) { + if (dc.getSlice(shardName).getReplicas().size() > 1 && !dc.isReadOnly()) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Collection must be in readOnly mode before installing data to shard"); + "Collection must be in readOnly mode before installing data to shard with more than 1 replica"); } - final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); + ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer, @@ -126,7 +126,10 @@ public static ZkNodeProps createRemoteMessage( if (requestBody != null) { messageTyped.location = requestBody.location; messageTyped.repository = requestBody.repository; + messageTyped.name = requestBody.name; + messageTyped.shardBackupId = requestBody.shardBackupId; messageTyped.asyncId = requestBody.async; + messageTyped.callingLockId = requestBody.callingLockId; } messageTyped.validate(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java index 3997b1971b4e..dcf1cfe85c1b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java @@ -132,12 +132,6 @@ private void doRestore(String coreName, RestoreCoreRequestBody requestBody) thro throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); } - // other replicas to-be-created will know that they are out of date by - // looking at their term : 0 compare to term of this core : 1 - coreContainer - .getZkController() - .getShardTerms(cd.getCollectionName(), cd.getShardId()) - .ensureHighestTermsAreNotZero(); // transitions state of update log to ACTIVE UpdateLog updateLog = core.getUpdateHandler().getUpdateLog(); diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index 17223f6deb26..6e959480db6f 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.slf4j.Logger; @@ -31,6 +32,9 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index a7c6043fb19a..3916d6ec5739 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -127,4 +127,6 @@ public interface CollectionAdminParams { String PROPERTY_PREFIX = "property."; String PER_REPLICA_STATE = CollectionStateProps.PER_REPLICA_STATE; + + String CALLING_LOCK_ID = "callingLockId"; } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 73e2e9518a1d..86dd98d8f24c 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -48,7 +48,8 @@ enum LockLevel { REPLICA(3, null), SHARD(2, REPLICA), COLLECTION(1, SHARD), - CLUSTER(0, COLLECTION); + CLUSTER(0, COLLECTION), + BASE(-1, CLUSTER); private final int height; private final LockLevel child; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index c74a9ccbc80f..73403943a88b 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -532,7 +532,10 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // And since all the nodes are hosting cores from all shards, the // distributed query formed by this node will select cores from the // local shards only - QueryResponse qResponse = cloudClient.query(collectionName, qRequest); + QueryResponse qResponse = null; + for (int i = 0; i < 100; i++) { + qResponse = cloudClient.query(collectionName, qRequest); + } Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); assertNotNull("Unable to obtain " + ShardParams.SHARDS_INFO, shardsInfo); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 1d7276a2e229..04e607f86613 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -71,6 +71,7 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -84,6 +85,9 @@ *

For a similar test harness for snapshot backup/restoration see {@link * AbstractCloudBackupRestoreTestCase} */ +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -127,6 +131,9 @@ public void setTestSuffix(String testSuffix) { public abstract String getBackupLocation(); @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSimple() throws Exception { setTestSuffix("testbackupincsimple"); final String backupCollectionName = getCollectionName(); @@ -193,6 +200,9 @@ public void testSimple() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testRestoreToOriginalCollection() throws Exception { setTestSuffix("testbackuprestoretooriginal"); final String backupCollectionName = getCollectionName(); @@ -355,6 +365,9 @@ public void testBackupIncremental() throws Exception { } @Test + @LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSkipConfigset() throws Exception { setTestSuffix("testskipconfigset"); final String backupCollectionName = getCollectionName(); From 2d5e12bac8243a9f5d09b12a00a5684f7d1d6e60 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 1 Aug 2025 15:11:02 -0700 Subject: [PATCH 09/67] Implement callingLock mirroring for distributed API Manager locking --- .../DistributedCollectionLockFactory.java | 4 +- .../apache/solr/cloud/DistributedLock.java | 4 ++ .../solr/cloud/DistributedMultiLock.java | 5 ++ .../ZkDistributedCollectionLockFactory.java | 7 +- .../ZkDistributedConfigSetLockFactory.java | 3 +- .../apache/solr/cloud/ZkDistributedLock.java | 56 ++++++++++----- .../solr/cloud/ZkDistributedLockFactory.java | 11 ++- .../collections/CollectionApiLockFactory.java | 19 ++++- .../cloud/api/collections/CreateAliasCmd.java | 4 +- ...butedCollectionConfigSetCommandRunner.java | 23 +++--- .../handler/admin/CollectionsHandler.java | 2 +- .../org/apache/solr/cloud/TestLockTree.java | 17 +++-- .../solr/cloud/ZkDistributedLockTest.java | 71 ++++++++++++++++--- .../collections/CollectionApiLockingTest.java | 16 ++--- .../admin/api/AddReplicaPropertyAPITest.java | 6 +- .../admin/api/MigrateReplicasAPITest.java | 9 ++- .../handler/admin/api/ReplaceNodeAPITest.java | 12 ++-- 17 files changed, 200 insertions(+), 69 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java index e49aba6c3f51..ec6fa179aeb5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.CollectionApiLockFactory; import org.apache.solr.common.params.CollectionParams; @@ -62,5 +63,6 @@ DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName); + String replicaName, + List callingLockIds); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java index 1929766e86ef..c26c5d499f03 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java @@ -24,4 +24,8 @@ public interface DistributedLock { void release(); boolean isAcquired(); + + String getLockId(); + + boolean isMirroringLock(); } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 9979c144e84b..1baea10051f5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.lang.invoke.MethodHandles; import java.util.List; +import java.util.stream.Collectors; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +71,10 @@ public boolean isAcquired() { return true; } + public String getLockId() { + return locks.stream().map(DistributedLock::getLockId).collect(Collectors.joining(",")); + } + @VisibleForTesting public int getCountInternalLocks() { return locks.size(); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java index fe77b18c3f2b..2e3e5c4b8d73 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import java.util.Objects; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; @@ -44,7 +45,8 @@ public DistributedLock createLock( CollectionParams.LockLevel level, String collName, String shardId, - String replicaName) { + String replicaName, + List callingLockIds) { Objects.requireNonNull(collName, "collName can't be null"); if (level != CollectionParams.LockLevel.COLLECTION) { Objects.requireNonNull( @@ -56,7 +58,8 @@ public DistributedLock createLock( } String lockPath = getLockPath(level, collName, shardId, replicaName); - return doCreateLock(isWriteLock, lockPath); + + return doCreateLock(isWriteLock, lockPath, callingLockIds); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index 884703a8829b..c21d8bdeb78c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.Objects; import org.apache.solr.common.cloud.SolrZkClient; @@ -40,7 +41,7 @@ public DistributedLock createLock(boolean isWriteLock, String configSetName) { Objects.requireNonNull(configSetName, "configSetName can't be null"); String lockPath = getLockPath(configSetName); - return doCreateLock(isWriteLock, lockPath); + return doCreateLock(isWriteLock, lockPath, Collections.emptyList()); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index b9fedc5e0763..505a05ef9e03 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -48,9 +48,9 @@ abstract class ZkDistributedLock implements DistributedLock { /** Read lock. */ static class Read extends ZkDistributedLock { - protected Read(SolrZkClient zkClient, String lockPath) + protected Read(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, READ_LOCK_PREFIX); + super(zkClient, lockPath, READ_LOCK_PREFIX, mirroredLockId); } @Override @@ -63,9 +63,9 @@ boolean isBlockedByNodeType(String otherLockName) { /** Write lock. */ static class Write extends ZkDistributedLock { - protected Write(SolrZkClient zkClient, String lockPath) + protected Write(SolrZkClient zkClient, String lockPath, String mirroredLockId) throws KeeperException, InterruptedException { - super(zkClient, lockPath, WRITE_LOCK_PREFIX); + super(zkClient, lockPath, WRITE_LOCK_PREFIX, mirroredLockId); } @Override @@ -80,23 +80,31 @@ boolean isBlockedByNodeType(String otherLockName) { private final String lockNode; protected final long sequence; protected volatile boolean released = false; + protected final boolean mirrored; - protected ZkDistributedLock(SolrZkClient zkClient, String lockDir, String lockNodePrefix) + protected ZkDistributedLock( + SolrZkClient zkClient, String lockDir, String lockNodePrefix, String mirroredLockId) throws KeeperException, InterruptedException { this.zkClient = zkClient; this.lockDir = lockDir; // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - lockNode = - zkClient.create( - lockDir - + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR - + lockNodePrefix, - null, - CreateMode.EPHEMERAL_SEQUENTIAL, - false); + if (mirroredLockId == null || mirroredLockId.startsWith(lockDir)) { + lockNode = + zkClient.create( + lockDir + + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + + lockNodePrefix, + null, + CreateMode.EPHEMERAL_SEQUENTIAL, + false); + mirrored = false; + } else { + lockNode = mirroredLockId; + mirrored = true; + } sequence = getSequenceFromNodename(lockNode); } @@ -159,8 +167,10 @@ public void waitUntilAcquired() { @Override public void release() { try { - zkClient.delete(lockNode, -1, true); - released = true; + if (!mirrored) { + zkClient.delete(lockNode, -1, true); + released = true; + } } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { @@ -213,7 +223,11 @@ String nodeToWatch() throws KeeperException, InterruptedException { if (!foundSelf) { // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only // here. - throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + if (mirrored) { + throw new SolrException(SERVER_ERROR, "Missing mirrored lock node " + lockNode); + } else { + throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); + } } // Didn't return early on any other blocking lock, means we own it @@ -241,6 +255,16 @@ static long getSequenceFromNodename(String lockNode) { return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH)); } + @Override + public String getLockId() { + return lockNode; + } + + @Override + public boolean isMirroringLock() { + return mirrored; + } + @Override public String toString() { return lockNode; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index 54434c8e6bcb..17b25c69edae 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.List; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; @@ -33,16 +34,20 @@ abstract class ZkDistributedLockFactory { this.rootPath = rootPath; } - protected DistributedLock doCreateLock(boolean isWriteLock, String lockPath) { + protected DistributedLock doCreateLock( + boolean isWriteLock, String lockPath, List callingLockIds) { try { // TODO optimize by first attempting to create the ZkDistributedLock without calling // makeLockPath() and only call it if the lock creation fails. This will be less costly on // high contention (and slightly more on low contention) makeLockPath(lockPath); + // If a callingLock has the same lockPath as this lock, we just want to mirror that lock + String mirroredLockId = + callingLockIds.stream().filter(p -> p.startsWith(lockPath)).findFirst().orElse(null); return isWriteLock - ? new ZkDistributedLock.Write(zkClient, lockPath) - : new ZkDistributedLock.Read(zkClient, lockPath); + ? new ZkDistributedLock.Write(zkClient, lockPath, mirroredLockId) + : new ZkDistributedLock.Read(zkClient, lockPath, mirroredLockId); } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index e1f6894f3011..d51c8ba5c174 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -17,19 +17,24 @@ package org.apache.solr.cloud.api.collections; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.solr.cloud.DistributedCollectionLockFactory; import org.apache.solr.cloud.DistributedLock; import org.apache.solr.cloud.DistributedMultiLock; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CollectionParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class implements a higher level locking abstraction for the Collection API using lower level * read and write locks. */ public class CollectionApiLockFactory { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final DistributedCollectionLockFactory lockFactory; @@ -54,7 +59,11 @@ public class CollectionApiLockFactory { * prevent other threads from locking. */ DistributedMultiLock createCollectionApiLock( - CollectionParams.LockLevel lockLevel, String collName, String shardId, String replicaName) { + CollectionParams.LockLevel lockLevel, + String collName, + String shardId, + String replicaName, + String callingLockId) { if (lockLevel == CollectionParams.LockLevel.NONE) { return new DistributedMultiLock(List.of()); } @@ -102,6 +111,9 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } + List callingLockIdList = + callingLockId == null ? Collections.emptyList() : List.of(callingLockId.split(",")); + // The first requested lock is a write one (on the target object for the action, depending on // lock level), then requesting read locks on "higher" levels (collection > shard > replica here // for the level. Note LockLevel "height" is other way around). @@ -121,7 +133,10 @@ DistributedMultiLock createCollectionApiLock( // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { - locks.add(lockFactory.createLock(requestWriteLock, level, collName, shardId, replicaName)); + DistributedLock lock = + lockFactory.createLock( + requestWriteLock, level, collName, shardId, replicaName, callingLockIdList); + locks.add(lock); requestWriteLock = false; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index ff5a533aea91..3c162fbea97a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -177,8 +177,8 @@ private void ensureAliasCollection( ZkStateReader zkStateReader, ClusterState state, Map aliasProperties, - String lockId, - String initialCollectionName) + String initialCollectionName, + String lockId) throws Exception { // Create the collection createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, lockId, ccc); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index ee36cec69eb8..6741fa450ecf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -257,7 +257,10 @@ public void runConfigSetCommand( * */ public OverseerSolrResponse runCollectionCommand( - ZkNodeProps message, CollectionParams.CollectionAction action, long timeoutMs) { + ZkNodeProps message, + CollectionParams.CollectionAction action, + long timeoutMs, + String callingLockId) { // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it // through this method earlier). See stopAndWaitForPendingTasksToComplete() below if (shuttingDown) { @@ -282,7 +285,8 @@ public OverseerSolrResponse runCollectionCommand( "Task with the same requestid already exists. (" + asyncId + ")"); } - CollectionCommandRunner commandRunner = new CollectionCommandRunner(message, action, asyncId); + CollectionCommandRunner commandRunner = + new CollectionCommandRunner(message, action, asyncId, callingLockId); final Future taskFuture; try { taskFuture = commandsExecutor.submit(commandRunner); @@ -366,12 +370,17 @@ private class CollectionCommandRunner implements Callable private final ZkNodeProps message; private final CollectionParams.CollectionAction action; private final String asyncId; + private final String callingLockId; private CollectionCommandRunner( - ZkNodeProps message, CollectionParams.CollectionAction action, String asyncId) { + ZkNodeProps message, + CollectionParams.CollectionAction action, + String asyncId, + String callingLockId) { this.message = message; this.action = action; this.asyncId = asyncId; + this.callingLockId = callingLockId; } /** @@ -404,7 +413,8 @@ public OverseerSolrResponse call() { new CollectionApiLockFactory( new ZkDistributedCollectionLockFactory( ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)) - .createCollectionApiLock(action.lockLevel, collName, shardId, replicaName); + .createCollectionApiLock( + action.lockLevel, collName, shardId, replicaName, callingLockId); try { log.debug( @@ -431,10 +441,7 @@ public OverseerSolrResponse call() { if (command != null) { // TODO: FIX command.call( - ccc.getSolrCloudManager().getClusterState(), - message, - lock.hashCode() + "", - results); + ccc.getSolrCloudManager().getClusterState(), message, lock.getLockId(), results); } else { asyncTaskTracker.cancelAsyncId(asyncId); // Seeing this is a bug, not bad user data diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index c0d8b17b205e..e861dd6ce655 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -372,7 +372,7 @@ public static SolrResponse submitCollectionApiCommand( if (distributedCollectionConfigSetCommandRunner.isPresent()) { return distributedCollectionConfigSetCommandRunner .get() - .runCollectionCommand(m, action, timeout); + .runCollectionCommand(m, action, timeout, m.getStr(CALLING_LOCK_ID)); } else { // Sending the Collection API message to Overseer via a Zookeeper queue String operation = m.getStr(QUEUE_OPERATION); if (operation == null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 33455cc2f0c7..795917e57acb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -40,25 +40,30 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); - Lock coll1Lock = lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1")); + Lock coll1Lock = + lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), null); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"))); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null)); assertNull( - lockTree.getSession().lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"))); + lockTree + .getSession() + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null)); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1")); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = - lockTree.getSession().lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2")); + lockTree + .getSession() + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -81,7 +86,7 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = session.lock(operation.first(), operation.second()); + final Lock lock = session.lock(operation.first(), operation.second(), null); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 2c767b91fef3..c6b132ccb94d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; @@ -75,17 +76,32 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Collection level locks DistributedLock collRL1 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); DistributedLock collRL2 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); DistributedLock collWL3 = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); @@ -100,7 +116,12 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock collRL4 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired()); @@ -110,14 +131,24 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // should see no impact. DistributedLock shardWL5 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertTrue( "shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired()); DistributedLock shardWL6 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null); + true, + CollectionParams.LockLevel.SHARD, + COLLECTION_NAME, + SHARD_NAME, + null, + Collections.emptyList()); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired()); @@ -125,12 +156,22 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Get a lock on a Replica. Again this is independent of collection or shard level DistributedLock replicaRL7 = factory.createLock( - false, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + false, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); DistributedLock replicaWL8 = factory.createLock( - true, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME); + true, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + Collections.emptyList()); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired()); @@ -164,13 +205,23 @@ private void multithreadedCollectionTests(DistributedCollectionLockFactory facto // Acquiring right away a read lock DistributedLock readLock = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + false, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertTrue("readLock should have been acquired", readLock.isAcquired()); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + true, + CollectionParams.LockLevel.COLLECTION, + COLLECTION_NAME, + null, + null, + Collections.emptyList()); assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); // Wait for acquisition of the write lock on another thread (and be notified via a latch) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index fb943f6a5e69..e3f38db7e2d3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -65,7 +65,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // hierarchy) DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); assertEquals( "Lock at collection level expected to need one distributed lock", @@ -76,7 +76,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // above DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); assertEquals( "Lock at shard level expected to need two distributed locks", @@ -87,7 +87,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // collection lock above DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null, null); assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); assertTrue("Collection should still be acquired", collLock.isAcquired()); @@ -104,7 +104,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); assertFalse( "replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired()); @@ -112,7 +112,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Now ask for a new lock on the collection collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertFalse( "Collection should not have been acquired, shard1 and shard2 locks preventing it", @@ -131,7 +131,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME, null); assertFalse( "replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired()); @@ -158,13 +158,13 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw // Lock on collection... DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null); + CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); assertFalse( "replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 6de9c8235366..31f92e92476a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -20,6 +20,7 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -70,7 +71,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); @@ -103,7 +104,8 @@ public void testCreatesValidOverseerMessage() throws Exception { addReplicaPropApi.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index f6124c99703e..9a7dadc9bacc 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -65,7 +66,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -81,7 +82,8 @@ public void testCreatesValidOverseerMessage() throws Exception { new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -98,7 +100,8 @@ public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index e49eb56de0de..cc045c2ddce7 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -62,7 +63,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getDistributedCollectionCommandRunner()) .thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -76,7 +77,8 @@ public void setUp() throws Exception { public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -91,7 +93,8 @@ public void testCreatesValidOverseerMessage() throws Exception { @Test public void testRequestBodyCanBeOmittedAltogether() throws Exception { replaceNodeApi.replaceNode("demoSourceNode", null); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -104,7 +107,8 @@ public void testRequestBodyCanBeOmittedAltogether() throws Exception { public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); From 242235abd56abcf146f8ac55716779a82ae276ed Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 2 Dec 2025 11:37:31 -0800 Subject: [PATCH 10/67] Changelog --- changelog/unreleased/solr-18011-locking-update.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/solr-18011-locking-update.yml diff --git a/changelog/unreleased/solr-18011-locking-update.yml b/changelog/unreleased/solr-18011-locking-update.yml new file mode 100644 index 000000000000..a19faee91d93 --- /dev/null +++ b/changelog/unreleased/solr-18011-locking-update.yml @@ -0,0 +1,10 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Allow locked Admin APIs to call other locked AdminAPIs without deadlocking +type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman + url: https://home.apache.org/phonebook.html?uid=houston +links: + - name: SOLR-18011 + url: https://issues.apache.org/jira/browse/SOLR-18011 From b4c3b7bb0ae71de4cd645c2097389a460bd3a7d0 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 4 Dec 2025 13:30:39 -0800 Subject: [PATCH 11/67] WIP - testing --- .../solr/cloud/DistributedMultiLock.java | 7 +++- .../apache/solr/cloud/ZkDistributedLock.java | 33 +++++++++++++++++-- .../api/collections/InstallShardDataCmd.java | 6 ++-- .../handler/admin/api/InstallCoreData.java | 4 --- .../solr/handler/admin/api/RestoreCore.java | 4 +++ .../apache/solr/cloud/SolrCloudTestCase.java | 2 +- 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 1baea10051f5..ef74b9862c86 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -47,7 +47,12 @@ public void waitUntilAcquired() { for (DistributedLock lock : locks) { log.debug("DistributedMultiLock.waitUntilAcquired. About to wait on lock {}", lock); lock.waitUntilAcquired(); - log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + if (lock.isMirroringLock()) { + log.debug( + "DistributedMultiLock.waitUntilAcquired. Mirroring already-acquired lock {}", lock); + } else { + log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index 505a05ef9e03..515028cf2274 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -41,10 +41,14 @@ abstract class ZkDistributedLock implements DistributedLock { static final char LOCK_PREFIX_SUFFIX = '_'; /** Prefix of EPHEMERAL read lock node names */ - static final String READ_LOCK_PREFIX = "R" + LOCK_PREFIX_SUFFIX; + static final char READ_LOCK_PREFIX_CHAR = 'R'; + + static final String READ_LOCK_PREFIX = "" + READ_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Prefix of EPHEMERAL write lock node names */ - static final String WRITE_LOCK_PREFIX = "W" + LOCK_PREFIX_SUFFIX; + static final char WRITE_LOCK_PREFIX_CHAR = 'W'; + + static final String WRITE_LOCK_PREFIX = "" + WRITE_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Read lock. */ static class Read extends ZkDistributedLock { @@ -59,6 +63,11 @@ boolean isBlockedByNodeType(String otherLockName) { // Lower numbered read locks are ok, they can coexist. return otherLockName.startsWith(WRITE_LOCK_PREFIX); } + + @Override + boolean canMirrorLock(String lockId) { + return true; + } } /** Write lock. */ @@ -73,6 +82,17 @@ boolean isBlockedByNodeType(String otherLockName) { // A write lock is blocked by another read or write lock with a lower sequence number return true; } + + @Override + boolean canMirrorLock(String lockId) { + // Only another Write lock can be mirrored + int lockTypeSuffixIndex = lockId.indexOf(LOCK_PREFIX_SUFFIX) - 1; + if (lockTypeSuffixIndex < 0) { + return false; + } else { + return lockId.charAt(lockTypeSuffixIndex) == WRITE_LOCK_PREFIX_CHAR; + } + } } private final SolrZkClient zkClient; @@ -90,7 +110,7 @@ protected ZkDistributedLock( // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - if (mirroredLockId == null || mirroredLockId.startsWith(lockDir)) { + if (mirroredLockId == null || !mirroredLockId.startsWith(lockDir)) { lockNode = zkClient.create( lockDir @@ -102,6 +122,11 @@ protected ZkDistributedLock( mirrored = false; } else { + if (!canMirrorLock(mirroredLockId)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Cannot mirror lock " + mirroredLockId + " with given lockPrefix: " + lockNodePrefix); + } lockNode = mirroredLockId; mirrored = true; } @@ -265,6 +290,8 @@ public boolean isMirroringLock() { return mirrored; } + abstract boolean canMirrorLock(String lockId); + @Override public String toString() { return lockNode; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 348c9cdf69fa..bfbaacc3c04c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.invoke.MethodHandles; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -31,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.cloud.ZkShardTerms; +import org.apache.solr.common.SolrErrorWrappingException; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -133,8 +135,8 @@ public void call( SolrException.ErrorCode.SERVER_ERROR, errorMessage + ". No leader-eligible replicas are live."); } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, errorMessage, (Throwable) failures.getVal(0)); + throw new SolrErrorWrappingException( + SolrException.ErrorCode.SERVER_ERROR, errorMessage, Collections.singletonList(failures.asMap(1))); } } else if (successfulReplicas.size() < leaderEligibleReplicas.size()) { // Some, but not all, leader-eligible replicas succeeded. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallCoreData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallCoreData.java index b679e0999f1f..5d6290e14975 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallCoreData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallCoreData.java @@ -57,10 +57,6 @@ public SolrJerseyResponse installCoreData(String coreName, InstallCoreDataReques throws Exception { final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - // TODO: This is for testing, we need to figure out how to get the leader to fail - // if (coreContainer.getCoreDescriptor(coreName).getCloudDescriptor().isLeader()) { - // throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Is leader"); - // } if (requestBody == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Required request body is missing"); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java index dcf1cfe85c1b..96af5387cb36 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java @@ -63,6 +63,10 @@ boolean isExpensive() { public SolrJerseyResponse restoreCore(String coreName, RestoreCoreRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + // TODO: This is for testing, we need to figure out how to get the leader to fail + if (coreContainer.getCoreDescriptor(coreName).getCloudDescriptor().isLeader()) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Is leader"); + } ensureRequiredParameterProvided("coreName", coreName); if (requestBody == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing required request body"); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index b2499fc7e2a6..68930fe2d590 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -123,7 +123,7 @@ protected static MiniSolrCloudCluster.Builder configureCluster(int nodeCount) { return new MiniSolrCloudCluster.Builder(nodeCount, createTempDir()) .withOverseer( EnvUtils.getPropertyAsBool( - "solr.cloud.overseer.enabled", LuceneTestCase.random().nextBoolean())); + "solr.cloud.overseer.enabled", false)); // LuceneTestCase.random().nextBoolean())); } public static void configurePrsDefault() { From 43d1091abc155ef0bcf68a5d2b8ec8fc17867f90 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 6 Jan 2026 11:15:22 -0800 Subject: [PATCH 12/67] Removing leadership starts a recovery, don't issue a second recovery --- .../solr/cloud/RecoveringCoreTermWatcher.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index 1e501e9e71ff..f640688d6829 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -56,12 +56,10 @@ public boolean onTermChanged(ShardTerms terms) { String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(); if (terms.haveHighestTermValue(coreNodeName)) return true; if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) { - log.info( - "Start recovery on {} because core's term is less than leader's term", coreNodeName); lastTermDoRecovery.set(terms.getTerm(coreNodeName)); if (coreDescriptor.getCloudDescriptor().isLeader()) { log.warn( - "Removing {} leader as leader, since its term is no longer the highest", + "Removing {} leader as leader, since its term is no longer the highest. This will initiate recovery", coreNodeName); coreContainer.getZkController().giveupLeadership(coreDescriptor); coreContainer @@ -75,11 +73,14 @@ public boolean onTermChanged(ShardTerms terms) { Replica leader = dc.getLeader(coreDescriptor.getCloudDescriptor().getShardId()); return leader == null || !leader.getCoreName().equals(coreDescriptor.getName()); }); + } else { + log.info( + "Start recovery on {} because core's term is less than leader's term", coreNodeName); + solrCore + .getUpdateHandler() + .getSolrCoreState() + .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); } - solrCore - .getUpdateHandler() - .getSolrCoreState() - .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); } } catch (Exception e) { if (log.isInfoEnabled()) { From e037654687d044cfa6df5c7b1ee99d7e0891a69c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 6 Jan 2026 11:16:17 -0800 Subject: [PATCH 13/67] Handle async success/failure better for collectionHandlingUtils --- .../collections/CollectionHandlingUtils.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 335e22d17eea..3184e2222ffe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -237,7 +237,8 @@ static void commit( parentShardLeader.getCoreName(), updateResponse, slice, - Collections.emptySet()); + Collections.emptySet(), + null); } catch (Exception e) { CollectionHandlingUtils.processResponse( results, @@ -246,7 +247,8 @@ static void commit( parentShardLeader.getCoreName(), updateResponse, slice, - Collections.emptySet()); + Collections.emptySet(), + null); throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unable to call distrib softCommit on: " + parentShardLeader.getCoreUrl(), @@ -435,7 +437,7 @@ static List collectionCmd( } static void processResponse( - NamedList results, ShardResponse srsp, Set okayExceptions) { + NamedList results, ShardResponse srsp, Set okayExceptions, String asyncId) { Throwable e = srsp.getException(); String nodeName = srsp.getNodeName(); // Use core or coreNodeName if given as a param, otherwise use nodeName @@ -443,7 +445,7 @@ static void processResponse( SolrResponse solrResponse = srsp.getSolrResponse(); String shard = srsp.getShard(); - processResponse(results, e, nodeName, coreName, solrResponse, shard, okayExceptions); + processResponse(results, e, nodeName, coreName, solrResponse, shard, okayExceptions, asyncId); } static void processResponse( @@ -453,7 +455,8 @@ static void processResponse( String coreName, SolrResponse solrResponse, String shard, - Set okayExceptions) { + Set okayExceptions, + String asyncId) { String rootThrowable = null; if (e instanceof RemoteSolrException remoteSolrException) { rootThrowable = remoteSolrException.getRootThrowable(); @@ -462,7 +465,8 @@ static void processResponse( if (e != null && (rootThrowable == null || !okayExceptions.contains(rootThrowable))) { log.error("Error from shard: {}", shard, e); addFailure(results, nodeName, coreName, e); - } else { + } else if (asyncId == null) { + // Do not add a success for async requests, that will be done when the async result is found addSuccess(results, nodeName, coreName, solrResponse.getResponse()); } } @@ -551,8 +555,6 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( do { srsp = shardHandler.takeCompletedOrError(); if (srsp != null) { - NamedList results = new NamedList<>(); - processResponse(results, srsp, Collections.emptySet()); if (srsp.getSolrResponse().getResponse() == null) { NamedList response = new NamedList<>(); response.add("STATUS", "failed"); @@ -760,7 +762,7 @@ void processResponses( ? shardHandler.takeCompletedOrError() : shardHandler.takeCompletedIncludingErrors(); if (srsp != null) { - processResponse(results, srsp, okayExceptions); + processResponse(results, srsp, okayExceptions, asyncId); Throwable exception = srsp.getException(); if (abortOnError && exception != null) { // drain pending requests @@ -782,6 +784,11 @@ void processResponses( private void waitForAsyncCallsToComplete(NamedList results) { for (AsyncCmdInfo asyncCmdInfo : shardAsyncCmds) { + Object failure = results._get("failure/" + requestKey(asyncCmdInfo.nodeName, asyncCmdInfo.coreName)); + // Do not wait for Async calls that have already failed + if (failure != null) { + return; + } final String node = asyncCmdInfo.nodeName; final String coreName = asyncCmdInfo.coreName; final String shardAsyncId = asyncCmdInfo.asyncId; From 8323e724da7f3ca08f1de7677cc0d606bc95384e Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 6 Jan 2026 11:16:45 -0800 Subject: [PATCH 14/67] Small fixes for installShardCmd --- .../api/collections/InstallShardDataCmd.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index bfbaacc3c04c..23f3977336b7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -25,10 +25,12 @@ import java.lang.invoke.MethodHandles; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.cloud.ZkShardTerms; @@ -145,22 +147,26 @@ public void call( ccc.getCoreContainer() .getZkController() .getShardTerms(typedMessage.collection, typedMessage.shard); + final Set replicasToStartRecovery = new HashSet<>(); + leaderEligibleReplicas.stream() + .filter(r -> !successfulReplicas.contains(r)) + .map(Replica::getName) + .forEach(replicasToStartRecovery::add); + log.info("Putting the unsuccessful replicas into recovery: {}", replicasToStartRecovery); shardTerms.ensureHighestTerms( successfulReplicas.stream().map(Replica::getName).collect(Collectors.toSet())); - Set replicasToRecover = - leaderEligibleReplicas.stream() - .filter(r -> !successfulReplicas.contains(r)) - .map(Replica::getName) - .collect(Collectors.toSet()); ccc.getZkStateReader() .waitForState( typedMessage.collection, 10, TimeUnit.SECONDS, - (liveNodes, collectionState) -> - collectionState.getSlice(typedMessage.shard).getReplicas().stream() - .filter(r -> replicasToRecover.contains(r.getName())) - .allMatch(r -> Replica.State.RECOVERING.equals(r.getState()))); + (liveNodes, collectionState) -> { + collectionState.getSlice(typedMessage.shard).getReplicas().stream() + .filter(r -> Replica.State.RECOVERING.equals(r.getState())) + .map(Replica::getName) + .forEach(replicasToStartRecovery::remove); + return replicasToStartRecovery.isEmpty(); + }); // In order for the async request to succeed, we need to ensure that there is no failure // message From e9c09d682d890f8319109a5d7737d765a0e9da77 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 7 Jan 2026 16:03:38 -0800 Subject: [PATCH 15/67] Add tests, fix recovery contention --- .../solr/cloud/RecoveringCoreTermWatcher.java | 63 +++++++--- .../cloud/ShardLeaderElectionContext.java | 4 +- .../AsyncCallRequestStatusResponseTest.java | 4 +- .../solr/s3/S3IncrementalBackupTest.java | 119 +++++++++++++++++- .../apache/solr/s3/S3InstallShardTest.java | 68 ++++++++++ .../solr/cloud/MiniSolrCloudCluster.java | 9 ++ .../AbstractIncrementalBackupTest.java | 17 +-- .../collections/AbstractInstallShardTest.java | 47 ++----- 8 files changed, 252 insertions(+), 79 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index f640688d6829..ec537d5b1dae 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -54,32 +54,55 @@ public boolean onTermChanged(ShardTerms terms) { if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true; String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(); - if (terms.haveHighestTermValue(coreNodeName)) return true; - if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) { - lastTermDoRecovery.set(terms.getTerm(coreNodeName)); - if (coreDescriptor.getCloudDescriptor().isLeader()) { - log.warn( - "Removing {} leader as leader, since its term is no longer the highest. This will initiate recovery", - coreNodeName); - coreContainer.getZkController().giveupLeadership(coreDescriptor); - coreContainer - .getZkController() - .getZkStateReader() - .waitForState( - coreDescriptor.getCollectionName(), - 20, - TimeUnit.SECONDS, - dc -> { - Replica leader = dc.getLeader(coreDescriptor.getCloudDescriptor().getShardId()); - return leader == null || !leader.getCoreName().equals(coreDescriptor.getName()); - }); - } else { + + // If we have the highest term, there is nothing to do + if (terms.haveHighestTermValue(coreNodeName)) { + return true; + } + + long lastRecoveryTerm; + long newTerm; + synchronized (lastTermDoRecovery) { + lastRecoveryTerm = lastTermDoRecovery.get(); + newTerm = terms.getTerm(coreNodeName); + if (lastRecoveryTerm < newTerm) { + lastTermDoRecovery.set(newTerm); + } + } + + if (coreDescriptor.getCloudDescriptor().isLeader()) { + log.warn( + "Removing {} leader as leader, since its term is no longer the highest. This will initiate recovery", + coreNodeName); + coreContainer.getZkController().giveupLeadership(coreDescriptor); + coreContainer + .getZkController() + .getZkStateReader() + .waitForState( + coreDescriptor.getCollectionName(), + 20, + TimeUnit.SECONDS, + dc -> { + Replica leader = dc.getLeader(coreDescriptor.getCloudDescriptor().getShardId()); + return leader == null || !leader.getCoreName().equals(coreDescriptor.getName()); + }); + return true; + } else if (lastRecoveryTerm < newTerm) { + CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor(); + Replica leaderReplica = solrCore.getCoreContainer().getZkController().getClusterState().getCollection(cloudDescriptor.getCollectionName()).getSlice(cloudDescriptor.getShardId()).getLeader(); + + // Only recover if the leader replica still has the highest term. + // If not, then the leader-election process will take care of recovery. + if (leaderReplica != null && terms.canBecomeLeader(leaderReplica.getName())) { log.info( "Start recovery on {} because core's term is less than leader's term", coreNodeName); solrCore .getUpdateHandler() .getSolrCoreState() .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); + } else { + log.info( + "Defer recovery on {} because leader-election will happen soon, old leader {}", coreNodeName, leaderReplica.getName()); } } } catch (Exception e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 16a29f89a586..40295b6b7998 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -507,12 +507,10 @@ private void rejoinLeaderElection(SolrCore core) throws InterruptedException, Ke return; } - log.info("There may be a better leader candidate than us - going back into recovery"); + log.info("There may be a better leader candidate than us - rejoin the election"); cancelElection(); - core.getUpdateHandler().getSolrCoreState().doRecovery(cc, core.getCoreDescriptor()); - leaderElector.joinElection(this, true); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java index 4c4f476e48ed..1fea2db07e2b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java @@ -68,8 +68,8 @@ public void testAsyncCallStatusResponse() throws Exception { final NamedList success = (NamedList) r.get("success"); assertNotNull("Expected 'success' response" + r, success); - final int actualSuccessElems = 2 * (numShards * numReplicas); - // every replica responds once on submit and once on complete + final int actualSuccessElems = numShards * numReplicas; + // every replica responds either once on submit (failure) or once on complete (if submit succeeds) assertEquals( "Expected " + actualSuccessElems + " elements in the success element" + success.jsonStr(), actualSuccessElems, diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index a34fb0ac9734..88827c572b3e 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -20,11 +20,19 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import java.lang.invoke.MethodHandles; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.Region; @@ -32,9 +40,6 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) -@LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -76,6 +81,22 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { + " REGION\n" + " ENDPOINT\n" + " \n" + + " \n" + + " s3BadNode\n" + + " \n" + + " \n" + + " BAD_BUCKET_ONE\n" + + " REGION\n" + + " ENDPOINT\n" + + " \n" + + " \n" + + " s3BadNodes\n" + + " \n" + + " \n" + + " BAD_BUCKET_ALL_BUT_ONE\n" + + " REGION\n" + + " ENDPOINT\n" + + " \n" + " \n" + " \n" + "\n"; @@ -111,6 +132,11 @@ public static void setupClass() throws Exception { .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml( SOLR_XML + // Only a single node will have a bad bucket name, all else should succeed. + // The bad node will be added later + .replace("BAD_BUCKET_ALL_BUT_ONE", "non-existent") + .replace("BAD_BUCKET_ONE", BUCKET_NAME) + .replace("BAD_BUCKET", BUCKET_NAME) .replace("BUCKET", BUCKET_NAME) .replace("REGION", Region.US_EAST_1.id()) .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())) @@ -126,4 +152,91 @@ public String getCollectionNamePrefix() { public String getBackupLocation() { return "/"; } + + @Test + public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { + JettySolrRunner badNodeJetty = cluster.startJettySolrRunner( + SOLR_XML + // The first solr node will not have a bad bucket + .replace("BAD_BUCKET_ALL_BUT_ONE", BUCKET_NAME) + .replace("BAD_BUCKET_ONE", "non-existent") + .replace("BUCKET", BUCKET_NAME) + .replace("REGION", Region.US_EAST_1.id()) + .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); + + try { + setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); + final String backupCollectionName = getCollectionName(); + final String backupName = BACKUPNAME_PREFIX + testSuffix; + + // Bootstrap the backup collection with seed docs + CollectionAdminRequest.createCollection(backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES + 1) + .process(cluster.getSolrClient()); + final int firstBatchNumDocs = indexDocs(backupCollectionName, true); + + // Backup and immediately add more docs to the collection + try (BackupRepository repository = + cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME)) { + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + final RequestStatusState result = + CollectionAdminRequest.backupCollection(backupCollectionName, backupName) + .setBackupConfigset(false) + .setLocation(backupLocation) + .setRepositoryName(BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 20); + assertEquals(RequestStatusState.COMPLETED, result); + } + int secondBatchNumDocs = indexDocs(backupCollectionName, true); + int maxDocs = secondBatchNumDocs + firstBatchNumDocs; + assertEquals(maxDocs, getNumDocsInCollection(backupCollectionName)); + + /* + Restore original docs and validate that doc count is correct + */ + // Test a single bad node + try (BackupRepository repository = + cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME); + SolrClient goodNodeClient = cluster.getJettySolrRunner(0).newClient()) { + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + final RequestStatusState result = + CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) + .setLocation(backupLocation) + .setRepositoryName(BACKUP_REPO_NAME + "BadNode") + .processAndWait(goodNodeClient, 30); + assertEquals(RequestStatusState.COMPLETED, result); + waitForState( + "The failed core-install should recover and become healthy", + backupCollectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * (NUM_NODES + 1))); + } + assertEquals(firstBatchNumDocs, getNumDocsInCollection(backupCollectionName)); + secondBatchNumDocs = indexDocs(backupCollectionName, true); + maxDocs = secondBatchNumDocs + firstBatchNumDocs; + assertEquals(maxDocs, getNumDocsInCollection(backupCollectionName)); + + // Test a single good node + try (BackupRepository repository = + cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME); + SolrClient goodNodeClient = badNodeJetty.newClient()) { + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + final RequestStatusState result = + CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) + .setLocation(backupLocation) + .setRepositoryName(BACKUP_REPO_NAME + "BadNodes") + .processAndWait(goodNodeClient, 30); + assertEquals(RequestStatusState.COMPLETED, result); + waitForState( + "The failed core-install should recover and become healthy", + backupCollectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * (NUM_NODES + 1))); + } + assertEquals(firstBatchNumDocs, getNumDocsInCollection(backupCollectionName)); + } finally { + badNodeJetty.stop(); + } + } } diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java index 194b2ffddc6c..9247330e3280 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java @@ -19,12 +19,21 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import org.apache.commons.io.file.PathUtils; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; +import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.handler.admin.api.InstallShardData; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Test; import software.amazon.awssdk.regions.Region; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; /** * Tests validating that the 'Install Shard API' works when used with {@link S3BackupRepository} @@ -49,6 +58,14 @@ public class S3InstallShardTest extends AbstractInstallShardTest { + " REGION\n" + " ENDPOINT\n" + " \n" + + " \n" + + " s3BadNode\n" + + " \n" + + " \n" + + " BAD_BUCKET\n" + + " REGION\n" + + " ENDPOINT\n" + + " \n" + " \n"; private static final String SOLR_XML = AbstractInstallShardTest.defaultSolrXmlTextWithBackupRepository(BACKUP_REPOSITORY_XML); @@ -69,6 +86,8 @@ public static void setupClass() throws Exception { .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml( SOLR_XML + // The first solr node will not have a bad bucket + .replace("BAD_BUCKET", BUCKET_NAME) .replace("BUCKET", BUCKET_NAME) .replace("REGION", Region.US_EAST_1.id()) .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())) @@ -76,4 +95,53 @@ public static void setupClass() throws Exception { bootstrapBackupRepositoryData("/"); } + + @Test + public void testInstallSucceedsOnASingleError() throws Exception { + JettySolrRunner jettySolrRunner = cluster.startJettySolrRunner( + SOLR_XML + // The first solr node will not have a bad bucket + .replace("BAD_BUCKET", "non-existent") + .replace("BUCKET", BUCKET_NAME) + .replace("REGION", Region.US_EAST_1.id()) + .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); + + try { + final String collectionName = createAndAwaitEmptyCollection(1, 2); + deleteAfterTest(collectionName); + enableReadOnly(collectionName); + + final String singleShardLocation = singleShard1Uri.toString(); + { // Test synchronous request error reporting + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME + "BadNode") + .process(cluster.getSolrClient()); + waitForState( + "The failed core-install should recover and become healthy", + collectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(1, 2)); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + + { // Test asynchronous request error reporting + final var requestStatusState = + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME + "BadNode") + .processAndWait(cluster.getSolrClient(), 15); + + assertEquals(RequestStatusState.COMPLETED, requestStatusState); + waitForState( + "The failed core-install should recover and become healthy", + collectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(1, 2)); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + } finally { + jettySolrRunner.stop(); + } + } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index ee8ced5a367d..61a43a416fe9 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -512,6 +512,15 @@ public JettySolrRunner startJettySolrRunner() throws Exception { return startJettySolrRunner(newNodeName(), jettyConfig, null); } + /** + * Start a new Solr instance, using the default config but with a custom Solr xml + * + * @return a JettySolrRunner + */ + public JettySolrRunner startJettySolrRunner(String solrXml) throws Exception { + return startJettySolrRunner(newNodeName(), jettyConfig, solrXml); + } + /** * Add a previously stopped node back to the cluster on a different port * diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 79bb122a8f8d..5fb666d36b3d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -71,7 +71,6 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -85,9 +84,6 @@ *

For a similar test harness for snapshot backup/restoration see {@link * AbstractCloudBackupRestoreTestCase} */ -@LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -131,9 +127,6 @@ public void setTestSuffix(String testSuffix) { public abstract String getBackupLocation(); @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSimple() throws Exception { setTestSuffix("testbackupincsimple"); final String backupCollectionName = getCollectionName(); @@ -200,9 +193,6 @@ public void testSimple() throws Exception { } @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testRestoreToOriginalCollection() throws Exception { setTestSuffix("testbackuprestoretooriginal"); final String backupCollectionName = getCollectionName(); @@ -244,7 +234,7 @@ public void testRestoreToOriginalCollection() throws Exception { @SuppressWarnings("unchecked") @Test - @Nightly + //@Nightly public void testBackupIncremental() throws Exception { setTestSuffix("testbackupinc"); CloudSolrClient solrClient = cluster.getSolrClient(); @@ -365,9 +355,6 @@ public void testBackupIncremental() throws Exception { } @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSkipConfigset() throws Exception { setTestSuffix("testskipconfigset"); final String backupCollectionName = getCollectionName(); @@ -617,7 +604,7 @@ private void randomlyPrecreateRestoreCollection( } } - private long getNumDocsInCollection(String collectionName) throws Exception { + protected long getNumDocsInCollection(String collectionName) throws Exception { return new QueryRequest(new SolrQuery("*:*")) .process(cluster.getSolrClient(), collectionName) .getResults() diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java index cf518e942511..ad61943cf0ed 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java @@ -93,20 +93,20 @@ public void deleteTestCollections() throws Exception { } } - private String deleteAfterTest(String collName) { + protected String deleteAfterTest(String collName) { collectionsToDelete.add(collName); return collName; } // Populated by 'bootstrapBackupRepositoryData' - private static int singleShardNumDocs = -1; - private static int replicasPerShard = -1; - private static int multiShardNumDocs = -1; - private static URI singleShard1Uri = null; - private static URI nonExistentLocationUri = null; - private static URI[] multiShardUris = null; + protected static int singleShardNumDocs = -1; + protected static int replicasPerShard = -1; + protected static int multiShardNumDocs = -1; + protected static URI singleShard1Uri = null; + protected static URI nonExistentLocationUri = null; + protected static URI[] multiShardUris = null; - private List collectionsToDelete; + protected List collectionsToDelete; public static void bootstrapBackupRepositoryData(String baseRepositoryLocation) throws Exception { final int numShards = /*random().nextInt(3) + 2*/ 4; @@ -216,31 +216,6 @@ public void testInstallReportsErrorsAppropriately() throws Exception { } } - @Test - public void testInstallSucceedsOnASingleError() throws Exception { - final String collectionName = createAndAwaitEmptyCollection(1, replicasPerShard); - deleteAfterTest(collectionName); - enableReadOnly(collectionName); - final String singleShardLocation = singleShard1Uri.toString(); - - { // Test synchronous request error reporting - CollectionAdminRequest.installDataToShard( - collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) - .process(cluster.getSolrClient()); - assertCollectionHasNumDocs(collectionName, singleShardNumDocs); - } - - { // Test asynchronous request error reporting - final var requestStatusState = - CollectionAdminRequest.installDataToShard( - collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME) - .processAndWait(cluster.getSolrClient(), 15); - - assertEquals(RequestStatusState.COMPLETED, requestStatusState); - assertCollectionHasNumDocs(collectionName, singleShardNumDocs); - } - } - @Test public void testSerialInstallToMultiShardCollection() throws Exception { final String collectionName = @@ -303,7 +278,7 @@ public static String defaultSolrXmlTextWithBackupRepository(String backupReposit + "\n"; } - private static void assertCollectionHasNumDocs(String collection, int expectedNumDocs) + protected static void assertCollectionHasNumDocs(String collection, int expectedNumDocs) throws Exception { final SolrClient solrClient = cluster.getSolrClient(); assertEquals( @@ -395,7 +370,7 @@ private static void indexDocs(String collectionName, int numDocs, boolean useUUI log.info("Indexed {} docs to collection: {}", numDocs, collectionName); } - private static String createAndAwaitEmptyCollection(int numShards, int replicasPerShard) + protected static String createAndAwaitEmptyCollection(int numShards, int replicasPerShard) throws Exception { final SolrClient solrClient = cluster.getSolrClient(); @@ -408,7 +383,7 @@ private static String createAndAwaitEmptyCollection(int numShards, int replicasP return collectionName; } - private static void enableReadOnly(String collectionName) throws Exception { + protected static void enableReadOnly(String collectionName) throws Exception { CollectionAdminRequest.modifyCollection(collectionName, Map.of("readOnly", true)) .process(cluster.getSolrClient()); } From 5e4c1be4e56c3db9ebf268f1a6a4d8216dec8f57 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 7 Jan 2026 16:04:00 -0800 Subject: [PATCH 16/67] Remove uneeded testing code --- .../java/org/apache/solr/handler/admin/api/RestoreCore.java | 4 ---- .../src/java/org/apache/solr/cloud/SolrCloudTestCase.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java index 96af5387cb36..dcf1cfe85c1b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java @@ -63,10 +63,6 @@ boolean isExpensive() { public SolrJerseyResponse restoreCore(String coreName, RestoreCoreRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - // TODO: This is for testing, we need to figure out how to get the leader to fail - if (coreContainer.getCoreDescriptor(coreName).getCloudDescriptor().isLeader()) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Is leader"); - } ensureRequiredParameterProvided("coreName", coreName); if (requestBody == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing required request body"); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 9401f30e18a1..7f589e48b232 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -124,7 +124,7 @@ protected static MiniSolrCloudCluster.Builder configureCluster(int nodeCount) { return new MiniSolrCloudCluster.Builder(nodeCount, createTempDir()) .withOverseer( EnvUtils.getPropertyAsBool( - "solr.cloud.overseer.enabled", false)); // LuceneTestCase.random().nextBoolean())); + "solr.cloud.overseer.enabled", LuceneTestCase.random().nextBoolean())); } public static void configurePrsDefault() { From 25e9ecd4ca0820324bd2c95a70bf3965733466f1 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 7 Jan 2026 16:33:02 -0800 Subject: [PATCH 17/67] Tidy --- .../solr/cloud/RecoveringCoreTermWatcher.java | 13 +++++-- .../collections/CollectionHandlingUtils.java | 3 +- .../api/collections/InstallShardDataCmd.java | 5 +-- .../AsyncCallRequestStatusResponseTest.java | 3 +- .../solr/s3/S3IncrementalBackupTest.java | 35 +++++++++++-------- .../apache/solr/s3/S3InstallShardTest.java | 20 +++++------ .../AbstractIncrementalBackupTest.java | 2 +- 7 files changed, 49 insertions(+), 32 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index ec537d5b1dae..7526520b29b1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -89,7 +89,14 @@ public boolean onTermChanged(ShardTerms terms) { return true; } else if (lastRecoveryTerm < newTerm) { CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor(); - Replica leaderReplica = solrCore.getCoreContainer().getZkController().getClusterState().getCollection(cloudDescriptor.getCollectionName()).getSlice(cloudDescriptor.getShardId()).getLeader(); + Replica leaderReplica = + solrCore + .getCoreContainer() + .getZkController() + .getClusterState() + .getCollection(cloudDescriptor.getCollectionName()) + .getSlice(cloudDescriptor.getShardId()) + .getLeader(); // Only recover if the leader replica still has the highest term. // If not, then the leader-election process will take care of recovery. @@ -102,7 +109,9 @@ public boolean onTermChanged(ShardTerms terms) { .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); } else { log.info( - "Defer recovery on {} because leader-election will happen soon, old leader {}", coreNodeName, leaderReplica.getName()); + "Defer recovery on {} because leader-election will happen soon, old leader {}", + coreNodeName, + leaderReplica.getName()); } } } catch (Exception e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 3184e2222ffe..bb36e61d0958 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -784,7 +784,8 @@ void processResponses( private void waitForAsyncCallsToComplete(NamedList results) { for (AsyncCmdInfo asyncCmdInfo : shardAsyncCmds) { - Object failure = results._get("failure/" + requestKey(asyncCmdInfo.nodeName, asyncCmdInfo.coreName)); + Object failure = + results._get("failure/" + requestKey(asyncCmdInfo.nodeName, asyncCmdInfo.coreName)); // Do not wait for Async calls that have already failed if (failure != null) { return; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 23f3977336b7..91b214c6beb6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -30,7 +30,6 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.cloud.ZkShardTerms; @@ -138,7 +137,9 @@ public void call( errorMessage + ". No leader-eligible replicas are live."); } else { throw new SolrErrorWrappingException( - SolrException.ErrorCode.SERVER_ERROR, errorMessage, Collections.singletonList(failures.asMap(1))); + SolrException.ErrorCode.SERVER_ERROR, + errorMessage, + Collections.singletonList(failures.asMap(1))); } } else if (successfulReplicas.size() < leaderEligibleReplicas.size()) { // Some, but not all, leader-eligible replicas succeeded. diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java index 1fea2db07e2b..ca683c4d764f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java @@ -69,7 +69,8 @@ public void testAsyncCallStatusResponse() throws Exception { assertNotNull("Expected 'success' response" + r, success); final int actualSuccessElems = numShards * numReplicas; - // every replica responds either once on submit (failure) or once on complete (if submit succeeds) + // every replica responds either once on submit (failure) or once on complete (if submit + // succeeds) assertEquals( "Expected " + actualSuccessElems + " elements in the success element" + success.jsonStr(), actualSuccessElems, diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index 88827c572b3e..a50e1920fa0d 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -29,7 +29,6 @@ import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -155,14 +154,15 @@ public String getBackupLocation() { @Test public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { - JettySolrRunner badNodeJetty = cluster.startJettySolrRunner( - SOLR_XML - // The first solr node will not have a bad bucket - .replace("BAD_BUCKET_ALL_BUT_ONE", BUCKET_NAME) - .replace("BAD_BUCKET_ONE", "non-existent") - .replace("BUCKET", BUCKET_NAME) - .replace("REGION", Region.US_EAST_1.id()) - .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); + JettySolrRunner badNodeJetty = + cluster.startJettySolrRunner( + SOLR_XML + // The first solr node will not have a bad bucket + .replace("BAD_BUCKET_ALL_BUT_ONE", BUCKET_NAME) + .replace("BAD_BUCKET_ONE", "non-existent") + .replace("BUCKET", BUCKET_NAME) + .replace("REGION", Region.US_EAST_1.id()) + .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); try { setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); @@ -170,7 +170,8 @@ public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { final String backupName = BACKUPNAME_PREFIX + testSuffix; // Bootstrap the backup collection with seed docs - CollectionAdminRequest.createCollection(backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES + 1) + CollectionAdminRequest.createCollection( + backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES + 1) .process(cluster.getSolrClient()); final int firstBatchNumDocs = indexDocs(backupCollectionName, true); @@ -191,11 +192,14 @@ public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { assertEquals(maxDocs, getNumDocsInCollection(backupCollectionName)); /* - Restore original docs and validate that doc count is correct - */ + Restore original docs and validate that doc count is correct + */ // Test a single bad node try (BackupRepository repository = - cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME); + cluster + .getJettySolrRunner(0) + .getCoreContainer() + .newBackupRepository(BACKUP_REPO_NAME); SolrClient goodNodeClient = cluster.getJettySolrRunner(0).newClient()) { final String backupLocation = repository.getBackupLocation(getBackupLocation()); final RequestStatusState result = @@ -218,7 +222,10 @@ public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { // Test a single good node try (BackupRepository repository = - cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME); + cluster + .getJettySolrRunner(0) + .getCoreContainer() + .newBackupRepository(BACKUP_REPO_NAME); SolrClient goodNodeClient = badNodeJetty.newClient()) { final String backupLocation = repository.getBackupLocation(getBackupLocation()); final RequestStatusState result = diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java index 9247330e3280..78000ac7fad7 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java @@ -19,7 +19,7 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import org.apache.commons.io.file.PathUtils; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.RequestStatusState; @@ -31,9 +31,6 @@ import org.junit.ClassRule; import org.junit.Test; import software.amazon.awssdk.regions.Region; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; /** * Tests validating that the 'Install Shard API' works when used with {@link S3BackupRepository} @@ -98,13 +95,14 @@ public static void setupClass() throws Exception { @Test public void testInstallSucceedsOnASingleError() throws Exception { - JettySolrRunner jettySolrRunner = cluster.startJettySolrRunner( - SOLR_XML - // The first solr node will not have a bad bucket - .replace("BAD_BUCKET", "non-existent") - .replace("BUCKET", BUCKET_NAME) - .replace("REGION", Region.US_EAST_1.id()) - .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); + JettySolrRunner jettySolrRunner = + cluster.startJettySolrRunner( + SOLR_XML + // The first solr node will not have a bad bucket + .replace("BAD_BUCKET", "non-existent") + .replace("BUCKET", BUCKET_NAME) + .replace("REGION", Region.US_EAST_1.id()) + .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); try { final String collectionName = createAndAwaitEmptyCollection(1, 2); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 5fb666d36b3d..cdc8359be1a6 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -234,7 +234,7 @@ public void testRestoreToOriginalCollection() throws Exception { @SuppressWarnings("unchecked") @Test - //@Nightly + // @Nightly public void testBackupIncremental() throws Exception { setTestSuffix("testbackupinc"); CloudSolrClient solrClient = cluster.getSolrClient(); From b0452f4b3cc1f64d94d918eb4e13c093734ec709 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 8 Jan 2026 11:02:15 -0800 Subject: [PATCH 18/67] Support distributed locks better --- .../solr/cloud/DistributedMultiLock.java | 7 +++- .../apache/solr/cloud/ZkDistributedLock.java | 33 +++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 1baea10051f5..ef74b9862c86 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -47,7 +47,12 @@ public void waitUntilAcquired() { for (DistributedLock lock : locks) { log.debug("DistributedMultiLock.waitUntilAcquired. About to wait on lock {}", lock); lock.waitUntilAcquired(); - log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + if (lock.isMirroringLock()) { + log.debug( + "DistributedMultiLock.waitUntilAcquired. Mirroring already-acquired lock {}", lock); + } else { + log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock); + } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index a8260e72bc20..8517661dba79 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -41,10 +41,14 @@ abstract class ZkDistributedLock implements DistributedLock { static final char LOCK_PREFIX_SUFFIX = '_'; /** Prefix of EPHEMERAL read lock node names */ - static final String READ_LOCK_PREFIX = "R" + LOCK_PREFIX_SUFFIX; + static final char READ_LOCK_PREFIX_CHAR = 'R'; + + static final String READ_LOCK_PREFIX = "" + READ_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Prefix of EPHEMERAL write lock node names */ - static final String WRITE_LOCK_PREFIX = "W" + LOCK_PREFIX_SUFFIX; + static final char WRITE_LOCK_PREFIX_CHAR = 'W'; + + static final String WRITE_LOCK_PREFIX = "" + WRITE_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; /** Read lock. */ static class Read extends ZkDistributedLock { @@ -59,6 +63,11 @@ boolean isBlockedByNodeType(String otherLockName) { // Lower numbered read locks are ok, they can coexist. return otherLockName.startsWith(WRITE_LOCK_PREFIX); } + + @Override + boolean canMirrorLock(String lockId) { + return true; + } } /** Write lock. */ @@ -73,6 +82,17 @@ boolean isBlockedByNodeType(String otherLockName) { // A write lock is blocked by another read or write lock with a lower sequence number return true; } + + @Override + boolean canMirrorLock(String lockId) { + // Only another Write lock can be mirrored + int lockTypeSuffixIndex = lockId.indexOf(LOCK_PREFIX_SUFFIX) - 1; + if (lockTypeSuffixIndex < 0) { + return false; + } else { + return lockId.charAt(lockTypeSuffixIndex) == WRITE_LOCK_PREFIX_CHAR; + } + } } private final SolrZkClient zkClient; @@ -90,7 +110,7 @@ protected ZkDistributedLock( // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - if (mirroredLockId == null || mirroredLockId.startsWith(lockDir)) { + if (mirroredLockId == null || !mirroredLockId.startsWith(lockDir)) { lockNode = zkClient.create( lockDir @@ -101,6 +121,11 @@ protected ZkDistributedLock( mirrored = false; } else { + if (!canMirrorLock(mirroredLockId)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Cannot mirror lock " + mirroredLockId + " with given lockPrefix: " + lockNodePrefix); + } lockNode = mirroredLockId; mirrored = true; } @@ -264,6 +289,8 @@ public boolean isMirroringLock() { return mirrored; } + abstract boolean canMirrorLock(String lockId); + @Override public String toString() { return lockNode; From 72401ca19cbffd0b85bfdebff7301b0292f0b523 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 8 Jan 2026 13:04:16 -0800 Subject: [PATCH 19/67] Start switching over to AdminCmdContext --- .../CreateCollectionSnapshotResponse.java | 2 +- .../DeleteCollectionSnapshotResponse.java | 2 +- .../api/collections/AdminCmdContext.java | 54 +++++++++++ .../collections/CollectionApiLockFactory.java | 12 ++- ...butedCollectionConfigSetCommandRunner.java | 95 +++++++++---------- .../collections/MaintainRoutedAliasCmd.java | 12 +-- .../handler/admin/CollectionsHandler.java | 38 ++++---- .../solr/handler/admin/RebalanceLeaders.java | 9 +- .../handler/admin/api/AddReplicaProperty.java | 19 ++-- .../solr/handler/admin/api/AdminAPIBase.java | 41 ++++++-- .../solr/handler/admin/api/AliasProperty.java | 57 +++++------ .../handler/admin/api/BalanceReplicas.java | 22 ++--- .../handler/admin/api/BalanceShardUnique.java | 3 - .../solr/handler/admin/api/CreateAlias.java | 25 ++--- .../handler/admin/api/CreateCollection.java | 27 +----- .../admin/api/CreateCollectionBackup.java | 20 ++-- .../admin/api/CreateCollectionSnapshot.java | 25 ++--- .../solr/handler/admin/api/CreateReplica.java | 2 - .../solr/handler/admin/api/DeleteAlias.java | 34 ++----- .../handler/admin/api/DeleteCollection.java | 35 ++----- .../admin/api/DeleteCollectionBackup.java | 12 +-- .../admin/api/DeleteCollectionSnapshot.java | 25 ++--- .../solr/handler/admin/api/DeleteNode.java | 34 ++----- .../admin/api/DeleteReplicaProperty.java | 19 +--- .../handler/admin/api/InstallShardData.java | 24 ++--- .../handler/admin/api/MigrateReplicas.java | 24 ++--- .../handler/admin/api/RenameCollection.java | 4 +- .../solr/handler/admin/api/ReplaceNode.java | 23 ++--- .../handler/admin/api/RestoreCollection.java | 22 +---- 29 files changed, 308 insertions(+), 413 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java diff --git a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java index 16944706498d..0392e0ed1ce5 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionSnapshotResponse.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; -public class CreateCollectionSnapshotResponse extends AsyncJerseyResponse { +public class CreateCollectionSnapshotResponse extends SubResponseAccumulatingJerseyResponse { @Schema(description = "The name of the collection.") public String collection; diff --git a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java index 905b0937f1de..d822e2ae0897 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java @@ -23,7 +23,7 @@ import io.swagger.v3.oas.annotations.media.Schema; /** The Response for {@link org.apache.solr.client.api.endpoint.CollectionSnapshotApis.Delete} */ -public class DeleteCollectionSnapshotResponse extends AsyncJerseyResponse { +public class DeleteCollectionSnapshotResponse extends SubResponseAccumulatingJerseyResponse { @Schema(description = "The name of the collection.") @JsonProperty(COLLECTION) public String collection; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java new file mode 100644 index 000000000000..33f91c46a17d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -0,0 +1,54 @@ +package org.apache.solr.cloud.api.collections; + +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.params.CollectionParams; +import java.util.List; + +public class AdminCmdContext { + final private CollectionParams.CollectionAction action; + final private String asyncId; + private String lockId; + private List callingLockIds; + private ClusterState clusterState; + + public AdminCmdContext(CollectionParams.CollectionAction action) { + this(action, null); + } + + public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) { + this.action = action; + this.asyncId = asyncId; + } + + public CollectionParams.CollectionAction getAction() { + return action; + } + + public String getAsyncId() { + return asyncId; + } + + public void setLockId(String lockId) { + this.lockId = lockId; + } + + public String getLockId() { + return lockId; + } + + public void setCallingLockIds(List callingLockIds) { + this.callingLockIds = callingLockIds; + } + + public List getCallingLockIds() { + return callingLockIds; + } + + public ClusterState getClusterState() { + return clusterState; + } + + public void setClusterState(ClusterState clusterState) { + this.clusterState = clusterState; + } +} diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index d51c8ba5c174..b9566bc499be 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -59,11 +59,11 @@ public class CollectionApiLockFactory { * prevent other threads from locking. */ DistributedMultiLock createCollectionApiLock( - CollectionParams.LockLevel lockLevel, + AdminCmdContext adminCmdContext, String collName, String shardId, - String replicaName, - String callingLockId) { + String replicaName) { + CollectionParams.LockLevel lockLevel = adminCmdContext.getAction().lockLevel; if (lockLevel == CollectionParams.LockLevel.NONE) { return new DistributedMultiLock(List.of()); } @@ -111,8 +111,10 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } - List callingLockIdList = - callingLockId == null ? Collections.emptyList() : List.of(callingLockId.split(",")); + List callingLockIdList = adminCmdContext.getCallingLockIds(); + if (callingLockIdList == null) { + callingLockIdList = Collections.emptyList(); + } // The first requested lock is a write one (on the target object for the action, depending on // lock level), then requesting read locks on "higher" levels (collection > shard > replica here diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 7b86c66ca378..5487d63ced46 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -252,10 +252,9 @@ public void runConfigSetCommand( * */ public OverseerSolrResponse runCollectionCommand( + AdminCmdContext adminCmdContext, ZkNodeProps message, - CollectionParams.CollectionAction action, - long timeoutMs, - String callingLockId) { + long timeoutMs) { // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it // through this method earlier). See stopAndWaitForPendingTasksToComplete() below if (shuttingDown) { @@ -264,36 +263,34 @@ public OverseerSolrResponse runCollectionCommand( "Solr is shutting down, no more Collection API tasks may be executed"); } - final String asyncId = message.getStr(ASYNC); - if (log.isInfoEnabled()) { log.info( - "Running Collection API locally for " + action.name() + " asyncId=" + asyncId); // nowarn + "Running Collection API locally for {} asyncId={}", adminCmdContext.getAction().name(), adminCmdContext.getAsyncId()); } // Following the call below returning true, we must eventually cancel or complete the task. // Happens either in the CollectionCommandRunner below or in the catch when the runner would not // execute. - if (!asyncTaskTracker.createNewAsyncJobTracker(asyncId)) { + if (!asyncTaskTracker.createNewAsyncJobTracker(adminCmdContext.getAsyncId())) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Task with the same requestid already exists. (" + asyncId + ")"); + "Task with the same requestid already exists. (" + adminCmdContext.getAsyncId() + ")"); } CollectionCommandRunner commandRunner = - new CollectionCommandRunner(message, action, asyncId, callingLockId); + new CollectionCommandRunner(adminCmdContext, message); final Future taskFuture; try { taskFuture = commandsExecutor.submit(commandRunner); } catch (RejectedExecutionException ree) { // The command will not run, need to cancel the async ID so it can be reused on a subsequent // attempt by the client - asyncTaskTracker.cancelAsyncId(asyncId); + asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); throw new SolrException( SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree); } - if (asyncId == null) { + if (adminCmdContext.getAsyncId() == null) { // Non async calls wait for a while in case the command completes. If they time out, there's // no way to track the job progress (improvement suggestion: decorrelate having a task ID from // the fact of waiting for the job to complete) @@ -301,18 +298,18 @@ public OverseerSolrResponse runCollectionCommand( return taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS); } catch (TimeoutException te) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, action + " timed out after " + timeoutMs + "ms"); + SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " timed out after " + timeoutMs + "ms"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " interrupted", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " interrupted", e); } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " failed", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " failed", e); } } else { // Async calls do not wait for the command to finish but get instead back the async id (that // they just sent...) NamedList resp = new NamedList<>(); - resp.add(CoreAdminParams.REQUESTID, asyncId); + resp.add(CoreAdminParams.REQUESTID, adminCmdContext.getAsyncId()); return new OverseerSolrResponse(resp); } } @@ -361,20 +358,12 @@ public static String getCollectionName(ZkNodeProps message) { * similar to the one provided by Overseer based Collection API execution. */ private class CollectionCommandRunner implements Callable { + private final AdminCmdContext adminCmdContext; private final ZkNodeProps message; - private final CollectionParams.CollectionAction action; - private final String asyncId; - private final String callingLockId; - private CollectionCommandRunner( - ZkNodeProps message, - CollectionParams.CollectionAction action, - String asyncId, - String callingLockId) { + private CollectionCommandRunner(AdminCmdContext adminCmdContext, ZkNodeProps message) { + this.adminCmdContext = adminCmdContext; this.message = message; - this.action = action; - this.asyncId = asyncId; - this.callingLockId = callingLockId; } /** @@ -407,39 +396,43 @@ public OverseerSolrResponse call() { new CollectionApiLockFactory( new ZkDistributedCollectionLockFactory( ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)) - .createCollectionApiLock( - action.lockLevel, collName, shardId, replicaName, callingLockId); + .createCollectionApiLock(adminCmdContext, collName, shardId, replicaName); try { - log.debug( - "CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}", - action, - action.lockLevel, - collName, - shardId, - replicaName); + if (log.isDebugEnabled()) { + log.debug( + "CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}", + adminCmdContext.getAction().name(), + adminCmdContext.getAction().lockLevel, + collName, + shardId, + replicaName); + } // Block this thread until all required locks are acquired. lock.waitUntilAcquired(); + adminCmdContext.setLockId(lock.getLockId()); // Got the lock so moving from submitted to running if we run for an async task (if // asyncId is null the asyncTaskTracker calls do nothing). - asyncTaskTracker.setTaskRunning(asyncId); + asyncTaskTracker.setTaskRunning(adminCmdContext.getAsyncId()); - log.debug( - "DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", - action, - message); + if (log.isDebugEnabled()) { + log.debug( + "DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", + adminCmdContext.getAction(), + message); + } - CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); + CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(adminCmdContext.getAction()); if (command != null) { - // TODO: FIX + adminCmdContext.setClusterState(ccc.getSolrCloudManager().getClusterState()); command.call( - ccc.getSolrCloudManager().getClusterState(), message, lock.getLockId(), results); + adminCmdContext, message, results); } else { - asyncTaskTracker.cancelAsyncId(asyncId); + asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); // Seeing this is a bug, not bad user data - String message = "Bug: Unknown operation " + action; + String message = "Bug: Unknown operation " + adminCmdContext.getAction(); log.error(message); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message); } @@ -451,21 +444,23 @@ public OverseerSolrResponse call() { // hierarchy do no harm, and there shouldn't be too many of those. lock.release(); } catch (SolrException se) { - log.error( - "Error when releasing collection locks for operation " + action, se); // nowarn + if (log.isErrorEnabled()) { + log.error( + "Error when releasing collection locks for operation {}", adminCmdContext.getAction(), se); + } } } } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } - logFailedOperation(action, e, collName); - addExceptionToNamedList(action, e, results); + logFailedOperation(adminCmdContext.getAction(), e, collName); + addExceptionToNamedList(adminCmdContext.getAction(), e, results); } OverseerSolrResponse res = new OverseerSolrResponse(results); // Following call marks success or failure depending on the contents of res - asyncTaskTracker.setTaskCompleted(asyncId, res); + asyncTaskTracker.setTaskCompleted(adminCmdContext.getAsyncId(), res); return res; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index a89601e402b9..df37051e757b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -57,14 +57,12 @@ public class MaintainRoutedAliasCmd extends AliasCmd { */ static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) throws Exception { - final CollectionParams.CollectionAction maintainroutedalias = - CollectionParams.CollectionAction.MAINTAINROUTEDALIAS; - Map msg = new HashMap<>(); - msg.put(Overseer.QUEUE_OPERATION, maintainroutedalias.toLower()); - msg.put(CollectionParams.NAME, aliasName); - msg.put(MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol); final SolrResponse rsp = - collHandler.submitCollectionApiCommand(new ZkNodeProps(msg), maintainroutedalias); + collHandler.submitCollectionApiCommand( + new AdminCmdContext(CollectionParams.CollectionAction.MAINTAINROUTEDALIAS), + new ZkNodeProps(Map.of( + CollectionParams.NAME, aliasName, + MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol))); if (rsp.getException() != null) { throw rsp.getException(); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index c0723cda4755..2a45ac4ffc7b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -140,6 +140,7 @@ import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkController.NotInClusterStateException; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.cloud.api.collections.ReindexCollectionCmd; @@ -317,17 +318,12 @@ void invokeAction( return; } - String asyncId = req.getParams().get(ASYNC); - if (asyncId != null) { - props.put(ASYNC, asyncId); - } - - props.put(QUEUE_OPERATION, operation.action.toLower()); + AdminCmdContext adminCmdContext = new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; - overseerResponse = submitCollectionApiCommand(zkProps, operation.action, operation.timeOut); + overseerResponse = submitCollectionApiCommand(adminCmdContext, zkProps, operation.timeOut); rsp.getValues().addAll(overseerResponse.getResponse()); Exception exp = overseerResponse.getException(); @@ -340,13 +336,13 @@ void invokeAction( public static long DEFAULT_COLLECTION_OP_TIMEOUT = 180 * 1000; - public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action) + public SolrResponse submitCollectionApiCommand(AdminCmdContext adminCmdContext, ZkNodeProps m) throws KeeperException, InterruptedException { - return submitCollectionApiCommand(m, action, DEFAULT_COLLECTION_OP_TIMEOUT); + return submitCollectionApiCommand(adminCmdContext, m, DEFAULT_COLLECTION_OP_TIMEOUT); } public static SolrResponse submitCollectionApiCommand( - ZkController zkController, ZkNodeProps m, CollectionAction action, long timeout) + ZkController zkController, AdminCmdContext adminCmdContext, ZkNodeProps m, long timeout) throws KeeperException, InterruptedException { // Collection API messages are either sent to Overseer and processed there, or processed // locally. Distributing Collection API implies we're also distributing Cluster State Updates. @@ -360,15 +356,17 @@ public static SolrResponse submitCollectionApiCommand( // updates (but the other way around is ok). See constructor of CloudConfig. Optional distribCommandRunner = zkController.getDistributedCommandRunner(); + String operation = adminCmdContext.getAction().lowerName; + if (adminCmdContext.getAsyncId() == null) { + m = m.plus(Map.of(QUEUE_OPERATION, operation)); + } else { + m = m.plus(Map.of(QUEUE_OPERATION, operation, ASYNC, adminCmdContext.getAsyncId())); + } if (distribCommandRunner.isPresent()) { - return distribCommandRunner.get().runCollectionCommand(m, action, timeout, m.getStr(CALLING_LOCK_ID)); + return distribCommandRunner.get().runCollectionCommand(adminCmdContext, m, timeout); } else { // Sending the Collection API message to Overseer via a Zookeeper queue - String operation = m.getStr(QUEUE_OPERATION); - if (operation == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION); - } - if (m.get(ASYNC) != null) { - String asyncId = m.getStr(ASYNC); + if (adminCmdContext.getAsyncId() != null) { + String asyncId = adminCmdContext.getAsyncId(); NamedList r = new NamedList<>(); if (zkController.claimAsyncId(asyncId)) { @@ -391,7 +389,7 @@ public static SolrResponse submitCollectionApiCommand( throw new SolrException( BAD_REQUEST, "Task with the same requestid already exists. (" + asyncId + ")"); } - r.add(CoreAdminParams.REQUESTID, m.get(ASYNC)); + r.add(CoreAdminParams.REQUESTID, asyncId); return new OverseerSolrResponse(r); } @@ -426,9 +424,9 @@ public static SolrResponse submitCollectionApiCommand( } public SolrResponse submitCollectionApiCommand( - ZkNodeProps m, CollectionAction action, long timeout) + AdminCmdContext adminCmdContext, ZkNodeProps m, long timeout) throws KeeperException, InterruptedException { - return submitCollectionApiCommand(coreContainer.getZkController(), m, action, timeout); + return submitCollectionApiCommand(coreContainer.getZkController(), adminCmdContext, m, timeout); } private boolean overseerCollectionQueueContains(String asyncId) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 2bebe2ba3ac8..f7033d62948d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -38,6 +38,7 @@ import java.util.concurrent.TimeUnit; import org.apache.solr.cloud.LeaderElector; import org.apache.solr.cloud.OverseerTaskProcessor; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -444,10 +445,8 @@ private void rejoinElectionQueue( Slice slice, String electionNode, String core, boolean rejoinAtHead) throws KeeperException, InterruptedException { Replica replica = slice.getReplica(LeaderElector.getNodeName(electionNode)); - final CollectionParams.CollectionAction rebalanceleaders = REBALANCELEADERS; Map propMap = new HashMap<>(); propMap.put(COLLECTION_PROP, collectionName); - propMap.put(QUEUE_OPERATION, rebalanceleaders.toLower()); propMap.put(CORE_NAME_PROP, core); propMap.put(CORE_NODE_NAME_PROP, replica.getName()); propMap.put(ZkStateReader.NODE_NAME_PROP, replica.getNodeName()); @@ -459,12 +458,10 @@ private void rejoinElectionQueue( .getBaseUrlForNodeName(replica.getNodeName())); propMap.put( REJOIN_AT_HEAD_PROP, Boolean.toString(rejoinAtHead)); // Get ourselves to be first in line. - String asyncId = rebalanceleaders.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); - propMap.put(ASYNC, asyncId); + String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); asyncRequests.add(asyncId); - collectionsHandler.submitCollectionApiCommand( - new ZkNodeProps(propMap), rebalanceleaders); // ignore response; we construct our own + collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId), new ZkNodeProps(propMap)); // ignore response; we construct our own } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java index 81b2dbce137a..1b15d2dd8184 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java @@ -34,7 +34,9 @@ import org.apache.solr.client.api.endpoint.AddReplicaPropertyApi; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; @@ -72,21 +74,17 @@ public SolrJerseyResponse addReplicaProperty( if (requestBody == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing required request body"); } - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, replicaName, propertyName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.ADDREPLICAPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + remoteMessage, + null); disableResponseCaching(); return response; @@ -104,7 +102,6 @@ public ZkNodeProps createRemoteMessage( remoteMessage.put(SHARD_ID_PROP, shardName); remoteMessage.put(REPLICA_PROP, replicaName); remoteMessage.put(PROPERTY_VALUE_PROP, requestBody.value); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICAPROP.toLower()); if (requestBody.shardUnique != null) { remoteMessage.put(SHARD_UNIQUE, requestBody.shardUnique); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index c4e001a3ad36..33a5a0904876 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -21,8 +21,11 @@ import java.util.Map; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.AsyncJerseyResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; @@ -130,12 +133,11 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( ZkNodeProps remoteMessage, String asyncId) throws Exception { - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), remoteMessage, action, DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + var remoteResponse = submitRemoteMessageAndHandleResponse( + response, + new AdminCmdContext(action, asyncId), + remoteMessage + ); if (asyncId != null) { response.requestId = asyncId; @@ -144,7 +146,34 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( // Values fetched from remoteResponse may be null response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + response.warning = (String) remoteResponse.getResponse().get("warning"); + + return remoteResponse; + } + + protected SolrResponse submitRemoteMessageAndHandleResponse( + SolrJerseyResponse response, + CollectionParams.CollectionAction action, + ZkNodeProps remoteMessage) + throws Exception { + return submitRemoteMessageAndHandleResponse( + response, + new AdminCmdContext(action, null), + remoteMessage + ); + } + protected SolrResponse submitRemoteMessageAndHandleResponse( + SolrJerseyResponse response, + AdminCmdContext adminCmdContext, + ZkNodeProps remoteMessage) + throws Exception { + final SolrResponse remoteResponse = + CollectionsHandler.submitCollectionApiCommand( + coreContainer.getZkController(), adminCmdContext, remoteMessage, DEFAULT_COLLECTION_OP_TIMEOUT); + if (remoteResponse.getException() != null) { + throw remoteResponse.getException(); + } return remoteResponse; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java index 56af9177a964..36bc6f27875c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java @@ -24,12 +24,14 @@ import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; import jakarta.inject.Inject; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.AliasPropertyApis; import org.apache.solr.client.api.model.GetAliasPropertyResponse; import org.apache.solr.client.api.model.GetAllAliasPropertiesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; import org.apache.solr.client.solrj.SolrResponse; @@ -112,8 +114,8 @@ public SolrJerseyResponse updateAliasProperties( recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperties(aliasName, requestBody.properties, requestBody.async); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperties(response, aliasName, requestBody.properties, requestBody.async); return response; } @@ -128,8 +130,8 @@ public SolrJerseyResponse createOrUpdateAliasProperty( recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperty(aliasName, propName, requestBody.value); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperty(response, aliasName, propName, requestBody.value); return response; } @@ -139,52 +141,41 @@ public SolrJerseyResponse deleteAliasProperty(String aliasName, String propName) throws Exception { recordCollectionForLogAndTracing(null, solrQueryRequest); - SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - modifyAliasProperty(aliasName, propName, null); + var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + modifyAliasProperty(response, aliasName, propName, null); return response; } - private void modifyAliasProperty(String alias, String proertyName, Object value) + private void modifyAliasProperty(SubResponseAccumulatingJerseyResponse response, String alias, String proertyName, Object value) throws Exception { Map props = new HashMap<>(); // value can be null props.put(proertyName, value); - modifyAliasProperties(alias, props, null); + modifyAliasProperties(response, alias, props, null); } + private static final String PROPERTIES = "property"; + /** * @param alias alias */ - private void modifyAliasProperties(String alias, Map properties, String async) + private void modifyAliasProperties(SubResponseAccumulatingJerseyResponse response, String alias, Map properties, String async) throws Exception { // Note: success/no-op in the event of no properties supplied is intentional. Keeps code // simple and one less case for api-callers to check for. - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - final ZkNodeProps remoteMessage = createRemoteMessage(alias, properties, async); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.ALIASPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); + if (properties == null) { + properties = Collections.emptyMap(); } + fetchAndValidateZooKeeperAwareCoreContainer(); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.ALIASPROP, + new ZkNodeProps(Map.of( + NAME, alias, + PROPERTIES, properties + )), + async); disableResponseCaching(); } - - private static final String PROPERTIES = "property"; - - public ZkNodeProps createRemoteMessage( - String alias, Map properties, String async) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ALIASPROP.toLower()); - remoteMessage.put(NAME, alias); - remoteMessage.put(PROPERTIES, properties); - if (async != null) { - remoteMessage.put(ASYNC, async); - } - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java index b582e39464bd..76bdec315a56 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java @@ -29,6 +29,7 @@ import org.apache.solr.client.api.endpoint.BalanceReplicasApi; import org.apache.solr.client.api.model.BalanceReplicasRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; @@ -53,19 +54,15 @@ public BalanceReplicas( @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse balanceReplicas(BalanceReplicasRequestBody requestBody) throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + recordCollectionForLogAndTracing(null, solrQueryRequest); + fetchAndValidateZooKeeperAwareCoreContainer(); final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionAction.BALANCE_REPLICAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionAction.BALANCE_REPLICAS, + remoteMessage, + requestBody.async); disableResponseCaching(); return response; @@ -76,7 +73,6 @@ public ZkNodeProps createRemoteMessage(BalanceReplicasRequestBody requestBody) { if (requestBody != null) { insertIfNotNull(remoteMessage, NODES, requestBody.nodes); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); } remoteMessage.put(QUEUE_OPERATION, CollectionAction.BALANCE_REPLICAS.toLower()); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java index ae54b0ab98da..ca627de75518 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java @@ -83,13 +83,10 @@ public SubResponseAccumulatingJerseyResponse balanceShardUnique( public static ZkNodeProps createRemoteMessage( String collectionName, BalanceShardUniqueRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put( - QUEUE_OPERATION, CollectionParams.CollectionAction.BALANCESHARDUNIQUE.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(PROPERTY_PROP, requestBody.property); insertIfNotNull(remoteMessage, ONLY_ACTIVE_NODES, requestBody.onlyActiveNodes); insertIfNotNull(remoteMessage, SHARD_UNIQUE, requestBody.shardUnique); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java index bbeafb210e55..168b2a51c0c0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java @@ -84,8 +84,7 @@ public CreateAlias( @Override @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception { - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); recordCollectionForLogAndTracing(null, solrQueryRequest); if (requestBody == null) { @@ -109,19 +108,11 @@ public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws remoteMessage = createRemoteMessageForRoutedAlias(requestBody); } - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.CREATEALIAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.CREATEALIAS, + remoteMessage, + requestBody.async); return response; } @@ -129,19 +120,15 @@ public static ZkNodeProps createRemoteMessageForTraditionalAlias( CreateAliasRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATEALIAS.toLower()); remoteMessage.put(NAME, requestBody.name); remoteMessage.put("collections", String.join(",", requestBody.collections)); - remoteMessage.put(ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } public static ZkNodeProps createRemoteMessageForRoutedAlias(CreateAliasRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATEALIAS.toLower()); remoteMessage.put(NAME, requestBody.name); - if (StrUtils.isNotBlank(requestBody.async)) remoteMessage.put(ASYNC, requestBody.async); if (requestBody.routers.size() > 1) { // Multi-dimensional alias for (int i = 0; i < requestBody.routers.size(); i++) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index fef25cfdb52f..3bd23946c052 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -99,8 +99,7 @@ public SubResponseAccumulatingJerseyResponse createCollection( throw new SolrException(BAD_REQUEST, "Request body is missing but required"); } - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(requestBody.name, solrQueryRequest); @@ -111,24 +110,11 @@ public SubResponseAccumulatingJerseyResponse createCollection( final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.CREATE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); - response.warning = (String) remoteResponse.getResponse().get("warning"); + remoteMessage, + requestBody.async); // Even if Overseer does wait for the collection to be created, it sees a different cluster // state than this node, so this wait is required to make sure the local node Zookeeper watches @@ -163,7 +149,6 @@ public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBod final Map rawProperties = new HashMap<>(); rawProperties.put("fromApi", "true"); - rawProperties.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); rawProperties.put(NAME, reqBody.name); rawProperties.put(COLL_CONF, reqBody.config); rawProperties.put(NUM_SLICES, reqBody.numShards); @@ -175,8 +160,6 @@ public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBod rawProperties.put(TLOG_REPLICAS, reqBody.tlogReplicas); rawProperties.put(WAIT_FOR_FINAL_STATE, reqBody.waitForFinalState); rawProperties.put(PER_REPLICA_STATE, reqBody.perReplicaState); - rawProperties.put(ALIAS, reqBody.alias); - rawProperties.put(ASYNC, reqBody.async); if (reqBody.createReplicas == null || reqBody.createReplicas) { // The remote message expects a single comma-delimited string, so nodeSet requires flattening if (reqBody.nodeSet != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java index 602bda58e7ed..83fb2e327e20 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java @@ -42,12 +42,14 @@ import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; import org.apache.solr.client.api.model.CreateCollectionBackupResponseBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; @@ -109,20 +111,15 @@ public SolrJerseyResponse createCollectionBackup( "Unknown index backup strategy " + requestBody.backupStrategy); } + final var response = instantiateJerseyResponse(CreateCollectionBackupResponseBody.class); final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, backupName, requestBody); final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.BACKUP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - final SolrJerseyResponse response = - objectMapper.convertValue( - remoteResponse.getResponse(), CreateCollectionBackupResponseBody.class); + remoteMessage, + requestBody.async); + objectMapper.updateValue(response, remoteResponse.getResponse()); return response; } @@ -130,7 +127,6 @@ public SolrJerseyResponse createCollectionBackup( public static ZkNodeProps createRemoteMessage( String collectionName, String backupName, CreateCollectionBackupRequestBody requestBody) { final Map remoteMessage = Utils.reflectToMap(requestBody); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.BACKUP.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(NAME, backupName); if (!StringUtils.isBlank(requestBody.backupStrategy)) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java index 96cf56ff3f62..ebf4fcb669f4 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java @@ -60,8 +60,7 @@ public CreateCollectionSnapshot( public CreateCollectionSnapshotResponse createCollectionSnapshot( String collName, String snapshotName, CreateCollectionSnapshotRequestBody requestBody) throws Exception { - final CreateCollectionSnapshotResponse response = - instantiateJerseyResponse(CreateCollectionSnapshotResponse.class); + final var response = instantiateJerseyResponse(CreateCollectionSnapshotResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); @@ -79,35 +78,27 @@ public CreateCollectionSnapshotResponse createCollectionSnapshot( } final ZkNodeProps remoteMessage = - createRemoteMessage(collName, requestBody.followAliases, snapshotName, requestBody.async); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.CREATESNAPSHOT, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + createRemoteMessage(collName, requestBody.followAliases, snapshotName); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.CREATESNAPSHOT, + remoteMessage, + requestBody.async); response.collection = collName; response.followAliases = requestBody.followAliases; response.snapshotName = snapshotName; - response.requestId = requestBody.async; return response; } public static ZkNodeProps createRemoteMessage( - String collectionName, boolean followAliases, String snapshotName, String asyncId) { + String collectionName, boolean followAliases, String snapshotName) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESNAPSHOT.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java index 72c8eb058b12..30f7d29103c8 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java @@ -100,7 +100,6 @@ public SubResponseAccumulatingJerseyResponse createReplica( public static ZkNodeProps createRemoteMessage( String collectionName, String shardName, CreateReplicaRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICA.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, CoreAdminParams.NAME, requestBody.name); @@ -119,7 +118,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, TLOG_REPLICAS, requestBody.tlogReplicas); insertIfNotNull(remoteMessage, PULL_REPLICAS, requestBody.pullReplicas); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); if (requestBody.properties != null) { requestBody diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java index aac01f61c846..50d54a430a7f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java @@ -29,6 +29,7 @@ import org.apache.solr.client.api.endpoint.DeleteAliasApi; import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -50,33 +51,14 @@ public DeleteAlias( @Override @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse deleteAlias(String aliasName, String asyncId) throws Exception { - final AsyncJerseyResponse response = instantiateJerseyResponse(AsyncJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - - final ZkNodeProps remoteMessage = createRemoteMessage(aliasName, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETEALIAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (asyncId != null) { - response.requestId = asyncId; - } + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETEALIAS, + new ZkNodeProps(Map.of(NAME, aliasName)), + asyncId); return response; } - - public static ZkNodeProps createRemoteMessage(String aliasName, String asyncId) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEALIAS.toLower()); - remoteMessage.put(NAME, aliasName); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); - - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java index 3b6d87ac33c4..90ae0c639b57 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java @@ -57,42 +57,25 @@ public DeleteCollection( @PermissionName(COLL_EDIT_PERM) public SubResponseAccumulatingJerseyResponse deleteCollection( String collectionName, Boolean followAliases, String asyncId) throws Exception { - final SubResponseAccumulatingJerseyResponse response = - instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collectionName, solrQueryRequest); - final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, followAliases, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (asyncId != null) { - response.requestId = asyncId; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, followAliases); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETE, + remoteMessage, + asyncId); return response; } - public static ZkNodeProps createRemoteMessage( - String collectionName, Boolean followAliases, String asyncId) { + public static ZkNodeProps createRemoteMessage(String collectionName, Boolean followAliases) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower()); remoteMessage.put(NAME, collectionName); if (followAliases != null) remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java index bb88c09853cb..f2362c83aa66 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java @@ -87,7 +87,7 @@ public BackupDeletionResponseBody deleteSingleBackupById( location = getAndValidateBackupLocation(repositoryName, location); final ZkNodeProps remoteMessage = - createRemoteMessage(backupName, backupId, null, null, location, repositoryName, asyncId); + createRemoteMessage(backupName, backupId, null, null, location, repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEBACKUP, remoteMessage, asyncId); @@ -114,7 +114,7 @@ public BackupDeletionResponseBody deleteMultipleBackupsByRecency( final ZkNodeProps remoteMessage = createRemoteMessage( - backupName, null, versionsToRetain, null, location, repositoryName, asyncId); + backupName, null, versionsToRetain, null, location, repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEBACKUP, remoteMessage, asyncId); @@ -144,8 +144,7 @@ public PurgeUnusedResponse garbageCollectUnusedBackupFiles( null, Boolean.TRUE, requestBody.location, - requestBody.repositoryName, - requestBody.async); + requestBody.repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, @@ -167,12 +166,10 @@ public static ZkNodeProps createRemoteMessage( Integer versionsToRetain, Boolean purgeUnused, String location, - String repositoryName, - String asyncId) { + String repositoryName) { final Map remoteMessage = new HashMap<>(); // Always provided - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEBACKUP.toLower()); remoteMessage.put(NAME, backupName); // Mutually exclusive assert backupId != null || versionsToRetain != null || purgeUnused != null; @@ -182,7 +179,6 @@ public static ZkNodeProps createRemoteMessage( // Remaining params are truly optional insertIfNotNull(remoteMessage, BACKUP_LOCATION, location); insertIfNotNull(remoteMessage, BACKUP_REPOSITORY, repositoryName); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java index dec728026a34..39e87894e5bf 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java @@ -56,23 +56,16 @@ public DeleteCollectionSnapshotResponse deleteCollectionSnapshot( String collName, String snapshotName, boolean followAliases, String asyncId) throws Exception { final var response = instantiateJerseyResponse(DeleteCollectionSnapshotResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); final String collectionName = resolveCollectionName(collName, followAliases); - final ZkNodeProps remoteMessage = - createRemoteMessage(collectionName, followAliases, snapshotName, asyncId); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETESNAPSHOT, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETESNAPSHOT, + createRemoteMessage(collectionName, followAliases, snapshotName), + asyncId); response.collection = collName; response.snapshotName = snapshotName; @@ -82,17 +75,13 @@ public DeleteCollectionSnapshotResponse deleteCollectionSnapshot( return response; } - public static ZkNodeProps createRemoteMessage( - String collectionName, boolean followAliases, String snapshotName, String asyncId) { + public static ZkNodeProps createRemoteMessage(String collectionName, boolean followAliases, String snapshotName) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETESNAPSHOT.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); remoteMessage.put(FOLLOW_ALIASES, followAliases); - if (asyncId != null) remoteMessage.put(ASYNC, asyncId); - return new ZkNodeProps(remoteMessage); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java index 78af96b4ec28..9cda7a408352 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java @@ -28,6 +28,7 @@ import org.apache.solr.client.api.endpoint.DeleteNodeApi; import org.apache.solr.client.api.model.DeleteNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -59,18 +60,13 @@ public DeleteNode( @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requestBody) throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - final ZkNodeProps remoteMessage = createRemoteMessage(nodeName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETENODE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETENODE, + new ZkNodeProps(Map.of(NODE, nodeName)), + requestBody.async); disableResponseCaching(); return response; } @@ -82,18 +78,4 @@ public static SolrJerseyResponse invokeUsingV1Inputs(DeleteNode apiInstance, Sol requestBody.async = params.get(ASYNC); return apiInstance.deleteNode(requiredParams.get(NODE), requestBody); } - - public static ZkNodeProps createRemoteMessage( - String nodeName, DeleteNodeRequestBody requestBody) { - Map remoteMessage = new HashMap<>(); - remoteMessage.put(NODE, nodeName); - if (requestBody != null) { - if (requestBody.async != null) { - remoteMessage.put(ASYNC, requestBody.async); - } - } - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETENODE.toLower()); - - return new ZkNodeProps(remoteMessage); - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java index 9c2486fdc39f..eb67a44986df 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java @@ -61,20 +61,13 @@ public DeleteReplicaProperty( public SolrJerseyResponse deleteReplicaProperty( String collName, String shardName, String replicaName, String propertyName) throws Exception { final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); - final ZkNodeProps remoteMessage = - createRemoteMessage(collName, shardName, replicaName, propertyName); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.DELETEREPLICAPROP, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.DELETEREPLICAPROP, + createRemoteMessage(collName, shardName, replicaName, propertyName)); return response; } @@ -99,8 +92,6 @@ public static ZkNodeProps createRemoteMessage( String collName, String shardName, String replicaName, String propName) { final Map messageProperties = Map.of( - QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETEREPLICAPROP.toLower(), COLLECTION_PROP, collName, SHARD_ID_PROP, diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index d24bb8991af3..d97b6e5590a2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -25,6 +25,7 @@ import org.apache.solr.client.api.endpoint.InstallShardDataApi; import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.InstallShardDataRequestBody; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.cloud.api.collections.InstallShardDataCmd; import org.apache.solr.common.SolrException; @@ -61,7 +62,7 @@ public InstallShardData( @PermissionName(COLL_EDIT_PERM) public AsyncJerseyResponse installShardData( String collName, String shardName, InstallShardDataRequestBody requestBody) throws Exception { - final var response = instantiateJerseyResponse(AsyncJerseyResponse.class); + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); if (requestBody == null) { @@ -87,21 +88,11 @@ public AsyncJerseyResponse installShardData( "Collection must be in readOnly mode before installing data to shard with more than 1 replica"); } - ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.INSTALLSHARDDATA, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.INSTALLSHARDDATA, + createRemoteMessage(collName, shardName, requestBody), + requestBody.async); return response; } @@ -127,7 +118,6 @@ public static ZkNodeProps createRemoteMessage( messageTyped.repository = requestBody.repository; messageTyped.name = requestBody.name; messageTyped.shardBackupId = requestBody.shardBackupId; - messageTyped.asyncId = requestBody.async; messageTyped.callingLockId = requestBody.callingLockId; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java index 670540b4f74f..fbdc482c41f2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java @@ -30,6 +30,7 @@ import org.apache.solr.client.api.endpoint.MigrateReplicasApi; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; @@ -55,19 +56,14 @@ public MigrateReplicas( @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse migrateReplicas(MigrateReplicasRequestBody requestBody) throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing - final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionAction.MIGRATE_REPLICAS, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(null, solrQueryRequest); + submitRemoteMessageAndHandleResponse( + response, + CollectionAction.MIGRATE_REPLICAS, + createRemoteMessage(requestBody), + requestBody.async); disableResponseCaching(); return response; @@ -84,13 +80,11 @@ public ZkNodeProps createRemoteMessage(MigrateReplicasRequestBody requestBody) { insertIfNotNull(remoteMessage, SOURCE_NODES, requestBody.sourceNodes); insertIfNotNull(remoteMessage, TARGET_NODES, requestBody.targetNodes); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No request body sent with request. The MigrateReplicas API requires a body."); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.MIGRATE_REPLICAS.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java index 1a6ed3a39466..902a8f218c28 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java @@ -72,18 +72,16 @@ public SubResponseAccumulatingJerseyResponse renameCollection( response, CollectionParams.CollectionAction.RENAME, remoteMessage, - requestBody != null ? requestBody.async : null); + requestBody.async); return response; } public static ZkNodeProps createRemoteMessage( String collectionName, RenameCollectionRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RENAME.toLower()); remoteMessage.put(NAME, collectionName); remoteMessage.put(TARGET, requestBody.to); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java index bb36f2036df4..3dd873b41753 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java @@ -30,6 +30,7 @@ import org.apache.solr.client.api.endpoint.ReplaceNodeApi; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -59,19 +60,14 @@ public ReplaceNode( @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse replaceNode(String sourceNodeName, ReplaceNodeRequestBody requestBody) throws Exception { - final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); - final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); - // TODO Record node for log and tracing - final ZkNodeProps remoteMessage = createRemoteMessage(sourceNodeName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, - CollectionParams.CollectionAction.REPLACENODE, - DEFAULT_COLLECTION_OP_TIMEOUT); - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } + final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(null, solrQueryRequest); + submitRemoteMessageAndHandleResponse( + response, + CollectionParams.CollectionAction.REPLACENODE, + createRemoteMessage(sourceNodeName, requestBody), + requestBody.async); disableResponseCaching(); return response; @@ -83,7 +79,6 @@ public ZkNodeProps createRemoteMessage(String nodeName, ReplaceNodeRequestBody r if (requestBody != null) { insertIfValueNotNull(remoteMessage, TARGET_NODE, requestBody.targetNodeName); insertIfValueNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); - insertIfValueNotNull(remoteMessage, ASYNC, requestBody.async); } remoteMessage.put(QUEUE_OPERATION, CollectionAction.REPLACENODE.toLower()); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java index fb39fafa02eb..71988d99e97e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java @@ -123,25 +123,11 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( } final ZkNodeProps remoteMessage = createRemoteMessage(backupName, requestBody); - final SolrResponse remoteResponse = - CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), - remoteMessage, + submitRemoteMessageAndHandleResponse( + response, CollectionParams.CollectionAction.RESTORE, - DEFAULT_COLLECTION_OP_TIMEOUT); - - if (remoteResponse.getException() != null) { - throw remoteResponse.getException(); - } - - if (requestBody.async != null) { - response.requestId = requestBody.async; - return response; - } - - // Values fetched from remoteResponse may be null - response.successfulSubResponsesByNodeName = remoteResponse.getResponse().get("success"); - response.failedSubResponsesByNodeName = remoteResponse.getResponse().get("failure"); + remoteMessage, + requestBody.async); return response; } From 6dac5939e179d42fd20329a3832365d4c4ba0d35 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 8 Jan 2026 16:13:59 -0800 Subject: [PATCH 20/67] Refactor how locking is passed through collections api commands. --- .../model/InstallShardDataRequestBody.java | 2 - .../java/org/apache/solr/api/V2HttpCall.java | 6 ++ .../java/org/apache/solr/cloud/LockTree.java | 6 +- .../solr/cloud/OverseerTaskProcessor.java | 1 + .../cloud/api/collections/AddReplicaCmd.java | 16 ++-- .../api/collections/AdminCmdContext.java | 36 +++++++- .../solr/cloud/api/collections/AliasCmd.java | 6 +- .../solr/cloud/api/collections/BackupCmd.java | 18 ++-- .../api/collections/BalanceReplicasCmd.java | 9 +- .../cloud/api/collections/CollApiCmds.java | 37 ++++---- .../collections/CollectionApiLockFactory.java | 6 +- .../collections/CollectionCommandContext.java | 4 +- .../collections/CollectionHandlingUtils.java | 52 ++++++----- .../cloud/api/collections/CreateAliasCmd.java | 23 ++--- .../api/collections/CreateCollectionCmd.java | 19 ++-- .../cloud/api/collections/CreateShardCmd.java | 18 ++-- .../api/collections/CreateSnapshotCmd.java | 10 +-- .../cloud/api/collections/DeleteAliasCmd.java | 5 +- .../api/collections/DeleteBackupCmd.java | 5 +- .../api/collections/DeleteCollectionCmd.java | 12 +-- .../cloud/api/collections/DeleteNodeCmd.java | 12 +-- .../api/collections/DeleteReplicaCmd.java | 25 +++--- .../cloud/api/collections/DeleteShardCmd.java | 15 ++-- .../api/collections/DeleteSnapshotCmd.java | 9 +- ...butedCollectionConfigSetCommandRunner.java | 5 +- .../api/collections/InstallShardDataCmd.java | 8 +- .../collections/MaintainRoutedAliasCmd.java | 25 +++--- .../cloud/api/collections/MigrateCmd.java | 69 ++++++-------- .../api/collections/MigrateReplicasCmd.java | 9 +- .../cloud/api/collections/MoveReplicaCmd.java | 48 ++++------ .../OverseerCollectionMessageHandler.java | 18 +++- .../api/collections/OverseerRoleCmd.java | 5 +- .../api/collections/OverseerStatusCmd.java | 5 +- .../api/collections/ReindexCollectionCmd.java | 51 +++-------- .../solr/cloud/api/collections/RenameCmd.java | 7 +- .../cloud/api/collections/ReplaceNodeCmd.java | 9 +- .../collections/ReplicaMigrationUtils.java | 24 ++--- .../cloud/api/collections/RestoreCmd.java | 90 +++++++------------ .../api/collections/SetAliasPropCmd.java | 5 +- .../cloud/api/collections/SplitShardCmd.java | 44 ++++----- .../handler/admin/CollectionsHandler.java | 22 +++-- .../handler/admin/api/InstallShardData.java | 1 - .../handler/component/HttpShardHandler.java | 3 + .../solr/handler/component/ShardRequest.java | 4 + .../common/params/CollectionAdminParams.java | 2 +- 45 files changed, 337 insertions(+), 469 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index a72b9c2bc920..05b27f1dcab3 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -29,6 +29,4 @@ public class InstallShardDataRequestBody { @JsonProperty public String shardBackupId; @JsonProperty public String async; - - @JsonProperty public String callingLockId; } diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 51d14b659932..8246bcfa39ce 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -18,6 +18,7 @@ package org.apache.solr.api; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN_OR_REMOTEPROXY; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PROCESS; @@ -216,6 +217,11 @@ private void initAdminRequest(String path) throws Exception { solrReq.getContext().put(CoreContainer.class.getName(), cores); requestType = AuthorizationContext.RequestType.ADMIN; action = ADMIN; + + String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); + if (callingLockIds != null && !callingLockIds.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + } } protected void parseRequest() throws Exception { diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 20a5686093f7..44fbd9fd263a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -88,15 +88,15 @@ public class Session { private SessionNode root = new SessionNode(LockLevel.CLUSTER); public Lock lock( - CollectionParams.CollectionAction action, List path, String callingLockId) { + CollectionParams.CollectionAction action, List path, List callingLockIds) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; - log.info("Calling lock level: {}", callingLockId); + log.info("Calling lock level: {}", callingLockIds); Node startingNode = LockTree.this.root; SessionNode startingSession = root; // If a callingLockId was passed in, validate it with the current lock path, and only start // locking below the calling lock - Lock callingLock = callingLockId != null ? allLocks.get(callingLockId) : null; + Lock callingLock = callingLockIds != null ? allLocks.get(callingLockIds.getLast()) : null; boolean ignoreCallingLock = false; if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { startingNode = ((LockImpl) callingLock).node; diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index b44c4f51ce24..4c2014149f96 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -37,6 +37,7 @@ import java.util.function.Predicate; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index 1ce8194d564d..6a55a3a95ed5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -25,7 +25,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; @@ -77,13 +76,13 @@ public AddReplicaCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - addReplica(state, message, results, null); + addReplica(adminCmdContext, message, results, null); } List addReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete) @@ -104,7 +103,7 @@ List addReplica( collectionName = extCollectionName; } - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); if (coll == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collectionName + " does not exist"); @@ -118,7 +117,6 @@ List addReplica( boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false); boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false); - final String asyncId = message.getStr(ASYNC); String node = message.getStr(CoreAdminParams.NODE); String createNodeSetStr = message.getStr(CREATE_NODE_SET); @@ -153,7 +151,7 @@ List addReplica( List createReplicas = buildReplicaPositions( ccc.getSolrCloudManager(), - clusterState, + adminCmdContext.getClusterState(), collectionName, message, numReplicas, @@ -162,14 +160,14 @@ List addReplica( .map( replicaPosition -> assignReplicaDetails( - ccc.getSolrCloudManager(), clusterState, message, replicaPosition)) + ccc.getSolrCloudManager(), adminCmdContext.getClusterState(), message, replicaPosition)) .collect(Collectors.toList()); ShardHandler shardHandler = ccc.newShardHandler(); ZkStateReader zkStateReader = ccc.getZkStateReader(); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (CreateReplica createReplica : createReplicas) { assert createReplica.coreName != null; ModifiableSolrParams params = diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 33f91c46a17d..f0d4dd34e9e3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -2,13 +2,16 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.util.StrUtils; +import java.util.ArrayList; import java.util.List; public class AdminCmdContext { final private CollectionParams.CollectionAction action; final private String asyncId; private String lockId; - private List callingLockIds; + private String callingLockIds; + private String subRequestCallingLockIds; private ClusterState clusterState; public AdminCmdContext(CollectionParams.CollectionAction action) { @@ -30,17 +33,27 @@ public String getAsyncId() { public void setLockId(String lockId) { this.lockId = lockId; + regenerateSubRequestCallingLockIds(); } public String getLockId() { return lockId; } - public void setCallingLockIds(List callingLockIds) { + public void setCallingLockIds(String callingLockIds) { this.callingLockIds = callingLockIds; + regenerateSubRequestCallingLockIds(); } - public List getCallingLockIds() { + private void regenerateSubRequestCallingLockIds() { + subRequestCallingLockIds = callingLockIds; + if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { + subRequestCallingLockIds += ","; + } + subRequestCallingLockIds += lockId; + } + + public String getCallingLockIds() { return callingLockIds; } @@ -48,7 +61,22 @@ public ClusterState getClusterState() { return clusterState; } - public void setClusterState(ClusterState clusterState) { + public AdminCmdContext withClusterState(ClusterState clusterState) { this.clusterState = clusterState; + return this; + } + + public String getSubRequestCallingLockIds() { + return subRequestCallingLockIds; + } + + public AdminCmdContext subRequestContext(CollectionParams.CollectionAction action) { + return subRequestContext(action, asyncId); + } + + public AdminCmdContext subRequestContext(CollectionParams.CollectionAction action, String asyncId) { + AdminCmdContext nextContext = new AdminCmdContext(action, asyncId); + nextContext.setCallingLockIds(subRequestCallingLockIds); + return nextContext.withClusterState(clusterState); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 98d4f1b311aa..e6c3f308336e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -28,6 +28,7 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CollectionsHandler; @@ -52,11 +53,10 @@ protected AliasCmd(CollectionCommandContext ccc) { * If the collection already exists then this is not an error. */ static NamedList createCollectionAndWait( - ClusterState clusterState, + AdminCmdContext adminCmdContext, String aliasName, Map aliasMetadata, String createCollName, - String lockId, CollectionCommandContext ccc) throws Exception { // Map alias metadata starting with a prefix to a create-collection API request @@ -84,7 +84,7 @@ static NamedList createCollectionAndWait( // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(clusterState, createMessage, lockId, results); + new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE), createMessage, results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 87507cee57be..3108a1c2b2d4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; @@ -34,7 +33,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; @@ -71,9 +69,7 @@ public BackupCmd(CollectionCommandContext ccc) { @SuppressWarnings("unchecked") @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -124,7 +120,7 @@ public void call( if (incremental) { try { incrementalCopyIndexFiles( - backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, backupUri, collectionName, message, results, backupProperties, backupMgr); } catch (SolrException e) { log.error( "Error happened during incremental backup for collection: {}", @@ -136,7 +132,7 @@ public void call( } } else { copyIndexFiles( - backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, backupUri, collectionName, message, results, backupProperties, backupMgr); } break; } @@ -289,6 +285,7 @@ private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMet } private void incrementalCopyIndexFiles( + AdminCmdContext adminCmdContext, URI backupUri, String collectionName, ZkNodeProps request, @@ -297,7 +294,6 @@ private void incrementalCopyIndexFiles( BackupManager backupManager) throws IOException { String backupName = request.getStr(NAME); - String asyncId = request.getStr(ASYNC); String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); ShardHandler shardHandler = ccc.newShardHandler(); @@ -309,7 +305,7 @@ private void incrementalCopyIndexFiles( Optional previousProps = backupManager.tryReadBackupProperties(); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); Collection slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices(); @@ -435,6 +431,7 @@ private ModifiableSolrParams coreBackupParams( } private void copyIndexFiles( + AdminCmdContext adminCmdContext, URI backupPath, String collectionName, ZkNodeProps request, @@ -443,7 +440,6 @@ private void copyIndexFiles( BackupManager backupManager) throws Exception { String backupName = request.getStr(NAME); - String asyncId = request.getStr(ASYNC); String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); ShardHandler shardHandler = ccc.newShardHandler(); @@ -485,7 +481,7 @@ private void copyIndexFiles( } final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); Collection slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices(); for (Slice slice : slices) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java index aaa0e684f18c..9d1acfb27b5e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BalanceReplicasCmd.java @@ -17,15 +17,12 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -41,8 +38,7 @@ public BalanceReplicasCmd(CollectionCommandContext ccc) { @SuppressWarnings({"unchecked"}) @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { Set nodes; Object nodesRaw = message.get(CollectionParams.NODES); @@ -61,7 +57,6 @@ public void call( + nodesRaw.getClass().getName()); } boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); @@ -80,7 +75,7 @@ public void call( boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index 1f8550d769ab..6649dfd35048 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -81,7 +81,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.OverseerNodePrioritizer; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; @@ -114,7 +113,7 @@ public class CollApiCmds { * classes whose names ends in {@code Cmd}. */ protected interface CollectionApiCommand { - void call(ClusterState state, ZkNodeProps message, String lockId, NamedList results) + void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception; } @@ -208,7 +207,7 @@ public TraceAwareCommand(CollectionApiCommand command) { @Override public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final Span localSpan; final Context localContext; @@ -219,7 +218,7 @@ public void call( String collection = Optional.ofNullable(message.getStr(COLLECTION_PROP, message.getStr(NAME))) .orElse("unknown"); - boolean isAsync = message.containsKey(ASYNC); + boolean isAsync = adminCmdContext.getAsyncId() != null; localSpan = TraceUtils.startCollectionApiCommandSpan( command.getClass().getSimpleName(), collection, isAsync); @@ -228,7 +227,7 @@ public void call( try (var scope = localContext.makeCurrent()) { assert scope != null; // prevent javac warning about scope being unused - command.call(state, message, lockId, results); + command.call(adminCmdContext, message, results); } finally { if (localSpan != null) { localSpan.end(); @@ -241,7 +240,7 @@ public static class MockOperationCmd implements CollectionApiCommand { @Override @SuppressForbidden(reason = "Needs currentTimeMillis for mock requests") public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws InterruptedException { // only for test purposes Thread.sleep(message.getInt("sleep", 1)); @@ -263,21 +262,18 @@ public ReloadCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); - String asyncId = message.getStr(ASYNC); CollectionHandlingUtils.collectionCmd( + adminCmdContext, message, params, results, Replica.State.ACTIVE, - asyncId, Collections.emptySet(), - ccc, - clusterState); + ccc); } } @@ -290,7 +286,7 @@ public RebalanceLeadersCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -332,7 +328,7 @@ public AddReplicaPropCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -367,7 +363,7 @@ public DeleteReplicaPropCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); @@ -397,7 +393,7 @@ public BalanceShardsUniqueCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { if (StrUtils.isBlank(message.getStr(COLLECTION_PROP)) || StrUtils.isBlank(message.getStr(PROPERTY_PROP))) { @@ -433,10 +429,7 @@ public ModifyCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { - + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); // the rest of the processing is based on writing cluster state properties String configName = (String) message.getProperties().get(COLL_CONF); @@ -506,7 +499,7 @@ public void call( SolrZkClient.checkInterrupted(e); log.debug( "modifyCollection(ClusterState={}, ZkNodeProps={}, NamedList={})", - clusterState, + adminCmdContext.getClusterState(), message, results, e); @@ -517,7 +510,7 @@ public void call( // if switching to/from read-only mode or configName is not null reload the collection if (message.keySet().contains(ZkStateReader.READ_ONLY) || configName != null) { new ReloadCollectionCmd(ccc) - .call(clusterState, new ZkNodeProps(NAME, collectionName), lockId, results); + .call(adminCmdContext.subRequestContext(RELOAD, null), new ZkNodeProps(NAME, collectionName), results); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index b9566bc499be..00b3d84d30da 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -111,9 +111,11 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } - List callingLockIdList = adminCmdContext.getCallingLockIds(); - if (callingLockIdList == null) { + List callingLockIdList; + if (adminCmdContext.getCallingLockIds() == null) { callingLockIdList = Collections.emptyList(); + } else { + callingLockIdList = List.of(adminCmdContext.getCallingLockIds().split(",")); } // The first requested lock is a write one (on the target object for the action, depending on diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java index f167d4dc6b44..025a5571ff06 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java @@ -55,9 +55,9 @@ public interface CollectionCommandContext { CoreContainer getCoreContainer(); - default ShardRequestTracker asyncRequestTracker(String asyncId) { + default ShardRequestTracker asyncRequestTracker(AdminCmdContext adminCmdContext) { return new ShardRequestTracker( - asyncId, getAdminPath(), getZkStateReader(), newShardHandler().getShardHandlerFactory()); + adminCmdContext.getAsyncId(), adminCmdContext.getSubRequestCallingLockIds(), getAdminPath(), getZkStateReader(), newShardHandler().getShardHandlerFactory()); } /** admin path passed to Overseer is a constant, including in tests. */ diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index d9a1b0111c16..aa58836a1ada 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -307,13 +308,13 @@ static void addPropertyParams(ZkNodeProps message, Map map) { } static void cleanupCollection( - String collectionName, NamedList results, CollectionCommandContext ccc) + AdminCmdContext adminCmdContext, String collectionName, NamedList results, CollectionCommandContext ccc) throws Exception { log.error("Cleaning up collection [{}].", collectionName); Map props = - Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collectionName); + Map.of(NAME, collectionName); new DeleteCollectionCmd(ccc) - .call(ccc.getZkStateReader().getClusterState(), new ZkNodeProps(props), null, results); + .call(adminCmdContext.subRequestContext(DELETE), new ZkNodeProps(props), results); } static Map waitToSeeReplicasInState( @@ -372,17 +373,6 @@ static void deleteBackup( new DeleteBackupCmd(ccc).keepNumberOfBackup(repository, backupPath, maxNumBackup, results); } - static List addReplica( - ClusterState clusterState, - ZkNodeProps message, - NamedList results, - Runnable onComplete, - CollectionCommandContext ccc) - throws Exception { - - return new AddReplicaCmd(ccc).addReplica(clusterState, message, results, onComplete); - } - static void validateConfigOrThrowSolrException( ConfigSetService configSetService, String configName) throws IOException { boolean isValid = configSetService.checkConfigExists(configName); @@ -399,24 +389,23 @@ static void validateConfigOrThrowSolrException( * @return List of replicas which is not live for receiving the request */ static List collectionCmd( + AdminCmdContext adminCmdContext, ZkNodeProps message, ModifiableSolrParams params, NamedList results, Replica.State stateMatcher, - String asyncId, Set okayExceptions, - CollectionCommandContext ccc, - ClusterState clusterState) { - log.info("Executing Collection Cmd={}, asyncId={}", params, asyncId); + CollectionCommandContext ccc) { + log.info("Executing Collection Cmd={}, asyncId={}", params, adminCmdContext.getAsyncId()); String collectionName = message.getStr(NAME); ShardHandler shardHandler = ccc.newShardHandler(); - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); List notLivesReplicas = new ArrayList<>(); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = - asyncRequestTracker(asyncId, ccc); + asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : coll.getSlices()) { notLivesReplicas.addAll( - shardRequestTracker.sliceCmd(clusterState, params, stateMatcher, slice, shardHandler)); + shardRequestTracker.sliceCmd(adminCmdContext.getClusterState(), params, stateMatcher, slice, shardHandler)); } shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions); @@ -574,14 +563,23 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( } while (true); } - public static ShardRequestTracker syncRequestTracker(CollectionCommandContext ccc) { - return asyncRequestTracker(null, ccc); + public static ShardRequestTracker syncRequestTracker(AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(),ccc); } public static ShardRequestTracker asyncRequestTracker( - String asyncId, CollectionCommandContext ccc) { + AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return requestTracker( + adminCmdContext.getAsyncId(), + adminCmdContext.getSubRequestCallingLockIds(), + ccc); + } + + protected static ShardRequestTracker requestTracker( + String asyncId, String lockIds, CollectionCommandContext ccc) { return new ShardRequestTracker( asyncId, + lockIds, ccc.getAdminPath(), ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); @@ -589,6 +587,7 @@ public static ShardRequestTracker asyncRequestTracker( public static class ShardRequestTracker { private final String asyncId; + private final String lockIdList; private final String adminPath; private final ZkStateReader zkStateReader; private final ShardHandlerFactory shardHandlerFactory; @@ -596,10 +595,12 @@ public static class ShardRequestTracker { public ShardRequestTracker( String asyncId, + String lockIdList, String adminPath, ZkStateReader zkStateReader, ShardHandlerFactory shardHandlerFactory) { this.asyncId = asyncId; + this.lockIdList = lockIdList; this.adminPath = adminPath; this.zkStateReader = zkStateReader; this.shardHandlerFactory = shardHandlerFactory; @@ -662,6 +663,9 @@ public void sendShardRequest( sreq.actualShards = sreq.shards; sreq.nodeName = nodeName; sreq.params = params; + if (lockIdList != null && !lockIdList.isBlank()) { + sreq.headers = Map.of(CALLING_LOCK_IDS_HEADER, lockIdList); + } shardHandler.submit(sreq, replica, sreq.params); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index 3c162fbea97a..963834348f58 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -50,9 +50,7 @@ public CreateAliasCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); // make sure we have the latest version of existing aliases @@ -64,7 +62,7 @@ public void call( if (!anyRoutingParams(message)) { callCreatePlainAlias(message, aliasName, zkStateReader); } else { - callCreateRoutedAlias(message, aliasName, zkStateReader, state, lockId); + callCreateRoutedAlias(adminCmdContext, message, aliasName, zkStateReader); } // Sleep a bit to allow ZooKeeper state propagation. @@ -113,11 +111,10 @@ private List parseCollectionsParameter(Object collections) { } private void callCreateRoutedAlias( + AdminCmdContext adminCmdContext, ZkNodeProps message, String aliasName, - ZkStateReader zkStateReader, - ClusterState state, - String lockId) + ZkStateReader zkStateReader) throws Exception { // Validate we got a basic minimum if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) { @@ -155,12 +152,11 @@ private void callCreateRoutedAlias( // Create the first collection. Prior validation ensures that this is not a standard alias collectionListStr = routedAlias.computeInitialCollectionName(); ensureAliasCollection( + adminCmdContext, aliasName, zkStateReader, - state, routedAlias.getAliasMetadata(), - collectionListStr, - lockId); + collectionListStr); } else { List collectionList = aliases.resolveAliases(aliasName); collectionListStr = String.join(",", collectionList); @@ -173,15 +169,14 @@ private void callCreateRoutedAlias( } private void ensureAliasCollection( + AdminCmdContext adminCmdContext, String aliasName, ZkStateReader zkStateReader, - ClusterState state, Map aliasProperties, - String initialCollectionName, - String lockId) + String initialCollectionName) throws Exception { // Create the collection - createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, lockId, ccc); + createCollectionAndWait(adminCmdContext, aliasName, aliasProperties, initialCollectionName, ccc); validateAllCollectionsExistAndNoDuplicates( Collections.singletonList(initialCollectionName), zkStateReader); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index d93083cd4ca5..ee778c0ac59a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -20,6 +20,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.ALIAS; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CommonParams.NAME; @@ -101,12 +102,11 @@ public CreateCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader ccc.getZkStateReader().aliasesManager.update(); } + ClusterState clusterState = adminCmdContext.getClusterState(); final Aliases aliases = ccc.getZkStateReader().getAliases(); final String collectionName = message.getStr(NAME); final boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false); @@ -152,9 +152,6 @@ public void call( final String collectionPath = DocCollection.getCollectionPath(collectionName); try { - - final String async = message.getStr(ASYNC); - ZkStateReader zkStateReader = ccc.getZkStateReader(); message.getProperties().put(COLL_CONF, configName); @@ -246,7 +243,7 @@ public void call( numReplicas); } catch (Assign.AssignmentException e) { ZkNodeProps deleteMessage = new ZkNodeProps("name", collectionName); - new DeleteCollectionCmd(ccc).call(clusterState, deleteMessage, lockId, results); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(DELETE).withClusterState(clusterState), deleteMessage, results); // unwrap the exception throw new SolrException(ErrorCode.BAD_REQUEST, e.getMessage(), e.getCause()); } @@ -257,7 +254,7 @@ public void call( } final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(async, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); if (log.isDebugEnabled()) { log.debug( formatString( @@ -356,8 +353,8 @@ public void call( params.set(CoreAdminParams.NEW_COLLECTION, "true"); params.set(CoreAdminParams.REPLICA_TYPE, replicaPosition.type.name()); - if (async != null) { - String coreAdminAsyncId = async + Math.abs(System.nanoTime()); + if (adminCmdContext.getAsyncId() != null) { + String coreAdminAsyncId = adminCmdContext.getAsyncId() + Math.abs(System.nanoTime()); params.add(ASYNC, coreAdminAsyncId); } CollectionHandlingUtils.addPropertyParams(message, params); @@ -439,7 +436,7 @@ public void call( // Let's cleanup as we hit an exception // We shouldn't be passing 'results' here for the cleanup as the response would then contain // 'success' element, which may be interpreted by the user as a positive ack - CollectionHandlingUtils.cleanupCollection(collectionName, new NamedList<>(), ccc); + CollectionHandlingUtils.cleanupCollection(adminCmdContext, collectionName, new NamedList<>(), ccc); log.info("Cleaned up artifacts for failed create collection for [{}]", collectionName); throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java index 2c2c4de3f997..10fe8ef7b3cf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.Map; @@ -29,6 +28,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.ReplicaCount; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; @@ -45,9 +45,7 @@ public CreateShardCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); String sliceName = message.getStr(SHARD_ID_PROP); boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); @@ -65,6 +63,7 @@ public void call( } else { collectionName = extCollectionName; } + ClusterState clusterState = adminCmdContext.getClusterState(); DocCollection collection = clusterState.getCollection(collectionName); ReplicaCount numReplicas = ReplicaCount.fromMessage(message, collection, 1); @@ -97,7 +96,7 @@ public void call( clusterState = CollectionHandlingUtils.waitForNewShard(collectionName, sliceName, ccc.getZkStateReader()); - String async = message.getStr(ASYNC); + String async = adminCmdContext.getAsyncId(); Map addReplicasProps = Utils.makeMap( COLLECTION_PROP, @@ -111,14 +110,11 @@ public void call( numReplicas.writeProps(addReplicasProps); CollectionHandlingUtils.addPropertyParams(message, addReplicasProps); - if (async != null) { - addReplicasProps.put(ASYNC, async); - } final NamedList addResult = new NamedList<>(); try { new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA, async).withClusterState(clusterState), new ZkNodeProps(addReplicasProps), addResult, () -> { @@ -148,9 +144,7 @@ public void call( }); } catch (Assign.AssignmentException e) { // clean up the slice that we created - ZkNodeProps deleteShard = - new ZkNodeProps(COLLECTION_PROP, collectionName, SHARD_ID_PROP, sliceName, ASYNC, async); - new DeleteShardCmd(ccc).call(clusterState, deleteShard, lockId, results); + new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, async).withClusterState(clusterState), new ZkNodeProps(COLLECTION_PROP, collectionName, SHARD_ID_PROP, sliceName), results); throw e; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java index e5960fc79307..64e308259140 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -33,7 +32,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; @@ -62,9 +60,7 @@ public CreateSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -76,7 +72,6 @@ public void call( } String commitName = message.getStr(CoreAdminParams.COMMIT_NAME); - String asyncId = message.getStr(ASYNC); SolrZkClient zkClient = ccc.getZkStateReader().getZkClient(); Date creationDate = new Date(); @@ -101,8 +96,7 @@ public void call( Map shardByCoreName = new HashMap<>(); ShardHandler shardHandler = ccc.newShardHandler(); - final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : ccc.getZkStateReader().getClusterState().getCollection(collectionName).getSlices()) { for (Replica replica : slice.getReplicas()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java index 2cfbdf7ffb60..cda76a8b603d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CommonParams.NAME; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -32,9 +31,7 @@ public DeleteAliasCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String aliasName = message.getStr(NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java index eaf97accf3ea..ff0593cd97e0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -36,7 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; @@ -71,9 +70,7 @@ public DeleteBackupCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); String backupName = message.getStr(NAME); String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 63c6e2f24c3d..72c08a16cb7d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -36,7 +35,6 @@ import org.apache.solr.common.NonExistentCoreException; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; @@ -64,9 +62,7 @@ public DeleteCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { Object o = message.get(MaintainRoutedAliasCmd.INVOKED_BY_ROUTED_ALIAS); if (o != null) { // this will ensure the collection is removed from the alias before it disappears. @@ -92,7 +88,7 @@ public void call( } // verify the placement modifications caused by the deletion are allowed - DocCollection coll = state.getCollectionOrNull(collection); + DocCollection coll = adminCmdContext.getClusterState().getCollectionOrNull(collection); if (coll != null) { Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ccc.getCoreContainer()); assignStrategy.verifyDeleteCollection(ccc.getSolrCloudManager(), coll); @@ -120,13 +116,11 @@ public void call( params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true); params.set(CoreAdminParams.DELETE_DATA_DIR, true); - String asyncId = message.getStr(ASYNC); - ZkNodeProps internalMsg = message.plus(NAME, collection); List failedReplicas = CollectionHandlingUtils.collectionCmd( - internalMsg, params, results, null, asyncId, okayExceptions, ccc, state); + adminCmdContext, internalMsg, params, results, null, okayExceptions, ccc); for (Replica failedReplica : failedReplicas) { boolean isSharedFS = failedReplica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java index 948c1893fe1c..cbd46c0627ca 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.List; import org.apache.solr.common.cloud.ClusterState; @@ -36,13 +34,11 @@ public DeleteNodeCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired(message, "node"); String node = message.getStr("node"); - List sourceReplicas = ReplicaMigrationUtils.getReplicasOfNode(node, state); - List singleReplicas = verifyReplicaAvailability(sourceReplicas, state); + List sourceReplicas = ReplicaMigrationUtils.getReplicasOfNode(node, adminCmdContext.getClusterState()); + List singleReplicas = verifyReplicaAvailability(sourceReplicas, adminCmdContext.getClusterState()); if (!singleReplicas.isEmpty()) { results.add( "failure", @@ -52,7 +48,7 @@ public void call( + singleReplicas); } else { ReplicaMigrationUtils.cleanupReplicas( - results, state, sourceReplicas, ccc, message.getStr(ASYNC)); + results, adminCmdContext, sourceReplicas, ccc); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index 1cf89549afd7..c506f4576071 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -21,7 +21,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -61,14 +60,12 @@ public DeleteReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { - deleteReplica(clusterState, message, results, null); + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + deleteReplica(adminCmdContext, message, results, null); } void deleteReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete) @@ -80,7 +77,7 @@ void deleteReplica( // If a count is specified the strategy needs be different if (message.getStr(COUNT_PROP) != null) { - deleteReplicaBasedOnCount(clusterState, message, results, onComplete, parallel); + deleteReplicaBasedOnCount(adminCmdContext, message, results, onComplete, parallel); return; } @@ -98,14 +95,14 @@ void deleteReplica( collectionName = extCollectionName; } - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); Slice slice = coll.getSlice(shard); if (slice == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Invalid shard name : " + shard + " in collection : " + collectionName); } - deleteCore(coll, shard, replicaName, message, results, onComplete, parallel, true); + deleteCore(adminCmdContext, coll, shard, replicaName, message, results, onComplete, parallel, true); } /** @@ -113,7 +110,7 @@ void deleteReplica( * deletes given num replicas across all shards for the given collection. */ void deleteReplicaBasedOnCount( - ClusterState clusterState, + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results, Runnable onComplete, @@ -123,7 +120,7 @@ void deleteReplicaBasedOnCount( int count = Integer.parseInt(message.getStr(COUNT_PROP)); String collectionName = message.getStr(COLLECTION_PROP); String shard = message.getStr(SHARD_ID_PROP); - DocCollection coll = clusterState.getCollection(collectionName); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); Slice slice = null; // Validate if shard is passed. if (shard != null) { @@ -173,7 +170,7 @@ void deleteReplicaBasedOnCount( for (String replica : replicas) { log.debug("Deleting replica {} for shard {} based on count {}", replica, shardId, count); // don't verify with the placement plugin - we already did it - deleteCore(coll, shardId, replica, message, results, onComplete, parallel, false); + deleteCore(adminCmdContext, coll, shardId, replica, message, results, onComplete, parallel, false); } results.add("shard_id", shardId); results.add("replicas_deleted", replicas); @@ -243,6 +240,7 @@ private void validateReplicaAvailability( } void deleteCore( + AdminCmdContext adminCmdContext, DocCollection coll, String shardId, String replicaName, @@ -296,7 +294,6 @@ void deleteCore( ShardHandler shardHandler = ccc.newShardHandler(); String core = replica.getStr(ZkStateReader.CORE_NAME_PROP); - String asyncId = message.getStr(ASYNC); ModifiableSolrParams params = new ModifiableSolrParams(); params.add(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString()); @@ -312,7 +309,7 @@ void deleteCore( boolean isLive = ccc.getZkStateReader().getClusterState().getLiveNodes().contains(replica.getNodeName()); final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); if (isLive) { shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java index 25b4fa45da65..0c18c08c4af1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java @@ -20,8 +20,8 @@ import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -35,7 +35,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; @@ -58,9 +57,7 @@ public DeleteShardCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP); String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); @@ -74,7 +71,7 @@ public void call( } log.info("Delete shard invoked"); - Slice slice = clusterState.getCollection(collectionName).getSlice(sliceId); + Slice slice = adminCmdContext.getClusterState().getCollection(collectionName).getSlice(sliceId); if (slice == null) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -126,14 +123,12 @@ public void call( } } - String asyncId = message.getStr(ASYNC); - try { List replicas = getReplicasForSlice(collectionName, slice); CountDownLatch cleanupLatch = new CountDownLatch(replicas.size()); for (ZkNodeProps r : replicas) { final ZkNodeProps replica = - r.plus(message.getProperties()).plus("parallel", "true").plus(ASYNC, asyncId); + r.plus(message.getProperties()).plus("parallel", "true"); if (log.isInfoEnabled()) { log.info( "Deleting replica for collection={} shard={} on node={}", @@ -145,7 +140,7 @@ public void call( try { new DeleteReplicaCmd(ccc) .deleteReplica( - clusterState, + adminCmdContext.subRequestContext(DELETEREPLICA), replica, deleteResult, () -> { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index 71678f2eb4a2..1a489722a2ea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -31,7 +30,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; import org.apache.solr.common.cloud.Slice; @@ -60,9 +58,7 @@ public DeleteSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); String collectionName; @@ -72,7 +68,6 @@ public void call( collectionName = extCollectionName; } String commitName = message.getStr(CoreAdminParams.COMMIT_NAME); - String asyncId = message.getStr(ASYNC); NamedList shardRequestResults = new NamedList<>(); ShardHandler shardHandler = ccc.newShardHandler(); SolrZkClient zkClient = ccc.getZkStateReader().getZkClient(); @@ -102,7 +97,7 @@ public void call( } final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); log.info( "Existing cores with snapshot for collection={} are {}", collectionName, existingCores); for (Slice slice : diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 5487d63ced46..a2185412c50d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -22,7 +22,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; @@ -46,7 +45,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ConfigSetParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.ExecutorUtil; @@ -426,9 +424,8 @@ public OverseerSolrResponse call() { CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(adminCmdContext.getAction()); if (command != null) { - adminCmdContext.setClusterState(ccc.getSolrCloudManager().getClusterState()); command.call( - adminCmdContext, message, results); + adminCmdContext.withClusterState(ccc.getSolrCloudManager().getClusterState()), message, results); } else { asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); // Seeing this is a bug, not bad user data diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 061bed68025d..e6db84478038 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -59,13 +59,11 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { @Override @SuppressWarnings("unchecked") - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { final RemoteMessage typedMessage = new ObjectMapper().convertValue(message.getProperties(), RemoteMessage.class); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(typedMessage.asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); final ClusterState clusterState = ccc.getZkStateReader().getClusterState(); typedMessage.validate(); @@ -99,8 +97,6 @@ public void call( /** A value-type representing the message received by {@link InstallShardDataCmd} */ public static class RemoteMessage implements JacksonReflectMapWriter { - @JsonProperty public String callingLockId; - @JsonProperty(QUEUE_OPERATION) public String operation = CollectionParams.CollectionAction.INSTALLSHARDDATA.toLower(); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index df37051e757b..921c0498e2d2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.cloud.Overseer; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterState; @@ -55,8 +54,7 @@ public class MaintainRoutedAliasCmd extends AliasCmd { * standard OCP timeout) to prevent large batches of add's from sending a message to the overseer * for every document added in RoutedAliasUpdateProcessor. */ - static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) - throws Exception { + static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) throws Exception { final SolrResponse rsp = collHandler.submitCollectionApiCommand( new AdminCmdContext(CollectionParams.CollectionAction.MAINTAINROUTEDALIAS), @@ -108,7 +106,7 @@ private void removeCollectionFromAlias( @Override public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { // ---- PARSE PRIMARY MESSAGE PARAMS // important that we use NAME for the alias as that is what the Overseer will get a lock on @@ -144,8 +142,7 @@ public void call( .execute( () -> { try { - deleteTargetCollection( - clusterState, results, aliasName, aliasesManager, action, lockId); + deleteTargetCollection(adminCmdContext, results, aliasName, aliasesManager, action); } catch (Exception e) { log.warn( "Deletion of {} by {} {} failed (this might be ok if two clients were", @@ -160,7 +157,7 @@ public void call( case ENSURE_EXISTS: if (!exists) { addTargetCollection( - clusterState, results, aliasName, aliasesManager, aliasMetadata, action, lockId); + adminCmdContext, results, aliasName, aliasesManager, aliasMetadata, action); } else { // check that the collection is properly integrated into the alias (see // TimeRoutedAliasUpdateProcessorTest.java:141). Presently we need to ensure inclusion @@ -188,17 +185,16 @@ public void call( } public void addTargetCollection( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, Map aliasMetadata, - RoutedAlias.Action action, - String lockId) + RoutedAlias.Action action) throws Exception { NamedList createResults = createCollectionAndWait( - clusterState, aliasName, aliasMetadata, action.targetCollection, lockId, ccc); + adminCmdContext, aliasName, aliasMetadata, action.targetCollection, ccc); if (createResults != null) { results.add("create", createResults); } @@ -206,12 +202,11 @@ public void addTargetCollection( } public void deleteTargetCollection( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, - RoutedAlias.Action action, - String lockId) + RoutedAlias.Action action) throws Exception { Map delProps = new HashMap<>(); delProps.put( @@ -220,6 +215,6 @@ public void deleteTargetCollection( () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection)); delProps.put(NAME, action.targetCollection); ZkNodeProps messageDelete = new ZkNodeProps(delProps); - new DeleteCollectionCmd(ccc).call(clusterState, messageDelete, lockId, results); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE), messageDelete, results); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index cc54aa9aa3df..1c0a34c1a02a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -23,7 +23,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.lang.invoke.MethodHandles; @@ -65,9 +64,7 @@ public MigrateCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extSourceCollectionName = message.getStr("collection"); String splitKey = message.getStr("split.key"); String extTargetCollectionName = message.getStr("target.collection"); @@ -90,13 +87,13 @@ public void call( targetCollectionName = extTargetCollectionName; } - DocCollection sourceCollection = clusterState.getCollection(sourceCollectionName); + DocCollection sourceCollection = adminCmdContext.getClusterState().getCollection(sourceCollectionName); if (sourceCollection == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName); } - DocCollection targetCollection = clusterState.getCollection(targetCollectionName); + DocCollection targetCollection = adminCmdContext.getClusterState().getCollection(targetCollectionName); if (targetCollection == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -137,9 +134,6 @@ public void call( + splitKey); } - String asyncId = null; - if (message.containsKey(ASYNC) && message.get(ASYNC) != null) asyncId = message.getStr(ASYNC); - for (Slice sourceSlice : sourceSlices) { for (Slice targetSlice : targetSlices) { log.info( @@ -148,7 +142,7 @@ public void call( targetSlice, splitKey); migrateKey( - clusterState, + adminCmdContext, sourceCollection, sourceSlice, targetCollection, @@ -156,15 +150,13 @@ public void call( splitKey, timeout, results, - asyncId, - message, - lockId); + message); } } } private void migrateKey( - ClusterState clusterState, + AdminCmdContext adminCmdContext, DocCollection sourceCollection, Slice sourceSlice, DocCollection targetCollection, @@ -172,22 +164,17 @@ private void migrateKey( String splitKey, int timeout, NamedList results, - String asyncId, - ZkNodeProps message, - String lockId) + ZkNodeProps message) throws Exception { String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName(); ZkStateReader zkStateReader = ccc.getZkStateReader(); - if (clusterState.hasCollection(tempSourceCollectionName)) { + if (adminCmdContext.getClusterState().hasCollection(tempSourceCollectionName)) { log.info("Deleting temporary collection: {}", tempSourceCollectionName); - Map props = - Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); - try { new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); - clusterState = zkStateReader.getClusterState(); + .call(adminCmdContext.subRequestContext(DELETE).withClusterState(zkStateReader.getClusterState()), new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), results); + adminCmdContext.withClusterState(zkStateReader.getClusterState()); } catch (Exception e) { log.warn( "Unable to clean up existing temporary collection: {}", tempSourceCollectionName, e); @@ -237,7 +224,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( @@ -315,17 +302,17 @@ private void migrateKey( configName, CollectionHandlingUtils.CREATE_NODE_SET, sourceLeader.getNodeName()); - if (asyncId != null) { - String internalAsyncId = asyncId + Math.abs(System.nanoTime()); - props.put(ASYNC, internalAsyncId); + String internalAsyncId = null; + if (adminCmdContext.getAsyncId() != null) { + internalAsyncId = adminCmdContext.getAsyncId() + Math.abs(System.nanoTime()); } log.info("Creating temporary collection: {}", props); - new CreateCollectionCmd(ccc).call(clusterState, new ZkNodeProps(props), lockId, results); + new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CREATE, internalAsyncId), new ZkNodeProps(props), results); // refresh cluster state - clusterState = zkStateReader.getClusterState(); + adminCmdContext.withClusterState(zkStateReader.getClusterState()); Slice tempSourceSlice = - clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next(); + adminCmdContext.getClusterState().getCollection(tempSourceCollectionName).getSlices().iterator().next(); Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 120000); @@ -352,7 +339,7 @@ private void migrateKey( cmd.setOnlyIfLeader(true); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); // we don't want this to happen asynchronously syncRequestTracker.sendShardRequest( tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler); @@ -377,7 +364,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(tempNodeName, params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command"); @@ -395,7 +382,6 @@ private void migrateKey( tempSourceSlice.getName(), Replica.Type.defaultType()); props = new HashMap<>(); - props.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower()); props.put(COLLECTION_PROP, tempSourceCollectionName); props.put(SHARD_ID_PROP, tempSourceSlice.getName()); props.put("node", targetLeader.getNodeName()); @@ -407,14 +393,11 @@ private void migrateKey( } } // add async param - if (asyncId != null) { - props.put(ASYNC, asyncId); - } - new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(props), results, null); + new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(props), results, null); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); syncRequestTracker.processResponses( results, shardHandler, @@ -445,7 +428,7 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( @@ -464,8 +447,7 @@ private void migrateKey( params.set(CoreAdminParams.SRC_CORE, tempCollectionReplica2); { - final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); String msg = @@ -485,16 +467,15 @@ private void migrateKey( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); shardRequestTracker.processResponses( results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates"); } try { log.info("Deleting temporary collection: {}", tempSourceCollectionName); - props = Map.of(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, tempSourceCollectionName); new DeleteCollectionCmd(ccc) - .call(zkStateReader.getClusterState(), new ZkNodeProps(props), lockId, results); + .call(adminCmdContext.subRequestContext(DELETE).withClusterState(zkStateReader.getClusterState()), new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), results); } catch (Exception e) { log.error( "Unable to delete temporary collection: {}. Please remove it manually", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java index c5080438a8a9..963a2215af5d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -48,9 +46,7 @@ public MigrateReplicasCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); Set sourceNodes = getNodesFromParam(message, CollectionParams.SOURCE_NODES); Set targetNodes = getNodesFromParam(message, CollectionParams.TARGET_NODES); @@ -59,7 +55,6 @@ public void call( throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "sourceNodes is a required param"); } - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); ClusterState clusterState = zkStateReader.getClusterState(); @@ -122,7 +117,7 @@ public void call( boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 1bacdcd3eeeb..9c2b0b43ee47 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -22,7 +22,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE; import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; @@ -60,14 +59,11 @@ public MoveReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { - moveReplica(ccc.getZkStateReader().getClusterState(), message, results); + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + moveReplica(adminCmdContext, message, results); } - private void moveReplica( - ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception { + private void moveReplica(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { if (log.isDebugEnabled()) { log.debug("moveReplica() : {}", Utils.toJSONString(message)); } @@ -78,8 +74,6 @@ private void moveReplica( boolean inPlaceMove = message.getBool(IN_PLACE_MOVE, true); int timeout = message.getInt(TIMEOUT, 10 * 60); // 10 minutes - String async = message.getStr(ASYNC); - boolean followAliases = message.getBool(FOLLOW_ALIASES, false); String collection; if (followAliases) { @@ -89,15 +83,15 @@ private void moveReplica( collection = extCollection; } - DocCollection coll = clusterState.getCollection(collection); + DocCollection coll = adminCmdContext.getClusterState().getCollection(collection); if (coll == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist"); } - if (!clusterState.getLiveNodes().contains(targetNode)) { + if (!adminCmdContext.getClusterState().getLiveNodes().contains(targetNode)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Target node: " + targetNode + " not in live nodes: " + clusterState.getLiveNodes()); + "Target node: " + targetNode + " not in live nodes: " + adminCmdContext.getClusterState().getLiveNodes()); } Replica replica = null; if (message.containsKey(REPLICA_PROP)) { @@ -157,21 +151,19 @@ private void moveReplica( if (isSharedFS && inPlaceMove) { log.debug("-- moveSharedFsReplica"); moveSharedFsReplica( - clusterState, + adminCmdContext, results, dataDir.toString(), targetNode, - async, replica, timeout, waitForFinalState); } else { log.debug("-- moveNormalReplica (inPlaceMove={}, isSharedFS={}", inPlaceMove, isSharedFS); moveNormalReplica( - clusterState, + adminCmdContext, results, targetNode, - async, coll, replica, slice, @@ -181,17 +173,16 @@ private void moveReplica( } private void moveSharedFsReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String dataDir, String targetNode, - String async, Replica replica, int timeout, boolean waitForFinalState) throws Exception { String skipCreateReplicaInClusterState = "true"; - if (clusterState.getLiveNodes().contains(replica.getNodeName())) { + if (adminCmdContext.getClusterState().getLiveNodes().contains(replica.getNodeName())) { skipCreateReplicaInClusterState = "false"; ZkNodeProps removeReplicasProps = new ZkNodeProps( @@ -200,11 +191,10 @@ private void moveSharedFsReplica( REPLICA_PROP, replica.getName()); removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_DATA_DIR, false); removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_INDEX, false); - if (async != null) removeReplicasProps.getProperties().put(ASYNC, async); NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(clusterState, removeReplicasProps, deleteResult, null); + .deleteReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), removeReplicasProps, deleteResult, null); } catch (SolrException e) { // assume this failed completely so there's nothing to roll back deleteResult.add("failure", e.toString()); @@ -260,11 +250,10 @@ private void moveSharedFsReplica( ZkStateReader.REPLICA_TYPE, replica.getType().name()); - if (async != null) addReplicasProps.getProperties().put(ASYNC, async); NamedList addResult = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, addResult, null); + .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, addResult, null); } catch (Exception e) { // fatal error - try rolling back String errorString = @@ -280,7 +269,7 @@ private void moveSharedFsReplica( addReplicasProps = addReplicasProps.plus(CoreAdminParams.NODE, replica.getNodeName()); NamedList rollback = new NamedList<>(); new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, rollback, null); + .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, rollback, null); if (rollback.get("failure") != null) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -308,7 +297,7 @@ private void moveSharedFsReplica( NamedList rollback = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(ccc.getZkStateReader().getClusterState(), addReplicasProps, rollback, null); + .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, rollback, null); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -338,10 +327,9 @@ private void moveSharedFsReplica( } private void moveNormalReplica( - ClusterState clusterState, + AdminCmdContext adminCmdContext, NamedList results, String targetNode, - String async, DocCollection coll, Replica replica, Slice slice, @@ -367,12 +355,11 @@ private void moveNormalReplica( ZkStateReader.REPLICA_TYPE, replica.getType().name()); - if (async != null) addReplicasProps.getProperties().put(ASYNC, async); NamedList addResult = new NamedList<>(); SolrCloseableLatch countDownLatch = new SolrCloseableLatch(1, ccc.getCloseableToLatchOn()); ActiveReplicaWatcher watcher = null; ZkNodeProps props = - new AddReplicaCmd(ccc).addReplica(clusterState, addReplicasProps, addResult, null).get(0); + new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA), addReplicasProps, addResult, null).get(0); log.debug("props {}", props); if (replica.equals(slice.getLeader()) || waitForFinalState) { watcher = @@ -428,11 +415,10 @@ private void moveNormalReplica( COLLECTION_PROP, coll.getName(), SHARD_ID_PROP, slice.getName(), REPLICA_PROP, replica.getName()); - if (async != null) removeReplicasProps.getProperties().put(ASYNC, async); NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(clusterState, removeReplicasProps, deleteResult, null); + .deleteReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), removeReplicasProps, deleteResult, null); } catch (SolrException e) { deleteResult.add("failure", e.toString()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index 4b6a1adb708e..8573bf637643 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -21,11 +21,14 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -41,11 +44,11 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.TimeSource; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.logging.MDCLoggingContext; @@ -128,7 +131,11 @@ public OverseerSolrResponse processMessage(ZkNodeProps message, String operation CollectionAction action = getCollectionAction(operation); CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - command.call(cloudManager.getClusterState(), message, lock.id(), results); + AdminCmdContext adminCmdContext = new AdminCmdContext(action, message.getStr(ASYNC)); + adminCmdContext.setLockId(lock.id()); + adminCmdContext.setCallingLockIds(message.getStr(CALLING_LOCK_IDS_HEADER)); + adminCmdContext.withClusterState(cloudManager.getClusterState()); + command.call(adminCmdContext, message, results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); } @@ -189,13 +196,18 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId) { sessionId = batchSessionId; } + List callingLockIds = null; + String callingLockIdsString = message.getStr(CALLING_LOCK_IDS_HEADER); + if (StrUtils.isNotBlank(callingLockIdsString)) { + callingLockIds = List.of(callingLockIdsString.split(",")); + } return lockSession.lock( getCollectionAction(message.getStr(Overseer.QUEUE_OPERATION)), Arrays.asList( getTaskKey(message), message.getStr(ZkStateReader.SHARD_ID_PROP), message.getStr(ZkStateReader.REPLICA_PROP)), - message.getStr(CollectionAdminParams.CALLING_LOCK_ID)); + callingLockIds); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java index 2009609cee96..9fad543952ad 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import org.apache.solr.cloud.OverseerNodePrioritizer; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -55,9 +54,7 @@ public OverseerRoleCmd( } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { if (ccc.isDistributedCollectionAPI()) { // No Overseer (not accessible from Collection API command execution in any case) so this // command can't be run... diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java index 489e8eda501a..2b1835466b2f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.solr.cloud.OverseerTaskProcessor; import org.apache.solr.cloud.Stats; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -157,9 +156,7 @@ public OverseerStatusCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { // If Collection API execution is distributed, we're not running on the Overseer node so can't // return any Overseer stats. if (ccc.getCoreContainer().getZkController().getDistributedCommandRunner().isPresent()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index a3f5a276e3a1..7fd87650dd27 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -169,9 +169,7 @@ public ReindexCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { log.debug("*** called: {}", message); @@ -189,6 +187,7 @@ public void call( } else { collection = extCollection; } + ClusterState clusterState = ccc.getZkStateReader().getClusterState(); if (!clusterState.hasCollection(collection)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Source collection name must exist"); @@ -294,13 +293,7 @@ public void call( } if (clusterState.hasCollection(chkCollection)) { // delete the checkpoint collection - cmd = - new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), - CommonParams.NAME, - chkCollection); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), new ZkNodeProps(CommonParams.NAME, chkCollection), cmdResults); CollectionHandlingUtils.checkResults( "deleting old checkpoint collection " + chkCollection, cmdResults, true); } @@ -311,7 +304,6 @@ public void call( } Map propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower()); propMap.put(CommonParams.NAME, targetCollection); propMap.put(ZkStateReader.NUM_SHARDS_PROP, numShards); propMap.put(CollectionAdminParams.COLL_CONF, configName); @@ -339,7 +331,7 @@ public void call( // create the target collection cmd = new ZkNodeProps(propMap); cmdResults = new NamedList<>(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); createdTarget = true; CollectionHandlingUtils.checkResults( "creating target collection " + targetCollection, cmdResults, true); @@ -347,14 +339,13 @@ public void call( // create the checkpoint collection - use RF=1 and 1 shard cmd = new ZkNodeProps( - Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), CommonParams.NAME, chkCollection, ZkStateReader.NUM_SHARDS_PROP, "1", ZkStateReader.REPLICATION_FACTOR, "1", CollectionAdminParams.COLL_CONF, "_default", CommonAdminParams.WAIT_FOR_FINAL_STATE, "true"); cmdResults = new NamedList<>(); - new CreateCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); CollectionHandlingUtils.checkResults( "creating checkpoint collection " + chkCollection, cmdResults, true); // wait for a while until we see both collections @@ -367,8 +358,6 @@ public void call( SolrException.ErrorCode.SERVER_ERROR, "Could not fully create temporary collection(s)"); } - clusterState = ccc.getSolrCloudManager().getClusterState(); - if (maybeAbort(collection)) { aborted = true; return; @@ -481,7 +470,7 @@ public void call( log.debug("- setting up alias from {} to {}", extCollection, targetCollection); cmd = new ZkNodeProps(CommonParams.NAME, extCollection, "collections", targetCollection); cmdResults = new NamedList<>(); - new CreateAliasCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new CreateAliasCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATEALIAS, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); CollectionHandlingUtils.checkResults( "setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true); reindexingState.put("alias", extCollection + " -> " + targetCollection); @@ -499,14 +488,8 @@ public void call( } // 6. delete the checkpoint collection log.debug("- deleting {}", chkCollection); - cmd = - new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), - CommonParams.NAME, - chkCollection); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), new ZkNodeProps(CommonParams.NAME, chkCollection), cmdResults); CollectionHandlingUtils.checkResults( "deleting checkpoint collection " + chkCollection, cmdResults, true); @@ -515,14 +498,12 @@ public void call( log.debug("- deleting source collection"); cmd = new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), CommonParams.NAME, collection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); CollectionHandlingUtils.checkResults( "deleting source collection " + collection, cmdResults, true); } else { @@ -576,13 +557,13 @@ public void call( } finally { if (aborted) { cleanup( + adminCmdContext, collection, targetCollection, chkCollection, daemonReplica, targetCollection, - createdTarget, - lockId); + createdTarget); if (exc != null) { results.add("error", exc.toString()); } @@ -871,13 +852,13 @@ private NamedList executeDaemonAction( } private void cleanup( + AdminCmdContext adminCmdContext, String collection, String targetCollection, String chkCollection, Replica daemonReplica, String daemonName, - boolean createdTarget, - String lockId) + boolean createdTarget) throws Exception { log.info("## Cleaning up after abort or error"); // 1. kill the daemon @@ -895,13 +876,11 @@ private void cleanup( log.debug(" -- removing {}", targetCollection); ZkNodeProps cmd = new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), CommonParams.NAME, targetCollection, FOLLOW_ALIASES, "false"); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), cmd, cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting target collection " + targetCollection, cmdResults, false); } @@ -910,14 +889,12 @@ private void cleanup( log.debug(" -- removing {}", chkCollection); ZkNodeProps cmd = new ZkNodeProps( - Overseer.QUEUE_OPERATION, - CollectionParams.CollectionAction.DELETE.toLower(), CommonParams.NAME, chkCollection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(clusterState, cmd, lockId, cmdResults); + new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), cmd, cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java index d76f74f3e653..ff99d0f8a43b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java @@ -22,7 +22,6 @@ import java.lang.invoke.MethodHandles; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; @@ -41,9 +40,7 @@ public RenameCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String extCollectionName = message.getStr(CoreAdminParams.NAME); String target = message.getStr(CollectionAdminParams.TARGET); @@ -65,7 +62,7 @@ public void call( } else { collectionName = extCollectionName; } - if (!state.hasCollection(collectionName)) { + if (!adminCmdContext.getClusterState().hasCollection(collectionName)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "source collection '" + collectionName + "' not found."); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index 0067a6708ccc..f44a9e092033 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,9 +43,7 @@ public ReplaceNodeCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); String source = message.getStr(CollectionParams.SOURCE_NODE); String target = message.getStr(CollectionParams.TARGET_NODE); @@ -56,7 +52,6 @@ public void call( throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "sourceNode is a required param"); } - String async = message.getStr(ASYNC); int timeout = message.getInt("timeout", 10 * 60); // 10 minutes boolean parallel = message.getBool("parallel", false); ClusterState clusterState = zkStateReader.getClusterState(); @@ -108,7 +103,7 @@ public void call( boolean migrationSuccessful = ReplicaMigrationUtils.migrateReplicas( - ccc, replicaMovements, parallel, waitForFinalState, timeout, async, results); + ccc, adminCmdContext, replicaMovements, parallel, waitForFinalState, timeout, results); if (migrationSuccessful) { results.add( "success", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java index d21aa3a5f953..7ac51487bc87 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -37,6 +35,7 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.zookeeper.KeeperException; @@ -53,22 +52,22 @@ public class ReplicaMigrationUtils { *

If an error occurs during the creation of new replicas, all new replicas will be deleted. * * @param ccc The collection command context to use from the API that calls this method + * @param adminCmdContext The context of the originating admin request * @param movements a map from replica to the new node that the replica should live on * @param parallel whether the replica creations should be done in parallel * @param waitForFinalState wait for the final state of all newly created replicas before * continuing * @param timeout the amount of time to wait for new replicas to be created - * @param asyncId If provided, the command will be run under the given asyncId * @param results push results (successful and failure) onto this list * @return whether the command was successful */ static boolean migrateReplicas( CollectionCommandContext ccc, + AdminCmdContext adminCmdContext, Map movements, boolean parallel, boolean waitForFinalState, int timeout, - String asyncId, NamedList results) throws IOException, InterruptedException, KeeperException { // how many leaders are we moving? for these replicas we have to make sure that either: @@ -92,8 +91,6 @@ static boolean migrateReplicas( SolrCloseableLatch replicasToRecover = new SolrCloseableLatch(numLeaders, ccc.getCloseableToLatchOn()); - ClusterState clusterState = ccc.getZkStateReader().getClusterState(); - for (Map.Entry movement : movements.entrySet()) { Replica sourceReplica = movement.getKey(); String targetNode = movement.getValue(); @@ -111,12 +108,11 @@ static boolean migrateReplicas( .toFullProps() .plus("parallel", String.valueOf(parallel)) .plus(CoreAdminParams.NODE, targetNode); - if (asyncId != null) msg.getProperties().put(ASYNC, asyncId); NamedList nl = new NamedList<>(); final ZkNodeProps addedReplica = new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), msg, nl, () -> { @@ -213,7 +209,7 @@ static boolean migrateReplicas( try { new DeleteReplicaCmd(ccc) .deleteReplica( - ccc.getZkStateReader().getClusterState(), + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), createdReplica.plus("parallel", "true"), deleteResult, () -> { @@ -242,15 +238,14 @@ static boolean migrateReplicas( // we have reached this far, meaning all replicas should have been recreated. // now cleanup the original replicas return cleanupReplicas( - results, ccc.getZkStateReader().getClusterState(), movements.keySet(), ccc, asyncId); + results, adminCmdContext, movements.keySet(), ccc); } static boolean cleanupReplicas( NamedList results, - ClusterState clusterState, + AdminCmdContext adminCmdContext, Collection sourceReplicas, - CollectionCommandContext ccc, - String async) + CollectionCommandContext ccc) throws IOException, InterruptedException { SolrCloseableLatch cleanupLatch = new SolrCloseableLatch(sourceReplicas.size(), ccc.getCloseableToLatchOn()); @@ -268,10 +263,9 @@ static boolean cleanupReplicas( NamedList deleteResult = new NamedList<>(); try { ZkNodeProps cmdMessage = sourceReplica.toFullProps(); - if (async != null) cmdMessage = cmdMessage.plus(ASYNC, async); new DeleteReplicaCmd(ccc) .deleteReplica( - clusterState, + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), cmdMessage.plus("parallel", "true"), deleteResult, () -> { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 11ab41495392..0019d5855c5c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -21,9 +21,10 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.Closeable; @@ -92,11 +93,9 @@ public RestoreCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { - try (RestoreContext restoreContext = new RestoreContext(message, lockId, ccc)) { - if (state.hasCollection(restoreContext.restoreCollectionName)) { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + try (RestoreContext restoreContext = new RestoreContext(adminCmdContext, message, ccc)) { + if (adminCmdContext.getClusterState().hasCollection(restoreContext.restoreCollectionName)) { RestoreOnExistingCollection restoreOnExistingCollection = new RestoreOnExistingCollection(restoreContext); restoreOnExistingCollection.process(restoreContext, results); @@ -112,14 +111,13 @@ public void call( private void requestReplicasToRestore( NamedList results, DocCollection restoreCollection, - ClusterState clusterState, + AdminCmdContext adminCmdContext, BackupProperties backupProperties, URI backupPath, String repo, - ShardHandler shardHandler, - String asyncId) { + ShardHandler shardHandler) { ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); // Copy data from backed up index to each replica for (Slice slice : restoreCollection.getSlices()) { ModifiableSolrParams params = new ModifiableSolrParams(); @@ -132,7 +130,7 @@ private void requestReplicasToRestore( } params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); + shardRequestTracker.sliceCmd(adminCmdContext.getClusterState(), params, null, slice, shardHandler); } shardRequestTracker.processResponses( new NamedList<>(), shardHandler, true, "Could not restore core"); @@ -141,10 +139,10 @@ private void requestReplicasToRestore( /** Encapsulates the parsing and access for common parameters restore parameters and values */ private static class RestoreContext implements Closeable { + final AdminCmdContext adminCmdContext; final String restoreCollectionName; final String backupName; final String backupCollection; - final String asyncId; final String repo; final String restoreConfigName; final int backupId; @@ -152,7 +150,6 @@ private static class RestoreContext implements Closeable { final URI backupPath; final List nodeList; - final String lockId; final CoreContainer container; final BackupRepository repository; final ZkStateReader zkStateReader; @@ -161,15 +158,14 @@ private static class RestoreContext implements Closeable { final DocCollection backupCollectionState; final ShardHandler shardHandler; - private RestoreContext(ZkNodeProps message, String lockId, CollectionCommandContext ccc) + private RestoreContext(AdminCmdContext adminCmdContext, ZkNodeProps message, CollectionCommandContext ccc) throws IOException { + this.adminCmdContext = adminCmdContext; this.restoreCollectionName = message.getStr(COLLECTION_PROP); this.backupName = message.getStr(NAME); // of backup - this.asyncId = message.getStr(ASYNC); this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1); - this.lockId = lockId; this.container = ccc.getCoreContainer(); this.repository = this.container.newBackupRepository(repo); @@ -240,11 +236,10 @@ public void process(NamedList results, RestoreContext rc) throws Excepti rc.backupName, rc.location); createCoreLessCollection( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.restoreCollectionName, rc.restoreConfigName, - rc.backupCollectionState, - rc.zkStateReader.getClusterState(), - rc.lockId); + rc.backupCollectionState); // note: when createCollection() returns, the collection exists (no race) // Restore collection properties @@ -253,8 +248,6 @@ public void process(NamedList results, RestoreContext rc) throws Excepti DocCollection restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName); markAllShardsAsConstruction(restoreCollection); - // TODO how do we leverage the RULE / SNITCH logic in createCollection? - ClusterState clusterState = rc.zkStateReader.getClusterState(); List sliceNames = new ArrayList<>(); restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName())); @@ -263,11 +256,11 @@ public void process(NamedList results, RestoreContext rc) throws Excepti getReplicaPositions(rc.restoreCollectionName, rc.nodeList, sliceNames); createSingleReplicaPerShard( - results, restoreCollection, rc.asyncId, clusterState, replicaPositions); + results, restoreCollection, rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), replicaPositions); Object failures = results.get("failure"); if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { log.error("Restore failed to create initial replicas."); - CollectionHandlingUtils.cleanupCollection(rc.restoreCollectionName, new NamedList<>(), ccc); + CollectionHandlingUtils.cleanupCollection(rc.adminCmdContext, rc.restoreCollectionName, new NamedList<>(), ccc); return; } @@ -277,14 +270,13 @@ public void process(NamedList results, RestoreContext rc) throws Excepti requestReplicasToRestore( results, restoreCollection, - clusterState, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.backupProperties, rc.backupPath, rc.repo, - rc.shardHandler, - rc.asyncId); + rc.shardHandler); markAllShardsAsActive(restoreCollection); - addReplicasToShards(results, clusterState, restoreCollection, replicaPositions, rc.asyncId); + addReplicasToShards(results, restoreCollection, replicaPositions, rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState())); restoringAlias(rc.backupProperties); log.info("Completed restoring collection={} backupName={}", restoreCollection, rc.backupName); @@ -315,11 +307,10 @@ private void uploadConfig( } private void createCoreLessCollection( + AdminCmdContext adminCmdContext, String restoreCollectionName, String restoreConfigName, - DocCollection backupCollectionState, - ClusterState clusterState, - String lockId) + DocCollection backupCollectionState) throws Exception { Map propMap = new HashMap<>(); propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString()); @@ -375,7 +366,7 @@ private void createCoreLessCollection( } new CreateCollectionCmd(ccc) - .call(clusterState, new ZkNodeProps(propMap), lockId, new NamedList<>()); + .call(adminCmdContext.subRequestContext(CREATE, null), new ZkNodeProps(propMap), new NamedList<>()); // note: when createCollection() returns, the collection exists (no race) } @@ -418,8 +409,7 @@ private List getReplicaPositions( private void createSingleReplicaPerShard( NamedList results, DocCollection restoreCollection, - String asyncId, - ClusterState clusterState, + AdminCmdContext adminCmdContext, List replicaPositions) throws Exception { CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); @@ -445,15 +435,11 @@ private void createSingleReplicaPerShard( } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } CollectionHandlingUtils.addPropertyParams(message, propMap); final NamedList addReplicaResult = new NamedList<>(); new AddReplicaCmd(ccc) .addReplica( - clusterState, + adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(propMap), addReplicaResult, () -> { @@ -515,10 +501,9 @@ private void markAllShardsAsActive(DocCollection restoreCollection) private void addReplicasToShards( NamedList results, - ClusterState clusterState, DocCollection restoreCollection, List replicaPositions, - String asyncId) + AdminCmdContext adminCmdContext) throws Exception { int totalReplicasPerShard = numReplicas.total(); if (totalReplicasPerShard > 1) { @@ -565,14 +550,10 @@ private void addReplicasToShards( } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } CollectionHandlingUtils.addPropertyParams(message, propMap); new AddReplicaCmd(ccc) - .addReplica(clusterState, new ZkNodeProps(propMap), results, null); + .addReplica(adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(propMap), results, null); } } } @@ -623,24 +604,22 @@ public void process(RestoreContext rc, NamedList results) throws Excepti ClusterState clusterState = rc.zkStateReader.getClusterState(); DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); - enableReadOnly(clusterState, restoreCollection, rc.lockId); + enableReadOnly(rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), restoreCollection); try { requestReplicasToRestore( results, restoreCollection, - clusterState, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), rc.backupProperties, rc.backupPath, rc.repo, - rc.shardHandler, - rc.asyncId); + rc.shardHandler); } finally { - disableReadOnly(clusterState, restoreCollection, rc.lockId); + disableReadOnly(rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), restoreCollection); } } - private void disableReadOnly( - ClusterState clusterState, DocCollection restoreCollection, String lockId) + private void disableReadOnly(AdminCmdContext adminCmdContext, DocCollection restoreCollection) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -649,11 +628,10 @@ private void disableReadOnly( ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, null); new CollApiCmds.ModifyCollectionCmd(ccc) - .call(clusterState, params, lockId, new NamedList<>()); + .call(adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } - private void enableReadOnly( - ClusterState clusterState, DocCollection restoreCollection, String lockId) + private void enableReadOnly(AdminCmdContext adminCmdContext, DocCollection restoreCollection) throws Exception { ZkNodeProps params = new ZkNodeProps( @@ -662,7 +640,7 @@ private void enableReadOnly( ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, "true"); new CollApiCmds.ModifyCollectionCmd(ccc) - .call(clusterState, params, lockId, new NamedList<>()); + .call(adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java index fe20b9e30a54..eac8df809e6a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java @@ -26,7 +26,6 @@ import java.util.Locale; import java.util.Map; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; @@ -46,9 +45,7 @@ public SetAliasPropCmd(CollectionCommandContext ccc) { } @Override - public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) - throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { String aliasName = message.getStr(NAME); final ZkStateReader.AliasesManager aliasesManager = ccc.getZkStateReader().aliasesManager; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 97450bd879e9..83cf10467f01 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -24,7 +24,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS; import java.lang.invoke.MethodHandles; @@ -102,9 +101,9 @@ public SplitShardCmd(CollectionCommandContext ccc) { @Override public void call( - ClusterState state, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - split(state, message, lockId, results); + split(adminCmdContext, message, results); } /** @@ -133,10 +132,8 @@ public void call( * illustrated with diagrams. */ public boolean split( - ClusterState clusterState, ZkNodeProps message, String lockId, NamedList results) + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { - final String asyncId = message.getStr(ASYNC); - boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); String methodStr = message.getStr( @@ -167,6 +164,7 @@ public boolean split( slice.set(message.getStr(ZkStateReader.SHARD_ID_PROP)); Set offlineSlices = new HashSet<>(); RTimerTree timings = new RTimerTree(); + ClusterState clusterState = zkStateReader.getClusterState(); String splitKey = message.getStr("split.key"); DocCollection collection = clusterState.getCollection(collectionName); @@ -277,7 +275,7 @@ public boolean split( { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest( parentShardLeader.getNodeName(), params, shardHandler); SimpleOrderedMap getRangesResults = new SimpleOrderedMap<>(); @@ -340,7 +338,7 @@ public boolean split( propMap.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(propMap); try { - new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList<>()); + new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), m, new NamedList<>()); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -418,17 +416,13 @@ public boolean split( propMap.put(key, message.getStr(key)); } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } new AddReplicaCmd(ccc) - .addReplica(clusterState, new ZkNodeProps((MapWriter) propMap), results, null); + .addReplica(adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), new ZkNodeProps((MapWriter) propMap), results, null); } { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); String msgOnError = "SPLITSHARD failed to create subshard leaders"; syncRequestTracker.processResponses(results, shardHandler, true, msgOnError); handleFailureOnAsyncRequest(results, msgOnError); @@ -439,7 +433,7 @@ public boolean split( t = timings.sub("waitForSubSliceLeadersAlive"); { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (String subShardName : subShardNames) { // wait for parent leader to acknowledge the sub-shard core log.debug( @@ -494,7 +488,7 @@ public boolean split( t = timings.sub("splitParentCore"); { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler); String msgOnError = "SPLITSHARD failed to invoke SPLIT core admin command"; @@ -511,7 +505,7 @@ public boolean split( t = timings.sub("applyBufferedUpdates"); { final ShardRequestTracker shardRequestTracker = - CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc); + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (int i = 0; i < subShardNames.size(); i++) { String subShardName = subShardNames.get(i); @@ -629,10 +623,6 @@ public boolean split( propMap.put(key, message.getStr(key)); } } - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } // special flag param to instruct addReplica not to create the replica in cluster state // again propMap.put(CollectionHandlingUtils.SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, "true"); @@ -777,14 +767,14 @@ public boolean split( t = timings.sub("createCoresForReplicas"); // 11. now actually create replica cores on sub shard nodes for (Map replica : replicas) { - new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(replica), results, null); + new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), new ZkNodeProps(replica), results, null); } assert TestInjection.injectSplitFailureAfterReplicaCreation(); { final ShardRequestTracker syncRequestTracker = - CollectionHandlingUtils.syncRequestTracker(ccc); + CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc); String msgOnError = "SPLITSHARD failed to create subshard replicas"; syncRequestTracker.processResponses(results, shardHandler, true, msgOnError); handleFailureOnAsyncRequest(results, msgOnError); @@ -824,7 +814,7 @@ public boolean split( } finally { if (!success) { cleanupAfterFailure( - zkStateReader, collectionName, parentSlice.getName(), subSlices, offlineSlices, lockId); + adminCmdContext, zkStateReader, collectionName, parentSlice.getName(), subSlices, offlineSlices); unlockForSplit(ccc.getSolrCloudManager(), collectionName, parentSlice.getName()); } } @@ -897,12 +887,12 @@ public static void checkDiskSpace( } private void cleanupAfterFailure( + AdminCmdContext adminCmdContext, ZkStateReader zkStateReader, String collectionName, String parentShard, List subSlices, - Set offlineSlices, - String lockId) { + Set offlineSlices) { log.info("Cleaning up after a failed split of {}/{}", collectionName, parentShard); // get the latest state try { @@ -999,7 +989,7 @@ private void cleanupAfterFailure( props.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(props); try { - new DeleteShardCmd(ccc).call(clusterState, m, lockId, new NamedList()); + new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), m, new NamedList<>()); } catch (Exception e) { log.warn( "Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 2a45ac4ffc7b..76a32ffc5422 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -34,7 +34,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -109,6 +109,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -117,6 +118,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; @@ -319,6 +321,7 @@ void invokeAction( } AdminCmdContext adminCmdContext = new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); + adminCmdContext.setCallingLockIds(req.getContext().get(CALLING_LOCK_IDS_HEADER).toString()); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; @@ -356,15 +359,19 @@ public static SolrResponse submitCollectionApiCommand( // updates (but the other way around is ok). See constructor of CloudConfig. Optional distribCommandRunner = zkController.getDistributedCommandRunner(); - String operation = adminCmdContext.getAction().lowerName; - if (adminCmdContext.getAsyncId() == null) { - m = m.plus(Map.of(QUEUE_OPERATION, operation)); - } else { - m = m.plus(Map.of(QUEUE_OPERATION, operation, ASYNC, adminCmdContext.getAsyncId())); - } if (distribCommandRunner.isPresent()) { return distribCommandRunner.get().runCollectionCommand(adminCmdContext, m, timeout); } else { // Sending the Collection API message to Overseer via a Zookeeper queue + String operation = adminCmdContext.getAction().lowerName; + HashMap additionalProps = new HashMap<>(); + additionalProps.put(QUEUE_OPERATION, operation); + if (adminCmdContext.getAsyncId() != null && !adminCmdContext.getAsyncId().isBlank()) { + additionalProps.put(ASYNC, adminCmdContext.getAsyncId()); + } + if (StringUtils.isNotBlank(adminCmdContext.getCallingLockIds())) { + additionalProps.put(CALLING_LOCK_IDS_HEADER, adminCmdContext.getCallingLockIds()); + } + m = m.plus(additionalProps); if (adminCmdContext.getAsyncId() != null) { String asyncId = adminCmdContext.getAsyncId(); NamedList r = new NamedList<>(); @@ -1063,7 +1070,6 @@ public Map execute( reqBody.location = req.getParams().get(BACKUP_LOCATION); reqBody.name = req.getParams().get(NAME); reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); - reqBody.callingLockId = req.getParams().get(CALLING_LOCK_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index d97b6e5590a2..a3ddeb4524bc 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -118,7 +118,6 @@ public static ZkNodeProps createRemoteMessage( messageTyped.repository = requestBody.repository; messageTyped.name = requestBody.name; messageTyped.shardBackupId = requestBody.shardBackupId; - messageTyped.callingLockId = requestBody.callingLockId; } messageTyped.validate(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index df0f39ccea5c..431652720277 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -188,6 +188,9 @@ private LBSolrClient.Req prepareLBRequest( params.remove(CommonParams.WT); // use default (currently javabin) QueryRequest req = createQueryRequest(sreq, params, shard); req.setMethod(SolrRequest.METHOD.POST); + if (sreq.headers != null) { + req.addHeaders(sreq.headers); + } SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo(); if (requestInfo != null) { req.setUserPrincipal(requestInfo.getUserPrincipal()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java index d7d7da04f224..385e7b720366 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.solr.common.params.ModifiableSolrParams; // todo... when finalized make accessors @@ -56,6 +57,9 @@ public class ShardRequest { /** may be null */ public String nodeName; + /** may be null */ + public Map headers; + // TODO: one could store a list of numbers to correlate where returned docs // go in the top-level response rather than looking up by id... // this would work well if we ever transitioned to using internal ids and diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index 60a111515d8d..629e56384aa6 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -125,5 +125,5 @@ public interface CollectionAdminParams { String PER_REPLICA_STATE = CollectionStateProps.PER_REPLICA_STATE; - String CALLING_LOCK_ID = "callingLockId"; + String CALLING_LOCK_IDS_HEADER = "callingLockIds"; } From e368c48e1fd4aaa01d5e9c339e46351f5050942f Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 9 Jan 2026 16:42:13 -0800 Subject: [PATCH 21/67] Make some bug fixes --- solr/core/src/java/org/apache/solr/cloud/LockTree.java | 2 +- .../cloud/api/collections/CollectionHandlingUtils.java | 8 +++----- .../solr/cloud/api/collections/DeleteCollectionCmd.java | 2 +- .../org/apache/solr/handler/admin/CollectionsHandler.java | 2 +- .../apache/solr/handler/admin/api/CreateCollection.java | 1 + .../org/apache/solr/handler/admin/api/DeleteNode.java | 2 +- .../org/apache/solr/handler/admin/api/ReplaceNode.java | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 44fbd9fd263a..7940b07ef7d4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -90,7 +90,7 @@ public class Session { public Lock lock( CollectionParams.CollectionAction action, List path, List callingLockIds) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; - log.info("Calling lock level: {}", callingLockIds); + log.debug("Calling lock level: {}", callingLockIds); Node startingNode = LockTree.this.root; SessionNode startingSession = root; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index aa58836a1ada..e28ae7a17c9d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -311,10 +311,8 @@ static void cleanupCollection( AdminCmdContext adminCmdContext, String collectionName, NamedList results, CollectionCommandContext ccc) throws Exception { log.error("Cleaning up collection [{}].", collectionName); - Map props = - Map.of(NAME, collectionName); new DeleteCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(DELETE), new ZkNodeProps(props), results); + .call(adminCmdContext.subRequestContext(DELETE), new ZkNodeProps(NAME, collectionName), results); } static Map waitToSeeReplicasInState( @@ -399,13 +397,13 @@ static List collectionCmd( log.info("Executing Collection Cmd={}, asyncId={}", params, adminCmdContext.getAsyncId()); String collectionName = message.getStr(NAME); ShardHandler shardHandler = ccc.newShardHandler(); - DocCollection coll = adminCmdContext.getClusterState().getCollection(collectionName); + DocCollection coll = ccc.getZkStateReader().getClusterState().getCollection(collectionName); List notLivesReplicas = new ArrayList<>(); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : coll.getSlices()) { notLivesReplicas.addAll( - shardRequestTracker.sliceCmd(adminCmdContext.getClusterState(), params, stateMatcher, slice, shardHandler)); + shardRequestTracker.sliceCmd(ccc.getZkStateReader().getClusterState(), params, stateMatcher, slice, shardHandler)); } shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 72c08a16cb7d..872f2419c655 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -88,7 +88,7 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList } // verify the placement modifications caused by the deletion are allowed - DocCollection coll = adminCmdContext.getClusterState().getCollectionOrNull(collection); + DocCollection coll = zkStateReader.getClusterState().getCollectionOrNull(collection); if (coll != null) { Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ccc.getCoreContainer()); assignStrategy.verifyDeleteCollection(ccc.getSolrCloudManager(), coll); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 76a32ffc5422..cc98e366cb9b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -321,7 +321,7 @@ void invokeAction( } AdminCmdContext adminCmdContext = new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); - adminCmdContext.setCallingLockIds(req.getContext().get(CALLING_LOCK_IDS_HEADER).toString()); + adminCmdContext.setCallingLockIds((String)req.getContext().get(CALLING_LOCK_IDS_HEADER)); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index 3bd23946c052..1c4953b57e5e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -160,6 +160,7 @@ public static ZkNodeProps createRemoteMessage(CreateCollectionRequestBody reqBod rawProperties.put(TLOG_REPLICAS, reqBody.tlogReplicas); rawProperties.put(WAIT_FOR_FINAL_STATE, reqBody.waitForFinalState); rawProperties.put(PER_REPLICA_STATE, reqBody.perReplicaState); + rawProperties.put(ALIAS, reqBody.alias); if (reqBody.createReplicas == null || reqBody.createReplicas) { // The remote message expects a single comma-delimited string, so nodeSet requires flattening if (reqBody.nodeSet != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java index 9cda7a408352..8c411b8618e0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java @@ -66,7 +66,7 @@ public SolrJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requ response, CollectionParams.CollectionAction.DELETENODE, new ZkNodeProps(Map.of(NODE, nodeName)), - requestBody.async); + requestBody != null ? requestBody.async : null); disableResponseCaching(); return response; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java index 3dd873b41753..60b2c601341b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java @@ -67,7 +67,7 @@ public SolrJerseyResponse replaceNode(String sourceNodeName, ReplaceNodeRequestB response, CollectionParams.CollectionAction.REPLACENODE, createRemoteMessage(sourceNodeName, requestBody), - requestBody.async); + requestBody != null ? requestBody.async : null); disableResponseCaching(); return response; From 97352f7cab88222dafb200284fc932e40921b08f Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 9 Jan 2026 16:42:31 -0800 Subject: [PATCH 22/67] Some test 'fixes', need to rethink v2 tests still --- .../collections/CollectionApiLockingTest.java | 16 ++++++++-------- .../admin/api/AddReplicaPropertyAPITest.java | 5 ++--- .../api/CreateCollectionSnapshotAPITest.java | 9 ++++----- .../admin/api/DeleteCollectionAPITest.java | 11 +++++------ .../admin/api/DeleteCollectionBackupAPITest.java | 8 +++----- .../api/DeleteCollectionSnapshotAPITest.java | 9 ++++----- .../admin/api/MigrateReplicasAPITest.java | 7 +++---- .../handler/admin/api/ReplaceNodeAPITest.java | 9 ++++----- 8 files changed, 33 insertions(+), 41 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index e3f38db7e2d3..9dcde25a5a06 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -65,7 +65,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // hierarchy) DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); assertEquals( "Lock at collection level expected to need one distributed lock", @@ -76,7 +76,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // above DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), COLLECTION_NAME, SHARD1_NAME, null); assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); assertEquals( "Lock at shard level expected to need two distributed locks", @@ -87,7 +87,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // collection lock above DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), COLLECTION_NAME, SHARD2_NAME, null); assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); assertTrue("Collection should still be acquired", collLock.isAcquired()); @@ -104,7 +104,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired()); @@ -112,7 +112,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Now ask for a new lock on the collection collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); assertFalse( "Collection should not have been acquired, shard1 and shard2 locks preventing it", @@ -131,7 +131,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); assertFalse( "replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired()); @@ -158,13 +158,13 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw // Lock on collection... DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); assertTrue("Collection should have been acquired", collLock.isAcquired()); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index b6e666b9137e..c1f114dfdf75 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -20,7 +20,6 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -74,7 +73,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getZkController()).thenReturn(mockZkController); when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); @@ -108,7 +107,7 @@ public void testCreatesValidOverseerMessage() throws Exception { addReplicaPropApi.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java index 27efa7bc0ef1..080be493b8fe 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -33,7 +33,7 @@ public class CreateCollectionSnapshotAPITest extends SolrTestCaseJ4 { @Test public void testConstructsValidOverseerMessage() { final ZkNodeProps messageOne = - CreateCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName", null); + CreateCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName"); final Map rawMessageOne = messageOne.getProperties(); assertEquals(4, rawMessageOne.size()); assertThat( @@ -47,17 +47,16 @@ public void testConstructsValidOverseerMessage() { final ZkNodeProps messageTwo = CreateCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName", "myAsyncId"); + "myCollName", true, "mySnapshotName"); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(5, rawMessageTwo.size()); + assertEquals(4, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("createsnapshot", rawMessageTwo.get(QUEUE_OPERATION)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); - assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java index fafc0a67c001..e50b772fd8f9 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java @@ -35,7 +35,7 @@ public class DeleteCollectionAPITest extends SolrTestCaseJ4 { public void testConstructsValidOverseerMessage() { // Only required properties provided { - final ZkNodeProps message = DeleteCollection.createRemoteMessage("someCollName", null, null); + final ZkNodeProps message = DeleteCollection.createRemoteMessage("someCollName", null); final Map rawMessage = message.getProperties(); assertEquals(2, rawMessage.size()); assertThat(rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME)); @@ -43,18 +43,17 @@ public void testConstructsValidOverseerMessage() { assertEquals("someCollName", rawMessage.get(NAME)); } - // Optional properties ('followAliases' and 'async') also provided + // Optional property 'followAliases' also provided { final ZkNodeProps message = - DeleteCollection.createRemoteMessage("someCollName", Boolean.TRUE, "someAsyncId"); + DeleteCollection.createRemoteMessage("someCollName", Boolean.TRUE); final Map rawMessage = message.getProperties(); - assertEquals(4, rawMessage.size()); + assertEquals(3, rawMessage.size()); assertThat( - rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, ASYNC, FOLLOW_ALIASES)); + rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, FOLLOW_ALIASES)); assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); assertEquals("someCollName", rawMessage.get(NAME)); assertEquals(Boolean.TRUE, rawMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", rawMessage.get(ASYNC)); } } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java index 7d083d1c0cfe..af8a5133ad64 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java @@ -125,11 +125,10 @@ public void testCreateRemoteMessageAllParams() { 123, true, "someLocation", - "someRepository", - "someAsyncId") + "someRepository") .getProperties(); - assertEquals(8, remoteMessage.size()); + assertEquals(7, remoteMessage.size()); assertEquals("deletebackup", remoteMessage.get(QUEUE_OPERATION)); assertEquals("someBackupName", remoteMessage.get(NAME)); assertEquals("someBackupId", remoteMessage.get(BACKUP_ID)); @@ -137,14 +136,13 @@ public void testCreateRemoteMessageAllParams() { assertEquals(Boolean.TRUE, remoteMessage.get(BACKUP_PURGE_UNUSED)); assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); } @Test public void testCreateRemoteMessageOnlyRequiredParams() { final var remoteMessage = DeleteCollectionBackup.createRemoteMessage( - "someBackupName", "someBackupId", null, null, null, null, null) + "someBackupName", "someBackupId", null, null, null, null) .getProperties(); assertEquals(3, remoteMessage.size()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java index 0ed8e42c12c9..cb8dfb76449f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -33,7 +33,7 @@ public class DeleteCollectionSnapshotAPITest extends SolrTestCaseJ4 { @Test public void testConstructsValidOverseerMessage() { final ZkNodeProps messageOne = - DeleteCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName", null); + DeleteCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName"); final Map rawMessageOne = messageOne.getProperties(); assertEquals(4, rawMessageOne.size()); assertThat( @@ -47,17 +47,16 @@ public void testConstructsValidOverseerMessage() { final ZkNodeProps messageTwo = DeleteCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName", "myAsyncId"); + "myCollName", true, "mySnapshotName"); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(5, rawMessageTwo.size()); + assertEquals(4, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("deletesnapshot", rawMessageTwo.get(QUEUE_OPERATION)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); - assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index db7afb4eedcf..f3f31feb7d93 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -18,7 +18,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -69,7 +68,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getZkController()).thenReturn(mockZkController); when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -86,7 +85,7 @@ public void testCreatesValidOverseerMessage() throws Exception { Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); migrateReplicasAPI.migrateReplicas(requestBody); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -104,7 +103,7 @@ public void testNoTargetNodes() throws Exception { new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); migrateReplicasAPI.migrateReplicas(requestBody); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index b27656998729..e895595f5084 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -18,7 +18,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -66,7 +65,7 @@ public void setUp() throws Exception { mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); when(mockCoreContainer.getZkController()).thenReturn(mockZkController); when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong(), nullable(String.class))) + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); queryResponse = new SolrQueryResponse(); @@ -81,7 +80,7 @@ public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); replaceNodeApi.replaceNode("demoSourceNode", requestBody); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -97,7 +96,7 @@ public void testCreatesValidOverseerMessage() throws Exception { public void testRequestBodyCanBeOmittedAltogether() throws Exception { replaceNodeApi.replaceNode("demoSourceNode", null); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -111,7 +110,7 @@ public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exce final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); replaceNodeApi.replaceNode("demoSourceNode", requestBody); verify(mockCommandRunner) - .runCollectionCommand(messageCapturer.capture(), any(), anyLong(), nullable(String.class)); + .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); From 6f2d61a2e5d5fa3925ee90c748ac8c0f7531a085 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 12 Jan 2026 13:16:24 -0800 Subject: [PATCH 23/67] Add real tests for DeleteAlias and DeleteNode v2 apis --- .../client/api/endpoint/DeleteAliasApi.java | 3 +- .../client/api/endpoint/DeleteNodeApi.java | 3 +- .../solr/handler/admin/api/DeleteAlias.java | 2 +- .../solr/handler/admin/api/DeleteNode.java | 2 +- .../java/org/apache/solr/util/TimeOut.java | 9 + .../handler/admin/api/DeleteAliasAPITest.java | 112 ++++++++-- .../handler/admin/api/DeleteNodeAPITest.java | 193 ++++++++++++++---- .../apache/solr/cloud/SolrCloudTestCase.java | 23 +++ 8 files changed, 283 insertions(+), 64 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java index badaea3c1178..b77293cc3b2c 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java @@ -24,6 +24,7 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.QueryParam; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/aliases/{aliasName}") public interface DeleteAliasApi { @@ -32,7 +33,7 @@ public interface DeleteAliasApi { @Operation( summary = "Deletes an alias by its name", tags = {"aliases"}) - SolrJerseyResponse deleteAlias( + SubResponseAccumulatingJerseyResponse deleteAlias( @Parameter(description = "The name of the alias to delete", required = true) @PathParam("aliasName") String aliasName, diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java index 64b5978afd27..5127f32d071f 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java @@ -25,6 +25,7 @@ import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.DeleteNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("cluster/nodes/{nodeName}/clear/") public interface DeleteNodeApi { @@ -33,7 +34,7 @@ public interface DeleteNodeApi { @Operation( summary = "Delete all replicas off of the specified SolrCloud node", tags = {"node"}) - SolrJerseyResponse deleteNode( + SubResponseAccumulatingJerseyResponse deleteNode( @Parameter( description = "The name of the node to be cleared. Usually of the form 'host:1234_solr'.", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java index 50d54a430a7f..d2471bb2ccf8 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java @@ -50,7 +50,7 @@ public DeleteAlias( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteAlias(String aliasName, String asyncId) throws Exception { + public SubResponseAccumulatingJerseyResponse deleteAlias(String aliasName, String asyncId) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java index 8c411b8618e0..2164e8399485 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java @@ -58,7 +58,7 @@ public DeleteNode( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requestBody) + public SubResponseAccumulatingJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/util/TimeOut.java b/solr/core/src/java/org/apache/solr/util/TimeOut.java index a1d682a6ed93..6fdc9bd26099 100644 --- a/solr/core/src/java/org/apache/solr/util/TimeOut.java +++ b/solr/core/src/java/org/apache/solr/util/TimeOut.java @@ -18,6 +18,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; +import java.time.Duration; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; @@ -30,6 +31,14 @@ public class TimeOut { private final long timeoutAt, startTime; private final TimeSource timeSource; + /** + * @param timeout after this maximum time, {@link #hasTimedOut()} will return true. + * @param timeSource the source of the time. + */ + public TimeOut(Duration timeout, TimeSource timeSource) { + this(timeout.toNanos(), NANOSECONDS, timeSource); + } + /** * @param timeout after this maximum time, {@link #hasTimedOut()} will return true. * @param unit the time unit of the timeout argument. diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java index dc93c29b4873..9b5ed2ab92b5 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java @@ -17,28 +17,102 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.common.params.CoreAdminParams.NAME; - -import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import java.lang.invoke.MethodHandles; +import java.time.Duration; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; +import org.apache.solr.client.solrj.request.AliasesApi; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.util.TimeOut; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration tests for {@link DeleteAlias} using the V2 API */ +public class DeleteAliasAPITest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig( + "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); + } + + @Test + public void testDeleteAlias() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String collectionName = "deletealiastest_coll"; + String aliasName = "deletealiastest_alias"; + + // Create a collection + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1) + .process(cloudClient); + cluster.waitForActiveCollection(collectionName, 1, 1); + + // Create an alias pointing to the collection + CollectionAdminRequest.createAlias(aliasName, collectionName).process(cloudClient); + + // Verify the alias exists + var clusterStateProvider = cloudClient.getClusterStateProvider(); + assertEquals("Alias should exist before deletion", collectionName, clusterStateProvider.resolveSimpleAlias(aliasName)); -/** Unit tests for {@link DeleteAlias}. */ -public class DeleteAliasAPITest extends SolrTestCaseJ4 { + // Delete the alias using the V2 API + var request = new AliasesApi.DeleteAlias(aliasName); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + assertNotNull(response); + assertNull("Expected request to not fail", response.failedSubResponsesByNodeName); + + // Verify the alias is gone + ZkStateReader.AliasesManager aliasesManager = ((ZkClientClusterStateProvider)clusterStateProvider).getZkStateReader().getAliasesManager(); + aliasesManager.update(); + assertFalse("Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); + } @Test - public void testConstructsValidRemoteMessage() { - Map props = DeleteAlias.createRemoteMessage("aliasName", null).getProperties(); - assertEquals(2, props.size()); - assertEquals("deletealias", props.get(QUEUE_OPERATION)); - assertEquals("aliasName", props.get(NAME)); - - props = DeleteAlias.createRemoteMessage("aliasName", "asyncId").getProperties(); - assertEquals(3, props.size()); - assertEquals("deletealias", props.get(QUEUE_OPERATION)); - assertEquals("aliasName", props.get(NAME)); - assertEquals("asyncId", props.get(ASYNC)); + public void testDeleteAliasAsync() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String collectionName = "deletealiastest_coll_async"; + String aliasName = "deletealiastest_alias_async"; + + // Create a collection + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1) + .process(cloudClient); + cluster.waitForActiveCollection(collectionName, 1, 1); + + // Create an alias pointing to the collection + CollectionAdminRequest.createAlias(aliasName, collectionName).process(cloudClient); + + // Verify the alias exists + var clusterStateProvider = cloudClient.getClusterStateProvider(); + + // Delete the alias using the V2 API with async + String asyncId = "deleteAlias001"; + var request = new AliasesApi.DeleteAlias(aliasName); + request.setAsync(asyncId); + var response = request.process(cloudClient); + assertNotNull(response); + assertNull("Expected request start to not fail", response.failedSubResponsesByNodeName); + + // Wait for the async request to complete + CollectionAdminRequest.RequestStatusResponse rsp = waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + + assertEquals( + "Expected async request to complete successfully", + RequestStatusState.COMPLETED, + rsp.getRequestStatus()); + + // Verify the alias is gone + ZkStateReader.AliasesManager aliasesManager = ((ZkClientClusterStateProvider)clusterStateProvider).getZkStateReader().getAliasesManager(); + aliasesManager.update(); + assertFalse("Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java index 74b6b3e927b9..1621f6442c13 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java @@ -16,59 +16,170 @@ */ package org.apache.solr.handler.admin.api; -import static org.mockito.Mockito.mock; - -import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.api.model.DeleteNodeRequestBody; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.ModifiableSolrParams; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.NodeApi; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.StrUtils; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** Unit tests for {@link DeleteNode} */ -public class DeleteNodeAPITest extends SolrTestCaseJ4 { +/** Integration tests for {@link DeleteNode} using the V2 API */ +public class DeleteNodeAPITest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); + public static void setupCluster() throws Exception { + configureCluster(6) + .addConfig( + "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) + .configure(); } - @Test - public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { - final var api = mock(DeleteNode.class); - final SolrException e = - expectThrows( - SolrException.class, - () -> { - DeleteNode.invokeUsingV1Inputs(api, new ModifiableSolrParams()); - }); - assertEquals("Missing required parameter: node", e.getMessage()); + @After + public void clearCollections() throws Exception { + cluster.deleteAllCollections(); } @Test - public void testValidOverseerMessageIsCreated() { - final var requestBody = new DeleteNodeRequestBody(); - requestBody.async = "async"; - final ZkNodeProps createdMessage = - DeleteNode.createRemoteMessage("nodeNameToDelete", requestBody); - final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(3, createdMessageProps.size()); - assertEquals("nodeNameToDelete", createdMessageProps.get("node")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("deletenode", createdMessageProps.get("operation")); + public void testDeleteNode() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String coll = "deletenodetest_coll"; + Set liveNodes = cloudClient.getClusterStateProvider().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + CollectionAdminRequest.Create create = + pickRandom( + CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1), + // check RF=1 + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)); + create.setCreateNodeSet(StrUtils.join(l, ',')); + cloudClient.request(create); + String nodeToBeDecommissioned = l.get(0); + + // check what replicas are on the node, and whether the call should fail + boolean shouldFail = false; + DocCollection docColl = cloudClient.getClusterStateProvider().getCollection(coll); + log.info("#### DocCollection: {}", docColl); + List replicas = docColl.getReplicasOnNode(nodeToBeDecommissioned); + for (Replica replica : replicas) { + String shard = replica.getShard(); + Slice slice = docColl.getSlice(shard); + boolean hasOtherNonPullReplicas = false; + for (Replica r : slice.getReplicas()) { + if (!r.getName().equals(replica.getName()) + && !r.getNodeName().equals(nodeToBeDecommissioned) + && r.getType().leaderEligible) { + hasOtherNonPullReplicas = true; + break; + } + } + if (!hasOtherNonPullReplicas) { + shouldFail = true; + break; + } + } + + var request = new NodeApi.DeleteNode(nodeToBeDecommissioned); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + + if (log.isInfoEnabled()) { + log.info( + "####### DocCollection after: {}", cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + } + + if (shouldFail) { + assertNotNull( + "Expected request to fail, there should be failures sent back", + response.failedSubResponsesByNodeName); + } else { + assertNull( + "Expected request to not fail, there should be no failures sent back", + response.failedSubResponsesByNodeName); + } } @Test - public void testRequestBodyCanBeOmitted() throws Exception { - final ZkNodeProps createdMessage = DeleteNode.createRemoteMessage("nodeNameToDelete", null); - final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); - assertEquals("nodeNameToDelete", createdMessageProps.get("node")); - assertEquals("deletenode", createdMessageProps.get("operation")); - assertFalse( - "Expected message to not contain value for async: " + createdMessageProps.get("async"), - createdMessageProps.containsKey("async")); + public void testDeleteNodeAsync() throws Exception { + CloudSolrClient cloudClient = cluster.getSolrClient(); + String coll = "deletenodetest_coll_async"; + Set liveNodes = cloudClient.getClusterStateProvider().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + CollectionAdminRequest.Create create = + pickRandom( + CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1), + // check RF=1 + CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0), + CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)); + create.setCreateNodeSet(StrUtils.join(l, ',')); + cloudClient.request(create); + String nodeToBeDecommissioned = l.get(0); + + // check what replicas are on the node, and whether the call should fail + boolean shouldFail = false; + DocCollection docColl = cloudClient.getClusterStateProvider().getCollection(coll); + log.info("#### DocCollection: {}", docColl); + List replicas = docColl.getReplicasOnNode(nodeToBeDecommissioned); + for (Replica replica : replicas) { + String shard = replica.getShard(); + Slice slice = docColl.getSlice(shard); + boolean hasOtherNonPullReplicas = false; + for (Replica r : slice.getReplicas()) { + if (!r.getName().equals(replica.getName()) + && !r.getNodeName().equals(nodeToBeDecommissioned) + && r.getType().leaderEligible) { + hasOtherNonPullReplicas = true; + break; + } + } + if (!hasOtherNonPullReplicas) { + shouldFail = true; + break; + } + } + + String asyncId = "003"; + var request = new NodeApi.DeleteNode(nodeToBeDecommissioned); + request.setAsync(asyncId); + SubResponseAccumulatingJerseyResponse response = request.process(cloudClient); + assertNull( + "Expected request to not fail, any failures will be returned in the async status response", + response.failedSubResponsesByNodeName); + + // Wait for the async request to complete + CollectionAdminRequest.RequestStatusResponse rsp = waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + + if (log.isInfoEnabled()) { + log.info( + "####### DocCollection after: {}", cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + } + + if (shouldFail) { + assertSame(String.valueOf(rsp), RequestStatusState.FAILED, rsp.getRequestStatus()); + } else { + assertNotSame(String.valueOf(rsp), RequestStatusState.FAILED, rsp.getRequestStatus()); + } } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 7f589e48b232..59d93c17ab62 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -26,6 +26,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.invoke.MethodHandles; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -47,7 +48,9 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.common.cloud.CollectionStatePredicate; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.LiveNodesPredicate; @@ -57,7 +60,9 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.TimeSource; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TimeOut; import org.junit.AfterClass; import org.junit.Before; import org.slf4j.Logger; @@ -415,6 +420,24 @@ protected static CoreStatusResponse.SingleCoreData getCoreStatus(Replica replica } } + protected CollectionAdminRequest.RequestStatusResponse waitForAsyncClusterRequest(String asyncId, Duration timeout) + throws SolrServerException, IOException, InterruptedException { + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + CollectionAdminRequest.RequestStatusResponse rsp = null; + TimeOut timeoutCheck = new TimeOut(timeout, TimeSource.NANO_TIME); + while (!timeoutCheck.hasTimedOut()) { + rsp = requestStatus.process(cluster.getSolrClient()); + if (rsp.getRequestStatus() == RequestStatusState.FAILED + || rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + return rsp; + } + Thread.sleep(50); + } + fail("Async request " + asyncId + " did not complete within duration: " + timeout.toString()); + return rsp; + } + @SuppressWarnings({"rawtypes"}) protected NamedList waitForResponse( Predicate predicate, From 74176a469e3df90189f9140d3148241a791c79c3 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 12 Jan 2026 13:24:10 -0800 Subject: [PATCH 24/67] Tidy --- .../client/api/endpoint/DeleteAliasApi.java | 1 - .../client/api/endpoint/DeleteNodeApi.java | 1 - .../solr/cloud/OverseerTaskProcessor.java | 1 - .../cloud/api/collections/AddReplicaCmd.java | 8 +- .../api/collections/AdminCmdContext.java | 9 +- .../solr/cloud/api/collections/AliasCmd.java | 7 +- .../solr/cloud/api/collections/BackupCmd.java | 19 +++- .../cloud/api/collections/CollApiCmds.java | 25 +++--- .../collections/CollectionApiLockFactory.java | 7 +- .../collections/CollectionCommandContext.java | 6 +- .../collections/CollectionHandlingUtils.java | 22 +++-- .../cloud/api/collections/CreateAliasCmd.java | 6 +- .../api/collections/CreateCollectionCmd.java | 12 ++- .../cloud/api/collections/CreateShardCmd.java | 15 +++- .../api/collections/CreateSnapshotCmd.java | 6 +- .../cloud/api/collections/DeleteAliasCmd.java | 3 +- .../api/collections/DeleteBackupCmd.java | 3 +- .../api/collections/DeleteCollectionCmd.java | 3 +- .../cloud/api/collections/DeleteNodeCmd.java | 12 +-- .../api/collections/DeleteReplicaCmd.java | 10 ++- .../cloud/api/collections/DeleteShardCmd.java | 6 +- .../api/collections/DeleteSnapshotCmd.java | 3 +- ...butedCollectionConfigSetCommandRunner.java | 31 ++++--- .../api/collections/InstallShardDataCmd.java | 3 +- .../collections/MaintainRoutedAliasCmd.java | 23 +++-- .../cloud/api/collections/MigrateCmd.java | 44 +++++++--- .../api/collections/MigrateReplicasCmd.java | 3 +- .../cloud/api/collections/MoveReplicaCmd.java | 66 ++++++++++---- .../api/collections/OverseerRoleCmd.java | 3 +- .../api/collections/OverseerStatusCmd.java | 3 +- .../api/collections/ReindexCollectionCmd.java | 87 +++++++++++++------ .../solr/cloud/api/collections/RenameCmd.java | 3 +- .../cloud/api/collections/ReplaceNodeCmd.java | 3 +- .../collections/ReplicaMigrationUtils.java | 15 ++-- .../cloud/api/collections/RestoreCmd.java | 48 +++++++--- .../api/collections/SetAliasPropCmd.java | 3 +- .../cloud/api/collections/SplitShardCmd.java | 37 ++++++-- .../handler/admin/CollectionsHandler.java | 5 +- .../solr/handler/admin/RebalanceLeaders.java | 7 +- .../handler/admin/api/AddReplicaProperty.java | 10 +-- .../solr/handler/admin/api/AdminAPIBase.java | 23 ++--- .../solr/handler/admin/api/AliasProperty.java | 25 +++--- .../handler/admin/api/BalanceReplicas.java | 9 +- .../handler/admin/api/BalanceShardUnique.java | 1 - .../solr/handler/admin/api/CreateAlias.java | 9 +- .../handler/admin/api/CreateCollection.java | 8 +- .../admin/api/CreateCollectionBackup.java | 10 +-- .../admin/api/CreateCollectionSnapshot.java | 5 -- .../solr/handler/admin/api/CreateReplica.java | 1 - .../solr/handler/admin/api/DeleteAlias.java | 11 +-- .../handler/admin/api/DeleteCollection.java | 10 +-- .../admin/api/DeleteCollectionBackup.java | 11 +-- .../admin/api/DeleteCollectionSnapshot.java | 8 +- .../solr/handler/admin/api/DeleteNode.java | 9 +- .../admin/api/DeleteReplicaProperty.java | 4 - .../handler/admin/api/InstallShardData.java | 3 - .../handler/admin/api/MigrateReplicas.java | 5 -- .../handler/admin/api/RenameCollection.java | 6 +- .../solr/handler/admin/api/ReplaceNode.java | 4 - .../handler/admin/api/RestoreCollection.java | 8 +- .../collections/CollectionApiLockingTest.java | 40 +++++++-- .../admin/api/AddReplicaPropertyAPITest.java | 3 +- .../api/CreateCollectionSnapshotAPITest.java | 4 +- .../handler/admin/api/DeleteAliasAPITest.java | 33 ++++--- .../admin/api/DeleteCollectionAPITest.java | 4 +- .../api/DeleteCollectionBackupAPITest.java | 8 +- .../api/DeleteCollectionSnapshotAPITest.java | 4 +- .../handler/admin/api/DeleteNodeAPITest.java | 11 +-- .../admin/api/MigrateReplicasAPITest.java | 6 +- .../handler/admin/api/ReplaceNodeAPITest.java | 9 +- .../apache/solr/cloud/SolrCloudTestCase.java | 3 +- 71 files changed, 498 insertions(+), 376 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java index b77293cc3b2c..50ec9e67b0bc 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteAliasApi.java @@ -23,7 +23,6 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.QueryParam; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/aliases/{aliasName}") diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java index 5127f32d071f..b42491b335f9 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteNodeApi.java @@ -24,7 +24,6 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.DeleteNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("cluster/nodes/{nodeName}/clear/") diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index 4c2014149f96..b44c4f51ce24 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -37,7 +37,6 @@ import java.util.function.Predicate; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; -import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index 6a55a3a95ed5..a13bce718d8a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -75,8 +75,7 @@ public AddReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { addReplica(adminCmdContext, message, results, null); } @@ -160,7 +159,10 @@ List addReplica( .map( replicaPosition -> assignReplicaDetails( - ccc.getSolrCloudManager(), adminCmdContext.getClusterState(), message, replicaPosition)) + ccc.getSolrCloudManager(), + adminCmdContext.getClusterState(), + message, + replicaPosition)) .collect(Collectors.toList()); ShardHandler shardHandler = ccc.newShardHandler(); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index f0d4dd34e9e3..e0db692bcff2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -3,12 +3,10 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; -import java.util.ArrayList; -import java.util.List; public class AdminCmdContext { - final private CollectionParams.CollectionAction action; - final private String asyncId; + private final CollectionParams.CollectionAction action; + private final String asyncId; private String lockId; private String callingLockIds; private String subRequestCallingLockIds; @@ -74,7 +72,8 @@ public AdminCmdContext subRequestContext(CollectionParams.CollectionAction actio return subRequestContext(action, asyncId); } - public AdminCmdContext subRequestContext(CollectionParams.CollectionAction action, String asyncId) { + public AdminCmdContext subRequestContext( + CollectionParams.CollectionAction action, String asyncId) { AdminCmdContext nextContext = new AdminCmdContext(action, asyncId); nextContext.setCallingLockIds(subRequestCallingLockIds); return nextContext.withClusterState(clusterState); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index e6c3f308336e..ceefa9e889be 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.solr.cloud.OverseerSolrResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; @@ -84,7 +83,11 @@ static NamedList createCollectionAndWait( // CreateCollectionCmd. // note: there's doesn't seem to be any point in locking on the collection name, so we don't. // We currently should already have a lock on the alias name which should be sufficient. - new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE), createMessage, results); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE), + createMessage, + results); } catch (SolrException e) { // The collection might already exist, and that's okay -- we can adopt it. if (!e.getMessage().contains("collection already exists")) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java index 3108a1c2b2d4..60ec1cb7e6c8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java @@ -69,7 +69,8 @@ public BackupCmd(CollectionCommandContext ccc) { @SuppressWarnings("unchecked") @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -120,7 +121,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList if (incremental) { try { incrementalCopyIndexFiles( - adminCmdContext, backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, + backupUri, + collectionName, + message, + results, + backupProperties, + backupMgr); } catch (SolrException e) { log.error( "Error happened during incremental backup for collection: {}", @@ -132,7 +139,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList } } else { copyIndexFiles( - adminCmdContext, backupUri, collectionName, message, results, backupProperties, backupMgr); + adminCmdContext, + backupUri, + collectionName, + message, + results, + backupProperties, + backupMgr); } break; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index 6649dfd35048..e360012ea01d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -65,7 +65,6 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.REPLACENODE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.RESTORE; import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import io.opentelemetry.api.trace.Span; @@ -262,7 +261,8 @@ public ReloadCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) { + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); @@ -285,8 +285,7 @@ public RebalanceLeadersCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -327,8 +326,7 @@ public AddReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, @@ -362,8 +360,7 @@ public DeleteReplicaPropCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { CollectionHandlingUtils.checkRequired( message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP); @@ -392,8 +389,7 @@ public BalanceShardsUniqueCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext clusterState, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext clusterState, ZkNodeProps message, NamedList results) throws Exception { if (StrUtils.isBlank(message.getStr(COLLECTION_PROP)) || StrUtils.isBlank(message.getStr(PROPERTY_PROP))) { @@ -429,7 +425,9 @@ public ModifyCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP); // the rest of the processing is based on writing cluster state properties String configName = (String) message.getProperties().get(COLL_CONF); @@ -510,7 +508,10 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList // if switching to/from read-only mode or configName is not null reload the collection if (message.keySet().contains(ZkStateReader.READ_ONLY) || configName != null) { new ReloadCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(RELOAD, null), new ZkNodeProps(NAME, collectionName), results); + .call( + adminCmdContext.subRequestContext(RELOAD, null), + new ZkNodeProps(NAME, collectionName), + results); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index 00b3d84d30da..9a7f2d6fe63e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -59,10 +59,7 @@ public class CollectionApiLockFactory { * prevent other threads from locking. */ DistributedMultiLock createCollectionApiLock( - AdminCmdContext adminCmdContext, - String collName, - String shardId, - String replicaName) { + AdminCmdContext adminCmdContext, String collName, String shardId, String replicaName) { CollectionParams.LockLevel lockLevel = adminCmdContext.getAction().lockLevel; if (lockLevel == CollectionParams.LockLevel.NONE) { return new DistributedMultiLock(List.of()); @@ -114,7 +111,7 @@ DistributedMultiLock createCollectionApiLock( List callingLockIdList; if (adminCmdContext.getCallingLockIds() == null) { callingLockIdList = Collections.emptyList(); - } else { + } else { callingLockIdList = List.of(adminCmdContext.getCallingLockIds().split(",")); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java index 025a5571ff06..0271f7bbe00c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java @@ -57,7 +57,11 @@ public interface CollectionCommandContext { default ShardRequestTracker asyncRequestTracker(AdminCmdContext adminCmdContext) { return new ShardRequestTracker( - adminCmdContext.getAsyncId(), adminCmdContext.getSubRequestCallingLockIds(), getAdminPath(), getZkStateReader(), newShardHandler().getShardHandlerFactory()); + adminCmdContext.getAsyncId(), + adminCmdContext.getSubRequestCallingLockIds(), + getAdminPath(), + getZkStateReader(), + newShardHandler().getShardHandlerFactory()); } /** admin path passed to Overseer is a constant, including in tests. */ diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index e28ae7a17c9d..0b3d4bb930af 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -308,11 +308,17 @@ static void addPropertyParams(ZkNodeProps message, Map map) { } static void cleanupCollection( - AdminCmdContext adminCmdContext, String collectionName, NamedList results, CollectionCommandContext ccc) + AdminCmdContext adminCmdContext, + String collectionName, + NamedList results, + CollectionCommandContext ccc) throws Exception { log.error("Cleaning up collection [{}].", collectionName); new DeleteCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(DELETE), new ZkNodeProps(NAME, collectionName), results); + .call( + adminCmdContext.subRequestContext(DELETE), + new ZkNodeProps(NAME, collectionName), + results); } static Map waitToSeeReplicasInState( @@ -403,7 +409,8 @@ static List collectionCmd( asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : coll.getSlices()) { notLivesReplicas.addAll( - shardRequestTracker.sliceCmd(ccc.getZkStateReader().getClusterState(), params, stateMatcher, slice, shardHandler)); + shardRequestTracker.sliceCmd( + ccc.getZkStateReader().getClusterState(), params, stateMatcher, slice, shardHandler)); } shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions); @@ -561,16 +568,15 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( } while (true); } - public static ShardRequestTracker syncRequestTracker(AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { - return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(),ccc); + public static ShardRequestTracker syncRequestTracker( + AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(), ccc); } public static ShardRequestTracker asyncRequestTracker( AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { return requestTracker( - adminCmdContext.getAsyncId(), - adminCmdContext.getSubRequestCallingLockIds(), - ccc); + adminCmdContext.getAsyncId(), adminCmdContext.getSubRequestCallingLockIds(), ccc); } protected static ShardRequestTracker requestTracker( diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java index 963834348f58..53de754285ff 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java @@ -50,7 +50,8 @@ public CreateAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); // make sure we have the latest version of existing aliases @@ -176,7 +177,8 @@ private void ensureAliasCollection( String initialCollectionName) throws Exception { // Create the collection - createCollectionAndWait(adminCmdContext, aliasName, aliasProperties, initialCollectionName, ccc); + createCollectionAndWait( + adminCmdContext, aliasName, aliasProperties, initialCollectionName, ccc); validateAllCollectionsExistAndNoDuplicates( Collections.singletonList(initialCollectionName), zkStateReader); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index ee778c0ac59a..b0f6754a15bc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -102,7 +102,8 @@ public CreateCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader ccc.getZkStateReader().aliasesManager.update(); } @@ -243,7 +244,11 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList numReplicas); } catch (Assign.AssignmentException e) { ZkNodeProps deleteMessage = new ZkNodeProps("name", collectionName); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(DELETE).withClusterState(clusterState), deleteMessage, results); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(DELETE).withClusterState(clusterState), + deleteMessage, + results); // unwrap the exception throw new SolrException(ErrorCode.BAD_REQUEST, e.getMessage(), e.getCause()); } @@ -436,7 +441,8 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList // Let's cleanup as we hit an exception // We shouldn't be passing 'results' here for the cleanup as the response would then contain // 'success' element, which may be interpreted by the user as a positive ack - CollectionHandlingUtils.cleanupCollection(adminCmdContext, collectionName, new NamedList<>(), ccc); + CollectionHandlingUtils.cleanupCollection( + adminCmdContext, collectionName, new NamedList<>(), ccc); log.info("Cleaned up artifacts for failed create collection for [{}]", collectionName); throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java index 10fe8ef7b3cf..ee7b2242f8c9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java @@ -45,7 +45,8 @@ public CreateShardCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); String sliceName = message.getStr(SHARD_ID_PROP); boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); @@ -114,7 +115,9 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList try { new AddReplicaCmd(ccc) .addReplica( - adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA, async).withClusterState(clusterState), + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA, async) + .withClusterState(clusterState), new ZkNodeProps(addReplicasProps), addResult, () -> { @@ -144,7 +147,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList }); } catch (Assign.AssignmentException e) { // clean up the slice that we created - new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, async).withClusterState(clusterState), new ZkNodeProps(COLLECTION_PROP, collectionName, SHARD_ID_PROP, sliceName), results); + new DeleteShardCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, async) + .withClusterState(clusterState), + new ZkNodeProps(COLLECTION_PROP, collectionName, SHARD_ID_PROP, sliceName), + results); throw e; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java index 64e308259140..97e24fc82795 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java @@ -60,7 +60,8 @@ public CreateSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); @@ -96,7 +97,8 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList Map shardByCoreName = new HashMap<>(); ShardHandler shardHandler = ccc.newShardHandler(); - final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); + final ShardRequestTracker shardRequestTracker = + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); for (Slice slice : ccc.getZkStateReader().getClusterState().getCollection(collectionName).getSlices()) { for (Replica replica : slice.getReplicas()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java index cda76a8b603d..1d90b4e87977 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteAliasCmd.java @@ -31,7 +31,8 @@ public DeleteAliasCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String aliasName = message.getStr(NAME); ZkStateReader zkStateReader = ccc.getZkStateReader(); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java index ff0593cd97e0..dfec0e3bb0e7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java @@ -70,7 +70,8 @@ public DeleteBackupCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION); String backupName = message.getStr(NAME); String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java index 872f2419c655..9c292863ece8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java @@ -62,7 +62,8 @@ public DeleteCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { Object o = message.get(MaintainRoutedAliasCmd.INVOKED_BY_ROUTED_ALIAS); if (o != null) { // this will ensure the collection is removed from the alias before it disappears. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java index cbd46c0627ca..656eb3747098 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java @@ -34,11 +34,14 @@ public DeleteNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { CollectionHandlingUtils.checkRequired(message, "node"); String node = message.getStr("node"); - List sourceReplicas = ReplicaMigrationUtils.getReplicasOfNode(node, adminCmdContext.getClusterState()); - List singleReplicas = verifyReplicaAvailability(sourceReplicas, adminCmdContext.getClusterState()); + List sourceReplicas = + ReplicaMigrationUtils.getReplicasOfNode(node, adminCmdContext.getClusterState()); + List singleReplicas = + verifyReplicaAvailability(sourceReplicas, adminCmdContext.getClusterState()); if (!singleReplicas.isEmpty()) { results.add( "failure", @@ -47,8 +50,7 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList + ": " + singleReplicas); } else { - ReplicaMigrationUtils.cleanupReplicas( - results, adminCmdContext, sourceReplicas, ccc); + ReplicaMigrationUtils.cleanupReplicas(results, adminCmdContext, sourceReplicas, ccc); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java index c506f4576071..5246401db6e9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java @@ -35,7 +35,6 @@ import org.apache.solr.cloud.api.collections.CollApiCmds.CollectionApiCommand; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -60,7 +59,8 @@ public DeleteReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { deleteReplica(adminCmdContext, message, results, null); } @@ -102,7 +102,8 @@ void deleteReplica( SolrException.ErrorCode.BAD_REQUEST, "Invalid shard name : " + shard + " in collection : " + collectionName); } - deleteCore(adminCmdContext, coll, shard, replicaName, message, results, onComplete, parallel, true); + deleteCore( + adminCmdContext, coll, shard, replicaName, message, results, onComplete, parallel, true); } /** @@ -170,7 +171,8 @@ void deleteReplicaBasedOnCount( for (String replica : replicas) { log.debug("Deleting replica {} for shard {} based on count {}", replica, shardId, count); // don't verify with the placement plugin - we already did it - deleteCore(adminCmdContext, coll, shardId, replica, message, results, onComplete, parallel, false); + deleteCore( + adminCmdContext, coll, shardId, replica, message, results, onComplete, parallel, false); } results.add("shard_id", shardId); results.add("replicas_deleted", replicas); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java index 0c18c08c4af1..1234925c94c7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java @@ -57,7 +57,8 @@ public DeleteShardCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP); String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP); @@ -127,8 +128,7 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList List replicas = getReplicasForSlice(collectionName, slice); CountDownLatch cleanupLatch = new CountDownLatch(replicas.size()); for (ZkNodeProps r : replicas) { - final ZkNodeProps replica = - r.plus(message.getProperties()).plus("parallel", "true"); + final ZkNodeProps replica = r.plus(message.getProperties()).plus("parallel", "true"); if (log.isInfoEnabled()) { log.info( "Deleting replica for collection={} shard={} on node={}", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java index 1a489722a2ea..ffb93c48fc2e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java @@ -58,7 +58,8 @@ public DeleteSnapshotCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(COLLECTION_PROP); boolean followAliases = message.getBool(FOLLOW_ALIASES, false); String collectionName; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index a2185412c50d..d0b3fbead5de 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -250,9 +250,7 @@ public void runConfigSetCommand( * */ public OverseerSolrResponse runCollectionCommand( - AdminCmdContext adminCmdContext, - ZkNodeProps message, - long timeoutMs) { + AdminCmdContext adminCmdContext, ZkNodeProps message, long timeoutMs) { // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it // through this method earlier). See stopAndWaitForPendingTasksToComplete() below if (shuttingDown) { @@ -263,7 +261,9 @@ public OverseerSolrResponse runCollectionCommand( if (log.isInfoEnabled()) { log.info( - "Running Collection API locally for {} asyncId={}", adminCmdContext.getAction().name(), adminCmdContext.getAsyncId()); + "Running Collection API locally for {} asyncId={}", + adminCmdContext.getAction().name(), + adminCmdContext.getAsyncId()); } // Following the call below returning true, we must eventually cancel or complete the task. @@ -275,8 +275,7 @@ public OverseerSolrResponse runCollectionCommand( "Task with the same requestid already exists. (" + adminCmdContext.getAsyncId() + ")"); } - CollectionCommandRunner commandRunner = - new CollectionCommandRunner(adminCmdContext, message); + CollectionCommandRunner commandRunner = new CollectionCommandRunner(adminCmdContext, message); final Future taskFuture; try { taskFuture = commandsExecutor.submit(commandRunner); @@ -296,12 +295,15 @@ public OverseerSolrResponse runCollectionCommand( return taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS); } catch (TimeoutException te) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " timed out after " + timeoutMs + "ms"); + SolrException.ErrorCode.SERVER_ERROR, + adminCmdContext.getAction() + " timed out after " + timeoutMs + "ms"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " interrupted", e); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " interrupted", e); } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " failed", e); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, adminCmdContext.getAction() + " failed", e); } } else { // Async calls do not wait for the command to finish but get instead back the async id (that @@ -422,10 +424,13 @@ public OverseerSolrResponse call() { message); } - CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(adminCmdContext.getAction()); + CollApiCmds.CollectionApiCommand command = + commandMapper.getActionCommand(adminCmdContext.getAction()); if (command != null) { command.call( - adminCmdContext.withClusterState(ccc.getSolrCloudManager().getClusterState()), message, results); + adminCmdContext.withClusterState(ccc.getSolrCloudManager().getClusterState()), + message, + results); } else { asyncTaskTracker.cancelAsyncId(adminCmdContext.getAsyncId()); // Seeing this is a bug, not bad user data @@ -443,7 +448,9 @@ public OverseerSolrResponse call() { } catch (SolrException se) { if (log.isErrorEnabled()) { log.error( - "Error when releasing collection locks for operation {}", adminCmdContext.getAction(), se); + "Error when releasing collection locks for operation {}", + adminCmdContext.getAction(), + se); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index e6db84478038..39b81cd434bd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -59,7 +59,8 @@ public InstallShardDataCmd(CollectionCommandContext ccc) { @Override @SuppressWarnings("unchecked") - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { final RemoteMessage typedMessage = new ObjectMapper().convertValue(message.getProperties(), RemoteMessage.class); final CollectionHandlingUtils.ShardRequestTracker shardRequestTracker = diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java index 921c0498e2d2..e6c316c2aa6c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java @@ -27,7 +27,6 @@ import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -54,13 +53,15 @@ public class MaintainRoutedAliasCmd extends AliasCmd { * standard OCP timeout) to prevent large batches of add's from sending a message to the overseer * for every document added in RoutedAliasUpdateProcessor. */ - static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) throws Exception { + static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol) + throws Exception { final SolrResponse rsp = collHandler.submitCollectionApiCommand( new AdminCmdContext(CollectionParams.CollectionAction.MAINTAINROUTEDALIAS), - new ZkNodeProps(Map.of( - CollectionParams.NAME, aliasName, - MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol))); + new ZkNodeProps( + Map.of( + CollectionParams.NAME, aliasName, + MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol))); if (rsp.getException() != null) { throw rsp.getException(); } @@ -105,8 +106,7 @@ private void removeCollectionFromAlias( } @Override - public void call( - AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { // ---- PARSE PRIMARY MESSAGE PARAMS // important that we use NAME for the alias as that is what the Overseer will get a lock on @@ -142,7 +142,8 @@ public void call( .execute( () -> { try { - deleteTargetCollection(adminCmdContext, results, aliasName, aliasesManager, action); + deleteTargetCollection( + adminCmdContext, results, aliasName, aliasesManager, action); } catch (Exception e) { log.warn( "Deletion of {} by {} {} failed (this might be ok if two clients were", @@ -215,6 +216,10 @@ public void deleteTargetCollection( () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection)); delProps.put(NAME, action.targetCollection); ZkNodeProps messageDelete = new ZkNodeProps(delProps); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE), messageDelete, results); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE), + messageDelete, + results); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index 1c0a34c1a02a..ba56c8ab9dda 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -37,7 +37,6 @@ import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CompositeIdRouter; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocRouter; @@ -64,7 +63,8 @@ public MigrateCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extSourceCollectionName = message.getStr("collection"); String splitKey = message.getStr("split.key"); String extTargetCollectionName = message.getStr("target.collection"); @@ -87,13 +87,15 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList targetCollectionName = extTargetCollectionName; } - DocCollection sourceCollection = adminCmdContext.getClusterState().getCollection(sourceCollectionName); + DocCollection sourceCollection = + adminCmdContext.getClusterState().getCollection(sourceCollectionName); if (sourceCollection == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName); } - DocCollection targetCollection = adminCmdContext.getClusterState().getCollection(targetCollectionName); + DocCollection targetCollection = + adminCmdContext.getClusterState().getCollection(targetCollectionName); if (targetCollection == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, @@ -173,7 +175,12 @@ private void migrateKey( log.info("Deleting temporary collection: {}", tempSourceCollectionName); try { new DeleteCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(DELETE).withClusterState(zkStateReader.getClusterState()), new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), results); + .call( + adminCmdContext + .subRequestContext(DELETE) + .withClusterState(zkStateReader.getClusterState()), + new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), + results); adminCmdContext.withClusterState(zkStateReader.getClusterState()); } catch (Exception e) { log.warn( @@ -308,11 +315,20 @@ private void migrateKey( } log.info("Creating temporary collection: {}", props); - new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CREATE, internalAsyncId), new ZkNodeProps(props), results); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext.subRequestContext(CREATE, internalAsyncId), + new ZkNodeProps(props), + results); // refresh cluster state adminCmdContext.withClusterState(zkStateReader.getClusterState()); Slice tempSourceSlice = - adminCmdContext.getClusterState().getCollection(tempSourceCollectionName).getSlices().iterator().next(); + adminCmdContext + .getClusterState() + .getCollection(tempSourceCollectionName) + .getSlices() + .iterator() + .next(); Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 120000); @@ -393,7 +409,9 @@ private void migrateKey( } } // add async param - new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(props), results, null); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(props), results, null); { final ShardRequestTracker syncRequestTracker = @@ -447,7 +465,8 @@ private void migrateKey( params.set(CoreAdminParams.SRC_CORE, tempCollectionReplica2); { - final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); + final ShardRequestTracker shardRequestTracker = + CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc); shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler); String msg = @@ -475,7 +494,12 @@ private void migrateKey( try { log.info("Deleting temporary collection: {}", tempSourceCollectionName); new DeleteCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(DELETE).withClusterState(zkStateReader.getClusterState()), new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), results); + .call( + adminCmdContext + .subRequestContext(DELETE) + .withClusterState(zkStateReader.getClusterState()), + new ZkNodeProps(Map.of(NAME, tempSourceCollectionName)), + results); } catch (Exception e) { log.error( "Unable to delete temporary collection: {}. Please remove it manually", diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java index 963a2215af5d..e4381ad888e7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateReplicasCmd.java @@ -46,7 +46,8 @@ public MigrateReplicasCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); Set sourceNodes = getNodesFromParam(message, CollectionParams.SOURCE_NODES); Set targetNodes = getNodesFromParam(message, CollectionParams.TARGET_NODES); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 9c2b0b43ee47..89a5f32c9309 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -36,7 +36,6 @@ import org.apache.solr.cloud.ActiveReplicaWatcher; import org.apache.solr.common.SolrCloseableLatch; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -59,11 +58,14 @@ public MoveReplicaCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { moveReplica(adminCmdContext, message, results); } - private void moveReplica(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + private void moveReplica( + AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { if (log.isDebugEnabled()) { log.debug("moveReplica() : {}", Utils.toJSONString(message)); } @@ -91,7 +93,10 @@ private void moveReplica(AdminCmdContext adminCmdContext, ZkNodeProps message, N if (!adminCmdContext.getClusterState().getLiveNodes().contains(targetNode)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Target node: " + targetNode + " not in live nodes: " + adminCmdContext.getClusterState().getLiveNodes()); + "Target node: " + + targetNode + + " not in live nodes: " + + adminCmdContext.getClusterState().getLiveNodes()); } Replica replica = null; if (message.containsKey(REPLICA_PROP)) { @@ -161,14 +166,7 @@ private void moveReplica(AdminCmdContext adminCmdContext, ZkNodeProps message, N } else { log.debug("-- moveNormalReplica (inPlaceMove={}, isSharedFS={}", inPlaceMove, isSharedFS); moveNormalReplica( - adminCmdContext, - results, - targetNode, - coll, - replica, - slice, - timeout, - waitForFinalState); + adminCmdContext, results, targetNode, coll, replica, slice, timeout, waitForFinalState); } } @@ -194,7 +192,11 @@ private void moveSharedFsReplica( NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), removeReplicasProps, deleteResult, null); + .deleteReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), + removeReplicasProps, + deleteResult, + null); } catch (SolrException e) { // assume this failed completely so there's nothing to roll back deleteResult.add("failure", e.toString()); @@ -253,7 +255,13 @@ private void moveSharedFsReplica( NamedList addResult = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, addResult, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + addResult, + null); } catch (Exception e) { // fatal error - try rolling back String errorString = @@ -269,7 +277,13 @@ private void moveSharedFsReplica( addReplicasProps = addReplicasProps.plus(CoreAdminParams.NODE, replica.getNodeName()); NamedList rollback = new NamedList<>(); new AddReplicaCmd(ccc) - .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, rollback, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + rollback, + null); if (rollback.get("failure") != null) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -297,7 +311,13 @@ private void moveSharedFsReplica( NamedList rollback = new NamedList<>(); try { new AddReplicaCmd(ccc) - .addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), addReplicasProps, rollback, null); + .addReplica( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), + addReplicasProps, + rollback, + null); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -359,7 +379,13 @@ private void moveNormalReplica( SolrCloseableLatch countDownLatch = new SolrCloseableLatch(1, ccc.getCloseableToLatchOn()); ActiveReplicaWatcher watcher = null; ZkNodeProps props = - new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA), addReplicasProps, addResult, null).get(0); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA), + addReplicasProps, + addResult, + null) + .get(0); log.debug("props {}", props); if (replica.equals(slice.getLeader()) || waitForFinalState) { watcher = @@ -418,7 +444,11 @@ private void moveNormalReplica( NamedList deleteResult = new NamedList<>(); try { new DeleteReplicaCmd(ccc) - .deleteReplica(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), removeReplicasProps, deleteResult, null); + .deleteReplica( + adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA), + removeReplicasProps, + deleteResult, + null); } catch (SolrException e) { deleteResult.add("failure", e.toString()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java index 9fad543952ad..32f10bc1c6f0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java @@ -54,7 +54,8 @@ public OverseerRoleCmd( } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { if (ccc.isDistributedCollectionAPI()) { // No Overseer (not accessible from Collection API command execution in any case) so this // command can't be run... diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java index 2b1835466b2f..031808b5b058 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java @@ -156,7 +156,8 @@ public OverseerStatusCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { // If Collection API execution is distributed, we're not running on the Overseer node so can't // return any Overseer stats. if (ccc.getCoreContainer().getZkController().getDistributedCommandRunner().isPresent()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index 7fd87650dd27..e72cfbe04a4c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -169,7 +169,8 @@ public ReindexCollectionCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { log.debug("*** called: {}", message); @@ -293,7 +294,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList } if (clusterState.hasCollection(chkCollection)) { // delete the checkpoint collection - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), new ZkNodeProps(CommonParams.NAME, chkCollection), cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(clusterState), + new ZkNodeProps(CommonParams.NAME, chkCollection), + cmdResults); CollectionHandlingUtils.checkResults( "deleting old checkpoint collection " + chkCollection, cmdResults, true); } @@ -331,7 +338,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList // create the target collection cmd = new ZkNodeProps(propMap); cmdResults = new NamedList<>(); - new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); createdTarget = true; CollectionHandlingUtils.checkResults( "creating target collection " + targetCollection, cmdResults, true); @@ -345,7 +358,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList CollectionAdminParams.COLL_CONF, "_default", CommonAdminParams.WAIT_FOR_FINAL_STATE, "true"); cmdResults = new NamedList<>(); - new CreateCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); + new CreateCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "creating checkpoint collection " + chkCollection, cmdResults, true); // wait for a while until we see both collections @@ -470,7 +489,13 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList log.debug("- setting up alias from {} to {}", extCollection, targetCollection); cmd = new ZkNodeProps(CommonParams.NAME, extCollection, "collections", targetCollection); cmdResults = new NamedList<>(); - new CreateAliasCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.CREATEALIAS, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); + new CreateAliasCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.CREATEALIAS, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true); reindexingState.put("alias", extCollection + " -> " + targetCollection); @@ -489,21 +514,28 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList // 6. delete the checkpoint collection log.debug("- deleting {}", chkCollection); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), new ZkNodeProps(CommonParams.NAME, chkCollection), cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + new ZkNodeProps(CommonParams.NAME, chkCollection), + cmdResults); CollectionHandlingUtils.checkResults( "deleting checkpoint collection " + chkCollection, cmdResults, true); // 7. optionally delete the source collection if (removeSource) { log.debug("- deleting source collection"); - cmd = - new ZkNodeProps( - CommonParams.NAME, - collection, - FOLLOW_ALIASES, - "false"); + cmd = new ZkNodeProps(CommonParams.NAME, collection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(ccc.getSolrCloudManager().getClusterState()), cmd, cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(ccc.getSolrCloudManager().getClusterState()), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "deleting source collection " + collection, cmdResults, true); } else { @@ -875,26 +907,29 @@ private void cleanup( && clusterState.hasCollection(targetCollection)) { log.debug(" -- removing {}", targetCollection); ZkNodeProps cmd = - new ZkNodeProps( - CommonParams.NAME, - targetCollection, - FOLLOW_ALIASES, - "false"); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), cmd, cmdResults); + new ZkNodeProps(CommonParams.NAME, targetCollection, FOLLOW_ALIASES, "false"); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(clusterState), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting target collection " + targetCollection, cmdResults, false); } // remove chk collection if (clusterState.hasCollection(chkCollection)) { log.debug(" -- removing {}", chkCollection); - ZkNodeProps cmd = - new ZkNodeProps( - CommonParams.NAME, - chkCollection, - FOLLOW_ALIASES, - "false"); + ZkNodeProps cmd = new ZkNodeProps(CommonParams.NAME, chkCollection, FOLLOW_ALIASES, "false"); cmdResults = new NamedList<>(); - new DeleteCollectionCmd(ccc).call(adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETE, null).withClusterState(clusterState), cmd, cmdResults); + new DeleteCollectionCmd(ccc) + .call( + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETE, null) + .withClusterState(clusterState), + cmd, + cmdResults); CollectionHandlingUtils.checkResults( "CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java index ff99d0f8a43b..be0c3cfa32b4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java @@ -40,7 +40,8 @@ public RenameCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String extCollectionName = message.getStr(CoreAdminParams.NAME); String target = message.getStr(CollectionAdminParams.TARGET); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index f44a9e092033..382f7dd9e2eb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -43,7 +43,8 @@ public ReplaceNodeCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { ZkStateReader zkStateReader = ccc.getZkStateReader(); String source = message.getStr(CollectionParams.SOURCE_NODE); String target = message.getStr(CollectionParams.TARGET_NODE); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java index 7ac51487bc87..498695f4f9fe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java @@ -112,7 +112,9 @@ static boolean migrateReplicas( final ZkNodeProps addedReplica = new AddReplicaCmd(ccc) .addReplica( - adminCmdContext.subRequestContext(CollectionParams.CollectionAction.ADDREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.ADDREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), msg, nl, () -> { @@ -209,7 +211,9 @@ static boolean migrateReplicas( try { new DeleteReplicaCmd(ccc) .deleteReplica( - adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), createdReplica.plus("parallel", "true"), deleteResult, () -> { @@ -237,8 +241,7 @@ static boolean migrateReplicas( // we have reached this far, meaning all replicas should have been recreated. // now cleanup the original replicas - return cleanupReplicas( - results, adminCmdContext, movements.keySet(), ccc); + return cleanupReplicas(results, adminCmdContext, movements.keySet(), ccc); } static boolean cleanupReplicas( @@ -265,7 +268,9 @@ static boolean cleanupReplicas( ZkNodeProps cmdMessage = sourceReplica.toFullProps(); new DeleteReplicaCmd(ccc) .deleteReplica( - adminCmdContext.subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA).withClusterState(ccc.getZkStateReader().getClusterState()), + adminCmdContext + .subRequestContext(CollectionParams.CollectionAction.DELETEREPLICA) + .withClusterState(ccc.getZkStateReader().getClusterState()), cmdMessage.plus("parallel", "true"), deleteResult, () -> { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index 0019d5855c5c..93df952eb271 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -93,7 +93,8 @@ public RestoreCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { try (RestoreContext restoreContext = new RestoreContext(adminCmdContext, message, ccc)) { if (adminCmdContext.getClusterState().hasCollection(restoreContext.restoreCollectionName)) { RestoreOnExistingCollection restoreOnExistingCollection = @@ -130,7 +131,8 @@ private void requestReplicasToRestore( } params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(adminCmdContext.getClusterState(), params, null, slice, shardHandler); + shardRequestTracker.sliceCmd( + adminCmdContext.getClusterState(), params, null, slice, shardHandler); } shardRequestTracker.processResponses( new NamedList<>(), shardHandler, true, "Could not restore core"); @@ -158,7 +160,8 @@ private static class RestoreContext implements Closeable { final DocCollection backupCollectionState; final ShardHandler shardHandler; - private RestoreContext(AdminCmdContext adminCmdContext, ZkNodeProps message, CollectionCommandContext ccc) + private RestoreContext( + AdminCmdContext adminCmdContext, ZkNodeProps message, CollectionCommandContext ccc) throws IOException { this.adminCmdContext = adminCmdContext; this.restoreCollectionName = message.getStr(COLLECTION_PROP); @@ -256,11 +259,15 @@ public void process(NamedList results, RestoreContext rc) throws Excepti getReplicaPositions(rc.restoreCollectionName, rc.nodeList, sliceNames); createSingleReplicaPerShard( - results, restoreCollection, rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), replicaPositions); + results, + restoreCollection, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + replicaPositions); Object failures = results.get("failure"); if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { log.error("Restore failed to create initial replicas."); - CollectionHandlingUtils.cleanupCollection(rc.adminCmdContext, rc.restoreCollectionName, new NamedList<>(), ccc); + CollectionHandlingUtils.cleanupCollection( + rc.adminCmdContext, rc.restoreCollectionName, new NamedList<>(), ccc); return; } @@ -276,7 +283,11 @@ public void process(NamedList results, RestoreContext rc) throws Excepti rc.repo, rc.shardHandler); markAllShardsAsActive(restoreCollection); - addReplicasToShards(results, restoreCollection, replicaPositions, rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState())); + addReplicasToShards( + results, + restoreCollection, + replicaPositions, + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState())); restoringAlias(rc.backupProperties); log.info("Completed restoring collection={} backupName={}", restoreCollection, rc.backupName); @@ -366,7 +377,10 @@ private void createCoreLessCollection( } new CreateCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(CREATE, null), new ZkNodeProps(propMap), new NamedList<>()); + .call( + adminCmdContext.subRequestContext(CREATE, null), + new ZkNodeProps(propMap), + new NamedList<>()); // note: when createCollection() returns, the collection exists (no race) } @@ -553,7 +567,11 @@ private void addReplicasToShards( CollectionHandlingUtils.addPropertyParams(message, propMap); new AddReplicaCmd(ccc) - .addReplica(adminCmdContext.subRequestContext(ADDREPLICA), new ZkNodeProps(propMap), results, null); + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA), + new ZkNodeProps(propMap), + results, + null); } } } @@ -604,7 +622,9 @@ public void process(RestoreContext rc, NamedList results) throws Excepti ClusterState clusterState = rc.zkStateReader.getClusterState(); DocCollection restoreCollection = clusterState.getCollection(rc.restoreCollectionName); - enableReadOnly(rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), restoreCollection); + enableReadOnly( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + restoreCollection); try { requestReplicasToRestore( results, @@ -615,7 +635,9 @@ public void process(RestoreContext rc, NamedList results) throws Excepti rc.repo, rc.shardHandler); } finally { - disableReadOnly(rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), restoreCollection); + disableReadOnly( + rc.adminCmdContext.withClusterState(rc.zkStateReader.getClusterState()), + restoreCollection); } } @@ -628,7 +650,8 @@ private void disableReadOnly(AdminCmdContext adminCmdContext, DocCollection rest ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, null); new CollApiCmds.ModifyCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); + .call( + adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } private void enableReadOnly(AdminCmdContext adminCmdContext, DocCollection restoreCollection) @@ -640,7 +663,8 @@ private void enableReadOnly(AdminCmdContext adminCmdContext, DocCollection resto ZkStateReader.COLLECTION_PROP, restoreCollection.getName(), ZkStateReader.READ_ONLY, "true"); new CollApiCmds.ModifyCollectionCmd(ccc) - .call(adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); + .call( + adminCmdContext.subRequestContext(MODIFYCOLLECTION, null), params, new NamedList<>()); } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java index eac8df809e6a..f43e9ceeef21 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SetAliasPropCmd.java @@ -45,7 +45,8 @@ public SetAliasPropCmd(CollectionCommandContext ccc) { } @Override - public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + throws Exception { String aliasName = message.getStr(NAME); final ZkStateReader.AliasesManager aliasesManager = ccc.getZkStateReader().aliasesManager; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 83cf10467f01..c2f0fd3d500b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -100,8 +100,7 @@ public SplitShardCmd(CollectionCommandContext ccc) { } @Override - public void call( - AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) + public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList results) throws Exception { split(adminCmdContext, message, results); } @@ -338,7 +337,13 @@ public boolean split( propMap.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(propMap); try { - new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), m, new NamedList<>()); + new DeleteShardCmd(ccc) + .call( + adminCmdContext + .subRequestContext(DELETESHARD, null) + .withClusterState(clusterState), + m, + new NamedList<>()); } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -417,7 +422,11 @@ public boolean split( } } new AddReplicaCmd(ccc) - .addReplica(adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), new ZkNodeProps((MapWriter) propMap), results, null); + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), + new ZkNodeProps((MapWriter) propMap), + results, + null); } { @@ -767,7 +776,12 @@ public boolean split( t = timings.sub("createCoresForReplicas"); // 11. now actually create replica cores on sub shard nodes for (Map replica : replicas) { - new AddReplicaCmd(ccc).addReplica(adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), new ZkNodeProps(replica), results, null); + new AddReplicaCmd(ccc) + .addReplica( + adminCmdContext.subRequestContext(ADDREPLICA).withClusterState(clusterState), + new ZkNodeProps(replica), + results, + null); } assert TestInjection.injectSplitFailureAfterReplicaCreation(); @@ -814,7 +828,12 @@ public boolean split( } finally { if (!success) { cleanupAfterFailure( - adminCmdContext, zkStateReader, collectionName, parentSlice.getName(), subSlices, offlineSlices); + adminCmdContext, + zkStateReader, + collectionName, + parentSlice.getName(), + subSlices, + offlineSlices); unlockForSplit(ccc.getSolrCloudManager(), collectionName, parentSlice.getName()); } } @@ -989,7 +1008,11 @@ private void cleanupAfterFailure( props.put(SHARD_ID_PROP, subSlice); ZkNodeProps m = new ZkNodeProps(props); try { - new DeleteShardCmd(ccc).call(adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), m, new NamedList<>()); + new DeleteShardCmd(ccc) + .call( + adminCmdContext.subRequestContext(DELETESHARD, null).withClusterState(clusterState), + m, + new NamedList<>()); } catch (Exception e) { log.warn( "Cleanup failed after failed split of {}/{} : (deleting existing sub shard{})", diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index cc98e366cb9b..20213193389b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -320,8 +320,9 @@ void invokeAction( return; } - AdminCmdContext adminCmdContext = new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); - adminCmdContext.setCallingLockIds((String)req.getContext().get(CALLING_LOCK_IDS_HEADER)); + AdminCmdContext adminCmdContext = + new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); + adminCmdContext.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index f7033d62948d..26afe771d5fc 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP; @@ -25,7 +24,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import java.lang.invoke.MethodHandles; import java.util.HashMap; @@ -47,7 +45,6 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; @@ -461,7 +458,9 @@ private void rejoinElectionQueue( String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); asyncRequests.add(asyncId); - collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId), new ZkNodeProps(propMap)); // ignore response; we construct our own + collectionsHandler.submitCollectionApiCommand( + new AdminCmdContext(REBALANCELEADERS, asyncId), + new ZkNodeProps(propMap)); // ignore response; we construct our own } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java index 1b15d2dd8184..70d21db5b1c0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; @@ -24,7 +23,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -35,14 +33,11 @@ import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -81,10 +76,7 @@ public SolrJerseyResponse addReplicaProperty( final ZkNodeProps remoteMessage = createRemoteMessage(collName, shardName, replicaName, propertyName, requestBody); submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.ADDREPLICAPROP, - remoteMessage, - null); + response, CollectionParams.CollectionAction.ADDREPLICAPROP, remoteMessage, null); disableResponseCaching(); return response; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 33a5a0904876..2e98fa18b46e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -21,7 +21,6 @@ import java.util.Map; import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; @@ -133,11 +132,9 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( ZkNodeProps remoteMessage, String asyncId) throws Exception { - var remoteResponse = submitRemoteMessageAndHandleResponse( - response, - new AdminCmdContext(action, asyncId), - remoteMessage - ); + var remoteResponse = + submitRemoteMessageAndHandleResponse( + response, new AdminCmdContext(action, asyncId), remoteMessage); if (asyncId != null) { response.requestId = asyncId; @@ -157,20 +154,18 @@ protected SolrResponse submitRemoteMessageAndHandleResponse( ZkNodeProps remoteMessage) throws Exception { return submitRemoteMessageAndHandleResponse( - response, - new AdminCmdContext(action, null), - remoteMessage - ); + response, new AdminCmdContext(action, null), remoteMessage); } protected SolrResponse submitRemoteMessageAndHandleResponse( - SolrJerseyResponse response, - AdminCmdContext adminCmdContext, - ZkNodeProps remoteMessage) + SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( - coreContainer.getZkController(), adminCmdContext, remoteMessage, DEFAULT_COLLECTION_OP_TIMEOUT); + coreContainer.getZkController(), + adminCmdContext, + remoteMessage, + DEFAULT_COLLECTION_OP_TIMEOUT); if (remoteResponse.getException() != null) { throw remoteResponse.getException(); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java index 36bc6f27875c..2e36aa703128 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java @@ -16,10 +16,7 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; @@ -34,14 +31,12 @@ import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -146,7 +141,11 @@ public SolrJerseyResponse deleteAliasProperty(String aliasName, String propName) return response; } - private void modifyAliasProperty(SubResponseAccumulatingJerseyResponse response, String alias, String proertyName, Object value) + private void modifyAliasProperty( + SubResponseAccumulatingJerseyResponse response, + String alias, + String proertyName, + Object value) throws Exception { Map props = new HashMap<>(); // value can be null @@ -159,7 +158,11 @@ private void modifyAliasProperty(SubResponseAccumulatingJerseyResponse response, /** * @param alias alias */ - private void modifyAliasProperties(SubResponseAccumulatingJerseyResponse response, String alias, Map properties, String async) + private void modifyAliasProperties( + SubResponseAccumulatingJerseyResponse response, + String alias, + Map properties, + String async) throws Exception { // Note: success/no-op in the event of no properties supplied is intentional. Keeps code // simple and one less case for api-callers to check for. @@ -170,10 +173,10 @@ private void modifyAliasProperties(SubResponseAccumulatingJerseyResponse respons submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.ALIASPROP, - new ZkNodeProps(Map.of( - NAME, alias, - PROPERTIES, properties - )), + new ZkNodeProps( + Map.of( + NAME, alias, + PROPERTIES, properties)), async); disableResponseCaching(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java index 76bdec315a56..c7f1d73816c3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java @@ -18,9 +18,7 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.NODES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -30,11 +28,9 @@ import org.apache.solr.client.api.model.BalanceReplicasRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -59,10 +55,7 @@ public SolrJerseyResponse balanceReplicas(BalanceReplicasRequestBody requestBody fetchAndValidateZooKeeperAwareCoreContainer(); final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); submitRemoteMessageAndHandleResponse( - response, - CollectionAction.BALANCE_REPLICAS, - remoteMessage, - requestBody.async); + response, CollectionAction.BALANCE_REPLICAS, remoteMessage, requestBody.async); disableResponseCaching(); return response; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java index ca627de75518..3b655bbb46fc 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceShardUnique.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_ACTIVE_NODES; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java index 168b2a51c0c0..ec877776ac29 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java @@ -18,7 +18,6 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.COLLECTIONS; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.RoutedAlias.CATEGORY; import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX; import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTER_TYPE_NAME; @@ -30,7 +29,6 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.START; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -46,7 +44,6 @@ import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.TimeRoutedAliasProperties; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.request.RoutedAliasTypes; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.cloud.api.collections.RoutedAlias; @@ -61,7 +58,6 @@ import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -109,10 +105,7 @@ public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws } submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.CREATEALIAS, - remoteMessage, - requestBody.async); + response, CollectionParams.CollectionAction.CREATEALIAS, remoteMessage, requestBody.async); return response; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index 1c4953b57e5e..1cf629836938 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -19,7 +19,6 @@ import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.ROUTER_KEY; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.SHARD_NAMES; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET_SHUFFLE; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; @@ -36,7 +35,6 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CoreAdminParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.handler.admin.CollectionsHandler.waitForActiveCollection; import static org.apache.solr.handler.api.V2ApiUtils.flattenMapWithPrefix; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -70,7 +68,6 @@ import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -111,10 +108,7 @@ public SubResponseAccumulatingJerseyResponse createCollection( final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); final SolrResponse remoteResponse = submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.CREATE, - remoteMessage, - requestBody.async); + response, CollectionParams.CollectionAction.CREATE, remoteMessage, requestBody.async); // Even if Overseer does wait for the collection to be created, it sees a different cluster // state than this node, so this wait is required to make sure the local node Zookeeper watches diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java index 83fb2e327e20..fa2e7a69b72d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.INDEX_BACKUP_STRATEGY; @@ -30,7 +29,6 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.COMMIT_NAME; import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.handler.admin.api.CreateCollection.copyPrefixedPropertiesWithoutPrefix; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -42,17 +40,14 @@ import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; import org.apache.solr.client.api.model.CreateCollectionBackupResponseBody; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.jersey.SolrJacksonMapper; import org.apache.solr.request.SolrQueryRequest; @@ -115,10 +110,7 @@ public SolrJerseyResponse createCollectionBackup( final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, backupName, requestBody); final SolrResponse remoteResponse = submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.BACKUP, - remoteMessage, - requestBody.async); + response, CollectionParams.CollectionAction.BACKUP, remoteMessage, requestBody.async); objectMapper.updateValue(response, remoteResponse.getResponse()); return response; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java index ebf4fcb669f4..88c747ffb932 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -29,7 +26,6 @@ import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; @@ -37,7 +33,6 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.snapshots.SolrSnapshotManager; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java index 30f7d29103c8..af0a6a139233 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateReplica.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java index d2471bb2ccf8..8ca11a7298ed 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteAlias.java @@ -17,24 +17,16 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteAliasApi; -import org.apache.solr.client.api.model.AsyncJerseyResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -50,7 +42,8 @@ public DeleteAlias( @Override @PermissionName(COLL_EDIT_PERM) - public SubResponseAccumulatingJerseyResponse deleteAlias(String aliasName, String asyncId) throws Exception { + public SubResponseAccumulatingJerseyResponse deleteAlias(String aliasName, String asyncId) + throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java index 90ae0c639b57..414ad539edae 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollection.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -28,11 +25,9 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteCollectionApi; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -63,10 +58,7 @@ public SubResponseAccumulatingJerseyResponse deleteCollection( final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, followAliases); submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.DELETE, - remoteMessage, - asyncId); + response, CollectionParams.CollectionAction.DELETE, remoteMessage, asyncId); return response; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java index f2362c83aa66..a2612984123d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionBackup.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; @@ -113,8 +112,7 @@ public BackupDeletionResponseBody deleteMultipleBackupsByRecency( location = getAndValidateBackupLocation(repositoryName, location); final ZkNodeProps remoteMessage = - createRemoteMessage( - backupName, null, versionsToRetain, null, location, repositoryName); + createRemoteMessage(backupName, null, versionsToRetain, null, location, repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEBACKUP, remoteMessage, asyncId); @@ -139,12 +137,7 @@ public PurgeUnusedResponse garbageCollectUnusedBackupFiles( final ZkNodeProps remoteMessage = createRemoteMessage( - backupName, - null, - null, - Boolean.TRUE, - requestBody.location, - requestBody.repositoryName); + backupName, null, null, Boolean.TRUE, requestBody.location, requestBody.repositoryName); final var remoteResponse = submitRemoteMessageAndHandleResponse( response, diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java index 39e87894e5bf..086fde0cff0b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java @@ -16,11 +16,8 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -28,12 +25,10 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.DeleteCollectionSnapshotResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -75,7 +70,8 @@ public DeleteCollectionSnapshotResponse deleteCollectionSnapshot( return response; } - public static ZkNodeProps createRemoteMessage(String collectionName, boolean followAliases, String snapshotName) { + public static ZkNodeProps createRemoteMessage( + String collectionName, boolean followAliases, String snapshotName) { final Map remoteMessage = new HashMap<>(); remoteMessage.put(COLLECTION_PROP, collectionName); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java index 2164e8399485..43c95ad3cb51 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteNode.java @@ -16,26 +16,21 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.NODE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteNodeApi; import org.apache.solr.client.api.model.DeleteNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -58,8 +53,8 @@ public DeleteNode( @Override @PermissionName(COLL_EDIT_PERM) - public SubResponseAccumulatingJerseyResponse deleteNode(String nodeName, DeleteNodeRequestBody requestBody) - throws Exception { + public SubResponseAccumulatingJerseyResponse deleteNode( + String nodeName, DeleteNodeRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); submitRemoteMessageAndHandleResponse( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java index eb67a44986df..d4da24690906 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplicaProperty.java @@ -17,25 +17,21 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import jakarta.inject.Inject; import java.util.Map; import org.apache.solr.client.api.endpoint.DeleteReplicaPropertyApi; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.RequiredSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index a3ddeb4524bc..3d5f9f5badf3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -26,7 +25,6 @@ import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.InstallShardDataRequestBody; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.cloud.api.collections.InstallShardDataCmd; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -36,7 +34,6 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java index fbdc482c41f2..2bfe996099f2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java @@ -16,12 +16,9 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.SOURCE_NODES; import static org.apache.solr.common.params.CollectionParams.TARGET_NODES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -31,12 +28,10 @@ import org.apache.solr.client.api.model.MigrateReplicasRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java index 902a8f218c28..6562ae1d8721 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RenameCollection.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.TARGET; @@ -69,10 +68,7 @@ public SubResponseAccumulatingJerseyResponse renameCollection( final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, requestBody); submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.RENAME, - remoteMessage, - requestBody.async); + response, CollectionParams.CollectionAction.RENAME, remoteMessage, requestBody.async); return response; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java index 60b2c601341b..93405a1ba64d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java @@ -19,9 +19,7 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE; import static org.apache.solr.common.params.CollectionParams.TARGET_NODE; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -31,12 +29,10 @@ import org.apache.solr.client.api.model.ReplaceNodeRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java index 71988d99e97e..1cbb9ce500dd 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java @@ -29,7 +29,6 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; -import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; import jakarta.inject.Inject; @@ -41,7 +40,6 @@ import org.apache.solr.client.api.model.RestoreCollectionRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; import org.apache.solr.common.SolrException; @@ -51,7 +49,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -124,10 +121,7 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( final ZkNodeProps remoteMessage = createRemoteMessage(backupName, requestBody); submitRemoteMessageAndHandleResponse( - response, - CollectionParams.CollectionAction.RESTORE, - remoteMessage, - requestBody.async); + response, CollectionParams.CollectionAction.RESTORE, remoteMessage, requestBody.async); return response; } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index 9dcde25a5a06..7f6f7e7daf42 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -65,7 +65,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // hierarchy) DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertTrue("Collection should have been acquired", collLock.isAcquired()); assertEquals( "Lock at collection level expected to need one distributed lock", @@ -76,7 +79,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // above DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), COLLECTION_NAME, SHARD1_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD1_NAME, + null); assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); assertEquals( "Lock at shard level expected to need two distributed locks", @@ -87,7 +93,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // collection lock above DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), COLLECTION_NAME, SHARD2_NAME, null); + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD2_NAME, + null); assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); assertTrue("Collection should still be acquired", collLock.isAcquired()); @@ -104,7 +113,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD1_NAME, + REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired()); @@ -112,7 +124,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Now ask for a new lock on the collection collLock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertFalse( "Collection should not have been acquired, shard1 and shard2 locks preventing it", @@ -131,7 +146,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD2_NAME, + REPLICA_NAME); assertFalse( "replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired()); @@ -158,13 +176,19 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw // Lock on collection... DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.CREATE), COLLECTION_NAME, null, null); + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); assertTrue("Collection should have been acquired", collLock.isAcquired()); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK), + COLLECTION_NAME, + SHARD1_NAME, + REPLICA_NAME); assertFalse( "replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index c1f114dfdf75..966f7fa2847f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -106,8 +106,7 @@ public void testCreatesValidOverseerMessage() throws Exception { addReplicaPropApi.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java index 080be493b8fe..836de542530d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -19,7 +19,6 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsInAnyOrder; import java.util.Map; @@ -46,8 +45,7 @@ public void testConstructsValidOverseerMessage() { assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); final ZkNodeProps messageTwo = - CreateCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName"); + CreateCollectionSnapshot.createRemoteMessage("myCollName", true, "mySnapshotName"); final Map rawMessageTwo = messageTwo.getProperties(); assertEquals(4, rawMessageTwo.size()); assertThat( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java index 9b5ed2ab92b5..fc50d5b33e53 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteAliasAPITest.java @@ -20,16 +20,13 @@ import java.lang.invoke.MethodHandles; import java.time.Duration; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; import org.apache.solr.client.solrj.request.AliasesApi; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.util.TimeOut; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -54,8 +51,7 @@ public void testDeleteAlias() throws Exception { String aliasName = "deletealiastest_alias"; // Create a collection - CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1) - .process(cloudClient); + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1).process(cloudClient); cluster.waitForActiveCollection(collectionName, 1, 1); // Create an alias pointing to the collection @@ -63,7 +59,10 @@ public void testDeleteAlias() throws Exception { // Verify the alias exists var clusterStateProvider = cloudClient.getClusterStateProvider(); - assertEquals("Alias should exist before deletion", collectionName, clusterStateProvider.resolveSimpleAlias(aliasName)); + assertEquals( + "Alias should exist before deletion", + collectionName, + clusterStateProvider.resolveSimpleAlias(aliasName)); // Delete the alias using the V2 API var request = new AliasesApi.DeleteAlias(aliasName); @@ -72,9 +71,13 @@ public void testDeleteAlias() throws Exception { assertNull("Expected request to not fail", response.failedSubResponsesByNodeName); // Verify the alias is gone - ZkStateReader.AliasesManager aliasesManager = ((ZkClientClusterStateProvider)clusterStateProvider).getZkStateReader().getAliasesManager(); + ZkStateReader.AliasesManager aliasesManager = + ((ZkClientClusterStateProvider) clusterStateProvider) + .getZkStateReader() + .getAliasesManager(); aliasesManager.update(); - assertFalse("Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); + assertFalse( + "Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); } @Test @@ -84,8 +87,7 @@ public void testDeleteAliasAsync() throws Exception { String aliasName = "deletealiastest_alias_async"; // Create a collection - CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1) - .process(cloudClient); + CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1).process(cloudClient); cluster.waitForActiveCollection(collectionName, 1, 1); // Create an alias pointing to the collection @@ -103,7 +105,8 @@ public void testDeleteAliasAsync() throws Exception { assertNull("Expected request start to not fail", response.failedSubResponsesByNodeName); // Wait for the async request to complete - CollectionAdminRequest.RequestStatusResponse rsp = waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + CollectionAdminRequest.RequestStatusResponse rsp = + waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); assertEquals( "Expected async request to complete successfully", @@ -111,8 +114,12 @@ public void testDeleteAliasAsync() throws Exception { rsp.getRequestStatus()); // Verify the alias is gone - ZkStateReader.AliasesManager aliasesManager = ((ZkClientClusterStateProvider)clusterStateProvider).getZkStateReader().getAliasesManager(); + ZkStateReader.AliasesManager aliasesManager = + ((ZkClientClusterStateProvider) clusterStateProvider) + .getZkStateReader() + .getAliasesManager(); aliasesManager.update(); - assertFalse("Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); + assertFalse( + "Alias should not exist after deletion", aliasesManager.getAliases().hasAlias(aliasName)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java index e50b772fd8f9..2df6516bfee0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java @@ -20,7 +20,6 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.NAME; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsInAnyOrder; import java.util.Map; @@ -49,8 +48,7 @@ public void testConstructsValidOverseerMessage() { DeleteCollection.createRemoteMessage("someCollName", Boolean.TRUE); final Map rawMessage = message.getProperties(); assertEquals(3, rawMessage.size()); - assertThat( - rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, FOLLOW_ALIASES)); + assertThat(rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, FOLLOW_ALIASES)); assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); assertEquals("someCollName", rawMessage.get(NAME)); assertEquals(Boolean.TRUE, rawMessage.get(FOLLOW_ALIASES)); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java index af8a5133ad64..3b9217ba2180 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java @@ -18,7 +18,6 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; @@ -120,12 +119,7 @@ public void testMultiVersionDeletionReportsErrorIfRetainParamMissing() { public void testCreateRemoteMessageAllParams() { final var remoteMessage = DeleteCollectionBackup.createRemoteMessage( - "someBackupName", - "someBackupId", - 123, - true, - "someLocation", - "someRepository") + "someBackupName", "someBackupId", 123, true, "someLocation", "someRepository") .getProperties(); assertEquals(7, remoteMessage.size()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java index cb8dfb76449f..6c28f2c8295b 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -19,7 +19,6 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsInAnyOrder; import java.util.Map; @@ -46,8 +45,7 @@ public void testConstructsValidOverseerMessage() { assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); final ZkNodeProps messageTwo = - DeleteCollectionSnapshot.createRemoteMessage( - "myCollName", true, "mySnapshotName"); + DeleteCollectionSnapshot.createRemoteMessage("myCollName", true, "mySnapshotName"); final Map rawMessageTwo = messageTwo.getProperties(); assertEquals(4, rawMessageTwo.size()); assertThat( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java index 1621f6442c13..8339e632cde2 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteNodeAPITest.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import java.io.IOException; import java.lang.invoke.MethodHandles; import java.time.Duration; import java.util.ArrayList; @@ -24,7 +23,6 @@ import java.util.List; import java.util.Set; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; -import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.NodeApi; @@ -104,7 +102,8 @@ public void testDeleteNode() throws Exception { if (log.isInfoEnabled()) { log.info( - "####### DocCollection after: {}", cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + "####### DocCollection after: {}", + cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); } if (shouldFail) { @@ -169,11 +168,13 @@ public void testDeleteNodeAsync() throws Exception { response.failedSubResponsesByNodeName); // Wait for the async request to complete - CollectionAdminRequest.RequestStatusResponse rsp = waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); + CollectionAdminRequest.RequestStatusResponse rsp = + waitForAsyncClusterRequest(asyncId, Duration.ofSeconds(5)); if (log.isInfoEnabled()) { log.info( - "####### DocCollection after: {}", cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); + "####### DocCollection after: {}", + cloudClient.getClusterStateProvider().getClusterState().getCollection(coll)); } if (shouldFail) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index f3f31feb7d93..62fec13856e7 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -84,8 +84,7 @@ public void testCreatesValidOverseerMessage() throws Exception { new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -102,8 +101,7 @@ public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index e895595f5084..d70db774dd09 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -79,8 +79,7 @@ public void setUp() throws Exception { public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -95,8 +94,7 @@ public void testCreatesValidOverseerMessage() throws Exception { @Test public void testRequestBodyCanBeOmittedAltogether() throws Exception { replaceNodeApi.replaceNode("demoSourceNode", null); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); @@ -109,8 +107,7 @@ public void testRequestBodyCanBeOmittedAltogether() throws Exception { public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner) - .runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 59d93c17ab62..1066d12e8931 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -420,7 +420,8 @@ protected static CoreStatusResponse.SingleCoreData getCoreStatus(Replica replica } } - protected CollectionAdminRequest.RequestStatusResponse waitForAsyncClusterRequest(String asyncId, Duration timeout) + protected CollectionAdminRequest.RequestStatusResponse waitForAsyncClusterRequest( + String asyncId, Duration timeout) throws SolrServerException, IOException, InterruptedException { CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId); From df0888ccedbb2a1d61189babecf996ba2ff4e94a Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 12 Jan 2026 17:51:15 -0800 Subject: [PATCH 25/67] Fix broken tests --- .../api/endpoint/AddReplicaPropertyApi.java | 4 +- .../api/endpoint/AliasPropertyApis.java | 8 +- .../api/endpoint/BalanceReplicasApi.java | 4 +- .../client/api/endpoint/CreateAliasApi.java | 5 +- .../api/endpoint/InstallShardDataApi.java | 4 +- .../api/endpoint/MigrateReplicasApi.java | 4 +- .../client/api/endpoint/ReplaceNodeApi.java | 4 +- .../handler/admin/api/AddReplicaProperty.java | 3 +- .../solr/handler/admin/api/AdminAPIBase.java | 14 +- .../solr/handler/admin/api/AliasProperty.java | 9 +- .../handler/admin/api/BalanceReplicas.java | 5 +- .../solr/handler/admin/api/CreateAlias.java | 4 +- .../handler/admin/api/CreateCollection.java | 2 +- .../admin/api/CreateCollectionBackup.java | 1 + .../admin/api/CreateCollectionSnapshot.java | 4 +- .../handler/admin/api/InstallShardData.java | 3 +- .../handler/admin/api/MigrateReplicas.java | 5 +- .../solr/handler/admin/api/ReplaceNode.java | 8 +- .../solr/handler/admin/api/RestoreCore.java | 6 + .../admin/api/AddReplicaPropertyAPITest.java | 57 ++----- .../admin/api/BalanceShardUniqueAPITest.java | 59 ++++--- .../handler/admin/api/CreateAliasAPITest.java | 101 ++++++++---- .../admin/api/CreateCollectionAPITest.java | 58 +++++-- .../api/CreateCollectionSnapshotAPITest.java | 67 ++++++-- .../admin/api/CreateReplicaAPITest.java | 59 ++++--- .../admin/api/DeleteCollectionAPITest.java | 62 ++++++-- .../api/DeleteCollectionBackupAPITest.java | 147 ++++++++++++------ .../api/DeleteCollectionSnapshotAPITest.java | 63 ++++++-- .../api/DeleteReplicaPropertyAPITest.java | 40 +++-- .../admin/api/MigrateReplicasAPITest.java | 60 ++----- .../handler/admin/api/ReplaceNodeAPITest.java | 65 ++------ .../admin/api/V2CollectionBackupApiTest.java | 85 +++++++--- 32 files changed, 611 insertions(+), 409 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java index 8e4d53e521ab..997613e292fc 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java @@ -24,7 +24,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/collections/{collName}/shards/{shardName}/replicas/{replicaName}/properties/{propName}") public interface AddReplicaPropertyApi { @@ -33,7 +33,7 @@ public interface AddReplicaPropertyApi { @Operation( summary = "Adds a property to the specified replica", tags = {"replica-properties"}) - public SolrJerseyResponse addReplicaProperty( + public SubResponseAccumulatingJerseyResponse addReplicaProperty( @Parameter( description = "The name of the collection the replica belongs to.", required = true) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java index 2c1f2ae82672..1710e2e0e485 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/AliasPropertyApis.java @@ -26,7 +26,7 @@ import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.GetAliasPropertyResponse; import org.apache.solr.client.api.model.GetAllAliasPropertiesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; @@ -56,7 +56,7 @@ GetAliasPropertyResponse getAliasProperty( @Operation( summary = "Update properties for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse updateAliasProperties( + SubResponseAccumulatingJerseyResponse updateAliasProperties( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @RequestBody(description = "Properties that need to be updated", required = true) UpdateAliasPropertiesRequestBody requestBody) @@ -67,7 +67,7 @@ SolrJerseyResponse updateAliasProperties( @Operation( summary = "Update a specific property for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse createOrUpdateAliasProperty( + SubResponseAccumulatingJerseyResponse createOrUpdateAliasProperty( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @Parameter(description = "Property Name") @PathParam("propName") String propName, @RequestBody(description = "Property value that needs to be updated", required = true) @@ -79,7 +79,7 @@ SolrJerseyResponse createOrUpdateAliasProperty( @Operation( summary = "Delete a specific property for a collection alias.", tags = {"alias-properties"}) - SolrJerseyResponse deleteAliasProperty( + SubResponseAccumulatingJerseyResponse deleteAliasProperty( @Parameter(description = "Alias Name") @PathParam("aliasName") String aliasName, @Parameter(description = "Property Name") @PathParam("propName") String propName) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java index 9cbfa5bf3a80..3fd863d1e0ab 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/BalanceReplicasApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.BalanceReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("cluster/replicas/balance") public interface BalanceReplicasApi { @@ -29,7 +29,7 @@ public interface BalanceReplicasApi { @Operation( summary = "Balance Replicas across the given set of Nodes.", tags = {"cluster"}) - SolrJerseyResponse balanceReplicas( + SubResponseAccumulatingJerseyResponse balanceReplicas( @RequestBody(description = "Contains user provided parameters") BalanceReplicasRequestBody requestBody) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java index 78b9b4376c02..b5bfc5afca11 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java @@ -20,7 +20,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.CreateAliasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; @Path("/aliases") public interface CreateAliasApi { @@ -28,5 +28,6 @@ public interface CreateAliasApi { @Operation( summary = "Create a traditional or 'routed' alias", tags = {"aliases"}) - SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception; + SubResponseAccumulatingJerseyResponse createAlias(CreateAliasRequestBody requestBody) + throws Exception; } diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java index f38c2bd00eea..e91c57054bf9 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/InstallShardDataApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.InstallShardDataRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** V2 API definition allowing users to import offline-constructed index into a shard. */ @Path("/collections/{collName}/shards/{shardName}/install") @@ -30,7 +30,7 @@ public interface InstallShardDataApi { @Operation( summary = "Install offline index into an existing shard", tags = {"shards"}) - SolrJerseyResponse installShardData( + SubResponseAccumulatingJerseyResponse installShardData( @PathParam("collName") String collName, @PathParam("shardName") String shardName, InstallShardDataRequestBody requestBody) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java index 566be87b76bd..10e918ba9ef6 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/MigrateReplicasApi.java @@ -21,7 +21,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** V2 API definition for migrating replicas from a set of nodes to another set of nodes. */ @Path("cluster/replicas/migrate") @@ -30,7 +30,7 @@ public interface MigrateReplicasApi { @Operation( summary = "Migrate Replicas from a given set of nodes.", tags = {"cluster"}) - SolrJerseyResponse migrateReplicas( + SubResponseAccumulatingJerseyResponse migrateReplicas( @RequestBody(description = "Contains user provided parameters", required = true) MigrateReplicasRequestBody requestBody) throws Exception; diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java index cee4032a1968..cb3c577783ba 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplaceNodeApi.java @@ -23,7 +23,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; /** * V2 API definition for recreating replicas in one node (the source) on another node(s) (the @@ -35,7 +35,7 @@ public interface ReplaceNodeApi { @Operation( summary = "'Replace' a specified node by moving all replicas elsewhere", tags = {"node"}) - SolrJerseyResponse replaceNode( + SubResponseAccumulatingJerseyResponse replaceNode( @Parameter(description = "The name of the node to be replaced.", required = true) @PathParam("sourceNodeName") String sourceNodeName, diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java index 70d21db5b1c0..baf66c3e4d3c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java @@ -31,7 +31,6 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.AddReplicaPropertyApi; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; @@ -59,7 +58,7 @@ public AddReplicaProperty( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse addReplicaProperty( + public SubResponseAccumulatingJerseyResponse addReplicaProperty( String collName, String shardName, String replicaName, diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 2e98fa18b46e..94f0e9c36ba5 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -72,11 +72,7 @@ protected String resolveAndValidateAliasIfEnabled( } private String resolveAlias(String aliasName) { - return coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(aliasName); + return coreContainer.getAliases().resolveSimpleAlias(aliasName); } public static void validateZooKeeperAwareCoreContainer(CoreContainer coreContainer) { @@ -94,13 +90,7 @@ public static void validateZooKeeperAwareCoreContainer(CoreContainer coreContain protected String resolveCollectionName(String collName, boolean followAliases) { final String collectionName = - followAliases - ? coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(collName) - : collName; + followAliases ? coreContainer.getAliases().resolveSimpleAlias(collName) : collName; final ClusterState clusterState = coreContainer.getZkController().getClusterState(); if (!clusterState.hasCollection(collectionName)) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java index 2e36aa703128..a400f0edb318 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AliasProperty.java @@ -27,7 +27,6 @@ import org.apache.solr.client.api.endpoint.AliasPropertyApis; import org.apache.solr.client.api.model.GetAliasPropertyResponse; import org.apache.solr.client.api.model.GetAllAliasPropertiesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateAliasPropertyRequestBody; @@ -100,7 +99,7 @@ private Aliases readAliasesFromZk() throws Exception { @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse updateAliasProperties( + public SubResponseAccumulatingJerseyResponse updateAliasProperties( String aliasName, UpdateAliasPropertiesRequestBody requestBody) throws Exception { if (requestBody == null) { @@ -116,7 +115,7 @@ public SolrJerseyResponse updateAliasProperties( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse createOrUpdateAliasProperty( + public SubResponseAccumulatingJerseyResponse createOrUpdateAliasProperty( String aliasName, String propName, UpdateAliasPropertyRequestBody requestBody) throws Exception { if (requestBody == null) { @@ -132,8 +131,8 @@ public SolrJerseyResponse createOrUpdateAliasProperty( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse deleteAliasProperty(String aliasName, String propName) - throws Exception { + public SubResponseAccumulatingJerseyResponse deleteAliasProperty( + String aliasName, String propName) throws Exception { recordCollectionForLogAndTracing(null, solrQueryRequest); var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java index c7f1d73816c3..088f6e225f83 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.BalanceReplicasApi; import org.apache.solr.client.api.model.BalanceReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams.CollectionAction; @@ -48,8 +47,8 @@ public BalanceReplicas( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse balanceReplicas(BalanceReplicasRequestBody requestBody) - throws Exception { + public SubResponseAccumulatingJerseyResponse balanceReplicas( + BalanceReplicasRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); recordCollectionForLogAndTracing(null, solrQueryRequest); fetchAndValidateZooKeeperAwareCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java index ec877776ac29..71879b7abb3d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java @@ -41,7 +41,6 @@ import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.RoutedAliasProperties; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.api.model.TimeRoutedAliasProperties; import org.apache.solr.client.solrj.request.RoutedAliasTypes; @@ -79,7 +78,8 @@ public CreateAlias( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception { + public SubResponseAccumulatingJerseyResponse createAlias(CreateAliasRequestBody requestBody) + throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); recordCollectionForLogAndTracing(null, solrQueryRequest); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index 1cf629836938..e0130c67c957 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -211,7 +211,7 @@ public static Map copyPrefixedPropertiesWithoutPrefix( private static Integer readIntegerDefaultFromClusterProp( CoreContainer coreContainer, String propName) throws IOException { final Object defaultValue = - new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient()) + new ClusterProperties(coreContainer.getZkController().getZkClient()) .getClusterProperty( List.of(CollectionAdminParams.DEFAULTS, CollectionAdminParams.COLLECTION, propName), null); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java index fa2e7a69b72d..5e856012819f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java @@ -119,6 +119,7 @@ public SolrJerseyResponse createCollectionBackup( public static ZkNodeProps createRemoteMessage( String collectionName, String backupName, CreateCollectionBackupRequestBody requestBody) { final Map remoteMessage = Utils.reflectToMap(requestBody); + remoteMessage.remove(ASYNC); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(NAME, backupName); if (!StringUtils.isBlank(requestBody.backupStrategy)) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java index 88c747ffb932..2cfa244c74a1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java @@ -72,12 +72,10 @@ public CreateCollectionSnapshotResponse createCollectionSnapshot( + "', no action taken."); } - final ZkNodeProps remoteMessage = - createRemoteMessage(collName, requestBody.followAliases, snapshotName); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.CREATESNAPSHOT, - remoteMessage, + createRemoteMessage(collName, requestBody.followAliases, snapshotName), requestBody.async); response.collection = collName; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java index 3d5f9f5badf3..4033fa6f0975 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/InstallShardData.java @@ -22,7 +22,6 @@ import jakarta.inject.Inject; import java.util.HashMap; import org.apache.solr.client.api.endpoint.InstallShardDataApi; -import org.apache.solr.client.api.model.AsyncJerseyResponse; import org.apache.solr.client.api.model.InstallShardDataRequestBody; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.cloud.api.collections.InstallShardDataCmd; @@ -57,7 +56,7 @@ public InstallShardData( @Override @PermissionName(COLL_EDIT_PERM) - public AsyncJerseyResponse installShardData( + public SubResponseAccumulatingJerseyResponse installShardData( String collName, String shardName, InstallShardDataRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java index 2bfe996099f2..ad9b96040801 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/MigrateReplicas.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.MigrateReplicasApi; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; @@ -49,8 +48,8 @@ public MigrateReplicas( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse migrateReplicas(MigrateReplicasRequestBody requestBody) - throws Exception { + public SubResponseAccumulatingJerseyResponse migrateReplicas( + MigrateReplicasRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(null, solrQueryRequest); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java index 93405a1ba64d..cd1626590871 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNode.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE; import static org.apache.solr.common.params.CollectionParams.TARGET_NODE; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; @@ -27,11 +26,9 @@ import java.util.Map; import org.apache.solr.client.api.endpoint.ReplaceNodeApi; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.core.CoreContainer; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; @@ -54,8 +51,8 @@ public ReplaceNode( @Override @PermissionName(COLL_EDIT_PERM) - public SolrJerseyResponse replaceNode(String sourceNodeName, ReplaceNodeRequestBody requestBody) - throws Exception { + public SubResponseAccumulatingJerseyResponse replaceNode( + String sourceNodeName, ReplaceNodeRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(null, solrQueryRequest); @@ -76,7 +73,6 @@ public ZkNodeProps createRemoteMessage(String nodeName, ReplaceNodeRequestBody r insertIfValueNotNull(remoteMessage, TARGET_NODE, requestBody.targetNodeName); insertIfValueNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.REPLACENODE.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java index dcf1cfe85c1b..3997b1971b4e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java @@ -132,6 +132,12 @@ private void doRestore(String coreName, RestoreCoreRequestBody requestBody) thro throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); } + // other replicas to-be-created will know that they are out of date by + // looking at their term : 0 compare to term of this core : 1 + coreContainer + .getZkController() + .getShardTerms(cd.getCollectionName(), cd.getShardId()) + .ensureHighestTermsAreNotZero(); // transitions state of update log to ACTIVE UpdateLog updateLog = core.getUpdateHandler().getUpdateLog(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 966f7fa2847f..05da868a7207 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -17,68 +17,32 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.opentelemetry.api.trace.Span; import java.util.Map; -import java.util.Optional; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link AddReplicaProperty} */ -public class AddReplicaPropertyAPITest extends SolrTestCaseJ4 { +public class AddReplicaPropertyAPITest extends MockAPITest { private static final AddReplicaPropertyRequestBody ANY_REQ_BODY = new AddReplicaPropertyRequestBody("anyValue"); - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; - private ArgumentCaptor messageCapturer; - private AddReplicaProperty addReplicaPropApi; - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } - @Override @Before public void setUp() throws Exception { super.setUp(); - - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); - queryResponse = new SolrQueryResponse(); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); addReplicaPropApi = new AddReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @@ -106,19 +70,20 @@ public void testCreatesValidOverseerMessage() throws Exception { addReplicaPropApi.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); - verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(6, createdMessageProps.size()); - assertEquals("addreplicaprop", createdMessageProps.get("operation")); + assertEquals(5, createdMessageProps.size()); assertEquals("someColl", createdMessageProps.get("collection")); assertEquals("someShard", createdMessageProps.get("shard")); assertEquals("someReplica", createdMessageProps.get("replica")); assertEquals("somePropName", createdMessageProps.get("property")); assertEquals("anyValue", createdMessageProps.get("property.value")); - assertFalse( - createdMessageProps.containsKey( - SHARD_UNIQUE)); // Omitted since not specified on request body + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.ADDREPLICAPROP, context.getAction()); + assertNull("asyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java index c986f332da83..f479c878e842 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java @@ -17,30 +17,43 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_ACTIVE_NODES; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.SHARD_UNIQUE; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.BalanceShardUniqueRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link BalanceShardUnique} */ -public class BalanceShardUniqueAPITest extends SolrTestCaseJ4 { +public class BalanceShardUniqueAPITest extends MockAPITest { + + private BalanceShardUnique balanceShardUnique; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + balanceShardUnique = new BalanceShardUnique(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique("someCollectionName", null); - }); + () -> balanceShardUnique.balanceShardUnique("someCollectionName", null)); assertEquals(400, thrown.code()); assertEquals("Missing required request body", thrown.getMessage()); @@ -52,11 +65,7 @@ public void testReportsErrorIfCollectionNameMissing() { requestBody.property = "preferredLeader"; final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique(null, requestBody); - }); + SolrException.class, () -> balanceShardUnique.balanceShardUnique(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -69,31 +78,35 @@ public void testReportsErrorIfPropertyToBalanceIsMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new BalanceShardUnique(null, null, null); - api.balanceShardUnique("someCollName", requestBody); - }); + () -> balanceShardUnique.balanceShardUnique("someCollName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: property", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new BalanceShardUniqueRequestBody(); requestBody.property = "someProperty"; requestBody.shardUnique = Boolean.TRUE; requestBody.onlyActiveNodes = Boolean.TRUE; requestBody.async = "someAsyncId"; - final var remoteMessage = - BalanceShardUnique.createRemoteMessage("someCollName", requestBody).getProperties(); - assertEquals(6, remoteMessage.size()); - assertEquals("balanceshardunique", remoteMessage.get(QUEUE_OPERATION)); + balanceShardUnique.balanceShardUnique("someCollName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(4, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someProperty", remoteMessage.get(PROPERTY_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(SHARD_UNIQUE)); assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_ACTIVE_NODES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BALANCESHARDUNIQUE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java index b4f5e2d2760e..1f6707c2d7a0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java @@ -18,23 +18,39 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.COLLECTIONS; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.RoutedAliasProperties; import org.apache.solr.client.api.model.TimeRoutedAliasProperties; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateAlias} */ -public class CreateAliasAPITest extends SolrTestCaseJ4 { +public class CreateAliasAPITest extends MockAPITest { + + private CreateAlias createAliasApi; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + createAliasApi = new CreateAlias(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { @@ -202,24 +218,29 @@ public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters( } @Test - public void testRemoteMessageCreationForTraditionalAlias() { + public void testRemoteMessageCreationForTraditionalAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; requestBody.collections = List.of("validColl1", "validColl2"); requestBody.async = "someAsyncId"; - final var remoteMessage = - CreateAlias.createRemoteMessageForTraditionalAlias(requestBody).getProperties(); + createAliasApi.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(4, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(2, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("validColl1,validColl2", remoteMessage.get(COLLECTIONS)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testRemoteMessageCreationForCategoryRoutedAlias() { + public void testRemoteMessageCreationForCategoryRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var categoryRouter = new CategoryRoutedAliasProperties(); @@ -230,25 +251,31 @@ public void testRemoteMessageCreationForCategoryRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + createAliasApi.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(6, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(5, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("category", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } @Test - public void testRemoteMessageCreationForTimeRoutedAlias() { + public void testRemoteMessageCreationForTimeRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; - timeRouter.start = "NOW"; + timeRouter.start = "NOW/HOUR"; timeRouter.interval = "+1MONTH"; timeRouter.maxFutureMs = 123456L; requestBody.routers = List.of(timeRouter); @@ -257,28 +284,34 @@ public void testRemoteMessageCreationForTimeRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + createAliasApi.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(9, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(8, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("time", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); - assertEquals("NOW", remoteMessage.get("router.start")); + assertEquals("NOW/HOUR", remoteMessage.get("router.start")); assertEquals("+1MONTH", remoteMessage.get("router.interval")); assertEquals(Long.valueOf(123456L), remoteMessage.get("router.maxFutureMs")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } @Test - public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { + public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() throws Exception { final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; - timeRouter.start = "NOW"; + timeRouter.start = "NOW/HOUR"; timeRouter.interval = "+1MONTH"; timeRouter.maxFutureMs = 123456L; final var categoryRouter = new CategoryRoutedAliasProperties(); @@ -289,21 +322,27 @@ public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var remoteMessage = - CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + createAliasApi.createAlias(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(11, remoteMessage.size()); - assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(10, remoteMessage.size()); assertEquals("someAliasName", remoteMessage.get("name")); assertEquals("time", remoteMessage.get("router.0.name")); assertEquals("someField", remoteMessage.get("router.0.field")); - assertEquals("NOW", remoteMessage.get("router.0.start")); + assertEquals("NOW/HOUR", remoteMessage.get("router.0.start")); assertEquals("+1MONTH", remoteMessage.get("router.0.interval")); assertEquals(Long.valueOf(123456L), remoteMessage.get("router.0.maxFutureMs")); assertEquals("category", remoteMessage.get("router.1.name")); assertEquals("someField", remoteMessage.get("router.1.field")); assertEquals(3, remoteMessage.get("create-collection.numShards")); assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATEALIAS, context.getAction()); + assertNull(context.getAsyncId()); } private CreateAliasRequestBody requestBodyWithProvidedRouter(RoutedAliasProperties router) { @@ -325,7 +364,7 @@ public void testConvertsV1ParamsForMultiDimensionalAliasToV2RequestBody() { v1Params.add("name", "someAliasName"); v1Params.add("router.name", "Dimensional[time,category]"); v1Params.add("router.0.field", "someField"); - v1Params.add("router.0.start", "NOW"); + v1Params.add("router.0.start", "NOW/HOUR"); v1Params.add("router.0.interval", "+1MONTH"); v1Params.add("router.0.maxFutureMs", "123456"); v1Params.add("router.1.field", "someOtherField"); @@ -342,7 +381,7 @@ public void testConvertsV1ParamsForMultiDimensionalAliasToV2RequestBody() { requestBody.routers.get(0) instanceof TimeRoutedAliasProperties); final var timeRouter = (TimeRoutedAliasProperties) requestBody.routers.get(0); assertEquals("someField", timeRouter.field); - assertEquals("NOW", timeRouter.start); + assertEquals("NOW/HOUR", timeRouter.start); assertEquals("+1MONTH", timeRouter.interval); assertEquals(Long.valueOf(123456L), timeRouter.maxFutureMs); assertTrue( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java index 80a2200131af..492548df197c 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.NUM_SLICES; import static org.apache.solr.common.cloud.DocCollection.CollectionStateProps.SHARDS; @@ -30,21 +29,42 @@ import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.CreateCollectionRouterProperties; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateCollection}. */ -public class CreateCollectionAPITest extends SolrTestCaseJ4 { +public class CreateCollectionAPITest extends MockAPITest { + + private CreateCollection createCollectionApi; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + createCollectionApi = new CreateCollection(mockCoreContainer, mockQueryRequest, queryResponse); + when(mockSolrZkClient.getData(eq("properties.json"), any(), any())) + .thenReturn("{}".getBytes(StandardCharsets.UTF_8)); + } @Test public void testReportsErrorIfRequestBodyMissing() { @@ -125,7 +145,7 @@ public void testReportsErrorIfShardNamesInvalid() { } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateCollectionRequestBody(); requestBody.name = "someName"; requestBody.replicationFactor = 123; @@ -145,9 +165,13 @@ public void testCreateRemoteMessageAllProperties() { requestBody.nodeSet = List.of("node1", "node2"); requestBody.shuffleNodes = false; - final var remoteMessage = CreateCollection.createRemoteMessage(requestBody).getProperties(); + createCollectionApi.createCollection(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(17, remoteMessage.size()); assertEquals("true", remoteMessage.get("fromApi")); assertEquals("someName", remoteMessage.get(NAME)); assertEquals(123, remoteMessage.get(REPLICATION_FACTOR)); @@ -161,24 +185,36 @@ public void testCreateRemoteMessageAllProperties() { assertEquals(true, remoteMessage.get(PER_REPLICA_STATE)); assertEquals("someAliasName", remoteMessage.get(ALIAS)); assertEquals("propValue", remoteMessage.get("property.propName")); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("someRouterName", remoteMessage.get("router.name")); assertEquals("someField", remoteMessage.get("router.field")); assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET)); assertEquals(false, remoteMessage.get(CREATE_NODE_SET_SHUFFLE_PARAM)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testNoReplicaCreationMessage() { + public void testNoReplicaCreationMessage() throws Exception { final var requestBody = new CreateCollectionRequestBody(); requestBody.name = "someName"; requestBody.createReplicas = false; + requestBody.async = "someAsyncId"; - final var remoteMessage = CreateCollection.createRemoteMessage(requestBody).getProperties(); + createCollectionApi.createCollection(requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals("create", remoteMessage.get(QUEUE_OPERATION)); assertEquals("someName", remoteMessage.get(NAME)); assertEquals("EMPTY", remoteMessage.get(CREATE_NODE_SET)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java index 836de542530d..139f8d52aa2c 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -16,45 +16,80 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -public class CreateCollectionSnapshotAPITest extends SolrTestCaseJ4 { +public class CreateCollectionSnapshotAPITest extends MockAPITest { + + private CreateCollectionSnapshot createCollectionSnapshot; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + createCollectionSnapshot = + new CreateCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { - final ZkNodeProps messageOne = - CreateCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName"); + public void testConstructsValidOverseerMessage() throws Exception { + when(mockClusterState.hasCollection("myCollName")).thenReturn(true); + when(mockSolrZkClient.exists(anyString())).thenReturn(false); + + CreateCollectionSnapshotRequestBody body = new CreateCollectionSnapshotRequestBody(); + body.followAliases = false; + createCollectionSnapshot.createCollectionSnapshot("myCollName", "mySnapshotName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageOne = messageCapturer.getValue(); final Map rawMessageOne = messageOne.getProperties(); - assertEquals(4, rawMessageOne.size()); + assertEquals(3, rawMessageOne.size()); assertThat( rawMessageOne.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("createsnapshot", rawMessageOne.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); - final ZkNodeProps messageTwo = - CreateCollectionSnapshot.createRemoteMessage("myCollName", true, "mySnapshotName"); + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESNAPSHOT, context.getAction()); + assertNull(context.getAsyncId()); + + body.followAliases = true; + body.async = "testId"; + Mockito.clearInvocations(mockCommandRunner); + createCollectionSnapshot.createCollectionSnapshot("myCollName", "mySnapshotName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageTwo = messageCapturer.getValue(); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(4, rawMessageTwo.size()); + assertEquals(3, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("createsnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); + + context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESNAPSHOT, context.getAction()); + assertEquals("testId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java index 2abe5d13397d..bdfc75cf9649 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; @@ -35,26 +34,42 @@ import static org.apache.solr.common.params.CoreAdminParams.NODE; import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR; import static org.apache.solr.common.params.ShardParams._ROUTE_; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateReplicaRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateReplica} */ -public class CreateReplicaAPITest extends SolrTestCaseJ4 { +public class CreateReplicaAPITest extends MockAPITest { + + private CreateReplica createReplica; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + createReplica = new CreateReplica(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica("someCollName", "someShardName", null); - }); + () -> createReplica.createReplica("someCollName", "someShardName", null)); assertEquals(400, thrown.code()); assertEquals("Required request-body is missing", thrown.getMessage()); @@ -65,11 +80,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new CreateReplicaRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica(null, "shardName", requestBody); - }); + SolrException.class, () -> createReplica.createReplica(null, "shardName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -81,17 +92,14 @@ public void testReportsErrorIfShardNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new CreateReplica(null, null, null); - api.createReplica("someCollectionName", null, requestBody); - }); + () -> createReplica.createReplica("someCollectionName", null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateReplicaRequestBody(); requestBody.name = "someName"; requestBody.type = "NRT"; @@ -110,12 +118,14 @@ public void testCreateRemoteMessageAllProperties() { requestBody.async = "someAsyncId"; requestBody.properties = Map.of("propName1", "propVal1", "propName2", "propVal2"); - final var remoteMessage = - CreateReplica.createRemoteMessage("someCollectionName", "someShardName", requestBody) - .getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + createReplica.createReplica("someCollectionName", "someShardName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(20, remoteMessage.size()); - assertEquals("addreplica", remoteMessage.get(QUEUE_OPERATION)); + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(18, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someName", remoteMessage.get(NAME)); @@ -132,9 +142,12 @@ public void testCreateRemoteMessageAllProperties() { assertEquals(Boolean.TRUE, remoteMessage.get(SKIP_NODE_ASSIGNMENT)); assertEquals(true, remoteMessage.get(WAIT_FOR_FINAL_STATE)); assertEquals(true, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("propVal1", remoteMessage.get("property.propName1")); assertEquals("propVal2", remoteMessage.get("property.propName2")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.ADDREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java index 2df6516bfee0..afff0677c341 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java @@ -17,41 +17,69 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionParams.NAME; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasEntry; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; /** Unit tests for {@link DeleteCollection} */ -public class DeleteCollectionAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionAPITest extends MockAPITest { + + private DeleteCollection deleteCollection; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + deleteCollection = new DeleteCollection(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { + public void testConstructsValidOverseerMessage() throws Exception { // Only required properties provided { - final ZkNodeProps message = DeleteCollection.createRemoteMessage("someCollName", null); + deleteCollection.deleteCollection("someCollName", null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); final Map rawMessage = message.getProperties(); - assertEquals(2, rawMessage.size()); - assertThat(rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME)); - assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); - assertEquals("someCollName", rawMessage.get(NAME)); + assertEquals(1, rawMessage.size()); + assertThat(rawMessage, hasEntry(NAME, "someCollName")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETE, context.getAction()); + assertNull(context.getAsyncId()); } // Optional property 'followAliases' also provided { - final ZkNodeProps message = - DeleteCollection.createRemoteMessage("someCollName", Boolean.TRUE); + Mockito.clearInvocations(mockCommandRunner); + deleteCollection.deleteCollection("someCollName", true, "test"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); final Map rawMessage = message.getProperties(); - assertEquals(3, rawMessage.size()); - assertThat(rawMessage.keySet(), containsInAnyOrder(QUEUE_OPERATION, NAME, FOLLOW_ALIASES)); - assertEquals("delete", rawMessage.get(QUEUE_OPERATION)); - assertEquals("someCollName", rawMessage.get(NAME)); - assertEquals(Boolean.TRUE, rawMessage.get(FOLLOW_ALIASES)); + assertEquals(2, rawMessage.size()); + assertThat(rawMessage, hasEntry(NAME, "someCollName")); + assertThat(rawMessage, hasEntry(FOLLOW_ALIASES, Boolean.TRUE)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETE, context.getAction()); + assertEquals("test", context.getAsyncId()); } } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java index 3b9217ba2180..f6a12d137523 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionBackupAPITest.java @@ -17,21 +17,53 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_PURGE_UNUSED; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS; - -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.util.Map; import org.apache.solr.client.api.model.PurgeUnusedFilesRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteCollectionBackup} */ -public class DeleteCollectionBackupAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionBackupAPITest extends MockAPITest { + + private DeleteCollectionBackup api; + private BackupRepository mockBackupRepository; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository(eq("someRepository"))) + .thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + when(mockBackupRepository.getBackupLocation(eq("someLocation"))).thenReturn("someLocation"); + URI uri = new URI("someLocation"); + when(mockBackupRepository.createDirectoryURI(eq("someLocation"))).thenReturn(uri); + when(mockBackupRepository.exists(eq(uri))).thenReturn(true); + + api = new DeleteCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfBackupNameMissing() { // Single delete @@ -39,11 +71,9 @@ public void testReportsErrorIfBackupNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteSingleBackupById( - null, "someBackupId", "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteSingleBackupById( + null, "someBackupId", "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -54,11 +84,9 @@ public void testReportsErrorIfBackupNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteMultipleBackupsByRecency( - null, 123, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteMultipleBackupsByRecency( + null, 123, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -72,11 +100,7 @@ public void testReportsErrorIfBackupNameMissing() { requestBody.async = "someAsyncId"; final SolrException thrown = expectThrows( - SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.garbageCollectUnusedBackupFiles(null, requestBody); - }); + SolrException.class, () -> api.garbageCollectUnusedBackupFiles(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: name", thrown.getMessage()); @@ -88,11 +112,9 @@ public void testDeletionByIdReportsErrorIfIdMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteSingleBackupById( - "someBackupName", null, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteSingleBackupById( + "someBackupName", null, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: backupId", thrown.getMessage()); @@ -103,11 +125,9 @@ public void testMultiVersionDeletionReportsErrorIfRetainParamMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteCollectionBackup(null, null, null); - api.deleteMultipleBackupsByRecency( - "someBackupName", null, "someLocation", "someRepository", "someAsyncId"); - }); + () -> + api.deleteMultipleBackupsByRecency( + "someBackupName", null, "someLocation", "someRepository", "someAsyncId")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: retainLatest", thrown.getMessage()); @@ -116,32 +136,63 @@ public void testMultiVersionDeletionReportsErrorIfRetainParamMissing() { // The message created in this test isn't valid in practice, since it contains mutually-exclusive // parameters, but that doesn't matter for the purposes of this test. @Test - public void testCreateRemoteMessageAllParams() { - final var remoteMessage = - DeleteCollectionBackup.createRemoteMessage( - "someBackupName", "someBackupId", 123, true, "someLocation", "someRepository") - .getProperties(); - - assertEquals(7, remoteMessage.size()); - assertEquals("deletebackup", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageSingle() throws Exception { + api.deleteSingleBackupById( + "someBackupName", "someBackupId", "someLocation", "someRepository", "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); assertEquals("someBackupName", remoteMessage.get(NAME)); assertEquals("someBackupId", remoteMessage.get(BACKUP_ID)); - assertEquals(Integer.valueOf(123), remoteMessage.get(MAX_NUM_BACKUP_POINTS)); - assertEquals(Boolean.TRUE, remoteMessage.get(BACKUP_PURGE_UNUSED)); assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testCreateRemoteMessageOnlyRequiredParams() { - final var remoteMessage = - DeleteCollectionBackup.createRemoteMessage( - "someBackupName", "someBackupId", null, null, null, null) - .getProperties(); - - assertEquals(3, remoteMessage.size()); - assertEquals("deletebackup", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageMultiple() throws Exception { + api.deleteMultipleBackupsByRecency("someBackupName", 2, "someLocation", "someRepository", null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); assertEquals("someBackupName", remoteMessage.get(NAME)); - assertEquals("someBackupId", remoteMessage.get(BACKUP_ID)); + assertEquals(2, remoteMessage.get(MAX_NUM_BACKUP_POINTS)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertNull(context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageGarbageCollect() throws Exception { + PurgeUnusedFilesRequestBody body = new PurgeUnusedFilesRequestBody(); + body.location = "someLocation"; + body.repositoryName = "someRepository"; + api.garbageCollectUnusedBackupFiles("someBackupName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(4, remoteMessage.size()); + assertEquals("someBackupName", remoteMessage.get(NAME)); + assertEquals(Boolean.TRUE, remoteMessage.get(BACKUP_PURGE_UNUSED)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEBACKUP, context.getAction()); + assertNull(context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java index 6c28f2c8295b..f04df2e4dbce 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -16,45 +16,76 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -public class DeleteCollectionSnapshotAPITest extends SolrTestCaseJ4 { +public class DeleteCollectionSnapshotAPITest extends MockAPITest { + + private DeleteCollectionSnapshot deleteCollectionSnapshot; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + deleteCollectionSnapshot = + new DeleteCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test - public void testConstructsValidOverseerMessage() { - final ZkNodeProps messageOne = - DeleteCollectionSnapshot.createRemoteMessage("myCollName", false, "mySnapshotName"); + public void testConstructsValidOverseerMessage() throws Exception { + when(mockClusterState.hasCollection("myCollName")).thenReturn(true); + when(mockSolrZkClient.exists(anyString())).thenReturn(false); + + deleteCollectionSnapshot.deleteCollectionSnapshot("myCollName", "mySnapshotName", false, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageOne = messageCapturer.getValue(); final Map rawMessageOne = messageOne.getProperties(); - assertEquals(4, rawMessageOne.size()); + assertEquals(3, rawMessageOne.size()); assertThat( rawMessageOne.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("deletesnapshot", rawMessageOne.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); - final ZkNodeProps messageTwo = - DeleteCollectionSnapshot.createRemoteMessage("myCollName", true, "mySnapshotName"); + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESNAPSHOT, context.getAction()); + assertNull(context.getAsyncId()); + + Mockito.clearInvocations(mockCommandRunner); + deleteCollectionSnapshot.deleteCollectionSnapshot( + "myCollName", "mySnapshotName", true, "testId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps messageTwo = messageCapturer.getValue(); final Map rawMessageTwo = messageTwo.getProperties(); - assertEquals(4, rawMessageTwo.size()); + assertEquals(3, rawMessageTwo.size()); assertThat( rawMessageTwo.keySet(), - containsInAnyOrder( - QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); - assertEquals("deletesnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + containsInAnyOrder(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); + + context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESNAPSHOT, context.getAction()); + assertEquals("testId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java index 19c85b85481c..e3950b1db42f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java @@ -17,20 +17,22 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; /** @@ -40,11 +42,18 @@ * here focus primarily on how the v1 code invokes the v2 API and how the v2 API crafts its * overseer/Distributed State Processing RPC message. */ -public class DeleteReplicaPropertyAPITest extends SolrTestCaseJ4 { +public class DeleteReplicaPropertyAPITest extends MockAPITest { - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); + private DeleteReplicaProperty deleteReplicaProperty; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + deleteReplicaProperty = + new DeleteReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -134,17 +143,22 @@ public void testV1InvocationTrimsPropertyNamePrefixIfProvided() throws Exception } @Test - public void testRPCMessageCreation() { - final ZkNodeProps message = - DeleteReplicaProperty.createRemoteMessage( - "someColl", "someShard", "someReplica", "somePropName"); + public void testRPCMessageCreation() throws Exception { + deleteReplicaProperty.deleteReplicaProperty( + "someColl", "someShard", "someReplica", "somePropName"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + final ZkNodeProps message = messageCapturer.getValue(); final Map props = message.getProperties(); - assertEquals(5, props.size()); - assertEquals("deletereplicaprop", props.get(QUEUE_OPERATION)); + assertEquals(4, props.size()); assertEquals("someColl", props.get(COLLECTION_PROP)); assertEquals("someShard", props.get(SHARD_ID_PROP)); assertEquals("someReplica", props.get(REPLICA_PROP)); assertEquals("somePropName", props.get(PROPERTY_PROP)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICAPROP, context.getAction()); + assertNull(context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index 62fec13856e7..eef0987cdcf4 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -16,66 +16,33 @@ */ package org.apache.solr.handler.admin.api; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.Collections; import java.util.Map; -import java.util.Optional; import java.util.Set; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.MigrateReplicasRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link ReplaceNode} */ -public class MigrateReplicasAPITest extends SolrTestCaseJ4 { +public class MigrateReplicasAPITest extends MockAPITest { - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; private MigrateReplicas migrateReplicasAPI; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private ArgumentCaptor messageCapturer; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } @Override @Before public void setUp() throws Exception { super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - queryResponse = new SolrQueryResponse(); migrateReplicasAPI = new MigrateReplicas(mockCoreContainer, mockQueryRequest, queryResponse); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - - when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); } @Test @@ -84,16 +51,18 @@ public void testCreatesValidOverseerMessage() throws Exception { new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(5, createdMessageProps.size()); + assertEquals(3, createdMessageProps.size()); assertEquals(Set.of("demoSourceNode"), createdMessageProps.get("sourceNodes")); assertEquals(Set.of("demoTargetNode"), createdMessageProps.get("targetNodes")); assertEquals(false, createdMessageProps.get("waitForFinalState")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("migrate_replicas", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.MIGRATE_REPLICAS, context.getAction()); + assertEquals("async", context.getAsyncId()); } @Test @@ -101,13 +70,16 @@ public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); migrateReplicasAPI.migrateReplicas(requestBody); - verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); + assertEquals(1, createdMessageProps.size()); assertEquals(Set.of("demoSourceNode"), createdMessageProps.get("sourceNodes")); - assertEquals("migrate_replicas", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.MIGRATE_REPLICAS, context.getAction()); + assertNull("There should be no asyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index d70db774dd09..0fd920b1bf89 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -18,77 +18,47 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.Map; -import java.util.Optional; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.CollectionParams; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; /** Unit tests for {@link ReplaceNode} */ -public class ReplaceNodeAPITest extends SolrTestCaseJ4 { +public class ReplaceNodeAPITest extends MockAPITest { - private CoreContainer mockCoreContainer; - private ZkController mockZkController; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; private ReplaceNode replaceNodeApi; - private DistributedCollectionConfigSetCommandRunner mockCommandRunner; - private ArgumentCaptor messageCapturer; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } @Override @Before public void setUp() throws Exception { super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - queryResponse = new SolrQueryResponse(); replaceNodeApi = new ReplaceNode(mockCoreContainer, mockQueryRequest, queryResponse); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - - when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); } @Test public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(5, createdMessageProps.size()); + assertEquals(3, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); assertEquals("demoTargetNode", createdMessageProps.get("targetNode")); assertEquals(false, createdMessageProps.get("waitForFinalState")); - assertEquals("async", createdMessageProps.get("async")); - assertEquals("replacenode", createdMessageProps.get("operation")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.REPLACENODE, context.getAction()); + assertEquals("async", context.getAsyncId()); } @Test @@ -98,30 +68,29 @@ public void testRequestBodyCanBeOmittedAltogether() throws Exception { final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(2, createdMessageProps.size()); + assertEquals(1, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); - assertEquals("replacenode", createdMessageProps.get("operation")); } @Test public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); replaceNodeApi.replaceNode("demoSourceNode", requestBody); - verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); final Map createdMessageProps = createdMessage.getProperties(); - assertEquals(3, createdMessageProps.size()); + assertEquals(2, createdMessageProps.size()); assertEquals("demoSourceNode", createdMessageProps.get("sourceNode")); assertEquals("demoTargetNode", createdMessageProps.get("targetNode")); - assertEquals("replacenode", createdMessageProps.get("operation")); assertFalse( "Expected message to not contain value for waitForFinalState: " + createdMessageProps.get("waitForFinalState"), createdMessageProps.containsKey("waitForFinalState")); - assertFalse( - "Expected message to not contain value for async: " + createdMessageProps.get("async"), - createdMessageProps.containsKey("async")); + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.REPLACENODE, context.getAction()); + assertNull("asyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java index 514423ba5df0..0e76cfd836fa 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java @@ -16,18 +16,44 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.COPY_FILES_STRATEGY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateCollectionBackup} */ -public class V2CollectionBackupApiTest extends SolrTestCaseJ4 { +public class V2CollectionBackupApiTest extends MockAPITest { + + private CreateCollectionBackup createCollectionBackup; + private BackupRepository mockBackupRepository; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository("someRepoName")).thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + + createCollectionBackup = + new CreateCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test - public void testCreateRemoteMessageWithAllProperties() { + public void testCreateRemoteMessageWithAllProperties() throws Exception { + final var requestBody = new CreateCollectionBackupRequestBody(); requestBody.location = "/some/location"; requestBody.repository = "someRepoName"; @@ -38,12 +64,19 @@ public void testCreateRemoteMessageWithAllProperties() { requestBody.maxNumBackupPoints = 123; requestBody.async = "someId"; - var message = - CreateCollectionBackup.createRemoteMessage( - "someCollectionName", "someBackupName", requestBody); - var messageProps = message.getProperties(); + when(mockClusterState.hasCollection("someCollectionName")).thenReturn(true); + when(mockBackupRepository.getBackupLocation(requestBody.location)) + .thenReturn(requestBody.location); + when(mockBackupRepository.exists(any())).thenReturn(true); + + createCollectionBackup.createCollectionBackup( + "someCollectionName", "someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); - assertEquals(11, messageProps.size()); + var message = messageCapturer.getValue(); + var messageProps = message.getProperties(); + assertEquals(messageProps.toString(), 9, messageProps.size()); assertEquals("someCollectionName", messageProps.get("collection")); assertEquals("/some/location", messageProps.get("location")); assertEquals("someRepoName", messageProps.get("repository")); @@ -52,30 +85,46 @@ public void testCreateRemoteMessageWithAllProperties() { assertEquals("someSnapshotName", messageProps.get("commitName")); assertEquals(true, messageProps.get("incremental")); assertEquals(123, messageProps.get("maxNumBackupPoints")); - assertEquals("someId", messageProps.get("async")); - assertEquals("backup", messageProps.get(QUEUE_OPERATION)); assertEquals("someBackupName", messageProps.get("name")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BACKUP, context.getAction()); + assertEquals("someId", context.getAsyncId()); } @Test - public void testCreateRemoteMessageOmitsNullValues() { + public void testCreateRemoteMessageOmitsNullValues() throws Exception { final var requestBody = new CreateCollectionBackupRequestBody(); requestBody.location = "/some/location"; - var message = - CreateCollectionBackup.createRemoteMessage( - "someCollectionName", "someBackupName", requestBody); - var messageProps = message.getProperties(); + when(mockClusterState.hasCollection("someCollectionName")).thenReturn(true); + when(mockBackupRepository.getBackupLocation(requestBody.location)) + .thenReturn(requestBody.location); + when(mockBackupRepository.exists(any())).thenReturn(true); - assertEquals(4, messageProps.size()); + createCollectionBackup.createCollectionBackup( + "someCollectionName", "someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + var message = messageCapturer.getValue(); + var messageProps = message.getProperties(); + assertEquals(5, messageProps.size()); assertEquals("someCollectionName", messageProps.get("collection")); assertEquals("/some/location", messageProps.get("location")); - assertEquals("backup", messageProps.get(QUEUE_OPERATION)); assertEquals("someBackupName", messageProps.get("name")); + assertEquals(true, messageProps.get("incremental")); + assertEquals("copy-files", messageProps.get("indexBackup")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.BACKUP, context.getAction()); + assertNull(context.getAsyncId()); } @Test public void testCanCreateV2RequestBodyFromV1Params() { + when(mockClusterState.hasCollection("demoSourceNode")).thenReturn(true); + final var params = new ModifiableSolrParams(); params.set("collection", "someCollectionName"); params.set("location", "/some/location"); From 3627d2962d71bb91132a3d34ba8bb8c3eb1eeec2 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 13 Jan 2026 09:26:11 -0800 Subject: [PATCH 26/67] Add missing test utility class --- .../solr/handler/admin/api/MockAPITest.java | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java new file mode 100644 index 000000000000..a47d4943137c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.trace.Span; +import java.util.Optional; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.OverseerSolrResponse; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.cloud.api.collections.AdminCmdContext; +import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; +import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.Before; +import org.junit.BeforeClass; +import org.mockito.ArgumentCaptor; + +/** + * Abstract test class to setup shared mocks for unit testing v2 API calls that go to the Overseer. + */ +public class MockAPITest extends SolrTestCaseJ4 { + + protected CoreContainer mockCoreContainer; + protected ZkController mockZkController; + protected ClusterState mockClusterState; + protected DistributedCollectionConfigSetCommandRunner mockCommandRunner; + protected SolrZkClient mockSolrZkClient; + protected ZkStateReader mockZkStateReader; + protected SolrQueryRequest mockQueryRequest; + protected SolrQueryResponse queryResponse; + protected ArgumentCaptor messageCapturer; + protected ArgumentCaptor contextCapturer; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + mockCoreContainer = mock(CoreContainer.class); + mockZkController = mock(ZkController.class); + mockClusterState = mock(ClusterState.class); + mockSolrZkClient = mock(SolrZkClient.class); + mockZkStateReader = mock(ZkStateReader.class); + mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); + when(mockCoreContainer.getZkController()).thenReturn(mockZkController); + when(mockCoreContainer.getAliases()).thenReturn(Aliases.EMPTY); + when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); + when(mockZkController.getClusterState()).thenReturn(mockClusterState); + when(mockZkController.getZkStateReader()).thenReturn(mockZkStateReader); + when(mockZkController.getZkClient()).thenReturn(mockSolrZkClient); + when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) + .thenReturn(new OverseerSolrResponse(new NamedList<>())); + mockQueryRequest = mock(SolrQueryRequest.class); + when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); + queryResponse = new SolrQueryResponse(); + messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); + contextCapturer = ArgumentCaptor.forClass(AdminCmdContext.class); + } +} From 4a49aa93c74edd807e101a1f267bf263c8fcb73e Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 13 Jan 2026 11:49:43 -0800 Subject: [PATCH 27/67] Move over remaining APIs and tests --- .../handler/admin/api/BalanceReplicas.java | 2 - .../handler/admin/api/CreateCollection.java | 6 +- .../solr/handler/admin/api/CreateShard.java | 3 - .../solr/handler/admin/api/DeleteReplica.java | 15 +- .../solr/handler/admin/api/DeleteShard.java | 3 - .../admin/api/ReloadCollectionAPI.java | 17 +- .../handler/admin/api/RestoreCollection.java | 4 +- .../admin/api/AddReplicaPropertyAPITest.java | 9 +- .../handler/admin/api/BackupCoreAPITest.java | 25 +-- .../admin/api/BalanceShardUniqueAPITest.java | 16 +- .../admin/api/CoreSnapshotAPITest.java | 30 ++-- .../handler/admin/api/CreateAliasAPITest.java | 20 +-- .../admin/api/CreateCollectionAPITest.java | 33 +--- .../api/CreateCollectionSnapshotAPITest.java | 9 +- .../admin/api/CreateReplicaAPITest.java | 15 +- .../handler/admin/api/CreateShardAPITest.java | 79 ++++---- .../admin/api/DeleteCollectionAPITest.java | 8 +- .../api/DeleteCollectionSnapshotAPITest.java | 10 +- .../admin/api/DeleteReplicaAPITest.java | 170 ++++++++++++------ .../api/DeleteReplicaPropertyAPITest.java | 33 ++-- .../handler/admin/api/DeleteShardAPITest.java | 81 ++++++--- .../handler/admin/api/ForceLeaderAPITest.java | 32 ++-- .../admin/api/MigrateReplicasAPITest.java | 12 +- .../admin/api/ReloadCollectionAPITest.java | 49 +++-- .../handler/admin/api/ReloadCoreAPITest.java | 11 +- .../handler/admin/api/ReplaceNodeAPITest.java | 10 +- .../admin/api/RestoreCollectionAPITest.java | 127 +++++++------ .../handler/admin/api/SyncShardAPITest.java | 32 ++-- .../handler/admin/api/UnloadCoreAPITest.java | 11 +- .../admin/api/V2CollectionBackupApiTest.java | 11 +- 30 files changed, 460 insertions(+), 423 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java index 088f6e225f83..d5ff9aa9ad4c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BalanceReplicas.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionParams.NODES; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; @@ -66,7 +65,6 @@ public ZkNodeProps createRemoteMessage(BalanceReplicasRequestBody requestBody) { insertIfNotNull(remoteMessage, NODES, requestBody.nodes); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); } - remoteMessage.put(QUEUE_OPERATION, CollectionAction.BALANCE_REPLICAS.toLower()); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java index e0130c67c957..18c95583275e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java @@ -105,10 +105,12 @@ public SubResponseAccumulatingJerseyResponse createCollection( // Populate any 'null' creation parameters that support COLLECTIONPROP defaults. populateDefaultsIfNecessary(coreContainer, requestBody); - final ZkNodeProps remoteMessage = createRemoteMessage(requestBody); final SolrResponse remoteResponse = submitRemoteMessageAndHandleResponse( - response, CollectionParams.CollectionAction.CREATE, remoteMessage, requestBody.async); + response, + CollectionParams.CollectionAction.CREATE, + createRemoteMessage(requestBody), + requestBody.async); // Even if Overseer does wait for the collection to be created, it sees a different cluster // state than this node, so this wait is required to make sure the local node Zookeeper watches diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java index 2912fbd8e079..410dcb30b05c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateShard.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; @@ -137,7 +136,6 @@ public static void invokeFromV1Params( public static ZkNodeProps createRemoteMessage( String collectionName, CreateShardRequestBody requestBody) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESHARD.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, requestBody.shardName); if (requestBody.createReplicas == null || requestBody.createReplicas) { @@ -154,7 +152,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, PULL_REPLICAS, requestBody.pullReplicas); insertIfNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, requestBody.followAliases); - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); if (requestBody.properties != null) { requestBody diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java index 857624b48526..2e6a2d648a92 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteReplica.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_IF_DOWN; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; @@ -92,8 +91,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicaByName( deleteInstanceDir, deleteDataDir, deleteIndex, - onlyIfDown, - async); + onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, remoteMessage, async); return response; @@ -129,8 +127,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicasByCount( deleteInstanceDir, deleteDataDir, deleteIndex, - onlyIfDown, - asyncId); + onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, remoteMessage, asyncId); return response; @@ -172,8 +169,7 @@ public SubResponseAccumulatingJerseyResponse deleteReplicasByCountAllShards( requestBody.deleteInstanceDir, requestBody.deleteDataDir, requestBody.deleteIndex, - requestBody.onlyIfDown, - requestBody.async); + requestBody.onlyIfDown); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.DELETEREPLICA, @@ -191,10 +187,8 @@ public static ZkNodeProps createRemoteMessage( Boolean deleteInstanceDir, Boolean deleteDataDir, Boolean deleteIndex, - Boolean onlyIfDown, - String asyncId) { + Boolean onlyIfDown) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETEREPLICA.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); insertIfNotNull(remoteMessage, SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, REPLICA, replicaName); @@ -204,7 +198,6 @@ public static ZkNodeProps createRemoteMessage( insertIfNotNull(remoteMessage, DELETE_DATA_DIR, deleteDataDir); insertIfNotNull(remoteMessage, DELETE_INDEX, deleteIndex); insertIfNotNull(remoteMessage, ONLY_IF_DOWN, onlyIfDown); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java index 5b9b5407274d..374738894f69 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteShard.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -96,14 +95,12 @@ public static ZkNodeProps createRemoteMessage( Boolean followAliases, String asyncId) { final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETESHARD.toLower()); remoteMessage.put(COLLECTION_PROP, collectionName); remoteMessage.put(SHARD_ID_PROP, shardName); insertIfNotNull(remoteMessage, FOLLOW_ALIASES, followAliases); insertIfNotNull(remoteMessage, DELETE_INSTANCE_DIR, deleteInstanceDir); insertIfNotNull(remoteMessage, DELETE_DATA_DIR, deleteDataDir); insertIfNotNull(remoteMessage, DELETE_INDEX, deleteIndex); - insertIfNotNull(remoteMessage, ASYNC, asyncId); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java index b620dd430a72..227d944e8e33 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReloadCollectionAPI.java @@ -16,7 +16,6 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -24,7 +23,6 @@ import jakarta.inject.Inject; import java.lang.invoke.MethodHandles; -import java.util.HashMap; import java.util.Map; import org.apache.solr.client.api.endpoint.ReloadCollectionApi; import org.apache.solr.client.api.model.ReloadCollectionRequestBody; @@ -66,27 +64,14 @@ public SubResponseAccumulatingJerseyResponse reloadCollection( fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collectionName, solrQueryRequest); - final ZkNodeProps remoteMessage = createRemoteMessage(collectionName, requestBody); submitRemoteMessageAndHandleResponse( response, CollectionParams.CollectionAction.RELOAD, - remoteMessage, + new ZkNodeProps(Map.of(NAME, collectionName)), requestBody != null ? requestBody.async : null); return response; } - public static ZkNodeProps createRemoteMessage( - String collectionName, ReloadCollectionRequestBody requestBody) { - final Map remoteMessage = new HashMap<>(); - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RELOAD.toLower()); - remoteMessage.put(NAME, collectionName); - if (requestBody != null) { - insertIfNotNull(remoteMessage, ASYNC, requestBody.async); - } - - return new ZkNodeProps(remoteMessage); - } - public static void invokeFromV1Params( CoreContainer coreContainer, SolrQueryRequest request, SolrQueryResponse response) throws Exception { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java index 1cbb9ce500dd..84cd3a3377cb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; @@ -110,6 +109,7 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( final var createRequestBody = requestBody.createCollectionParams; if (createRequestBody != null) { + createRequestBody.name = collectionName; CreateCollection.populateDefaultsIfNecessary(coreContainer, createRequestBody); CreateCollection.validateRequestBody(createRequestBody); if (Boolean.FALSE.equals(createRequestBody.createReplicas)) { @@ -128,6 +128,7 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( public ZkNodeProps createRemoteMessage( String backupName, RestoreCollectionRequestBody requestBody) { final Map remoteMessage = Utils.reflectToMap(requestBody); + remoteMessage.remove(ASYNC); // If the RESTORE is setup to create a new collection, copy those parameters first final var createReqBody = requestBody.createCollectionParams; @@ -144,7 +145,6 @@ public ZkNodeProps createRemoteMessage( } // Copy restore-specific parameters - remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RESTORE.toLower()); remoteMessage.put(NAME, backupName); return new ZkNodeProps(remoteMessage); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 05da868a7207..4b4cd0033153 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -36,7 +36,7 @@ public class AddReplicaPropertyAPITest extends MockAPITest { private static final AddReplicaPropertyRequestBody ANY_REQ_BODY = new AddReplicaPropertyRequestBody("anyValue"); - private AddReplicaProperty addReplicaPropApi; + private AddReplicaProperty api; @Override @Before @@ -44,7 +44,7 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - addReplicaPropApi = new AddReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); + api = new AddReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -55,7 +55,7 @@ public void testReportsErrorWhenCalledInStandaloneMode() { expectThrows( SolrException.class, () -> { - addReplicaPropApi.addReplicaProperty( + api.addReplicaProperty( "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); }); assertEquals(400, e.code()); @@ -68,8 +68,7 @@ public void testReportsErrorWhenCalledInStandaloneMode() { public void testCreatesValidOverseerMessage() throws Exception { when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - addReplicaPropApi.addReplicaProperty( - "someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); + api.addReplicaProperty("someColl", "someShard", "someReplica", "somePropName", ANY_REQ_BODY); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java index 2e5af32ff5d4..8bb81e676cbd 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/BackupCoreAPITest.java @@ -38,7 +38,7 @@ public class BackupCoreAPITest extends SolrTestCaseJ4 { - private CreateCoreBackup backupCoreAPI; + private CreateCoreBackup api; private static final String backupName = "my-new-backup"; @BeforeClass @@ -57,7 +57,7 @@ public void setUp() throws Exception { CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - backupCoreAPI = + api = new CreateCoreBackup( coreContainer, solrQueryRequest, solrQueryResponse, coreAdminAsyncTracker); } @@ -68,8 +68,7 @@ public void testCreateNonIncrementalBackupReturnsValidResponse() throws Exceptio backupCoreRequestBody.incremental = false; backupCoreRequestBody.backupName = backupName; SnapShooter.CoreSnapshotResponse response = - (SnapShooter.CoreSnapshotResponse) - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); + (SnapShooter.CoreSnapshotResponse) api.createBackup(coreName, backupCoreRequestBody); assertEquals(backupName, response.snapshotName); assertEquals("snapshot." + backupName, response.directoryName); @@ -84,11 +83,7 @@ public void testMissingLocationParameter() throws Exception { backupCoreRequestBody.incremental = false; backupCoreRequestBody.backupName = backupName; final SolrException solrException = - expectThrows( - SolrException.class, - () -> { - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); - }); + expectThrows(SolrException.class, () -> api.createBackup(coreName, backupCoreRequestBody)); assertEquals(500, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -105,11 +100,7 @@ public void testMissingCoreNameParameter() throws Exception { backupCoreRequestBody.backupName = backupName; final SolrException solrException = - expectThrows( - SolrException.class, - () -> { - backupCoreAPI.createBackup(null, backupCoreRequestBody); - }); + expectThrows(SolrException.class, () -> api.createBackup(null, backupCoreRequestBody)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -125,9 +116,7 @@ public void testNonIncrementalBackupForNonExistentCore() throws Exception { final SolrException solrException = expectThrows( SolrException.class, - () -> { - backupCoreAPI.createBackup("non-existent-core", backupCoreRequestBody); - }); + () -> api.createBackup("non-existent-core", backupCoreRequestBody)); assertEquals(500, solrException.code()); } @@ -138,7 +127,7 @@ public void testCreateIncrementalBackupReturnsValidResponse() throws Exception { backupCoreRequestBody.shardBackupId = "md_shard1_0"; IncrementalShardBackup.IncrementalShardSnapshotResponse response = (IncrementalShardBackup.IncrementalShardSnapshotResponse) - backupCoreAPI.createBackup(coreName, backupCoreRequestBody); + api.createBackup(coreName, backupCoreRequestBody); assertEquals(1, response.indexFileCount); assertEquals(1, response.uploadedIndexFileCount); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java index f479c878e842..fd1ea94d874d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/BalanceShardUniqueAPITest.java @@ -37,7 +37,7 @@ /** Unit tests for {@link BalanceShardUnique} */ public class BalanceShardUniqueAPITest extends MockAPITest { - private BalanceShardUnique balanceShardUnique; + private BalanceShardUnique api; @Override @Before @@ -45,15 +45,13 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - balanceShardUnique = new BalanceShardUnique(mockCoreContainer, mockQueryRequest, queryResponse); + api = new BalanceShardUnique(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> balanceShardUnique.balanceShardUnique("someCollectionName", null)); + expectThrows(SolrException.class, () -> api.balanceShardUnique("someCollectionName", null)); assertEquals(400, thrown.code()); assertEquals("Missing required request body", thrown.getMessage()); @@ -64,8 +62,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new BalanceShardUniqueRequestBody(); requestBody.property = "preferredLeader"; final SolrException thrown = - expectThrows( - SolrException.class, () -> balanceShardUnique.balanceShardUnique(null, requestBody)); + expectThrows(SolrException.class, () -> api.balanceShardUnique(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -77,8 +74,7 @@ public void testReportsErrorIfPropertyToBalanceIsMissing() { final var requestBody = new BalanceShardUniqueRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> balanceShardUnique.balanceShardUnique("someCollName", requestBody)); + SolrException.class, () -> api.balanceShardUnique("someCollName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: property", thrown.getMessage()); @@ -92,7 +88,7 @@ public void testCreateRemoteMessageAllProperties() throws Exception { requestBody.onlyActiveNodes = Boolean.TRUE; requestBody.async = "someAsyncId"; - balanceShardUnique.balanceShardUnique("someCollName", requestBody); + api.balanceShardUnique("someCollName", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java index 5ada271d324b..12b72b2c5636 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreSnapshotAPITest.java @@ -34,7 +34,7 @@ public class CoreSnapshotAPITest extends SolrTestCaseJ4 { - private CoreSnapshot coreSnapshotAPI; + private CoreSnapshot api; @BeforeClass public static void initializeCoreAndRequestFactory() throws Exception { @@ -54,7 +54,7 @@ public void setUp() throws Exception { CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - coreSnapshotAPI = + api = new CoreSnapshot(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); } @@ -63,7 +63,7 @@ public void setUp() throws Exception { @After public void deleteSnapshots() throws Exception { for (String snapshotName : snapshotsToCleanup) { - coreSnapshotAPI.deleteSnapshot(coreName, snapshotName, null); + api.deleteSnapshot(coreName, snapshotName, null); } snapshotsToCleanup.clear(); @@ -73,8 +73,7 @@ public void deleteSnapshots() throws Exception { public void testCreateSnapshotReturnsValidResponse() throws Exception { final String snapshotName = "my-new-snapshot"; - final CreateCoreSnapshotResponse response = - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + final CreateCoreSnapshotResponse response = api.createSnapshot(coreName, snapshotName, null); snapshotsToCleanup.add(snapshotName); assertEquals(coreName, response.core); @@ -91,9 +90,7 @@ public void testReportsErrorWhenCreatingSnapshotForNonexistentCore() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - coreSnapshotAPI.createSnapshot(nonExistentCoreName, "my-new-snapshot", null); - }); + () -> api.createSnapshot(nonExistentCoreName, "my-new-snapshot", null)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), @@ -106,11 +103,11 @@ public void testListSnapshotsReturnsValidResponse() throws Exception { for (int i = 0; i < 5; i++) { final String snapshotName = snapshotNameBase + i; - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + api.createSnapshot(coreName, snapshotName, null); snapshotsToCleanup.add(snapshotName); } - final ListCoreSnapshotsResponse response = coreSnapshotAPI.listSnapshots(coreName); + final ListCoreSnapshotsResponse response = api.listSnapshots(coreName); assertEquals(5, response.snapshots.size()); } @@ -123,7 +120,7 @@ public void testReportsErrorWhenListingSnapshotsForNonexistentCore() { expectThrows( SolrException.class, () -> { - coreSnapshotAPI.listSnapshots(nonExistentCoreName); + api.listSnapshots(nonExistentCoreName); }); assertEquals(400, solrException.code()); assertTrue( @@ -135,15 +132,14 @@ public void testReportsErrorWhenListingSnapshotsForNonexistentCore() { public void testDeleteSnapshotReturnsValidResponse() throws Exception { final String snapshotName = "my-new-snapshot"; - coreSnapshotAPI.createSnapshot(coreName, snapshotName, null); + api.createSnapshot(coreName, snapshotName, null); - final DeleteSnapshotResponse deleteResponse = - coreSnapshotAPI.deleteSnapshot(coreName, snapshotName, null); + final DeleteSnapshotResponse deleteResponse = api.deleteSnapshot(coreName, snapshotName, null); assertEquals(coreName, deleteResponse.coreName); assertEquals(snapshotName, deleteResponse.commitName); - final ListCoreSnapshotsResponse response = coreSnapshotAPI.listSnapshots(coreName); + final ListCoreSnapshotsResponse response = api.listSnapshots(coreName); assertEquals(0, response.snapshots.size()); } @@ -155,9 +151,7 @@ public void testReportsErrorWhenDeletingSnapshotForNonexistentCore() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - coreSnapshotAPI.deleteSnapshot(nonExistentCoreName, "non-existent-snapshot", null); - }); + () -> api.deleteSnapshot(nonExistentCoreName, "non-existent-snapshot", null)); assertEquals(400, solrException.code()); assertTrue( "Exception message differed from expected: " + solrException.getMessage(), diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java index 1f6707c2d7a0..bcfdf9ca0683 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java @@ -41,7 +41,7 @@ /** Unit tests for {@link CreateAlias} */ public class CreateAliasAPITest extends MockAPITest { - private CreateAlias createAliasApi; + private CreateAlias api; @Override @Before @@ -49,18 +49,12 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - createAliasApi = new CreateAlias(mockCoreContainer, mockQueryRequest, queryResponse); + api = new CreateAlias(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testReportsErrorIfRequestBodyMissing() { - final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateAlias(null, null, null); - api.createAlias(null); - }); + final SolrException thrown = expectThrows(SolrException.class, () -> api.createAlias(null)); assertEquals(400, thrown.code()); assertEquals("Request body is required but missing", thrown.getMessage()); @@ -224,7 +218,7 @@ public void testRemoteMessageCreationForTraditionalAlias() throws Exception { requestBody.collections = List.of("validColl1", "validColl2"); requestBody.async = "someAsyncId"; - createAliasApi.createAlias(requestBody); + api.createAlias(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -251,7 +245,7 @@ public void testRemoteMessageCreationForCategoryRoutedAlias() throws Exception { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - createAliasApi.createAlias(requestBody); + api.createAlias(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -284,7 +278,7 @@ public void testRemoteMessageCreationForTimeRoutedAlias() throws Exception { createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - createAliasApi.createAlias(requestBody); + api.createAlias(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -322,7 +316,7 @@ public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() throws Exc createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - createAliasApi.createAlias(requestBody); + api.createAlias(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java index 492548df197c..6fdbaa744192 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionAPITest.java @@ -53,7 +53,7 @@ /** Unit tests for {@link CreateCollection}. */ public class CreateCollectionAPITest extends MockAPITest { - private CreateCollection createCollectionApi; + private CreateCollection api; @Override @Before @@ -61,7 +61,7 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - createCollectionApi = new CreateCollection(mockCoreContainer, mockQueryRequest, queryResponse); + api = new CreateCollection(mockCoreContainer, mockQueryRequest, queryResponse); when(mockSolrZkClient.getData(eq("properties.json"), any(), any())) .thenReturn("{}".getBytes(StandardCharsets.UTF_8)); } @@ -69,12 +69,7 @@ public void setUp() throws Exception { @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateCollection(null, null, null); - api.createCollection(null); - }); + expectThrows(SolrException.class, () -> api.createCollection(null)); assertEquals(400, thrown.code()); assertEquals("Request body is missing but required", thrown.getMessage()); @@ -91,11 +86,7 @@ public void testReportsErrorIfReplicationFactorAndNrtReplicasConflict() { requestBody.nrtReplicas = 321; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals( @@ -110,11 +101,7 @@ public void testReportsErrorIfCollectionNameInvalid() { requestBody.config = "someConfig"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertTrue( @@ -132,11 +119,7 @@ public void testReportsErrorIfShardNamesInvalid() { requestBody.shardNames = List.of("good-name", "bad;name"); final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - CreateCollection.validateRequestBody(requestBody); - }); + expectThrows(SolrException.class, () -> CreateCollection.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertTrue( @@ -165,7 +148,7 @@ public void testCreateRemoteMessageAllProperties() throws Exception { requestBody.nodeSet = List.of("node1", "node2"); requestBody.shuffleNodes = false; - createCollectionApi.createCollection(requestBody); + api.createCollection(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -202,7 +185,7 @@ public void testNoReplicaCreationMessage() throws Exception { requestBody.createReplicas = false; requestBody.async = "someAsyncId"; - createCollectionApi.createCollection(requestBody); + api.createCollection(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java index 139f8d52aa2c..f3e1e8f8fdba 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -36,7 +36,7 @@ public class CreateCollectionSnapshotAPITest extends MockAPITest { - private CreateCollectionSnapshot createCollectionSnapshot; + private CreateCollectionSnapshot api; @Override @Before @@ -44,8 +44,7 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - createCollectionSnapshot = - new CreateCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + api = new CreateCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -55,7 +54,7 @@ public void testConstructsValidOverseerMessage() throws Exception { CreateCollectionSnapshotRequestBody body = new CreateCollectionSnapshotRequestBody(); body.followAliases = false; - createCollectionSnapshot.createCollectionSnapshot("myCollName", "mySnapshotName", body); + api.createCollectionSnapshot("myCollName", "mySnapshotName", body); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps messageOne = messageCapturer.getValue(); @@ -75,7 +74,7 @@ public void testConstructsValidOverseerMessage() throws Exception { body.followAliases = true; body.async = "testId"; Mockito.clearInvocations(mockCommandRunner); - createCollectionSnapshot.createCollectionSnapshot("myCollName", "mySnapshotName", body); + api.createCollectionSnapshot("myCollName", "mySnapshotName", body); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps messageTwo = messageCapturer.getValue(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java index bdfc75cf9649..eebf54dc464a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateReplicaAPITest.java @@ -53,7 +53,7 @@ /** Unit tests for {@link CreateReplica} */ public class CreateReplicaAPITest extends MockAPITest { - private CreateReplica createReplica; + private CreateReplica api; @Override @Before @@ -61,15 +61,14 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - createReplica = new CreateReplica(mockCoreContainer, mockQueryRequest, queryResponse); + api = new CreateReplica(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = expectThrows( - SolrException.class, - () -> createReplica.createReplica("someCollName", "someShardName", null)); + SolrException.class, () -> api.createReplica("someCollName", "someShardName", null)); assertEquals(400, thrown.code()); assertEquals("Required request-body is missing", thrown.getMessage()); @@ -79,8 +78,7 @@ public void testReportsErrorIfRequestBodyMissing() { public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new CreateReplicaRequestBody(); final SolrException thrown = - expectThrows( - SolrException.class, () -> createReplica.createReplica(null, "shardName", requestBody)); + expectThrows(SolrException.class, () -> api.createReplica(null, "shardName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -91,8 +89,7 @@ public void testReportsErrorIfShardNameMissing() { final var requestBody = new CreateReplicaRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> createReplica.createReplica("someCollectionName", null, requestBody)); + SolrException.class, () -> api.createReplica("someCollectionName", null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); @@ -119,7 +116,7 @@ public void testCreateRemoteMessageAllProperties() throws Exception { requestBody.properties = Map.of("propName1", "propVal1", "propName2", "propVal2"); when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); - createReplica.createReplica("someCollectionName", "someShardName", requestBody); + api.createReplica("someCollectionName", "someShardName", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java index a8bc3edf658d..b17a3ee59eda 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateShardAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; @@ -29,27 +28,44 @@ import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateShardRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ImplicitDocRouter; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link CreateShard} */ -public class CreateShardAPITest extends SolrTestCaseJ4 { +public class CreateShardAPITest extends MockAPITest { + + private CreateShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new CreateShard(mockCoreContainer, mockQueryRequest, queryResponse); + } @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollName", null); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollName", null)); assertEquals(400, thrown.code()); assertEquals("Required request-body is missing", thrown.getMessage()); @@ -60,12 +76,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "someShardName"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard(null, requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -76,12 +87,7 @@ public void testReportsErrorIfShardNameMissing() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = null; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollectionName", requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollectionName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); @@ -92,19 +98,14 @@ public void testReportsErrorIfShardNameIsInvalid() { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "invalid$shard@name"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new CreateShard(null, null, null); - api.createShard("someCollectionName", requestBody); - }); + expectThrows(SolrException.class, () -> api.createShard("someCollectionName", requestBody)); assertEquals(400, thrown.code()); assertThat(thrown.getMessage(), containsString("Invalid shard: [invalid$shard@name]")); } @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new CreateShardRequestBody(); requestBody.shardName = "someShardName"; requestBody.replicationFactor = 123; @@ -118,11 +119,20 @@ public void testCreateRemoteMessageAllProperties() { requestBody.async = "someAsyncId"; requestBody.properties = Map.of("propName1", "propVal1", "propName2", "propVal2"); - final var remoteMessage = - CreateShard.createRemoteMessage("someCollectionName", requestBody).getProperties(); + DocCollection mockCollection = mock(DocCollection.class); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + when(mockClusterState.getCollection(eq("someCollectionName"))).thenReturn(mockCollection); + when(mockCollection.get(DocCollection.CollectionStateProps.DOC_ROUTER)) + .thenReturn(Map.of(CommonParams.NAME, ImplicitDocRouter.NAME)); - assertEquals(13, remoteMessage.size()); - assertEquals("createshard", remoteMessage.get(QUEUE_OPERATION)); + api.createShard("someCollectionName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(11, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals(123, remoteMessage.get(REPLICATION_FACTOR)); @@ -132,9 +142,12 @@ public void testCreateRemoteMessageAllProperties() { assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM)); assertEquals(true, remoteMessage.get(WAIT_FOR_FINAL_STATE)); assertEquals(true, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); assertEquals("propVal1", remoteMessage.get("property.propName1")); assertEquals("propVal2", remoteMessage.get("property.propName2")); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.CREATESHARD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java index afff0677c341..e4a2109c6fcb 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionAPITest.java @@ -35,7 +35,7 @@ /** Unit tests for {@link DeleteCollection} */ public class DeleteCollectionAPITest extends MockAPITest { - private DeleteCollection deleteCollection; + private DeleteCollection api; @Override @Before @@ -43,14 +43,14 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - deleteCollection = new DeleteCollection(mockCoreContainer, mockQueryRequest, queryResponse); + api = new DeleteCollection(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testConstructsValidOverseerMessage() throws Exception { // Only required properties provided { - deleteCollection.deleteCollection("someCollName", null, null); + api.deleteCollection("someCollName", null, null); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -67,7 +67,7 @@ public void testConstructsValidOverseerMessage() throws Exception { // Optional property 'followAliases' also provided { Mockito.clearInvocations(mockCommandRunner); - deleteCollection.deleteCollection("someCollName", true, "test"); + api.deleteCollection("someCollName", true, "test"); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java index f04df2e4dbce..a7d820775766 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -35,7 +35,7 @@ public class DeleteCollectionSnapshotAPITest extends MockAPITest { - private DeleteCollectionSnapshot deleteCollectionSnapshot; + private DeleteCollectionSnapshot api; @Override @Before @@ -43,8 +43,7 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - deleteCollectionSnapshot = - new DeleteCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); + api = new DeleteCollectionSnapshot(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -52,7 +51,7 @@ public void testConstructsValidOverseerMessage() throws Exception { when(mockClusterState.hasCollection("myCollName")).thenReturn(true); when(mockSolrZkClient.exists(anyString())).thenReturn(false); - deleteCollectionSnapshot.deleteCollectionSnapshot("myCollName", "mySnapshotName", false, null); + api.deleteCollectionSnapshot("myCollName", "mySnapshotName", false, null); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps messageOne = messageCapturer.getValue(); @@ -70,8 +69,7 @@ public void testConstructsValidOverseerMessage() throws Exception { assertNull(context.getAsyncId()); Mockito.clearInvocations(mockCommandRunner); - deleteCollectionSnapshot.deleteCollectionSnapshot( - "myCollName", "mySnapshotName", true, "testId"); + api.deleteCollectionSnapshot("myCollName", "mySnapshotName", true, "testId"); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps messageTwo = messageCapturer.getValue(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java index d0dca0bce8e1..710c9673d348 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaAPITest.java @@ -17,34 +17,50 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ONLY_IF_DOWN; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; import static org.apache.solr.common.params.CollectionAdminParams.REPLICA; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; +import org.apache.solr.client.api.model.ScaleCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteReplica} */ -public class DeleteReplicaAPITest extends SolrTestCaseJ4 { +public class DeleteReplicaAPITest extends MockAPITest { + + private DeleteReplica api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteReplica(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - null, "someShard", "someReplica", null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + null, "someShard", "someReplica", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -55,11 +71,9 @@ public void testReportsErrorIfShardNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - "someCollection", null, "someReplica", null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + "someCollection", null, "someReplica", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); @@ -70,66 +84,112 @@ public void testReportsErrorIfReplicaNameMissingWhenDeletingByName() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteReplica(null, null, null); - api.deleteReplicaByName( - "someCollection", "someShard", null, null, null, null, null, null, null); - }); + () -> + api.deleteReplicaByName( + "someCollection", "someShard", null, null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: replica", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { - final var remoteMessage = - DeleteReplica.createRemoteMessage( - "someCollName", - "someShardName", - "someReplicaName", - 123, - true, - false, - true, - false, - true, - "someAsyncId") - .getProperties(); - - assertEquals(11, remoteMessage.size()); - assertEquals("deletereplica", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageByName() throws Exception { + api.deleteReplicaByName( + "someCollName", + "someShardName", + "someReplicaName", + true, + false, + true, + false, + true, + "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(8, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someReplicaName", remoteMessage.get(REPLICA)); - assertEquals(Integer.valueOf(123), remoteMessage.get(COUNT_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INSTANCE_DIR)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_DATA_DIR)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INDEX)); assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_IF_DOWN)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageByCount() throws Exception { + api.deleteReplicasByCount( + "someCollName", "someShardName", 123, true, false, true, false, true, "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(8, remoteMessage.size()); + assertEquals("someCollName", remoteMessage.get(COLLECTION)); + assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); + assertEquals(123, remoteMessage.get(COUNT_PROP)); + assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); + assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INSTANCE_DIR)); + assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_DATA_DIR)); + assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_INDEX)); + assertEquals(Boolean.TRUE, remoteMessage.get(ONLY_IF_DOWN)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); + } + + @Test + public void testCreateRemoteMessageByCountAllShards() throws Exception { + ScaleCollectionRequestBody body = new ScaleCollectionRequestBody(); + body.numToDelete = 123; + body.followAliases = true; + body.deleteIndex = true; + body.onlyIfDown = false; + body.async = "someAsyncId"; + api.deleteReplicasByCountAllShards("someCollName", body); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(5, remoteMessage.size()); + assertEquals("someCollName", remoteMessage.get(COLLECTION)); + assertEquals(123, remoteMessage.get(COUNT_PROP)); + assertEquals(Boolean.TRUE, remoteMessage.get(FOLLOW_ALIASES)); + assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INDEX)); + assertEquals(Boolean.FALSE, remoteMessage.get(ONLY_IF_DOWN)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testMissingValuesExcludedFromRemoteMessage() { - final var remoteMessage = - DeleteReplica.createRemoteMessage( - "someCollName", - "someShardName", - "someReplicaName", - null, - null, - null, - null, - null, - null, - null) - .getProperties(); - - assertEquals(4, remoteMessage.size()); - assertEquals("deletereplica", remoteMessage.get(QUEUE_OPERATION)); + public void testMissingValuesExcludedFromRemoteMessage() throws Exception { + api.deleteReplicaByName( + "someCollName", "someShardName", "someReplicaName", null, null, null, null, null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps message = messageCapturer.getValue(); + final Map remoteMessage = message.getProperties(); + assertEquals(3, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals("someReplicaName", remoteMessage.get(REPLICA)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, context.getAction()); + assertNull("AsyncId should be null", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java index e3950b1db42f..b5562f521186 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteReplicaPropertyAPITest.java @@ -44,7 +44,7 @@ */ public class DeleteReplicaPropertyAPITest extends MockAPITest { - private DeleteReplicaProperty deleteReplicaProperty; + private DeleteReplicaProperty api; @Override @Before @@ -52,13 +52,11 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - deleteReplicaProperty = - new DeleteReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); + api = new DeleteReplicaProperty(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { - final var api = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); @@ -71,9 +69,7 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noCollectionParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noCollectionParams)); assertEquals("Missing required parameter: " + COLLECTION_PROP, e.getMessage()); } @@ -83,9 +79,7 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noShardParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noShardParams)); assertEquals("Missing required parameter: " + SHARD_ID_PROP, e.getMessage()); } @@ -107,45 +101,42 @@ public void testV1InvocationThrowsErrorsIfRequiredParametersMissing() { final SolrException e = expectThrows( SolrException.class, - () -> { - DeleteReplicaProperty.invokeUsingV1Inputs(api, noPropertyParams); - }); + () -> DeleteReplicaProperty.invokeUsingV1Inputs(api, noPropertyParams)); assertEquals("Missing required parameter: " + PROPERTY_PROP, e.getMessage()); } } @Test public void testV1InvocationPassesAllProvidedParameters() throws Exception { - final var api = mock(DeleteReplicaProperty.class); + final var mockApi = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); allParams.add(REPLICA_PROP, "someReplica"); allParams.add(PROPERTY_PROP, "somePropName"); - DeleteReplicaProperty.invokeUsingV1Inputs(api, allParams); + DeleteReplicaProperty.invokeUsingV1Inputs(mockApi, allParams); - verify(api).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); + verify(mockApi).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); } @Test public void testV1InvocationTrimsPropertyNamePrefixIfProvided() throws Exception { - final var api = mock(DeleteReplicaProperty.class); + final var mockApi = mock(DeleteReplicaProperty.class); final var allParams = new ModifiableSolrParams(); allParams.add(COLLECTION_PROP, "someColl"); allParams.add(SHARD_ID_PROP, "someShard"); allParams.add(REPLICA_PROP, "someReplica"); allParams.add(PROPERTY_PROP, "property.somePropName"); // NOTE THE "property." prefix! - DeleteReplicaProperty.invokeUsingV1Inputs(api, allParams); + DeleteReplicaProperty.invokeUsingV1Inputs(mockApi, allParams); - verify(api).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); + verify(mockApi).deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); } @Test public void testRPCMessageCreation() throws Exception { - deleteReplicaProperty.deleteReplicaProperty( - "someColl", "someShard", "someReplica", "somePropName"); + api.deleteReplicaProperty("someColl", "someShard", "someReplica", "somePropName"); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); final ZkNodeProps message = messageCapturer.getValue(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java index e26653e6d894..fa161880fb29 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteShardAPITest.java @@ -17,30 +17,45 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link DeleteShard} */ -public class DeleteShardAPITest extends SolrTestCaseJ4 { +public class DeleteShardAPITest extends MockAPITest { + + private DeleteShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new DeleteShard(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteShard(null, null, null); - api.deleteShard(null, "someShard", null, null, null, null, null); - }); + () -> api.deleteShard(null, "someShard", null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -51,43 +66,51 @@ public void testReportsErrorIfShardNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new DeleteShard(null, null, null); - api.deleteShard("someCollection", null, null, null, null, null, null); - }); + () -> api.deleteShard("someCollection", null, null, null, null, null, null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); } @Test - public void testCreateRemoteMessageAllProperties() { - final var remoteMessage = - DeleteShard.createRemoteMessage( - "someCollName", "someShardName", true, false, true, false, "someAsyncId") - .getProperties(); - - assertEquals(8, remoteMessage.size()); - assertEquals("deleteshard", remoteMessage.get(QUEUE_OPERATION)); + public void testCreateRemoteMessageAllProperties() throws Exception { + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.deleteShard("someCollName", "someShardName", true, false, true, false, "someAsyncId"); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(6, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INSTANCE_DIR)); assertEquals(Boolean.FALSE, remoteMessage.get(DELETE_DATA_DIR)); assertEquals(Boolean.TRUE, remoteMessage.get(DELETE_INDEX)); assertEquals(Boolean.FALSE, remoteMessage.get(FOLLOW_ALIASES)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESHARD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testMissingValuesExcludedFromRemoteMessage() { - final var remoteMessage = - DeleteShard.createRemoteMessage( - "someCollName", "someShardName", null, null, null, null, null) - .getProperties(); - - assertEquals(3, remoteMessage.size()); - assertEquals("deleteshard", remoteMessage.get(QUEUE_OPERATION)); + public void testMissingValuesExcludedFromRemoteMessage() throws Exception { + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.deleteShard("someCollName", "someShardName", null, null, null, null, null); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + + assertEquals(2, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(COLLECTION)); assertEquals("someShardName", remoteMessage.get(SHARD_ID_PROP)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.DELETESHARD, context.getAction()); + assertNull("There should be no asyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java index be1fce44fb81..5158041282e9 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ForceLeaderAPITest.java @@ -17,21 +17,30 @@ package org.apache.solr.handler.admin.api; -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.Mockito.when; + import org.apache.solr.common.SolrException; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link ForceLeader} */ -public class ForceLeaderAPITest extends SolrTestCaseJ4 { +public class ForceLeaderAPITest extends MockAPITest { + + private ForceLeader api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new ForceLeader(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new ForceLeader(null, null, null); - api.forceShardLeader(null, "someShard"); - }); + expectThrows(SolrException.class, () -> api.forceShardLeader(null, "someShard")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -40,12 +49,7 @@ public void testReportsErrorIfCollectionNameMissing() { @Test public void testReportsErrorIfShardNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new ForceLeader(null, null, null); - api.forceShardLeader("someCollection", null); - }); + expectThrows(SolrException.class, () -> api.forceShardLeader("someCollection", null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java index eef0987cdcf4..3e49723fa3dc 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/MigrateReplicasAPITest.java @@ -34,7 +34,7 @@ /** Unit tests for {@link ReplaceNode} */ public class MigrateReplicasAPITest extends MockAPITest { - private MigrateReplicas migrateReplicasAPI; + private MigrateReplicas api; @Override @Before @@ -42,7 +42,7 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - migrateReplicasAPI = new MigrateReplicas(mockCoreContainer, mockQueryRequest, queryResponse); + api = new MigrateReplicas(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -50,7 +50,7 @@ public void testCreatesValidOverseerMessage() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody( Set.of("demoSourceNode"), Set.of("demoTargetNode"), false, "async"); - migrateReplicasAPI.migrateReplicas(requestBody); + api.migrateReplicas(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -69,7 +69,7 @@ public void testCreatesValidOverseerMessage() throws Exception { public void testNoTargetNodes() throws Exception { MigrateReplicasRequestBody requestBody = new MigrateReplicasRequestBody(Set.of("demoSourceNode"), null, null, null); - migrateReplicasAPI.migrateReplicas(requestBody); + api.migrateReplicas(requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -87,9 +87,9 @@ public void testNoSourceNodesThrowsError() throws Exception { MigrateReplicasRequestBody requestBody1 = new MigrateReplicasRequestBody( Collections.emptySet(), Set.of("demoTargetNode"), null, null); - assertThrows(SolrException.class, () -> migrateReplicasAPI.migrateReplicas(requestBody1)); + assertThrows(SolrException.class, () -> api.migrateReplicas(requestBody1)); MigrateReplicasRequestBody requestBody2 = new MigrateReplicasRequestBody(null, Set.of("demoTargetNode"), null, null); - assertThrows(SolrException.class, () -> migrateReplicasAPI.migrateReplicas(requestBody2)); + assertThrows(SolrException.class, () -> api.migrateReplicas(requestBody2)); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java index 5900fece7a4e..03a8f3e6d7d7 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCollectionAPITest.java @@ -17,42 +17,61 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.NAME; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import org.apache.solr.SolrTestCaseJ4; +import java.util.Map; import org.apache.solr.client.api.model.ReloadCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link ReloadCollectionAPI} */ -public class ReloadCollectionAPITest extends SolrTestCaseJ4 { +public class ReloadCollectionAPITest extends MockAPITest { + + private ReloadCollectionAPI api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new ReloadCollectionAPI(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = expectThrows( SolrException.class, - () -> { - final var api = new ReloadCollectionAPI(null, null, null); - api.reloadCollection(null, new ReloadCollectionRequestBody()); - }); + () -> api.reloadCollection(null, new ReloadCollectionRequestBody())); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); } - // TODO message creation @Test - public void testCreateRemoteMessageAllProperties() { + public void testCreateRemoteMessageAllProperties() throws Exception { final var requestBody = new ReloadCollectionRequestBody(); requestBody.async = "someAsyncId"; - final var remoteMessage = - ReloadCollectionAPI.createRemoteMessage("someCollName", requestBody).getProperties(); - assertEquals(3, remoteMessage.size()); - assertEquals("reload", remoteMessage.get(QUEUE_OPERATION)); + api.reloadCollection("someCollName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); + assertEquals(1, remoteMessage.size()); assertEquals("someCollName", remoteMessage.get(NAME)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + + final AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RELOAD, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java index fc73c6781f9d..1589f97310aa 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java @@ -31,7 +31,7 @@ public class ReloadCoreAPITest extends SolrTestCaseJ4 { - private ReloadCore reloadCoreAPI; + private ReloadCore api; private static final String NON_EXISTENT_CORE = "non_existent_core"; @BeforeClass @@ -49,13 +49,12 @@ public void setUp() throws Exception { CoreContainer coreContainer = h.getCoreContainer(); CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - reloadCoreAPI = - new ReloadCore(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); + api = new ReloadCore(solrQueryRequest, solrQueryResponse, coreContainer, coreAdminAsyncTracker); } @Test public void testValidReloadCoreAPIResponse() throws Exception { - SolrJerseyResponse response = reloadCoreAPI.reloadCore(coreName, new ReloadCoreRequestBody()); + SolrJerseyResponse response = api.reloadCore(coreName, new ReloadCoreRequestBody()); assertEquals(0, response.responseHeader.status); } @@ -64,9 +63,7 @@ public void testNonExistentCoreExceptionResponse() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - reloadCoreAPI.reloadCore(NON_EXISTENT_CORE, new ReloadCoreRequestBody()); - }); + () -> api.reloadCore(NON_EXISTENT_CORE, new ReloadCoreRequestBody())); assertEquals(400, solrException.code()); assertTrue(solrException.getMessage().contains("No such core: " + NON_EXISTENT_CORE)); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java index 0fd920b1bf89..2ac1998ddd4a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java @@ -32,7 +32,7 @@ /** Unit tests for {@link ReplaceNode} */ public class ReplaceNodeAPITest extends MockAPITest { - private ReplaceNode replaceNodeApi; + private ReplaceNode api; @Override @Before @@ -40,13 +40,13 @@ public void setUp() throws Exception { super.setUp(); when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); - replaceNodeApi = new ReplaceNode(mockCoreContainer, mockQueryRequest, queryResponse); + api = new ReplaceNode(mockCoreContainer, mockQueryRequest, queryResponse); } @Test public void testCreatesValidOverseerMessage() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", false, "async"); - replaceNodeApi.replaceNode("demoSourceNode", requestBody); + api.replaceNode("demoSourceNode", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -63,7 +63,7 @@ public void testCreatesValidOverseerMessage() throws Exception { @Test public void testRequestBodyCanBeOmittedAltogether() throws Exception { - replaceNodeApi.replaceNode("demoSourceNode", null); + api.replaceNode("demoSourceNode", null); verify(mockCommandRunner).runCollectionCommand(any(), messageCapturer.capture(), anyLong()); final ZkNodeProps createdMessage = messageCapturer.getValue(); @@ -75,7 +75,7 @@ public void testRequestBodyCanBeOmittedAltogether() throws Exception { @Test public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception { final var requestBody = new ReplaceNodeRequestBody("demoTargetNode", null, null); - replaceNodeApi.replaceNode("demoSourceNode", requestBody); + api.replaceNode("demoSourceNode", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java index ce9833771f9c..166bb6aa90ff 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; @@ -25,40 +24,52 @@ import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS; import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; import static org.apache.solr.common.params.CoreAdminParams.NAME; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.net.URI; import java.util.List; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; import org.apache.solr.client.api.model.RestoreCollectionRequestBody; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.NodeConfig; -import org.apache.solr.request.LocalSolrQueryRequest; -import org.junit.BeforeClass; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link RestoreCollection} */ -public class RestoreCollectionAPITest extends SolrTestCaseJ4 { - - private static RestoreCollection restoreCollectionAPI; - - @BeforeClass - public static void setUpApi() { - restoreCollectionAPI = - new RestoreCollection( - new CoreContainer( - new NodeConfig.NodeConfigBuilder("testnode", createTempDir()).build()), - new LocalSolrQueryRequest(null, new NamedList<>()), - null); +public class RestoreCollectionAPITest extends MockAPITest { + + private static RestoreCollection api; + private BackupRepository mockBackupRepository; + + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + mockBackupRepository = mock(BackupRepository.class); + when(mockCoreContainer.newBackupRepository(eq("someRepository"))) + .thenReturn(mockBackupRepository); + when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); + when(mockBackupRepository.getBackupLocation(eq("someLocation"))).thenReturn("someLocation"); + URI uri = new URI("someLocation"); + when(mockBackupRepository.createDirectoryURI(eq("someLocation"))).thenReturn(uri); + when(mockBackupRepository.exists(eq(uri))).thenReturn(true); + + api = new RestoreCollection(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -66,11 +77,7 @@ public void testReportsErrorIfBackupNameMissing() { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollection"; final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection(null, requestBody); - }); + expectThrows(SolrException.class, () -> api.restoreCollection(null, requestBody)); assertEquals(400, thrown.code()); assertEquals("Required parameter 'backupName' missing", thrown.getMessage()); @@ -79,11 +86,7 @@ public void testReportsErrorIfBackupNameMissing() { @Test public void testReportsErrorIfRequestBodyMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", null); - }); + expectThrows(SolrException.class, () -> api.restoreCollection("someBackupName", null)); assertEquals(400, thrown.code()); assertEquals("Missing required request body", thrown.getMessage()); @@ -95,10 +98,7 @@ public void testReportsErrorIfCollectionNameMissing() { final var requestBody = new RestoreCollectionRequestBody(); final SolrException thrown = expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", requestBody); - }); + SolrException.class, () -> api.restoreCollection("someBackupName", requestBody)); assertEquals(400, thrown.code()); assertEquals("Required parameter 'collection' missing", thrown.getMessage()); @@ -110,10 +110,7 @@ public void testReportsErrorIfProvidedCollectionNameIsInvalid() { requestBody.collection = "invalid$collection@name"; final SolrException thrown = expectThrows( - SolrException.class, - () -> { - restoreCollectionAPI.restoreCollection("someBackupName", requestBody); - }); + SolrException.class, () -> api.restoreCollection("someBackupName", requestBody)); assertEquals(400, thrown.code()); assertThat( @@ -121,34 +118,41 @@ public void testReportsErrorIfProvidedCollectionNameIsInvalid() { } @Test - public void testCreatesValidRemoteMessageForExistingCollectionRestore() { + public void testCreatesValidRemoteMessageForExistingCollectionRestore() throws Exception { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; - requestBody.location = "/some/location/path"; + requestBody.location = "someLocation"; requestBody.backupId = 123; - requestBody.repository = "someRepositoryName"; + requestBody.repository = "someRepository"; requestBody.async = "someAsyncId"; - final var remoteMessage = - restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.restoreCollection("someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals(7, remoteMessage.size()); - assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); + assertEquals(5, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); - assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals(Integer.valueOf(123), remoteMessage.get(BACKUP_ID)); - assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); assertEquals("someBackupName", remoteMessage.get(NAME)); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RESTORE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test - public void testCreatesValidRemoteMessageForNewCollectionRestore() { + public void testCreatesValidRemoteMessageForNewCollectionRestore() throws Exception { final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; - requestBody.location = "/some/location/path"; + requestBody.location = "someLocation"; requestBody.backupId = 123; - requestBody.repository = "someRepositoryName"; + requestBody.repository = "someRepository"; requestBody.async = "someAsyncId"; final var createParams = new CreateCollectionRequestBody(); requestBody.createCollectionParams = createParams; @@ -159,16 +163,19 @@ public void testCreatesValidRemoteMessageForNewCollectionRestore() { createParams.nodeSet = List.of("node1", "node2"); createParams.properties = Map.of("foo", "bar"); - final var remoteMessage = - restoreCollectionAPI.createRemoteMessage("someBackupName", requestBody).getProperties(); + when(mockClusterState.hasCollection(eq("someCollectionName"))).thenReturn(true); + api.restoreCollection("someBackupName", requestBody); + verify(mockCommandRunner) + .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); + + final ZkNodeProps createdMessage = messageCapturer.getValue(); + final Map remoteMessage = createdMessage.getProperties(); - assertEquals(14, remoteMessage.size()); - assertEquals("restore", remoteMessage.get(QUEUE_OPERATION)); + assertEquals(12, remoteMessage.size()); assertEquals("someCollectionName", remoteMessage.get(COLLECTION)); - assertEquals("/some/location/path", remoteMessage.get(BACKUP_LOCATION)); + assertEquals("someLocation", remoteMessage.get(BACKUP_LOCATION)); assertEquals(Integer.valueOf(123), remoteMessage.get(BACKUP_ID)); - assertEquals("someRepositoryName", remoteMessage.get(BACKUP_REPOSITORY)); - assertEquals("someAsyncId", remoteMessage.get(ASYNC)); + assertEquals("someRepository", remoteMessage.get(BACKUP_REPOSITORY)); assertEquals("someBackupName", remoteMessage.get(NAME)); assertEquals("someConfig", remoteMessage.get(COLL_CONF)); assertEquals(Integer.valueOf(123), remoteMessage.get(NRT_REPLICAS)); @@ -177,6 +184,10 @@ public void testCreatesValidRemoteMessageForNewCollectionRestore() { assertEquals(Integer.valueOf(789), remoteMessage.get(PULL_REPLICAS)); assertEquals("node1,node2", remoteMessage.get(CREATE_NODE_SET_PARAM)); assertEquals("bar", remoteMessage.get("property.foo")); + + AdminCmdContext context = contextCapturer.getValue(); + assertEquals(CollectionParams.CollectionAction.RESTORE, context.getAction()); + assertEquals("someAsyncId", context.getAsyncId()); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java index 23d3f8982a25..e9c605de183b 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/SyncShardAPITest.java @@ -17,21 +17,30 @@ package org.apache.solr.handler.admin.api; -import org.apache.solr.SolrTestCaseJ4; +import static org.mockito.Mockito.when; + import org.apache.solr.common.SolrException; +import org.junit.Before; import org.junit.Test; /** Unit tests for {@link SyncShard} */ -public class SyncShardAPITest extends SolrTestCaseJ4 { +public class SyncShardAPITest extends MockAPITest { + + private SyncShard api; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + when(mockCoreContainer.isZooKeeperAware()).thenReturn(true); + + api = new SyncShard(mockCoreContainer, mockQueryRequest, queryResponse); + } + @Test public void testReportsErrorIfCollectionNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new SyncShard(null, null, null); - api.syncShard(null, "someShard"); - }); + expectThrows(SolrException.class, () -> api.syncShard(null, "someShard")); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: collection", thrown.getMessage()); @@ -40,12 +49,7 @@ public void testReportsErrorIfCollectionNameMissing() { @Test public void testReportsErrorIfShardNameMissing() { final SolrException thrown = - expectThrows( - SolrException.class, - () -> { - final var api = new SyncShard(null, null, null); - api.syncShard("someCollection", null); - }); + expectThrows(SolrException.class, () -> api.syncShard("someCollection", null)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: shard", thrown.getMessage()); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java index c823f33dea12..c2c8a3b126e4 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java @@ -30,7 +30,7 @@ import org.junit.Test; public class UnloadCoreAPITest extends SolrTestCaseJ4 { - private UnloadCore unloadCoreAPI; + private UnloadCore api; private static final String NON_EXISTENT_CORE = "non_existent_core"; @BeforeClass @@ -48,13 +48,12 @@ public void setUp() throws Exception { CoreContainer coreContainer = h.getCoreContainer(); CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker = new CoreAdminHandler.CoreAdminAsyncTracker(); - unloadCoreAPI = - new UnloadCore(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse); + api = new UnloadCore(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse); } @Test public void testValidUnloadCoreAPIResponse() throws Exception { - SolrJerseyResponse response = unloadCoreAPI.unloadCore(coreName, getUnloadCoreRequestBodyObj()); + SolrJerseyResponse response = api.unloadCore(coreName, getUnloadCoreRequestBodyObj()); assertEquals(0, response.responseHeader.status); } @@ -63,9 +62,7 @@ public void testNonExistentCoreExceptionResponse() { final SolrException solrException = expectThrows( SolrException.class, - () -> { - unloadCoreAPI.unloadCore(NON_EXISTENT_CORE, getUnloadCoreRequestBodyObj()); - }); + () -> api.unloadCore(NON_EXISTENT_CORE, getUnloadCoreRequestBodyObj())); assertEquals(400, solrException.code()); assertTrue(solrException.getMessage().contains("Cannot unload non-existent core")); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java index 0e76cfd836fa..535bf7747433 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/V2CollectionBackupApiTest.java @@ -34,7 +34,7 @@ /** Unit tests for {@link CreateCollectionBackup} */ public class V2CollectionBackupApiTest extends MockAPITest { - private CreateCollectionBackup createCollectionBackup; + private CreateCollectionBackup api; private BackupRepository mockBackupRepository; @Override @@ -47,8 +47,7 @@ public void setUp() throws Exception { when(mockCoreContainer.newBackupRepository("someRepoName")).thenReturn(mockBackupRepository); when(mockCoreContainer.newBackupRepository(null)).thenReturn(mockBackupRepository); - createCollectionBackup = - new CreateCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); + api = new CreateCollectionBackup(mockCoreContainer, mockQueryRequest, queryResponse); } @Test @@ -69,8 +68,7 @@ public void testCreateRemoteMessageWithAllProperties() throws Exception { .thenReturn(requestBody.location); when(mockBackupRepository.exists(any())).thenReturn(true); - createCollectionBackup.createCollectionBackup( - "someCollectionName", "someBackupName", requestBody); + api.createCollectionBackup("someCollectionName", "someBackupName", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); @@ -102,8 +100,7 @@ public void testCreateRemoteMessageOmitsNullValues() throws Exception { .thenReturn(requestBody.location); when(mockBackupRepository.exists(any())).thenReturn(true); - createCollectionBackup.createCollectionBackup( - "someCollectionName", "someBackupName", requestBody); + api.createCollectionBackup("someCollectionName", "someBackupName", requestBody); verify(mockCommandRunner) .runCollectionCommand(contextCapturer.capture(), messageCapturer.capture(), anyLong()); From 4c9a76676aabb453dd65f5f7cce9a241d5e3e29b Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 16 Jan 2026 16:33:16 -0800 Subject: [PATCH 28/67] Fix changelog entry --- changelog/unreleased/solr-18011-locking-update.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/changelog/unreleased/solr-18011-locking-update.yml b/changelog/unreleased/solr-18011-locking-update.yml index a19faee91d93..3b714ae59345 100644 --- a/changelog/unreleased/solr-18011-locking-update.yml +++ b/changelog/unreleased/solr-18011-locking-update.yml @@ -4,7 +4,6 @@ type: changed # added, changed, fixed, deprecated, removed, dependency_update, s authors: - name: Houston Putman nick: HoustonPutman - url: https://home.apache.org/phonebook.html?uid=houston links: - name: SOLR-18011 url: https://issues.apache.org/jira/browse/SOLR-18011 From c7cfe19daad0a43c31461ac9037ab44fcf1d77be Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 16 Jan 2026 16:40:55 -0800 Subject: [PATCH 29/67] Remove files that shouldn't be changed. --- .../api/endpoint/AddReplicaPropertyApi.java | 4 +- .../model/InstallShardDataRequestBody.java | 4 - .../api/collections/InstallShardDataCmd.java | 4 - .../solr/handler/admin/api/MockAPITest.java | 91 ------------------- .../solr/s3/S3IncrementalBackupTest.java | 4 - .../solrj/impl/CloudHttp2SolrClientTest.java | 5 +- .../AbstractIncrementalBackupTest.java | 13 --- 7 files changed, 3 insertions(+), 122 deletions(-) delete mode 100644 solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java index 997613e292fc..8e4d53e521ab 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java @@ -24,7 +24,7 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; -import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; @Path("/collections/{collName}/shards/{shardName}/replicas/{replicaName}/properties/{propName}") public interface AddReplicaPropertyApi { @@ -33,7 +33,7 @@ public interface AddReplicaPropertyApi { @Operation( summary = "Adds a property to the specified replica", tags = {"replica-properties"}) - public SubResponseAccumulatingJerseyResponse addReplicaProperty( + public SolrJerseyResponse addReplicaProperty( @Parameter( description = "The name of the collection the replica belongs to.", required = true) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index 05b27f1dcab3..31bec8eb4346 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -24,9 +24,5 @@ public class InstallShardDataRequestBody { @JsonProperty public String repository; - @JsonProperty public String name; - - @JsonProperty public String shardBackupId; - @JsonProperty public String async; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 39b81cd434bd..ca654a150aed 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -109,10 +109,6 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String location; - @JsonProperty public String name = ""; - - @JsonProperty public String shardBackupId; - @JsonProperty(ASYNC) public String asyncId; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java deleted file mode 100644 index a47d4943137c..000000000000 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/MockAPITest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.handler.admin.api; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.opentelemetry.api.trace.Span; -import java.util.Optional; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.OverseerSolrResponse; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.api.collections.AdminCmdContext; -import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; -import org.apache.solr.common.cloud.Aliases; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.junit.Before; -import org.junit.BeforeClass; -import org.mockito.ArgumentCaptor; - -/** - * Abstract test class to setup shared mocks for unit testing v2 API calls that go to the Overseer. - */ -public class MockAPITest extends SolrTestCaseJ4 { - - protected CoreContainer mockCoreContainer; - protected ZkController mockZkController; - protected ClusterState mockClusterState; - protected DistributedCollectionConfigSetCommandRunner mockCommandRunner; - protected SolrZkClient mockSolrZkClient; - protected ZkStateReader mockZkStateReader; - protected SolrQueryRequest mockQueryRequest; - protected SolrQueryResponse queryResponse; - protected ArgumentCaptor messageCapturer; - protected ArgumentCaptor contextCapturer; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - - mockCoreContainer = mock(CoreContainer.class); - mockZkController = mock(ZkController.class); - mockClusterState = mock(ClusterState.class); - mockSolrZkClient = mock(SolrZkClient.class); - mockZkStateReader = mock(ZkStateReader.class); - mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class); - when(mockCoreContainer.getZkController()).thenReturn(mockZkController); - when(mockCoreContainer.getAliases()).thenReturn(Aliases.EMPTY); - when(mockZkController.getDistributedCommandRunner()).thenReturn(Optional.of(mockCommandRunner)); - when(mockZkController.getClusterState()).thenReturn(mockClusterState); - when(mockZkController.getZkStateReader()).thenReturn(mockZkStateReader); - when(mockZkController.getZkClient()).thenReturn(mockSolrZkClient); - when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) - .thenReturn(new OverseerSolrResponse(new NamedList<>())); - mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); - queryResponse = new SolrQueryResponse(); - messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); - contextCapturer = ArgumentCaptor.forClass(AdminCmdContext.class); - } -} diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index a34fb0ac9734..80c5207505b1 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -22,7 +22,6 @@ import java.lang.invoke.MethodHandles; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; -import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.slf4j.Logger; @@ -32,9 +31,6 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) -@LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 3531bc0cd081..e80c60242a7f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -581,10 +581,7 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // And since all the nodes are hosting cores from all shards, the // distributed query formed by this node will select cores from the // local shards only - QueryResponse qResponse = null; - for (int i = 0; i < 100; i++) { - qResponse = cloudClient.query(collectionName, qRequest); - } + QueryResponse qResponse = cloudClient.query(collectionName, qRequest); Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); assertNotNull("Unable to obtain " + ShardParams.SHARDS_INFO, shardsInfo); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 79bb122a8f8d..0df483d8b35d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -71,7 +71,6 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -85,9 +84,6 @@ *

For a similar test harness for snapshot backup/restoration see {@link * AbstractCloudBackupRestoreTestCase} */ -@LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -131,9 +127,6 @@ public void setTestSuffix(String testSuffix) { public abstract String getBackupLocation(); @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSimple() throws Exception { setTestSuffix("testbackupincsimple"); final String backupCollectionName = getCollectionName(); @@ -200,9 +193,6 @@ public void testSimple() throws Exception { } @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testRestoreToOriginalCollection() throws Exception { setTestSuffix("testbackuprestoretooriginal"); final String backupCollectionName = getCollectionName(); @@ -365,9 +355,6 @@ public void testBackupIncremental() throws Exception { } @Test - @LogLevel( - value = - "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public void testSkipConfigset() throws Exception { setTestSuffix("testskipconfigset"); final String backupCollectionName = getCollectionName(); From 08326cc90511820acd85cf920c88fe1051fdcc69 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 16 Jan 2026 16:47:59 -0800 Subject: [PATCH 30/67] One more that shouldn't be changed --- .../apache/solr/handler/admin/CollectionsHandler.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 20213193389b..05361725ad89 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -101,7 +101,6 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; -import static org.apache.solr.common.params.CoreAdminParams.SHARD_BACKUP_ID; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -118,7 +117,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.commons.lang3.StringUtils; import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; @@ -166,6 +164,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; @@ -248,7 +247,9 @@ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { if (action == null) return PermissionNameProvider.Name.COLL_READ_PERM; CollectionParams.CollectionAction collectionAction = CollectionParams.CollectionAction.get(action); - if (collectionAction == null) return null; + if (collectionAction == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + action); + } return collectionAction.isWrite ? PermissionNameProvider.Name.COLL_EDIT_PERM : PermissionNameProvider.Name.COLL_READ_PERM; @@ -369,7 +370,7 @@ public static SolrResponse submitCollectionApiCommand( if (adminCmdContext.getAsyncId() != null && !adminCmdContext.getAsyncId().isBlank()) { additionalProps.put(ASYNC, adminCmdContext.getAsyncId()); } - if (StringUtils.isNotBlank(adminCmdContext.getCallingLockIds())) { + if (StrUtils.isNotBlank(adminCmdContext.getCallingLockIds())) { additionalProps.put(CALLING_LOCK_IDS_HEADER, adminCmdContext.getCallingLockIds()); } m = m.plus(additionalProps); @@ -1069,8 +1070,6 @@ public Map execute( reqBody.async = req.getParams().get(ASYNC); reqBody.repository = req.getParams().get(BACKUP_REPOSITORY); reqBody.location = req.getParams().get(BACKUP_LOCATION); - reqBody.name = req.getParams().get(NAME); - reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = From b90c283003252423d6ffcee5f38855ac73cfd238 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 16 Jan 2026 17:46:20 -0800 Subject: [PATCH 31/67] Some more fixes --- .../model/InstallShardDataRequestBody.java | 2 -- .../collections/CollectionHandlingUtils.java | 27 ++++++++++++------- .../api/collections/InstallShardDataCmd.java | 10 +++++++ .../api/collections/ReindexCollectionCmd.java | 6 ++--- .../cloud/api/collections/RestoreCmd.java | 1 - .../cloud/api/collections/SplitShardCmd.java | 3 +-- .../handler/admin/CollectionsHandler.java | 1 - 7 files changed, 30 insertions(+), 20 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java index a72b9c2bc920..05b27f1dcab3 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/InstallShardDataRequestBody.java @@ -29,6 +29,4 @@ public class InstallShardDataRequestBody { @JsonProperty public String shardBackupId; @JsonProperty public String async; - - @JsonProperty public String callingLockId; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 5972802b6596..3e541a24696a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -624,31 +624,38 @@ private static NamedList waitForCoreAdminAsyncCallToComplete( public static ShardRequestTracker syncRequestTracker( AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { - return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(), ccc); + return syncRequestTracker(adminCmdContext, ccc.getAdminPath(), ccc); + } + + public static ShardRequestTracker syncRequestTracker( + AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) { + return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(), adminPath, ccc); } public static ShardRequestTracker asyncRequestTracker( AdminCmdContext adminCmdContext, CollectionCommandContext ccc) { + return asyncRequestTracker(adminCmdContext, ccc.getAdminPath(), ccc); + } + + public static ShardRequestTracker asyncRequestTracker( + AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) { return requestTracker( - adminCmdContext.getAsyncId(), adminCmdContext.getSubRequestCallingLockIds(), ccc); + adminCmdContext.getAsyncId(), + adminCmdContext.getSubRequestCallingLockIds(), + adminPath, + ccc); } protected static ShardRequestTracker requestTracker( - String asyncId, String lockIds, CollectionCommandContext ccc) { + String asyncId, String lockIds, String adminPath, CollectionCommandContext ccc) { return new ShardRequestTracker( asyncId, lockIds, - ccc.getAdminPath(), + adminPath, ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); } - public static ShardRequestTracker asyncRequestTracker( - String asyncId, String adminPath, CollectionCommandContext ccc) { - return new ShardRequestTracker( - asyncId, adminPath, ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); - } - public static class ShardRequestTracker { private final String asyncId; private final String lockIdList; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 3aebdaaf5b9b..149f6b88cef5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -17,6 +17,9 @@ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.invoke.MethodHandles; @@ -37,6 +40,7 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -285,6 +289,9 @@ public static void handleCoreRestoreResponses( /** A value-type representing the message received by {@link InstallShardDataCmd} */ public static class RemoteMessage implements JacksonReflectMapWriter { + @JsonProperty(QUEUE_OPERATION) + public String operation = CollectionParams.CollectionAction.INSTALLSHARDDATA.toLower(); + @JsonProperty public String collection; @JsonProperty public String shard; @@ -297,6 +304,9 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String shardBackupId; + @JsonProperty(ASYNC) + public String asyncId; + public void validate() { if (StrUtils.isBlank(collection)) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index b3a7a4fc1762..e72cfbe04a4c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -595,8 +595,7 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList chkCollection, daemonReplica, targetCollection, - createdTarget, - lockId); + createdTarget); if (exc != null) { results.add("error", exc.toString()); } @@ -891,8 +890,7 @@ private void cleanup( String chkCollection, Replica daemonReplica, String daemonName, - boolean createdTarget, - String lockId) + boolean createdTarget) throws Exception { log.info("## Cleaning up after abort or error"); // 1. kill the daemon diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index d595b2507c6f..9beb40931228 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -168,7 +168,6 @@ private static class RestoreContext implements Closeable { final URI backupPath; final List nodeList; - final String lockId; final CoreContainer container; final BackupRepository repository; final ZkStateReader zkStateReader; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 3bde5dec9722..baf0bf408478 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -910,8 +910,7 @@ private void cleanupAfterFailure( String collectionName, String parentShard, List subSlices, - Set offlineSlices, - String lockId) { + Set offlineSlices) { log.info("Cleaning up after a failed split of {}/{}", collectionName, parentShard); // get the latest state try { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 8771b5af263f..bd4c9d7bcee6 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -1073,7 +1073,6 @@ public Map execute( reqBody.location = req.getParams().get(BACKUP_LOCATION); reqBody.name = req.getParams().get(NAME); reqBody.shardBackupId = req.getParams().get(SHARD_BACKUP_ID); - reqBody.callingLockId = req.getParams().get(CALLING_LOCK_ID); final InstallShardData installApi = new InstallShardData(h.coreContainer, req, rsp); final SolrJerseyResponse installResponse = From b13d97e9bf8254377ab7ab0e249967b878752174 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 21 Jan 2026 12:10:59 -0800 Subject: [PATCH 32/67] SOLR-18080: Initiate Leader election for ShardTerms --- .../solr/cloud/RecoveringCoreTermWatcher.java | 59 ++++- .../cloud/ShardLeaderElectionContext.java | 4 +- .../org/apache/solr/cloud/ShardTerms.java | 22 ++ .../org/apache/solr/cloud/SyncStrategy.java | 39 +++- .../org/apache/solr/cloud/ZkShardTerms.java | 5 + .../org/apache/solr/handler/IndexFetcher.java | 2 +- .../solr/update/SolrCmdDistributor.java | 5 + .../DistributedZkUpdateProcessor.java | 22 +- .../org/apache/solr/cloud/ShardTermsTest.java | 24 +++ .../solr/cloud/ZkShardTermsRecoveryTest.java | 202 ++++++++++++++++++ .../apache/solr/cloud/ZkShardTermsTest.java | 33 +++ .../solr/common/cloud/ZkStateReader.java | 29 ++- .../solr/common/cloud/ZkCoreNodeProps.java | 4 + 13 files changed, 399 insertions(+), 51 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index 40b340b283ce..515110b95866 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -19,6 +19,7 @@ import java.lang.invoke.MethodHandles; import java.util.concurrent.atomic.AtomicLong; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; @@ -52,15 +53,55 @@ public boolean onTermChanged(ShardTerms terms) { if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true; String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(); - if (terms.haveHighestTermValue(coreNodeName)) return true; - if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) { - log.info( - "Start recovery on {} because core's term is less than leader's term", coreNodeName); - lastTermDoRecovery.set(terms.getTerm(coreNodeName)); - solrCore - .getUpdateHandler() - .getSolrCoreState() - .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); + + // If we have the highest term, there is nothing to do + if (terms.haveHighestTermValue(coreNodeName)) { + return true; + } + + long lastRecoveryTerm; + long newTerm; + synchronized (lastTermDoRecovery) { + lastRecoveryTerm = lastTermDoRecovery.get(); + newTerm = terms.getTerm(coreNodeName); + if (lastRecoveryTerm < newTerm) { + lastTermDoRecovery.set(newTerm); + } + } + + if (coreDescriptor.getCloudDescriptor().isLeader()) { + log.warn( + "Removing {} leader as leader, since its term is no longer the highest. This will initiate recovery", + coreNodeName); + coreContainer.getZkController().giveupLeadership(coreDescriptor); + } else if (lastRecoveryTerm < newTerm) { + CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor(); + Replica leaderReplica = + solrCore + .getCoreContainer() + .getZkController() + .getClusterState() + .getCollection(cloudDescriptor.getCollectionName()) + .getSlice(cloudDescriptor.getShardId()) + .getLeader(); + + // Only recover if the leader replica still has the highest term. + // If not, then the leader-election process will take care of recovery. + if (leaderReplica != null && terms.canBecomeLeader(leaderReplica.getName())) { + log.info( + "Start recovery on {} because core's term is less than leader's term", coreNodeName); + solrCore + .getUpdateHandler() + .getSolrCoreState() + .doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor()); + } else { + if (log.isInfoEnabled()) { + log.info( + "Defer recovery on {} because leader-election will happen soon, old leader: {}", + coreNodeName, + leaderReplica == null ? null : leaderReplica.getName()); + } + } } } catch (Exception e) { if (log.isInfoEnabled()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 16a29f89a586..5b956eac5cfe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -507,12 +507,10 @@ private void rejoinLeaderElection(SolrCore core) throws InterruptedException, Ke return; } - log.info("There may be a better leader candidate than us - going back into recovery"); + log.info("There may be a better leader candidate than us - rejoining the election"); cancelElection(); - core.getUpdateHandler().getSolrCoreState().doRecovery(cc, core.getCoreDescriptor()); - leaderElector.joinElection(this, true); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ShardTerms.java index 323342c83535..dcc687f237f6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardTerms.java @@ -124,6 +124,28 @@ private boolean skipIncreaseTermOf(String key, Set replicasNeedingRecove return replicasNeedingRecovery.contains(key); } + public ShardTerms setHighestTerms(Set highestTermKeys) { + long newMaxTerm = maxTerm + 1; + boolean keyFound = false; + HashMap newValues = new HashMap<>(values); + long nextHighestTerm = -1; + for (String key : values.keySet()) { + if (highestTermKeys.contains(key)) { + newValues.put(key, newMaxTerm); + keyFound = true; + } else { + nextHighestTerm = Math.max(nextHighestTerm, values.get(key)); + } + } + // We only want to update if increasing the maxTerm makes an impact. + // If the nextHighestTerm is already < maxTerm, then upping the maxTerm doesn't do anything. + if (nextHighestTerm == maxTerm && keyFound) { + return new ShardTerms(newValues, version); + } else { + return null; + } + } + /** * Return a new {@link ShardTerms} in which the highest terms are not zero * diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index 1b2789cb3a0e..c58bafd2602c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -28,6 +28,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; @@ -173,7 +174,7 @@ private PeerSync.PeerSyncResult syncWithReplicas( String shardId, boolean peerSyncOnlyWithActive) throws Exception { - List nodes = + List replicas = zkController .getZkStateReader() .getReplicaProps( @@ -186,13 +187,13 @@ private PeerSync.PeerSyncResult syncWithReplicas( return PeerSync.PeerSyncResult.failure(); } - if (nodes == null) { + if (replicas == null) { // I have no replicas return PeerSync.PeerSyncResult.success(); } - List syncWith = new ArrayList<>(nodes.size()); - for (ZkCoreNodeProps node : nodes) { + List syncWith = new ArrayList<>(replicas.size()); + for (Replica node : replicas) { syncWith.add(node.getCoreUrl()); } @@ -230,11 +231,11 @@ private void syncToMe( // sync everyone else // TODO: we should do this in parallel at least - List nodes = + List replicas = zkController .getZkStateReader() .getReplicaProps(collection, shardId, cd.getCloudDescriptor().getCoreNodeName()); - if (nodes == null) { + if (replicas == null) { if (log.isInfoEnabled()) { log.info("{} has no replicas", ZkCoreNodeProps.getCoreUrl(leaderProps)); } @@ -242,20 +243,36 @@ private void syncToMe( } ZkCoreNodeProps zkLeader = new ZkCoreNodeProps(leaderProps); - for (ZkCoreNodeProps node : nodes) { + ZkShardTerms shardTerms = zkController.getShardTerms(collection, shardId); + for (Replica replica : replicas) { try { + if (shardTerms.registered(replica.getName()) + && !shardTerms.canBecomeLeader(replica.getName())) { + if (log.isInfoEnabled()) { + log.info( + "{}: do NOT ask {} to sync, as it is not of the same shardTerm. Issue a recovery instead.", + ZkCoreNodeProps.getCoreUrl(leaderProps), + replica.getCoreUrl()); + } + RecoveryRequest rr = new RecoveryRequest(); + rr.leaderProps = leaderProps; + rr.baseUrl = replica.getBaseUrl(); + rr.coreName = replica.getCoreName(); + recoveryRequests.add(rr); + continue; + } if (log.isInfoEnabled()) { log.info( "{}: try and ask {} to sync", ZkCoreNodeProps.getCoreUrl(leaderProps), - node.getCoreUrl()); + replica.getCoreUrl()); } requestSync( - node.getBaseUrl(), - node.getCoreUrl(), + replica.getBaseUrl(), + replica.getCoreUrl(), zkLeader.getCoreUrl(), - node.getCoreName(), + replica.getCoreName(), nUpdates); } catch (Exception e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java index 6ec5b09afd41..ffc444dda134 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java @@ -120,6 +120,11 @@ public void ensureTermsIsHigher(String leader, Set replicasNeedingRecove mutate(terms -> terms.increaseTerms(leader, replicasNeedingRecovery)); } + public void ensureHighestTerms(Set mostUpToDateCores) { + if (mostUpToDateCores.isEmpty()) return; + mutate(terms -> terms.setHighestTerms(mostUpToDateCores)); + } + public ShardTerms getShardTerms() { return terms.get(); } diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 704806028075..a5dc2985acdc 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -494,7 +494,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // TODO: make sure that getLatestCommit only returns commit points for the main index (i.e. no // side-car indexes) - IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit(); + IndexCommit commit = solrCore.getDeletionPolicy().getAndSaveLatestCommit(); if (commit == null) { // Presumably the IndexWriter hasn't been opened yet, and hence the deletion policy hasn't // been updated with commit points diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index 839c8a9c6192..40c972aefcbb 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -39,6 +39,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.JavaBinResponseParser; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; @@ -526,6 +527,10 @@ public StdNode(ZkCoreNodeProps nodeProps) { this(nodeProps, null, null, 0); } + public StdNode(Replica replica, String collection, String shardId) { + this(new ZkCoreNodeProps(replica), collection, shardId); + } + public StdNode(ZkCoreNodeProps nodeProps, String collection, String shardId) { this(nodeProps, collection, shardId, 0); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 34532421bf56..b8febecd2856 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -546,7 +546,7 @@ protected void doDistribDeleteByQuery( Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, myShardId); // DBQ forwarded to NRT and TLOG replicas - List replicaProps = + List replicaProps = zkController .getZkStateReader() .getReplicaProps( @@ -558,8 +558,8 @@ protected void doDistribDeleteByQuery( EnumSet.of(Replica.Type.NRT, Replica.Type.TLOG)); if (replicaProps != null) { final List myReplicas = new ArrayList<>(replicaProps.size()); - for (ZkCoreNodeProps replicaProp : replicaProps) { - myReplicas.add(new SolrCmdDistributor.StdNode(replicaProp, collection, myShardId)); + for (Replica replica : replicaProps) { + myReplicas.add(new SolrCmdDistributor.StdNode(replica, collection, myShardId)); } cmdDistrib.distribDelete( cmd, myReplicas, params, false, rollupReplicationTracker, leaderReplicationTracker); @@ -620,7 +620,7 @@ private List setupRequestForDBQ() { // TODO: what if we are no longer the leader? forwardToLeader = false; - List replicaProps = + List replicas = zkController .getZkStateReader() .getReplicaProps( @@ -630,10 +630,10 @@ private List setupRequestForDBQ() { null, Replica.State.DOWN, EnumSet.of(Replica.Type.NRT, Replica.Type.TLOG)); - if (replicaProps != null) { - nodes = new ArrayList<>(replicaProps.size()); - for (ZkCoreNodeProps props : replicaProps) { - nodes.add(new SolrCmdDistributor.StdNode(props, collection, shardId)); + if (replicas != null) { + nodes = new ArrayList<>(replicas.size()); + for (Replica replica : replicas) { + nodes.add(new SolrCmdDistributor.StdNode(replica, collection, shardId)); } } } catch (InterruptedException e) { @@ -1271,14 +1271,14 @@ protected void doDistribFinish() { getLeaderExc); } - List myReplicas = + List myReplicas = zkController .getZkStateReader() .getReplicaProps(collection, cloudDesc.getShardId(), cloudDesc.getCoreNodeName()); boolean foundErrorNodeInReplicaList = false; if (myReplicas != null) { - for (ZkCoreNodeProps replicaProp : myReplicas) { - if (((Replica) replicaProp.getNodeProps()) + for (Replica replica : myReplicas) { + if (replica .getName() .equals(((Replica) stdNode.getNodeProps().getNodeProps()).getName())) { foundErrorNodeInReplicaList = true; diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardTermsTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardTermsTest.java index 586706a92fba..579285d7207d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ShardTermsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ShardTermsTest.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.apache.solr.SolrTestCase; import org.junit.Test; @@ -43,4 +44,27 @@ public void testIncreaseTerms() { assertEquals(2L, terms.getTerm("leader").longValue()); assertEquals(1L, terms.getTerm("dead-replica").longValue()); } + + @Test + public void testSetHighestTerms() { + Map map = new HashMap<>(); + map.put("leader", 0L); + ShardTerms terms = new ShardTerms(map, 0); + terms = terms.setHighestTerms(Set.of("leader")); + assertNull(terms); + + map.put("leader", 2L); + map.put("live-replica", 2L); + map.put("another-replica", 2L); + map.put("bad-replica", 2L); + map.put("dead-replica", 1L); + terms = new ShardTerms(map, 0); + + terms = terms.setHighestTerms(Set.of("live-replica", "another-replica")); + assertEquals(3L, terms.getTerm("live-replica").longValue()); + assertEquals(3L, terms.getTerm("another-replica").longValue()); + assertEquals(2L, terms.getTerm("leader").longValue()); + assertEquals(2L, terms.getTerm("bad-replica").longValue()); + assertEquals(1L, terms.getTerm("dead-replica").longValue()); + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java new file mode 100644 index 000000000000..4d78d28c7543 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import static org.hamcrest.Matchers.in; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ZkShardTermsRecoveryTest extends SolrCloudTestCase { + private static final String COLLECTION = "collection1"; + private static final int NUM_SHARDS = 2; + private static final int NUM_REPLICAS = 5; + private static int NUM_DOCS = 0; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(3).addConfig("conf", configset("cloud-minimal")).configure(); + assertEquals( + 0, + CollectionAdminRequest.createCollection(COLLECTION, "conf", NUM_SHARDS, NUM_REPLICAS) + .process(cluster.getSolrClient()) + .getStatus()); + cluster.waitForActiveCollection(COLLECTION, 10, TimeUnit.SECONDS, 2, NUM_SHARDS * NUM_REPLICAS); + + UpdateRequest up = new UpdateRequest(); + for (int i = 0; i < 200; i++) { + up.add("id", "id-" + i); + } + up.commit(cluster.getSolrClient(), COLLECTION); + NUM_DOCS += 200; + } + + @Before + public void waitForActiveState() { + cluster.waitForActiveCollection(COLLECTION, 10, TimeUnit.SECONDS, 2, NUM_SHARDS * NUM_REPLICAS); + } + + @Test + public void testShardTermsInducedReplication() throws Exception { + String shard = "shard2"; + if (random().nextBoolean()) { + // Add uncommitted documents, to test that part of the recovery + UpdateRequest up = new UpdateRequest(); + for (int i = 0; i < 1000; i++) { + up.add("id", "id2-" + i); + } + up.process(cluster.getSolrClient(), COLLECTION); + NUM_DOCS += 1000; + } + + DocCollection docCollection = cluster.getZkStateReader().getCollection(COLLECTION); + JettySolrRunner jetty = cluster.getRandomJetty(random()); + + Slice shard1 = docCollection.getSlice(shard); + Replica leader = shard1.getLeader(); + Replica replica = shard1.getReplicas(r -> !r.isLeader()).getFirst(); + List recoveryReplicas = + shard1.getReplicas(r -> r.getType().leaderEligible && (r != leader && r != replica)); + + ZkShardTerms shardTerms = + jetty.getCoreContainer().getZkController().getShardTerms(COLLECTION, shard); + // Increase the leader and another replica's shardTerms + shardTerms.ensureHighestTerms(Set.of(leader.getName(), replica.getName())); + + waitForState( + "Waiting for replicas to go into recovery", + COLLECTION, + 5, + TimeUnit.SECONDS, + state -> { + Slice shardState = state.getSlice(shard); + for (Replica r : recoveryReplicas) { + if (shardState.getReplica(r.name).getState() != Replica.State.RECOVERING) { + return false; + } + } + return true; + }); + shardTerms.refreshTerms(); + for (Replica r : recoveryReplicas) { + assertTrue(shardTerms.isRecovering(r.getName())); + } + + // Recovery should succeed relatively quickly + cluster.waitForActiveCollection(COLLECTION, 5, TimeUnit.SECONDS, 2, NUM_SHARDS * NUM_REPLICAS); + shardTerms.refreshTerms(); + long maxTerm = shardTerms.getHighestTerm(); + for (Replica r : recoveryReplicas) { + assertFalse(shardTerms.isRecovering(r.getName())); + assertEquals(maxTerm, shardTerms.getTerm(r.getName())); + } + + new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION); + waitForNumDocsInAllReplicas(NUM_DOCS, shard1.getReplicas(), "*:*"); + } + + @Test + public void testShardTermsInducedLeaderElection() throws IOException, SolrServerException { + String shard = "shard1"; + if (random().nextBoolean()) { + // Add uncommitted documents, to test that part of the recovery + UpdateRequest up = new UpdateRequest(); + for (int i = 0; i < 1000; i++) { + up.add("id", "id3-" + i); + } + up.process(cluster.getSolrClient(), COLLECTION); + NUM_DOCS += 1000; + } + + DocCollection docCollection = cluster.getZkStateReader().getCollection(COLLECTION); + JettySolrRunner jetty = cluster.getRandomJetty(random()); + + // Increase the leader and another replica's shardTerms + Slice shard1 = docCollection.getSlice(shard); + Set replicasToSetHighest = + shard1.getReplicas(r -> !r.isLeader()).subList(1, 3).stream() + .map(Replica::getName) + .collect(Collectors.toSet()); + List recoveryReplicas = + shard1.getReplicas(r -> !replicasToSetHighest.contains(r.getName())); + ZkShardTerms shardTerms = + jetty.getCoreContainer().getZkController().getShardTerms(COLLECTION, shard); + shardTerms.ensureHighestTerms(replicasToSetHighest); + waitForState( + "Wait for leadership to be given up", COLLECTION, dc -> dc.getLeader(shard) == null); + waitForState( + "Waiting for replicas to go into recovery", + COLLECTION, + 5, + TimeUnit.SECONDS, + state -> { + Slice shardState = state.getSlice(shard); + for (Replica r : recoveryReplicas) { + if (shardState.getReplica(r.name).getState() != Replica.State.RECOVERING) { + return false; + } + } + return true; + }); + waitForState("Wait for leadership to be taken", COLLECTION, dc -> dc.getLeader(shard) != null); + cluster.waitForActiveCollection(COLLECTION, 5, TimeUnit.SECONDS, 2, NUM_SHARDS * NUM_REPLICAS); + // Make sure that a leader election took place + assertThat( + cluster.getZkStateReader().getCollection(COLLECTION).getLeader(shard).getName(), + in(replicasToSetHighest)); + shardTerms.refreshTerms(); + long maxTerm = shardTerms.getHighestTerm(); + for (Replica r : recoveryReplicas) { + assertFalse(shardTerms.isRecovering(r.getName())); + assertEquals(maxTerm, shardTerms.getTerm(r.getName())); + } + + new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION); + waitForNumDocsInAllReplicas(NUM_DOCS, shard1.getReplicas(), "*:*"); + } + + private void waitForNumDocsInAllReplicas(int numDocs, Collection replicas, String query) + throws IOException, SolrServerException { + for (Replica r : replicas) { + if (!r.isActive(cluster.getSolrClient().getClusterState().getLiveNodes())) { + continue; + } + try (SolrClient replicaClient = getHttpSolrClient(r)) { + assertEquals( + "Replica " + r.getName() + " not up to date", + numDocs, + replicaClient.query(new SolrQuery(query)).getResults().getNumFound()); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java index ead1a49e6ea6..99d365f230cb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -329,6 +330,38 @@ public void testSetTermEqualsToLeader() throws InterruptedException { replicaTerms.close(); } + public void testSetHighestTerms() throws InterruptedException { + String collection = "setHighestTerms"; + ZkShardTerms leaderTerms = new ZkShardTerms(collection, "shard1", cluster.getZkClient()); + ZkShardTerms replica1Terms = new ZkShardTerms(collection, "shard1", cluster.getZkClient()); + ZkShardTerms replica2Terms = new ZkShardTerms(collection, "shard1", cluster.getZkClient()); + ZkShardTerms replica3Terms = new ZkShardTerms(collection, "shard1", cluster.getZkClient()); + leaderTerms.registerTerm("leader"); + replica1Terms.registerTerm("replica1"); + replica2Terms.registerTerm("replica2"); + replica3Terms.registerTerm("replica3"); + + leaderTerms.ensureHighestTerms(Set.of("leader", "replica1")); + waitFor(true, () -> leaderTerms.canBecomeLeader("leader")); + waitFor(true, () -> replica1Terms.canBecomeLeader("replica1")); + waitFor(false, () -> replica2Terms.canBecomeLeader("replica2")); + waitFor(false, () -> replica3Terms.canBecomeLeader("replica3")); + waitFor(false, () -> leaderTerms.skipSendingUpdatesTo("replica1")); + waitFor(true, () -> leaderTerms.skipSendingUpdatesTo("replica2")); + waitFor(true, () -> leaderTerms.skipSendingUpdatesTo("replica3")); + + leaderTerms.ensureHighestTerms(Set.of("replica2", "replica3")); + waitFor(false, () -> leaderTerms.canBecomeLeader("leader")); + waitFor(false, () -> replica1Terms.canBecomeLeader("replica1")); + waitFor(true, () -> replica2Terms.canBecomeLeader("replica2")); + waitFor(true, () -> replica3Terms.canBecomeLeader("replica3")); + + leaderTerms.close(); + replica1Terms.close(); + replica2Terms.close(); + replica3Terms.close(); + } + private void waitFor(T expected, Supplier supplier) throws InterruptedException { TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, new TimeSource.CurrentTimeSource()); while (!timeOut.hasTimedOut()) { diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 2f60c7aad9b7..306fb3c44d7e 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -1047,8 +1047,7 @@ public static String getShardLeadersElectPath(String collection, String shardId) + (shardId != null ? ("/" + shardId + "/" + ELECTION_NODE) : ""); } - public List getReplicaProps( - String collection, String shardId, String thisCoreNodeName) { + public List getReplicaProps(String collection, String shardId, String thisCoreNodeName) { // TODO: It's odd that the default is to return replicas of type TLOG and NRT only return getReplicaProps( collection, @@ -1059,7 +1058,7 @@ public List getReplicaProps( EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT)); } - public List getReplicaProps( + public List getReplicaProps( String collection, String shardId, String thisCoreNodeName, @@ -1078,39 +1077,37 @@ public List getReplicaProps( } Map slices = docCollection.getSlicesMap(); - Slice replicas = slices.get(shardId); - if (replicas == null) { + Slice shard = slices.get(shardId); + if (shard == null) { throw new ZooKeeperException( ErrorCode.BAD_REQUEST, "Could not find shardId in zk: " + shardId); } - Map shardMap = replicas.getReplicasMap(); - List nodes = new ArrayList<>(shardMap.size()); + Map shardMap = shard.getReplicasMap(); + List replicas = new ArrayList<>(shardMap.size()); for (Entry entry : shardMap.entrySet().stream() .filter((e) -> acceptReplicaType.contains(e.getValue().getType())) .collect(Collectors.toList())) { - ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue()); + Replica replica = entry.getValue(); String coreNodeName = entry.getValue().getName(); - if (clusterState.liveNodesContain(nodeProps.getNodeName()) + if (clusterState.liveNodesContain(replica.getNodeName()) && !coreNodeName.equals(thisCoreNodeName)) { - if (mustMatchStateFilter == null - || mustMatchStateFilter == Replica.State.getState(nodeProps.getState())) { - if (mustNotMatchStateFilter == null - || mustNotMatchStateFilter != Replica.State.getState(nodeProps.getState())) { - nodes.add(nodeProps); + if (mustMatchStateFilter == null || mustMatchStateFilter == replica.getState()) { + if (mustNotMatchStateFilter == null || mustNotMatchStateFilter != replica.getState()) { + replicas.add(replica); } } } } - if (nodes.size() == 0) { + if (replicas.size() == 0) { // no replicas return null; } - return nodes; + return replicas; } public SolrZkClient getZkClient() { diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java index 34da8d239a02..788cb3866f17 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java @@ -47,6 +47,10 @@ public String getCoreName() { return nodeProps.getStr(ReplicaStateProps.CORE_NAME); } + public String getCoreNodeName() { + return nodeProps.getStr(ReplicaStateProps.CORE_NODE_NAME); + } + private static String getBaseUrl(ZkNodeProps nodeProps) { // if storing baseUrl in ZK is enabled, and it's stored, just use what's stored, i.e. no // self-healing here From 7522caa0affd61c306d944991c7cce131a058f38 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 21 Jan 2026 12:11:49 -0800 Subject: [PATCH 33/67] Add changelog entry --- .../solr-18080-shard-term-induce-leader-election.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml diff --git a/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml b/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml new file mode 100644 index 000000000000..49c3f8f4db3e --- /dev/null +++ b/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: ShardTerms can now induce a leader election if needed +type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman +links: + - name: SOLR-18080 + url: https://issues.apache.org/jira/browse/SOLR-18080 From 776c28b7d1580d3b831a57be592d3bb9ef3a7694 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 21 Jan 2026 12:44:52 -0800 Subject: [PATCH 34/67] Fix precommit issues --- .../test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java index 4d78d28c7543..4e337cdfb572 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java @@ -86,7 +86,8 @@ public void testShardTermsInducedReplication() throws Exception { Replica leader = shard1.getLeader(); Replica replica = shard1.getReplicas(r -> !r.isLeader()).getFirst(); List recoveryReplicas = - shard1.getReplicas(r -> r.getType().leaderEligible && (r != leader && r != replica)); + shard1.getReplicas( + r -> r.getType().leaderEligible && !(r.equals(leader) || r.equals(replica))); ZkShardTerms shardTerms = jetty.getCoreContainer().getZkController().getShardTerms(COLLECTION, shard); From f3e6ff80ab93979aac78974fa38144bb4cab43c8 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 22 Jan 2026 14:04:07 -0800 Subject: [PATCH 35/67] Fixes for passing callingLockIds around --- .../api/collections/AdminCmdContext.java | 19 ++++++++++++++++--- .../handler/admin/CollectionsHandler.java | 3 +-- .../solr/handler/admin/RebalanceLeaders.java | 6 +++--- .../solr/handler/admin/api/AdminAPIBase.java | 4 +++- .../org/apache/solr/servlet/HttpSolrCall.java | 5 +++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 857f4d91dbad..24f85ef18c6e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -20,6 +20,9 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SolrQueryRequest; + +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; public class AdminCmdContext { private final CollectionParams.CollectionAction action; @@ -38,6 +41,12 @@ public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) this.asyncId = asyncId; } + public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { + this.action = action; + this.asyncId = asyncId; + this.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); + } + public CollectionParams.CollectionAction getAction() { return action; } @@ -61,11 +70,15 @@ public void setCallingLockIds(String callingLockIds) { } private void regenerateSubRequestCallingLockIds() { - subRequestCallingLockIds = callingLockIds; if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { - subRequestCallingLockIds += ","; + subRequestCallingLockIds += "," + lockId; + } else if (StrUtils.isNotBlank(callingLockIds)) { + subRequestCallingLockIds = callingLockIds; + } else if (StrUtils.isNotBlank(lockId)) { + subRequestCallingLockIds = lockId; + } else { + subRequestCallingLockIds = null; } - subRequestCallingLockIds += lockId; } public String getCallingLockIds() { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index bd4c9d7bcee6..6e8c2df41a7f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -323,8 +323,7 @@ void invokeAction( } AdminCmdContext adminCmdContext = - new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); - adminCmdContext.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); + new AdminCmdContext(operation.action, req.getParams().get(ASYNC), req); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 26afe771d5fc..5923fc810c13 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -23,6 +23,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_AT_ONCE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; import java.lang.invoke.MethodHandles; @@ -458,9 +459,8 @@ private void rejoinElectionQueue( String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); asyncRequests.add(asyncId); - collectionsHandler.submitCollectionApiCommand( - new AdminCmdContext(REBALANCELEADERS, asyncId), - new ZkNodeProps(propMap)); // ignore response; we construct our own + // ignore response; we construct our own + collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 3d627c4cfed4..9783c28a1ab0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.admin.api; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import java.util.Map; @@ -120,6 +121,7 @@ public void disableResponseCaching() { protected SolrResponse submitRemoteMessageAndHandleException( SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { + adminCmdContext.setCallingLockIds((String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer.getZkController(), @@ -138,7 +140,7 @@ protected SolrResponse submitRemoteMessageAndHandleException( ZkNodeProps remoteMessage) throws Exception { return submitRemoteMessageAndHandleException( - response, new AdminCmdContext(action, null), remoteMessage); + response, new AdminCmdContext(action), remoteMessage); } protected SolrResponse submitRemoteMessageAndHandleAsync( diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index db998ff9b8c8..9736ad5efbcf 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -19,6 +19,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH; @@ -766,6 +767,10 @@ protected QueryResponseWriter getResponseWriter() { protected void handleAdmin(SolrQueryResponse solrResp) { SolrCore.preDecorateResponse(solrReq, solrResp); + String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); + if (callingLockIds != null && !callingLockIds.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + } handler.handleRequest(solrReq, solrResp); SolrCore.postDecorateResponse(handler, solrReq, solrResp); } From 2513cd4bfc3cf4e5b23c35c43ac997796483da87 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 22 Jan 2026 14:04:07 -0800 Subject: [PATCH 36/67] Fixes for passing callingLockIds around --- .../api/collections/AdminCmdContext.java | 19 ++++++++++++++++--- .../handler/admin/CollectionsHandler.java | 3 +-- .../solr/handler/admin/RebalanceLeaders.java | 6 +++--- .../solr/handler/admin/api/AdminAPIBase.java | 4 +++- .../org/apache/solr/servlet/HttpSolrCall.java | 5 +++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 857f4d91dbad..24f85ef18c6e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -20,6 +20,9 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SolrQueryRequest; + +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; public class AdminCmdContext { private final CollectionParams.CollectionAction action; @@ -38,6 +41,12 @@ public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) this.asyncId = asyncId; } + public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { + this.action = action; + this.asyncId = asyncId; + this.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); + } + public CollectionParams.CollectionAction getAction() { return action; } @@ -61,11 +70,15 @@ public void setCallingLockIds(String callingLockIds) { } private void regenerateSubRequestCallingLockIds() { - subRequestCallingLockIds = callingLockIds; if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { - subRequestCallingLockIds += ","; + subRequestCallingLockIds += "," + lockId; + } else if (StrUtils.isNotBlank(callingLockIds)) { + subRequestCallingLockIds = callingLockIds; + } else if (StrUtils.isNotBlank(lockId)) { + subRequestCallingLockIds = lockId; + } else { + subRequestCallingLockIds = null; } - subRequestCallingLockIds += lockId; } public String getCallingLockIds() { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 05361725ad89..33190986f4d1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -322,8 +322,7 @@ void invokeAction( } AdminCmdContext adminCmdContext = - new AdminCmdContext(operation.action, req.getParams().get(ASYNC)); - adminCmdContext.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); + new AdminCmdContext(operation.action, req.getParams().get(ASYNC), req); ZkNodeProps zkProps = new ZkNodeProps(props); final SolrResponse overseerResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 26afe771d5fc..5923fc810c13 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -23,6 +23,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_AT_ONCE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; import java.lang.invoke.MethodHandles; @@ -458,9 +459,8 @@ private void rejoinElectionQueue( String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime()); asyncRequests.add(asyncId); - collectionsHandler.submitCollectionApiCommand( - new AdminCmdContext(REBALANCELEADERS, asyncId), - new ZkNodeProps(propMap)); // ignore response; we construct our own + // ignore response; we construct our own + collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 3d627c4cfed4..9783c28a1ab0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.admin.api; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import java.util.Map; @@ -120,6 +121,7 @@ public void disableResponseCaching() { protected SolrResponse submitRemoteMessageAndHandleException( SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { + adminCmdContext.setCallingLockIds((String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer.getZkController(), @@ -138,7 +140,7 @@ protected SolrResponse submitRemoteMessageAndHandleException( ZkNodeProps remoteMessage) throws Exception { return submitRemoteMessageAndHandleException( - response, new AdminCmdContext(action, null), remoteMessage); + response, new AdminCmdContext(action), remoteMessage); } protected SolrResponse submitRemoteMessageAndHandleAsync( diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index db998ff9b8c8..9736ad5efbcf 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -19,6 +19,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH; @@ -766,6 +767,10 @@ protected QueryResponseWriter getResponseWriter() { protected void handleAdmin(SolrQueryResponse solrResp) { SolrCore.preDecorateResponse(solrReq, solrResp); + String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); + if (callingLockIds != null && !callingLockIds.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + } handler.handleRequest(solrReq, solrResp); SolrCore.postDecorateResponse(handler, solrReq, solrResp); } From d2dad8e55e234e2d162c7fb775d06fc8a003b4a8 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 23 Jan 2026 14:48:55 -0800 Subject: [PATCH 37/67] Fixes for syncStrategy with empty indexes --- .../cloud/ShardLeaderElectionContext.java | 35 ++----------------- .../cloud/ShardLeaderElectionContextBase.java | 13 +++++-- .../org/apache/solr/cloud/SyncStrategy.java | 29 ++++++++++++--- .../handler/admin/RequestSyncShardOp.java | 4 +-- 4 files changed, 41 insertions(+), 40 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 5b956eac5cfe..fc6d3db2f194 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -195,7 +195,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup // first cancel any current recovery core.getUpdateHandler().getSolrCoreState().cancelRecovery(); - if (weAreReplacement) { + if (weAreReplacement && !core.readOnly) { // wait a moment for any floating updates to finish try { Thread.sleep(2500); @@ -205,41 +205,12 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup } } - PeerSync.PeerSyncResult result = null; boolean success = false; try { - result = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement); + PeerSync.PeerSyncResult result = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement, true); success = result.isSuccess(); } catch (Exception e) { log.error("Exception while trying to sync", e); - result = PeerSync.PeerSyncResult.failure(); - } - - UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); - - if (!success) { - boolean hasRecentUpdates = false; - if (ulog != null) { - // TODO: we could optimize this if necessary - try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) { - hasRecentUpdates = !recentUpdates.getVersions(1).isEmpty(); - } - } - - if (!hasRecentUpdates) { - // we failed sync, but we have no versions - we can't sync in that case - // - we were active - // before, so become leader anyway if no one else has any versions either - if (result.getOtherHasVersions().orElse(false)) { - log.info( - "We failed sync, but we have no versions - we can't sync in that case. But others have some versions, so we should not become leader"); - success = false; - } else { - log.info( - "We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway"); - success = true; - } - } } // solrcloud_debug @@ -250,7 +221,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup try { if (log.isDebugEnabled()) { log.debug( - "{} synched {}", + "{} synced {}", core.getCoreContainer().getZkController().getNodeName(), searcher.count(new MatchAllDocsQuery())); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java index b91733845954..0925461cce07 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java @@ -19,6 +19,8 @@ import java.lang.invoke.MethodHandles; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.curator.framework.api.transaction.CuratorTransactionResult; import org.apache.curator.framework.api.transaction.OperationType; import org.apache.solr.cloud.overseer.OverseerAction; @@ -182,8 +184,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup .getClusterState() .getCollection(collection) .getSlice(shardId) - .getReplicas() - .size() + .getNumLeaderReplicas() < 2) { Replica leader = zkStateReader.getLeader(collection, shardId); if (leader != null @@ -239,6 +240,14 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup prs) .persist(coll.getZNode(), zkStateReader.getZkClient()); } + try { + zkStateReader.waitForState(collection, 10, TimeUnit.SECONDS, state -> { + Replica leader = state.getLeader(shardId); + return leader != null && id.equals(leader.getName()); + }); + } catch (TimeoutException e) { + throw new RuntimeException(e); + } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index dae1a5f7a3c8..c8050658c77e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -41,6 +41,7 @@ import org.apache.solr.handler.component.ShardRequest; import org.apache.solr.handler.component.ShardResponse; import org.apache.solr.update.PeerSync; +import org.apache.solr.update.UpdateLog; import org.apache.solr.update.UpdateShardHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,14 +76,15 @@ public SyncStrategy(CoreContainer cc) { public PeerSync.PeerSyncResult sync( ZkController zkController, SolrCore core, ZkNodeProps leaderProps) { - return sync(zkController, core, leaderProps, false); + return sync(zkController, core, leaderProps, false, false); } public PeerSync.PeerSyncResult sync( ZkController zkController, SolrCore core, ZkNodeProps leaderProps, - boolean peerSyncOnlyWithActive) { + boolean peerSyncOnlyWithActive, + boolean ignoreNoVersionsFailure) { if (SKIP_AUTO_RECOVERY) { return PeerSync.PeerSyncResult.success(); } @@ -103,14 +105,15 @@ public PeerSync.PeerSyncResult sync( return PeerSync.PeerSyncResult.failure(); } - return syncReplicas(zkController, core, leaderProps, peerSyncOnlyWithActive); + return syncReplicas(zkController, core, leaderProps, peerSyncOnlyWithActive, ignoreNoVersionsFailure); } private PeerSync.PeerSyncResult syncReplicas( ZkController zkController, SolrCore core, ZkNodeProps leaderProps, - boolean peerSyncOnlyWithActive) { + boolean peerSyncOnlyWithActive, + boolean ignoreNoVersionsFailure) { if (isClosed) { log.info("We have been closed, won't sync with replicas"); return PeerSync.PeerSyncResult.failure(); @@ -128,6 +131,24 @@ private PeerSync.PeerSyncResult syncReplicas( result = syncWithReplicas( zkController, core, leaderProps, collection, shardId, peerSyncOnlyWithActive); + + if (!result.isSuccess() && ignoreNoVersionsFailure) { + UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); + boolean hasRecentUpdates = false; + if (ulog != null) { + // TODO: we could optimize this if necessary + try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) { + hasRecentUpdates = !recentUpdates.getVersions(1).isEmpty(); + } + } + // we failed sync, but we have no versions - we can't sync in that case + // - we were active before, so continue if no one else has any versions either + if (!hasRecentUpdates && !result.getOtherHasVersions().orElse(false)) { + log.info( + "We failed sync, but we have no versions - we can't sync in that case - so continue"); + result = PeerSync.PeerSyncResult.success(); + } + } success = result.isSuccess(); } catch (Exception e) { log.error("Sync Failed", e); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RequestSyncShardOp.java b/solr/core/src/java/org/apache/solr/handler/admin/RequestSyncShardOp.java index d781834e3e79..815a3cc76246 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RequestSyncShardOp.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RequestSyncShardOp.java @@ -65,7 +65,7 @@ public void execute(CallInfo it) throws Exception { zkController.getZkStateReader().getBaseUrlForNodeName(zkController.getNodeName())); boolean success = - syncStrategy.sync(zkController, core, new ZkNodeProps(props), true).isSuccess(); + syncStrategy.sync(zkController, core, new ZkNodeProps(props), true, false).isSuccess(); // solrcloud_debug if (log.isDebugEnabled()) { try { @@ -74,7 +74,7 @@ public void execute(CallInfo it) throws Exception { try { if (log.isDebugEnabled()) { log.debug( - "{} synched {}", + "{} synced {}", core.getCoreContainer().getZkController().getNodeName(), searcher.count(new MatchAllDocsQuery())); } From 198f01bd97027dda82ae9d916ee50536bf5d15c5 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 23 Jan 2026 17:05:48 -0800 Subject: [PATCH 38/67] Improve tests for recovery --- .../LocalFSCloudIncrementalBackupTest.java | 4 + .../solr/gcs/GCSIncrementalBackupTest.java | 4 + .../solr/s3/S3IncrementalBackupTest.java | 130 +++--------------- .../AbstractIncrementalBackupTest.java | 128 ++++++++++++++++- 4 files changed, 154 insertions(+), 112 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java index eabd81014806..57a82b57c36c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java @@ -58,6 +58,10 @@ public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackup + " \n" + " \n" + " \n" + + " \n" + + " localfs\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " localfs\n" + " \n" diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java index 846563b929f5..1ba2f114b3f7 100644 --- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java @@ -55,6 +55,10 @@ public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest { + " \n" + " \n" + " \n" + + " \n" + + " localfs\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " localfs\n" + " \n" diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index a50e1920fa0d..5393a52bf2ae 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -19,16 +19,28 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.PropertiesUtil; +import org.apache.solr.core.TrackingBackupRepository; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -39,6 +51,9 @@ // Backups do checksum validation against a footer value not present in 'SimpleText' @LuceneTestCase.SuppressCodecs({"SimpleText"}) @ThreadLeakLingering(linger = 10) +@LogLevel( + value = + "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.overseer=DEBUG") public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -72,6 +87,10 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { + " \n" + " \n" + " \n" + + " \n" + + " s3\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " s3\n" + " \n" @@ -80,22 +99,6 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { + " REGION\n" + " ENDPOINT\n" + " \n" - + " \n" - + " s3BadNode\n" - + " \n" - + " \n" - + " BAD_BUCKET_ONE\n" - + " REGION\n" - + " ENDPOINT\n" - + " \n" - + " \n" - + " s3BadNodes\n" - + " \n" - + " \n" - + " BAD_BUCKET_ALL_BUT_ONE\n" - + " REGION\n" - + " ENDPOINT\n" - + " \n" + " \n" + " \n" + "\n"; @@ -151,99 +154,4 @@ public String getCollectionNamePrefix() { public String getBackupLocation() { return "/"; } - - @Test - public void testRestoreToOriginalSucceedsOnASingleError() throws Exception { - JettySolrRunner badNodeJetty = - cluster.startJettySolrRunner( - SOLR_XML - // The first solr node will not have a bad bucket - .replace("BAD_BUCKET_ALL_BUT_ONE", BUCKET_NAME) - .replace("BAD_BUCKET_ONE", "non-existent") - .replace("BUCKET", BUCKET_NAME) - .replace("REGION", Region.US_EAST_1.id()) - .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); - - try { - setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); - final String backupCollectionName = getCollectionName(); - final String backupName = BACKUPNAME_PREFIX + testSuffix; - - // Bootstrap the backup collection with seed docs - CollectionAdminRequest.createCollection( - backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES + 1) - .process(cluster.getSolrClient()); - final int firstBatchNumDocs = indexDocs(backupCollectionName, true); - - // Backup and immediately add more docs to the collection - try (BackupRepository repository = - cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME)) { - final String backupLocation = repository.getBackupLocation(getBackupLocation()); - final RequestStatusState result = - CollectionAdminRequest.backupCollection(backupCollectionName, backupName) - .setBackupConfigset(false) - .setLocation(backupLocation) - .setRepositoryName(BACKUP_REPO_NAME) - .processAndWait(cluster.getSolrClient(), 20); - assertEquals(RequestStatusState.COMPLETED, result); - } - int secondBatchNumDocs = indexDocs(backupCollectionName, true); - int maxDocs = secondBatchNumDocs + firstBatchNumDocs; - assertEquals(maxDocs, getNumDocsInCollection(backupCollectionName)); - - /* - Restore original docs and validate that doc count is correct - */ - // Test a single bad node - try (BackupRepository repository = - cluster - .getJettySolrRunner(0) - .getCoreContainer() - .newBackupRepository(BACKUP_REPO_NAME); - SolrClient goodNodeClient = cluster.getJettySolrRunner(0).newClient()) { - final String backupLocation = repository.getBackupLocation(getBackupLocation()); - final RequestStatusState result = - CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) - .setLocation(backupLocation) - .setRepositoryName(BACKUP_REPO_NAME + "BadNode") - .processAndWait(goodNodeClient, 30); - assertEquals(RequestStatusState.COMPLETED, result); - waitForState( - "The failed core-install should recover and become healthy", - backupCollectionName, - 30, - TimeUnit.SECONDS, - SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * (NUM_NODES + 1))); - } - assertEquals(firstBatchNumDocs, getNumDocsInCollection(backupCollectionName)); - secondBatchNumDocs = indexDocs(backupCollectionName, true); - maxDocs = secondBatchNumDocs + firstBatchNumDocs; - assertEquals(maxDocs, getNumDocsInCollection(backupCollectionName)); - - // Test a single good node - try (BackupRepository repository = - cluster - .getJettySolrRunner(0) - .getCoreContainer() - .newBackupRepository(BACKUP_REPO_NAME); - SolrClient goodNodeClient = badNodeJetty.newClient()) { - final String backupLocation = repository.getBackupLocation(getBackupLocation()); - final RequestStatusState result = - CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) - .setLocation(backupLocation) - .setRepositoryName(BACKUP_REPO_NAME + "BadNodes") - .processAndWait(goodNodeClient, 30); - assertEquals(RequestStatusState.COMPLETED, result); - waitForState( - "The failed core-install should recover and become healthy", - backupCollectionName, - 30, - TimeUnit.SECONDS, - SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * (NUM_NODES + 1))); - } - assertEquals(firstBatchNumDocs, getNumDocsInCollection(backupCollectionName)); - } finally { - badNodeJetty.stop(); - } - } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index cdc8359be1a6..5829779e40f4 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -48,7 +48,9 @@ import org.apache.lucene.store.IndexInput; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CollectionsApi; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -60,6 +62,7 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.core.TrackingBackupRepository; @@ -71,6 +74,7 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -88,12 +92,13 @@ public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static long docsSeed; // see indexDocs() - protected static final int NUM_NODES = 2; + protected static final int NUM_NODES = 3; protected static final int NUM_SHARDS = 2; // granted we sometimes shard split to get more protected static final int LARGE_NUM_SHARDS = 11; // Periodically chosen via randomization protected static final int REPL_FACTOR = 2; protected static final String BACKUPNAME_PREFIX = "mytestbackup"; protected static final String BACKUP_REPO_NAME = "trackingBackupRepository"; + protected static final String ERROR_BACKUP_REPO_NAME = "errorBackupRepository"; protected String testSuffix = "test1"; @@ -490,6 +495,116 @@ public void testBackupProperties() throws IOException { } } + static Set portsToFailOn = new HashSet<>(); + @Test + public void testRestoreToOriginalSucceedsWithErrors() throws Exception { + setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); + final String backupCollectionName = getCollectionName(); + final String backupName = BACKUPNAME_PREFIX + testSuffix; + + // Bootstrap the backup collection with seed docs + CollectionAdminRequest.createCollection( + backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES) + .process(cluster.getSolrClient()); + int backupDocs = indexDocs(backupCollectionName, true); + + // Backup and immediately add more docs to the collection + try (BackupRepository repository = + cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(ERROR_BACKUP_REPO_NAME)) { + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + final RequestStatusState result = + CollectionAdminRequest.backupCollection(backupCollectionName, backupName) + .setBackupConfigset(false) + .setLocation(backupLocation) + .setRepositoryName(ERROR_BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 20); + assertEquals(RequestStatusState.COMPLETED, result); + } + assertEquals(backupDocs, getNumDocsInCollection(backupCollectionName)); + clearDocs(backupCollectionName); + assertEquals(0, getNumDocsInCollection(backupCollectionName)); + + /* + Restore original docs and validate that doc count is correct + */ + // Test a single bad node + try (BackupRepository repository = + cluster + .getJettySolrRunner(0) + .getCoreContainer() + .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { + // Only the first jetty will fail + portsToFailOn = Set.of(cluster.getJettySolrRunner(0).getLocalPort()); + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + final RequestStatusState result = + CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) + .setLocation(backupLocation) + .setRepositoryName(ERROR_BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 30); + assertEquals(RequestStatusState.COMPLETED, result); + waitForState( + "The failed core-install should recover and become healthy", + backupCollectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * NUM_NODES)); + } + assertEquals(backupDocs, getNumDocsInCollection(backupCollectionName)); + clearDocs(backupCollectionName); + assertEquals(0, getNumDocsInCollection(backupCollectionName)); + + // Test a single good node + try (BackupRepository repository = + cluster + .getJettySolrRunner(0) + .getCoreContainer() + .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { + final String backupLocation = repository.getBackupLocation(getBackupLocation()); + // All but the first jetty will fail + portsToFailOn = cluster.getJettySolrRunners().subList(1, NUM_NODES).stream().map(JettySolrRunner::getLocalPort).collect(Collectors.toSet()); + final RequestStatusState result = + CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) + .setLocation(backupLocation) + .setRepositoryName(ERROR_BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 30); + assertEquals(RequestStatusState.COMPLETED, result); + waitForState( + "The failed core-install should recover and become healthy", + backupCollectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(NUM_SHARDS, NUM_SHARDS * NUM_NODES)); + } + assertEquals(backupDocs, getNumDocsInCollection(backupCollectionName)); + } + + public static class ErrorThrowingTrackingBackupRepository extends TrackingBackupRepository { + + private int port; + @Override + public void init(NamedList args) { + super.init(args); + port = Integer.parseInt((String) args.get("hostPort")); + } + + @Override + public void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException { + if (portsToFailOn.contains(port)) { + throw new UnsupportedOperationException(); + } + super.copyFileTo(sourceRepo, fileName, dest); + } + + @Override + public void copyIndexFileTo( + URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { + if (portsToFailOn.contains(port)) { + throw new UnsupportedOperationException(); + } + super.copyIndexFileTo(sourceRepo, sourceFileName, dest, destFileName); + } + } + protected void corruptIndexFiles() throws IOException { List slices = new ArrayList<>(getCollectionState(getCollectionName()).getSlices()); Replica leader = slices.get(random().nextInt(slices.size())).getLeader(); @@ -566,6 +681,17 @@ private void simpleRestoreAndCheckDocCount( CollectionAdminRequest.deleteCollection(restoreCollectionName).process(solrClient); } + protected void clearDocs(String collectionName) throws Exception { + CollectionAdminRequest.deleteCollection( + collectionName) + .process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection( + collectionName, "conf1", NUM_SHARDS, NUM_NODES) + .process(cluster.getSolrClient()); + + log.info("Cleared all docs in collection: {}", collectionName); + } + private void indexDocs(String collectionName, int numDocs, boolean useUUID) throws Exception { Random random = new Random(docsSeed); From a667bb48e06cfb44ea024040637d866b268347a3 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 23 Jan 2026 17:07:16 -0800 Subject: [PATCH 39/67] Make read only check better --- .../java/org/apache/solr/cloud/RecoveryStrategy.java | 11 ++++------- .../org/apache/solr/handler/RequestHandlerUtils.java | 1 + .../org/apache/solr/update/CommitUpdateCommand.java | 5 ++++- .../processor/DistributedZkUpdateProcessor.java | 7 ++++++- .../org/apache/solr/common/params/UpdateParams.java | 3 +++ 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index b517d63603b9..9f7cb74e1fb5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -231,14 +231,9 @@ private void replicate(String nodeName, SolrCore core, ZkNodeProps leaderprops) log.info("Attempting to replicate from core [{}] on node [{}].", leaderCore, leaderBaseUrl); - // send commit if replica could be a leader and this collection is not in a read-only state + // send commit if replica could be a leader if (replicaType.leaderEligible) { - if (!zkController - .getClusterState() - .getCollection(coreDescriptor.getCollectionName()) - .isReadOnly()) { - commitOnLeader(leaderBaseUrl, leaderCore); - } + commitOnLeader(leaderBaseUrl, leaderCore); } // use rep handler directly, so we can do this sync rather than async @@ -308,6 +303,8 @@ private void commitOnLeader(String leaderBaseUrl, String coreName) // ureq.getParams().set(UpdateParams.OPEN_SEARCHER, onlyLeaderIndexes); // Why do we need to open searcher if "onlyLeaderIndexes"? ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false); + // If the leader is readOnly, do not fail since the core is already committed. + ureq.getParams().set(UpdateParams.FAIL_ON_READ_ONLY, false); ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(client); } } diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java index 7b5d791ae423..7a04edad33ed 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java @@ -99,6 +99,7 @@ public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) { cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); cmd.prepareCommit = params.getBool(UpdateParams.PREPARE_COMMIT, cmd.prepareCommit); + cmd.failOnReadOnly = params.getBool(UpdateParams.FAIL_ON_READ_ONLY, cmd.failOnReadOnly); } /** diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java index 6a749af2ac91..470740583eee 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java @@ -39,6 +39,7 @@ public class CommitUpdateCommand extends UpdateCommand { public boolean expungeDeletes = false; public boolean softCommit = false; public boolean prepareCommit = false; + public boolean failOnReadOnly = true; // fail the commit if the core or collection is readOnly /** * User provided commit data. Can be let to null if there is none. It is possible to commit this @@ -97,7 +98,9 @@ public String toString() { .append(",softCommit=") .append(softCommit) .append(",prepareCommit=") - .append(prepareCommit); + .append(prepareCommit) + .append(",failOnReadOnly=") + .append(failOnReadOnly); if (commitData != null) { sb.append(",commitData=").append(commitData); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index b8febecd2856..9051917c58da 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -161,7 +161,12 @@ public void processCommit(CommitUpdateCommand cmd) throws IOException { assert TestInjection.injectFailUpdateRequests(); if (isReadOnly()) { - throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only."); + if (cmd.failOnReadOnly) { + throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only."); + } else { + // Committing on a readOnly core/collection is a no-op, since the core was committed when becoming read-only and hasn't had any updates since. + return; + } } List nodes = null; diff --git a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java index f14252fb9708..df6b11f1f3a3 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java @@ -46,6 +46,9 @@ public interface UpdateParams { /** expert: calls IndexWriter.prepareCommit */ public static String PREPARE_COMMIT = "prepareCommit"; + /** Fail a commit when the core or collection is in read-only mode */ + public static String FAIL_ON_READ_ONLY = "failOnReadOnly"; + /** Rollback update commands */ public static String ROLLBACK = "rollback"; From ed8c839767da66adefc2ff4e820bea65856a3beb Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 23 Jan 2026 17:08:55 -0800 Subject: [PATCH 40/67] Improve logging for results --- .../solr/cloud/api/collections/CollectionHandlingUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 3e541a24696a..d1272d72e9cd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -820,7 +820,7 @@ private void waitForAsyncCallsToComplete(NamedList results) { status = (String) reqResult._get("status/state"); } if ("failed".equalsIgnoreCase(status)) { - log.error("Error from shard {}: {}", node, reqResult); + log.error("Error from shard {}/{}: {}", node, coreName, reqResult); addFailure(results, node, coreName, reqResult); } else { addSuccess(results, node, coreName, reqResult); From 3b898f93dd04404f82e24502d44c1ca80a84ee7c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 23 Jan 2026 17:10:43 -0800 Subject: [PATCH 41/67] Various fixes --- .../apache/solr/cloud/api/collections/CollApiCmds.java | 2 +- .../cloud/api/collections/InstallShardDataCmd.java | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index 9dffda37472a..a65d369a98e2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -279,7 +279,7 @@ public void call( message, params, results, - Replica.State.ACTIVE, + null, Collections.emptySet(), ccc); } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 149f6b88cef5..f7f09bbb64bf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -20,6 +20,7 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import java.lang.invoke.MethodHandles; @@ -158,7 +159,7 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList ccc.getZkStateReader() .waitForState( typedMessage.collection, - 10, + 30, TimeUnit.SECONDS, (liveNodes, collectionState) -> { collectionState.getSlice(typedMessage.shard).getReplicas().stream() @@ -287,11 +288,9 @@ public static void handleCoreRestoreResponses( */ /** A value-type representing the message received by {@link InstallShardDataCmd} */ + @JsonIgnoreProperties(ignoreUnknown = true) public static class RemoteMessage implements JacksonReflectMapWriter { - @JsonProperty(QUEUE_OPERATION) - public String operation = CollectionParams.CollectionAction.INSTALLSHARDDATA.toLower(); - @JsonProperty public String collection; @JsonProperty public String shard; @@ -304,9 +303,6 @@ public static class RemoteMessage implements JacksonReflectMapWriter { @JsonProperty public String shardBackupId; - @JsonProperty(ASYNC) - public String asyncId; - public void validate() { if (StrUtils.isBlank(collection)) { throw new SolrException( From bb371206503797de06b5c566c5fb807e523703e0 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 26 Jan 2026 18:04:37 -0800 Subject: [PATCH 42/67] Let some parts of solr fetch an index writer for a read-only core --- .../src/java/org/apache/solr/core/SolrCore.java | 2 +- .../java/org/apache/solr/handler/IndexFetcher.java | 4 ++-- .../apache/solr/handler/ReplicationHandler.java | 2 +- .../apache/solr/update/DefaultSolrCoreState.java | 11 ++++++----- .../java/org/apache/solr/update/SolrCoreState.java | 14 +++++++++++++- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 24c7dc83b0c9..5e246e69ff15 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -2476,7 +2476,7 @@ public RefCounted openNewSearcher( true, directoryFactory); } else { - RefCounted writer = getSolrCoreState().getIndexWriter(this); + RefCounted writer = getSolrCoreState().getIndexWriter(this, true); DirectoryReader newReader = null; try { newReader = indexReaderFactory.newReader(writer.get(), this); diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index a5dc2985acdc..a2fd75c6c732 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -530,7 +530,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // we just clear ours and commit log.info("New index in Leader. Deleting mine..."); RefCounted iw = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, true); try { iw.get().deleteAll(); } finally { @@ -624,7 +624,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // are successfully deleted solrCore.getUpdateHandler().newIndexWriter(true); RefCounted writer = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, true); try { IndexWriter indexWriter = writer.get(); int c = 0; diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index cdc5de9313bb..7f50064ffe1d 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1378,7 +1378,7 @@ public void inform(SolrCore core) { // ensure the writer is initialized so that we have a list of commit points RefCounted iw = - core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); + core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, true); iw.decref(); } catch (IOException e) { diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 2a848f7c0913..4feb43abd9d1 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -106,11 +106,12 @@ private void closeIndexWriter(IndexWriterCloser closer) { } @Override - public RefCounted getIndexWriter(SolrCore core) throws IOException { - // if (core != null && (!core.indexEnabled || core.readOnly)) { - // throw new SolrException( - // SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled"); - // } + public RefCounted getIndexWriter(SolrCore core, boolean readOnlyCompatible) + throws IOException { + if (core != null && (!core.indexEnabled || (!readOnlyCompatible && core.readOnly))) { + throw new SolrException( + SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled"); + } boolean succeeded = false; lock(iwLock.readLock()); try { diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 4d6f02d229fb..5016f9fdbb6f 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -178,7 +178,19 @@ public void deregisterInFlightUpdate() { * * @throws IOException If there is a low-level I/O error. */ - public abstract RefCounted getIndexWriter(SolrCore core) throws IOException; + public RefCounted getIndexWriter(SolrCore core) throws IOException { + return getIndexWriter(core, false); + } + + /** + * Get the current IndexWriter. If a new IndexWriter must be created, use the settings from the + * given {@link SolrCore}. Be very careful using the {@code readOnlyCompatible} flag, by default + * it should be false if the returned indexWriter will be used for writing. + * + * @throws IOException If there is a low-level I/O error. + */ + public abstract RefCounted getIndexWriter(SolrCore core, boolean readOnlyCompatible) + throws IOException; /** * Rollback the current IndexWriter. When creating the new IndexWriter use the settings from the From 3135c5bf116387bb25d9e145bca65eba416fc100 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 26 Jan 2026 18:05:06 -0800 Subject: [PATCH 43/67] Tidy --- .../solr/cloud/RecoveringCoreTermWatcher.java | 1 - .../cloud/ShardLeaderElectionContext.java | 3 +- .../cloud/ShardLeaderElectionContextBase.java | 12 +++++--- .../org/apache/solr/cloud/SyncStrategy.java | 3 +- .../api/collections/AdminCmdContext.java | 7 +++-- .../cloud/api/collections/CollApiCmds.java | 9 +----- .../api/collections/InstallShardDataCmd.java | 4 --- .../solr/handler/admin/RebalanceLeaders.java | 4 +-- .../solr/handler/admin/api/AdminAPIBase.java | 3 +- .../handler/admin/api/GetSegmentData.java | 4 +-- .../DistributedZkUpdateProcessor.java | 3 +- .../LocalFSCloudIncrementalBackupTest.java | 4 ++- .../solr/gcs/GCSIncrementalBackupTest.java | 4 ++- .../s3-repository/src/test-files/log4j2.xml | 2 +- .../solr/s3/S3IncrementalBackupTest.java | 2 +- .../AbstractIncrementalBackupTest.java | 28 ++++++++++--------- 16 files changed, 48 insertions(+), 45 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index ac924daffde8..515110b95866 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -18,7 +18,6 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.solr.common.cloud.Replica; import org.apache.solr.core.CoreContainer; diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index fc6d3db2f194..3d7b9ce756f5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -207,7 +207,8 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup boolean success = false; try { - PeerSync.PeerSyncResult result = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement, true); + PeerSync.PeerSyncResult result = + syncStrategy.sync(zkController, core, leaderProps, weAreReplacement, true); success = result.isSuccess(); } catch (Exception e) { log.error("Exception while trying to sync", e); diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java index 0925461cce07..d50dd58d10cd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java @@ -241,10 +241,14 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup .persist(coll.getZNode(), zkStateReader.getZkClient()); } try { - zkStateReader.waitForState(collection, 10, TimeUnit.SECONDS, state -> { - Replica leader = state.getLeader(shardId); - return leader != null && id.equals(leader.getName()); - }); + zkStateReader.waitForState( + collection, + 10, + TimeUnit.SECONDS, + state -> { + Replica leader = state.getLeader(shardId); + return leader != null && id.equals(leader.getName()); + }); } catch (TimeoutException e) { throw new RuntimeException(e); } diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index c8050658c77e..536c11709d07 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -105,7 +105,8 @@ public PeerSync.PeerSyncResult sync( return PeerSync.PeerSyncResult.failure(); } - return syncReplicas(zkController, core, leaderProps, peerSyncOnlyWithActive, ignoreNoVersionsFailure); + return syncReplicas( + zkController, core, leaderProps, peerSyncOnlyWithActive, ignoreNoVersionsFailure); } private PeerSync.PeerSyncResult syncReplicas( diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 24f85ef18c6e..02a0aa29d58c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -17,13 +17,13 @@ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; + import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; - public class AdminCmdContext { private final CollectionParams.CollectionAction action; private final String asyncId; @@ -41,7 +41,8 @@ public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) this.asyncId = asyncId; } - public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { + public AdminCmdContext( + CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { this.action = action; this.asyncId = asyncId; this.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java index a65d369a98e2..de6b965b5005 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java @@ -80,7 +80,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.OverseerNodePrioritizer; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -275,13 +274,7 @@ public void call( params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RELOAD.toString()); CollectionHandlingUtils.collectionCmd( - adminCmdContext, - message, - params, - results, - null, - Collections.emptySet(), - ccc); + adminCmdContext, message, params, results, null, Collections.emptySet(), ccc); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index f7f09bbb64bf..c3e8a0abf780 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -17,9 +17,6 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; @@ -41,7 +38,6 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 5923fc810c13..bff0be76ad3e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -23,7 +23,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_AT_ONCE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; import java.lang.invoke.MethodHandles; @@ -460,7 +459,8 @@ private void rejoinElectionQueue( asyncRequests.add(asyncId); // ignore response; we construct our own - collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); + collectionsHandler.submitCollectionApiCommand( + new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 9783c28a1ab0..b6766696cdef 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -121,7 +121,8 @@ public void disableResponseCaching() { protected SolrResponse submitRemoteMessageAndHandleException( SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { - adminCmdContext.setCallingLockIds((String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); + adminCmdContext.setCallingLockIds( + (String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer.getZkController(), diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java index fb11371c6e81..693a65402271 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java @@ -153,7 +153,7 @@ public GetSegmentDataResponse getSegmentData( coreSummary.indexDir = core.getIndexDir(); coreSummary.sizeInGB = (double) core.getIndexSize() / GB; - RefCounted iwRef = core.getSolrCoreState().getIndexWriter(core); + RefCounted iwRef = core.getSolrCoreState().getIndexWriter(core, true); if (iwRef != null) { try { IndexWriter iw = iwRef.get(); @@ -257,7 +257,7 @@ private Map getMergeInformation( SolrQueryRequest req, SegmentInfos infos, List mergeCandidates) throws IOException { final var result = new HashMap(); RefCounted refCounted = - req.getCore().getSolrCoreState().getIndexWriter(req.getCore()); + req.getCore().getSolrCoreState().getIndexWriter(req.getCore(), true ); try { IndexWriter indexWriter = refCounted.get(); if (indexWriter instanceof SolrIndexWriter) { diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 9051917c58da..462e699821bd 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -164,7 +164,8 @@ public void processCommit(CommitUpdateCommand cmd) throws IOException { if (cmd.failOnReadOnly) { throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only."); } else { - // Committing on a readOnly core/collection is a no-op, since the core was committed when becoming read-only and hasn't had any updates since. + // Committing on a readOnly core/collection is a no-op, since the core was committed when + // becoming read-only and hasn't had any updates since. return; } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java index 57a82b57c36c..6cf4e994d684 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java @@ -58,7 +58,9 @@ public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackup + " \n" + " \n" + " \n" - + " \n" + + " \n" + " localfs\n" + " ${hostPort:8983}\n" + " \n" diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java index 1ba2f114b3f7..d955da11e1eb 100644 --- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java @@ -55,7 +55,9 @@ public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest { + " \n" + " \n" + " \n" - + " \n" + + " \n" + " localfs\n" + " ${hostPort:8983}\n" + " \n" diff --git a/solr/modules/s3-repository/src/test-files/log4j2.xml b/solr/modules/s3-repository/src/test-files/log4j2.xml index 528299e3e0bd..64039f916dc3 100644 --- a/solr/modules/s3-repository/src/test-files/log4j2.xml +++ b/solr/modules/s3-repository/src/test-files/log4j2.xml @@ -22,7 +22,7 @@ %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ - =>%ex{short}}}{10240}%n + =>%ex}}{10240}%n diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index 5393a52bf2ae..fd3d532aab75 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Set; +import java.util.Set;solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.lucene.store.Directory; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 5829779e40f4..f702e37ade2b 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -48,9 +48,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CollectionsApi; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -74,7 +72,6 @@ import org.apache.solr.core.backup.ShardBackupMetadata; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.LogLevel; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -496,6 +493,7 @@ public void testBackupProperties() throws IOException { } static Set portsToFailOn = new HashSet<>(); + @Test public void testRestoreToOriginalSucceedsWithErrors() throws Exception { setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); @@ -503,14 +501,16 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { final String backupName = BACKUPNAME_PREFIX + testSuffix; // Bootstrap the backup collection with seed docs - CollectionAdminRequest.createCollection( - backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES) + CollectionAdminRequest.createCollection(backupCollectionName, "conf1", NUM_SHARDS, NUM_NODES) .process(cluster.getSolrClient()); int backupDocs = indexDocs(backupCollectionName, true); // Backup and immediately add more docs to the collection try (BackupRepository repository = - cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(ERROR_BACKUP_REPO_NAME)) { + cluster + .getJettySolrRunner(0) + .getCoreContainer() + .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { final String backupLocation = repository.getBackupLocation(getBackupLocation()); final RequestStatusState result = CollectionAdminRequest.backupCollection(backupCollectionName, backupName) @@ -561,7 +561,10 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { final String backupLocation = repository.getBackupLocation(getBackupLocation()); // All but the first jetty will fail - portsToFailOn = cluster.getJettySolrRunners().subList(1, NUM_NODES).stream().map(JettySolrRunner::getLocalPort).collect(Collectors.toSet()); + portsToFailOn = + cluster.getJettySolrRunners().subList(1, NUM_NODES).stream() + .map(JettySolrRunner::getLocalPort) + .collect(Collectors.toSet()); final RequestStatusState result = CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) .setLocation(backupLocation) @@ -581,6 +584,7 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { public static class ErrorThrowingTrackingBackupRepository extends TrackingBackupRepository { private int port; + @Override public void init(NamedList args) { super.init(args); @@ -597,7 +601,8 @@ public void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws I @Override public void copyIndexFileTo( - URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException { + URI sourceRepo, String sourceFileName, Directory dest, String destFileName) + throws IOException { if (portsToFailOn.contains(port)) { throw new UnsupportedOperationException(); } @@ -682,11 +687,8 @@ private void simpleRestoreAndCheckDocCount( } protected void clearDocs(String collectionName) throws Exception { - CollectionAdminRequest.deleteCollection( - collectionName) - .process(cluster.getSolrClient()); - CollectionAdminRequest.createCollection( - collectionName, "conf1", NUM_SHARDS, NUM_NODES) + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection(collectionName, "conf1", NUM_SHARDS, NUM_NODES) .process(cluster.getSolrClient()); log.info("Cleared all docs in collection: {}", collectionName); From e3d1d265cc6c8172cd8034c5c2f063c0cbbf0903 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 15:00:17 -0800 Subject: [PATCH 44/67] Fix collection and shard term deletion --- .../org/apache/solr/cloud/ZkCollectionTerms.java | 11 ++++++++++- .../java/org/apache/solr/cloud/ZkShardTerms.java | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java index a6c6a857a9e1..5aa1eba074a3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java @@ -38,7 +38,15 @@ class ZkCollectionTerms implements AutoCloseable { public ZkShardTerms getShard(String shardId) { synchronized (terms) { - return terms.computeIfAbsent(shardId, shard -> new ZkShardTerms(collection, shard, zkClient)); + return terms.compute( + shardId, + (shard, existingTerms) -> { + if (existingTerms == null || existingTerms.isClosed()) { + return new ZkShardTerms(collection, shard, zkClient); + } else { + return existingTerms; + } + }); } } @@ -60,6 +68,7 @@ public void remove(String shardId, CoreDescriptor coreDescriptor) { public void close() { synchronized (terms) { terms.values().forEach(ZkShardTerms::close); + terms.clear(); } assert ObjectReleaseTracker.release(this); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java index ffc444dda134..116efc785fc1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java @@ -166,9 +166,14 @@ public void close() { synchronized (listeners) { listeners.clear(); } + terms.set(new ShardTerms()); assert ObjectReleaseTracker.release(this); } + public boolean isClosed() { + return isClosed.get(); + } + // package private for testing, only used by tests Map getTerms() { return new HashMap<>(terms.get().getTerms()); @@ -331,7 +336,12 @@ private boolean saveTerms(ShardTerms newTerms, String action) log.info("Successful update of terms at {} to {} for {}", znodePath, newTerms, action); return true; } catch (KeeperException.BadVersionException e) { - log.info("Failed to save terms, version is not a match, retrying"); + if (log.isInfoEnabled()) { + log.info( + "Failed to save terms for {}, version {} is not a match, retrying", + action, + newTerms.getVersion()); + } refreshTerms(); } catch (KeeperException.NoNodeException e) { throw e; @@ -417,6 +427,10 @@ private void registerWatcher() throws KeeperException { if (Watcher.Event.EventType.None == event.getType()) { return; } + // If the node is deleted, we should close the ZkShardTerms + if (Watcher.Event.EventType.NodeDeleted == event.getType()) { + close(); + } // Some events may be missed during registering a watcher, so it is safer to refresh terms // after registering watcher retryRegisterWatcher(); From 40b62f6c36c99f14b71056f65d3f2e67ff77b3e0 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 15:03:02 -0800 Subject: [PATCH 45/67] Tidy --- .../handler/admin/api/GetSegmentData.java | 2 +- .../solr/s3/S3IncrementalBackupTest.java | 23 +++---------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java index 693a65402271..ffafc81e5c02 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java @@ -257,7 +257,7 @@ private Map getMergeInformation( SolrQueryRequest req, SegmentInfos infos, List mergeCandidates) throws IOException { final var result = new HashMap(); RefCounted refCounted = - req.getCore().getSolrCoreState().getIndexWriter(req.getCore(), true ); + req.getCore().getSolrCoreState().getIndexWriter(req.getCore(), true); try { IndexWriter indexWriter = refCounted.get(); if (indexWriter instanceof SolrIndexWriter) { diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index fd3d532aab75..c35dbc17ab0e 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -19,31 +19,12 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.URI; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set;solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.PropertiesUtil; -import org.apache.solr.core.TrackingBackupRepository; -import org.apache.solr.core.backup.repository.BackupRepository; -import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.util.LogLevel; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.Region; @@ -87,7 +68,9 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest { + " \n" + " \n" + " \n" - + " \n" + + " \n" + " s3\n" + " ${hostPort:8983}\n" + " \n" From da598a5160db168cc99de3e33f278dc71c4271d4 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 15:23:02 -0800 Subject: [PATCH 46/67] make another wait conditional on not readOnly --- .../apache/solr/cloud/RecoveryStrategy.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 9f7cb74e1fb5..bf023d3cb090 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -659,15 +659,17 @@ public final void doSyncOrReplicateRecovery(SolrCore core) throws Exception { break; } - // we wait a bit so that any updates on the leader - // that started before they saw recovering state - // are sure to have finished (see SOLR-7141 for - // discussion around current value) - // TODO since SOLR-11216, we probably won't need this - try { - Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + if (!core.readOnly) { + // we wait a bit so that any updates on the leader + // that started before they saw recovering state + // are sure to have finished (see SOLR-7141 for + // discussion around current value) + // TODO since SOLR-11216, we probably won't need this + try { + Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } // first thing we just try to sync From 1b71b7219d6f75a7585608d536df040cc50b1828 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 15:30:14 -0800 Subject: [PATCH 47/67] Add changelog entry, remove others --- .../unreleased/solr-17821-fix-restore-error-scenario.yml | 9 +++++++++ changelog/unreleased/solr-18011-locking-update.yml | 9 --------- .../solr-18080-shard-term-induce-leader-election.yml | 9 --------- 3 files changed, 9 insertions(+), 18 deletions(-) create mode 100644 changelog/unreleased/solr-17821-fix-restore-error-scenario.yml delete mode 100644 changelog/unreleased/solr-18011-locking-update.yml delete mode 100644 changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml diff --git a/changelog/unreleased/solr-17821-fix-restore-error-scenario.yml b/changelog/unreleased/solr-17821-fix-restore-error-scenario.yml new file mode 100644 index 000000000000..dbadb3d6fe38 --- /dev/null +++ b/changelog/unreleased/solr-17821-fix-restore-error-scenario.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Fix error scenario in InstallShardData and Restore +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman +links: + - name: SOLR-17821 + url: https://issues.apache.org/jira/browse/SOLR-17821 diff --git a/changelog/unreleased/solr-18011-locking-update.yml b/changelog/unreleased/solr-18011-locking-update.yml deleted file mode 100644 index 3b714ae59345..000000000000 --- a/changelog/unreleased/solr-18011-locking-update.yml +++ /dev/null @@ -1,9 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: Allow locked Admin APIs to call other locked AdminAPIs without deadlocking -type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Houston Putman - nick: HoustonPutman -links: - - name: SOLR-18011 - url: https://issues.apache.org/jira/browse/SOLR-18011 diff --git a/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml b/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml deleted file mode 100644 index 49c3f8f4db3e..000000000000 --- a/changelog/unreleased/solr-18080-shard-term-induce-leader-election.yml +++ /dev/null @@ -1,9 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: ShardTerms can now induce a leader election if needed -type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Houston Putman - nick: HoustonPutman -links: - - name: SOLR-18080 - url: https://issues.apache.org/jira/browse/SOLR-18080 From 7dc8a98d459d67c62b0845c7d05e368eb0bda32c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 16:03:33 -0800 Subject: [PATCH 48/67] Tidy --- .../apache/solr/cloud/api/collections/AdminCmdContext.java | 7 ++++--- .../org/apache/solr/handler/admin/RebalanceLeaders.java | 4 ++-- .../org/apache/solr/handler/admin/api/AdminAPIBase.java | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 24f85ef18c6e..02a0aa29d58c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -17,13 +17,13 @@ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; + import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; - public class AdminCmdContext { private final CollectionParams.CollectionAction action; private final String asyncId; @@ -41,7 +41,8 @@ public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId) this.asyncId = asyncId; } - public AdminCmdContext(CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { + public AdminCmdContext( + CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { this.action = action; this.asyncId = asyncId; this.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 5923fc810c13..bff0be76ad3e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -23,7 +23,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.MAX_AT_ONCE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_WAIT_SECONDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS; import java.lang.invoke.MethodHandles; @@ -460,7 +459,8 @@ private void rejoinElectionQueue( asyncRequests.add(asyncId); // ignore response; we construct our own - collectionsHandler.submitCollectionApiCommand(new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); + collectionsHandler.submitCollectionApiCommand( + new AdminCmdContext(REBALANCELEADERS, asyncId, req), new ZkNodeProps(propMap)); } // maxWaitSecs - How long are we going to wait? Defaults to 30 seconds. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 9783c28a1ab0..b6766696cdef 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -121,7 +121,8 @@ public void disableResponseCaching() { protected SolrResponse submitRemoteMessageAndHandleException( SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { - adminCmdContext.setCallingLockIds((String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); + adminCmdContext.setCallingLockIds( + (String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer.getZkController(), From f6062da882971be1baabb5b10dd209da6057205c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 16:01:15 -0800 Subject: [PATCH 49/67] Fix no uLog error case --- .../java/org/apache/solr/cloud/SyncStrategy.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index 536c11709d07..b08dea55abfb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -100,11 +100,6 @@ public PeerSync.PeerSyncResult sync( log.info("Sync replicas to {}", ZkCoreNodeProps.getCoreUrl(leaderProps)); } - if (core.getUpdateHandler().getUpdateLog() == null) { - log.error("No UpdateLog found - cannot sync"); - return PeerSync.PeerSyncResult.failure(); - } - return syncReplicas( zkController, core, leaderProps, peerSyncOnlyWithActive, ignoreNoVersionsFailure); } @@ -129,9 +124,14 @@ private PeerSync.PeerSyncResult syncReplicas( // first sync ourselves - we are the potential leader after all try { - result = - syncWithReplicas( - zkController, core, leaderProps, collection, shardId, peerSyncOnlyWithActive); + if (core.getUpdateHandler().getUpdateLog() == null) { + log.error("No UpdateLog found - cannot sync"); + result = PeerSync.PeerSyncResult.failure(); + } else { + result = + syncWithReplicas( + zkController, core, leaderProps, collection, shardId, peerSyncOnlyWithActive); + } if (!result.isSuccess() && ignoreNoVersionsFailure) { UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); From 714a7056af964eacd3c15559673d991dde348724 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 16:36:52 -0800 Subject: [PATCH 50/67] Fix test --- .../solr/opentelemetry/TestDistributedTracing.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java index 97916c007ca9..6391b8d7db9b 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java @@ -228,7 +228,15 @@ private void verifyCollectionCreation(String collection) throws Exception { // db.instance=testInternalCollectionApiCommands_shard2_replica_n1 // db.instance=testInternalCollectionApiCommands_shard1_replica_n6 // - // 7..8 (2 times) name=post:/{core}/get + // 7..8 (2 times) name=post:/{core}/get (FingerPrinting to get versions from non-leaders) + // db.instance=testInternalCollectionApiCommands_shard2_replica_n1 + // db.instance=testInternalCollectionApiCommands_shard1_replica_n6 + // + // 9..10 (2 times) name=post:/{core}/get (PeerSync request to non-leaders) + // db.instance=testInternalCollectionApiCommands_shard2_replica_n1 + // db.instance=testInternalCollectionApiCommands_shard1_replica_n6 + // + // 11..12 (2 times) name=post:/{core}/get (FingerPrinting to get versions from leaders PeerSync) // db.instance=testInternalCollectionApiCommands_shard2_replica_n4 // db.instance=testInternalCollectionApiCommands_shard1_replica_n2 @@ -238,7 +246,7 @@ private void verifyCollectionCreation(String collection) throws Exception { assertEquals("create:/admin/collections", s0.getName()); Map ops = new HashMap<>(); - assertEquals(7, finishedSpans.size()); + assertEquals(11, finishedSpans.size()); var parentTraceId = getRootTraceId(finishedSpans); for (var span : finishedSpans) { if (isRootSpan(span)) { @@ -251,7 +259,7 @@ private void verifyCollectionCreation(String collection) throws Exception { ops.put(span.getName(), ops.getOrDefault(span.getName(), 0) + 1); } var expectedOps = - Map.of("CreateCollectionCmd", 1, "post:/admin/cores", 4, "post:/{core}/get", 2); + Map.of("CreateCollectionCmd", 1, "post:/admin/cores", 4, "post:/{core}/get", 6); assertEquals(expectedOps, ops); } From 201d31d6210139596505da329817230c1d9d96b7 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 27 Jan 2026 16:54:13 -0800 Subject: [PATCH 51/67] InstallShardTest failure scenario should not just be s3 --- .../collections/LocalFSInstallShardTest.java | 8 +- .../apache/solr/gcs/GCSInstallShardTest.java | 9 ++- .../apache/solr/s3/S3InstallShardTest.java | 75 ++----------------- .../AbstractIncrementalBackupTest.java | 9 ++- .../collections/AbstractInstallShardTest.java | 41 ++++++++++ 5 files changed, 69 insertions(+), 73 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSInstallShardTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSInstallShardTest.java index 690ff4471946..989c894b9783 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSInstallShardTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSInstallShardTest.java @@ -30,6 +30,12 @@ public class LocalFSInstallShardTest extends AbstractInstallShardTest { + " \n" + " localfs\n" + " \n" + + " \n" + + " localfs\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " \n" + " \n"; @@ -43,7 +49,7 @@ public static void setupClass() throws Exception { final String tmpDirPrefix = whitespacesInPath ? "my install" : "myinstall"; final String backupLocation = createTempDir(tmpDirPrefix).toAbsolutePath().toString(); - configureCluster(1) // nodes + configureCluster(2) // nodes .addConfig( "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) .withSolrXml(SOLR_XML.replace("ALLOWPATHS_TEMPLATE_VAL", backupLocation)) diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java index ecb08fa01926..4b78c0cc8053 100644 --- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java @@ -19,6 +19,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; import org.apache.solr.handler.admin.api.InstallShardData; import org.junit.AfterClass; @@ -40,6 +41,12 @@ public class GCSInstallShardTest extends AbstractInstallShardTest { + " \n" + " localfs\n" + " \n" + + " \n" + + " localfs\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " someBucketName\n" + " backup1\n" @@ -51,7 +58,7 @@ public class GCSInstallShardTest extends AbstractInstallShardTest { @BeforeClass public static void setupClass() throws Exception { - configureCluster(1) // nodes + configureCluster(2) // nodes .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml(SOLR_XML) .configure(); diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java index 78000ac7fad7..c44e2170a39e 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java @@ -19,17 +19,12 @@ import com.adobe.testing.s3mock.junit4.S3MockRule; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest; import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; -import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.handler.admin.api.InstallShardData; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Test; import software.amazon.awssdk.regions.Region; /** @@ -50,19 +45,17 @@ public class S3InstallShardTest extends AbstractInstallShardTest { + " \n" + " s3\n" + " \n" + + " \n" + + " s3\n" + + " ${hostPort:8983}\n" + + " \n" + " \n" + " BUCKET\n" + " REGION\n" + " ENDPOINT\n" + " \n" - + " \n" - + " s3BadNode\n" - + " \n" - + " \n" - + " BAD_BUCKET\n" - + " REGION\n" - + " ENDPOINT\n" - + " \n" + " \n"; private static final String SOLR_XML = AbstractInstallShardTest.defaultSolrXmlTextWithBackupRepository(BACKUP_REPOSITORY_XML); @@ -79,12 +72,10 @@ public static void setupClass() throws Exception { AbstractS3ClientTest.setS3ConfFile(); - configureCluster(1) // nodes + configureCluster(2) // nodes .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml( SOLR_XML - // The first solr node will not have a bad bucket - .replace("BAD_BUCKET", BUCKET_NAME) .replace("BUCKET", BUCKET_NAME) .replace("REGION", Region.US_EAST_1.id()) .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())) @@ -92,54 +83,4 @@ public static void setupClass() throws Exception { bootstrapBackupRepositoryData("/"); } - - @Test - public void testInstallSucceedsOnASingleError() throws Exception { - JettySolrRunner jettySolrRunner = - cluster.startJettySolrRunner( - SOLR_XML - // The first solr node will not have a bad bucket - .replace("BAD_BUCKET", "non-existent") - .replace("BUCKET", BUCKET_NAME) - .replace("REGION", Region.US_EAST_1.id()) - .replace("ENDPOINT", "http://localhost:" + S3_MOCK_RULE.getHttpPort())); - - try { - final String collectionName = createAndAwaitEmptyCollection(1, 2); - deleteAfterTest(collectionName); - enableReadOnly(collectionName); - - final String singleShardLocation = singleShard1Uri.toString(); - { // Test synchronous request error reporting - CollectionAdminRequest.installDataToShard( - collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME + "BadNode") - .process(cluster.getSolrClient()); - waitForState( - "The failed core-install should recover and become healthy", - collectionName, - 30, - TimeUnit.SECONDS, - SolrCloudTestCase.activeClusterShape(1, 2)); - assertCollectionHasNumDocs(collectionName, singleShardNumDocs); - } - - { // Test asynchronous request error reporting - final var requestStatusState = - CollectionAdminRequest.installDataToShard( - collectionName, "shard1", singleShardLocation, BACKUP_REPO_NAME + "BadNode") - .processAndWait(cluster.getSolrClient(), 15); - - assertEquals(RequestStatusState.COMPLETED, requestStatusState); - waitForState( - "The failed core-install should recover and become healthy", - collectionName, - 30, - TimeUnit.SECONDS, - SolrCloudTestCase.activeClusterShape(1, 2)); - assertCollectionHasNumDocs(collectionName, singleShardNumDocs); - } - } finally { - jettySolrRunner.stop(); - } - } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index f702e37ade2b..91c46d67874a 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -492,8 +492,6 @@ public void testBackupProperties() throws IOException { } } - static Set portsToFailOn = new HashSet<>(); - @Test public void testRestoreToOriginalSucceedsWithErrors() throws Exception { setTestSuffix("testRestoreToOriginalSucceedsOnASingleError"); @@ -534,7 +532,8 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { .getCoreContainer() .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { // Only the first jetty will fail - portsToFailOn = Set.of(cluster.getJettySolrRunner(0).getLocalPort()); + ErrorThrowingTrackingBackupRepository.portsToFailOn = + Set.of(cluster.getJettySolrRunner(0).getLocalPort()); final String backupLocation = repository.getBackupLocation(getBackupLocation()); final RequestStatusState result = CollectionAdminRequest.restoreCollection(backupCollectionName, backupName) @@ -561,7 +560,7 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { .newBackupRepository(ERROR_BACKUP_REPO_NAME)) { final String backupLocation = repository.getBackupLocation(getBackupLocation()); // All but the first jetty will fail - portsToFailOn = + ErrorThrowingTrackingBackupRepository.portsToFailOn = cluster.getJettySolrRunners().subList(1, NUM_NODES).stream() .map(JettySolrRunner::getLocalPort) .collect(Collectors.toSet()); @@ -583,6 +582,8 @@ public void testRestoreToOriginalSucceedsWithErrors() throws Exception { public static class ErrorThrowingTrackingBackupRepository extends TrackingBackupRepository { + public static Set portsToFailOn = new HashSet<>(); + private int port; @Override diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java index ad61943cf0ed..50a5fc633a61 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractInstallShardTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -72,6 +73,7 @@ public abstract class AbstractInstallShardTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected static final String BACKUP_REPO_NAME = "trackingBackupRepository"; + protected static final String ERROR_BACKUP_REPO_NAME = "errorBackupRepository"; private static long docsSeed; // see indexDocs() @@ -244,6 +246,45 @@ public void testParallelInstallToMultiShardCollection() throws Exception { assertCollectionHasNumDocs(collectionName, multiShardNumDocs); } + @Test + public void testInstallSucceedsOnASingleError() throws Exception { + final String collectionName = createAndAwaitEmptyCollection(1, 2); + deleteAfterTest(collectionName); + enableReadOnly(collectionName); + + AbstractIncrementalBackupTest.ErrorThrowingTrackingBackupRepository.portsToFailOn = + Set.of(cluster.getJettySolrRunner(0).getLocalPort()); + final String singleShardLocation = singleShard1Uri.toString(); + { // Test synchronous request error reporting + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", singleShardLocation, ERROR_BACKUP_REPO_NAME) + .process(cluster.getSolrClient()); + waitForState( + "The failed core-install should recover and become healthy", + collectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(1, 2)); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + + { // Test asynchronous request error reporting + final var requestStatusState = + CollectionAdminRequest.installDataToShard( + collectionName, "shard1", singleShardLocation, ERROR_BACKUP_REPO_NAME) + .processAndWait(cluster.getSolrClient(), 15); + + assertEquals(RequestStatusState.COMPLETED, requestStatusState); + waitForState( + "The failed core-install should recover and become healthy", + collectionName, + 30, + TimeUnit.SECONDS, + SolrCloudTestCase.activeClusterShape(1, 2)); + assertCollectionHasNumDocs(collectionName, singleShardNumDocs); + } + } + /** * Builds a string representation of a valid solr.xml configuration, with the provided * backup-repository configuration inserted From 71aae5b40d0ec3ee0e5c5383c33dc53b4def5cef Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 28 Jan 2026 12:42:50 -0800 Subject: [PATCH 52/67] Make some fixes, primarily indexFetcher checking files that are still open --- .../cloud/ShardLeaderElectionContextBase.java | 14 ---- .../org/apache/solr/handler/IndexFetcher.java | 83 ++++++++----------- solr/core/src/test-files/log4j2.xml | 2 +- .../solr/cloud/ZkShardTermsRecoveryTest.java | 4 +- 4 files changed, 39 insertions(+), 64 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java index d50dd58d10cd..4ed392b3a852 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java @@ -19,8 +19,6 @@ import java.lang.invoke.MethodHandles; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.apache.curator.framework.api.transaction.CuratorTransactionResult; import org.apache.curator.framework.api.transaction.OperationType; import org.apache.solr.cloud.overseer.OverseerAction; @@ -240,18 +238,6 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup prs) .persist(coll.getZNode(), zkStateReader.getZkClient()); } - try { - zkStateReader.waitForState( - collection, - 10, - TimeUnit.SECONDS, - state -> { - Replica leader = state.getLeader(shardId); - return leader != null && id.equals(leader.getName()); - }); - } catch (TimeoutException e) { - throw new RuntimeException(e); - } } } diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index a2fd75c6c732..3f250dc45c04 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -494,7 +494,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // TODO: make sure that getLatestCommit only returns commit points for the main index (i.e. no // side-car indexes) - IndexCommit commit = solrCore.getDeletionPolicy().getAndSaveLatestCommit(); + IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit(); if (commit == null) { // Presumably the IndexWriter hasn't been opened yet, and hence the deletion policy hasn't // been updated with commit points @@ -1300,11 +1300,11 @@ protected static CompareResult compareFile( Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) { CompareResult compareResult = new CompareResult(); try { - try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) { - long indexFileLen = indexInput.length(); - long indexFileChecksum = 0; - - if (backupIndexFileChecksum != null) { + long indexFileLen; + long indexFileChecksum = 0; + if (backupIndexFileChecksum != null) { + try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) { + indexFileLen = indexInput.length(); try { indexFileChecksum = CodecUtil.retrieveChecksum(indexInput); compareResult.checkSummed = true; @@ -1312,47 +1312,49 @@ protected static CompareResult compareFile( log.warn("Could not retrieve checksum from file.", e); } } + } else { + indexFileLen = indexDir.fileLength(filename); + } - if (!compareResult.checkSummed) { - // we don't have checksums to compare - - if (indexFileLen == backupIndexFileLen) { - compareResult.equal = true; - return compareResult; - } else { - log.info( - "File {} did not match. expected length is {} and actual length is {}", - filename, - backupIndexFileLen, - indexFileLen); - compareResult.equal = false; - return compareResult; - } - } - - // we have checksums to compare + if (!compareResult.checkSummed) { + // we don't have checksums to compare - if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) { + if (indexFileLen == backupIndexFileLen) { compareResult.equal = true; return compareResult; } else { - log.warn( - "File {} did not match. expected checksum is {} and actual is checksum {}. " - + "expected length is {} and actual length is {}", + log.info( + "File {} did not match. expected length is {} and actual length is {}", filename, - backupIndexFileChecksum, - indexFileChecksum, backupIndexFileLen, indexFileLen); compareResult.equal = false; return compareResult; } } + + // we have checksums to compare + + if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) { + compareResult.equal = true; + return compareResult; + } else { + log.warn( + "File {} did not match. expected checksum is {} and actual is checksum {}. " + + "expected length is {} and actual length is {}", + filename, + backupIndexFileChecksum, + indexFileChecksum, + backupIndexFileLen, + indexFileLen); + compareResult.equal = false; + return compareResult; + } } catch (NoSuchFileException | FileNotFoundException e) { compareResult.equal = false; return compareResult; } catch (IOException e) { - log.error("Could not read file {}. Downloading it again", filename, e); + log.error("Could not read file {}. Assuming it is out of date", filename, e); compareResult.equal = false; return compareResult; } @@ -1383,22 +1385,9 @@ private boolean isIndexStale(Directory dir) throws IOException { String filename = (String) file.get(NAME); Long length = (Long) file.get(SIZE); Long checksum = (Long) file.get(CHECKSUM); - if (slowFileExists(dir, filename)) { - if (checksum != null) { - if (!(compareFile(dir, filename, length, checksum).equal)) { - // file exists and size or checksum is different, therefore we must download it again - return true; - } - } else { - if (length != dir.fileLength(filename)) { - log.warn( - "File {} did not match. expected length is {} and actual length is {}", - filename, - length, - dir.fileLength(filename)); - return true; - } - } + if (!(compareFile(dir, filename, length, checksum).equal)) { + // file exists and size or checksum is different, therefore we must download it again + return true; } } return false; diff --git a/solr/core/src/test-files/log4j2.xml b/solr/core/src/test-files/log4j2.xml index 9078571bce0d..d42c102b1ac4 100644 --- a/solr/core/src/test-files/log4j2.xml +++ b/solr/core/src/test-files/log4j2.xml @@ -22,7 +22,7 @@ %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ - =>%ex{short}}}{10240}%n + =>%ex}}{10240}%n diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java index 4e337cdfb572..c483a6c2a12e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsRecoveryTest.java @@ -70,7 +70,7 @@ public void waitForActiveState() { public void testShardTermsInducedReplication() throws Exception { String shard = "shard2"; if (random().nextBoolean()) { - // Add uncommitted documents, to test that part of the recovery + // Add uncommitted/committed documents, to test that part of the recovery UpdateRequest up = new UpdateRequest(); for (int i = 0; i < 1000; i++) { up.add("id", "id2-" + i); @@ -130,7 +130,7 @@ public void testShardTermsInducedReplication() throws Exception { public void testShardTermsInducedLeaderElection() throws IOException, SolrServerException { String shard = "shard1"; if (random().nextBoolean()) { - // Add uncommitted documents, to test that part of the recovery + // Add uncommitted/committed documents, to test that part of the recovery UpdateRequest up = new UpdateRequest(); for (int i = 0; i < 1000; i++) { up.add("id", "id3-" + i); From 062ef49c91e4d26b6441ddf67b7993b34e58a3e6 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 28 Jan 2026 13:53:32 -0800 Subject: [PATCH 53/67] Address some review comments --- .../java/org/apache/solr/update/CommitUpdateCommand.java | 7 ++++--- .../solr/update/processor/DistributedUpdateProcessor.java | 2 +- .../update/processor/DistributedZkUpdateProcessor.java | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java index 470740583eee..2e4d0be5fb2a 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java @@ -98,9 +98,10 @@ public String toString() { .append(",softCommit=") .append(softCommit) .append(",prepareCommit=") - .append(prepareCommit) - .append(",failOnReadOnly=") - .append(failOnReadOnly); + .append(prepareCommit); + if (!failOnReadOnly) { + sb.append(",failOnReadOnly=").append(failOnReadOnly); + } if (commitData != null) { sb.append(",commitData=").append(commitData); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index 82424a5f06a4..fe6f44501e71 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -127,7 +127,7 @@ public static DistribPhase parseParam(final String param) { protected final SolrQueryResponse rsp; private final AtomicUpdateDocumentMerger docMerger; - private final UpdateLog ulog; + protected final UpdateLog ulog; private final VersionInfo vinfo; private final boolean versionsStored; private boolean returnVersions; diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 462e699821bd..631f67836e36 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -166,6 +166,8 @@ public void processCommit(CommitUpdateCommand cmd) throws IOException { } else { // Committing on a readOnly core/collection is a no-op, since the core was committed when // becoming read-only and hasn't had any updates since. + assert ulog == null || !ulog.hasUncommittedChanges() + : "Uncommitted changes found when trying to commit on a read-only collection"; return; } } From 3f200acbf99426d91e76b3d1cec4ca42609c05c5 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 28 Jan 2026 13:58:10 -0800 Subject: [PATCH 54/67] Change flag to failOnReadOnly --- solr/core/src/java/org/apache/solr/core/SolrCore.java | 2 +- .../src/java/org/apache/solr/handler/IndexFetcher.java | 4 ++-- .../java/org/apache/solr/handler/ReplicationHandler.java | 2 +- .../java/org/apache/solr/update/DefaultSolrCoreState.java | 4 ++-- .../src/java/org/apache/solr/update/SolrCoreState.java | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 5e246e69ff15..4e73bd530f36 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -2476,7 +2476,7 @@ public RefCounted openNewSearcher( true, directoryFactory); } else { - RefCounted writer = getSolrCoreState().getIndexWriter(this, true); + RefCounted writer = getSolrCoreState().getIndexWriter(this, false); DirectoryReader newReader = null; try { newReader = indexReaderFactory.newReader(writer.get(), this); diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 3f250dc45c04..969d9991108c 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -530,7 +530,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // we just clear ours and commit log.info("New index in Leader. Deleting mine..."); RefCounted iw = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, true); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, false); try { iw.get().deleteAll(); } finally { @@ -624,7 +624,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // are successfully deleted solrCore.getUpdateHandler().newIndexWriter(true); RefCounted writer = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, true); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, false); try { IndexWriter indexWriter = writer.get(); int c = 0; diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 7f50064ffe1d..e803bdd1d6b8 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1378,7 +1378,7 @@ public void inform(SolrCore core) { // ensure the writer is initialized so that we have a list of commit points RefCounted iw = - core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, true); + core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, false); iw.decref(); } catch (IOException e) { diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 4feb43abd9d1..e7c9474f5084 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -106,9 +106,9 @@ private void closeIndexWriter(IndexWriterCloser closer) { } @Override - public RefCounted getIndexWriter(SolrCore core, boolean readOnlyCompatible) + public RefCounted getIndexWriter(SolrCore core, boolean failOnReadOnly) throws IOException { - if (core != null && (!core.indexEnabled || (!readOnlyCompatible && core.readOnly))) { + if (core != null && (!core.indexEnabled || (core.readOnly && failOnReadOnly))) { throw new SolrException( SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled"); } diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 5016f9fdbb6f..23adf63bd979 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -179,17 +179,17 @@ public void deregisterInFlightUpdate() { * @throws IOException If there is a low-level I/O error. */ public RefCounted getIndexWriter(SolrCore core) throws IOException { - return getIndexWriter(core, false); + return getIndexWriter(core, true); } /** * Get the current IndexWriter. If a new IndexWriter must be created, use the settings from the - * given {@link SolrCore}. Be very careful using the {@code readOnlyCompatible} flag, by default - * it should be false if the returned indexWriter will be used for writing. + * given {@link SolrCore}. Be very careful using the {@code failOnReadOnly=false} flag, by default + * it should be true if the returned indexWriter will be used for writing. * * @throws IOException If there is a low-level I/O error. */ - public abstract RefCounted getIndexWriter(SolrCore core, boolean readOnlyCompatible) + public abstract RefCounted getIndexWriter(SolrCore core, boolean failOnReadOnly) throws IOException; /** From dbb01cd2459942cdb4d0c8c2e7f3a747ba2f7a22 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 29 Jan 2026 13:03:35 -0800 Subject: [PATCH 55/67] Deserialize callingLockIds in context class --- .../java/org/apache/solr/cloud/LockTree.java | 2 +- .../OverseerConfigSetMessageHandler.java | 2 +- .../solr/cloud/OverseerMessageHandler.java | 2 +- .../solr/cloud/OverseerTaskProcessor.java | 12 +++++++++- .../api/collections/AdminCmdContext.java | 14 +++++++++++ .../collections/CollectionApiLockFactory.java | 8 +------ .../OverseerCollectionMessageHandler.java | 8 +------ .../org/apache/solr/cloud/TestLockTree.java | 23 +++++++++++++++---- 8 files changed, 48 insertions(+), 23 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 7940b07ef7d4..20efff349e44 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -96,7 +96,7 @@ public Lock lock( // If a callingLockId was passed in, validate it with the current lock path, and only start // locking below the calling lock - Lock callingLock = callingLockIds != null ? allLocks.get(callingLockIds.getLast()) : null; + Lock callingLock = callingLockIds.isEmpty() ? null : allLocks.get(callingLockIds.getLast()); boolean ignoreCallingLock = false; if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { startingNode = ((LockImpl) callingLock).node; diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java index 849626a30b42..6500805e0527 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java @@ -114,7 +114,7 @@ public String getTimerName(String operation) { } @Override - public Lock lockTask(ZkNodeProps message, long ignored) { + public Lock lockTask(ZkNodeProps message, long ignored, List callingLockIds) { String configSetName = getTaskKey(message); if (canExecute(configSetName, message)) { markExclusiveTask(configSetName, message); diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java index 4bc13e2fe6a6..b883211a8671 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java @@ -53,7 +53,7 @@ interface Lock { * * @return null if locking is not possible. */ - Lock lockTask(ZkNodeProps message, long batchSessionId); + Lock lockTask(ZkNodeProps message, long batchSessionId, List callingLockIds); /** * @param message the message being processed diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index b44c4f51ce24..aef841c543aa 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.ID; @@ -37,6 +38,7 @@ import java.util.function.Predicate; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; +import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -326,8 +328,16 @@ public void run() { workQueue.remove(head, asyncId == null); continue; } + if (operation == null) { + log.error("Msg does not have required {} : {}", Overseer.QUEUE_OPERATION, message); + workQueue.remove(head, asyncId == null); + continue; + } + List callingLockIds = + AdminCmdContext.parseCallingLockIds(message.getStr(CALLING_LOCK_IDS_HEADER)); OverseerMessageHandler messageHandler = selector.selectOverseerMessageHandler(message); - OverseerMessageHandler.Lock lock = messageHandler.lockTask(message, batchSessionId); + OverseerMessageHandler.Lock lock = + messageHandler.lockTask(message, batchSessionId, callingLockIds); if (lock == null) { if (log.isDebugEnabled()) { log.debug("Exclusivity check failed for [{}]", message); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 02a0aa29d58c..d0310ad36f16 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -19,6 +19,8 @@ import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import java.util.Collections; +import java.util.List; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.StrUtils; @@ -86,6 +88,18 @@ public String getCallingLockIds() { return callingLockIds; } + public List getCallingLockIdList() { + return parseCallingLockIds(callingLockIds); + } + + public static List parseCallingLockIds(String callingLockIdsString) { + if (StrUtils.isBlank(callingLockIdsString)) { + return Collections.emptyList(); + } else { + return List.of(callingLockIdsString.split(",")); + } + } + public ClusterState getClusterState() { return clusterState; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index 9a7f2d6fe63e..485f149f0da9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -19,7 +19,6 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.apache.solr.cloud.DistributedCollectionLockFactory; import org.apache.solr.cloud.DistributedLock; @@ -108,12 +107,7 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } - List callingLockIdList; - if (adminCmdContext.getCallingLockIds() == null) { - callingLockIdList = Collections.emptyList(); - } else { - callingLockIdList = List.of(adminCmdContext.getCallingLockIds().split(",")); - } + List callingLockIdList = adminCmdContext.getCallingLockIdList(); // The first requested lock is a write one (on the target object for the action, depending on // lock level), then requesting read locks on "higher" levels (collection > shard > replica here diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index 8573bf637643..e642ff49d528 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -48,7 +48,6 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.TimeSource; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.logging.MDCLoggingContext; @@ -187,7 +186,7 @@ public String getTaskKey(ZkNodeProps message) { * because it happens that a lock got released). */ @Override - public Lock lockTask(ZkNodeProps message, long batchSessionId) { + public Lock lockTask(ZkNodeProps message, long batchSessionId, List callingLockIds) { if (sessionId != batchSessionId) { // this is always called in the same thread. // Each batch is supposed to have a new taskBatch @@ -196,11 +195,6 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId) { sessionId = batchSessionId; } - List callingLockIds = null; - String callingLockIdsString = message.getStr(CALLING_LOCK_IDS_HEADER); - if (StrUtils.isNotBlank(callingLockIdsString)) { - callingLockIds = List.of(callingLockIdsString.split(",")); - } return lockSession.lock( getCollectionAction(message.getStr(Overseer.QUEUE_OPERATION)), Arrays.asList( diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 08ab76b7f7df..83241b3e0508 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -26,6 +26,7 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -42,25 +43,36 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); Lock coll1Lock = - lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), null); + lockTree + .getSession() + .lock(CollectionAction.CREATE, Arrays.asList("coll1"), Collections.emptyList()); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null)); + .lock( + CollectionAction.BALANCESHARDUNIQUE, + Arrays.asList("coll1", "shard1"), + Collections.emptyList())); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null); + .lock( + CollectionAction.BALANCESHARDUNIQUE, + Arrays.asList("coll1", "shard1"), + Collections.emptyList()); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = lockTree .getSession() - .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null); + .lock( + ADDREPLICAPROP, + Arrays.asList("coll1", "shard1", "core_node2"), + Collections.emptyList()); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -83,7 +95,8 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = session.lock(operation.first(), operation.second(), null); + final Lock lock = + session.lock(operation.first(), operation.second(), Collections.emptyList()); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); From e87052e210c34041b59a6d9d6888b5fd739b30f8 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 29 Jan 2026 14:16:43 -0800 Subject: [PATCH 56/67] Make some fixes for LockTree, add tests to validate --- .../java/org/apache/solr/cloud/LockTree.java | 16 +-- .../org/apache/solr/cloud/TestLockTree.java | 100 ++++++++++++++++++ .../solr/common/params/CollectionParams.java | 3 +- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 20efff349e44..002d1333b4a1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -97,18 +97,18 @@ public Lock lock( // If a callingLockId was passed in, validate it with the current lock path, and only start // locking below the calling lock Lock callingLock = callingLockIds.isEmpty() ? null : allLocks.get(callingLockIds.getLast()); - boolean ignoreCallingLock = false; + boolean reuseCurrentLock = false; if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { startingNode = ((LockImpl) callingLock).node; startingSession = startingSession.find(startingNode.level.getHeight(), path); if (startingSession == null) { startingSession = root; } - ignoreCallingLock = true; + reuseCurrentLock = true; } synchronized (LockTree.this) { if (startingSession.isBusy(action.lockLevel, path)) return null; - Lock lockObject = startingNode.lock(action.lockLevel, path, ignoreCallingLock); + Lock lockObject = startingNode.lock(action.lockLevel, path, reuseCurrentLock); if (lockObject == null) { startingSession.markBusy(action.lockLevel, path); } else { @@ -207,11 +207,13 @@ void unlock(LockImpl lockObject) { } } - Lock lock(LockLevel lockLevel, List path, boolean ignoreCurrentLock) { - if (myLock != null && !ignoreCurrentLock) - return null; // I'm already locked. no need to go any further + Lock lock(LockLevel lockLevel, List path, boolean reuseCurrentLock) { if (lockLevel == level) { // lock is supposed to be acquired at this level + if (myLock != null && reuseCurrentLock) { + // I am already locked, and I want to be re-used + return myLock; + } // If I am locked or any of my children or grandchildren are locked // it is not possible to acquire a lock if (isLocked()) return null; @@ -226,7 +228,7 @@ Lock lock(LockLevel lockLevel, List path, boolean ignoreCurrentLock) { } boolean validateSubpath(int lockLevel, List path) { - return level.getHeight() < lockLevel + return level.getHeight() <= lockLevel && (level.getHeight() == 0 || name.equals(path.get(level.getHeight() - 1))) && (mom == null || mom.validateSubpath(lockLevel, path)); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 83241b3e0508..bb752dbeed9e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -122,6 +122,106 @@ public void testLocks() throws Exception { } } + public void testCallingLockIdSubLocks() throws Exception { + LockTree lockTree = new LockTree(); + Lock coll1Lock = + lockTree + .getSession() + .lock(CollectionAction.CREATE, List.of("coll1"), Collections.emptyList()); + assertNotNull(coll1Lock); + + // Test sub-locks at the same level + assertNull( + "Should not be able to lock coll1 without using a callingLockId", + lockTree + .getSession() + .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); + Lock coll1Lock2 = + lockTree + .getSession() + .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(coll1Lock.id())); + assertNotNull(coll1Lock2); + coll1Lock2.unlock(); + + // Test locks underneath + Lock replica1Lock = + lockTree + .getSession() + .lock( + CollectionAction.ADDREPLICA, + List.of("coll1", "shard1", "replica1"), + List.of(coll1Lock.id())); + assertNotNull(replica1Lock); + assertNull( + "Should not be able to lock coll1/shard1/replica2 since our callingLockId is only coll1, not shard1", + lockTree + .getSession() + .lock( + CollectionAction.ADDREPLICA, + List.of("coll1", "shard1", "replica2"), + List.of(coll1Lock.id()))); + Lock replica2Lock = + lockTree + .getSession() + .lock( + CollectionAction.ADDREPLICA, + List.of("coll1", "shard2", "replica1"), + List.of(coll1Lock.id())); + assertNotNull(replica2Lock); + replica2Lock.unlock(); + replica1Lock.unlock(); + coll1Lock.unlock(); + + // Test difference at a higher level + Lock shard1Lock1 = + lockTree + .getSession() + .lock( + CollectionAction.INSTALLSHARDDATA, + List.of("coll1", "shard1"), + Collections.emptyList()); + assertNotNull(shard1Lock1); + Lock shard1Lock2 = + lockTree + .getSession() + .lock( + CollectionAction.INSTALLSHARDDATA, + List.of("coll2", "shard1"), + Collections.emptyList()); + assertNotNull(shard1Lock2); + assertNull( + "Should not be able to lock coll1/shard1 since our callingLockId is coll2", + lockTree + .getSession() + .lock( + CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), List.of(shard1Lock2.id()))); + Lock shard1Lock3 = + lockTree + .getSession() + .lock( + CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), List.of(shard1Lock1.id())); + assertNotNull(shard1Lock3); + shard1Lock1.unlock(); + shard1Lock2.unlock(); + shard1Lock3.unlock(); + + // Test difference at a higher level + shard1Lock1 = + lockTree + .getSession() + .lock( + CollectionAction.INSTALLSHARDDATA, + List.of("coll1", "shard1"), + Collections.emptyList()); + assertNotNull(shard1Lock1); + assertNull( + "Should not be able to lock coll1 since our callingLockId is coll1/shar1. Cannot move up", + lockTree + .getSession() + .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); + shard1Lock1.unlock(); + } + private Runnable getRunnable( List>> completedOps, Pair> operation, diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 82d9f80909b4..6ae82508df4b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -48,8 +48,7 @@ enum LockLevel { REPLICA(3, null), SHARD(2, REPLICA), COLLECTION(1, SHARD), - CLUSTER(0, COLLECTION), - BASE(-1, CLUSTER); + CLUSTER(0, COLLECTION); private final int height; private final LockLevel child; From 5a2b445051bebe42b0b1db6d2731a991c49862ef Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 29 Jan 2026 16:46:40 -0800 Subject: [PATCH 57/67] Improve tests, fix errors in locking --- .../java/org/apache/solr/cloud/LockTree.java | 18 +++++- .../org/apache/solr/cloud/TestLockTree.java | 64 +++++++++++++------ 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 002d1333b4a1..1e6e9c09bf13 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -53,8 +53,9 @@ private class LockImpl implements Lock { @Override public void unlock() { synchronized (LockTree.this) { - node.unlock(this); - allLocks.remove(id); + if (node.unlock(this)) { + allLocks.remove(id); + } } } @@ -184,6 +185,7 @@ private class Node { final String name; final Node mom; final LockLevel level; + int refCount = 0; HashMap children = new HashMap<>(); LockImpl myLock; @@ -200,23 +202,33 @@ boolean isLocked() { return false; } - void unlock(LockImpl lockObject) { + boolean unlock(LockImpl lockObject) { + if (--refCount > 0) { + return false; + } if (myLock == lockObject) myLock = null; else { log.info("Unlocked multiple times : {}", lockObject); } + return true; } Lock lock(LockLevel lockLevel, List path, boolean reuseCurrentLock) { + if (myLock != null && !reuseCurrentLock) { + // I'm already locked. no need to go any further + return null; + } if (lockLevel == level) { // lock is supposed to be acquired at this level if (myLock != null && reuseCurrentLock) { // I am already locked, and I want to be re-used + refCount++; return myLock; } // If I am locked or any of my children or grandchildren are locked // it is not possible to acquire a lock if (isLocked()) return null; + refCount++; return myLock = new LockImpl(this); } else { String childName = path.get(level.getHeight()); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index bb752dbeed9e..ea3ac296859a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -144,29 +144,51 @@ public void testCallingLockIdSubLocks() throws Exception { coll1Lock2.unlock(); // Test locks underneath + Lock shard1Lock = + lockTree + .getSession() + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id())); + assertNotNull(shard1Lock); + assertNull( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1", + lockTree + .getSession() + .lock( + CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id()))); + Lock shard2Lock = + lockTree + .getSession() + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard2"), List.of(coll1Lock.id())); + assertNotNull(shard2Lock); + shard2Lock.unlock(); + shard1Lock.unlock(); + + // Test locks 2 underneath Lock replica1Lock = lockTree .getSession() .lock( - CollectionAction.ADDREPLICA, - List.of("coll1", "shard1", "replica1"), - List.of(coll1Lock.id())); - assertNotNull(replica1Lock); + MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica1"), List.of(coll1Lock.id())); assertNull( - "Should not be able to lock coll1/shard1/replica2 since our callingLockId is only coll1, not shard1", + "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", lockTree .getSession() .lock( - CollectionAction.ADDREPLICA, - List.of("coll1", "shard1", "replica2"), + CollectionAction.MOCK_REPLICA_TASK, + List.of("coll1", "shard1", "replica1"), List.of(coll1Lock.id()))); + assertNull( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", + lockTree + .getSession() + .lock( + CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id()))); + assertNotNull(replica1Lock); Lock replica2Lock = lockTree .getSession() .lock( - CollectionAction.ADDREPLICA, - List.of("coll1", "shard2", "replica1"), - List.of(coll1Lock.id())); + MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica2"), List.of(coll1Lock.id())); assertNotNull(replica2Lock); replica2Lock.unlock(); replica1Lock.unlock(); @@ -201,24 +223,28 @@ public void testCallingLockIdSubLocks() throws Exception { .lock( CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), List.of(shard1Lock1.id())); assertNotNull(shard1Lock3); - shard1Lock1.unlock(); shard1Lock2.unlock(); shard1Lock3.unlock(); // Test difference at a higher level - shard1Lock1 = + assertNull( + "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", lockTree .getSession() - .lock( - CollectionAction.INSTALLSHARDDATA, - List.of("coll1", "shard1"), - Collections.emptyList()); - assertNotNull(shard1Lock1); + .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); assertNull( - "Should not be able to lock coll1 since our callingLockId is coll1/shar1. Cannot move up", + "Should not be able to lock coll1 since our callingLockId is coll1/shard1. Cannot move up", lockTree .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); + .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(shard1Lock1.id()))); + + // Test an unrelated lock + Lock coll2Lock = + lockTree + .getSession() + .lock(CollectionAction.CREATE, List.of("coll2"), List.of(shard1Lock1.id())); + assertNotNull(coll2Lock); + coll2Lock.unlock(); shard1Lock1.unlock(); } From 1e4767af4f4a18cd4c06b060ce020b5c3eae1ba1 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 29 Jan 2026 16:46:59 -0800 Subject: [PATCH 58/67] Add tests for Distributed locking. Make fixes --- .../apache/solr/cloud/ZkDistributedLock.java | 2 +- .../api/collections/AdminCmdContext.java | 5 + .../collections/CollectionApiLockFactory.java | 5 - .../collections/CollectionApiLockingTest.java | 202 ++++++++++++++++++ 4 files changed, 208 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index 8517661dba79..28ffa7d58a65 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -86,7 +86,7 @@ boolean isBlockedByNodeType(String otherLockName) { @Override boolean canMirrorLock(String lockId) { // Only another Write lock can be mirrored - int lockTypeSuffixIndex = lockId.indexOf(LOCK_PREFIX_SUFFIX) - 1; + int lockTypeSuffixIndex = lockId.lastIndexOf(LOCK_PREFIX_SUFFIX) - 1; if (lockTypeSuffixIndex < 0) { return false; } else { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index d0310ad36f16..43851438dedd 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -72,6 +72,11 @@ public void setCallingLockIds(String callingLockIds) { regenerateSubRequestCallingLockIds(); } + public AdminCmdContext withCallingLockIds(String callingLockIds) { + setCallingLockIds(callingLockIds); + return this; + } + private void regenerateSubRequestCallingLockIds() { if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { subRequestCallingLockIds += "," + lockId; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index 485f149f0da9..a6aa4057f9cf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -120,11 +120,6 @@ DistributedMultiLock createCollectionApiLock( }; List locks = new ArrayList<>(iterationOrder.length); for (CollectionParams.LockLevel level : iterationOrder) { - // We do not want to re-lock from the parent lock level - // TODO: Fix - // if (calledFromLockLevel.isHigherOrEqual(level)) { - // continue; - // } // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index 7f6f7e7daf42..6dcf5e4311a2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -17,12 +17,14 @@ package org.apache.solr.cloud.api.collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.DistributedMultiLock; import org.apache.solr.cloud.ZkDistributedCollectionLockFactory; import org.apache.solr.cloud.ZkTestServer; +import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CollectionParams; import org.junit.Test; @@ -54,6 +56,7 @@ public void monothreadedApiLockTests() throws Exception { monothreadedTests(apiLockFactory); multithreadedTests(apiLockFactory); + testCallingLockIdSubLocks(apiLockFactory); } } finally { server.shutdown(); @@ -229,4 +232,203 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw assertEquals( "we should have been notified that replica lock was acquired", 0, latch.getCount()); } + + public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) + throws Exception { + DistributedMultiLock coll1Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), "coll1", null, null); + assertEquals("Wrong number of internalLocks", 1, coll1Lock.getCountInternalLocks()); + assertTrue("Lock should be acquired", coll1Lock.isAcquired()); + + // Test sub-locks at the same level + DistributedMultiLock coll1Lock2 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); + assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1 without using a callingLockId", coll1Lock2.isAcquired()); + coll1Lock2.release(); + coll1Lock2 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.RELOAD) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + null, + null); + assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); + assertTrue("Should be able to lock coll1 when using a callingLockId", coll1Lock2.isAcquired()); + coll1Lock2.release(); + + // Test locks underneath + DistributedMultiLock shard1Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll1/shard1 when using a callingLockId on coll1", + shard1Lock.isAcquired()); + DistributedMultiLock shard1Lock2 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock2.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, since shard1 is already locked", + shard1Lock2.isAcquired()); + shard1Lock2.release(); + DistributedMultiLock shard2Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard2", + null); + assertEquals("Wrong number of internalLocks", 2, shard2Lock.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll1/shard2 when using a callingLockId on coll1, since shard2 has not been locked yet", + shard2Lock.isAcquired()); + shard2Lock.release(); + shard1Lock.release(); + + // Test locks 2 underneath + DistributedMultiLock replica1Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + "replica1"); + assertEquals("Wrong number of internalLocks", 3, replica1Lock.getCountInternalLocks()); + assertTrue( + "Should be able to lock shard1/replica1 when using a callingLockId on coll1", + replica1Lock.isAcquired()); + DistributedMultiLock replica2Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + "replica1"); + assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1/shard1/replica2 since our callingLockId is only coll1, not replica1, which is already locked", + replica2Lock.isAcquired()); + replica2Lock.release(); + shard1Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", + shard1Lock.isAcquired()); + shard1Lock.release(); + replica2Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) + .withCallingLockIds(coll1Lock.getLockId()), + "coll1", + "shard1", + "replica2"); + assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); + assertTrue( + "Should be able to lock shard2/replica2 when using a callingLockId on coll1, since shard2 has not been locked yet", + replica2Lock.isAcquired()); + replica2Lock.release(); + replica1Lock.release(); + coll1Lock.release(); + + // Test difference at a higher level + DistributedMultiLock shard1Lock1 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock1.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll1/shard1 when not using a callingLockId since shard1 has not been locked yet", + shard1Lock1.isAcquired()); + shard1Lock2 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA), + "coll2", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock2.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll2/shard1 when not using a callingLockId since shard1 has not been locked yet for coll2", + shard1Lock2.isAcquired()); + DistributedMultiLock shard1Lock3 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) + .withCallingLockIds(shard1Lock2.getLockId()), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock3.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1/shard1 since our callingLockId is coll2", + shard1Lock3.isAcquired()); + shard1Lock3 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) + .withCallingLockIds(shard1Lock1.getLockId()), + "coll1", + "shard1", + null); + assertEquals("Wrong number of internalLocks", 2, shard1Lock3.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll1/shard1 since our callingLockId is coll1/shard1", + shard1Lock3.isAcquired()); + shard1Lock2.release(); + shard1Lock3.release(); + + // Test difference at a higher level + coll1Lock2 = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); + assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); + assertFalse( + "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", + coll1Lock2.isAcquired()); + coll1Lock2.release(); + assertExceptionThrownWithMessageContaining( + SolrException.class, + List.of("Cannot mirror lock"), + () -> + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.RELOAD) + .withCallingLockIds(shard1Lock1.getLockId()), + "coll1", + null, + null)); + shard1Lock3.release(); + + // Test an unrelated lock + DistributedMultiLock coll2Lock = + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.CREATE) + .withCallingLockIds(shard1Lock1.getLockId()), + "coll2", + null, + null); + assertEquals("Wrong number of internalLocks", 1, coll2Lock.getCountInternalLocks()); + assertTrue( + "Should be able to lock coll2even though callingLockId is coll1 and unrelated", + coll2Lock.isAcquired()); + coll2Lock.release(); + shard1Lock1.release(); + } } From ea3aa3f17f34fe32a0b21704f58287bf0b839309 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 29 Jan 2026 17:05:05 -0800 Subject: [PATCH 59/67] Add back in ShardRequest.coreName --- .../java/org/apache/solr/handler/component/ShardRequest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java index 5222b38abee3..ecaee01c7fa8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java @@ -60,6 +60,9 @@ public class ShardRequest { /** may be null */ public String coreNodeName; + /** may be null */ + public String coreName; + /** may be null */ public Map headers; From 1bf192cd5e499e1bd96fbc9768e559199ae08f64 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 24 Mar 2026 12:52:50 -0700 Subject: [PATCH 60/67] Fix precommit issues --- .../ZkDistributedConfigSetLockFactory.java | 4 +- .../api/collections/AdminCmdContext.java | 3 +- .../org/apache/solr/cloud/TestLockTree.java | 44 +++---------- .../solr/cloud/ZkDistributedLockTest.java | 62 ++++--------------- 4 files changed, 24 insertions(+), 89 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index c21d8bdeb78c..896091b68c68 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -17,7 +17,7 @@ package org.apache.solr.cloud; -import java.util.Collections; +import java.util.List; import java.util.Objects; import org.apache.solr.common.cloud.SolrZkClient; @@ -41,7 +41,7 @@ public DistributedLock createLock(boolean isWriteLock, String configSetName) { Objects.requireNonNull(configSetName, "configSetName can't be null"); String lockPath = getLockPath(configSetName); - return doCreateLock(isWriteLock, lockPath, Collections.emptyList()); + return doCreateLock(isWriteLock, lockPath, List.of()); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 43851438dedd..69bca970db4a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; -import java.util.Collections; import java.util.List; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; @@ -99,7 +98,7 @@ public List getCallingLockIdList() { public static List parseCallingLockIds(String callingLockIdsString) { if (StrUtils.isBlank(callingLockIdsString)) { - return Collections.emptyList(); + return List.of(); } else { return List.of(callingLockIdsString.split(",")); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index ea3ac296859a..958211b4568f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -26,7 +26,6 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -43,36 +42,26 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); Lock coll1Lock = - lockTree - .getSession() - .lock(CollectionAction.CREATE, Arrays.asList("coll1"), Collections.emptyList()); + lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), List.of()); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() .lock( - CollectionAction.BALANCESHARDUNIQUE, - Arrays.asList("coll1", "shard1"), - Collections.emptyList())); + CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), List.of())); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock( - CollectionAction.BALANCESHARDUNIQUE, - Arrays.asList("coll1", "shard1"), - Collections.emptyList()); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), List.of()); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = lockTree .getSession() - .lock( - ADDREPLICAPROP, - Arrays.asList("coll1", "shard1", "core_node2"), - Collections.emptyList()); + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), List.of()); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -95,8 +84,7 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = - session.lock(operation.first(), operation.second(), Collections.emptyList()); + final Lock lock = session.lock(operation.first(), operation.second(), List.of()); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); @@ -125,17 +113,13 @@ public void testLocks() throws Exception { public void testCallingLockIdSubLocks() throws Exception { LockTree lockTree = new LockTree(); Lock coll1Lock = - lockTree - .getSession() - .lock(CollectionAction.CREATE, List.of("coll1"), Collections.emptyList()); + lockTree.getSession().lock(CollectionAction.CREATE, List.of("coll1"), List.of()); assertNotNull(coll1Lock); // Test sub-locks at the same level assertNull( "Should not be able to lock coll1 without using a callingLockId", - lockTree - .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); + lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), List.of())); Lock coll1Lock2 = lockTree .getSession() @@ -198,18 +182,12 @@ public void testCallingLockIdSubLocks() throws Exception { Lock shard1Lock1 = lockTree .getSession() - .lock( - CollectionAction.INSTALLSHARDDATA, - List.of("coll1", "shard1"), - Collections.emptyList()); + .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll1", "shard1"), List.of()); assertNotNull(shard1Lock1); Lock shard1Lock2 = lockTree .getSession() - .lock( - CollectionAction.INSTALLSHARDDATA, - List.of("coll2", "shard1"), - Collections.emptyList()); + .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll2", "shard1"), List.of()); assertNotNull(shard1Lock2); assertNull( "Should not be able to lock coll1/shard1 since our callingLockId is coll2", @@ -229,9 +207,7 @@ public void testCallingLockIdSubLocks() throws Exception { // Test difference at a higher level assertNull( "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", - lockTree - .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), Collections.emptyList())); + lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), List.of())); assertNull( "Should not be able to lock coll1 since our callingLockId is coll1/shard1. Cannot move up", lockTree diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index c6b132ccb94d..ce9e9edb78a6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,7 +17,7 @@ package org.apache.solr.cloud; -import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; @@ -76,32 +76,17 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Collection level locks DistributedLock collRL1 = factory.createLock( - false, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); DistributedLock collRL2 = factory.createLock( - false, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); DistributedLock collWL3 = factory.createLock( - true, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertFalse( "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); @@ -116,12 +101,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock collRL4 = factory.createLock( - false, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired()); @@ -131,24 +111,14 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // should see no impact. DistributedLock shardWL5 = factory.createLock( - true, - CollectionParams.LockLevel.SHARD, - COLLECTION_NAME, - SHARD_NAME, - null, - Collections.emptyList()); + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, List.of()); assertTrue( "shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired()); DistributedLock shardWL6 = factory.createLock( - true, - CollectionParams.LockLevel.SHARD, - COLLECTION_NAME, - SHARD_NAME, - null, - Collections.emptyList()); + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, List.of()); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired()); @@ -161,7 +131,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor COLLECTION_NAME, SHARD_NAME, REPLICA_NAME, - Collections.emptyList()); + List.of()); assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); DistributedLock replicaWL8 = @@ -171,7 +141,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor COLLECTION_NAME, SHARD_NAME, REPLICA_NAME, - Collections.emptyList()); + List.of()); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired()); @@ -205,23 +175,13 @@ private void multithreadedCollectionTests(DistributedCollectionLockFactory facto // Acquiring right away a read lock DistributedLock readLock = factory.createLock( - false, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertTrue("readLock should have been acquired", readLock.isAcquired()); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( - true, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - Collections.emptyList()); + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); // Wait for acquisition of the write lock on another thread (and be notified via a latch) From a9b4cdb09cec29825712d1ff9526108f03aee804 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 14 Apr 2026 13:03:00 -0700 Subject: [PATCH 61/67] Make sure that locking must be top-down, and cannot introduce dead-locks --- .../DistributedCollectionLockFactory.java | 4 +- .../java/org/apache/solr/cloud/LockTree.java | 24 +++++++--- .../ZkDistributedCollectionLockFactory.java | 5 +- .../ZkDistributedConfigSetLockFactory.java | 3 +- .../apache/solr/cloud/ZkDistributedLock.java | 27 +++++++---- .../solr/cloud/ZkDistributedLockFactory.java | 21 ++++---- .../collections/CollectionApiLockFactory.java | 25 ++++++---- .../org/apache/solr/cloud/TestLockTree.java | 48 +++++++++++++------ .../solr/cloud/ZkDistributedLockTest.java | 21 ++++---- .../collections/CollectionApiLockingTest.java | 46 +++++++++--------- .../solr/common/params/CollectionParams.java | 6 ++- 11 files changed, 138 insertions(+), 92 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java index ec6fa179aeb5..f2ab771232ed 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud; -import java.util.List; import org.apache.solr.cloud.api.collections.CollectionApiLockFactory; import org.apache.solr.common.params.CollectionParams; @@ -55,6 +54,7 @@ public interface DistributedCollectionLockFactory { * @param replicaName is ignored and can be {@code null} if {@code level} is {@link * org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION} or {@link * org.apache.solr.common.params.CollectionParams.LockLevel#SHARD} + * @param callingLockId the lockId from the caller that should be mirrored by this lock * @return a lock instance that must be {@link DistributedLock#release()}'ed in a {@code finally}, * regardless of the lock having been acquired or not. */ @@ -64,5 +64,5 @@ DistributedLock createLock( String collName, String shardId, String replicaName, - List callingLockIds); + String callingLockId); } diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index 1e6e9c09bf13..ae727ad523ff 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -21,9 +21,11 @@ import java.util.ArrayDeque; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import org.apache.solr.cloud.OverseerMessageHandler.Lock; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.LockLevel; import org.apache.solr.common.util.StrUtils; @@ -99,13 +101,23 @@ public Lock lock( // locking below the calling lock Lock callingLock = callingLockIds.isEmpty() ? null : allLocks.get(callingLockIds.getLast()); boolean reuseCurrentLock = false; - if (callingLock != null && callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { - startingNode = ((LockImpl) callingLock).node; - startingSession = startingSession.find(startingNode.level.getHeight(), path); - if (startingSession == null) { - startingSession = root; + if (callingLock != null) { + if (callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { + startingNode = ((LockImpl) callingLock).node; + startingSession = startingSession.find(startingNode.level.getHeight(), path); + if (startingSession == null) { + startingSession = root; + } + reuseCurrentLock = true; + } else { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + String.format( + Locale.ROOT, + "Cannot lock an action under a different path than the calling action. Calling action lock path: %s, Requested action lock path: %s", + callingLock, + String.join("/", path))); } - reuseCurrentLock = true; } synchronized (LockTree.this) { if (startingSession.isBusy(action.lockLevel, path)) return null; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java index 2e3e5c4b8d73..098545b6afcf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud; -import java.util.List; import java.util.Objects; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; @@ -46,7 +45,7 @@ public DistributedLock createLock( String collName, String shardId, String replicaName, - List callingLockIds) { + String lockIdToMirror) { Objects.requireNonNull(collName, "collName can't be null"); if (level != CollectionParams.LockLevel.COLLECTION) { Objects.requireNonNull( @@ -59,7 +58,7 @@ public DistributedLock createLock( String lockPath = getLockPath(level, collName, shardId, replicaName); - return doCreateLock(isWriteLock, lockPath, callingLockIds); + return doCreateLock(isWriteLock, lockPath, lockIdToMirror); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index 896091b68c68..3458ea322153 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud; -import java.util.List; import java.util.Objects; import org.apache.solr.common.cloud.SolrZkClient; @@ -41,7 +40,7 @@ public DistributedLock createLock(boolean isWriteLock, String configSetName) { Objects.requireNonNull(configSetName, "configSetName can't be null"); String lockPath = getLockPath(configSetName); - return doCreateLock(isWriteLock, lockPath, List.of()); + return doCreateLock(isWriteLock, lockPath, null); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index 28ffa7d58a65..30b70f66b82b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -24,6 +24,7 @@ import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -52,9 +53,9 @@ abstract class ZkDistributedLock implements DistributedLock { /** Read lock. */ static class Read extends ZkDistributedLock { - protected Read(SolrZkClient zkClient, String lockPath, String mirroredLockId) + protected Read(SolrZkClient zkClient, String lockPath, String lockIdToMirror) throws KeeperException, InterruptedException { - super(zkClient, lockPath, READ_LOCK_PREFIX, mirroredLockId); + super(zkClient, lockPath, READ_LOCK_PREFIX, lockIdToMirror); } @Override @@ -72,9 +73,9 @@ boolean canMirrorLock(String lockId) { /** Write lock. */ static class Write extends ZkDistributedLock { - protected Write(SolrZkClient zkClient, String lockPath, String mirroredLockId) + protected Write(SolrZkClient zkClient, String lockPath, String lockIdToMirror) throws KeeperException, InterruptedException { - super(zkClient, lockPath, WRITE_LOCK_PREFIX, mirroredLockId); + super(zkClient, lockPath, WRITE_LOCK_PREFIX, lockIdToMirror); } @Override @@ -103,14 +104,14 @@ boolean canMirrorLock(String lockId) { protected final boolean mirrored; protected ZkDistributedLock( - SolrZkClient zkClient, String lockDir, String lockNodePrefix, String mirroredLockId) + SolrZkClient zkClient, String lockDir, String lockNodePrefix, String lockIdToMirror) throws KeeperException, InterruptedException { this.zkClient = zkClient; this.lockDir = lockDir; // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually // call release() or we block others. - if (mirroredLockId == null || !mirroredLockId.startsWith(lockDir)) { + if (StrUtils.isBlank(lockIdToMirror)) { lockNode = zkClient.create( lockDir @@ -121,12 +122,20 @@ protected ZkDistributedLock( mirrored = false; } else { - if (!canMirrorLock(mirroredLockId)) { + if (!lockIdToMirror.startsWith(lockDir)) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Cannot mirror lock " + mirroredLockId + " with given lockPrefix: " + lockNodePrefix); + "Requested lock with path: " + + lockDir + + " cannot mirror the callingLock with id: " + + lockIdToMirror); } - lockNode = mirroredLockId; + if (!canMirrorLock(lockIdToMirror)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Cannot mirror lock " + lockIdToMirror + " with given lockPrefix: " + lockNodePrefix); + } + lockNode = lockIdToMirror; mirrored = true; } sequence = getSequenceFromNodename(lockNode); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index f7935a3c0115..969e65a6aaa3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,10 +17,10 @@ package org.apache.solr.cloud; -import java.util.List; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -35,19 +35,18 @@ abstract class ZkDistributedLockFactory { } protected DistributedLock doCreateLock( - boolean isWriteLock, String lockPath, List callingLockIds) { + boolean isWriteLock, String lockPath, String lockIdToMirror) { try { - // TODO optimize by first attempting to create the ZkDistributedLock without calling - // makeLockPath() and only call it if the lock creation fails. This will be less costly on - // high contention (and slightly more on low contention) - makeLockPath(lockPath); - // If a callingLock has the same lockPath as this lock, we just want to mirror that lock - String mirroredLockId = - callingLockIds.stream().filter(p -> p.startsWith(lockPath)).findFirst().orElse(null); + if (StrUtils.isBlank(lockIdToMirror)) { + // TODO optimize by first attempting to create the ZkDistributedLock without calling + // makeLockPath() and only call it if the lock creation fails. This will be less costly on + // high contention (and slightly more on low contention) + makeLockPath(lockPath); + } return isWriteLock - ? new ZkDistributedLock.Write(zkClient, lockPath, mirroredLockId) - : new ZkDistributedLock.Read(zkClient, lockPath, mirroredLockId); + ? new ZkDistributedLock.Write(zkClient, lockPath, lockIdToMirror) + : new ZkDistributedLock.Read(zkClient, lockPath, lockIdToMirror); } catch (KeeperException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index a6aa4057f9cf..eff4d37be7b1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -109,25 +109,34 @@ DistributedMultiLock createCollectionApiLock( List callingLockIdList = adminCmdContext.getCallingLockIdList(); - // The first requested lock is a write one (on the target object for the action, depending on - // lock level), then requesting read locks on "higher" levels (collection > shard > replica here - // for the level. Note LockLevel "height" is other way around). - boolean requestWriteLock = true; final CollectionParams.LockLevel[] iterationOrder = { - CollectionParams.LockLevel.REPLICA, + CollectionParams.LockLevel.COLLECTION, CollectionParams.LockLevel.SHARD, - CollectionParams.LockLevel.COLLECTION + CollectionParams.LockLevel.REPLICA }; List locks = new ArrayList<>(iterationOrder.length); + int lockLevelCount = 0; + for (CollectionParams.LockLevel level : iterationOrder) { // This comparison is based on the LockLevel height value that classifies replica > shard > // collection. if (lockLevel.isHigherOrEqual(level)) { + // The last requested lock is either a write or read one (on the target object for the + // action, depending on lock level) depending on what the action is. All "higher" levels of + // locks are reads (collection > shard > replica here for the level. Note LockLevel "height" + // is other way around). + boolean requestWriteLock = lockLevel.isEqual(level) && adminCmdContext.getAction().isWrite; + // Find the matching callingLockId for this level, if it was provided. All levels must be + // provided in order by the caller, so when we run out of callingLockIds, we are done + // mirroring and should start getting new locks. + String callingLockIdMatch = null; + if (lockLevelCount < callingLockIdList.size()) { + callingLockIdMatch = callingLockIdList.get(lockLevelCount++); + } DistributedLock lock = lockFactory.createLock( - requestWriteLock, level, collName, shardId, replicaName, callingLockIdList); + requestWriteLock, level, collName, shardId, replicaName, callingLockIdMatch); locks.add(lock); - requestWriteLock = false; } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 958211b4568f..79402cb48eba 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -31,6 +31,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.OverseerMessageHandler.Lock; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.util.Pair; import org.slf4j.Logger; @@ -189,12 +190,16 @@ public void testCallingLockIdSubLocks() throws Exception { .getSession() .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll2", "shard1"), List.of()); assertNotNull(shard1Lock2); - assertNull( + assertThrows( "Should not be able to lock coll1/shard1 since our callingLockId is coll2", - lockTree - .getSession() - .lock( - CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), List.of(shard1Lock2.id()))); + SolrException.class, + () -> + lockTree + .getSession() + .lock( + CollectionAction.SYNCSHARD, + List.of("coll1", "shard1"), + List.of(shard1Lock2.id()))); Lock shard1Lock3 = lockTree .getSession() @@ -208,19 +213,32 @@ public void testCallingLockIdSubLocks() throws Exception { assertNull( "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), List.of())); - assertNull( + assertThrows( "Should not be able to lock coll1 since our callingLockId is coll1/shard1. Cannot move up", - lockTree - .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(shard1Lock1.id()))); + SolrException.class, + () -> + lockTree + .getSession() + .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(shard1Lock1.id()))); // Test an unrelated lock - Lock coll2Lock = - lockTree - .getSession() - .lock(CollectionAction.CREATE, List.of("coll2"), List.of(shard1Lock1.id())); - assertNotNull(coll2Lock); - coll2Lock.unlock(); + assertThrows( + "Should not be able to lock coll2 since our callingLockId is coll1/shard1. Cannot lock an unrelated resource", + SolrException.class, + () -> + lockTree + .getSession() + .lock(CollectionAction.CREATE, List.of("coll2"), List.of(shard1Lock1.id()))); + assertThrows( + "Should not be able to lock coll1/shard2 since our callingLockId is coll1/shard1. Cannot lock an unrelated resource", + SolrException.class, + () -> + lockTree + .getSession() + .lock( + CollectionAction.SYNCSHARD, + List.of("coll1", "shard2"), + List.of(shard1Lock1.id()))); shard1Lock1.unlock(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index ce9e9edb78a6..e31f042bd090 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud; -import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; @@ -76,17 +75,17 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // Collection level locks DistributedLock collRL1 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); DistributedLock collRL2 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); DistributedLock collWL3 = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertFalse( "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); @@ -101,7 +100,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock collRL4 = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired()); @@ -111,14 +110,14 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor // should see no impact. DistributedLock shardWL5 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, List.of()); + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); assertTrue( "shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired()); DistributedLock shardWL6 = factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, List.of()); + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired()); @@ -131,7 +130,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor COLLECTION_NAME, SHARD_NAME, REPLICA_NAME, - List.of()); + null); assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); DistributedLock replicaWL8 = @@ -141,7 +140,7 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor COLLECTION_NAME, SHARD_NAME, REPLICA_NAME, - List.of()); + null); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired()); @@ -175,13 +174,13 @@ private void multithreadedCollectionTests(DistributedCollectionLockFactory facto // Acquiring right away a read lock DistributedLock readLock = factory.createLock( - false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertTrue("readLock should have been acquired", readLock.isAcquired()); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, List.of()); + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); // Wait for acquisition of the write lock on another thread (and be notified via a latch) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index 6dcf5e4311a2..e850b9df14dd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -319,7 +319,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "replica1"); assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); assertFalse( - "Should not be able to lock coll1/shard1/replica2 since our callingLockId is only coll1, not replica1, which is already locked", + "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", replica2Lock.isAcquired()); replica2Lock.release(); shard1Lock = @@ -370,18 +370,18 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) assertTrue( "Should be able to lock coll2/shard1 when not using a callingLockId since shard1 has not been locked yet for coll2", shard1Lock2.isAcquired()); - DistributedMultiLock shard1Lock3 = - apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) - .withCallingLockIds(shard1Lock2.getLockId()), - "coll1", - "shard1", - null); - assertEquals("Wrong number of internalLocks", 2, shard1Lock3.getCountInternalLocks()); - assertFalse( + String badLockId = shard1Lock2.getLockId(); + assertThrows( "Should not be able to lock coll1/shard1 since our callingLockId is coll2", - shard1Lock3.isAcquired()); - shard1Lock3 = + SolrException.class, + () -> + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) + .withCallingLockIds(badLockId), + "coll1", + "shard1", + null)); + DistributedMultiLock shard1Lock3 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) .withCallingLockIds(shard1Lock1.getLockId()), @@ -417,18 +417,16 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) shard1Lock3.release(); // Test an unrelated lock - DistributedMultiLock coll2Lock = - apiLockingHelper.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.CREATE) - .withCallingLockIds(shard1Lock1.getLockId()), - "coll2", - null, - null); - assertEquals("Wrong number of internalLocks", 1, coll2Lock.getCountInternalLocks()); - assertTrue( - "Should be able to lock coll2even though callingLockId is coll1 and unrelated", - coll2Lock.isAcquired()); - coll2Lock.release(); + assertThrows( + "Should not be able to lock coll2even since callingLockId is coll1 and unrelated", + SolrException.class, + () -> + apiLockingHelper.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.CREATE) + .withCallingLockIds(shard1Lock1.getLockId()), + "coll2", + null, + null)); shard1Lock1.release(); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 6ae82508df4b..a066ff1e89b3 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -69,6 +69,10 @@ public int getHeight() { public boolean isHigherOrEqual(LockLevel that) { return height >= that.height; } + + public boolean isEqual(LockLevel that) { + return height == that.height; + } } /** @@ -136,7 +140,7 @@ enum CollectionAction { // TODO when we have a node level lock use it here BALANCE_REPLICAS(true, LockLevel.NONE), DELETENODE(true, LockLevel.NONE), - MOCK_REPLICA_TASK(false, LockLevel.REPLICA), + MOCK_REPLICA_TASK(true, LockLevel.REPLICA), NONE(false, LockLevel.NONE), // TODO: not implemented yet MERGESHARDS(true, LockLevel.SHARD), From 76ea4fd5f2e41de88e3b1c1bbcfdff1c7144c2a6 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 14 Apr 2026 13:40:33 -0700 Subject: [PATCH 62/67] Only pass relevant lockId, not all lockIds --- .../java/org/apache/solr/api/V2HttpCall.java | 8 +-- .../solr/cloud/DistributedMultiLock.java | 8 +++ .../java/org/apache/solr/cloud/LockTree.java | 6 +- .../OverseerConfigSetMessageHandler.java | 2 +- .../solr/cloud/OverseerMessageHandler.java | 2 +- .../solr/cloud/OverseerTaskProcessor.java | 8 +-- .../api/collections/AdminCmdContext.java | 61 ++++--------------- .../collections/CollectionApiLockFactory.java | 3 +- .../collections/CollectionCommandContext.java | 2 +- .../collections/CollectionHandlingUtils.java | 11 ++-- ...butedCollectionConfigSetCommandRunner.java | 2 +- .../OverseerCollectionMessageHandler.java | 16 ++--- .../handler/admin/CollectionsHandler.java | 6 +- .../solr/handler/admin/api/AdminAPIBase.java | 7 +-- .../org/apache/solr/servlet/HttpSolrCall.java | 6 +- .../org/apache/solr/cloud/TestLockTree.java | 61 +++++++------------ .../collections/CollectionApiLockingTest.java | 24 ++++---- .../common/params/CollectionAdminParams.java | 2 +- 18 files changed, 93 insertions(+), 142 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 6bbef9177244..df88f992e8e5 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -18,7 +18,7 @@ package org.apache.solr.api; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.servlet.HttpSolrCall.Action.ADMIN; import static org.apache.solr.servlet.HttpSolrCall.Action.ADMIN_OR_REMOTEPROXY; import static org.apache.solr.servlet.HttpSolrCall.Action.PROCESS; @@ -214,9 +214,9 @@ private void initAdminRequest(String path) throws Exception { requestType = AuthorizationContext.RequestType.ADMIN; action = ADMIN; - String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); - if (callingLockIds != null && !callingLockIds.isBlank()) { - solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + String callingLockId = req.getHeader(CALLING_LOCK_ID_HEADER); + if (callingLockId != null && !callingLockId.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_ID_HEADER, callingLockId); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index ef74b9862c86..97be2293399c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.StrUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +81,13 @@ public String getLockId() { return locks.stream().map(DistributedLock::getLockId).collect(Collectors.joining(",")); } + public static List splitLockIds(String lockIds) { + if (StrUtils.isBlank(lockIds)) { + return List.of(); + } + return List.of(lockIds.split(",")); + } + @VisibleForTesting public int getCountInternalLocks() { return locks.size(); diff --git a/solr/core/src/java/org/apache/solr/cloud/LockTree.java b/solr/core/src/java/org/apache/solr/cloud/LockTree.java index ae727ad523ff..6bb432abd8e8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LockTree.java +++ b/solr/core/src/java/org/apache/solr/cloud/LockTree.java @@ -91,15 +91,15 @@ public class Session { private SessionNode root = new SessionNode(LockLevel.CLUSTER); public Lock lock( - CollectionParams.CollectionAction action, List path, List callingLockIds) { + CollectionParams.CollectionAction action, List path, String callingLockId) { if (action.lockLevel == LockLevel.NONE) return FREELOCK; - log.debug("Calling lock level: {}", callingLockIds); Node startingNode = LockTree.this.root; SessionNode startingSession = root; // If a callingLockId was passed in, validate it with the current lock path, and only start // locking below the calling lock - Lock callingLock = callingLockIds.isEmpty() ? null : allLocks.get(callingLockIds.getLast()); + Lock callingLock = StrUtils.isBlank(callingLockId) ? null : allLocks.get(callingLockId); + log.debug("Calling lock id: {}, level: {}", callingLockId, callingLock); boolean reuseCurrentLock = false; if (callingLock != null) { if (callingLock.validateSubpath(action.lockLevel.getHeight(), path)) { diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java index 6500805e0527..750fc62effa1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java @@ -114,7 +114,7 @@ public String getTimerName(String operation) { } @Override - public Lock lockTask(ZkNodeProps message, long ignored, List callingLockIds) { + public Lock lockTask(ZkNodeProps message, long ignored, String callingLockId) { String configSetName = getTaskKey(message); if (canExecute(configSetName, message)) { markExclusiveTask(configSetName, message); diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java index b883211a8671..c01f365a3d76 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java @@ -53,7 +53,7 @@ interface Lock { * * @return null if locking is not possible. */ - Lock lockTask(ZkNodeProps message, long batchSessionId, List callingLockIds); + Lock lockTask(ZkNodeProps message, long batchSessionId, String callingLockId); /** * @param message the message being processed diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index e6dea14b3636..4601f132c5d8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -16,7 +16,7 @@ */ package org.apache.solr.cloud; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.ID; @@ -38,7 +38,6 @@ import java.util.function.Predicate; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; -import org.apache.solr.cloud.api.collections.AdminCmdContext; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -334,11 +333,10 @@ public void run() { workQueue.remove(head, asyncId == null); continue; } - List callingLockIds = - AdminCmdContext.parseCallingLockIds(message.getStr(CALLING_LOCK_IDS_HEADER)); + String callingLockId = message.getStr(CALLING_LOCK_ID_HEADER); OverseerMessageHandler messageHandler = selector.selectOverseerMessageHandler(message); OverseerMessageHandler.Lock lock = - messageHandler.lockTask(message, batchSessionId, callingLockIds); + messageHandler.lockTask(message, batchSessionId, callingLockId); if (lock == null) { if (log.isDebugEnabled()) { log.debug("Exclusivity check failed for [{}]", message); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java index 69bca970db4a..ee5f56deca1f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AdminCmdContext.java @@ -17,20 +17,17 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; -import java.util.List; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; public class AdminCmdContext { private final CollectionParams.CollectionAction action; private final String asyncId; private String lockId; - private String callingLockIds; - private String subRequestCallingLockIds; + private String callingLockId; private ClusterState clusterState; public AdminCmdContext(CollectionParams.CollectionAction action) { @@ -46,7 +43,7 @@ public AdminCmdContext( CollectionParams.CollectionAction action, String asyncId, SolrQueryRequest req) { this.action = action; this.asyncId = asyncId; - this.setCallingLockIds((String) req.getContext().get(CALLING_LOCK_IDS_HEADER)); + this.withCallingLockId((String) req.getContext().get(CALLING_LOCK_ID_HEADER)); } public CollectionParams.CollectionAction getAction() { @@ -57,51 +54,22 @@ public String getAsyncId() { return asyncId; } - public void setLockId(String lockId) { + public AdminCmdContext withLockId(String lockId) { this.lockId = lockId; - regenerateSubRequestCallingLockIds(); + return this; } public String getLockId() { return lockId; } - public void setCallingLockIds(String callingLockIds) { - this.callingLockIds = callingLockIds; - regenerateSubRequestCallingLockIds(); - } - - public AdminCmdContext withCallingLockIds(String callingLockIds) { - setCallingLockIds(callingLockIds); + public AdminCmdContext withCallingLockId(String callingLockId) { + this.callingLockId = callingLockId; return this; } - private void regenerateSubRequestCallingLockIds() { - if (StrUtils.isNotBlank(callingLockIds) && StrUtils.isNotBlank(lockId)) { - subRequestCallingLockIds += "," + lockId; - } else if (StrUtils.isNotBlank(callingLockIds)) { - subRequestCallingLockIds = callingLockIds; - } else if (StrUtils.isNotBlank(lockId)) { - subRequestCallingLockIds = lockId; - } else { - subRequestCallingLockIds = null; - } - } - - public String getCallingLockIds() { - return callingLockIds; - } - - public List getCallingLockIdList() { - return parseCallingLockIds(callingLockIds); - } - - public static List parseCallingLockIds(String callingLockIdsString) { - if (StrUtils.isBlank(callingLockIdsString)) { - return List.of(); - } else { - return List.of(callingLockIdsString.split(",")); - } + public String getCallingLockId() { + return callingLockId; } public ClusterState getClusterState() { @@ -113,18 +81,15 @@ public AdminCmdContext withClusterState(ClusterState clusterState) { return this; } - public String getSubRequestCallingLockIds() { - return subRequestCallingLockIds; - } - public AdminCmdContext subRequestContext(CollectionParams.CollectionAction action) { return subRequestContext(action, asyncId); } public AdminCmdContext subRequestContext( CollectionParams.CollectionAction action, String asyncId) { - AdminCmdContext nextContext = new AdminCmdContext(action, asyncId); - nextContext.setCallingLockIds(subRequestCallingLockIds); - return nextContext.withClusterState(clusterState); + return new AdminCmdContext(action, asyncId) + .withCallingLockId(callingLockId) + .withLockId(lockId) + .withClusterState(clusterState); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index eff4d37be7b1..62cf0dd65af1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -107,7 +107,8 @@ DistributedMultiLock createCollectionApiLock( // CollectionParams.LockLevel.COLLECTION; } - List callingLockIdList = adminCmdContext.getCallingLockIdList(); + List callingLockIdList = + DistributedMultiLock.splitLockIds(adminCmdContext.getCallingLockId()); final CollectionParams.LockLevel[] iterationOrder = { CollectionParams.LockLevel.COLLECTION, diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java index 0271f7bbe00c..94dae81bc727 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java @@ -58,7 +58,7 @@ public interface CollectionCommandContext { default ShardRequestTracker asyncRequestTracker(AdminCmdContext adminCmdContext) { return new ShardRequestTracker( adminCmdContext.getAsyncId(), - adminCmdContext.getSubRequestCallingLockIds(), + adminCmdContext.getLockId(), getAdminPath(), getZkStateReader(), newShardHandler().getShardHandlerFactory()); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 314b4c30c925..6df49cc1a1bc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -17,7 +17,7 @@ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -630,7 +630,7 @@ public static ShardRequestTracker syncRequestTracker( public static ShardRequestTracker syncRequestTracker( AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) { - return requestTracker(null, adminCmdContext.getSubRequestCallingLockIds(), adminPath, ccc); + return requestTracker(null, adminCmdContext.getLockId(), adminPath, ccc); } public static ShardRequestTracker asyncRequestTracker( @@ -641,10 +641,7 @@ public static ShardRequestTracker asyncRequestTracker( public static ShardRequestTracker asyncRequestTracker( AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) { return requestTracker( - adminCmdContext.getAsyncId(), - adminCmdContext.getSubRequestCallingLockIds(), - adminPath, - ccc); + adminCmdContext.getAsyncId(), adminCmdContext.getLockId(), adminPath, ccc); } protected static ShardRequestTracker requestTracker( @@ -746,7 +743,7 @@ public void sendShardRequest( sreq.coreNodeName = coreNodeName; sreq.params = params; if (lockIdList != null && !lockIdList.isBlank()) { - sreq.headers = Map.of(CALLING_LOCK_IDS_HEADER, lockIdList); + sreq.headers = Map.of(CALLING_LOCK_ID_HEADER, lockIdList); } shardHandler.submit(sreq, replica, sreq.params); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index d0b3fbead5de..a626af3c45c6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -411,7 +411,7 @@ public OverseerSolrResponse call() { // Block this thread until all required locks are acquired. lock.waitUntilAcquired(); - adminCmdContext.setLockId(lock.getLockId()); + adminCmdContext.withLockId(lock.getLockId()); // Got the lock so moving from submitted to running if we run for an async task (if // asyncId is null the asyncTaskTracker calls do nothing). diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index e642ff49d528..20d06bd4890c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -21,14 +21,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Arrays; -import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -130,10 +129,11 @@ public OverseerSolrResponse processMessage(ZkNodeProps message, String operation CollectionAction action = getCollectionAction(operation); CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action); if (command != null) { - AdminCmdContext adminCmdContext = new AdminCmdContext(action, message.getStr(ASYNC)); - adminCmdContext.setLockId(lock.id()); - adminCmdContext.setCallingLockIds(message.getStr(CALLING_LOCK_IDS_HEADER)); - adminCmdContext.withClusterState(cloudManager.getClusterState()); + AdminCmdContext adminCmdContext = + new AdminCmdContext(action, message.getStr(ASYNC)) + .withLockId(lock.id()) + .withCallingLockId(message.getStr(CALLING_LOCK_ID_HEADER)) + .withClusterState(cloudManager.getClusterState()); command.call(adminCmdContext, message, results); } else { throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation); @@ -186,7 +186,7 @@ public String getTaskKey(ZkNodeProps message) { * because it happens that a lock got released). */ @Override - public Lock lockTask(ZkNodeProps message, long batchSessionId, List callingLockIds) { + public Lock lockTask(ZkNodeProps message, long batchSessionId, String callingLockId) { if (sessionId != batchSessionId) { // this is always called in the same thread. // Each batch is supposed to have a new taskBatch @@ -201,7 +201,7 @@ public Lock lockTask(ZkNodeProps message, long batchSessionId, List call getTaskKey(message), message.getStr(ZkStateReader.SHARD_ID_PROP), message.getStr(ZkStateReader.REPLICA_PROP)), - callingLockIds); + callingLockId); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 50a54145f486..07f129737cfc 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -34,7 +34,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; @@ -368,8 +368,8 @@ public static SolrResponse submitCollectionApiCommand( if (adminCmdContext.getAsyncId() != null && !adminCmdContext.getAsyncId().isBlank()) { additionalProps.put(ASYNC, adminCmdContext.getAsyncId()); } - if (StrUtils.isNotBlank(adminCmdContext.getCallingLockIds())) { - additionalProps.put(CALLING_LOCK_IDS_HEADER, adminCmdContext.getCallingLockIds()); + if (StrUtils.isNotBlank(adminCmdContext.getLockId())) { + additionalProps.put(CALLING_LOCK_ID_HEADER, adminCmdContext.getCallingLockId()); } m = m.plus(additionalProps); if (adminCmdContext.getAsyncId() != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index b6766696cdef..faee1e61d72e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import java.util.Map; @@ -121,8 +120,6 @@ public void disableResponseCaching() { protected SolrResponse submitRemoteMessageAndHandleException( SolrJerseyResponse response, AdminCmdContext adminCmdContext, ZkNodeProps remoteMessage) throws Exception { - adminCmdContext.setCallingLockIds( - (String) solrQueryRequest.getContext().get(CALLING_LOCK_IDS_HEADER)); final SolrResponse remoteResponse = CollectionsHandler.submitCollectionApiCommand( coreContainer.getZkController(), @@ -141,7 +138,7 @@ protected SolrResponse submitRemoteMessageAndHandleException( ZkNodeProps remoteMessage) throws Exception { return submitRemoteMessageAndHandleException( - response, new AdminCmdContext(action), remoteMessage); + response, new AdminCmdContext(action, null, solrQueryRequest), remoteMessage); } protected SolrResponse submitRemoteMessageAndHandleAsync( @@ -152,7 +149,7 @@ protected SolrResponse submitRemoteMessageAndHandleAsync( throws Exception { var remoteResponse = submitRemoteMessageAndHandleException( - response, new AdminCmdContext(action, asyncId), remoteMessage); + response, new AdminCmdContext(action, asyncId, solrQueryRequest), remoteMessage); if (asyncId != null) { response.requestId = asyncId; diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 12a06739db29..2562fce1020c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -19,7 +19,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; -import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_IDS_HEADER; +import static org.apache.solr.common.params.CollectionAdminParams.CALLING_LOCK_ID_HEADER; import static org.apache.solr.security.AuditEvent.EventType.COMPLETED; import static org.apache.solr.security.AuditEvent.EventType.ERROR; import static org.apache.solr.servlet.HttpSolrCall.Action.ADMIN; @@ -739,9 +739,9 @@ protected QueryResponseWriter getResponseWriter() { protected void handleAdmin(SolrQueryResponse solrResp) { SolrCore.preDecorateResponse(solrReq, solrResp); - String callingLockIds = req.getHeader(CALLING_LOCK_IDS_HEADER); + String callingLockIds = req.getHeader(CALLING_LOCK_ID_HEADER); if (callingLockIds != null && !callingLockIds.isBlank()) { - solrReq.getContext().put(CALLING_LOCK_IDS_HEADER, callingLockIds); + solrReq.getContext().put(CALLING_LOCK_ID_HEADER, callingLockIds); } handler.handleRequest(solrReq, solrResp); SolrCore.postDecorateResponse(handler, solrReq, solrResp); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java index 79402cb48eba..f56da34f780f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLockTree.java @@ -43,26 +43,25 @@ public class TestLockTree extends SolrTestCaseJ4 { public void testLocks() throws Exception { LockTree lockTree = new LockTree(); Lock coll1Lock = - lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), List.of()); + lockTree.getSession().lock(CollectionAction.CREATE, Arrays.asList("coll1"), null); assertNotNull(coll1Lock); assertNull( "Should not be able to lock coll1/shard1", lockTree .getSession() - .lock( - CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), List.of())); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null)); coll1Lock.unlock(); Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), List.of()); + .lock(CollectionAction.BALANCESHARDUNIQUE, Arrays.asList("coll1", "shard1"), null); assertNotNull(shard1Lock); shard1Lock.unlock(); Lock replica1Lock = lockTree .getSession() - .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), List.of()); + .lock(ADDREPLICAPROP, Arrays.asList("coll1", "shard1", "core_node2"), null); assertNotNull(replica1Lock); List>> operations = new ArrayList<>(); @@ -85,7 +84,7 @@ public void testLocks() throws Exception { List locks = new CopyOnWriteArrayList<>(); List threads = new ArrayList<>(); for (Pair> operation : operations) { - final Lock lock = session.lock(operation.first(), operation.second(), List.of()); + final Lock lock = session.lock(operation.first(), operation.second(), null); if (lock != null) { Thread thread = new Thread(getRunnable(completedOps, operation, locks, lock)); threads.add(thread); @@ -113,18 +112,15 @@ public void testLocks() throws Exception { public void testCallingLockIdSubLocks() throws Exception { LockTree lockTree = new LockTree(); - Lock coll1Lock = - lockTree.getSession().lock(CollectionAction.CREATE, List.of("coll1"), List.of()); + Lock coll1Lock = lockTree.getSession().lock(CollectionAction.CREATE, List.of("coll1"), null); assertNotNull(coll1Lock); // Test sub-locks at the same level assertNull( "Should not be able to lock coll1 without using a callingLockId", - lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), List.of())); + lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), null)); Lock coll1Lock2 = - lockTree - .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(coll1Lock.id())); + lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), coll1Lock.id()); assertNotNull(coll1Lock2); coll1Lock2.unlock(); @@ -132,18 +128,17 @@ public void testCallingLockIdSubLocks() throws Exception { Lock shard1Lock = lockTree .getSession() - .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id())); + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), coll1Lock.id()); assertNotNull(shard1Lock); assertNull( "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1", lockTree .getSession() - .lock( - CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id()))); + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), coll1Lock.id())); Lock shard2Lock = lockTree .getSession() - .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard2"), List.of(coll1Lock.id())); + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard2"), coll1Lock.id()); assertNotNull(shard2Lock); shard2Lock.unlock(); shard1Lock.unlock(); @@ -152,8 +147,7 @@ public void testCallingLockIdSubLocks() throws Exception { Lock replica1Lock = lockTree .getSession() - .lock( - MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica1"), List.of(coll1Lock.id())); + .lock(MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica1"), coll1Lock.id()); assertNull( "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", lockTree @@ -161,19 +155,17 @@ public void testCallingLockIdSubLocks() throws Exception { .lock( CollectionAction.MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica1"), - List.of(coll1Lock.id()))); + coll1Lock.id())); assertNull( "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", lockTree .getSession() - .lock( - CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), List.of(coll1Lock.id()))); + .lock(CollectionAction.ADDREPLICA, List.of("coll1", "shard1"), coll1Lock.id())); assertNotNull(replica1Lock); Lock replica2Lock = lockTree .getSession() - .lock( - MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica2"), List.of(coll1Lock.id())); + .lock(MOCK_REPLICA_TASK, List.of("coll1", "shard1", "replica2"), coll1Lock.id()); assertNotNull(replica2Lock); replica2Lock.unlock(); replica1Lock.unlock(); @@ -183,12 +175,12 @@ public void testCallingLockIdSubLocks() throws Exception { Lock shard1Lock1 = lockTree .getSession() - .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll1", "shard1"), List.of()); + .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll1", "shard1"), null); assertNotNull(shard1Lock1); Lock shard1Lock2 = lockTree .getSession() - .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll2", "shard1"), List.of()); + .lock(CollectionAction.INSTALLSHARDDATA, List.of("coll2", "shard1"), null); assertNotNull(shard1Lock2); assertThrows( "Should not be able to lock coll1/shard1 since our callingLockId is coll2", @@ -196,15 +188,11 @@ public void testCallingLockIdSubLocks() throws Exception { () -> lockTree .getSession() - .lock( - CollectionAction.SYNCSHARD, - List.of("coll1", "shard1"), - List.of(shard1Lock2.id()))); + .lock(CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), shard1Lock2.id())); Lock shard1Lock3 = lockTree .getSession() - .lock( - CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), List.of(shard1Lock1.id())); + .lock(CollectionAction.SYNCSHARD, List.of("coll1", "shard1"), shard1Lock1.id()); assertNotNull(shard1Lock3); shard1Lock2.unlock(); shard1Lock3.unlock(); @@ -212,14 +200,14 @@ public void testCallingLockIdSubLocks() throws Exception { // Test difference at a higher level assertNull( "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", - lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), List.of())); + lockTree.getSession().lock(CollectionAction.RELOAD, List.of("coll1"), null)); assertThrows( "Should not be able to lock coll1 since our callingLockId is coll1/shard1. Cannot move up", SolrException.class, () -> lockTree .getSession() - .lock(CollectionAction.RELOAD, List.of("coll1"), List.of(shard1Lock1.id()))); + .lock(CollectionAction.RELOAD, List.of("coll1"), shard1Lock1.id())); // Test an unrelated lock assertThrows( @@ -228,17 +216,14 @@ public void testCallingLockIdSubLocks() throws Exception { () -> lockTree .getSession() - .lock(CollectionAction.CREATE, List.of("coll2"), List.of(shard1Lock1.id()))); + .lock(CollectionAction.CREATE, List.of("coll2"), shard1Lock1.id())); assertThrows( "Should not be able to lock coll1/shard2 since our callingLockId is coll1/shard1. Cannot lock an unrelated resource", SolrException.class, () -> lockTree .getSession() - .lock( - CollectionAction.SYNCSHARD, - List.of("coll1", "shard2"), - List.of(shard1Lock1.id()))); + .lock(CollectionAction.SYNCSHARD, List.of("coll1", "shard2"), shard1Lock1.id())); shard1Lock1.unlock(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index e850b9df14dd..ee0da8853b82 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -252,7 +252,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) coll1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", null, null); @@ -264,7 +264,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", null); @@ -275,7 +275,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) DistributedMultiLock shard1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", null); @@ -287,7 +287,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard2", null); @@ -302,7 +302,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) DistributedMultiLock replica1Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", "replica1"); @@ -313,7 +313,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) DistributedMultiLock replica2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", "replica1"); @@ -325,7 +325,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) shard1Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", null); @@ -337,7 +337,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) replica2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) - .withCallingLockIds(coll1Lock.getLockId()), + .withCallingLockId(coll1Lock.getLockId()), "coll1", "shard1", "replica2"); @@ -377,14 +377,14 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) () -> apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) - .withCallingLockIds(badLockId), + .withCallingLockId(badLockId), "coll1", "shard1", null)); DistributedMultiLock shard1Lock3 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA) - .withCallingLockIds(shard1Lock1.getLockId()), + .withCallingLockId(shard1Lock1.getLockId()), "coll1", "shard1", null); @@ -410,7 +410,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) () -> apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD) - .withCallingLockIds(shard1Lock1.getLockId()), + .withCallingLockId(shard1Lock1.getLockId()), "coll1", null, null)); @@ -423,7 +423,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) () -> apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.CREATE) - .withCallingLockIds(shard1Lock1.getLockId()), + .withCallingLockId(shard1Lock1.getLockId()), "coll2", null, null)); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index 629e56384aa6..050e32bb977a 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -125,5 +125,5 @@ public interface CollectionAdminParams { String PER_REPLICA_STATE = CollectionStateProps.PER_REPLICA_STATE; - String CALLING_LOCK_IDS_HEADER = "callingLockIds"; + String CALLING_LOCK_ID_HEADER = "callingLockId"; } From 629d9c4acb1a19365f9d02a2279c386c82c99865 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 14 Apr 2026 14:59:31 -0700 Subject: [PATCH 63/67] One missed refactored name --- .../core/src/java/org/apache/solr/servlet/HttpSolrCall.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 2562fce1020c..3e638f56de90 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -739,9 +739,9 @@ protected QueryResponseWriter getResponseWriter() { protected void handleAdmin(SolrQueryResponse solrResp) { SolrCore.preDecorateResponse(solrReq, solrResp); - String callingLockIds = req.getHeader(CALLING_LOCK_ID_HEADER); - if (callingLockIds != null && !callingLockIds.isBlank()) { - solrReq.getContext().put(CALLING_LOCK_ID_HEADER, callingLockIds); + String callingLockId = req.getHeader(CALLING_LOCK_ID_HEADER); + if (callingLockId != null && !callingLockId.isBlank()) { + solrReq.getContext().put(CALLING_LOCK_ID_HEADER, callingLockId); } handler.handleRequest(solrReq, solrResp); SolrCore.postDecorateResponse(handler, solrReq, solrResp); From 2687168ad0d5484b6512d285aef6dd8dc0e807fc Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 14 Apr 2026 15:43:57 -0700 Subject: [PATCH 64/67] Some fixes --- .../api/collections/CollectionHandlingUtils.java | 14 +++++++------- .../solr/handler/admin/CollectionsHandler.java | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java index 6df49cc1a1bc..6ba0bc39a6fe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java @@ -645,10 +645,10 @@ public static ShardRequestTracker asyncRequestTracker( } protected static ShardRequestTracker requestTracker( - String asyncId, String lockIds, String adminPath, CollectionCommandContext ccc) { + String asyncId, String lockId, String adminPath, CollectionCommandContext ccc) { return new ShardRequestTracker( asyncId, - lockIds, + lockId, adminPath, ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory()); @@ -656,7 +656,7 @@ protected static ShardRequestTracker requestTracker( public static class ShardRequestTracker { private final String asyncId; - private final String lockIdList; + private final String lockId; private final String adminPath; private final ZkStateReader zkStateReader; private final ShardHandlerFactory shardHandlerFactory; @@ -664,12 +664,12 @@ public static class ShardRequestTracker { public ShardRequestTracker( String asyncId, - String lockIdList, + String lockId, String adminPath, ZkStateReader zkStateReader, ShardHandlerFactory shardHandlerFactory) { this.asyncId = asyncId; - this.lockIdList = lockIdList; + this.lockId = lockId; this.adminPath = adminPath; this.zkStateReader = zkStateReader; this.shardHandlerFactory = shardHandlerFactory; @@ -742,8 +742,8 @@ public void sendShardRequest( sreq.nodeName = nodeName; sreq.coreNodeName = coreNodeName; sreq.params = params; - if (lockIdList != null && !lockIdList.isBlank()) { - sreq.headers = Map.of(CALLING_LOCK_ID_HEADER, lockIdList); + if (StrUtils.isNotBlank(lockId)) { + sreq.headers = Map.of(CALLING_LOCK_ID_HEADER, lockId); } shardHandler.submit(sreq, replica, sreq.params); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 07f129737cfc..be9f394d26c5 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -368,7 +368,7 @@ public static SolrResponse submitCollectionApiCommand( if (adminCmdContext.getAsyncId() != null && !adminCmdContext.getAsyncId().isBlank()) { additionalProps.put(ASYNC, adminCmdContext.getAsyncId()); } - if (StrUtils.isNotBlank(adminCmdContext.getLockId())) { + if (StrUtils.isNotBlank(adminCmdContext.getCallingLockId())) { additionalProps.put(CALLING_LOCK_ID_HEADER, adminCmdContext.getCallingLockId()); } m = m.plus(additionalProps); From edf4a12cc9b429417d921976f1a5ab6c0a336a7a Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 15 Apr 2026 09:51:45 -0700 Subject: [PATCH 65/67] Improve changelog entry --- changelog/unreleased/solr-18011-locking-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/solr-18011-locking-update.yml b/changelog/unreleased/solr-18011-locking-update.yml index 3b714ae59345..7d2ad4cb4722 100644 --- a/changelog/unreleased/solr-18011-locking-update.yml +++ b/changelog/unreleased/solr-18011-locking-update.yml @@ -1,5 +1,5 @@ # See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: Allow locked Admin APIs to call other locked AdminAPIs without deadlocking +title: Allow locked Admin APIs to call other locked AdminAPIs. These locked Admin APIs can only call other APIs on the same resource tree (Collection > Shard > Replica) to protect against deadlocks. type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other authors: - name: Houston Putman From 50f97a2bef940b0a440ad8871d1ba7f769c9e858 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 20 Apr 2026 16:22:48 -0700 Subject: [PATCH 66/67] Fix error scenario for overseer, add test --- .../solr/cloud/OverseerTaskProcessor.java | 23 +++- ...rseerCollectionConfigSetProcessorTest.java | 108 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index 4601f132c5d8..30ff9a31996e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -38,11 +38,13 @@ import java.util.function.Predicate; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; +import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; @@ -335,8 +337,25 @@ public void run() { } String callingLockId = message.getStr(CALLING_LOCK_ID_HEADER); OverseerMessageHandler messageHandler = selector.selectOverseerMessageHandler(message); - OverseerMessageHandler.Lock lock = - messageHandler.lockTask(message, batchSessionId, callingLockId); + OverseerMessageHandler.Lock lock; + try { + lock = messageHandler.lockTask(message, batchSessionId, callingLockId); + } catch (SolrException e) { + // Lock acquisition can throw if e.g. callingLockId references an unrelated + // action. In that case, fail the task immediately rather than retrying. + log.error( + "Error occurred while trying to acquire lock for task [{}]", head.getId(), e); + NamedList errResp = new NamedList<>(); + errResp.add("exception", e.getMessage()); + OverseerSolrResponse response = new OverseerSolrResponse(errResp); + if (asyncId != null) { + failureMap.put(asyncId, OverseerSolrResponseSerializer.serialize(response)); + } else { + head.setBytes(OverseerSolrResponseSerializer.serialize(response)); + } + workQueue.remove(head, asyncId == null); + continue; + } if (lock == null) { if (log.isDebugEnabled()) { log.debug("Exclusivity check failed for [{}]", message); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java index 371568db8697..2608a4e86350 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; @@ -57,6 +58,7 @@ import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; +import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler; import org.apache.solr.cluster.placement.PlacementPluginFactory; import org.apache.solr.cluster.placement.plugins.SimplePlacementFactory; import org.apache.solr.common.MapWriter; @@ -74,6 +76,7 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.ObjectCache; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.TimeSource; @@ -1431,4 +1434,109 @@ public void testFloodQueue() throws Exception { waitForEmptyQueue(); stopProcessor(); } + + /** + * Verify that when lockTask throws a SolrException due to a callingLockId from an unrelated + * collection, the async task is properly marked as failed rather than silently swallowed or + * retried forever. This test exercises the real LockTree validation code path. + */ + @Test + public void testLockTaskExceptionFailsAsyncTask() throws Exception { + commonMocks(2, false); + + String asyncId = "lock-fail-test-async-123"; + + // Create a real OverseerCollectionMessageHandler so we exercise the real LockTree locking + OverseerCollectionMessageHandler collHandler = + new OverseerCollectionMessageHandler( + zkStateReaderMock, + "1234", + shardHandlerFactoryMock, + ADMIN_PATH, + new Stats(), + overseerMock, + new OverseerNodePrioritizer( + zkStateReaderMock, overseerMock, ADMIN_PATH, shardHandlerFactoryMock)); + + // Acquire a lock on collA by calling lockTask directly. + // This puts a real lock into the LockTree's allLocks map. + ZkNodeProps collAMessage = + new ZkNodeProps( + Map.of( + Overseer.QUEUE_OPERATION, + CollectionParams.CollectionAction.MOCK_COLL_TASK.toLower(), + "name", + "collA")); + OverseerMessageHandler.Lock collALock = collHandler.lockTask(collAMessage, 1, null); + assertNotNull("Should have acquired lock on collA", collALock); + String collALockId = collALock.id(); + + // Build a selector that always returns our real handler + OverseerTaskProcessor.OverseerMessageHandlerSelector selector = + new OverseerTaskProcessor.OverseerMessageHandlerSelector() { + @Override + public void close() { + IOUtils.closeQuietly(collHandler); + } + + @Override + public OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message) { + return collHandler; + } + }; + + // Create a processor using the real handler + OverseerTaskProcessor processor = + new OverseerTaskProcessor( + zkStateReaderMock, + "1234", + new Stats(), + selector, + mock(OverseerNodePrioritizer.class), + workQueueMock, + runningMapMock, + completedMapMock, + failureMapMock, + solrMetricsContextMock) { + @Override + protected LeaderStatus amILeader() { + return LeaderStatus.YES; + } + }; + + Thread processorThread = new Thread(processor); + processorThread.start(); + + try { + // Submit an async task for collB, but with collA's lock ID as the callingLockId. + // The real LockTree will find collA's lock, call validateSubpath, and throw SolrException + // because collB != collA. + Map propMap = + Map.of( + Overseer.QUEUE_OPERATION, + CollectionParams.CollectionAction.MOCK_COLL_TASK.toLower(), + "name", + "collB", + CollectionAdminParams.CALLING_LOCK_ID_HEADER, + collALockId, + "async", + asyncId); + ZkNodeProps props = new ZkNodeProps(propMap); + QueueEvent qe = new QueueEvent("lockFailTask", Utils.toJSON(props), null); + queue.add(qe); + + waitForEmptyQueue(); + + // Verify the task was put in the failure map + verify(failureMapMock, times(1)).put(eq(asyncId), any(byte[].class)); + + // Verify the task was NOT put in the running map (it should fail before reaching that point) + verify(runningMapMock, times(0)).put(eq(asyncId), any()); + } finally { + collALock.unlock(); + processor.close(); + processorThread.interrupt(); + processorThread.join(MAX_WAIT_MS); + } + } } From 089281dbc93453676b528cbdfc2e9663e0e6e6dd Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 21 Apr 2026 12:07:35 -0700 Subject: [PATCH 67/67] Remove commented out and unrelated code --- .../api/collections/InstallShardDataCmd.java | 93 ------------------- .../handler/admin/api/GetSegmentData.java | 4 +- .../solrj/impl/CloudHttp2SolrClientTest.java | 5 +- 3 files changed, 3 insertions(+), 99 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java index 18986e4ad1b5..cc58912dc2a7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/InstallShardDataCmd.java @@ -191,99 +191,6 @@ public void call(AdminCmdContext adminCmdContext, ZkNodeProps message, NamedList } } - /* - public static void handleCoreRestoreResponses( - CoreContainer coreContainer, - String collection, - List shards, - NamedList results, - List notLiveReplicas, - String errorMessage) { - DocCollection collectionState = coreContainer.getZkController().getZkStateReader().getCollectionLive(collection); - Collection allReplicas = - shards.stream() - .flatMap(shard -> collectionState.getSlice(shard).getReplicas().stream()) - .toList(); - - // Ensure that terms are correct for this shard after the execution is done - // We only care about leader eligible replicas, all others will eventually get updated. - List leaderEligibleReplicas = - allReplicas.stream() - .filter(r -> r.getType().leaderEligible) - .toList(); - - NamedList failures = (NamedList) results.get("failure"); - Set successfulReplicas = - leaderEligibleReplicas.stream() - .filter(replica -> !notLiveReplicas.contains(replica)) - .filter( - replica -> - failures == null - || failures.get(CollectionHandlingUtils.requestKey(replica)) == null) - .collect(Collectors.toSet()); - - if (successfulReplicas.isEmpty()) { - // No leader-eligible replicas succeeded, return failure - if (failures == null) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - errorMessage + ". No leader-eligible replicas are live."); - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, errorMessage, (Throwable) failures.getVal(0)); - } - } else if (successfulReplicas.size() < leaderEligibleReplicas.size()) { - // Some, but not all, leader-eligible replicas succeeded. - // Ensure the shard terms are correct so that the non-successful replicas go into recovery - ZkShardTerms shardTerms = - coreContainer - .getZkController() - .getShardTerms(typedMessage.collection, typedMessage.shard); - shardTerms.ensureHighestTerms( - successfulReplicas.stream().map(Replica::getName).collect(Collectors.toSet())); - Set replicasToRecover = - leaderEligibleReplicas.stream() - .filter(r -> !successfulReplicas.contains(r)) - .map(Replica::getName) - .collect(Collectors.toSet()); - coreContainer - .getZkController() - .getZkStateReader() - .waitForState( - collection, - 10, - TimeUnit.SECONDS, - (liveNodes, colState) -> - colState.getSlice(typedMessage.shard).getReplicas().stream() - .filter(r -> replicasToRecover.contains(r.getName())) - .allMatch(r -> Replica.State.RECOVERING.equals(r.getState()))); - - // In order for the async request to succeed, we need to ensure that there is no failure - // message - NamedList successes = (NamedList) results.get("success"); - failures.forEach( - (replicaKey, value) -> { - successes.add( - replicaKey, - new NamedList<>( - Map.of( - "explanation", - "Core install failed, but is now recovering from the leader", - "failure", - value))); - }); - results.remove("failure"); - } else { - // other replicas to-be-created will know that they are out of date by - // looking at their term : 0 compare to term of this core : 1 - coreContainer - .getZkController() - .getShardTerms(collection, typedMessage.shard) - .ensureHighestTermsAreNotZero(); - } - } - */ - /** A value-type representing the message received by {@link InstallShardDataCmd} */ @JsonIgnoreProperties(ignoreUnknown = true) public static class RemoteMessage implements JacksonReflectMapWriter { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java index ffafc81e5c02..fb11371c6e81 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSegmentData.java @@ -153,7 +153,7 @@ public GetSegmentDataResponse getSegmentData( coreSummary.indexDir = core.getIndexDir(); coreSummary.sizeInGB = (double) core.getIndexSize() / GB; - RefCounted iwRef = core.getSolrCoreState().getIndexWriter(core, true); + RefCounted iwRef = core.getSolrCoreState().getIndexWriter(core); if (iwRef != null) { try { IndexWriter iw = iwRef.get(); @@ -257,7 +257,7 @@ private Map getMergeInformation( SolrQueryRequest req, SegmentInfos infos, List mergeCandidates) throws IOException { final var result = new HashMap(); RefCounted refCounted = - req.getCore().getSolrCoreState().getIndexWriter(req.getCore(), true); + req.getCore().getSolrCoreState().getIndexWriter(req.getCore()); try { IndexWriter indexWriter = refCounted.get(); if (indexWriter instanceof SolrIndexWriter) { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 7c78f450a007..317fc48dbed7 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -602,10 +602,7 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // And since all the nodes are hosting cores from all shards, the // distributed query formed by this node will select cores from the // local shards only - QueryResponse qResponse = null; - for (int i = 0; i < 100; i++) { - qResponse = cloudClient.query(collectionName, qRequest); - } + QueryResponse qResponse = cloudClient.query(collectionName, qRequest); Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); assertNotNull("Unable to obtain " + ShardParams.SHARDS_INFO, shardsInfo);