fix: cache version probe result to prevent reconciliation deadlock

api/v1alpha1/clickhousecluster_types.go

@@ -199,6 +199,11 @@ type ClickHouseClusterStatus struct {
 	// +optional
 	// +operator-sdk:csv:customresourcedefinitions:type=status
 	Version string `json:"version,omitempty"`
+	// VersionProbeRevision is the image hash of the last successful version probe.
+	// When this matches the current image hash, the cached Version is used directly.
+	// +optional
+	// +operator-sdk:csv:customresourcedefinitions:type=status
+	VersionProbeRevision string `json:"versionProbeRevision,omitempty"`
 }
 
 // ClickHouseCluster is the Schema for the `clickhouseclusters` API.

api/v1alpha1/keepercluster_types.go

@@ -160,6 +160,11 @@ type KeeperClusterStatus struct {
 	// +optional
 	// +operator-sdk:csv:customresourcedefinitions:type=status
 	Version string `json:"version,omitempty"`
+	// VersionProbeRevision is the image hash of the last successful version probe.
+	// When this matches the current image hash, the cached Version is used directly.
+	// +optional
+	// +operator-sdk:csv:customresourcedefinitions:type=status
+	VersionProbeRevision string `json:"versionProbeRevision,omitempty"`
 }
 
 // KeeperCluster is the Schema for the `keeperclusters` API.

config/crd/bases/clickhouse.com_clickhouseclusters.yaml

@@ -6963,6 +6963,11 @@ spec:
                 description: Version indicates the version reported by the container
                   image.
                 type: string
+              versionProbeRevision:
+                description: |-
+                  VersionProbeRevision is the image hash of the last successful version probe.
+                  When this matches the current image hash, the cached Version is used directly.
+                type: string
             type: object
         type: object
     served: true

config/crd/bases/clickhouse.com_keeperclusters.yaml

@@ -6875,6 +6875,11 @@ spec:
                 description: Version indicates the version reported by the container
                   image.
                 type: string
+              versionProbeRevision:
+                description: |-
+                  VersionProbeRevision is the image hash of the last successful version probe.
+                  When this matches the current image hash, the cached Version is used directly.
+                type: string
             type: object
         type: object
     served: true

dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml

@@ -6966,6 +6966,11 @@ spec:
                 description: Version indicates the version reported by the container
                   image.
                 type: string
+              versionProbeRevision:
+                description: |-
+                  VersionProbeRevision is the image hash of the last successful version probe.
+                  When this matches the current image hash, the cached Version is used directly.
+                type: string
             type: object
         type: object
     served: true

dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml

@@ -6878,6 +6878,11 @@ spec:
                 description: Version indicates the version reported by the container
                   image.
                 type: string
+              versionProbeRevision:
+                description: |-
+                  VersionProbeRevision is the image hash of the last successful version probe.
+                  When this matches the current image hash, the cached Version is used directly.
+                type: string
             type: object
         type: object
     served: true

docs/api_reference.md

@@ -78,6 +78,7 @@ ClickHouseClusterStatus defines the observed state of ClickHouseCluster.
 | `updateRevision` | string | UpdateRevision indicates latest requested ClickHouseCluster spec revision. | true |  |
 | `observedGeneration` | integer | ObservedGeneration indicates latest generation observed by controller. | true |  |
 | `version` | string | Version indicates the version reported by the container image. | false |  |
+| `versionProbeRevision` | string | VersionProbeRevision is the image hash of the last successful version probe.<br />When this matches the current image hash, the cached Version is used directly. | false |  |
 
 Appears in:
 - [ClickHouseCluster](#clickhousecluster)
@@ -278,6 +279,7 @@ KeeperClusterStatus defines the observed state of KeeperCluster.
 | `updateRevision` | string | CurrentRevision indicates latest requested KeeperCluster spec revision. | true |  |
 | `observedGeneration` | integer | ObservedGeneration indicates latest generation observed by controller. | true |  |
 | `version` | string | Version indicates the version reported by the container image. | false |  |
+| `versionProbeRevision` | string | VersionProbeRevision is the image hash of the last successful version probe.<br />When this matches the current image hash, the cached Version is used directly. | false |  |
 
 Appears in:
 - [KeeperCluster](#keepercluster)

internal/controller/clickhouse/controller_test.go

@@ -182,6 +182,12 @@ var _ = When("reconciling ClickHouseCluster", Ordered, func() {
 		}
 		Expect(suite.Client.Update(ctx, updatedCR)).To(Succeed())
 
+		// Clear the cached version probe revision to force the probe to re-run,
+		// since the image didn't change but the overrides did.
+		Expect(suite.Client.Get(ctx, cr.NamespacedName(), updatedCR)).To(Succeed())
+		updatedCR.Status.VersionProbeRevision = ""
+		Expect(suite.Client.Status().Update(ctx, updatedCR)).To(Succeed())
+
 		// Delete old job so new one is created with overrides.
 		for _, j := range jobs.Items {
 			Expect(suite.Client.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground))).To(Succeed())

internal/controller/clickhouse/sync.go

@@ -414,6 +414,8 @@ func (r *clickhouseReconciler) reconcileVersionProbe(ctx context.Context, log ct
 		PodTemplate:       r.Cluster.Spec.PodTemplate,
 		ContainerTemplate: r.Cluster.Spec.ContainerTemplate,
 		VersionProbe:      r.Cluster.Spec.VersionProbeTemplate,
+		CachedVersion:     r.Cluster.Status.Version,
+		CachedRevision:    r.Cluster.Status.VersionProbeRevision,
 	})
 	if err != nil {
 		return chctrl.StepResult{}, fmt.Errorf("run version probe: %w", err)
@@ -422,6 +424,7 @@ func (r *clickhouseReconciler) reconcileVersionProbe(ctx context.Context, log ct
 	r.versionProbe = probeResult
 	if probeResult.Completed() {
 		r.Cluster.Status.Version = probeResult.Version
+		r.Cluster.Status.VersionProbeRevision = probeResult.Revision
 	}
 
 	return chctrl.StepContinue(), nil

internal/controller/keeper/sync.go

@@ -216,13 +216,18 @@ func (r *keeperReconciler) reconcileClusterRevisions(ctx context.Context, log ct
 		PodTemplate:       r.Cluster.Spec.PodTemplate,
 		ContainerTemplate: r.Cluster.Spec.ContainerTemplate,
 		VersionProbe:      r.Cluster.Spec.VersionProbeTemplate,
+		CachedVersion:     r.Cluster.Status.Version,
+		CachedRevision:    r.Cluster.Status.VersionProbeRevision,
 	})
 	if err != nil {
 		return chctrl.StepResult{}, fmt.Errorf("run version probe: %w", err)
 	}
 
 	r.versionProbe = probeResult
-	r.Cluster.Status.Version = r.versionProbe.Version
+	if probeResult.Completed() {
+		r.Cluster.Status.Version = probeResult.Version
+		r.Cluster.Status.VersionProbeRevision = probeResult.Revision
+	}
 
 	if r.Checker != nil {
 		cond, event := chctrl.GetUpgradeCondition(*r.Checker, r.versionProbe, r.Cluster.Spec.UpgradeChannel)

internal/controller/versionprobe.go

@@ -42,6 +42,10 @@ type VersionProbeConfig struct {
 	ContainerTemplate v1.ContainerTemplateSpec
 	// VersionProbe is the user-provided override for the version probe Job.
 	VersionProbe *v1.VersionProbeTemplate
+	// CachedVersion is the previously detected version stored in CR Status.
+	CachedVersion string
+	// CachedRevision is the image hash that produced CachedVersion.
+	CachedRevision string
 }
 
 // VersionProbeResult holds the outcome of a version probe reconciliation.
@@ -52,6 +56,8 @@ type VersionProbeResult struct {
 	Pending bool
 	// Err if version probe failed it contains the error.
 	Err error
+	// Revision is the image hash of the probe, set on successful completion.
+	Revision string
 }
 
 // Completed returns true if probe completed successfully with a detected version, false otherwise.
@@ -61,12 +67,27 @@ func (r *VersionProbeResult) Completed() bool {
 
 // VersionProbe manages a one-time Job to detect the version from a container image.
 // Returns the version string when available, or empty string if the Job is pending/running.
+// When CachedRevision matches the current image hash and CachedVersion is non-empty,
+// returns the cached version immediately without creating or reading a Job.
 func (rm *ResourceManager) VersionProbe(
 	ctx context.Context,
 	log controllerutil.Logger,
 	cfg VersionProbeConfig,
 ) (VersionProbeResult, error) {
-	job, err := rm.buildVersionProbeJob(cfg)
+	// Compute revision once — used for cache check, Job name, and result.
+	revision, err := imageRevision(cfg)
+	if err != nil {
+		return VersionProbeResult{}, fmt.Errorf("compute image revision: %w", err)
+	}
+
+	// Fast path: return cached version when image hasn't changed.
+	// Intentionally skips cleanupVersionProbeJobs and Job/Pod API calls.
+	// Orphaned Jobs from previous images are cleaned on next cache miss.
+	if cfg.CachedRevision == revision && cfg.CachedVersion != "" {
+		return VersionProbeResult{Version: cfg.CachedVersion, Revision: revision}, nil
+	}
+
+	job, err := rm.buildVersionProbeJob(cfg, revision)
 	if err != nil {
 		return VersionProbeResult{}, fmt.Errorf("build version probe job: %w", err)
 	}
@@ -124,7 +145,7 @@ func (rm *ResourceManager) VersionProbe(
 		return VersionProbeResult{Err: err}, nil
 	}
 
-	return VersionProbeResult{Version: version}, nil
+	return VersionProbeResult{Version: version, Revision: revision}, nil
 }
 
 // GetVersionSyncCondition evaluates the VersionInSync condition based on the probe result and replica versions.
@@ -209,7 +230,27 @@ func (rm *ResourceManager) cleanupVersionProbeJobs(ctx context.Context, log cont
 	}
 }
 
-func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig) (batchv1.Job, error) {
+// imageRevision computes a hash of the image and pull policy to use as the cache key
+// and Job name suffix. Extracted from buildVersionProbeJob to allow VersionProbe to
+// compute it once and pass it through.
+func imageRevision(cfg VersionProbeConfig) (string, error) {
+	type imageKey struct {
+		Image      string
+		PullPolicy corev1.PullPolicy
+	}
+
+	hash, err := controllerutil.DeepHashObject(imageKey{
+		Image:      cfg.ContainerTemplate.Image.String(),
+		PullPolicy: cfg.ContainerTemplate.ImagePullPolicy,
+	})
+	if err != nil {
+		return "", fmt.Errorf("hash image key: %w", err)
+	}
+
+	return hash, nil
+}
+
+func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig, revision string) (batchv1.Job, error) {
 	job := batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace:   rm.owner.GetNamespace(),
@@ -269,21 +310,7 @@ func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig) (batchv1
 		return batchv1.Job{}, fmt.Errorf("set version probe job controller reference: %w", err)
 	}
 
-	// Recreate successful probe only on Image or PullPolicy changes
-	type imageKey struct {
-		Image      string
-		PullPolicy corev1.PullPolicy
-	}
-
-	imageHash, err := controllerutil.DeepHashObject(imageKey{
-		Image:      cfg.ContainerTemplate.Image.String(),
-		PullPolicy: cfg.ContainerTemplate.ImagePullPolicy,
-	})
-	if err != nil {
-		return batchv1.Job{}, fmt.Errorf("hash version probe job image: %w", err)
-	}
-
-	job.Name = fmt.Sprintf("%s-version-probe-%s", rm.specificName, imageHash[:8])
+	job.Name = fmt.Sprintf("%s-version-probe-%s", rm.specificName, revision[:8])
 
 	// Set reserved labels after overrides to ensure they are not modified by user overrides.
 	job.Labels = controllerutil.MergeMaps(job.Labels, map[string]string{