feat: downgrade uncaught error log when reconciler handles the error

When a reconciler's updateErrorStatus returns any ErrorStatusUpdateControl
other than defaultErrorProcessing(), the error is now considered handled by
the reconciler. Native retry (including @GradualRetry exponential backoff)
is kept intact, but the framework logs the error on DEBUG level instead of
emitting the "Uncaught error during event processing" WARN.

This lets a reconciler keep retrying an expected, recoverable condition
without producing a continuous stream of WARN messages, while it remains
free to log the error at whatever level it wants inside updateErrorStatus.
Returning defaultErrorProcessing() preserves the previous behavior,
including the warning.

- PostExecutionControl carries an errorHandledByReconciler flag
- ReconciliationDispatcher marks the exception control as handled instead
  of rethrowing when a non-default control still wants retry
- EventProcessor downgrades the retry-aware and no-retry-configured error
  logs to DEBUG when the error was handled by the reconciler

Signed-off-by: Attila Mészáros <a_meszaros@apple.com>

Author: csviri

docs/content/en/docs/documentation/error-handling-retries.md

@@ -108,6 +108,16 @@ Retry can be skipped in cases of unrecoverable errors:
  ErrorStatusUpdateControl.patchStatus(customResource).withNoRetry();
 ```
 
+When `updateErrorStatus` returns any `ErrorStatusUpdateControl` other than
+`ErrorStatusUpdateControl.defaultErrorProcessing()`, the framework considers the error handled by
+the reconciler. In that case the native retry (including the exponential backoff from
+`@GradualRetry`) is still performed, but the framework no longer logs the "Uncaught error during
+event processing" warning; the error is logged on `DEBUG` level instead. This lets a reconciler keep
+retrying an expected, recoverable condition without producing a continuous stream of `WARN` messages,
+while it remains free to log the error at whatever level it deems appropriate inside
+`updateErrorStatus`. Returning `ErrorStatusUpdateControl.defaultErrorProcessing()` preserves the
+default behavior, including the warning.
+
 ### Correctness and Automatic Retries
 
 While it is possible to deactivate automatic retries, this is not desirable unless there is a particular reason.

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java

@@ -297,7 +297,9 @@ synchronized void eventProcessingFinished(
         && postExecutionControl.exceptionDuringExecution()
         && (!state.deleteEventPresent() || triggerOnAllEvents())) {
       handleRetryOnException(
-          executionScope, postExecutionControl.getRuntimeException().orElseThrow());
+          executionScope,
+          postExecutionControl.getRuntimeException().orElseThrow(),
+          postExecutionControl.isErrorHandledByReconciler());
       return;
     }
     cleanupOnSuccessfulExecution(executionScope);
@@ -327,10 +329,17 @@ private boolean isTriggerOnAllEventAndDeleteEventPresent(ResourceState state) {
   private void logErrorIfNoRetryConfigured(
       ExecutionScope<P> executionScope, PostExecutionControl<P> postExecutionControl) {
     if (!isRetryConfigured() && postExecutionControl.exceptionDuringExecution()) {
-      log.error(
-          "Error during event processing {}",
-          executionScope,
-          postExecutionControl.getRuntimeException().orElseThrow());
+      if (postExecutionControl.isErrorHandledByReconciler()) {
+        log.debug(
+            "Error during event processing {}, but was handled by the reconciler",
+            executionScope,
+            postExecutionControl.getRuntimeException().orElseThrow());
+      } else {
+        log.error(
+            "Error during event processing {}",
+            executionScope,
+            postExecutionControl.getRuntimeException().orElseThrow());
+      }
     }
   }
 
@@ -369,14 +378,16 @@ TimerEventSource<P> retryEventSource() {
    * events (received meanwhile retry is in place or already in buffer) instantly or always wait
    * according to the retry timing if there was an exception.
    */
-  private void handleRetryOnException(ExecutionScope<P> executionScope, Exception exception) {
+  private void handleRetryOnException(
+      ExecutionScope<P> executionScope, Exception exception, boolean errorHandledByReconciler) {
     final var state = getOrInitRetryExecution(executionScope);
     var resourceID = state.getId();
     boolean eventPresent =
         state.eventPresent()
             || (triggerOnAllEvents() && state.isAdditionalEventPresentAfterDeleteEvent());
     state.markEventReceived(triggerOnAllEvents());
-    retryAwareErrorLogging(state.getRetry(), eventPresent, exception, executionScope);
+    retryAwareErrorLogging(
+        state.getRetry(), eventPresent, errorHandledByReconciler, exception, executionScope);
     metrics.reconciliationFailed(
         executionScope.getResource(), state.getRetry(), exception, metricsMetadata);
     if (eventPresent) {
@@ -408,9 +419,17 @@ private void handleRetryOnException(ExecutionScope<P> executionScope, Exception
   private void retryAwareErrorLogging(
       RetryExecution retry,
       boolean eventPresent,
+      boolean errorHandledByReconciler,
       Exception exception,
       ExecutionScope<P> executionScope) {
-    if (!retry.isLastAttempt()
+    if (errorHandledByReconciler) {
+      // The reconciler already handled the error in updateErrorStatus (and had the chance to log it
+      // as needed), so the framework only logs it on debug level while still retrying.
+      log.debug(
+          "Error during event processing {}, but was handled by the reconciler",
+          executionScope,
+          exception);
+    } else if (!retry.isLastAttempt()
         && exception instanceof KubernetesClientException ex
         && ex.getCode() == HttpURLConnection.HTTP_CONFLICT) {
       log.debug("Full client conflict error during event processing {}", executionScope, exception);

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/PostExecutionControl.java

@@ -27,6 +27,7 @@ final class PostExecutionControl<R extends HasMetadata> {
   private final Exception runtimeException;
 
   private Long reScheduleDelay = null;
+  private boolean errorHandledByReconciler = false;
 
   private PostExecutionControl(
       boolean finalizerRemoved,
@@ -68,6 +69,22 @@ public static <R extends HasMetadata> PostExecutionControl<R> exceptionDuringExe
     return new PostExecutionControl<>(false, null, false, exception);
   }
 
+  /**
+   * Marks that the exception was handled by the reconciler's {@code updateErrorStatus} (i.e. the
+   * reconciler did not return {@link
+   * io.javaoperatorsdk.operator.api.reconciler.ErrorStatusUpdateControl#defaultErrorProcessing()}),
+   * but the error is still retried. In this case the framework logs the error on a lower level,
+   * since the reconciler already had the chance to handle and log it as needed.
+   */
+  public PostExecutionControl<R> withErrorHandledByReconciler() {
+    this.errorHandledByReconciler = true;
+    return this;
+  }
+
+  public boolean isErrorHandledByReconciler() {
+    return errorHandledByReconciler;
+  }
+
   public Optional<R> getUpdatedCustomResource() {
     return Optional.ofNullable(updatedCustomResource);
   }

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java

@@ -266,7 +266,12 @@ public boolean isLastAttempt() {
       errorStatusUpdateControl.getScheduleDelay().ifPresent(postExecutionControl::withReSchedule);
       return postExecutionControl;
     }
-    throw e;
+    // The reconciler handled the error via updateErrorStatus (it did not return
+    // defaultErrorProcessing()) but still wants the error to be retried. The retry (and its
+    // backoff) is kept intact, but since the reconciler already had the chance to handle and log
+    // the error, the framework logs it on a lower level instead of emitting an "uncaught error"
+    // warning.
+    return PostExecutionControl.<P>exceptionDuringExecution(e).withErrorHandledByReconciler();
   }
 
   private PostExecutionControl<P> createPostExecutionControl(

operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcherTest.java

@@ -481,6 +481,40 @@ void callErrorStatusHandlerEvenOnFirstError() {
     assertThat(postExecControl.exceptionDuringExecution()).isTrue();
   }
 
+  @Test
+  void errorHandledByReconcilerMarkedWhenErrorStatusHandledButRetried() {
+    testCustomResource.addFinalizer(DEFAULT_FINALIZER);
+    reconciler.reconcile =
+        (r, c) -> {
+          throw new IllegalStateException("Error Status Test");
+        };
+    reconciler.errorHandler = () -> ErrorStatusUpdateControl.patchStatus(testCustomResource);
+
+    var postExecControl =
+        reconciliationDispatcher.handleExecution(
+            new ExecutionScope(null, null, false, false).setResource(testCustomResource));
+
+    assertThat(postExecControl.exceptionDuringExecution()).isTrue();
+    assertThat(postExecControl.isErrorHandledByReconciler()).isTrue();
+  }
+
+  @Test
+  void errorNotHandledByReconcilerOnDefaultErrorProcessing() {
+    testCustomResource.addFinalizer(DEFAULT_FINALIZER);
+    reconciler.reconcile =
+        (r, c) -> {
+          throw new IllegalStateException("Error Status Test");
+        };
+    reconciler.errorHandler = () -> ErrorStatusUpdateControl.defaultErrorProcessing();
+
+    var postExecControl =
+        reconciliationDispatcher.handleExecution(
+            new ExecutionScope(null, null, false, false).setResource(testCustomResource));
+
+    assertThat(postExecControl.exceptionDuringExecution()).isTrue();
+    assertThat(postExecControl.isErrorHandledByReconciler()).isFalse();
+  }
+
   @Test
   void errorHandlerCanInstructNoRetryWithUpdate() {
     testCustomResource.addFinalizer(DEFAULT_FINALIZER);