Merge pull request #5251 from MichaelEischer/rclone-retries

Retry temporary rclone backend errors

internal/backend/retry/backend_retry.go

@@ -127,12 +127,20 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
 		b = backoff.WithMaxRetries(b, 10)
 	}
 
+	permanentErrorAttempts := 1
+	if be.Backend.Properties().HasFlakyErrors {
+		permanentErrorAttempts = 5
+	}
+
 	err := retryNotifyErrorWithSuccess(
 		func() error {
 			err := f()
 			// don't retry permanent errors as those very likely cannot be fixed by retrying
 			// TODO remove IsNotExist(err) special cases when removing the feature flag
 			if feature.Flag.Enabled(feature.BackendErrorRedesign) && !errors.Is(err, &backoff.PermanentError{}) && be.Backend.IsPermanentError(err) {
+				permanentErrorAttempts--
+			}
+			if permanentErrorAttempts <= 0 {
 				return backoff.Permanent(err)
 			}
 			return err
@@ -166,7 +174,7 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
 			return nil
 		}
 
-		if be.Backend.HasAtomicReplace() {
+		if be.Backend.Properties().HasAtomicReplace {
 			debug.Log("Save(%v) failed with error: %v", h, err)
 			// there is no need to remove files from backends which can atomically replace files
 			// in fact if something goes wrong at the backend side the delete operation might delete the wrong instance of the file

internal/backend/retry/backend_retry_test.go

@@ -69,7 +69,12 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
 			calledRemove = true
 			return nil
 		},
-		HasAtomicReplaceFn: func() bool { return true },
+		PropertiesFn: func() backend.Properties {
+			return backend.Properties{
+				Connections:      2,
+				HasAtomicReplace: true,
+			}
+		},
 	}
 
 	TestFastRetries(t)
@@ -278,32 +283,52 @@ func TestBackendLoadRetry(t *testing.T) {
 	test.Equals(t, 2, attempt)
 }
 
-func TestBackendLoadNotExists(t *testing.T) {
+func testBackendLoadNotExists(t *testing.T, hasFlakyErrors bool) {
 	// load should not retry if the error matches IsNotExist
 	notFound := errors.New("not found")
 	attempt := 0
+	expectedAttempts := 1
+	if hasFlakyErrors {
+		expectedAttempts = 5
+	}
 
 	be := mock.NewBackend()
 	be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
 		attempt++
-		if attempt > 1 {
+		if attempt > expectedAttempts {
 			t.Fail()
 			return nil, errors.New("must not retry")
 		}
 		return nil, notFound
 	}
+	be.PropertiesFn = func() backend.Properties {
+		return backend.Properties{
+			Connections:    2,
+			HasFlakyErrors: hasFlakyErrors,
+		}
+	}
 	be.IsPermanentErrorFn = func(err error) bool {
 		return errors.Is(err, notFound)
 	}
 
 	TestFastRetries(t)
-	retryBackend := New(be, 10, nil, nil)
+	retryBackend := New(be, time.Second, nil, nil)
 
 	err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
 		return nil
 	})
 	test.Assert(t, be.IsPermanentErrorFn(err), "unexpected error %v", err)
-	test.Equals(t, 1, attempt)
+	test.Equals(t, expectedAttempts, attempt)
+}
+
+func TestBackendLoadNotExists(t *testing.T) {
+	// Without HasFlakyErrors, should fail after 1 attempt
+	testBackendLoadNotExists(t, false)
+}
+
+func TestBackendLoadNotExistsFlakyErrors(t *testing.T) {
+	// With HasFlakyErrors, should fail after attempt number 5
+	testBackendLoadNotExists(t, true)
 }
 
 func TestBackendLoadCircuitBreaker(t *testing.T) {