backend: factor out connection limiting and parameter validation

The SemaphoreBackend now uniformly enforces the limit of concurrent
backend operations. In addition, it unifies the parameter validation.

The List() methods no longer uses a semaphore. Restic already never runs
multiple list operations in parallel.

By managing the semaphore in a wrapper backend, the sections that hold a
semaphore token grow slightly. However, the main bottleneck is IO, so
this shouldn't make much of a difference.

The key insight that enables the SemaphoreBackend is that all of the
complex semaphore handling in `openReader()` still happens within the
original call to `Load()`. Thus, getting and releasing the semaphore
tokens can be refactored to happen directly in `Load()`. This eliminates
the need for wrapping the reader in `openReader()` to release the token.

internal/backend/sema/semaphore.go

@@ -2,64 +2,30 @@
 package sema
 
 import (
-	"context"
-	"io"
-
+	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 )
 
-// A Semaphore limits access to a restricted resource.
-type Semaphore struct {
+// A semaphore limits access to a restricted resource.
+type semaphore struct {
 	ch chan struct{}
 }
 
-// New returns a new semaphore with capacity n.
-func New(n uint) (Semaphore, error) {
+// newSemaphore returns a new semaphore with capacity n.
+func newSemaphore(n uint) (semaphore, error) {
 	if n == 0 {
-		return Semaphore{}, errors.New("capacity must be a positive number")
+		return semaphore{}, errors.New("capacity must be a positive number")
 	}
-	return Semaphore{
+	return semaphore{
 		ch: make(chan struct{}, n),
 	}, nil
 }
 
 // GetToken blocks until a Token is available.
-func (s Semaphore) GetToken() { s.ch <- struct{}{} }
+func (s semaphore) GetToken() {
+	s.ch <- struct{}{}
+	debug.Log("acquired token")
+}
 
 // ReleaseToken returns a token.
-func (s Semaphore) ReleaseToken() { <-s.ch }
-
-// ReleaseTokenOnClose wraps an io.ReadCloser to return a token on Close.
-// Before returning the token, cancel, if not nil, will be run
-// to free up context resources.
-func (s Semaphore) ReleaseTokenOnClose(rc io.ReadCloser, cancel context.CancelFunc) io.ReadCloser {
-	return &wrapReader{ReadCloser: rc, sem: s, cancel: cancel}
-}
-
-type wrapReader struct {
-	io.ReadCloser
-	eofSeen bool
-	sem     Semaphore
-	cancel  context.CancelFunc
-}
-
-func (wr *wrapReader) Read(p []byte) (int, error) {
-	if wr.eofSeen { // XXX Why do we do this?
-		return 0, io.EOF
-	}
-
-	n, err := wr.ReadCloser.Read(p)
-	if err == io.EOF {
-		wr.eofSeen = true
-	}
-	return n, err
-}
-
-func (wr *wrapReader) Close() error {
-	err := wr.ReadCloser.Close()
-	if wr.cancel != nil {
-		wr.cancel()
-	}
-	wr.sem.ReleaseToken()
-	return err
-}
+func (s semaphore) ReleaseToken() { <-s.ch }