archiver: Return low-level errors

This commit changes the archiver so that low-level errors saving data to
the repo are returned to the caller (instead of being handled by the
error callback function). This correctly bubbles up errors like a full
temp file system and makes restic abort early and makes all other worker
goroutines exit.

internal/archiver/blob_saver.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"sync"
 
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 )
 
@@ -21,12 +23,11 @@ type BlobSaver struct {
 	knownBlobs restic.BlobSet
 
 	ch chan<- saveBlobJob
-	wg sync.WaitGroup
 }
 
 // NewBlobSaver returns a new blob. A worker pool is started, it is stopped
 // when ctx is cancelled.
-func NewBlobSaver(ctx context.Context, repo Saver, workers uint) *BlobSaver {
+func NewBlobSaver(ctx context.Context, g Goer, repo Saver, workers uint) *BlobSaver {
 	ch := make(chan saveBlobJob)
 	s := &BlobSaver{
 		repo:       repo,
@@ -35,8 +36,9 @@ func NewBlobSaver(ctx context.Context, repo Saver, workers uint) *BlobSaver {
 	}
 
 	for i := uint(0); i < workers; i++ {
-		s.wg.Add(1)
-		go s.worker(ctx, &s.wg, ch)
+		g.Go(func() error {
+			return s.worker(ctx, ch)
+		})
 	}
 
 	return s
@@ -47,7 +49,13 @@ func NewBlobSaver(ctx context.Context, repo Saver, workers uint) *BlobSaver {
 // previously unknown.
 func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer) FutureBlob {
 	ch := make(chan saveBlobResponse, 1)
-	s.ch <- saveBlobJob{BlobType: t, buf: buf, ch: ch}
+	select {
+	case s.ch <- saveBlobJob{BlobType: t, buf: buf, ch: ch}:
+	case <-ctx.Done():
+		debug.Log("not sending job, context is cancelled")
+		close(ch)
+		return FutureBlob{ch: ch}
+	}
 
 	return FutureBlob{ch: ch, length: len(buf.Data)}
 }
@@ -59,31 +67,28 @@ type FutureBlob struct {
 	res    saveBlobResponse
 }
 
-func (s *FutureBlob) wait() {
-	res, ok := <-s.ch
-	if ok {
-		s.res = res
+// Wait blocks until the result is available or the context is cancelled.
+func (s *FutureBlob) Wait(ctx context.Context) {
+	select {
+	case <-ctx.Done():
+		return
+	case res, ok := <-s.ch:
+		if ok {
+			s.res = res
+		}
 	}
 }
 
 // ID returns the ID of the blob after it has been saved.
 func (s *FutureBlob) ID() restic.ID {
-	s.wait()
 	return s.res.id
 }
 
 // Known returns whether or not the blob was already known.
 func (s *FutureBlob) Known() bool {
-	s.wait()
 	return s.res.known
 }
 
-// Err returns the error which may have occurred during save.
-func (s *FutureBlob) Err() error {
-	s.wait()
-	return s.res.err
-}
-
 // Length returns the length of the blob.
 func (s *FutureBlob) Length() int {
 	return s.length
@@ -98,10 +103,9 @@ type saveBlobJob struct {
 type saveBlobResponse struct {
 	id    restic.ID
 	known bool
-	err   error
 }
 
-func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) saveBlobResponse {
+func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) {
 	id := restic.Hash(buf)
 	h := restic.BlobHandle{ID: id, Type: t}
 
@@ -121,7 +125,7 @@ func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte)
 		return saveBlobResponse{
 			id:    id,
 			known: true,
-		}
+		}, nil
 	}
 
 	// check if the repo knows this blob
@@ -129,29 +133,38 @@ func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte)
 		return saveBlobResponse{
 			id:    id,
 			known: true,
-		}
+		}, nil
 	}
 
 	// otherwise we're responsible for saving it
 	_, err := s.repo.SaveBlob(ctx, t, buf, id)
+	if err != nil {
+		return saveBlobResponse{}, errors.Fatalf("unable to save data: %v", err)
+	}
+
 	return saveBlobResponse{
 		id:    id,
 		known: false,
-		err:   err,
-	}
+	}, nil
 }
 
-func (s *BlobSaver) worker(ctx context.Context, wg *sync.WaitGroup, jobs <-chan saveBlobJob) {
-	defer wg.Done()
+func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
 	for {
 		var job saveBlobJob
 		select {
 		case <-ctx.Done():
-			return
+			debug.Log("context is cancelled, exiting: %v", ctx.Err())
+			return nil
 		case job = <-jobs:
 		}
 
-		job.ch <- s.saveBlob(ctx, job.BlobType, job.buf.Data)
+		res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data)
+		if err != nil {
+			debug.Log("saveBlob returned error: %v", err)
+			close(job.ch)
+			return err
+		}
+		job.ch <- res
 		close(job.ch)
 		job.buf.Release()
 	}