Reduce memory usage for fuse mount

This changes `repository.LoadBlob()` so that a destination buffer must
be provided, which enables the fuse code to use a buffer from a
`sync.Pool`. In addition, release the buffers when the file is closed.
At the moment, the max memory usage is defined by the max file size that
is read in one go (e.g. with `cat`). It could be further optimized by
implementing a LRU caching scheme.

cmd/restic/cmd_cat.go

@@ -162,9 +162,18 @@ func (cmd CmdCat) Execute(args []string) error {
 		return err
 
 	case "blob":
-		data, err := repo.LoadBlob(pack.Data, id)
-		if err == nil {
-			_, err = os.Stdout.Write(data)
+		_, tpe, _, length, err := repo.Index().Lookup(id)
+		if err != nil {
+			return err
+		}
+
+		if tpe != pack.Data {
+			return errors.New("wrong type for blob")
+		}
+
+		buf := make([]byte, length)
+		data, err := repo.LoadBlob(pack.Data, id, buf)
+		if err != nil {
 			return err
 		}
 

cmd/restic/fuse/file.go

@@ -1,6 +1,8 @@
 package fuse
 
 import (
+	"sync"
+
 	"github.com/restic/restic"
 	"github.com/restic/restic/pack"
 	"github.com/restic/restic/repository"
@@ -12,6 +14,7 @@ import (
 
 // Statically ensure that *file implements the given interface
 var _ = fs.HandleReader(&file{})
+var _ = fs.HandleReleaser(&file{})
 
 type file struct {
 	repo *repository.Repository
@@ -21,6 +24,14 @@ type file struct {
 	blobs [][]byte
 }
 
+const defaultBlobSize = 128 * 1024
+
+var blobPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, defaultBlobSize)
+	},
+}
+
 func newFile(repo *repository.Repository, node *restic.Node) (*file, error) {
 	sizes := make([]uint32, len(node.Content))
 	for i, blobID := range node.Content {
@@ -48,50 +59,69 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
 	if f.blobs[i] != nil {
-		blob = f.blobs[i]
-	} else {
-		blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i])
-		if err != nil {
-			return nil, err
-		}
-		f.blobs[i] = blob
+		return f.blobs[i], nil
 	}
 
+	buf := blobPool.Get().([]byte)
+	buf = buf[:cap(buf)]
+
+	if uint32(len(buf)) < f.sizes[i] {
+		if len(buf) > defaultBlobSize {
+			blobPool.Put(buf)
+		}
+		buf = make([]byte, f.sizes[i])
+	}
+
+	blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf)
+	if err != nil {
+		return nil, err
+	}
+	f.blobs[i] = blob
+
 	return blob, nil
 }
 
 func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
-	off := req.Offset
+	offset := req.Offset
 
 	// Skip blobs before the offset
 	startContent := 0
-	for off > int64(f.sizes[startContent]) {
-		off -= int64(f.sizes[startContent])
+	for offset > int64(f.sizes[startContent]) {
+		offset -= int64(f.sizes[startContent])
 		startContent++
 	}
 
-	content := make([]byte, req.Size)
-	allContent := content
-	for i := startContent; i < len(f.sizes); i++ {
+	dst := resp.Data[0:req.Size]
+	readBytes := 0
+	remainingBytes := req.Size
+	for i := startContent; remainingBytes > 0 && i < len(f.sizes); i++ {
 		blob, err := f.getBlobAt(i)
 		if err != nil {
 			return err
 		}
 
-		blob = blob[off:]
-		off = 0
+		if offset > 0 {
+			blob = blob[offset:len(blob)]
+			offset = 0
+		}
 
-		var copied int
-		if len(blob) > len(content) {
-			copied = copy(content[0:], blob[:len(content)])
-		} else {
-			copied = copy(content[0:], blob)
-		}
-		content = content[copied:]
-		if len(content) == 0 {
-			break
-		}
+		copied := copy(dst, blob)
+		remainingBytes -= copied
+		readBytes += copied
+
+		dst = dst[copied:]
+	}
+	resp.Data = resp.Data[:readBytes]
+
+	return nil
+}
+
+func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
+	for i := range f.blobs {
+		if f.blobs[i] != nil {
+			blobPool.Put(f.blobs[i])
+			f.blobs[i] = nil
+		}
 	}
-	resp.Data = allContent
 	return nil
 }