restorer: pre-allocate files before loading chunks

internal/restorer/filerestorer.go

@@ -33,6 +33,7 @@ const (
 type fileInfo struct {
 	lock     sync.Mutex
 	flags    int
+	size     int64
 	location string      // file on local filesystem relative to restorer basedir
 	blobs    interface{} // blobs of the file
 }
@@ -74,8 +75,8 @@ func newFileRestorer(dst string,
 	}
 }
 
-func (r *fileRestorer) addFile(location string, content restic.IDs) {
-	r.files = append(r.files, &fileInfo{location: location, blobs: content})
+func (r *fileRestorer) addFile(location string, content restic.IDs, size int64) {
+	r.files = append(r.files, &fileInfo{location: location, blobs: content, size: size})
 }
 
 func (r *fileRestorer) targetPath(location string) string {
@@ -275,13 +276,15 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) {
 					// write other blobs after releasing the lock
 					file.lock.Lock()
 					create := file.flags&fileProgress == 0
+					createSize := int64(-1)
 					if create {
 						defer file.lock.Unlock()
 						file.flags |= fileProgress
+						createSize = file.size
 					} else {
 						file.lock.Unlock()
 					}
-					return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, create)
+					return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize)
 				}
 				err := writeToFile()
 				if err != nil {

internal/restorer/fileswriter.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/cespare/xxhash"
+	"github.com/restic/restic/internal/debug"
 )
 
 // writes blobs to target files.
@@ -33,7 +34,7 @@ func newFilesWriter(count int) *filesWriter {
 	}
 }
 
-func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bool) error {
+func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64) error {
 	bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
 
 	acquireWriter := func() (*os.File, error) {
@@ -46,7 +47,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 		}
 
 		var flags int
-		if create {
+		if createSize >= 0 {
 			flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY
 		} else {
 			flags = os.O_WRONLY
@@ -60,6 +61,18 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 		bucket.files[path] = wr
 		bucket.users[path] = 1
 
+		if createSize >= 0 {
+			err := preallocateFile(wr, createSize)
+			if err != nil {
+				// Just log the preallocate error but don't let it cause the restore process to fail.
+				// Preallocate might return an error if the filesystem (implementation) does not
+				// support preallocation or our parameters combination to the preallocate call
+				// This should yield a syscall.ENOTSUP error, but some other errors might also
+				// show up.
+				debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
+			}
+		}
+
 		return wr, nil
 	}
 

internal/restorer/fileswriter_test.go

@@ -16,19 +16,19 @@ func TestFilesWriterBasic(t *testing.T) {
 	f1 := dir + "/f1"
 	f2 := dir + "/f2"
 
-	rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, true))
+	rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, 2))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, true))
+	rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, 2))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, false))
+	rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, -1))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 
-	rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, false))
+	rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, -1))
 	rtest.Equals(t, 0, len(w.buckets[0].files))
 	rtest.Equals(t, 0, len(w.buckets[0].users))
 

internal/restorer/preallocate_darwin.go

@@ -0,0 +1,33 @@
+package restorer
+
+import (
+	"os"
+	"runtime"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+func preallocateFile(wr *os.File, size int64) error {
+	// try contiguous first
+	fst := unix.Fstore_t{
+		Flags:   unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL,
+		Posmode: unix.F_PEOFPOSMODE,
+		Offset:  0,
+		Length:  size,
+	}
+	_, err := unix.FcntlInt(wr.Fd(), unix.F_PREALLOCATE, int(uintptr(unsafe.Pointer(&fst))))
+
+	if err == nil {
+		return nil
+	}
+
+	// just take preallocation in any form, but still ask for everything
+	fst.Flags = unix.F_ALLOCATEALL
+	_, err = unix.FcntlInt(wr.Fd(), unix.F_PREALLOCATE, int(uintptr(unsafe.Pointer(&fst))))
+
+	// Keep struct alive until fcntl has returned
+	runtime.KeepAlive(fst)
+
+	return err
+}

internal/restorer/preallocate_linux.go

@@ -0,0 +1,16 @@
+package restorer
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func preallocateFile(wr *os.File, size int64) error {
+	if size <= 0 {
+		return nil
+	}
+	// int fallocate(int fd, int mode, off_t offset, off_t len)
+	// use mode = 0 to also change the file size
+	return unix.Fallocate(int(wr.Fd()), 0, 0, size)
+}

internal/restorer/preallocate_other.go

@@ -0,0 +1,11 @@
+// +build !linux,!darwin
+
+package restorer
+
+import "os"
+
+func preallocateFile(wr *os.File, size int64) error {
+	// Maybe truncate can help?
+	// Windows: This calls SetEndOfFile which preallocates space on disk
+	return wr.Truncate(size)
+}

internal/restorer/restorer.go

@@ -238,7 +238,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
 				idx.Add(node.Inode, node.DeviceID, location)
 			}
 
-			filerestorer.addFile(location, node.Content)
+			filerestorer.addFile(location, node.Content, int64(node.Size))
 
 			return nil
 		},