fs / archiver: convert to handle based interface

The actual implementation still relies on file paths, but with the
abstraction layer in place, an FS implementation can ensure atomic file
accesses in the future.
This commit is contained in:
Michael Eischer
2024-11-02 20:27:38 +01:00
parent 2f2ce9add2
commit 48dbefc37e
18 changed files with 356 additions and 284 deletions

View File

@@ -66,6 +66,11 @@ func (s *ItemStats) Add(other ItemStats) {
s.TreeSizeInRepo += other.TreeSizeInRepo
}
// ToNoder returns a restic.Node for a File.
type ToNoder interface {
ToNode(ignoreXattrListError bool) (*restic.Node, error)
}
type archiverRepo interface {
restic.Loader
restic.BlobSaver
@@ -257,8 +262,8 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
}
// nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
node, err := arch.FS.NodeFromFileInfo(filename, fi, ignoreXattrListError)
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
node, err := meta.ToNode(ignoreXattrListError)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
@@ -308,20 +313,14 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
debug.Log("%v %v", snPath, dir)
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
if err != nil {
return futureNode{}, err
}
names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
if err != nil {
return futureNode{}, err
}
sort.Strings(names)
nodes := make([]futureNode, 0, len(names))
for _, name := range names {
@@ -359,6 +358,29 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi
return fn, nil
}
func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *restic.Node, names []string, err error) {
err = meta.MakeReadable()
if err != nil {
return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}
node, err = arch.nodeFromFileInfo(snPath, dir, meta, false)
if err != nil {
return nil, nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, nil, fmt.Errorf("directory %v changed type, refusing to archive", snPath)
}
names, err = meta.Readdirnames(-1)
if err != nil {
return nil, nil, fmt.Errorf("readdirnames %v failed: %w", dir, err)
}
sort.Strings(names)
return node, names, nil
}
// futureNode holds a reference to a channel that returns a FutureNodeResult
// or a reference to an already existing result. If the result is available
// immediately, then storing a reference directly requires less memory than
@@ -448,8 +470,23 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, true, nil
}
meta, err := arch.FS.OpenFile(target, fs.O_NOFOLLOW, true)
if err != nil {
debug.Log("open metadata for %v returned error: %v", target, err)
return filterError(err)
}
closeFile := true
defer func() {
if closeFile {
cerr := meta.Close()
if err == nil {
err = cerr
}
}
}()
// get file info and run remaining select functions that require file information
fi, err := arch.FS.Lstat(target)
fi, err := meta.Stat()
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
return filterError(err)
@@ -470,7 +507,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
@@ -497,28 +534,28 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW)
err := meta.MakeReadable()
if err != nil {
debug.Log("Openfile() for %v returned error: %v", target, err)
debug.Log("MakeReadable() for %v returned error: %v", target, err)
return filterError(err)
}
fi, err = file.Stat()
fi, err := meta.Stat()
if err != nil {
debug.Log("stat() on opened file %v returned error: %v", target, err)
_ = file.Close()
return filterError(err)
}
// make sure it's still a file
if !fi.Mode().IsRegular() {
err = errors.Errorf("file %v changed type, refusing to archive", target)
_ = file.Close()
return filterError(err)
}
closeFile = false
// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
fn = arch.fileSaver.Save(ctx, snPath, target, meta, func() {
arch.StartFile(snPath)
}, func() {
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
@@ -538,7 +575,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, false, err
}
fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.trackItem(snItem, previous, node, stats, time.Since(start))
})
@@ -554,7 +591,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
default:
debug.Log(" %v other", target)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
@@ -688,7 +725,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
}
func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, err error) {
meta, err := arch.FS.OpenFile(target, fs.O_RDONLY)
meta, err := arch.FS.OpenFile(target, 0, true)
if err != nil {
return nil, err
}
@@ -700,14 +737,9 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e
}()
debug.Log("%v, reading dir node data from %v", snPath, target)
fi, err := meta.Stat()
if err != nil {
return nil, errors.WithStack(err)
}
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, target, fi, true)
node, err = arch.nodeFromFileInfo(snPath, target, meta, true)
if err != nil {
return nil, err
}

View File

@@ -76,17 +76,12 @@ func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS
startCallback = true
}
file, err := arch.FS.OpenFile(filename, fs.O_RDONLY|fs.O_NOFOLLOW)
file, err := arch.FS.OpenFile(filename, fs.O_NOFOLLOW, false)
if err != nil {
t.Fatal(err)
}
fi, err := file.Stat()
if err != nil {
t.Fatal(err)
}
res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, completeReading, complete)
res := arch.fileSaver.Save(ctx, "/", filename, file, start, completeReading, complete)
fnr := res.take(ctx)
if fnr.err != nil {
@@ -556,11 +551,12 @@ func rename(t testing.TB, oldname, newname string) {
}
}
func nodeFromFI(t testing.TB, fs fs.FS, filename string, fi os.FileInfo) *restic.Node {
node, err := fs.NodeFromFileInfo(filename, fi, false)
if err != nil {
t.Fatal(err)
}
func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *restic.Node {
meta, err := localFs.OpenFile(filename, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
node, err := meta.ToNode(false)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
return node
}
@@ -688,7 +684,7 @@ func TestFileChanged(t *testing.T) {
fs := &fs.Local{}
fiBefore := lstat(t, filename)
node := nodeFromFI(t, fs, filename, fiBefore)
node := nodeFromFile(t, fs, filename)
if fileChanged(fs, fiBefore, node, 0) {
t.Fatalf("unchanged file detected as changed")
@@ -729,8 +725,8 @@ func TestFilChangedSpecialCases(t *testing.T) {
t.Run("type-change", func(t *testing.T) {
fi := lstat(t, filename)
node := nodeFromFI(t, &fs.Local{}, filename, fi)
node.Type = "restic.NodeTypeSymlink"
node := nodeFromFile(t, &fs.Local{}, filename)
node.Type = restic.NodeTypeSymlink
if !fileChanged(&fs.Local{}, fi, node, 0) {
t.Fatal("node with changed type detected as unchanged")
}
@@ -834,7 +830,8 @@ func TestArchiverSaveDir(t *testing.T) {
wg, ctx := errgroup.WithContext(context.Background())
repo.StartPackUploader(ctx, wg)
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
testFS := fs.Track{FS: fs.Local{}}
arch := New(repo, testFS, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
@@ -846,15 +843,11 @@ func TestArchiverSaveDir(t *testing.T) {
back := rtest.Chdir(t, chdir)
defer back()
fi, err := os.Lstat(test.target)
if err != nil {
t.Fatal(err)
}
ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
meta, err := testFS.OpenFile(test.target, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", test.target, meta, nil, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fnr := ft.take(ctx)
node, stats := fnr.node, fnr.stats
@@ -916,19 +909,16 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
wg, ctx := errgroup.WithContext(context.TODO())
repo.StartPackUploader(ctx, wg)
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
testFS := fs.Track{FS: fs.Local{}}
arch := New(repo, testFS, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
fi, err := os.Lstat(tempdir)
if err != nil {
t.Fatal(err)
}
ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
meta, err := testFS.OpenFile(tempdir, fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", tempdir, meta, nil, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fnr := ft.take(ctx)
node, stats := fnr.node, fnr.stats
@@ -1665,8 +1655,8 @@ type MockFS struct {
bytesRead map[string]int // tracks bytes read from all opened files
}
func (m *MockFS) OpenFile(name string, flag int) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag)
func (m *MockFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return f, err
}
@@ -2056,12 +2046,12 @@ type TrackFS struct {
m sync.Mutex
}
func (m *TrackFS) OpenFile(name string, flag int) (fs.File, error) {
func (m *TrackFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
m.m.Lock()
m.opened[name]++
m.m.Unlock()
return m.FS.OpenFile(name, flag)
return m.FS.OpenFile(name, flag, metadataOnly)
}
type failSaveRepo struct {
@@ -2210,48 +2200,39 @@ func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot
return snapshot, node
}
// StatFS allows overwriting what is returned by the Lstat function.
type StatFS struct {
type overrideFS struct {
fs.FS
OverrideLstat map[string]os.FileInfo
OnlyOverrideStat bool
overrideFI os.FileInfo
overrideNode *restic.Node
overrideErr error
}
func (fs *StatFS) Lstat(name string) (os.FileInfo, error) {
if !fs.OnlyOverrideStat {
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
return fi, nil
}
func (m *overrideFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := m.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return f, err
}
return fs.FS.Lstat(name)
}
func (fs *StatFS) OpenFile(name string, flags int) (fs.File, error) {
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
f, err := fs.FS.OpenFile(name, flags)
if err != nil {
return nil, err
}
wrappedFile := fileStat{
File: f,
fi: fi,
}
return wrappedFile, nil
if filepath.Base(name) == "testfile" {
return &overrideFile{f, m}, nil
}
return fs.FS.OpenFile(name, flags)
return f, nil
}
type fileStat struct {
type overrideFile struct {
fs.File
fi os.FileInfo
ofs *overrideFS
}
func (f fileStat) Stat() (os.FileInfo, error) {
return f.fi, nil
func (f overrideFile) Stat() (os.FileInfo, error) {
return f.ofs.overrideFI, nil
}
func (f overrideFile) ToNode(ignoreXattrListError bool) (*restic.Node, error) {
if f.ofs.overrideNode == nil {
return f.File.ToNode(ignoreXattrListError)
}
return f.ofs.overrideNode, f.ofs.overrideErr
}
// used by wrapFileInfo, use untyped const in order to avoid having a version
@@ -2279,17 +2260,18 @@ func TestMetadataChanged(t *testing.T) {
// get metadata
fi := lstat(t, "testfile")
localFS := &fs.Local{}
want, err := localFS.NodeFromFileInfo("testfile", fi, false)
if err != nil {
t.Fatal(err)
}
meta, err := localFS.OpenFile("testfile", fs.O_NOFOLLOW, true)
rtest.OK(t, err)
want, err := meta.ToNode(false)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fs := &StatFS{
FS: localFS,
OverrideLstat: map[string]os.FileInfo{
"testfile": fi,
},
fs := &overrideFS{
FS: localFS,
overrideFI: fi,
overrideNode: &restic.Node{},
}
*fs.overrideNode = *want
sn, node2 := snapshot(t, repo, fs, nil, "testfile")
@@ -2309,7 +2291,8 @@ func TestMetadataChanged(t *testing.T) {
}
// modify the mode by wrapping it in a new struct, uses the consts defined above
fs.OverrideLstat["testfile"] = wrapFileInfo(fi)
fs.overrideFI = wrapFileInfo(fi)
rtest.Assert(t, !fileChanged(fs, fs.overrideFI, node2, 0), "testfile must not be considered as changed")
// set the override values in the 'want' node which
want.Mode = 0400
@@ -2318,16 +2301,13 @@ func TestMetadataChanged(t *testing.T) {
want.UID = 51234
want.GID = 51235
}
// no user and group name
want.User = ""
want.Group = ""
// update mock node accordingly
fs.overrideNode.Mode = 0400
fs.overrideNode.UID = want.UID
fs.overrideNode.GID = want.GID
// make another snapshot
_, node3 := snapshot(t, repo, fs, sn, "testfile")
// Override username and group to empty string - in case underlying system has user with UID 51234
// See https://github.com/restic/restic/issues/2372
node3.User = ""
node3.Group = ""
// make sure that metadata was recorded successfully
if !cmp.Equal(want, node3) {
@@ -2342,7 +2322,7 @@ func TestMetadataChanged(t *testing.T) {
func TestRacyFileSwap(t *testing.T) {
files := TestDir{
"file": TestFile{
"testfile": TestFile{
Content: "foo bar test file",
},
}
@@ -2354,14 +2334,11 @@ func TestRacyFileSwap(t *testing.T) {
// get metadata of current folder
fi := lstat(t, ".")
tempfile := filepath.Join(tempdir, "file")
tempfile := filepath.Join(tempdir, "testfile")
statfs := &StatFS{
FS: fs.Local{},
OverrideLstat: map[string]os.FileInfo{
tempfile: fi,
},
OnlyOverrideStat: true,
statfs := &overrideFS{
FS: fs.Local{},
overrideFI: fi,
}
ctx, cancel := context.WithCancel(context.Background())
@@ -2388,14 +2365,19 @@ func TestRacyFileSwap(t *testing.T) {
}
}
type mockToNoder struct {
node *restic.Node
err error
}
func (m *mockToNoder) ToNode(_ bool) (*restic.Node, error) {
return m.node, m.err
}
func TestMetadataBackupErrorFiltering(t *testing.T) {
tempdir := t.TempDir()
repo := repository.TestRepository(t)
filename := filepath.Join(tempdir, "file")
rtest.OK(t, os.WriteFile(filename, []byte("example"), 0o600))
fi, err := os.Stat(filename)
rtest.OK(t, err)
repo := repository.TestRepository(t)
arch := New(repo, fs.Local{}, Options{})
@@ -2406,15 +2388,24 @@ func TestMetadataBackupErrorFiltering(t *testing.T) {
return replacementErr
}
nonExistNoder := &mockToNoder{
node: &restic.Node{Type: restic.NodeTypeFile},
err: fmt.Errorf("not found"),
}
// check that errors from reading extended metadata are properly filtered
node, err := arch.nodeFromFileInfo("file", filename+"invalid", fi, false)
node, err := arch.nodeFromFileInfo("file", filename+"invalid", nonExistNoder, false)
rtest.Assert(t, node != nil, "node is missing")
rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err)
rtest.Assert(t, filteredErr != nil, "missing inner error")
// check that errors from reading irregular file are not filtered
filteredErr = nil
node, err = arch.nodeFromFileInfo("file", filename, wrapIrregularFileInfo(fi), false)
nonExistNoder = &mockToNoder{
node: &restic.Node{Type: restic.NodeTypeIrregular},
err: fmt.Errorf(`unsupported file type "irregular"`),
}
node, err = arch.nodeFromFileInfo("file", filename, nonExistNoder, false)
rtest.Assert(t, node != nil, "node is missing")
rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered")
rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err)
@@ -2434,17 +2425,19 @@ func TestIrregularFile(t *testing.T) {
tempfile := filepath.Join(tempdir, "testfile")
fi := lstat(t, "testfile")
statfs := &StatFS{
FS: fs.Local{},
OverrideLstat: map[string]os.FileInfo{
tempfile: wrapIrregularFileInfo(fi),
override := &overrideFS{
FS: fs.Local{},
overrideFI: wrapIrregularFileInfo(fi),
overrideNode: &restic.Node{
Type: restic.NodeTypeIrregular,
},
overrideErr: fmt.Errorf(`unsupported file type "irregular"`),
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
arch := New(repo, fs.Track{FS: statfs}, Options{})
arch := New(repo, fs.Track{FS: override}, Options{})
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
if err == nil {
t.Fatalf("Save() should have failed")

View File

@@ -57,12 +57,8 @@ func wrapIrregularFileInfo(fi os.FileInfo) os.FileInfo {
}
func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) {
fi := lstat(t, name)
fs := &fs.Local{}
want, err := fs.NodeFromFileInfo(name, fi, false)
rtest.OK(t, err)
_, node := snapshot(t, repo, fs, nil, name)
want := nodeFromFile(t, &fs.Local{}, name)
_, node := snapshot(t, repo, &fs.Local{}, nil, name)
return want, node
}

View File

@@ -135,9 +135,9 @@ func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache,
return rejected
}
func isDirExcludedByFile(dir, tagFilename, header string, fs fs.FS, warnf func(msg string, args ...interface{})) bool {
tf := fs.Join(dir, tagFilename)
_, err := fs.Lstat(tf)
func isDirExcludedByFile(dir, tagFilename, header string, fsInst fs.FS, warnf func(msg string, args ...interface{})) bool {
tf := fsInst.Join(dir, tagFilename)
_, err := fsInst.Lstat(tf)
if errors.Is(err, os.ErrNotExist) {
return false
}
@@ -153,7 +153,7 @@ func isDirExcludedByFile(dir, tagFilename, header string, fs fs.FS, warnf func(m
// From this stage, errors mean tagFilename exists but it is malformed.
// Warnings will be generated so that the user is informed that the
// indented ignore-action is not performed.
f, err := fs.OpenFile(tf, os.O_RDONLY)
f, err := fsInst.OpenFile(tf, fs.O_RDONLY, false)
if err != nil {
warnf("could not open exclusion tagfile: %v", err)
return false

View File

@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"os"
"sync"
"github.com/restic/chunker"
@@ -29,7 +28,7 @@ type fileSaver struct {
CompleteBlob func(bytes uint64)
NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error)
NodeFromFileInfo func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error)
}
// newFileSaver returns a new file saver. A worker pool with fileWorkers is
@@ -71,13 +70,12 @@ type fileCompleteFunc func(*restic.Node, ItemStats)
// file is closed by Save. completeReading is only called if the file was read
// successfully. complete is always called. If completeReading is called, then
// this will always happen before calling complete.
func (s *fileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), completeReading func(), complete fileCompleteFunc) futureNode {
func (s *fileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, start func(), completeReading func(), complete fileCompleteFunc) futureNode {
fn, ch := newFutureNode()
job := saveFileJob{
snPath: snPath,
target: target,
file: file,
fi: fi,
ch: ch,
start: start,
@@ -100,7 +98,6 @@ type saveFileJob struct {
snPath string
target string
file fs.File
fi os.FileInfo
ch chan<- futureNodeResult
start func()
@@ -109,7 +106,7 @@ type saveFileJob struct {
}
// saveFile stores the file f in the repo, then closes it.
func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func(), finishReading func(), finish func(res futureNodeResult)) {
func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, start func(), finishReading func(), finish func(res futureNodeResult)) {
start()
fnr := futureNodeResult{
@@ -156,7 +153,7 @@ func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
debug.Log("%v", snPath)
node, err := s.NodeFromFileInfo(snPath, target, fi, false)
node, err := s.NodeFromFileInfo(snPath, target, f, false)
if err != nil {
_ = f.Close()
completeError(err)
@@ -262,7 +259,7 @@ func (s *fileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
}
}
s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.fi, job.start, func() {
s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.start, func() {
if job.completeReading != nil {
job.completeReading()
}

View File

@@ -30,7 +30,7 @@ func createTestFiles(t testing.TB, num int) (files []string) {
return files
}
func startFileSaver(ctx context.Context, t testing.TB, fs fs.FS) (*fileSaver, context.Context, *errgroup.Group) {
func startFileSaver(ctx context.Context, t testing.TB, fsInst fs.FS) (*fileSaver, context.Context, *errgroup.Group) {
wg, ctx := errgroup.WithContext(ctx)
saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *buffer, _ string, cb func(saveBlobResponse)) {
@@ -49,8 +49,8 @@ func startFileSaver(ctx context.Context, t testing.TB, fs fs.FS) (*fileSaver, co
}
s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers)
s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
return fs.NodeFromFileInfo(filename, fi, ignoreXattrListError)
s.NodeFromFileInfo = func(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
return meta.ToNode(ignoreXattrListError)
}
return s, ctx, wg
@@ -72,17 +72,12 @@ func TestFileSaver(t *testing.T) {
var results []futureNode
for _, filename := range files {
f, err := testFs.OpenFile(filename, os.O_RDONLY)
f, err := testFs.OpenFile(filename, os.O_RDONLY, false)
if err != nil {
t.Fatal(err)
}
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
ff := s.Save(ctx, filename, filename, f, fi, startFn, completeReadingFn, completeFn)
ff := s.Save(ctx, filename, filename, f, startFn, completeReadingFn, completeFn)
results = append(results, ff)
}