TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-14 13:11:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fs / archiver: convert to handle based interface

Browse Source 
The actual implementation still relies on file paths, but with the
abstraction layer in place, an FS implementation can ensure atomic file
accesses in the future.
This commit is contained in:
Michael Eischer
2024-11-02 20:27:38 +01:00
parent 2f2ce9add2
commit 48dbefc37e
18 changed files with 356 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
internal/archiver/archiver.go
View File

@@ -66,6 +66,11 @@ func (s *ItemStats) Add(other ItemStats) {
s.TreeSizeInRepo += other.TreeSizeInRepo
}
// ToNoder returns a restic.Node for a File.
type ToNoder interface {
ToNode(ignoreXattrListError bool) (*restic.Node, error)
}
type archiverRepo interface {
restic.Loader
restic.BlobSaver
@@ -257,8 +262,8 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I
}
// nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
node, err := arch.FS.NodeFromFileInfo(filename, fi, ignoreXattrListError)
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, meta ToNoder, ignoreXattrListError bool) (*restic.Node, error) {
node, err := meta.ToNode(ignoreXattrListError)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
@@ -308,20 +313,14 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, meta fs.File, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) {
debug.Log("%v %v", snPath, dir)
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
treeNode, names, err := arch.dirToNodeAndEntries(snPath, dir, meta)
if err != nil {
return futureNode{}, err
}
names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
if err != nil {
return futureNode{}, err
}
sort.Strings(names)
nodes := make([]futureNode, 0, len(names))
for _, name := range names {
@@ -359,6 +358,29 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi
return fn, nil
}
func (arch *Archiver) dirToNodeAndEntries(snPath, dir string, meta fs.File) (node *restic.Node, names []string, err error) {
err = meta.MakeReadable()
if err != nil {
return nil, nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}
node, err = arch.nodeFromFileInfo(snPath, dir, meta, false)
if err != nil {
return nil, nil, err
}
if node.Type != restic.NodeTypeDir {
return nil, nil, fmt.Errorf("directory %v changed type, refusing to archive", snPath)
}
names, err = meta.Readdirnames(-1)
if err != nil {
return nil, nil, fmt.Errorf("readdirnames %v failed: %w", dir, err)
}
sort.Strings(names)
return node, names, nil
}
// futureNode holds a reference to a channel that returns a FutureNodeResult
// or a reference to an already existing result. If the result is available
// immediately, then storing a reference directly requires less memory than
@@ -448,8 +470,23 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, true, nil
}
meta, err := arch.FS.OpenFile(target, fs.O_NOFOLLOW, true)
if err != nil {
debug.Log("open metadata for %v returned error: %v", target, err)
return filterError(err)
}
closeFile := true
defer func() {
if closeFile {
cerr := meta.Close()
if err == nil {
err = cerr
}
}
}()
// get file info and run remaining select functions that require file information
fi, err := arch.FS.Lstat(target)
fi, err := meta.Stat()
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
return filterError(err)
@@ -470,7 +507,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
@@ -497,28 +534,28 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW)
err := meta.MakeReadable()
if err != nil {
debug.Log("Openfile() for %v returned error: %v", target, err)
debug.Log("MakeReadable() for %v returned error: %v", target, err)
return filterError(err)
}
fi, err = file.Stat()
fi, err := meta.Stat()
if err != nil {
debug.Log("stat() on opened file %v returned error: %v", target, err)
_ = file.Close()
return filterError(err)
}
// make sure it's still a file
if !fi.Mode().IsRegular() {
err = errors.Errorf("file %v changed type, refusing to archive", target)
_ = file.Close()
return filterError(err)
}
closeFile = false
// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
fn = arch.fileSaver.Save(ctx, snPath, target, meta, func() {
arch.StartFile(snPath)
}, func() {
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
@@ -538,7 +575,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
return futureNode{}, false, err
}
fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
fn, err = arch.saveDir(ctx, snPath, target, meta, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.trackItem(snItem, previous, node, stats, time.Since(start))
})
@@ -554,7 +591,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous
default:
debug.Log(" %v other", target)
node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
node, err := arch.nodeFromFileInfo(snPath, target, meta, false)
if err != nil {
return futureNode{}, false, err
}
@@ -688,7 +725,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree,
}
func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, err error) {
meta, err := arch.FS.OpenFile(target, fs.O_RDONLY)
meta, err := arch.FS.OpenFile(target, 0, true)
if err != nil {
return nil, err
}
@@ -700,14 +737,9 @@ func (arch *Archiver) dirPathToNode(snPath, target string) (node *restic.Node, e
}()
debug.Log("%v, reading dir node data from %v", snPath, target)
fi, err := meta.Stat()
if err != nil {
return nil, errors.WithStack(err)
}
// in some cases reading xattrs for directories above the backup source is not allowed
// thus ignore errors for such folders.
node, err = arch.nodeFromFileInfo(snPath, target, fi, true)
node, err = arch.nodeFromFileInfo(snPath, target, meta, true)
if err != nil {
return nil, err
}