TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-08-25 20:47:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5143 from MichaelEischer/fs-handle-interface

Browse Source 
fs: rework FS interface to be handle based
This commit is contained in:
Michael Eischer
2024-11-30 15:29:31 +01:00
committed by GitHub
parent 8644bb145b b51bf0c0c4
commit 8642049532
31 changed files with 817 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/fs/const_unix.go
View File

@@ -7,3 +7,6 @@ import "syscall"
// O_NOFOLLOW instructs the kernel to not follow symlinks when opening a file.
const O_NOFOLLOW int = syscall.O_NOFOLLOW
// O_DIRECTORY instructs the kernel to only open directories.
const O_DIRECTORY int = syscall.O_DIRECTORY

11
internal/fs/const_windows.go
View File

@@ -3,5 +3,12 @@
package fs
// O_NOFOLLOW is a noop on Windows.
const O_NOFOLLOW int = 0
// TODO honor flags when opening files
// O_NOFOLLOW is currently only interpreted by FS.OpenFile in metadataOnly mode and ignored by OpenFile.
// The value of the constant is invented and only for use within this fs package. It must not be used in other contexts.
// It must not conflict with the other O_* values from go/src/syscall/types_windows.go
const O_NOFOLLOW int = 0x40000000
// O_DIRECTORY is a noop on Windows.
const O_DIRECTORY int = 0

9
internal/fs/file.go
View File

@@ -3,6 +3,7 @@ package fs
import (
"fmt"
"os"
"runtime"
)
// MkdirAll creates a directory named path, along with any necessary parents,
@@ -47,6 +48,9 @@ func Lstat(name string) (os.FileInfo, error) {
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
if runtime.GOOS == "windows" {
flag &^= O_NOFOLLOW
}
return os.OpenFile(fixpath(name), flag, perm)
}
@@ -64,9 +68,10 @@ func ResetPermissions(path string) error {
return nil
}
// Readdirnames returns a list of file in a directory. Flags are passed to fs.OpenFile. O_RDONLY is implied.
// Readdirnames returns a list of file in a directory. Flags are passed to fs.OpenFile.
// O_RDONLY and O_DIRECTORY are implied.
func Readdirnames(filesystem FS, dir string, flags int) ([]string, error) {
f, err := filesystem.OpenFile(dir, O_RDONLY|flags, 0)
f, err := filesystem.OpenFile(dir, O_RDONLY|O_DIRECTORY|flags, false)
if err != nil {
return nil, fmt.Errorf("openfile for readdirnames failed: %w", err)
}

22
internal/fs/file_unix_test.go Normal file
View File

@@ -0,0 +1,22 @@
//go:build unix
package fs
import (
"path/filepath"
"syscall"
"testing"
"github.com/restic/restic/internal/errors"
rtest "github.com/restic/restic/internal/test"
)
func TestReaddirnamesFifo(t *testing.T) {
// should not block when reading from a fifo instead of a directory
tempdir := t.TempDir()
fifoFn := filepath.Join(tempdir, "fifo")
rtest.OK(t, mkfifo(fifoFn, 0o600))
_, err := Readdirnames(&Local{}, fifoFn, 0)
rtest.Assert(t, errors.Is(err, syscall.ENOTDIR), "unexpected error %v", err)
}

116
internal/fs/fs_local.go
View File

@@ -20,24 +20,16 @@ func (fs Local) VolumeName(path string) string {
return filepath.VolumeName(path)
}
// OpenFile is the generalized open call; most users will use Open
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func (fs Local) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
f, err := os.OpenFile(fixpath(name), flag, perm)
if err != nil {
return nil, err
}
_ = setFlags(f)
return f, nil
}
// Stat returns a FileInfo describing the named file. If there is an error, it
// will be of type *PathError.
func (fs Local) Stat(name string) (os.FileInfo, error) {
return os.Stat(fixpath(name))
// OpenFile opens a file or directory for reading.
//
// If metadataOnly is set, an implementation MUST return a File object for
// arbitrary file types including symlinks. The implementation may internally use
// the given file path or a file handle. In particular, an implementation may
// delay actually accessing the underlying filesystem.
//
// Only the O_NOFOLLOW and O_DIRECTORY flags are supported.
func (fs Local) OpenFile(name string, flag int, metadataOnly bool) (File, error) {
return newLocalFile(name, flag, metadataOnly)
}
// Lstat returns the FileInfo structure describing the named file.
@@ -59,10 +51,6 @@ func (fs Local) ExtendedStat(fi os.FileInfo) ExtendedFileInfo {
return ExtendedStat(fi)
}
func (fs Local) NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
return nodeFromFileInfo(path, fi, ignoreXattrListError)
}
// Join joins any number of path elements into a single path, adding a
// Separator if necessary. Join calls Clean on the result; in particular, all
// empty strings are ignored. On Windows, the result is a UNC path if and only
@@ -103,3 +91,87 @@ func (fs Local) Base(path string) string {
func (fs Local) Dir(path string) string {
return filepath.Dir(path)
}
type localFile struct {
name string
flag int
f *os.File
fi os.FileInfo
}
// See the File interface for a description of each method
var _ File = &localFile{}
func newLocalFile(name string, flag int, metadataOnly bool) (*localFile, error) {
var f *os.File
if !metadataOnly {
var err error
f, err = os.OpenFile(fixpath(name), flag, 0)
if err != nil {
return nil, err
}
_ = setFlags(f)
}
return &localFile{
name: name,
flag: flag,
f: f,
}, nil
}
func (f *localFile) MakeReadable() error {
if f.f != nil {
panic("file is already readable")
}
newF, err := newLocalFile(f.name, f.flag, false)
if err != nil {
return err
}
// replace state and also reset cached FileInfo
*f = *newF
return nil
}
func (f *localFile) cacheFI() error {
if f.fi != nil {
return nil
}
var err error
if f.f != nil {
f.fi, err = f.f.Stat()
} else if f.flag&O_NOFOLLOW != 0 {
f.fi, err = os.Lstat(f.name)
} else {
f.fi, err = os.Stat(f.name)
}
return err
}
func (f *localFile) Stat() (os.FileInfo, error) {
err := f.cacheFI()
// the call to cacheFI MUST happen before reading from f.fi
return f.fi, err
}
func (f *localFile) ToNode(ignoreXattrListError bool) (*restic.Node, error) {
if err := f.cacheFI(); err != nil {
return nil, err
}
return nodeFromFileInfo(f.name, f.fi, ignoreXattrListError)
}
func (f *localFile) Read(p []byte) (n int, err error) {
return f.f.Read(p)
}
func (f *localFile) Readdirnames(n int) ([]string, error) {
return f.f.Readdirnames(n)
}
func (f *localFile) Close() error {
if f.f != nil {
return f.f.Close()
}
return nil
}

222
internal/fs/fs_local_test.go Normal file
View File

@@ -0,0 +1,222 @@
package fs
import (
"io"
"os"
"path/filepath"
"slices"
"testing"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type fsLocalMetadataTestcase struct {
name string
follow bool
setup func(t *testing.T, path string)
nodeType restic.NodeType
}
func TestFSLocalMetadata(t *testing.T) {
for _, test := range []fsLocalMetadataTestcase{
{
name: "file",
setup: func(t *testing.T, path string) {
rtest.OK(t, os.WriteFile(path, []byte("example"), 0o600))
},
nodeType: restic.NodeTypeFile,
},
{
name: "directory",
setup: func(t *testing.T, path string) {
rtest.OK(t, os.Mkdir(path, 0o600))
},
nodeType: restic.NodeTypeDir,
},
{
name: "symlink",
setup: func(t *testing.T, path string) {
rtest.OK(t, os.Symlink(path+"old", path))
},
nodeType: restic.NodeTypeSymlink,
},
{
name: "symlink file",
follow: true,
setup: func(t *testing.T, path string) {
rtest.OK(t, os.WriteFile(path+"file", []byte("example"), 0o600))
rtest.OK(t, os.Symlink(path+"file", path))
},
nodeType: restic.NodeTypeFile,
},
} {
runFSLocalTestcase(t, test)
}
}
func runFSLocalTestcase(t *testing.T, test fsLocalMetadataTestcase) {
t.Run(test.name, func(t *testing.T) {
tmp := t.TempDir()
path := filepath.Join(tmp, "item")
test.setup(t, path)
testFs := &Local{}
flags := 0
if !test.follow {
flags |= O_NOFOLLOW
}
f, err := testFs.OpenFile(path, flags, true)
rtest.OK(t, err)
checkMetadata(t, f, path, test.follow, test.nodeType)
rtest.OK(t, f.Close())
})
}
func checkMetadata(t *testing.T, f File, path string, follow bool, nodeType restic.NodeType) {
fi, err := f.Stat()
rtest.OK(t, err)
var fi2 os.FileInfo
if follow {
fi2, err = os.Stat(path)
} else {
fi2, err = os.Lstat(path)
}
rtest.OK(t, err)
assertFIEqual(t, fi2, fi)
node, err := f.ToNode(false)
rtest.OK(t, err)
// ModTime is likely unique per file, thus it provides a good indication that it is from the correct file
rtest.Equals(t, fi.ModTime(), node.ModTime, "node ModTime")
rtest.Equals(t, nodeType, node.Type, "node Type")
}
func assertFIEqual(t *testing.T, want os.FileInfo, got os.FileInfo) {
t.Helper()
rtest.Equals(t, want.Name(), got.Name(), "Name")
rtest.Equals(t, want.IsDir(), got.IsDir(), "IsDir")
rtest.Equals(t, want.ModTime(), got.ModTime(), "ModTime")
rtest.Equals(t, want.Mode(), got.Mode(), "Mode")
rtest.Equals(t, want.Size(), got.Size(), "Size")
}
func TestFSLocalRead(t *testing.T) {
testFSLocalRead(t, false)
testFSLocalRead(t, true)
}
func testFSLocalRead(t *testing.T, makeReadable bool) {
tmp := t.TempDir()
path := filepath.Join(tmp, "item")
testdata := "example"
rtest.OK(t, os.WriteFile(path, []byte(testdata), 0o600))
f := openReadable(t, path, makeReadable)
checkMetadata(t, f, path, false, restic.NodeTypeFile)
data, err := io.ReadAll(f)
rtest.OK(t, err)
rtest.Equals(t, testdata, string(data), "file content mismatch")
rtest.OK(t, f.Close())
}
func openReadable(t *testing.T, path string, useMakeReadable bool) File {
testFs := &Local{}
f, err := testFs.OpenFile(path, O_NOFOLLOW, useMakeReadable)
rtest.OK(t, err)
if useMakeReadable {
// file was opened as metadataOnly. open for reading
rtest.OK(t, f.MakeReadable())
}
return f
}
func TestFSLocalReaddir(t *testing.T) {
testFSLocalReaddir(t, false)
testFSLocalReaddir(t, true)
}
func testFSLocalReaddir(t *testing.T, makeReadable bool) {
tmp := t.TempDir()
path := filepath.Join(tmp, "item")
rtest.OK(t, os.Mkdir(path, 0o700))
entries := []string{"testfile"}
rtest.OK(t, os.WriteFile(filepath.Join(path, entries[0]), []byte("example"), 0o600))
f := openReadable(t, path, makeReadable)
checkMetadata(t, f, path, false, restic.NodeTypeDir)
names, err := f.Readdirnames(-1)
rtest.OK(t, err)
slices.Sort(names)
rtest.Equals(t, entries, names, "directory content mismatch")
rtest.OK(t, f.Close())
}
func TestFSLocalReadableRace(t *testing.T) {
tmp := t.TempDir()
path := filepath.Join(tmp, "item")
testdata := "example"
rtest.OK(t, os.WriteFile(path, []byte(testdata), 0o600))
testFs := &Local{}
f, err := testFs.OpenFile(path, O_NOFOLLOW, true)
rtest.OK(t, err)
pathNew := path + "new"
rtest.OK(t, os.Rename(path, pathNew))
err = f.MakeReadable()
if err == nil {
// a file handle based implementation should still work
checkMetadata(t, f, pathNew, false, restic.NodeTypeFile)
data, err := io.ReadAll(f)
rtest.OK(t, err)
rtest.Equals(t, testdata, string(data), "file content mismatch")
}
rtest.OK(t, f.Close())
}
func TestFSLocalTypeChange(t *testing.T) {
tmp := t.TempDir()
path := filepath.Join(tmp, "item")
testdata := "example"
rtest.OK(t, os.WriteFile(path, []byte(testdata), 0o600))
testFs := &Local{}
f, err := testFs.OpenFile(path, O_NOFOLLOW, true)
rtest.OK(t, err)
// cache metadata
_, err = f.Stat()
rtest.OK(t, err)
pathNew := path + "new"
// rename instead of unlink to let the test also work on windows
rtest.OK(t, os.Rename(path, pathNew))
rtest.OK(t, os.Mkdir(path, 0o700))
rtest.OK(t, f.MakeReadable())
fi, err := f.Stat()
rtest.OK(t, err)
if !fi.IsDir() {
// a file handle based implementation should still reference the file
checkMetadata(t, f, pathNew, false, restic.NodeTypeFile)
data, err := io.ReadAll(f)
rtest.OK(t, err)
rtest.Equals(t, testdata, string(data), "file content mismatch")
}
// else:
// path-based implementation
// nothing to test here. stat returned the new file type
rtest.OK(t, f.Close())
}

40
internal/fs/fs_local_unix_test.go Normal file
View File

@@ -0,0 +1,40 @@
//go:build unix
package fs
import (
"syscall"
"testing"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestFSLocalMetadataUnix(t *testing.T) {
for _, test := range []fsLocalMetadataTestcase{
{
name: "socket",
setup: func(t *testing.T, path string) {
fd, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
rtest.OK(t, err)
defer func() {
_ = syscall.Close(fd)
}()
addr := &syscall.SockaddrUnix{Name: path}
rtest.OK(t, syscall.Bind(fd, addr))
},
nodeType: restic.NodeTypeSocket,
},
{
name: "fifo",
setup: func(t *testing.T, path string) {
rtest.OK(t, mkfifo(path, 0o600))
},
nodeType: restic.NodeTypeFifo,
},
// device files can only be created as root
} {
runFSLocalTestcase(t, test)
}
}

16
internal/fs/fs_local_vss.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// VSSConfig holds extended options of windows volume shadow copy service.
@@ -126,14 +125,9 @@ func (fs *LocalVss) DeleteSnapshots() {
fs.snapshots = activeSnapshots
}
// OpenFile wraps the Open method of the underlying file system.
func (fs *LocalVss) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
return fs.FS.OpenFile(fs.snapshotPath(name), flag, perm)
}
// Stat wraps the Stat method of the underlying file system.
func (fs *LocalVss) Stat(name string) (os.FileInfo, error) {
return fs.FS.Stat(fs.snapshotPath(name))
// OpenFile wraps the OpenFile method of the underlying file system.
func (fs *LocalVss) OpenFile(name string, flag int, metadataOnly bool) (File, error) {
return fs.FS.OpenFile(fs.snapshotPath(name), flag, metadataOnly)
}
// Lstat wraps the Lstat method of the underlying file system.
@@ -141,10 +135,6 @@ func (fs *LocalVss) Lstat(name string) (os.FileInfo, error) {
return fs.FS.Lstat(fs.snapshotPath(name))
}
func (fs *LocalVss) NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
return fs.FS.NodeFromFileInfo(fs.snapshotPath(path), fi, ignoreXattrListError)
}
// isMountPointIncluded is true if given mountpoint included by user.
func (fs *LocalVss) isMountPointIncluded(mountPoint string) bool {
if fs.excludeVolumes == nil {

15
internal/fs/fs_local_vss_test.go
View File

@@ -317,28 +317,25 @@ func TestVSSFS(t *testing.T) {
// trigger snapshot creation and
// capture FI while file still exists (should already be within the snapshot)
origFi, err := localVss.Stat(tempfile)
origFi, err := localVss.Lstat(tempfile)
rtest.OK(t, err)
// remove original file
rtest.OK(t, os.Remove(tempfile))
statFi, err := localVss.Stat(tempfile)
rtest.OK(t, err)
rtest.Equals(t, origFi.Mode(), statFi.Mode())
lstatFi, err := localVss.Lstat(tempfile)
rtest.OK(t, err)
rtest.Equals(t, origFi.Mode(), lstatFi.Mode())
f, err := localVss.OpenFile(tempfile, os.O_RDONLY, 0)
f, err := localVss.OpenFile(tempfile, os.O_RDONLY, false)
rtest.OK(t, err)
data, err := io.ReadAll(f)
rtest.OK(t, err)
rtest.Equals(t, "example", string(data), "unexpected file content")
rtest.OK(t, f.Close())
node, err := localVss.NodeFromFileInfo(tempfile, statFi, false)
node, err := f.ToNode(false)
rtest.OK(t, err)
rtest.Equals(t, node.Mode, statFi.Mode())
rtest.Equals(t, node.Mode, lstatFi.Mode())
rtest.OK(t, f.Close())
}

39
internal/fs/fs_reader.go
View File

@@ -49,12 +49,7 @@ func (fs *Reader) fi() os.FileInfo {
}
}
// OpenFile is the generalized open call; most users will use Open
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *os.PathError.
func (fs *Reader) OpenFile(name string, flag int, _ os.FileMode) (f File, err error) {
func (fs *Reader) OpenFile(name string, flag int, _ bool) (f File, err error) {
if flag & ^(O_RDONLY|O_NOFOLLOW) != 0 {
return nil, pathError("open", name,
fmt.Errorf("invalid combination of flags 0x%x", flag))
@@ -81,12 +76,6 @@ func (fs *Reader) OpenFile(name string, flag int, _ os.FileMode) (f File, err er
return nil, pathError("open", name, syscall.ENOENT)
}
// Stat returns a FileInfo describing the named file. If there is an error, it
// will be of type *os.PathError.
func (fs *Reader) Stat(name string) (os.FileInfo, error) {
return fs.Lstat(name)
}
// Lstat returns the FileInfo structure describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link. Lstat makes no attempt to follow the link.
@@ -133,17 +122,6 @@ func (fs *Reader) ExtendedStat(fi os.FileInfo) ExtendedFileInfo {
}
}
func (fs *Reader) NodeFromFileInfo(path string, fi os.FileInfo, _ bool) (*restic.Node, error) {
node := buildBasicNode(path, fi)
// fill minimal info with current values for uid, gid
node.UID = uint32(os.Getuid())
node.GID = uint32(os.Getgid())
node.ChangeTime = node.ModTime
return node, nil
}
// Join joins any number of path elements into a single path, adding a
// Separator if necessary. Join calls Clean on the result; in particular, all
// empty strings are ignored. On Windows, the result is a UNC path if and only
@@ -241,6 +219,10 @@ type fakeFile struct {
// ensure that fakeFile implements File
var _ File = fakeFile{}
func (f fakeFile) MakeReadable() error {
return nil
}
func (f fakeFile) Readdirnames(_ int) ([]string, error) {
return nil, pathError("readdirnames", f.name, os.ErrInvalid)
}
@@ -257,6 +239,17 @@ func (f fakeFile) Stat() (os.FileInfo, error) {
return f.FileInfo, nil
}
func (f fakeFile) ToNode(_ bool) (*restic.Node, error) {
node := buildBasicNode(f.name, f.FileInfo)
// fill minimal info with current values for uid, gid
node.UID = uint32(os.Getuid())
node.GID = uint32(os.Getgid())
node.ChangeTime = node.ModTime
return node, nil
}
// fakeDir implements Readdirnames and Readdir, everything else is delegated to fakeFile.
type fakeDir struct {
entries []os.FileInfo

8
internal/fs/fs_reader_test.go
View File

@@ -16,7 +16,7 @@ import (
)
func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte) {
f, err := fs.OpenFile(filename, O_RDONLY, 0)
f, err := fs.OpenFile(filename, O_RDONLY, false)
if err != nil {
t.Fatal(err)
}
@@ -37,7 +37,7 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte
}
func verifyDirectoryContents(t testing.TB, fs FS, dir string, want []string) {
f, err := fs.OpenFile(dir, os.O_RDONLY, 0)
f, err := fs.OpenFile(dir, O_RDONLY, false)
if err != nil {
t.Fatal(err)
}
@@ -123,7 +123,7 @@ func TestFSReader(t *testing.T) {
{
name: "file/Stat",
f: func(t *testing.T, fs FS) {
f, err := fs.OpenFile(filename, os.O_RDONLY, 0)
f, err := fs.OpenFile(filename, O_RDONLY, true)
if err != nil {
t.Fatal(err)
}
@@ -295,7 +295,7 @@ func TestFSReaderMinFileSize(t *testing.T) {
AllowEmptyFile: test.allowEmpty,
}
f, err := fs.OpenFile("testfile", os.O_RDONLY, 0)
f, err := fs.OpenFile("testfile", O_RDONLY, false)
if err != nil {
t.Fatal(err)
}

6
internal/fs/fs_track.go
View File

@@ -16,8 +16,8 @@ type Track struct {
}
// OpenFile wraps the OpenFile method of the underlying file system.
func (fs Track) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
f, err := fs.FS.OpenFile(fixpath(name), flag, perm)
func (fs Track) OpenFile(name string, flag int, metadataOnly bool) (File, error) {
f, err := fs.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return nil, err
}
@@ -31,7 +31,7 @@ type trackFile struct {
func newTrackFile(stack []byte, filename string, file File) *trackFile {
f := &trackFile{file}
runtime.SetFinalizer(f, func(_ *trackFile) {
runtime.SetFinalizer(f, func(_ any) {
fmt.Fprintf(os.Stderr, "file %s not closed\n\nStacktrack:\n%s\n", filename, stack)
panic("file " + filename + " not closed")
})

26
internal/fs/interface.go
View File

@@ -9,12 +9,18 @@ import (
// FS bundles all methods needed for a file system.
type FS interface {
OpenFile(name string, flag int, perm os.FileMode) (File, error)
Stat(name string) (os.FileInfo, error)
// OpenFile opens a file or directory for reading.
//
// If metadataOnly is set, an implementation MUST return a File object for
// arbitrary file types including symlinks. The implementation may internally use
// the given file path or a file handle. In particular, an implementation may
// delay actually accessing the underlying filesystem.
//
// Only the O_NOFOLLOW and O_DIRECTORY flags are supported.
OpenFile(name string, flag int, metadataOnly bool) (File, error)
Lstat(name string) (os.FileInfo, error)
DeviceID(fi os.FileInfo) (deviceID uint64, err error)
ExtendedStat(fi os.FileInfo) ExtendedFileInfo
NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error)
Join(elem ...string) string
Separator() string
@@ -27,11 +33,23 @@ type FS interface {
Base(path string) string
}
// File is an open file on a file system.
// File is an open file on a file system. When opened as metadataOnly, an
// implementation may opt to perform filesystem operations using the filepath
// instead of actually opening the file.
type File interface {
// MakeReadable reopens a File that was opened metadataOnly for reading.
// The method must not be called for files that are opened for reading.
// If possible, the underlying file should be reopened atomically.
// MakeReadable must work for files and directories.
MakeReadable() error
io.Reader
io.Closer
Readdirnames(n int) ([]string, error)
Stat() (os.FileInfo, error)
// ToNode returns a restic.Node for the File. The internally used os.FileInfo
// must be consistent with that returned by Stat(). In particular, the metadata
// returned by consecutive calls to Stat() and ToNode() must match.
ToNode(ignoreXattrListError bool) (*restic.Node, error)
}

7
internal/fs/node.go
View File

@@ -23,11 +23,8 @@ func nodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*
return node, err
}
allowExtended, err := nodeFillGenericAttributes(node, path, &stat)
if allowExtended {
// Skip processing ExtendedAttributes if allowExtended is false.
err = errors.Join(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError))
}
err := nodeFillGenericAttributes(node, path, &stat)
err = errors.Join(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError))
return node, err
}

26
internal/fs/node_aix.go
View File

@@ -1,26 +0,0 @@
//go:build aix
// +build aix
package fs
import "github.com/restic/restic/internal/restic"
// nodeRestoreExtendedAttributes is a no-op on AIX.
func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error {
return nil
}
// nodeFillExtendedAttributes is a no-op on AIX.
func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error {
return nil
}
// nodeRestoreGenericAttributes is no-op on AIX.
func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error {
return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn)
}
// nodeFillGenericAttributes is a no-op on AIX.
func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) (allowExtended bool, err error) {
return true, nil
}

23
internal/fs/node_netbsd.go
View File

@@ -1,23 +0,0 @@
package fs
import "github.com/restic/restic/internal/restic"
// nodeRestoreExtendedAttributes is a no-op on netbsd.
func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error {
return nil
}
// nodeFillExtendedAttributes is a no-op on netbsd.
func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error {
return nil
}
// nodeRestoreGenericAttributes is no-op on netbsd.
func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error {
return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn)
}
// nodeFillGenericAttributes is a no-op on netbsd.
func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) (allowExtended bool, err error) {
return true, nil
}

18
internal/fs/node_noxattr.go Normal file
View File

@@ -0,0 +1,18 @@
//go:build aix || netbsd || openbsd
// +build aix netbsd openbsd
package fs
import (
"github.com/restic/restic/internal/restic"
)
// nodeRestoreExtendedAttributes is a no-op
func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error {
return nil
}
// nodeFillExtendedAttributes is a no-op
func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error {
return nil
}

23
internal/fs/node_openbsd.go
View File

@@ -1,23 +0,0 @@
package fs
import "github.com/restic/restic/internal/restic"
// nodeRestoreExtendedAttributes is a no-op on openbsd.
func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error {
return nil
}
// nodeFillExtendedAttributes is a no-op on openbsd.
func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error {
return nil
}
// nodeRestoreGenericAttributes is no-op on openbsd.
func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error {
return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn)
}
// fillGenericAttributes is a no-op on openbsd.
func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) (allowExtended bool, err error) {
return true, nil
}

62
internal/fs/node_test.go
View File

@@ -17,56 +17,26 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func BenchmarkNodeFillUser(t *testing.B) {
tempfile, err := os.CreateTemp("", "restic-test-temp-")
if err != nil {
t.Fatal(err)
}
fi, err := tempfile.Stat()
if err != nil {
t.Fatal(err)
}
func BenchmarkNodeFromFileInfo(t *testing.B) {
tempfile, err := os.CreateTemp(t.TempDir(), "restic-test-temp-")
rtest.OK(t, err)
path := tempfile.Name()
rtest.OK(t, tempfile.Close())
fs := Local{}
f, err := fs.OpenFile(path, O_NOFOLLOW, true)
rtest.OK(t, err)
_, err = f.Stat()
rtest.OK(t, err)
t.ResetTimer()
for i := 0; i < t.N; i++ {
_, err := fs.NodeFromFileInfo(path, fi, false)
_, err := f.ToNode(false)
rtest.OK(t, err)
}
rtest.OK(t, tempfile.Close())
rtest.RemoveAll(t, tempfile.Name())
}
func BenchmarkNodeFromFileInfo(t *testing.B) {
tempfile, err := os.CreateTemp("", "restic-test-temp-")
if err != nil {
t.Fatal(err)
}
fi, err := tempfile.Stat()
if err != nil {
t.Fatal(err)
}
path := tempfile.Name()
fs := Local{}
t.ResetTimer()
for i := 0; i < t.N; i++ {
_, err := fs.NodeFromFileInfo(path, fi, false)
if err != nil {
t.Fatal(err)
}
}
rtest.OK(t, tempfile.Close())
rtest.RemoveAll(t, tempfile.Name())
rtest.OK(t, f.Close())
}
func parseTime(s string) time.Time {
@@ -249,14 +219,14 @@ func TestNodeRestoreAt(t *testing.T) {
rtest.OK(t, NodeCreateAt(&test, nodePath))
rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }))
fi, err := os.Lstat(nodePath)
rtest.OK(t, err)
fs := &Local{}
n2, err := fs.NodeFromFileInfo(nodePath, fi, false)
meta, err := fs.OpenFile(nodePath, O_NOFOLLOW, true)
rtest.OK(t, err)
n3, err := fs.NodeFromFileInfo(nodePath, fi, true)
n2, err := meta.ToNode(false)
rtest.OK(t, err)
n3, err := meta.ToNode(true)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3))
rtest.Assert(t, test.Name == n2.Name,

12
internal/fs/node_unix.go
View File

@@ -5,8 +5,20 @@ package fs
import (
"os"
"github.com/restic/restic/internal/restic"
)
func lchown(name string, uid, gid int) error {
return os.Lchown(name, uid, gid)
}
// nodeRestoreGenericAttributes is no-op.
func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error {
return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn)
}
// nodeFillGenericAttributes is a no-op.
func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) error {
return nil
}

16
internal/fs/node_unix_test.go
View File

@@ -114,16 +114,14 @@ func TestNodeFromFileInfo(t *testing.T) {
return
}
if fi.Sys() == nil {
t.Skip("fi.Sys() is nil")
return
}
fs := &Local{}
node, err := fs.NodeFromFileInfo(test.filename, fi, false)
if err != nil {
t.Fatal(err)
}
meta, err := fs.OpenFile(test.filename, O_NOFOLLOW, true)
rtest.OK(t, err)
node, err := meta.ToNode(false)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
rtest.OK(t, err)
switch node.Type {
case restic.NodeTypeFile, restic.NodeTypeSymlink:

46
internal/fs/node_windows.go
View File

@@ -83,8 +83,28 @@ func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) {
return nil
}
// fill extended attributes in the node. This also includes the Generic attributes for windows.
// fill extended attributes in the node
// It also checks if the volume supports extended attributes and stores the result in a map
// so that it does not have to be checked again for subsequent calls for paths in the same volume.
func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool) (err error) {
if strings.Contains(filepath.Base(path), ":") {
// Do not process for Alternate Data Streams in Windows
return nil
}
// only capture xattrs for file/dir
if node.Type != restic.NodeTypeFile && node.Type != restic.NodeTypeDir {
return nil
}
allowExtended, err := checkAndStoreEASupport(path)
if err != nil {
return err
}
if !allowExtended {
return nil
}
var fileHandle windows.Handle
if fileHandle, err = openHandleForEA(node.Type, path, false); fileHandle == 0 {
return nil
@@ -316,40 +336,28 @@ func decryptFile(pathPointer *uint16) error {
// nodeFillGenericAttributes fills in the generic attributes for windows like File Attributes,
// Created time and Security Descriptors.
// It also checks if the volume supports extended attributes and stores the result in a map
// so that it does not have to be checked again for subsequent calls for paths in the same volume.
func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFileInfo) (allowExtended bool, err error) {
func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFileInfo) error {
if strings.Contains(filepath.Base(path), ":") {
// Do not process for Alternate Data Streams in Windows
// Also do not allow processing of extended attributes for ADS.
return false, nil
return nil
}
isVolume, err := isVolumePath(path)
if err != nil {
return false, err
return err
}
if isVolume {
// Do not process file attributes, created time and sd for windows root volume paths
// Security descriptors are not supported for root volume paths.
// Though file attributes and created time are supported for root volume paths,
// we ignore them and we do not want to replace them during every restore.
allowExtended, err = checkAndStoreEASupport(path)
if err != nil {
return false, err
}
return allowExtended, err
return nil
}
var sd *[]byte
if node.Type == restic.NodeTypeFile || node.Type == restic.NodeTypeDir {
// Check EA support and get security descriptor for file/dir only
allowExtended, err = checkAndStoreEASupport(path)
if err != nil {
return false, err
}
if sd, err = getSecurityDescriptor(path); err != nil {
return allowExtended, err
return err
}
}
@@ -361,7 +369,7 @@ func nodeFillGenericAttributes(node *restic.Node, path string, stat *ExtendedFil
FileAttributes: &winFI.FileAttributes,
SecurityDescriptor: sd,
})
return allowExtended, err
return err
}
// checkAndStoreEASupport checks if the volume of the path supports extended attributes and stores the result in a map

8
internal/fs/node_windows_test.go
View File

@@ -222,11 +222,11 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warn
test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath))
fs := &Local{}
fi, err := fs.Lstat(testPath)
test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath))
nodeFromFileInfo, err := fs.NodeFromFileInfo(testPath, fi, false)
meta, err := fs.OpenFile(testPath, O_NOFOLLOW, true)
test.OK(t, err)
nodeFromFileInfo, err := meta.ToNode(false)
test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath))
test.OK(t, meta.Close())
return testPath, nodeFromFileInfo
}

10
internal/fs/node_xattr.go
View File

@@ -65,16 +65,6 @@ func handleXattrErr(err error) error {
}
}
// nodeRestoreGenericAttributes is no-op.
func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error {
return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn)
}
// nodeFillGenericAttributes is a no-op.
func nodeFillGenericAttributes(_ *restic.Node, _ string, _ *ExtendedFileInfo) (allowExtended bool, err error) {
return true, nil
}
func nodeRestoreExtendedAttributes(node *restic.Node, path string) error {
expectedAttrs := map[string]struct{}{}
for _, attr := range node.ExtendedAttributes {