fs: rewrite Reader to build fs tree up front

This adds proper support for filenames that include directories. For
example, `/foo/bar` would result in an error when trying to open `/foo`.

The directory tree is now build upfront. This ensures let's the
directory tree construction be handled only once. All accessors then
only have to look up the constructed directory entries.

cmd/restic/cmd_backup.go

@@ -591,12 +591,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 				return err
 			}
 		}
-		targetFS = &fs.Reader{
+		targetFS = fs.NewReader(filename, source, fs.ReaderOptions{
-			ModTime:    timeStamp,
+			ModTime: timeStamp,
-			Name:       filename,
+			Mode:    0644,
-			Mode:       0644,
+		})
-			ReadCloser: source,
-		}
 		targets = []string{filename}
 	}
 

internal/archiver/archiver_test.go

@@ -174,12 +174,10 @@ func TestArchiverSaveFileReaderFS(t *testing.T) {
 
 			ts := time.Now()
 			filename := "xx"
-			readerFs := &fs.Reader{
+			readerFs := fs.NewReader(filename, io.NopCloser(strings.NewReader(test.Data)), fs.ReaderOptions{
-				ModTime:    ts,
+				ModTime: ts,
-				Mode:       0123,
+				Mode:    0123,
-				Name:       filename,
+			})
-				ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
-			}
 
 			node, stats := saveFile(t, repo, filename, readerFs)
 
@@ -288,12 +286,10 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 
 			ts := time.Now()
 			filename := "xx"
-			readerFs := &fs.Reader{
+			readerFs := fs.NewReader(filename, io.NopCloser(strings.NewReader(test.Data)), fs.ReaderOptions{
-				ModTime:    ts,
+				ModTime: ts,
-				Mode:       0123,
+				Mode:    0123,
-				Name:       filename,
+			})
-				ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
-			}
 
 			arch := New(repo, readerFs, Options{})
 			arch.Error = func(item string, err error) error {

internal/fs/fs_reader.go

@@ -19,97 +19,129 @@ import (
 // be opened once, all subsequent open calls return syscall.EIO. For Lstat(),
 // the provided FileInfo is returned.
 type Reader struct {
-	Name string
+	items map[string]readerItem
-	io.ReadCloser
+}
 
-	// for FileInfo
+type ReaderOptions struct {
 	Mode    os.FileMode
 	ModTime time.Time
 	Size    int64
 
 	AllowEmptyFile bool
+}
 
-	open sync.Once
+type readerItem struct {
+	open           *sync.Once
+	fi             *ExtendedFileInfo
+	rc             io.ReadCloser
+	allowEmptyFile bool
+
+	children []string
 }
 
 // statically ensure that Local implements FS.
 var _ FS = &Reader{}
 
+func NewReader(name string, r io.ReadCloser, opts ReaderOptions) *Reader {
+	items := make(map[string]readerItem)
+	name = readerCleanPath(name)
+
+	isFile := true
+	for {
+		if isFile {
+			fi := &ExtendedFileInfo{
+				Name:    path.Base(name),
+				Mode:    opts.Mode,
+				ModTime: opts.ModTime,
+				Size:    opts.Size,
+			}
+			items[name] = readerItem{
+				open:           &sync.Once{},
+				fi:             fi,
+				rc:             r,
+				allowEmptyFile: opts.AllowEmptyFile,
+			}
+			isFile = false
+		} else {
+			fi := &ExtendedFileInfo{
+				Name:    path.Base(name),
+				Mode:    os.ModeDir | 0755,
+				ModTime: time.Now(), // FIXME
+				Size:    0,
+			}
+			items[name] = readerItem{
+				fi: fi,
+				// keep the children set during the previous iteration
+				children: items[name].children,
+			}
+		}
+
+		parent := path.Dir(name)
+		if parent == name {
+			break
+		}
+		// add the current file to the children of the parent directory
+		item := items[parent]
+		item.children = append(item.children, path.Base(name))
+		items[parent] = item
+
+		name = parent
+	}
+	return &Reader{
+		items: items,
+	}
+}
+
+func readerCleanPath(name string) string {
+	return path.Clean("/" + name)
+}
+
 // VolumeName returns leading volume name, for the Reader file system it's
 // always the empty string.
 func (fs *Reader) VolumeName(_ string) string {
 	return ""
 }
 
-func (fs *Reader) fi() *ExtendedFileInfo {
-	return &ExtendedFileInfo{
-		Name:    fs.Name,
-		Mode:    fs.Mode,
-		ModTime: fs.ModTime,
-		Size:    fs.Size,
-	}
-}
-
 func (fs *Reader) OpenFile(name string, flag int, _ bool) (f File, err error) {
 	if flag & ^(O_RDONLY|O_NOFOLLOW) != 0 {
 		return nil, pathError("open", name,
 			fmt.Errorf("invalid combination of flags 0x%x", flag))
 	}
 
-	switch name {
+	name = readerCleanPath(name)
-	case fs.Name:
+	item, ok := fs.items[name]
-		fs.open.Do(func() {
+	if !ok {
-			f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile)
+		return nil, pathError("open", name, syscall.ENOENT)
+	}
+
+	// Check if the path matches our target file
+	if item.rc != nil {
+		item.open.Do(func() {
+			f = newReaderFile(item.rc, item.fi, item.allowEmptyFile)
 		})
 
 		if f == nil {
 			return nil, pathError("open", name, syscall.EIO)
 		}
 
-		return f, nil
-	case "/", ".":
-		f = fakeDir{
-			entries: []string{fs.fi().Name},
-		}
 		return f, nil
 	}
 
-	return nil, pathError("open", name, syscall.ENOENT)
+	f = fakeDir{
+		entries: slices.Clone(item.children),
+	}
+	return f, nil
 }
 
 // Lstat returns the FileInfo structure describing the named file.
-// If the file is a symbolic link, the returned FileInfo
-// describes the symbolic link.  Lstat makes no attempt to follow the link.
 // If there is an error, it will be of type *os.PathError.
 func (fs *Reader) Lstat(name string) (*ExtendedFileInfo, error) {
-	getDirInfo := func(name string) *ExtendedFileInfo {
+	name = readerCleanPath(name)
-		return &ExtendedFileInfo{
+	item, ok := fs.items[name]
-			Name:    fs.Base(name),
+	if !ok {
-			Size:    0,
+		return nil, pathError("lstat", name, os.ErrNotExist)
-			Mode:    os.ModeDir | 0755,
-			ModTime: time.Now(),
-		}
 	}
-
+	return item.fi, nil
-	switch name {
-	case fs.Name:
-		return fs.fi(), nil
-	case "/", ".":
-		return getDirInfo(name), nil
-	}
-
-	dir := fs.Dir(fs.Name)
-	for {
-		if dir == "/" || dir == "." {
-			break
-		}
-		if name == dir {
-			return getDirInfo(name), nil
-		}
-		dir = fs.Dir(dir)
-	}
-
-	return nil, pathError("lstat", name, os.ErrNotExist)
 }
 
 // Join joins any number of path elements into a single path, adding a
@@ -137,7 +169,7 @@ func (fs *Reader) IsAbs(_ string) bool {
 //
 // For the Reader, all paths are absolute.
 func (fs *Reader) Abs(p string) (string, error) {
-	return path.Clean(p), nil
+	return readerCleanPath(p), nil
 }
 
 // Clean returns the cleaned path. For details, see filepath.Clean.

internal/fs/fs_reader_test.go

@@ -82,27 +82,30 @@ func checkFileInfo(t testing.TB, fi *ExtendedFileInfo, filename string, modtime
 	}
 }
 
-func TestFSReader(t *testing.T) {
+type fsTest []struct {
-	data := test.Random(55, 1<<18+588)
+	name string
-	now := time.Now()
+	f    func(t *testing.T, fs FS)
-	filename := "foobar"
+}
 
-	var tests = []struct {
+func createReadDirTest(fpath, filename string) fsTest {
-		name string
+	return fsTest{
-		f    func(t *testing.T, fs FS)
-	}{
 		{
-			name: "Readdirnames-slash",
+			name: "Readdirnames-slash-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				verifyDirectoryContents(t, fs, "/", []string{filename})
+				verifyDirectoryContents(t, fs, "/"+fpath, []string{filename})
 			},
 		},
 		{
-			name: "Readdirnames-current",
+			name: "Readdirnames-current-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				verifyDirectoryContents(t, fs, ".", []string{filename})
+				verifyDirectoryContents(t, fs, path.Clean(fpath), []string{filename})
 			},
 		},
+	}
+}
+
+func createFileTest(filename string, now time.Time, data []byte) fsTest {
+	return fsTest{
 		{
 			name: "file/OpenFile",
 			f: func(t *testing.T, fs FS) {
@@ -141,70 +144,108 @@ func TestFSReader(t *testing.T) {
 				checkFileInfo(t, fi, filename, now, 0644, false)
 			},
 		},
+	}
+}
+
+func createDirTest(fpath string) fsTest {
+	return fsTest{
 		{
-			name: "dir/Lstat-slash",
+			name: "dir/Lstat-slash-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				fi, err := fs.Lstat("/")
+				fi, err := fs.Lstat("/" + fpath)
 				if err != nil {
 					t.Fatal(err)
 				}
 
-				checkFileInfo(t, fi, "/", time.Time{}, os.ModeDir|0755, true)
+				checkFileInfo(t, fi, "/"+fpath, time.Time{}, os.ModeDir|0755, true)
 			},
 		},
 		{
-			name: "dir/Lstat-current",
+			name: "dir/Lstat-current-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				fi, err := fs.Lstat(".")
+				fi, err := fs.Lstat("./" + fpath)
 				if err != nil {
 					t.Fatal(err)
 				}
 
-				checkFileInfo(t, fi, ".", time.Time{}, os.ModeDir|0755, true)
+				checkFileInfo(t, fi, "/"+fpath, time.Time{}, os.ModeDir|0755, true)
 			},
 		},
 		{
-			name: "dir/Lstat-error-not-exist",
+			name: "dir/Lstat-error-not-exist-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				_, err := fs.Lstat("other")
+				_, err := fs.Lstat(fpath + "/other")
 				if !errors.Is(err, os.ErrNotExist) {
 					t.Fatal(err)
 				}
 			},
 		},
 		{
-			name: "dir/Open-slash",
+			name: "dir/Open-slash-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				fi, err := fs.Lstat("/")
+				fi, err := fs.Lstat("/" + fpath)
 				if err != nil {
 					t.Fatal(err)
 				}
 
-				checkFileInfo(t, fi, "/", time.Time{}, os.ModeDir|0755, true)
+				checkFileInfo(t, fi, "/"+fpath, time.Time{}, os.ModeDir|0755, true)
 			},
 		},
 		{
-			name: "dir/Open-current",
+			name: "dir/Open-current-" + fpath,
 			f: func(t *testing.T, fs FS) {
-				fi, err := fs.Lstat(".")
+				fi, err := fs.Lstat("./" + fpath)
 				if err != nil {
 					t.Fatal(err)
 				}
 
-				checkFileInfo(t, fi, ".", time.Time{}, os.ModeDir|0755, true)
+				checkFileInfo(t, fi, "/"+fpath, time.Time{}, os.ModeDir|0755, true)
 			},
 		},
 	}
+}
+
+func TestFSReader(t *testing.T) {
+	data := test.Random(55, 1<<18+588)
+	now := time.Now()
+	filename := "foobar"
+
+	tests := createReadDirTest("", filename)
+	tests = append(tests, createFileTest(filename, now, data)...)
+	tests = append(tests, createDirTest("")...)
 
 	for _, test := range tests {
-		fs := &Reader{
+		fs := NewReader(filename, io.NopCloser(bytes.NewReader(data)), ReaderOptions{
-			Name:       filename,
-			ReadCloser: io.NopCloser(bytes.NewReader(data)),
-
 			Mode:    0644,
 			Size:    int64(len(data)),
 			ModTime: now,
-		}
+		})
+
+		t.Run(test.name, func(t *testing.T) {
+			test.f(t, fs)
+		})
+	}
+}
+
+func TestFSReaderNested(t *testing.T) {
+	data := test.Random(55, 1<<18+588)
+	now := time.Now()
+	filename := "foo/sub/bar"
+
+	tests := createReadDirTest("", "foo")
+	tests = append(tests, createReadDirTest("foo", "sub")...)
+	tests = append(tests, createReadDirTest("foo/sub", "bar")...)
+	tests = append(tests, createFileTest(filename, now, data)...)
+	tests = append(tests, createDirTest("")...)
+	tests = append(tests, createDirTest("foo")...)
+	tests = append(tests, createDirTest("foo/sub")...)
+
+	for _, test := range tests {
+		fs := NewReader(filename, io.NopCloser(bytes.NewReader(data)), ReaderOptions{
+			Mode:    0644,
+			Size:    int64(len(data)),
+			ModTime: now,
+		})
 
 		t.Run(test.name, func(t *testing.T) {
 			test.f(t, fs)
@@ -232,16 +273,13 @@ func TestFSReaderDir(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			fs := &Reader{
+			fs := NewReader(test.filename, io.NopCloser(bytes.NewReader(data)), ReaderOptions{
-				Name:       test.filename,
-				ReadCloser: io.NopCloser(bytes.NewReader(data)),
-
 				Mode:    0644,
 				Size:    int64(len(data)),
 				ModTime: now,
-			}
+			})
 
-			dir := path.Dir(fs.Name)
+			dir := path.Dir(test.filename)
 			for {
 				if dir == "/" || dir == "." {
 					break
@@ -287,13 +325,11 @@ func TestFSReaderMinFileSize(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			fs := &Reader{
+			fs := NewReader("testfile", io.NopCloser(strings.NewReader(test.data)), ReaderOptions{
-				Name:           "testfile",
-				ReadCloser:     io.NopCloser(strings.NewReader(test.data)),
 				Mode:           0644,
 				ModTime:        time.Now(),
 				AllowEmptyFile: test.allowEmpty,
-			}
+			})
 
 			f, err := fs.OpenFile("testfile", O_RDONLY, false)
 			if err != nil {

internal/restorer/restorer_test.go

@@ -908,11 +908,9 @@ func TestRestorerSparseFiles(t *testing.T) {
 
 	var zeros [1<<20 + 13]byte
 
-	target := &fs.Reader{
+	target := fs.NewReader("/zeros", io.NopCloser(bytes.NewReader(zeros[:])), fs.ReaderOptions{
-		Mode:       0600,
+		Mode: 0600,
-		Name:       "/zeros",
+	})
-		ReadCloser: io.NopCloser(bytes.NewReader(zeros[:])),
-	}
 	sc := archiver.NewScanner(target)
 	err := sc.Scan(context.TODO(), []string{"/zeros"})
 	rtest.OK(t, err)