TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-10-27 12:18:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #249 from restic/reduce-fuse-memory-usage

Browse Source 
Reduce memory usage for fuse mount
This commit is contained in:
Alexander Neumann
2015-07-26 19:20:13 +02:00
parent 7608610d72 b85927576b
commit 05e2afba0b
12 changed files with 344 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/restic/cmd_cat.go
View File

@@ -162,9 +162,18 @@ func (cmd CmdCat) Execute(args []string) error {
return err
case "blob":
data, err := repo.LoadBlob(pack.Data, id)
if err == nil {
_, err = os.Stdout.Write(data)
_, blobType, _, length, err := repo.Index().Lookup(id)
if err != nil {
return err
}
if blobType != pack.Data {
return errors.New("wrong type for blob")
}
buf := make([]byte, length)
data, err := repo.LoadBlob(pack.Data, id, buf)
if err != nil {
return err
}

2
cmd/restic/cmd_mount.go
View File

@@ -6,7 +6,7 @@ import (
"fmt"
"os"
"github.com/restic/restic/cmd/restic/fuse"
"github.com/restic/restic/fuse"
systemFuse "bazil.org/fuse"
"bazil.org/fuse/fs"

103
cmd/restic/fuse/dir.go
View File

@@ -1,103 +0,0 @@
package fuse
import (
"os"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"golang.org/x/net/context"
"github.com/restic/restic"
"github.com/restic/restic/repository"
)
// Statically ensure that *dir implement those interface
var _ = fs.HandleReadDirAller(&dir{})
var _ = fs.NodeStringLookuper(&dir{})
type dir struct {
repo *repository.Repository
items map[string]*restic.Node
inode uint64
}
func newDir(repo *repository.Repository, node *restic.Node) (*dir, error) {
tree, err := restic.LoadTree(repo, *node.Subtree)
if err != nil {
return nil, err
}
items := make(map[string]*restic.Node)
for _, node := range tree.Nodes {
items[node.Name] = node
}
return &dir{
repo: repo,
items: items,
inode: node.Inode,
}, nil
}
func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId) (*dir, error) {
tree, err := restic.LoadTree(repo, *snapshot.Tree)
if err != nil {
return nil, err
}
items := make(map[string]*restic.Node)
for _, node := range tree.Nodes {
items[node.Name] = node
}
return &dir{
repo: repo,
items: items,
inode: inodeFromBackendId(snapshot.ID),
}, nil
}
func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = d.inode
a.Mode = os.ModeDir | 0555
return nil
}
func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
ret := make([]fuse.Dirent, 0, len(d.items))
for _, node := range d.items {
var typ fuse.DirentType
switch node.Type {
case "dir":
typ = fuse.DT_Dir
case "file":
typ = fuse.DT_File
case "symlink":
typ = fuse.DT_Link
}
ret = append(ret, fuse.Dirent{
Inode: node.Inode,
Type: typ,
Name: node.Name,
})
}
return ret, nil
}
func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
node, ok := d.items[name]
if !ok {
return nil, fuse.ENOENT
}
switch node.Type {
case "dir":
return newDir(d.repo, node)
case "file":
return newFile(d.repo, node)
case "symlink":
return newLink(d.repo, node)
default:
return nil, fuse.ENOENT
}
}

97
cmd/restic/fuse/file.go
View File

@@ -1,97 +0,0 @@
package fuse
import (
"github.com/restic/restic"
"github.com/restic/restic/pack"
"github.com/restic/restic/repository"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"golang.org/x/net/context"
)
// Statically ensure that *file implements the given interface
var _ = fs.HandleReader(&file{})
type file struct {
repo *repository.Repository
node *restic.Node
sizes []uint32
blobs [][]byte
}
func newFile(repo *repository.Repository, node *restic.Node) (*file, error) {
sizes := make([]uint32, len(node.Content))
for i, blobID := range node.Content {
length, err := repo.Index().LookupSize(blobID)
if err != nil {
return nil, err
}
sizes[i] = uint32(length)
}
return &file{
repo: repo,
node: node,
sizes: sizes,
blobs: make([][]byte, len(node.Content)),
}, nil
}
func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = f.node.Inode
a.Mode = f.node.Mode
a.Size = f.node.Size
return nil
}
func (f *file) getBlobAt(i int) (blob []byte, err error) {
if f.blobs[i] != nil {
blob = f.blobs[i]
} else {
blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i])
if err != nil {
return nil, err
}
f.blobs[i] = blob
}
return blob, nil
}
func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
off := req.Offset
// Skip blobs before the offset
startContent := 0
for off > int64(f.sizes[startContent]) {
off -= int64(f.sizes[startContent])
startContent++
}
content := make([]byte, req.Size)
allContent := content
for i := startContent; i < len(f.sizes); i++ {
blob, err := f.getBlobAt(i)
if err != nil {
return err
}
blob = blob[off:]
off = 0
var copied int
if len(blob) > len(content) {
copied = copy(content[0:], blob[:len(content)])
} else {
copied = copy(content[0:], blob)
}
content = content[copied:]
if len(content) == 0 {
break
}
}
resp.Data = allContent
return nil
}

14
cmd/restic/fuse/fuse.go
View File

@@ -1,14 +0,0 @@
package fuse
import (
"encoding/binary"
"github.com/restic/restic/backend"
)
// inodeFromBackendId returns a unique uint64 from a backend id.
// Endianness has no specific meaning, it is just the simplest way to
// transform a []byte to an uint64
func inodeFromBackendId(id backend.ID) uint64 {
return binary.BigEndian.Uint64(id[:8])
}

30
cmd/restic/fuse/link.go
View File

@@ -1,30 +0,0 @@
package fuse
import (
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/restic/restic"
"github.com/restic/restic/repository"
"golang.org/x/net/context"
)
// Statically ensure that *file implements the given interface
var _ = fs.NodeReadlinker(&link{})
type link struct {
node *restic.Node
}
func newLink(repo *repository.Repository, node *restic.Node) (*link, error) {
return &link{node: node}, nil
}
func (l *link) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
return l.node.LinkTarget, nil
}
func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = l.node.Inode
a.Mode = l.node.Mode
return nil
}

108
cmd/restic/fuse/snapshot.go
View File

@@ -1,108 +0,0 @@
package fuse
import (
"os"
"sync"
"time"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/restic/restic"
"github.com/restic/restic/backend"
"github.com/restic/restic/repository"
"golang.org/x/net/context"
)
type SnapshotWithId struct {
*restic.Snapshot
backend.ID
}
// These lines statically ensure that a *SnapshotsDir implement the given
// interfaces; a misplaced refactoring of the implementation that breaks
// the interface will be catched by the compiler
var _ = fs.HandleReadDirAller(&SnapshotsDir{})
var _ = fs.NodeStringLookuper(&SnapshotsDir{})
type SnapshotsDir struct {
repo *repository.Repository
// knownSnapshots maps snapshot timestamp to the snapshot
sync.RWMutex
knownSnapshots map[string]SnapshotWithId
}
func NewSnapshotsDir(repo *repository.Repository) *SnapshotsDir {
return &SnapshotsDir{
repo: repo,
knownSnapshots: make(map[string]SnapshotWithId),
}
}
func (sn *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
attr.Inode = 0
attr.Mode = os.ModeDir | 0555
return nil
}
func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
sn.Lock()
defer sn.Unlock()
for id := range sn.repo.List(backend.Snapshot, ctx.Done()) {
snapshot, err := restic.LoadSnapshot(sn.repo, id)
if err != nil {
return err
}
sn.knownSnapshots[snapshot.Time.Format(time.RFC3339)] = SnapshotWithId{snapshot, id}
}
return nil
}
func (sn *SnapshotsDir) get(name string) (snapshot SnapshotWithId, ok bool) {
sn.RLock()
snapshot, ok = sn.knownSnapshots[name]
sn.RUnlock()
return snapshot, ok
}
func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
err := sn.updateCache(ctx)
if err != nil {
return nil, err
}
sn.RLock()
defer sn.RUnlock()
ret := make([]fuse.Dirent, 0)
for _, snapshot := range sn.knownSnapshots {
ret = append(ret, fuse.Dirent{
Inode: inodeFromBackendId(snapshot.ID),
Type: fuse.DT_Dir,
Name: snapshot.Time.Format(time.RFC3339),
})
}
return ret, nil
}
func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
snapshot, ok := sn.get(name)
if !ok {
// We don't know about it, update the cache
err := sn.updateCache(ctx)
if err != nil {
return nil, err
}
snapshot, ok = sn.get(name)
if !ok {
// We still don't know about it, this time it really doesn't exist
return nil, fuse.ENOENT
}
}
return newDirFromSnapshot(sn.repo, snapshot)
}