Merge pull request #252 from restic/repack-blobs

WIP: Repack blobs

cmd/restic/cmd_cat.go

@@ -167,12 +167,8 @@ func (cmd CmdCat) Execute(args []string) error {
 			return err
 		}
 
-		if blob.Type != pack.Data {
-			return errors.New("wrong type for blob")
-		}
-
 		buf := make([]byte, blob.Length)
-		data, err := repo.LoadBlob(pack.Data, id, buf)
+		data, err := repo.LoadBlob(blob.Type, id, buf)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_check.go

@@ -5,13 +5,12 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/restic/restic/backend"
 	"github.com/restic/restic/checker"
 )
 
 type CmdCheck struct {
-	ReadData       bool `long:"read-data"  description:"Read data blobs" default:"false"`
-	RemoveOrphaned bool `long:"remove"     description:"Remove data that isn't used" default:"false"`
+	ReadData    bool `long:"read-data"   description:"Read data blobs" default:"false"`
+	CheckUnused bool `long:"check-unused" description:"Check for unused blobs" default:"false"`
 
 	global *GlobalOptions
 }
@@ -80,14 +79,9 @@ func (cmd CmdCheck) Execute(args []string) error {
 	cmd.global.Verbosef("Check all packs\n")
 	go chkr.Packs(errChan, done)
 
-	foundOrphanedPacks := false
 	for err := range errChan {
 		errorsFound = true
 		fmt.Fprintf(os.Stderr, "%v\n", err)
-
-		if e, ok := err.(checker.PackError); ok && e.Orphaned {
-			foundOrphanedPacks = true
-		}
 	}
 
 	cmd.global.Verbosef("Check snapshots, trees and blobs\n")
@@ -106,21 +100,11 @@ func (cmd CmdCheck) Execute(args []string) error {
 		}
 	}
 
-	for _, id := range chkr.UnusedBlobs() {
-		cmd.global.Verbosef("unused blob %v\n", id.Str())
-	}
-
-	if foundOrphanedPacks && cmd.RemoveOrphaned {
-		IDs := chkr.OrphanedPacks()
-		cmd.global.Verbosef("Remove %d orphaned packs... ", len(IDs))
-
-		for _, id := range IDs {
-			if err := repo.Backend().Remove(backend.Data, id.String()); err != nil {
-				fmt.Fprintf(os.Stderr, "%v\n", err)
-			}
+	if cmd.CheckUnused {
+		for _, id := range chkr.UnusedBlobs() {
+			cmd.global.Verbosef("unused blob %v\n", id.Str())
+			errorsFound = true
 		}
-
-		cmd.global.Verbosef("done\n")
 	}
 
 	if errorsFound {

cmd/restic/cmd_optimize.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/checker"
+)
+
+type CmdOptimize struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("optimize",
+		"optimize the repository",
+		"The optimize command reorganizes the repository and removes uneeded data",
+		&CmdOptimize{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdOptimize) Usage() string {
+	return "[optimize-options]"
+}
+
+func (cmd CmdOptimize) Execute(args []string) error {
+	if len(args) != 0 {
+		return errors.New("optimize has no arguments")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("Create exclusive lock for repository\n")
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	chkr := checker.New(repo)
+
+	cmd.global.Verbosef("Load indexes\n")
+	_, errs := chkr.LoadIndex()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+		return fmt.Errorf("LoadIndex returned errors")
+	}
+
+	done := make(chan struct{})
+	errChan := make(chan error)
+	go chkr.Structure(errChan, done)
+
+	for err := range errChan {
+		if e, ok := err.(checker.TreeError); ok {
+			cmd.global.Warnf("error for tree %v:\n", e.ID.Str())
+			for _, treeErr := range e.Errors {
+				cmd.global.Warnf("  %v\n", treeErr)
+			}
+		} else {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+	}
+
+	unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...)
+	cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs))
+
+	repacker := checker.NewRepacker(repo, unusedBlobs)
+	err = repacker.Repack()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("repacking done\n")
+	return nil
+}

cmd/restic/cmd_rebuild_index.go

@@ -90,7 +90,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 			}
 
 			blobsDone[b] = struct{}{}
-			combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
+			combinedIndex.Store(packedBlob)
 		}
 
 		combinedIndex.AddToSupersedes(indexID)
@@ -162,7 +162,13 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 
 		for _, blob := range up.Entries {
 			debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
-			combinedIndex.Store(blob.Type, blob.ID, packID, blob.Offset, blob.Length)
+			combinedIndex.Store(repository.PackedBlob{
+				Type:   blob.Type,
+				ID:     blob.ID,
+				PackID: packID,
+				Offset: blob.Offset,
+				Length: blob.Length,
+			})
 		}
 
 		err = rd.Close()

cmd/restic/integration_helpers_test.go

@@ -216,3 +216,13 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 
 	RemoveAll(t, tempdir)
 }
+
+// removeFile resets the read-only flag and then deletes the file.
+func removeFile(fn string) error {
+	err := os.Chmod(fn, 0666)
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(fn)
+}

cmd/restic/integration_test.go

@@ -61,7 +61,7 @@ func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, pare
 	OK(t, cmd.Execute(target))
 }
 
-func cmdList(t testing.TB, global GlobalOptions, tpe string) []backend.ID {
+func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs {
 	var buf bytes.Buffer
 	global.stdout = &buf
 	cmd := &CmdList{global: &global}
@@ -87,7 +87,11 @@ func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshot
 }
 
 func cmdCheck(t testing.TB, global GlobalOptions) {
-	cmd := &CmdCheck{global: &global, ReadData: true}
+	cmd := &CmdCheck{
+		global:      &global,
+		ReadData:    true,
+		CheckUnused: true,
+	}
 	OK(t, cmd.Execute(nil))
 }
 
@@ -105,6 +109,11 @@ func cmdRebuildIndex(t testing.TB, global GlobalOptions) {
 	OK(t, cmd.Execute(nil))
 }
 
+func cmdOptimize(t testing.TB, global GlobalOptions) {
+	cmd := &CmdOptimize{global: &global}
+	OK(t, cmd.Execute(nil))
+}
+
 func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string {
 	var buf bytes.Buffer
 	global.stdout = &buf
@@ -739,3 +748,43 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 	repository.IndexFull = func(*repository.Index) bool { return true }
 	TestRebuildIndex(t)
 }
+
+var optimizeTests = []struct {
+	testFilename string
+	snapshots    backend.IDSet
+}{
+	{
+		filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz"),
+		backend.NewIDSet(ParseID("a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43")),
+	},
+	{
+		filepath.Join("testdata", "old-index-repo.tar.gz"),
+		nil,
+	},
+	{
+		filepath.Join("testdata", "old-index-repo.tar.gz"),
+		backend.NewIDSet(
+			ParseID("f7d83db709977178c9d1a09e4009355e534cde1a135b8186b8b118a3fc4fcd41"),
+			ParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"),
+		),
+	},
+}
+
+func TestOptimizeRemoveUnusedBlobs(t *testing.T) {
+	for i, test := range optimizeTests {
+		withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+			SetupTarTestFixture(t, env.base, test.testFilename)
+
+			for id := range test.snapshots {
+				OK(t, removeFile(filepath.Join(env.repo, "snapshots", id.String())))
+			}
+
+			cmdOptimize(t, global)
+			output := cmdCheckOutput(t, global)
+
+			if len(output) > 0 {
+				t.Errorf("expected no output for check in test %d, got:\n%v", i, output)
+			}
+		})
+	}
+}