Merge pull request #2876 from aawsome/new-repair-command

Add repair command
This commit is contained in:
Michael Eischer
2023-05-05 23:22:24 +02:00
committed by GitHub
17 changed files with 901 additions and 108 deletions

View File

@@ -245,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
}
if suggestIndexRebuild {
Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
}
if mixedFound {
Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")

View File

@@ -488,7 +488,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
// Pack size does not fit and pack is needed => error
// If the pack is not needed, this is no error, the pack can
// and will be simply removed, see below.
Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
id.Str(), p.unusedSize+p.usedSize, packSize)
return errorSizeNotMatching
}

14
cmd/restic/cmd_repair.go Normal file
View File

@@ -0,0 +1,14 @@
package main
import (
"github.com/spf13/cobra"
)
var cmdRepair = &cobra.Command{
Use: "repair",
Short: "Repair the repository",
}
func init() {
cmdRoot.AddCommand(cmdRepair)
}

View File

@@ -7,15 +7,15 @@ import (
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
var cmdRebuildIndex = &cobra.Command{
Use: "rebuild-index [flags]",
var cmdRepairIndex = &cobra.Command{
Use: "index [flags]",
Short: "Build a new index",
Long: `
The "rebuild-index" command creates a new index based on the pack files in the
The "repair index" command creates a new index based on the pack files in the
repository.
EXIT STATUS
@@ -25,25 +25,37 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
},
}
// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
var cmdRebuildIndex = &cobra.Command{
Use: "rebuild-index [flags]",
Short: cmdRepairIndex.Short,
Long: cmdRepairIndex.Long,
Deprecated: `Use "repair index" instead`,
DisableAutoGenTag: true,
RunE: cmdRepairIndex.RunE,
}
// RepairIndexOptions collects all options for the repair index command.
type RepairIndexOptions struct {
ReadAllPacks bool
}
var rebuildIndexOptions RebuildIndexOptions
var repairIndexOptions RepairIndexOptions
func init() {
cmdRepair.AddCommand(cmdRepairIndex)
// add alias for old name
cmdRoot.AddCommand(cmdRebuildIndex)
f := cmdRebuildIndex.Flags()
f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} {
f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
}
}
func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
@@ -58,7 +70,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global
return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
}
func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
var obsoleteIndexes restic.IDs
packSizeFromList := make(map[restic.ID]int64)
packSizeFromIndex := make(map[restic.ID]int64)

View File

@@ -0,0 +1,176 @@
package main
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
"github.com/spf13/cobra"
)
var cmdRepairSnapshots = &cobra.Command{
Use: "snapshots [flags] [snapshot ID] [...]",
Short: "Repair snapshots",
Long: `
The "repair snapshots" command repairs broken snapshots. It scans the given
snapshots and generates new ones with damaged directories and file contents
removed. If the broken snapshots are deleted, a prune run will be able to
clean up the repository.
The command depends on a correct index, thus make sure to run "repair index"
first!
WARNING
=======
Repairing and deleting broken snapshots causes data loss! It will remove broken
directories and modify broken files in the modified snapshots.
If the contents of directories and files are still available, the better option
is to run "backup" which in that case is able to heal existing snapshots. Only
use the "repair snapshots" command if you need to recover an old and broken
snapshot!
EXIT STATUS
===========
Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args)
},
}
// RepairOptions collects all options for the repair command.
type RepairOptions struct {
DryRun bool
Forget bool
restic.SnapshotFilter
}
var repairSnapshotOptions RepairOptions
func init() {
cmdRepair.AddCommand(cmdRepairSnapshots)
flags := cmdRepairSnapshots.Flags()
flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true)
}
func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
repo, err := OpenRepository(ctx, globalOptions)
if err != nil {
return err
}
if !opts.DryRun {
var lock *restic.Lock
var err error
lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
defer unlockRepo(lock)
if err != nil {
return err
}
} else {
repo.SetDryRun()
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
if err := repo.LoadIndex(ctx); err != nil {
return err
}
// Three error cases are checked:
// - tree is a nil tree (-> will be replaced by an empty tree)
// - trees which cannot be loaded (-> the tree contents will be removed)
// - files whose contents are not fully available (-> file will be modified)
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if node.Type != "file" {
return node
}
ok := true
var newContent restic.IDs = restic.IDs{}
var newSize uint64
// check all contents and remove if not available
for _, id := range node.Content {
if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found {
ok = false
} else {
newContent = append(newContent, id)
newSize += uint64(size)
}
}
if !ok {
Verbosef(" file %q: removed missing content\n", path)
} else if newSize != node.Size {
Verbosef(" file %q: fixed incorrect size\n", path)
}
// no-ops if already correct
node.Content = newContent
node.Size = newSize
return node
},
RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) {
if path == "/" {
Verbosef(" dir %q: not readable\n", path)
// remove snapshots with invalid root node
return restic.ID{}, nil
}
// If a subtree fails to load, remove it
Verbosef(" dir %q: replaced with empty directory\n", path)
emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{})
if err != nil {
return restic.ID{}, err
}
return emptyID, nil
},
AllowUnstableSerialization: true,
})
changedCount := 0
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
}, opts.DryRun, opts.Forget, "repaired")
if err != nil {
return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
}
if changed {
changedCount++
}
}
Verbosef("\n")
if changedCount == 0 {
if !opts.DryRun {
Verbosef("no snapshots were modified\n")
} else {
Verbosef("no snapshots would be modified\n")
}
} else {
if !opts.DryRun {
Verbosef("modified %v snapshots\n", changedCount)
} else {
Verbosef("would modify %v snapshots\n", changedCount)
}
}
return nil
}

View File

@@ -87,36 +87,67 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
return true
}
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if selectByName(path) {
return node
}
Verbosef(fmt.Sprintf("excluding %s\n", path))
return nil
},
DisableNodeCache: true,
})
return filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
}, opts.DryRun, opts.Forget, "rewrite")
}
func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) {
wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(wgCtx, wg)
var filteredTree restic.ID
wg.Go(func() error {
filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
SelectByName: selectByName,
PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
})
var err error
filteredTree, err = filter(ctx, sn)
if err != nil {
return err
}
return repo.Flush(wgCtx)
})
err = wg.Wait()
err := wg.Wait()
if err != nil {
return false, err
}
if filteredTree.IsNull() {
if dryRun {
Verbosef("would delete empty snapshot\n")
} else {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}
debug.Log("removed empty snapshot %v", sn.ID())
Verbosef("removed empty snapshot %v\n", sn.ID().Str())
}
return true, nil
}
if filteredTree == *sn.Tree {
debug.Log("Snapshot %v not modified", sn)
return false, nil
}
debug.Log("Snapshot %v modified", sn)
if opts.DryRun {
if dryRun {
Verbosef("would save new snapshot\n")
if opts.Forget {
if forget {
Verbosef("would remove old snapshot\n")
}
@@ -125,10 +156,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
// Always set the original snapshot id as this essentially a new snapshot.
sn.Original = sn.ID()
*sn.Tree = filteredTree
sn.Tree = &filteredTree
if !opts.Forget {
sn.AddTags([]string{"rewrite"})
if !forget {
sn.AddTags([]string{addTag})
}
// Save the new snapshot.
@@ -138,7 +169,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
}
Verbosef("saved new snapshot %v\n", id.Str())
if opts.Forget {
if forget {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err

View File

@@ -0,0 +1,135 @@
package main
import (
"context"
"hash/fnv"
"io"
"math/rand"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func testRunRepairSnapshot(t testing.TB, gopts GlobalOptions, forget bool) {
opts := RepairOptions{
Forget: forget,
}
rtest.OK(t, runRepairSnapshots(context.TODO(), gopts, opts, nil))
}
func createRandomFile(t testing.TB, env *testEnvironment, path string, size int) {
fn := filepath.Join(env.testdata, path)
rtest.OK(t, os.MkdirAll(filepath.Dir(fn), 0o755))
h := fnv.New64()
_, err := h.Write([]byte(path))
rtest.OK(t, err)
r := rand.New(rand.NewSource(int64(h.Sum64())))
f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0o644)
rtest.OK(t, err)
_, err = io.Copy(f, io.LimitReader(r, int64(size)))
rtest.OK(t, err)
rtest.OK(t, f.Close())
}
func TestRepairSnapshotsWithLostData(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
createRandomFile(t, env, "foo/bar/file", 512*1024)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
testListSnapshots(t, env.gopts, 1)
// damage repository
removePacksExcept(env.gopts, t, restic.NewIDSet(), false)
createRandomFile(t, env, "foo/bar/file2", 256*1024)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
snapshotIDs := testListSnapshots(t, env.gopts, 2)
testRunCheckMustFail(t, env.gopts)
// repair but keep broken snapshots
testRunRebuildIndex(t, env.gopts)
testRunRepairSnapshot(t, env.gopts, false)
testListSnapshots(t, env.gopts, 4)
testRunCheckMustFail(t, env.gopts)
// repository must be ok after removing the broken snapshots
testRunForget(t, env.gopts, snapshotIDs[0].String(), snapshotIDs[1].String())
testListSnapshots(t, env.gopts, 2)
_, err := testRunCheckOutput(env.gopts)
rtest.OK(t, err)
}
func TestRepairSnapshotsWithLostTree(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
createRandomFile(t, env, "foo/bar/file", 12345)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
oldSnapshot := testListSnapshots(t, env.gopts, 1)
oldPacks := testRunList(t, "packs", env.gopts)
// keep foo/bar unchanged
createRandomFile(t, env, "foo/bar2", 1024)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
testListSnapshots(t, env.gopts, 2)
// remove tree for foo/bar and the now completely broken first snapshot
removePacks(env.gopts, t, restic.NewIDSet(oldPacks...))
testRunForget(t, env.gopts, oldSnapshot[0].String())
testRunCheckMustFail(t, env.gopts)
// repair
testRunRebuildIndex(t, env.gopts)
testRunRepairSnapshot(t, env.gopts, true)
testListSnapshots(t, env.gopts, 1)
_, err := testRunCheckOutput(env.gopts)
rtest.OK(t, err)
}
func TestRepairSnapshotsWithLostRootTree(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
createRandomFile(t, env, "foo/bar/file", 12345)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
testListSnapshots(t, env.gopts, 1)
oldPacks := testRunList(t, "packs", env.gopts)
// remove all trees
removePacks(env.gopts, t, restic.NewIDSet(oldPacks...))
testRunCheckMustFail(t, env.gopts)
// repair
testRunRebuildIndex(t, env.gopts)
testRunRepairSnapshot(t, env.gopts, true)
testListSnapshots(t, env.gopts, 0)
_, err := testRunCheckOutput(env.gopts)
rtest.OK(t, err)
}
func TestRepairSnapshotsIntact(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
oldSnapshotIDs := testListSnapshots(t, env.gopts, 1)
// use an exclude that will not exclude anything
testRunRepairSnapshot(t, env.gopts, false)
snapshotIDs := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, reflect.DeepEqual(oldSnapshotIDs, snapshotIDs), "unexpected snapshot id mismatch %v vs. %v", oldSnapshotIDs, snapshotIDs)
testRunCheck(t, env.gopts)
}

View File

@@ -100,6 +100,13 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
return parseIDsFromReader(t, buf)
}
func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs {
t.Helper()
snapshotIDs := testRunList(t, "snapshots", opts)
rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs)
return snapshotIDs
}
func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) {
testRunRestoreExcludes(t, opts, dir, snapshotID, nil)
}
@@ -164,6 +171,11 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) {
return buf.String(), err
}
func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
_, err := testRunCheckOutput(gopts)
rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
}
func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) {
buf := bytes.NewBuffer(nil)
@@ -188,7 +200,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
globalOptions.stdout = os.Stdout
}()
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts))
}
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
@@ -486,7 +498,16 @@ func TestBackupNonExistingFile(t *testing.T) {
testRunBackup(t, "", dirs, opts, env.gopts)
}
func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) {
func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
for id := range remove {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
}
}
func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) {
r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
@@ -1504,8 +1525,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
t.Fatalf("expected no error from checker for test repository, got %v", err)
}
if !strings.Contains(out, "restic rebuild-index") {
t.Fatalf("did not find hint for rebuild-index command")
if !strings.Contains(out, "restic repair index") {
t.Fatalf("did not find hint for repair index command")
}
env.gopts.backendTestHook = backendTestHook
@@ -1518,7 +1539,7 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
}
if err != nil {
t.Fatalf("expected no error from checker after rebuild-index, got: %v", err)
t.Fatalf("expected no error from checker after repair index, got: %v", err)
}
}
@@ -1599,7 +1620,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
return &appendOnlyBackend{r}, nil
}
err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)
err := runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
if err == nil {
t.Error("expected rebuildIndex to fail")
}
@@ -1887,8 +1908,8 @@ func TestListOnce(t *testing.T) {
testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts))
}
func TestHardLink(t *testing.T) {