init: Add --copy-chunker-params option

This allows creating multiple repositories with identical chunker
parameters which is required for working deduplication when copying
snapshots between different repositories.

cmd/restic/cmd_copy.go

@@ -18,7 +18,8 @@ The "copy" command copies one or more snapshots from one repository to another
 repository. Note that this will have to read (download) and write (upload) the
 entire snapshot(s) due to the different encryption keys on the source and
 destination, and that transferred files are not re-chunked, which may break
-their deduplication.
+their deduplication. This can be mitigated by the "--copy-chunker-params"
+option when initializing a new destination repository using the "init" command.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCopy(copyOptions, globalOptions, args)

cmd/restic/cmd_init.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/restic/chunker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 
@@ -20,19 +21,36 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runInit(globalOptions, args)
+		return runInit(initOptions, globalOptions, args)
 	},
 }
 
-func init() {
-	cmdRoot.AddCommand(cmdInit)
+// InitOptions bundles all options for the init command.
+type InitOptions struct {
+	secondaryRepoOptions
+	CopyChunkerParameters bool
 }
 
-func runInit(gopts GlobalOptions, args []string) error {
+var initOptions InitOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdInit)
+
+	f := cmdInit.Flags()
+	initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from")
+	f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
+}
+
+func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 	if gopts.Repo == "" {
 		return errors.Fatal("Please specify repository location (-r)")
 	}
 
+	chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
+	if err != nil {
+		return err
+	}
+
 	be, err := create(gopts.Repo, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", gopts.Repo, err)
@@ -47,7 +65,7 @@ func runInit(gopts GlobalOptions, args []string) error {
 
 	s := repository.New(be)
 
-	err = s.Init(gopts.ctx, gopts.password)
+	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
 	if err != nil {
 		return errors.Fatalf("create key in repository at %s failed: %v\n", gopts.Repo, err)
 	}
@@ -60,3 +78,25 @@ func runInit(gopts GlobalOptions, args []string) error {
 
 	return nil
 }
+
+func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
+	if opts.CopyChunkerParameters {
+		otherGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
+		if err != nil {
+			return nil, err
+		}
+
+		otherRepo, err := OpenRepository(otherGopts)
+		if err != nil {
+			return nil, err
+		}
+
+		pol := otherRepo.Config().ChunkerPolynomial
+		return &pol, nil
+	}
+
+	if opts.Repo != "" {
+		return nil, errors.Fatal("Secondary repository must only be specified when copying the chunker parameters")
+	}
+	return nil, nil
+}

cmd/restic/integration_test.go

@@ -51,7 +51,7 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
 	restic.TestDisableCheckPolynomial(t)
 	restic.TestSetLockTimeout(t, 0)
 
-	rtest.OK(t, runInit(opts, nil))
+	rtest.OK(t, runInit(InitOptions{}, opts, nil))
 	t.Logf("repository initialized at %v", opts.Repo)
 }
 
@@ -731,6 +731,36 @@ func TestCopyIncremental(t *testing.T) {
 		len(copiedSnapshotIDs), len(snapshotIDs))
 }
 
+func TestInitCopyChunkerParams(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	env2, cleanup2 := withTestEnvironment(t)
+	defer cleanup2()
+
+	testRunInit(t, env2.gopts)
+
+	initOpts := InitOptions{
+		secondaryRepoOptions: secondaryRepoOptions{
+			Repo:     env2.gopts.Repo,
+			password: env2.gopts.password,
+		},
+	}
+	rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
+
+	initOpts.CopyChunkerParameters = true
+	rtest.OK(t, runInit(initOpts, env.gopts, nil))
+
+	repo, err := OpenRepository(env.gopts)
+	rtest.OK(t, err)
+
+	otherRepo, err := OpenRepository(env2.gopts)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
+		"expected equal chunker polynomials, got %v expected %v", repo.Config().ChunkerPolynomial,
+		otherRepo.Config().ChunkerPolynomial)
+}
+
 func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
 	rtest.OK(t, runTag(opts, gopts, []string{}))
 }