Compare commits

...

853 Commits

Author SHA1 Message Date
Alexander Neumann
37d0e1fe58 Add version for 0.15.0 2023-01-12 20:51:19 +01:00
Alexander Neumann
da196aa43e Update manpages and auto-completion 2023-01-12 20:51:19 +01:00
Alexander Neumann
099774c2aa Generate CHANGELOG.md for 0.15.0 2023-01-12 20:50:45 +01:00
Alexander Neumann
cd2f53e3f9 Prepare changelog for 0.15.0 2023-01-12 20:50:44 +01:00
Alexander Neumann
0c5a55d1bd Merge pull request #4142 from restic/update-deps
Upgrade dependencies
2023-01-12 20:23:25 +01:00
Michael Eischer
9ddca65f6d update dependencies 2023-01-11 23:22:10 +01:00
Michael Eischer
06fee601bc Merge pull request #4141 from restic/doc-no-scan-correction
doc: Correct heading level for --no-scan
2023-01-11 22:01:05 +01:00
Leo R. Lundgren
1cb920cc57 doc: Correct heading level for --no-scan 2023-01-11 21:47:38 +01:00
Michael Eischer
8f53ffb921 Merge pull request #4140 from restic/doc-manual
doc: Update manual page with --no-scan
2023-01-11 21:42:42 +01:00
Michael Eischer
351cbb4f94 Merge pull request #4139 from restic/doc-no-scan
doc: Move and update documentation for --no-scan
2023-01-11 21:41:20 +01:00
Leo R. Lundgren
b8b5508d15 doc: Update manual page with --no-scan 2023-01-11 02:27:43 +01:00
Leo R. Lundgren
c5542ddcd2 doc: Move and update documentation for --no-scan 2023-01-11 00:09:24 +01:00
rawtaz
dffb8e0c14 Merge pull request #4138 from MichaelEischer/doc-sparse
doc: add description for restore --sparse
2023-01-10 23:39:51 +01:00
Michael Eischer
b151fa498a doc: add description for restore --sparse 2023-01-10 23:27:42 +01:00
Michael Eischer
c354b55e62 Merge pull request #4136 from restic/doc-small-files
doc: Clarify text about tuning backups for small files
2023-01-08 22:00:09 +01:00
Michael Eischer
375953a001 Merge pull request #4088 from MichaelEischer/doc-cifs-backup-source
doc: reading from CIFS can be a problem on linux
2023-01-08 21:56:48 +01:00
Leo R. Lundgren
6306797238 doc: Clarify text about tuning backups for small files 2023-01-08 21:49:55 +01:00
Michael Eischer
ef9164fcbb doc: reading from CIFS can be a problem on linux 2023-01-08 21:47:34 +01:00
rawtaz
e2bcfd68dd Merge pull request #4135 from restic/changelogs
Polish changelogs
2023-01-08 20:04:24 +01:00
Leo R. Lundgren
33fb351386 Polish changelogs 2023-01-08 19:48:51 +01:00
Michael Eischer
c9840da4f8 Merge pull request #4134 from MichaelEischer/fix-verbose-help-text
Correct maximum verbosity level in help message
2023-01-07 22:24:04 +01:00
Michael Eischer
732184a849 Correct maximum verbosity level in help message
The maximum for `--verbose=n` is n=2. Internally it is translated into a
scale from 0 to 3. However, the default (without verbose) is 1, thus the
verbosity level can only be increased two times.
2023-01-07 22:02:13 +01:00
Michael Eischer
24178c97e9 Merge pull request #4117 from MichaelEischer/prune-dry-run-help
prune: make it clearer when prune is used in dry-run mode
2023-01-04 23:18:53 +01:00
Michael Eischer
7a36306901 forget: Clarify log message for --dry-run --prune 2023-01-04 00:44:46 +01:00
Michael Eischer
b404ad4eaa prune: make it clearer when prune is used in dry-run mode 2023-01-04 00:44:46 +01:00
Michael Eischer
e02a10c58a Merge pull request #4109 from MichaelEischer/fix-prune-uncompressed-accounting
prune: Fix calculation of remaining uncompressed data
2023-01-03 23:28:10 +01:00
Michael Eischer
81dc8c8d13 prune: Fix calculation of remaining uncompressed data
Only the repacking of *un*compressed packs reduces the amount of
uncompressed data. Previously the counter even overflowed for fully
compressed repositories.
2023-01-03 22:34:36 +01:00
Michael Eischer
89a8006578 Merge pull request #4104 from philaris/fix_max_uint32_uid_gid_to_zero
in tar dump, convert uid, gid of value -1 to zero
2023-01-02 22:28:28 +01:00
Panagiotis Cheilaris
3b516d4b70 convert uid/gid -1 to 0 only in 32-bit tar dump
Only for a 32-bit build of restic, convert a uid or gid value of -1 to 0.
2022-12-30 18:12:12 +01:00
Michael Eischer
0de3b24756 Merge pull request #4110 from MichaelEischer/remove-exitf
Remove Exitf function
2022-12-29 12:07:51 +01:00
Michael Eischer
0fbff39ae8 Merge pull request #4108 from MichaelEischer/cleanup-check-output
Cleanup check output
2022-12-29 11:59:18 +01:00
Michael Eischer
68b1f30733 Remove Exitf function
Commands should use the normal shutdown path. In addition, the Exitf
function was only used by `dump` and `restore` but not any other command
which introduces the risk of inconsistent behavior.
2022-12-28 21:42:38 +01:00
Panagiotis Cheilaris
a86a56cf3b fix lint issue with function name 'tarId'
See https://github.com/golang/lint/issues/89 and
https://github.com/golang/lint/issues/124
2022-12-28 18:46:58 +01:00
Panagiotis Cheilaris
050ed616ae be more explicit with uid or gid of value -1 2022-12-28 18:44:36 +01:00
Michael Eischer
8430399fce check: Partially fix garbled output
When reporting an error for a tree, the output message can overlap with
the progress bar output, e.g. `error for tree e91ef6fb:napshots`.

The fix only applies for this specific message and does not work on
Windows.
2022-12-28 17:47:27 +01:00
Michael Eischer
aea96b7d86 check: Slightly improve help message
If a repository has both pack/index related warnings and errors, then
the help message is quite misleading. Reword it slightly to be more
clear.
2022-12-28 17:46:06 +01:00
Michael Eischer
bcae28afb4 Merge pull request #4100 from klemensn/tag-self-update
Reinstate selfupdate tag to make builds without self-update work
2022-12-28 15:49:11 +01:00
Panagiotis Cheilaris
10fa5cde0a in tar dump, convert uid, gid of value -1 to zero 2022-12-27 16:36:04 +01:00
Klemens Nanni
61e7386384 Bugfix: Make distribution package builds without self-update work 2022-12-26 21:52:24 +04:00
Klemens Nanni
94f6e7d4a6 Reinstate selfupdate tag to make builds without self-update work
Revert what seems to be a typo introduced as part of the fix for #2041
in 2018 7d0f2eaf24.

`xbuild` does not look like a go build/tag keyword to me, I failed to
find documentation for it and using `go install -tags '!selfupdate' ...`
has no effect, i.e. self-update code is still compiled.

`+build` however works;  updating the OpenBSD port/binary package
security/restic to apply this PR works as expected:

```
	$ restic help | grep self
	$ restic self-update
	unknown command "self-update" for "restic"
```

(Using `go:build` now as per restic's style and gofmt.)

Previously, using `restic-0.14.0p1` on OpenBSD/amd64 7.2-current would
check for a newer version and probably attempt replacing the system wide
root-owned executable (on a read-only filesystem) as unprivileged user:

```
	$ restic version
	restic 0.14.0 compiled with go1.19.2 on openbsd/amd64
	$ restic help | grep self
	  self-update   Update the restic binary
	$ restic self-update
	writing restic to /usr/local/bin/restic
	find latest release of restic at GitHub
	restic is up to date
```

(It never tried to actually write besaid path;  doing so would fail, so
the current message can be considered misleading.)
2022-12-26 21:46:22 +04:00
Michael Eischer
90fb6f70b4 Merge pull request #4089 from greatroar/errors
Clean up error handling further
2022-12-24 10:41:56 +01:00
Michael Eischer
29b8500254 Merge pull request #4090 from restic/upgrade-dependencies
Upgrade dependencies
2022-12-23 22:34:39 +01:00
Michael Eischer
705cabb304 Merge pull request #3981 from MichaelEischer/prune-uncompressed-stats
prune: report how much data must be repacked to compressed the repo
2022-12-23 22:34:04 +01:00
Michael Eischer
a6f3ae5790 Merge pull request #4094 from googol42/master
remove duplicated init
2022-12-23 22:33:01 +01:00
Andreas Dominik Preikschat
ea37240597 remove duplicated init
the documentation contained the `init` command twice
2022-12-20 17:24:56 +01:00
Michael Eischer
bd2f6aaac3 azure: downgrade azblob dependency due to build breakages on Solaris 2022-12-17 23:35:07 +01:00
Michael Eischer
583372956b Upgrade dependencies
Nothing special has changed.
2022-12-17 15:23:11 +01:00
greatroar
1678392a6d checker: Make ErrLegacyLayout a value, not a type 2022-12-17 09:41:07 +01:00
greatroar
d9002f050e backend: Don't Wrap errors from url.Parse
The messages from url.Error.Error already start with the word "parse".
2022-12-17 09:41:07 +01:00
greatroar
b150dd0235 all: Replace some errors.Wrap calls by errors.WithStack
Mostly changed the ones that repeat the name of a system call, which is
already contained in os.PathError.Op. internal/fs.Reader had to be
changed to actually return such errors.
2022-12-17 09:41:07 +01:00
Michael Eischer
cccc17e4e9 Merge pull request #4086 from blackpiglet/modify_access_denied_code
Fix: change error code in function isAccessDenied to AccessDenied
2022-12-16 21:55:52 +01:00
Michael Eischer
2723159ed4 Merge pull request #3931 from kjetilho/feature/optional_scanner
add --no-scan to backup command
2022-12-16 21:42:22 +01:00
Xun Jiang/Bruce Jiang
ecc62c8be2 Update changelog/unreleased/issue-4085
Co-authored-by: greatroar <61184462+greatroar@users.noreply.github.com>
Signed-off-by: Xun Jiang <blackpiglet@gmail.com>
2022-12-16 21:41:16 +01:00
Xun Jiang
cc5325d22b Fix: change error code in function isAccessDenied to AccessDenied
Signed-off-by: Xun Jiang <blackpiglet@gmail.com>
2022-12-16 21:41:16 +01:00
Michael Eischer
da0e45cf40 Merge pull request #4083 from greatroar/cleanup
repository: Remove empty cleanup functions in tests
2022-12-16 21:39:30 +01:00
Kjetil Torgrim Homme
14aa6f2a00 add --disable-scanner to backup command
The scanner process has only cosmetic effect for the progress printer,
and can be disabled without impacting functionality when the user does
not need an estimate of completion.

In many cases the scanner process can provide beneficial priming of
the file system cache, so as general advice it should not be disabled.
However, tests have shown that backup of NFS and fuse based filesystems,
where stat(2) is relatively expensive, can be significantly faster
without the scanner.
2022-12-16 21:29:59 +01:00
Michael Eischer
7bdb985dde Merge pull request #4079 from MichaelEischer/rewrite-set-original
rewrite: Always set the Original field in a rewritten snapshot
2022-12-13 22:56:20 +01:00
Michael Eischer
1bfe98bdc0 Merge pull request #2398 from DanielG/b2-hide-file
b2: Fallback to b2_hide_file when delete returns unauthorized
2022-12-13 22:52:23 +01:00
Michael Eischer
1c071a462e Merge pull request #4084 from ekarlso/azure-stat-fix
fix: Make create not error out when ContainerNotFound
2022-12-13 22:49:05 +01:00
Michael Eischer
25d22d5241 Merge pull request #4082 from MichaelEischer/unbuffered-logger-for-testing
Don't buffer the golang `log` package output when running tests
2022-12-13 22:45:50 +01:00
Endre Karlson
7dd33c0ecc azure: Make create not error out when ContainerNotFound 2022-12-11 22:57:23 +01:00
greatroar
c0b5ec55ab repository: Remove empty cleanup functions in tests
TestRepository and its variants always returned no-op cleanup functions.
If they ever do need to do cleanup, using testing.T.Cleanup is easier
than passing these functions around.
2022-12-11 11:06:25 +01:00
Michael Eischer
2e3d4640be Don't buffer the golang log output when running tests 2022-12-10 16:08:27 +01:00
Michael Eischer
38b2e9b42c rewrite: Always set the Original field in a rewritten snapshot
The Original field is meant to remember the original snapshot id if e.g.
changing its tags. It was only set by the `rewrite` command if it was
not set previously. However, a rewritten snapshot is potentially rather
different from the original snapshot. Thus just always set the Original
field. This also makes it easier to later on detect and potentially
remove the original snapshots.
2022-12-10 12:47:00 +01:00
Michael Eischer
049a105ba5 Merge pull request #4077 from greatroar/cleanup
test: Use testing.T.Cleanup to remove tempdirs
2022-12-09 22:17:46 +01:00
Michael Eischer
4b98b5562d Merge pull request #4075 from greatroar/sftp-enospc
sftp: Fix ENOSPC check
2022-12-09 22:00:13 +01:00
greatroar
f90bf84ba7 test: Use testing.T.Cleanup to remove tempdirs 2022-12-09 14:23:55 +01:00
greatroar
83d23b3ae8 Changelog for ENOSPC handling bug 2022-12-09 08:50:30 +01:00
Michael Eischer
eae7366563 Merge pull request #4028 from ekarlso/use-az-blob-sdk
Switch to azblob sdk
2022-12-07 21:58:03 +01:00
Endre Karlson
25648e2501 azure: Switch to azblob sdk 2022-12-07 21:46:07 +01:00
greatroar
62520bb7b4 sftp: Fix ENOSPC check
We now check for space that is not reserved for the root user on the
remote, and the check is no longer in a defer block because it wouldn't
fire. Some change in the surrounding code may have led the deferred
function to capture the wrong err variable.

Fixes #3336.
2022-12-07 21:06:46 +01:00
rawtaz
4ba31df08f Merge pull request #4074 from greatroar/lobaro-docker
doc: Remove ref to Lobaro's Docker image
2022-12-04 18:00:08 +01:00
greatroar
5efcbe143c doc: Remove ref to Lobaro's Docker image
It hasn't been updated for a while and has restic 0.12.0. Fixes #4002.
2022-12-04 16:20:42 +01:00
Michael Eischer
0df585dd99 Merge pull request #4066 from sedlund/fix#4033
fix#4033 cmd: copy no longer lists skipped existing snapshots by default
2022-12-03 19:22:11 +01:00
Michael Eischer
223da7344e Merge pull request #4070 from restic/fix-cloud-tests
Fix cloud tests
2022-12-03 19:21:25 +01:00
Michael Eischer
2b67862420 backend/test: check that IsNotExist actually works 2022-12-03 18:56:55 +01:00
Michael Eischer
2f934f5803 gs: check against the correct error in IsNotExist 2022-12-03 18:49:54 +01:00
Michael Eischer
04d101fa94 gs/s3: remove useless os.IsNotExist check 2022-12-03 18:49:54 +01:00
Michael Eischer
579cd6dc64 azure: fix totally broken IsNotExist 2022-12-03 18:49:54 +01:00
Michael Eischer
3ebdadc58f Merge pull request #4069 from greatroar/cleanup
cache, prune, restic: Cleanup
2022-12-03 17:50:52 +01:00
Michael Eischer
bc8b2455b9 Merge pull request #4064 from MichaelEischer/flaky-abort-early-on-error
archiver: Fix flaky TestArchiverAbortEarlyOnError
2022-12-03 17:43:44 +01:00
Michael Eischer
60c6a09324 Merge pull request #4065 from MichaelEischer/flaky-rclone-failed-start
rclone: treat "file already closed" as command startup error
2022-12-03 17:42:56 +01:00
Michael Eischer
8bf6b2b80d Merge pull request #4067 from MichaelEischer/remove-backend-test-method
Remove `Test()` method from Backend
2022-12-03 17:40:55 +01:00
Michael Eischer
78ea69082a Merge pull request #4068 from MichaelEischer/debug-lock-refresh-test
Add more debug logging to `TestLockSuccessfulRefresh`
2022-12-03 17:38:21 +01:00
Scott Edlund
cbe73ace3f Update changelog/unreleased/issue-4033
Co-authored-by: greatroar <61184462+greatroar@users.noreply.github.com>
2022-12-03 20:07:37 +08:00
greatroar
63bed34608 restic: Clean up restic.IDs type
IDs.Less can be rewritten as

	string(list[i][:]) < string(list[j][:])

Note that this does not copy the ID's.

The Uniq method was no longer used.

The String method has been reimplemented without first copying into a
separate slice of a custom type.
2022-12-03 12:38:20 +01:00
greatroar
0c749dd358 prune: Pass fewer options around 2022-12-03 12:14:04 +01:00
greatroar
d45a2475e1 cache: Rewrite unnecessary if-else 2022-12-03 12:13:54 +01:00
Michael Eischer
6b5d6b9f2c Add more debug logging to TestLockSuccessfulRefresh
The test fails from time to time. Add some more logging to hopefully get
an idea where things go wrong.
2022-12-03 12:05:38 +01:00
Michael Eischer
648edeca40 retry: Do not retry Stat() if file does not exist
In non test/debug code, Stat() is used exclusively to check whether a
file exists. Thus, do not retry if a file is reported as not existing.
2022-12-03 11:42:48 +01:00
Michael Eischer
40ac678252 backend: remove Test method
The Test method was only used in exactly one place, namely when trying
to create a new repository it was used to check whether a config file
already exists.

Use a combination of Stat() and IsNotExist() instead.
2022-12-03 11:28:10 +01:00
sedlund
06ee0339aa fix#4033 cmd: copy no longer lists skipped existing snapshots by default 2022-12-03 09:55:39 +08:00
Michael Eischer
57d8eedb88 Merge pull request #4020 from greatroar/fuse-inode
fuse: Better inode generation
2022-12-02 22:28:15 +01:00
Michael Eischer
ca1803cacb Merge pull request #4063 from MichaelEischer/replace-ioutil-usage
Replace ioutil usage
2022-12-02 21:49:40 +01:00
Michael Eischer
0af89a5738 Merge pull request #3132 from metalsp0rk/init-json
Init command JSON output
2022-12-02 21:49:22 +01:00
Michael Eischer
364a396fd6 init: use standard name message_type to distinguish JSON messages 2022-12-02 21:33:03 +01:00
Michael Eischer
9a9f559806 init: cleanup json print code 2022-12-02 21:33:03 +01:00
Kyle Brennan
933c9af328 create changelog entry for issue-3124 and pull-3132 2022-12-02 21:32:30 +01:00
Kyle Brennan
a6ae79b39e support json output for init command 2022-12-02 21:32:30 +01:00
Michael Eischer
f3d964a8c1 rclone: treat "file already closed" as command startup error
Since #3940 the rclone backend returns the commands exit code if it
fails to start. The list of expected errors was missing the "file
already closed"-error which can occur if the http test request first
learns about the closed pipe to rclone before noticing the canceled
context.

Go internally makes sure that a file descriptor is unusable once it was
closed, thus this cannot have unintended side effects (like accidentally
reading from the wrong file due to a reused file descriptor).
2022-12-02 20:46:02 +01:00
Michael Eischer
a9972dbe7d archiver: Fix flaky TestArchiverAbortEarlyOnError
Saving the blobs of a file by now happens asynchronously to the
processing in the FileSaver. Thus we have to account for the blobs
queued for saving.
2022-12-02 20:07:34 +01:00
Michael Eischer
f755233210 Replace usages of ioutil.ReadDir
This changes the return type to []fs.DirEntry. However, as we only use
the filenames anyways, this doesn't make a difference.
2022-12-02 19:54:27 +01:00
Michael Eischer
fa20a78bb6 Merge pull request #4056 from greatroar/cleanup
backend, fs, options: Minor cleanup
2022-12-02 19:44:54 +01:00
Michael Eischer
ff7ef5007e Replace most usages of ioutil with the underlying function
The ioutil functions are deprecated since Go 1.17 and only wrap another
library function. Thus directly call the underlying function.

This commit only mechanically replaces the function calls.
2022-12-02 19:36:43 +01:00
greatroar
65612d797c backend, options: Prefer strings.Cut to SplitN
Also realigned the various "split into host🪣prefix"
implementations.
2022-12-02 19:19:14 +01:00
Michael Eischer
2d5e28e777 Merge pull request #4059 from restic/dependabot/go_modules/github.com/minio/minio-go/v7-7.0.45
build(deps): bump github.com/minio/minio-go/v7 from 7.0.44 to 7.0.45
2022-12-01 21:20:57 +01:00
dependabot[bot]
4fefa2ade2 build(deps): bump github.com/minio/minio-go/v7 from 7.0.44 to 7.0.45
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.44 to 7.0.45.
- [Release notes](https://github.com/minio/minio-go/releases)
- [Commits](https://github.com/minio/minio-go/compare/v7.0.44...v7.0.45)

---
updated-dependencies:
- dependency-name: github.com/minio/minio-go/v7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-12-01 01:03:42 +00:00
Michael Eischer
3c5d1eabe9 Merge pull request #4051 from restic/dependabot/go_modules/github.com/klauspost/compress-1.15.12
build(deps): bump github.com/klauspost/compress from 1.15.9 to 1.15.12
2022-11-28 21:47:55 +01:00
Michael Eischer
bec391ee26 Merge pull request #4053 from greatroar/xattr
Upgrade pkg/xattr to version with Solaris FIFO fix
2022-11-28 21:06:19 +01:00
greatroar
daafcaf380 Upgrade pkg/xattr to version with Solaris FIFO fix
This version doesn't have a release tag yet, but it's 0.4.9 + one patch.

Fixes #4003.
2022-11-28 20:43:51 +01:00
Alexander Neumann
1d7e7fcd6b Merge pull request #4049 from MichaelEischer/fix-rewrite-docs
rewrite: fix link anchors in documentation
2022-11-28 19:39:38 +01:00
dependabot[bot]
57d59c71e3 build(deps): bump github.com/klauspost/compress from 1.15.9 to 1.15.12
Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.15.9 to 1.15.12.
- [Release notes](https://github.com/klauspost/compress/releases)
- [Changelog](https://github.com/klauspost/compress/blob/master/.goreleaser.yml)
- [Commits](https://github.com/klauspost/compress/compare/v1.15.9...v1.15.12)

---
updated-dependencies:
- dependency-name: github.com/klauspost/compress
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-28 18:39:35 +00:00
Alexander Neumann
bb83c78ee5 Merge pull request #4047 from MichaelEischer/clean-ci-configuration
Cleanup CI configuration
2022-11-28 19:38:33 +01:00
greatroar
60aa87bbab fs: Remove explicit type check in extendedStat
Without comma-ok, the runtime inserts the same check with a similar
enough panic message:

    interface conversion: interface {} is nil, not *syscall.Stat_t
2022-11-27 19:58:06 +01:00
Michael Eischer
34609bca0e Merge pull request #4050 from greatroar/lruv2
bloblru: Upgrade to hashicorp/golang-lru/v2
2022-11-27 17:37:14 +01:00
greatroar
e5d597fd22 bloblru: Upgrade to hashicorp/golang-lru/v2
The new genericized LRU cache no longer needs to have the IDs separately
allocated:

name   old time/op    new time/op    delta
Add-8     494ns ± 2%     388ns ± 2%  -21.46%  (p=0.000 n=10+9)

name   old alloc/op   new alloc/op   delta
Add-8      176B ± 0%      152B ± 0%  -13.64%  (p=0.000 n=10+10)

name   old allocs/op  new allocs/op  delta
Add-8      5.00 ± 0%      3.00 ± 0%  -40.00%  (p=0.000 n=10+10)
2022-11-27 17:18:13 +01:00
Michael Eischer
0eddc89e98 doc: design.rst: Fix highlighting for index snippet
JSON does not support comments. As JSON is a subset of Javascript, use
the latter instead.
2022-11-27 17:01:27 +01:00
Michael Eischer
41b0f1d43a doc: fix link to amazon s3 section 2022-11-27 17:01:22 +01:00
Michael Eischer
6a793db9ca rewrite: fix link anchors in documentation 2022-11-27 16:38:10 +01:00
Michael Eischer
05cebc1c4b Merge pull request #4044 from restic/dependabot/go_modules/cloud.google.com/go/storage-1.28.0
build(deps): bump cloud.google.com/go/storage from 1.25.0 to 1.28.0
2022-11-27 15:23:06 +01:00
Michael Eischer
ce39727846 Merge pull request #4036 from restic/dependabot/go_modules/github.com/pkg/profile-1.7.0
build(deps): bump github.com/pkg/profile from 1.6.0 to 1.7.0
2022-11-27 15:22:16 +01:00
Michael Eischer
9aa06ce959 CI: remove option to configure command used to install go tools
With the minimum required go version of 1.18, we always use `go
install`.
2022-11-27 15:07:29 +01:00
Michael Eischer
5968971313 CI: remove dependabot ignore for bazil.org/fuse
We've switched to a fork of the original library, thus the ignore is no
longer necessary.
2022-11-27 15:06:30 +01:00
dependabot[bot]
95374767de build(deps): bump github.com/pkg/profile from 1.6.0 to 1.7.0
Bumps [github.com/pkg/profile](https://github.com/pkg/profile) from 1.6.0 to 1.7.0.
- [Release notes](https://github.com/pkg/profile/releases)
- [Commits](https://github.com/pkg/profile/compare/v1.6.0...v1.7.0)

---
updated-dependencies:
- dependency-name: github.com/pkg/profile
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-27 14:02:32 +00:00
dependabot[bot]
c100a62ebf build(deps): bump cloud.google.com/go/storage from 1.25.0 to 1.28.0
Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.25.0 to 1.28.0.
- [Release notes](https://github.com/googleapis/google-cloud-go/releases)
- [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.25.0...spanner/v1.28.0)

---
updated-dependencies:
- dependency-name: cloud.google.com/go/storage
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-27 14:01:36 +00:00
Michael Eischer
c41a1b66e1 Merge pull request #4037 from restic/dependabot/go_modules/google.golang.org/api-0.103.0
build(deps): bump google.golang.org/api from 0.93.0 to 0.103.0
2022-11-27 15:00:16 +01:00
dependabot[bot]
705aed0ecb build(deps): bump google.golang.org/api from 0.93.0 to 0.103.0
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.93.0 to 0.103.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.93.0...v0.103.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-27 13:45:08 +00:00
Michael Eischer
28d6de648c Merge pull request #4040 from restic/dependabot/go_modules/github.com/spf13/cobra-1.6.1
build(deps): bump github.com/spf13/cobra from 1.5.0 to 1.6.1
2022-11-27 14:44:25 +01:00
Michael Eischer
bb40b55d1c Merge pull request #4038 from restic/dependabot/go_modules/github.com/cenkalti/backoff/v4-4.2.0
build(deps): bump github.com/cenkalti/backoff/v4 from 4.1.3 to 4.2.0
2022-11-27 14:13:39 +01:00
dependabot[bot]
a24c1e99a6 build(deps): bump github.com/cenkalti/backoff/v4 from 4.1.3 to 4.2.0
Bumps [github.com/cenkalti/backoff/v4](https://github.com/cenkalti/backoff) from 4.1.3 to 4.2.0.
- [Release notes](https://github.com/cenkalti/backoff/releases)
- [Commits](https://github.com/cenkalti/backoff/compare/v4.1.3...v4.2.0)

---
updated-dependencies:
- dependency-name: github.com/cenkalti/backoff/v4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-27 12:57:41 +00:00
dependabot[bot]
fd56ead4a8 build(deps): bump github.com/spf13/cobra from 1.5.0 to 1.6.1
Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.5.0 to 1.6.1.
- [Release notes](https://github.com/spf13/cobra/releases)
- [Commits](https://github.com/spf13/cobra/compare/v1.5.0...v1.6.1)

---
updated-dependencies:
- dependency-name: github.com/spf13/cobra
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-27 12:57:23 +00:00
Michael Eischer
cc679c6494 Merge pull request #4041 from MichaelEischer/require-go-1.18
Require go 1.18
2022-11-27 13:56:43 +01:00
greatroar
c9c7671c58 fuse: Clean up inode generation 2022-11-27 13:53:42 +01:00
Michael Eischer
530f129a39 rest: remove workaround for content-length handling bug 2022-11-27 13:18:44 +01:00
Michael Eischer
8ad231bcad bump version numbers in instructions to reproduce binaries 2022-11-27 13:18:44 +01:00
Michael Eischer
a1eb923876 remove no longer necessary conditional compiles 2022-11-27 13:18:44 +01:00
Michael Eischer
bcdfc2a8ea CI: allow dependabot update of oauth2
Our minimum go version is new enough to allow updating the library.
2022-11-27 13:18:44 +01:00
Michael Eischer
686b0b2a3e update the minimum required go version to 1.18 2022-11-27 13:18:43 +01:00
Michael Eischer
69a2e81bd3 Merge pull request #4039 from restic/dependabot/go_modules/github.com/google/go-cmp-0.5.9
build(deps): bump github.com/google/go-cmp from 0.5.8 to 0.5.9
2022-11-26 17:39:27 +01:00
dependabot[bot]
278e93f738 build(deps): bump github.com/google/go-cmp from 0.5.8 to 0.5.9
Bumps [github.com/google/go-cmp](https://github.com/google/go-cmp) from 0.5.8 to 0.5.9.
- [Release notes](https://github.com/google/go-cmp/releases)
- [Commits](https://github.com/google/go-cmp/compare/v0.5.8...v0.5.9)

---
updated-dependencies:
- dependency-name: github.com/google/go-cmp
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-11-26 11:35:05 +00:00
Michael Eischer
747d2ecd7b Merge pull request #4042 from restic/skip-cloud-test-for-dependabot
CI: skip cloud tests for dependabot pull requests
2022-11-26 12:34:16 +01:00
Michael Eischer
98c6ca9d8f CI: skip cloud tests for dependabot pull requests 2022-11-26 12:23:55 +01:00
Michael Eischer
9113b2620f Merge pull request #4024 from MichaelEischer/macos-fuse
mount: switch to anacrolix fork of bazil/fuse
2022-11-26 12:15:14 +01:00
Michael Eischer
f115d64634 Merge pull request #4022 from MichaelEischer/race-checker
CI: Run the golang race checker
2022-11-26 12:13:50 +01:00
Michael Eischer
923c06cea0 Merge pull request #4025 from MichaelEischer/update-minio
Update minio library to add `credential_process` support
2022-11-25 23:21:57 +01:00
Michael Eischer
f4d3ed77c4 update minio library 2022-11-25 22:36:21 +01:00
greatroar
189e0fe5a9 fuse: Better inode generation
Hard links to the same file now get the same inode within the FUSE
mount. Also, inode generation is faster and, more importantly, no longer
allocates.

Benchmarked on Linux/amd64. Old means the benchmark with

        sink = fs.GenerateDynamicInode(1, sub.node.Name)

instead of calling inodeFromNode. Results:

name                   old time/op    new time/op    delta
Inode/no_hard_links-8     137ns ± 4%      34ns ± 1%   -75.20%  (p=0.000 n=10+10)
Inode/hard_link-8        33.6ns ± 1%     9.5ns ± 0%   -71.82%  (p=0.000 n=9+8)

name                   old alloc/op   new alloc/op   delta
Inode/no_hard_links-8     48.0B ± 0%      0.0B       -100.00%  (p=0.000 n=10+10)
Inode/hard_link-8         0.00B          0.00B           ~     (all equal)

name                   old allocs/op  new allocs/op  delta
Inode/no_hard_links-8      1.00 ± 0%      0.00       -100.00%  (p=0.000 n=10+10)
Inode/hard_link-8          0.00           0.00           ~     (all equal)
2022-11-16 08:35:01 +01:00
Michael Eischer
32ffcd86a2 Merge pull request #3993 from MichaelEischer/backup-json-full-snapshot-id
backup: print full snapshot id in JSON summary
2022-11-12 20:42:35 +01:00
Michael Eischer
f032a9d0ad prune: report how much data must be repacked to compressed the repo
prune now reports the remaining size of pack files containing
uncompressed blobs. The displayed value is suitable for use with `--max-repack-size`.
2022-11-12 20:20:23 +01:00
Michael Eischer
66818a8f98 Merge pull request #3980 from MichaelEischer/prune-compression-stats
prune: Correctly count used/duplicate blobs for partially compressed repos
2022-11-12 20:06:56 +01:00
Michael Eischer
4b5234924b Merge pull request #2875 from fgma/issue2699
issue2699: restore symlinks on windows when run as admin user
2022-11-12 20:06:45 +01:00
Michael Eischer
726a1969cd Merge pull request #2731 from dionorgua/rewrite-snapshot
Implement 'rewrite' command to exclude files from existing snapshots
2022-11-12 20:06:35 +01:00
Michael Eischer
bb0fa76c06 Cleanup exclude pattern collection 2022-11-12 19:55:22 +01:00
Michael Eischer
537cfe2e4c rewrite: Fix check that an exclude pattern was passed
The old check did not consider files containing case insensitive
excludes. The check is now implemented as a function of the
excludePatternOptions struct to improve cohesion.
2022-11-12 19:55:22 +01:00
Leo R. Lundgren
f175da2756 rewrite: Polish documentation 2022-11-12 19:55:22 +01:00
Leo R. Lundgren
f86ef4d3dd rewrite: Polish code and add missing messages 2022-11-12 19:55:22 +01:00
Leo R. Lundgren
c15bedccc0 rewrite: Revert unrelated documentation change 2022-11-12 19:55:22 +01:00
Michael Eischer
f88acd4503 rewrite: Fail if a tree contains an unknown field
In principle, the JSON format of Tree objects is extensible without
requiring a format change. In order to not loose information just play
it safe and reject rewriting trees for which we could loose data.
2022-11-12 19:55:22 +01:00
Michael Eischer
11b8c3a158 rewrite: add documentation 2022-11-12 19:55:22 +01:00
Michael Eischer
ec0c91e233 rewrite: Add tests for further ways to use the command 2022-11-12 19:55:22 +01:00
Michael Eischer
0224e276ec walker: Add tests for FilterTree 2022-11-12 19:55:22 +01:00
Michael Eischer
73f54cc5ea rewrite: rename --inplace to --forget 2022-11-12 19:55:22 +01:00
Michael Eischer
a47d9a1c40 rewrite: use unified snapshot filter options 2022-11-12 19:55:22 +01:00
Michael Eischer
b044649118 rewrite: add minimal test 2022-11-12 19:55:22 +01:00
Michael Eischer
375a3db64d rewrite: non-exclusive lock if snapshots are only added 2022-11-12 19:55:22 +01:00
Michael Eischer
327f418a9c rewrite: cleanup err handling and output 2022-11-12 19:55:22 +01:00
Michael Eischer
ad14d6e4ac rewrite: use SelectByName like in the backup command 2022-11-12 19:55:22 +01:00
Michael Eischer
7ebaf6e899 rewrite: start repository uploader goroutines 2022-11-12 19:55:22 +01:00
Michael Eischer
559acea0d8 unify exclude pattern options 2022-11-12 19:55:22 +01:00
Michael Eischer
4cace1ffe9 unify exclude patterns with backup command 2022-11-12 19:55:22 +01:00
Michael Eischer
2b69a1c53b rewrite: filter all snapshots if none are specified 2022-11-12 19:55:22 +01:00
Michael Eischer
f6339b88af rewrite: extract tree filtering 2022-11-12 19:55:22 +01:00
Michael Eischer
c0f7ba2388 rewrite: simplify dryrun 2022-11-12 19:55:22 +01:00
Michael Eischer
4d6ab83019 rewrite: use treejsonbuilder 2022-11-12 19:55:22 +01:00
Michael Eischer
82592b88b5 rewrite: address most review comments 2022-11-12 19:55:22 +01:00
Michael Eischer
b922774343 rewrite: fix compilation 2022-11-12 19:55:22 +01:00
Dmitry Nezhevenko
dc29709742 Implement 'rewrite' command to exclude files from existing snapshots 2022-11-12 19:55:22 +01:00
Michael Eischer
220eaee76b mount: switch to anacrolix fork of bazil/fuse
The anacrolix fork contains the latest changes from bazil/fuse and
additionally provides support for recent versions of macFUSE.
2022-11-12 19:22:31 +01:00
Michael Eischer
6fa45d0d39 Merge pull request #4011 from greatroar/backup-stdin-password
cmd: Don't read password from stdin for backup --stdin
2022-11-12 19:18:56 +01:00
Michael Eischer
bbd180ae21 Merge pull request #4017 from Rajpratik71/Rajpratik71-patch-1
feat: dependabot workflow automation for updating dependency
2022-11-12 15:48:48 +01:00
Pratik Raj
bef1064b8e chore: ignore upgrade for 'bazil/fuse' and 'golang.org/x/oauth2' 2022-11-12 19:39:16 +05:30
Michael Eischer
7b4fe7bad5 Merge pull request #4021 from greatroar/mac-fsync
backend/local: Ignore ENOTTY for fsync on Mac
2022-11-11 23:10:37 +01:00
greatroar
348e966daa backend/local: Ignore ENOTTY for fsync on Mac
Fixes #4016.
2022-11-11 22:51:51 +01:00
Michael Eischer
0e5fe4c6ab CI: run golang race checker 2022-11-11 22:15:22 +01:00
Michael Eischer
13fbc96ed3 lock: Synchronize Refresh() and Stale()
The lock test creates a lock and checks that it is not stale. However,
it is possible that the lock is refreshed concurrently, which updates
the lock timestamp. Checking the timestamp in `Stale()` without
synchronization results in a data race. Thus add a lock to prevent
concurrent accesses.
2022-11-11 21:52:53 +01:00
Michael Eischer
e1ba7ab684 lock: Don't copy the lock when checking for process existence
The lock test creates a lock and checks that it is not stale. This also
tests whether the corresponding process still exists. However, it is
possible that the lock is refreshed concurrently, which updates the lock
timestamp. Calling `processExists()` with a value receiver, however,
creates an unsynchronized copy of this field. Thus call the method using
a pointer receiver.
2022-11-11 21:45:55 +01:00
Michael Eischer
dc060356c2 mount: only start next test after mount command cleanup is complete
The test did not wait for the mount command to fully shutdown all
running goroutines. This caused the go race detector to report a data
race related to lock refreshes.

==================
WARNING: DATA RACE
Write at 0x0000021bdfdb by goroutine 667:
  github.com/restic/restic/internal/backend/retry.TestFastRetries()
      /restic/restic/internal/backend/retry/testing.go:7 +0x18f
  github.com/restic/restic/cmd/restic.withTestEnvironment()
      /restic/restic/cmd/restic/integration_helpers_test.go:175 +0x183
  github.com/restic/restic/cmd/restic.TestMountSameTimestamps()
      /restic/restic/cmd/restic/integration_fuse_test.go:202 +0xac
  testing.tRunner()
      /usr/lib/go/src/testing/testing.go:1446 +0x216
  testing.(*T).Run.func1()
      /usr/lib/go/src/testing/testing.go:1493 +0x47

Previous read at 0x0000021bdfdb by goroutine 609:
  github.com/restic/restic/internal/backend/retry.(*Backend).retry()
      /restic/restic/internal/backend/retry/backend_retry.go:72 +0x9e
  github.com/restic/restic/internal/backend/retry.(*Backend).Remove()
      /restic/restic/internal/backend/retry/backend_retry.go:149 +0x17d
  github.com/restic/restic/internal/cache.(*Backend).Remove()
      /restic/restic/internal/cache/backend.go:38 +0x11d
  github.com/restic/restic/internal/restic.(*Lock).Unlock()
      /restic/restic/internal/restic/lock.go:190 +0x249
  github.com/restic/restic/cmd/restic.refreshLocks.func1()
      /restic/restic/cmd/restic/lock.go:86 +0xae
  runtime.deferreturn()
      /usr/lib/go/src/runtime/panic.go:476 +0x32
  github.com/restic/restic/cmd/restic.lockRepository.func2()
      /restic/restic/cmd/restic/lock.go:61 +0x71

[...]

Goroutine 609 (finished) created at:
  github.com/restic/restic/cmd/restic.lockRepository()
      /restic/restic/cmd/restic/lock.go:61 +0x488
  github.com/restic/restic/cmd/restic.lockRepo()
      /restic/restic/cmd/restic/lock.go:25 +0x219
  github.com/restic/restic/cmd/restic.runMount()
      /restic/restic/cmd/restic/cmd_mount.go:126 +0x1f8
  github.com/restic/restic/cmd/restic.testRunMount()
      /restic/restic/cmd/restic/integration_fuse_test.go:61 +0x1ce
  github.com/restic/restic/cmd/restic.checkSnapshots.func1()
      /restic/restic/cmd/restic/integration_fuse_test.go:90 +0x124
==================
2022-11-11 21:43:01 +01:00
Michael Eischer
32c9667990 Merge pull request #4019 from MichaelEischer/fix-file-saver-race
archiver: Fix race condition resulting in files containing null IDs
2022-11-11 20:52:33 +01:00
Michael Eischer
d268552a0a Merge pull request #4014 from MichaelEischer/fix-debug-examine
debug: fix crash in `debug examine --reupload-blobs`
2022-11-10 20:37:32 +01:00
Michael Eischer
5756c96c9f archiver: Fix race condition resulting in files containing null IDs
In some rare cases files could be created which contain null IDs (all
zero) in their content list. This was caused by a race condition between
growing the `Content` slice and inserting the blob IDs into it. In some
cases the blob ID was written to the old slice, which a short time
afterwards was replaced with a larger copy, that did not yet contain the
blob ID.
2022-11-10 20:19:37 +01:00
Pratik Raj
df614fff26 feat: dependabot workflow automation for updating dependency
Signed-off-by: Pratik Raj <Rajpratik71@gmail.com>
2022-11-10 16:02:03 +05:30
Michael Eischer
11a4bb051e debug: fix crash in debug examine --reupload-blobs 2022-11-09 22:13:17 +01:00
Michael Eischer
5f9ac2b165 Merge pull request #4010 from MichaelEischer/file-saver-sanity-check
archiver: Check that saved file does not have null IDs in content
2022-11-08 23:07:32 +01:00
Michael Eischer
b1d1202b1d archiver: Check that saved file does not have null IDs in content
Null IDs in the file content indicate that something went wrong. Thus
fails before saving the affected file.
2022-11-08 22:57:41 +01:00
greatroar
5dceadeb72 cmd: Don't read password from stdin for backup --stdin 2022-11-06 14:55:57 +01:00
Michael Eischer
1ccab95bc4 b2: Support file hiding instead of deleting them permanently
Automatically fall back to hiding files if not authorized to permanently
delete files. This allows using restic with an append-only application
key with B2.  Thus, an attacker cannot directly delete backups with the
API key used by restic.

To use this feature create an application key without the deleteFiles
capability. It is recommended to restrict the key to just one bucket.
For example using the b2 command line tool:

    b2 create-key --bucket <bucketName> <keyName> listBuckets,readFiles,writeFiles,listFiles

Suggested-by: Daniel Gröber <dxld@darkboxed.org>
2022-11-05 20:10:45 +01:00
Michael Eischer
24a2e5cab9 Merge pull request #4008 from MichaelEischer/tweak-lock-refresh-test
lock: Tweak timeouts for lock refresh test
2022-11-05 10:53:13 +01:00
Michael Eischer
403390479c Merge pull request #3997 from greatroar/fuse-hash
fuse: Better check for whether snapshots changed
2022-11-05 10:52:11 +01:00
Michael Eischer
d29abc1a31 Merge pull request #4007 from MichaelEischer/hide-compression-level-for-v1-repo
Only print compression level starting from repository version 2
2022-11-05 10:33:25 +01:00
greatroar
c091e43b33 fuse: Better check for whether snapshots changed
We previously checked whether the set of snapshots might have changed
based only on their number, which fails when as many snapshots are
forgotten as are added. Check for the SHA-256 of their id's instead.
2022-11-05 09:32:45 +01:00
Michael Eischer
aaac63da8d lock: Tweak timeouts for lock refresh test
For some reason the test fails from time to time. Increase the timeouts
to hopefully avoid this issue.
2022-11-04 22:48:18 +01:00
Michael Eischer
fd4d23460f only print compression level starting from repository version 2 2022-11-04 22:40:07 +01:00
Alexander Neumann
8dd95b710e Merge pull request #3992 from MichaelEischer/err-on-invalid-compression
Return error if RESTIC_COMPRESSION env variable is invalid
2022-11-04 19:41:34 +01:00
Alexander Neumann
783b8781a7 Merge pull request #4000 from restic/min-go-version
build: Correct checks for minimum Go version
2022-11-04 10:31:02 +01:00
Alexander Neumann
543649f2f2 Merge pull request #4001 from restic/docker-go-version
docker: Increase Go version to 1.19
2022-11-04 10:30:11 +01:00
Leo R. Lundgren
0a4cddb34d docker: Increase Go version to 1.19 2022-11-03 22:59:59 +01:00
Leo R. Lundgren
333c2c6ed4 build: Correct checks for minimum Go version 2022-11-03 22:50:07 +01:00
rawtaz
92df039e5d Merge pull request #3996 from MichaelEischer/fix-ui-progress
backup: fix stuck status bar
2022-11-02 21:48:16 +01:00
Michael Eischer
9354262b1b backup: fix stuck status bar
The status bar got stuck once the first error was reported, the scanner
completed or some file was backed up. Either case sets a flag that the
scanner has started.

This flag is used to hide the progress bar until the flag is set. Due to
an inverted condition, the opposite happened and the status stopped
refreshing once the flag was set.

In addition, the scannerStarted flag was not set when the scanner just
reported progress information.
2022-11-02 21:31:13 +01:00
Michael Eischer
06141ce1f4 backup: print full snapshot id in JSON summary 2022-10-31 19:03:42 +01:00
Michael Eischer
59a90943bb Merge pull request #3983 from greatroar/formatting
Centralize and fix formatting of bytes, percentages, durations
2022-10-31 18:52:24 +01:00
greatroar
5ab3e6276a ui: Fix FormatBytes at exactly 1024 time a unit
1024 would be displayed as "1024 bytes" instead of "1.000 KiB", etc.
2022-10-31 18:39:28 +01:00
rawtaz
4f1fae9c98 Merge pull request #3982 from MichaelEischer/show-compression-mode
Show selected compression level when opening repository
2022-10-30 21:29:42 +01:00
Michael Eischer
8fe159cc5a enable ysmlink tests for windows 2022-10-30 18:43:04 +01:00
Michael Eischer
3499c6354e Merge pull request #3955 from MichaelEischer/async-futurefile-completion
Improve archiver performance for small files
2022-10-30 18:38:04 +01:00
Michael Eischer
144257f8bd restore symlink timestamps on windows 2022-10-30 11:04:04 +01:00
Michael Eischer
c0f34af9db backup: hide files from status which are read completely but not saved
As the FileSaver is asynchronously waiting for all blobs of a file to be
stored, the number of active files is higher than the number of files
from which restic is reading concurrently. Thus to not confuse users,
only display files in the status from which restic is currently reading.
2022-10-30 10:29:12 +01:00
Michael Eischer
a571fc4aa1 add changelog for faster backups with small files 2022-10-30 10:29:12 +01:00
Michael Eischer
b52a8ff05c ui: Properly clear lines no longer used for status
Previously, the old status text remained until it was overwritten.
2022-10-30 10:29:12 +01:00
Michael Eischer
b4de902596 archiver: Asynchronously complete FutureFile
After reading and chunking all data in a file, the FutureFile still has
to wait until the FutureBlobs are completed. This was done synchronously
which results in blocking the file saver and prevents the next file from
being read.

By replacing the FutureBlob with a callback, it becomes possible to
complete the FutureFile asynchronously.
2022-10-30 10:29:11 +01:00
Michael Eischer
47e05080a9 Merge pull request #3990 from MichaelEischer/lock-refresh-test
lock: add test to check that refreshing works
2022-10-30 10:15:44 +01:00
Michael Eischer
c7ace314f6 Merge pull request #3989 from greatroar/eachbypack
More compact data structure for Index.EachByPack
2022-10-30 00:02:55 +02:00
greatroar
0e8893dae9 index: Compact data structure for Index.EachByPack 2022-10-29 23:09:17 +02:00
greatroar
137f0bc944 repository: Fix benchmarkSaveAndEncrypt 2022-10-29 23:09:17 +02:00
Michael Eischer
01f0db4e56 return error if RESTIC_COMPRESSION env variable is invalid 2022-10-29 22:03:39 +02:00
Michael Eischer
7c87fb941c Merge pull request #3986 from greatroar/counter
ui/progress: Load both values in a single Lock/Unlock
2022-10-29 21:50:55 +02:00
Michael Eischer
3b0bb02a68 Merge pull request #3977 from greatroar/progress
ui/backup: Replace channels with a mutex
2022-10-29 21:33:04 +02:00
Michael Eischer
0d260cfd82 enable symlink test on windows 2022-10-29 21:26:34 +02:00
fgma
8e5eb1090c issue2699: restore symlinks on windows when run as admin user 2022-10-29 21:19:33 +02:00
rawtaz
af3f7c866f Merge pull request #3988 from FelixBurkhard/FelixBurkhard-patch-1
Clearify what Azure account name means
2022-10-29 13:32:44 +02:00
Michael Eischer
24267e9a9d lock: add test to check that refreshing works 2022-10-29 11:26:00 +02:00
Michael Eischer
8e51e1e605 shorten 'repository opened' output 2022-10-29 11:22:00 +02:00
FelixBurkhard
575d26ec87 Clearify what Azure account name means
When reading it first it was not clear to me the 'account name' meant the name of the
Azure Storage Account and not the Azure account itself.
2022-10-29 00:27:43 +02:00
greatroar
2dafda9164 ui/progress: Load both values in a single Lock/Unlock
We always need both values, except in a test, so we don't need to lock
twice and risk scheduling in between.

Also, removed the resetting in Done. This copied a mutex, which isn't
allowed. Static analyzers tend to trip over that.
2022-10-25 07:55:24 +02:00
Michael Eischer
f8910bc4ff Merge pull request #3985 from saltsa/fix_lock_refresh
Fix bug in lock refresh monitoring
2022-10-24 22:59:18 +02:00
Joonas Aunola
b06427c9f6 fix Unix to UnixNano 2022-10-23 23:40:21 +03:00
greatroar
006380199e cmd, ui: Deduplicate formatting utilities 2022-10-23 13:40:07 +02:00
greatroar
04216eb9aa ui/backup: Replace channels with a mutex
The channel-based algorithm had grown quite complicated. This is easier
to reason about and likely to be more performant with very many
CompleteBlob calls.
2022-10-23 13:28:41 +02:00
Michael Eischer
4fea3a413d show selected compression level when opening repository 2022-10-22 20:18:46 +02:00
Michael Eischer
ba58ccbe07 prune: add remark about non-deterministic blob selection 2022-10-22 19:46:10 +02:00
Michael Eischer
05651d6d4f prune: Correctly count used/duplicate blobs for partially compressed repos
Counting the first occurrence of a duplicate blob as used and counting
all other as duplicates, independent of which instance of the blob is
kept, is only accurate if all copies of the blob have the same size. This
is no longer the case for a repository containing both compressed and
uncompressed blobs.

Thus for duplicated blobs first count all instances as duplicates and
then subtract the actually used instance later on.
2022-10-22 19:24:36 +02:00
Michael Eischer
b57d42905c Merge pull request #3899 from MichaelEischer/less-prune-mem
Optimize prune memory usage
2022-10-22 18:56:02 +02:00
Michael Eischer
d966c52707 prune: allow gc of set of repacked blobs before index rebuild 2022-10-22 18:45:12 +02:00
Michael Eischer
1e2794fa55 add prune memory optimization changelog 2022-10-22 18:45:12 +02:00
Michael Eischer
68c9cb9c6a prune: Shrink keepBlobs set if possible
As long as only a small fraction of the data in a repository is
rewritten, the keepBlobs set will be rather small after cleaning it up.
As golang maps do not shrink their memory usage, just copy the contents
over to a new map. However, only copy the map if the cleanup removed at
least half the entries.
2022-10-22 18:45:12 +02:00
Michael Eischer
c4fc5c97f9 prune: Use a single CountedBlobSet to track blobs
The set covers necessary, existing and duplicate blobs. This removes the
duplicate sets used to track whether all necessary blobs also exist.
This reduces the memory usage of prune by about 20-30%.
2022-10-22 18:45:12 +02:00
Michael Eischer
b21241ec1c restic: Add CountedBlobSet type
This allows maintaining a usage counter for each blob.
2022-10-22 18:45:12 +02:00
Michael Eischer
ee6688a9f6 Merge pull request #3915 from plumbeo/compression-stats
restic stats: print uncompressed size in mode raw-data
2022-10-21 22:10:29 +02:00
Michael Eischer
27634a1a68 Merge pull request #3978 from MichaelEischer/fix-negative-pattern-example
Remove misleading wildcard from negative exclude pattern example
2022-10-21 22:04:30 +02:00
Michael Eischer
aa77702e49 Merge pull request #3971 from MichaelEischer/parallel-list
Unify ForAllIndex/Snapshot/Lock functions
2022-10-21 21:58:33 +02:00
Michael Eischer
6877aaa8aa Merge pull request #3967 from MichaelEischer/archiver-extract-exclude-options
backup: extract exclude pattern options
2022-10-21 21:50:00 +02:00
Michael Eischer
2e9ee8577a Merge pull request #3970 from MichaelEischer/split-retry-backend
Split backend package into smaller parts
2022-10-21 21:49:46 +02:00
Michael Eischer
59d46bb3f5 backup: extract exclude pattern options
This is a preparation to make the exclude options usable for the
upcoming `rewrite` command.
2022-10-21 21:40:59 +02:00
Michael Eischer
5c7a9a739a backend: Split RetryBackend into own package
The RetryBackend tests depend on the mock backend. When the Backend
interface is eventually split from the restic package, this will lead to
a dependency cycle between backend and backend/mock. Thus split the
RetryBackend into a separate package to avoid this problem.
2022-10-21 21:38:17 +02:00
Michael Eischer
32603d49c4 backend: remove unused ErrorBackend 2022-10-21 21:36:05 +02:00
Michael Eischer
8c18c65b3b backend: remove unused Paths variable 2022-10-21 21:36:05 +02:00
Michael Eischer
4ccd5e806b backend: split layout code into own subpackage 2022-10-21 21:36:05 +02:00
Michael Eischer
b361284f28 Merge pull request #3979 from MichaelEischer/backup-less-time-now
backup: reduce calls to time.Now
2022-10-21 21:33:34 +02:00
Michael Eischer
738b2a0445 parallelize more List usages 2022-10-21 21:26:45 +02:00
Michael Eischer
ae45f3b04f restic: Unify code to load Index/Lock/Snapshot 2022-10-21 21:25:11 +02:00
Michael Eischer
8e2695be0b Merge pull request #3973 from MichaelEischer/speedup-integration-tests
speed-up integration tests by reducing the RetryBackend timeout
2022-10-21 21:17:35 +02:00
Michael Eischer
35d968bcde Merge pull request #3969 from MichaelEischer/key-by-id
Port restic.Find to return IDs and identify keys by restic.ID
2022-10-21 21:15:40 +02:00
Michael Eischer
4133fee6f9 Merge pull request #3972 from MichaelEischer/fix-flaky-lock-cancel-test
lock: fix flaky TestLockFailedRefresh
2022-10-21 21:12:34 +02:00
Michael Eischer
c8c8391b21 Merge pull request #3974 from greatroar/cleanup
More cleanups and a micro-optimization
2022-10-21 21:11:37 +02:00
Michael Eischer
ee7c28f5e6 backup: reduce calls to time.Now
Archiver.Save queries the current time multiple times. This commit
removes one of these calls as they showed up while profiling a backup of
a nearly unchanged dataset containing 3 million files.
2022-10-21 20:55:01 +02:00
Michael Eischer
3e60d38a23 Remove misleading wildcard from negative exclude pattern example
There is no need to use a special wildcard `**` to demonstrate negative
patterns. Actually, it is both slower than the simpler variant and seems
to confuse users.
2022-10-21 20:48:45 +02:00
greatroar
9adae5521d cache: Call interface method once 2022-10-21 14:32:46 +02:00
greatroar
201e5c7e74 backup: Clean up progress reporting code 2022-10-21 13:48:30 +02:00
plumbeo
a6f83e0011 Add changelog 2022-10-17 15:38:42 +02:00
plumbeo
bc945d0bf0 restic stats: add more compression statistics
Calculate and display compression ratio, space saving and progress
2022-10-17 15:38:38 +02:00
greatroar
b513597546 internal/restic: Make FileType a uint8 instead of a string
The string form was presumably useful before the introduction of
layouts, but right now it just makes call sequences and garbage
collection more expensive (the latter because every string contains
a pointer to be scanned).
2022-10-16 10:59:01 +02:00
greatroar
22147e1e02 all: Minor cleanups
if x { return true } return false => return x

	fmt.Sprintf("%v", x) => fmt.Sprint(x) or x.String()

The fmt.Sprintf idiom is still used in the SecretString tests, where it
serves security hardening.
2022-10-16 10:50:39 +02:00
greatroar
d03460010f internal/restic: Fix ID.UnmarshalJSON, ParseID
ID.UnmarshalJSON accepted non-JSON input with ' as the string delimiter.
Also, the error message for non-hex input was less informative than it
could be and it performed too many checks.

Changed ParseID to keep the error messages consistent.
2022-10-16 10:39:52 +02:00
Michael Eischer
aa39bf3cf6 backend/test: remove duplicate test
The test is identical to the tests for the mem backend.
2022-10-15 23:15:07 +02:00
Michael Eischer
28e1c4574b mem: use cheaper hash for backend 2022-10-15 23:14:33 +02:00
Michael Eischer
c3400d3c55 backend: speedup RetryBackend tests 2022-10-15 23:13:44 +02:00
Michael Eischer
99547518cd lock: fix flaky TestLockFailedRefresh
The comparison of the current time and the last lock refresh were using
seconds represented as integers. As the test only waits for up to one
second, the associated number truncation can cause the test to take
longer than once second and thus to fail.

Switch to nanoseconds to avoid this problem. This also slightly speeds
up the test.
2022-10-15 22:36:32 +02:00
Michael Eischer
e10420553b speed-up integration tests by reducing the RetryBackend timeout
On my machine this decreases the runtime for `./cmd/restic` from 9.5s to
6.5s.
2022-10-15 22:29:58 +02:00
Michael Eischer
367f35db27 Merge pull request #3968 from MichaelEischer/cleanup-complete-blob
backup: Remove unused filename parameter from CompleteBlob callback
2022-10-15 16:11:16 +02:00
Michael Eischer
8d62a7adb4 identify keys by ID and not name 2022-10-15 16:07:43 +02:00
Michael Eischer
02634dce7a restic: change Find to return ids
That way consumers no longer have to manually convert the returned name
to an id.
2022-10-15 16:06:54 +02:00
Michael Eischer
964977677f backup: Remove unused filename parameter from CompleteBlob callback 2022-10-15 15:21:17 +02:00
Michael Eischer
258b487d8f Merge pull request #3951 from MichaelEischer/rework-snapshot-filter
Rework snapshot filtering
2022-10-15 14:47:47 +02:00
Michael Eischer
de9bc031df add changelog for ls handling of missing snapshots 2022-10-15 13:34:50 +02:00
Michael Eischer
246d3032ae restic: Don't list snapshots if FindSnapshot gets full id 2022-10-15 13:34:34 +02:00
Michael Eischer
d8c00b9726 add comment 2022-10-15 13:34:21 +02:00
Michael Eischer
a3113c6097 restic: Change FindSnapshot functions to return the snapshot 2022-10-15 13:34:04 +02:00
Michael Eischer
b50f48594d restic: cleanup arguments of findLatestSnapshot 2022-10-15 13:33:48 +02:00
Michael Eischer
61e827ae4f restic: hide findLatestSnapshot 2022-10-15 13:33:32 +02:00
Michael Eischer
fcad5e6f5d backup: use unified FindFilteredSnapshot 2022-10-15 13:33:29 +02:00
Michael Eischer
0aa73bbd39 ls: proper error handling for non-existent snapshot
Use restic.FindFilteredSnapshot to resolve the snapshot ID. This ensures
consistent behavior for all commands using initSingleSnapshotFilterOptions.
2022-10-15 13:32:00 +02:00
Michael Eischer
a81f0432e9 restic: Add unified method to resolve a single snapshot 2022-10-15 13:31:45 +02:00
Michael Eischer
95a1bb4261 restic: Rework error handling of FindFilteredSnapshots and handle snapshotIDs
FindFilteredSnapshots no longer prints errors during snapshot loading on
stderr, but instead passes the error to the callback to allow the caller
to decide on what to do.

In addition, it moves the logic to handle an explicit snapshot list from
the main package to restic.
2022-10-15 13:31:26 +02:00
Michael Eischer
cff22a5f01 dump: use correct help text for filter options 2022-10-15 13:31:10 +02:00
Michael Eischer
7a6dcb4831 Merge pull request #3966 from MichaelEischer/cleanup-walker-test
walker: Convert tests to use TreeJSONBuilder
2022-10-15 11:25:11 +02:00
Michael Eischer
7cf042118f walker: Convert tests to use TreeJSONBuilder
The old code marshalled the tree blobs different than other places in
restic. The hashed tree blob did not contain a final newline character.
2022-10-15 11:04:13 +02:00
Michael Eischer
cea7191995 Merge pull request #3959 from MichaelEischer/buffered-backup-progress
backup: Use buffered channels to collect backup status
2022-10-15 10:57:19 +02:00
Michael Eischer
ba688aad20 Merge pull request #3961 from greatroar/cleanup
Misc. cleanup
2022-10-14 21:49:35 +02:00
Michael Eischer
9c290a8093 Merge pull request #3960 from greatroar/errors
errors: Drop WithMessage
2022-10-14 21:41:28 +02:00
greatroar
0e155fd9a6 internal/restic: Fix UID/GID parsing
The helper function uidGidInt used strconv.ParseInt instead of
ParseUint, so it silently ignored some invalid user/group IDs.

Also, improve the error message. "Invalid UID" is more informative than
having "ParseInt" twice (*strconv.NumError displays the function name).

Finally, the user.User struct can be passed by pointer to get reduce
code size.
2022-10-14 18:21:00 +02:00
greatroar
e0b743c64d internal/restic: Remove unused ID.EqualString 2022-10-14 18:20:11 +02:00
greatroar
6922360179 ui/backup: Remove unused ProgressReporter type, Progress field 2022-10-14 14:36:19 +02:00
greatroar
d4aadfa389 all: Drop ctxhttp
This package is no longer needed, since we can use the stdlib's
http.NewRequestWithContext.

backend/rclone already did, but it needed a different error check due to
a difference between net/http and ctxhttp.

Also, store the http.Client by value in the REST backend (changed to a
pointer when ctxhttp was introduced) and use errors.WithStack instead
of errors.Wrap where the message was no longer accurate. Errors from
http.NewRequestWithContext will start with "net/http" or "net/url", so
they're easy to identify.
2022-10-14 14:33:49 +02:00
greatroar
16849d5361 internal/archiver: Missing argument to errors.Errorf 2022-10-14 14:18:52 +02:00
greatroar
09c14f33c8 internal/checker: Pass Error.Error pointer receiver 2022-10-14 14:13:32 +02:00
greatroar
feb790f497 internal/restic: Use errors.New when no formatting is needed 2022-10-14 14:07:20 +02:00
greatroar
ba44666704 errors: Drop WithMessage 2022-10-14 14:06:47 +02:00
Michael Eischer
1a6160d152 Merge pull request #3880 from MichaelEischer/archiver-savedir-cleanup
archiver: Improve handling of "file xxx already present" error
2022-10-08 21:48:14 +02:00
Michael Eischer
21b1d7a880 Merge pull request #3948 from MichaelEischer/split-index
repository: split index into a separate package
2022-10-08 21:41:57 +02:00
Michael Eischer
5278ab51c8 archiver: Check that duplicates are only ignored if identical 2022-10-08 21:38:36 +02:00
Michael Eischer
403b01b788 backup: Only return a warning for duplicate directory entries
The backup command failed if a directory contains duplicate entries.
Downgrade the severity of this problem from fatal error to a warning.
This allows users to still create a backup.
2022-10-08 21:38:21 +02:00
Michael Eischer
d7d7b4ab27 archiver: refactor TreeSaverTest 2022-10-08 21:29:32 +02:00
Michael Eischer
8e38c43c27 archiver: let FutureNode.Take return an error if no data is available
This ensures that we cannot accidentally store an invalid node.
2022-10-08 21:28:39 +02:00
Michael Eischer
2b88cd6eab archiver: Restructure SaveTree to work like SaveDir
SaveTree did not use the TreeSaver but rather managed the tree
collection and upload itself. This prevents using the parallelism
offered by the TreeSaver and duplicates all related code. Using the
TreeSaver can provide some speed-ups as all steps within the backup tree
now rely on FutureNodes. This can be especially relevant for backups
with large amounts of explicitly specified files.

The main difference between SaveTree and SaveDir is, that only the
former can save tree blobs in which nodes have a different name than the
actual file on disk. This is the result of resolving name conflicts
between multiple files with the same name. The filename that must be
used within the snapshot is now passed directly to
restic.NodeFromFileInfo. This ensures that a FutureNode already contains
the correct filename.
2022-10-08 21:28:39 +02:00
Michael Eischer
2e3f1c08c5 repository: split index into a separate package 2022-10-08 21:15:34 +02:00
Michael Eischer
5760ba6989 Merge pull request #3949 from MichaelEischer/simplify-mixedpacks
repository: remove IsMixedPack and add replacement for checker
2022-10-08 21:14:14 +02:00
Michael Eischer
5ee25e669a Merge pull request #3940 from MichaelEischer/better-rclone-error
Better error message if connection to rclone fails
2022-10-08 21:14:00 +02:00
Michael Eischer
5600f11696 rclone: Fix stderr handling if command exits unexpectedly
According to the documentation of exec.Cmd Wait() must not be called
before completing all reads from the pipe returned by StdErrPipe(). Thus
return a context that is canceled once rclone has exited and use that as
a precondition to calling Wait(). This should ensure that all errors
printed to stderr have been copied first.
2022-10-08 20:16:06 +02:00
Michael Eischer
b8acad4da0 rclone: return rclone error instead of canceled context
When rclone fails during the connection setup this currently often
results in a context canceled error. Replace this error with the exit
code from rclone.
2022-10-08 20:15:24 +02:00
Michael Eischer
d3ebec8f21 backup: Use buffered channels to collect backup status
When backing up many small files, the unbuffered channels frequently
cause the FileSaver to block when reporting progress information. Thus,
add buffers to these channels to avoid unnecessary scheduling.

As the status information is purely informational, it doesn't matter
that the status reporting shutdown is somewhat racy and could miss a few
final updates.
2022-10-08 18:20:41 +02:00
Michael Eischer
f9d4e0c2af Merge pull request #3958 from greatroar/errors
errors: Drop Cause in favor of Go 1.13 error handling
2022-10-08 18:06:35 +02:00
Michael Eischer
119e6aee01 Merge pull request #3957 from greatroar/typo
cmd: Typo in --read-concurrency description
2022-10-08 14:41:35 +02:00
greatroar
07e5c38361 errors: Drop Cause in favor of Go 1.13 error handling
The only use cases in the code were in errors.IsFatal, backend/b2,
which needs a workaround, and backend.ParseLayout. The last of these
requires all backends to implement error unwrapping in IsNotExist.
All backends except gs already did that.
2022-10-08 13:08:08 +02:00
greatroar
4eae4d3e1a cmd: Typo in --read-concurrency description 2022-10-08 11:27:39 +02:00
Michael Eischer
83cb58b4f3 Merge pull request #3956 from MichaelEischer/fix-lock-refresh
lock: Use the correct duration to check for expired locks
2022-10-07 22:58:10 +02:00
Michael Eischer
7c5d63a794 lock: Use the correct duration to check for expired locks 2022-10-07 22:39:53 +02:00
Michael Eischer
8b7c952f17 Merge pull request #3953 from keachi/typo
Fix typo
2022-10-07 22:18:32 +02:00
Michael Eischer
e43d2d45f7 Merge pull request #3952 from hoelzro/master
Update copy documentation to use --from-repo option
2022-10-07 22:18:06 +02:00
Rob Hoelz
03e9a26018 Update copy documentation to use --from-repo option
Removing the last references to the deprecated --repo2 option
2022-10-07 22:00:12 +02:00
tr
43cc01d63e doc: Fix typo 2022-10-05 21:03:14 +02:00
Michael Eischer
7112a132c3 Merge pull request #3950 from MichaelEischer/misc-cleanups
Cleanups for cmd_debug/repository and remove dead code from restic package
2022-10-03 12:46:32 +02:00
Michael Eischer
4bb5240720 repository: remove unused PrefixLength 2022-10-03 12:15:53 +02:00
Michael Eischer
999fe29976 repository: hide prepareCache 2022-10-03 12:15:53 +02:00
Michael Eischer
9197c63007 debug: use repository.ListPack wrapper 2022-10-03 12:09:08 +02:00
Michael Eischer
ddcf549eba repository: remove IsMixedPack and add replacement for checker
Repositories with mixed packs are probably quite rare by now. When
loading data blobs from a mixed pack file, this will no longer trigger
caching that file. However, usually tree blobs are accessed first such
that this shouldn't make much of a difference.

The checker gets a simpler replacement.
2022-10-03 12:03:59 +02:00
Michael Eischer
a61fbd287a Merge pull request #3569 from MichaelEischer/strict-locking
Strict repository lock handling
2022-10-03 00:44:44 +02:00
Michael Eischer
6d2d297215 pass global context through cobra 2022-10-03 00:19:46 +02:00
Michael Eischer
49126796d0 lock: fix timer expiry monitoring during standby
Monotonic timers are paused during standby. Thus these timers won't fire
after waking up. Fall back to periodic polling to detect too large clock
jumps. See https://github.com/golang/go/issues/35012 for a discussion of
go timers during standby.
2022-10-03 00:19:46 +02:00
Michael Eischer
401e432e9d lock: Do not ignore invalid lock files
While searching for lock file from concurrently running restic
instances, restic ignored unreadable lock files. These can either be
in fact invalid or just be temporarily unreadable. As it is not really
possible to differentiate between both cases, just err on the side of
caution and consider the repository as already locked.

The code retries searching for other locks up to three times to smooth
out temporarily unreadable lock files.
2022-10-03 00:19:46 +02:00
Michael Eischer
aeed420e1a add changelog 2022-10-03 00:19:46 +02:00
Michael Eischer
9959190e39 lock: Add integration test
The tests check that the wrapped context is properly canceled whenever
the repository is unlock or when the lock refresh fails.
2022-10-03 00:19:46 +02:00
Michael Eischer
c3538b063a lock: Use repository interface instead of struct 2022-10-03 00:19:46 +02:00
Michael Eischer
d92957dd78 lock: Implement strict lock expiry monitoring
Restic continued e.g. a backup task even when it failed to renew the
lock or failed to do so in time. For example if a backup client enters
standby during the backup this can allow other operations like `prune`
to run in the meantime (after calling `unlock`). After leaving standby
the backup client will continue its backup and upload indexes which
refer pack files that were removed in the meantime.

This commit introduces a goroutine explicitly monitoring for locks that
are not refreshed in time. To simplify the implementation there's now a
separate goroutine to refresh the lock and monitor for timeouts for each
lock. The monitoring goroutine would now cause the backup to fail as the
client has lost it's lock in the meantime.

The lock refresh goroutines are bound to the context used to lock the
repository initially. The context returned by `lockRepo` is also
cancelled when any of the goroutines exits. This ensures that the
context is cancelled whenever for any reason the lock is no longer
refreshed.
2022-10-03 00:19:46 +02:00
Michael Eischer
928914f821 Prepare for context bound to lock lifetime 2022-10-03 00:19:46 +02:00
Michael Eischer
985722b102 Remove ctx from globalOptions
Previously the global context was either accessed via gopts.ctx,
stored in a local variable and then used within that function or
sometimes both. This makes it very hard to follow which ctx or a wrapped
version of it reaches which method.

Thus just drop the context from the globalOptions struct and pass it
explicitly to every command line handler method.
2022-10-03 00:19:46 +02:00
Michael Eischer
ab819b2344 key: Cleanup method signatures 2022-10-03 00:19:46 +02:00
Michael Eischer
d0668b695d Remove unnecessary context.WithCancel calls
The gopts.ctx is cancelled when the main() method of restic exits.
2022-10-03 00:19:46 +02:00
Michael Eischer
7ce4cb7908 Merge pull request #3947 from MichaelEischer/fix-cache-verify-test
cache: Fix file descriptor leak in TestBackendRemoveBroken
2022-10-03 00:19:26 +02:00
Michael Eischer
430ab32941 cache: Fix file descriptor leak in TestBackendRemoveBroken 2022-10-03 00:06:44 +02:00
Michael Eischer
e99ad39b34 Merge pull request #2750 from metalsp0rk/min-packsize
Add `backup --file-read-concurrency` flag
2022-10-02 23:11:47 +02:00
Michael Eischer
2e606ca70b backup: rework read concurrency 2022-10-02 22:55:14 +02:00
Kyle Brennan
4a501d7118 backup: add option for file read concurrency 2022-10-02 22:51:45 +02:00
Michael Eischer
9ec7eee803 Merge pull request #3521 from MichaelEischer/redownload-broken-files
Redownload files with wrong hash
2022-10-02 22:50:03 +02:00
Michael Eischer
b25d0773b6 Merge pull request #3944 from MichaelEischer/fix-linter-errors
CI: ignore warning about missing package comment
2022-09-27 21:41:55 +02:00
Michael Eischer
5265550ff3 CI: ignore warning about missing package comment 2022-09-27 21:31:37 +02:00
Michael Eischer
e89fc2a29d Merge pull request #3943 from MichaelEischer/find-match-only-valid-ids
ignore filenames which are not IDs when expanding a prefix
2022-09-27 20:56:48 +02:00
Michael Eischer
67e4620cd6 Merge pull request #3938 from restic/errdot
rclone/sftp: Improve handling of ErrDot errors
2022-09-27 20:33:42 +02:00
Michael Eischer
5d3c5b9e50 restic: ignore filenames which are not IDs when expanding a prefix
Some backends generate additional files for each existing file, e.g.

1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef
1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.sha256

For some commands this leads to an "multiple IDs with prefix" error when
trying to reference a snapshot.
2022-09-27 20:30:40 +02:00
Leo R. Lundgren
ebe9f2c969 rclone/sftp: Improve handling of ErrDot errors
Restic now yields a more informative error message when exec.ErrDot occurs.
2022-09-25 16:19:03 +02:00
Michael Eischer
d114e483c4 Add changelog for corrupt data downloads 2022-09-25 11:55:09 +02:00
Michael Eischer
34c1a83340 cache: Drop cache entry if it cannot be processed
Failing to process data requested from the cache usually indicates a
problem with the returned data. Assume that the cache entry is somehow
damaged and retry downloading it once.
2022-09-25 11:55:09 +02:00
Michael Eischer
aa3b1925b4 cache: Simplify loadFromCacheOrDelegate 2022-09-25 11:35:35 +02:00
Michael Eischer
5c6b6edefe retry index, lock and snapshot loading on hash mismatch 2022-09-25 11:35:35 +02:00
Michael Eischer
822422ef03 retry key loading on hash mismatch 2022-09-25 11:35:35 +02:00
Michael Eischer
d6575f53ca Merge pull request #3942 from MichaelEischer/split-cross-compile-test
Split cross compile test
2022-09-24 22:27:08 +02:00
Michael Eischer
78d2312ee9 Merge pull request #3854 from MichaelEischer/sparsefiles
restore: Add support for sparse files
2022-09-24 22:04:02 +02:00
Michael Eischer
46b30b9826 split cross compilation into three parts
The cross compilation tasks are currently the slowest part of the CI
runs. Splitting it into three parts should reduce its time to roughly
that of the windows CI run.
2022-09-24 22:00:25 +02:00
Michael Eischer
bd191ec60b update golang-ci to version 1.49 2022-09-24 22:00:08 +02:00
Michael Eischer
519059cca4 update ci actions 2022-09-24 21:59:36 +02:00
Michael Eischer
19afad8a09 restore: support sparse restores also on windows 2022-09-24 21:39:39 +02:00
Michael Eischer
0f89f443c7 update sparse restore changelog 2022-09-24 21:39:39 +02:00
Michael Eischer
c147422ba5 repository: special case SaveBlob for all zero chunks
Sparse files contain large regions containing only zero bytes. Checking
that a blob only contains zeros is possible with over 100GB/s for modern
x86 CPUs. Calculating sha256 hashes is only possible with 500MB/s (or
2GB/s using hardware acceleration). Thus we can speed up the hash
calculation for all zero blobs (which always have length
chunker.MinSize) by checking for zero bytes and then using the
precomputed hash.

The all zeros check is only performed for blobs with the minimal chunk
size, and thus should add no overhead most of the time. For chunks which
are not all zero but have the minimal chunks size, the overhead will be
below 2% based on the above performance numbers.

This allows reading sparse sections of files as fast as the kernel can
return data to us. On my system using BTRFS this resulted in about
4GB/s.
2022-09-24 21:39:39 +02:00
Michael Eischer
34fe1362da restorer: move zeroPrefixLen to restic package 2022-09-24 21:39:39 +02:00
Michael Eischer
a5ebd5de4b restorer: Fix race condition in partialFile.WriteAt
The restorer can issue multiple calls to WriteAt in parallel. This can
result in unexpected orderings of the Truncate and WriteAt calls and
sometimes too short restored files.
2022-09-24 21:39:39 +02:00
Michael Eischer
5b6a77058a Enable sparseness only conditionally
We can either preallocate storage for a file or sparsify it. This
detects a pack file as sparse if it contains an all zero block or
consists of only one block. As the file sparsification is just an
approximation, hide it behind a `--sparse` parameter.
2022-09-24 21:20:00 +02:00
greatroar
3047bf611c Changelog entry for sparse file restoring 2022-09-24 21:18:48 +02:00
greatroar
5d4568d393 Write sparse files in restorer
This writes files by using (*os.File).Truncate, which resolves to the
truncate system call on Unix.

Compared to the naive loop,

	for _, b := range p {
		if b != 0 {
			return false
		}
	}

the optimized allZero is about 10× faster:

name       old time/op    new time/op     delta
AllZero-8    1.09ms ± 1%     0.09ms ± 1%    -92.10%  (p=0.000 n=10+10)

name       old speed      new speed       delta
AllZero-8  3.84GB/s ± 1%  48.59GB/s ± 1%  +1166.51%  (p=0.000 n=10+10)
2022-09-24 21:18:48 +02:00
Michael Eischer
eb83402d39 Merge pull request #3935 from miles170/master
Only display the message if there were locks to be removed
2022-09-24 20:53:13 +02:00
Michael Eischer
ef58ddd7b1 Merge pull request #3923 from MichaelEischer/fix-flaky-cache-test
cache: fix flaky TestFileSaveConcurrent on windows
2022-09-24 20:52:55 +02:00
Michael Eischer
7fc178aaf4 internal/cache: extend description of cache sharing test failure 2022-09-24 13:07:01 +02:00
Miles Liu
1acbda18f8 Only display the message if there were locks to be removed
`restic unlock` now only shows `successfully removed locks` if there were locks to be removed.
In addition, it also reports the number of the removed lock files.
2022-09-24 19:02:24 +08:00
Michael Eischer
da1a359c8b Merge pull request #3927 from MichaelEischer/faster-index-each
Speed up MasterIndex.Each
2022-09-24 12:35:23 +02:00
Michael Eischer
041a51512a Merge pull request #3780 from jkmw/fix/2578
Remove existing path before restoring a symlink
2022-09-24 12:34:42 +02:00
Michael Eischer
1ebd57247a repository: optimize MasterIndex.Each
Sending data through a channel at very high frequency is extremely
inefficient. Thus use simple callbacks instead of channels.

> name                old time/op  new time/op  delta
> MasterIndexEach-16   6.68s ±24%   0.96s ± 2%  -85.64%  (p=0.008 n=5+5)
2022-09-24 12:21:59 +02:00
Michael Eischer
825b95e313 repository: add benchmark for MasterIndex.Each 2022-09-24 12:21:59 +02:00
greatroar
1220fe9650 internal/cache: Concurrent use of cache not working on Windows 2022-09-17 19:49:44 +02:00
Jerome Küttner
ef618bdd3f use os.Remove if path already exists on symlink restore 2022-09-14 08:14:31 +02:00
Michael Eischer
b48766d7b8 Merge pull request #3928 from restic/rawtaz-doc-b2-s3
doc: Clarify S3 recommendation for B2 slightly
2022-09-13 20:59:50 +02:00
rawtaz
20f1913ef7 doc: Clarify S3 recommendation for B2 slightly
This gives slightly more background to the recommendation for using restic's S3 backend with Backblaze B2.
2022-09-12 17:48:59 +02:00
rawtaz
d79e61ce5d Merge pull request #3925 from hgraeber/add-powershell-completion
Add powershell completion
2022-09-11 01:04:57 +02:00
Herbert Graeber
988b386e8b Add powershell completion
- Add code for powersehll complition available in cobra
- Add documentation for powershell completion
- Add changelog for pr3925
2022-09-11 00:44:12 +02:00
rawtaz
14d09a6081 Merge pull request #3912 from MichaelEischer/cleanup-snapshot-filter-options
Clean up snapshot filter options
2022-09-11 00:18:42 +02:00
Michael Eischer
381da0443a tweak snapshot filter descriptions 2022-09-10 23:50:20 +02:00
Michael Eischer
8b9778d537 Merge pull request #3900 from MichaelEischer/b2-init-timeout
Add timeout for the initial connection to B2
2022-09-10 23:28:59 +02:00
Michael Eischer
17c27400f8 Merge pull request #3921 from MichaelEischer/filter-cleanup-error-handling
filter: deduplicate error handling for pattern validation
2022-09-10 23:24:50 +02:00
Michael Eischer
f76643bd2e Merge pull request #3894 from MichaelEischer/filter-mount-exit-code
Mount should return exit code 0 after pressing Ctrl-C
2022-09-10 23:22:01 +02:00
Michael Eischer
be9ccc186e Merge pull request #3875 from MichaelEischer/fix-fuse-context-cancel
mount: Fix input/output errors for canceled syscalls
2022-09-10 23:20:29 +02:00
Michael Eischer
2363e5c083 Merge pull request #3913 from MichaelEischer/better-migrate-error-message
migrate: Report why an migration cannot be applied
2022-09-09 23:37:25 +02:00
Michael Eischer
8e0ca80547 filter: deduplicate error handling for pattern validation 2022-09-09 23:12:41 +02:00
plumbeo
d66e755ac7 Change uncompressed size calculation to account for the encryption overhead 2022-09-08 10:15:19 +02:00
plumbeo
837b816358 restic stats: print uncompressed size in mode raw-data 2022-09-05 17:38:32 +02:00
Michael Eischer
d6309961c5 deduplicate the snapshot filter cli option setup 2022-09-04 10:27:33 +02:00
Michael Eischer
8b4dd70013 migrate: Report why an migration cannot be applied
Just returning that `Migration upgrade cannot be applied: check failed`
is not too useful when running `migrate upgrade_repo_v2`.
2022-09-03 11:49:31 +02:00
Michael Eischer
7689d6c679 normalize help text for host, tag and path options 2022-09-03 00:06:38 +02:00
Michael Eischer
6c69f08a7b Merge pull request #3905 from DRON-666/haspaths-linear
Reduce quadratic time complexity of `Snapshot.HasPaths`
2022-08-30 20:35:56 +02:00
Michael Eischer
3e70bac56e Merge pull request #3898 from MichaelEischer/fix-copy-hang
don't hang when `copy` uses a single connection
2022-08-30 20:23:39 +02:00
DRON-666
2a630c51c1 Add changelog 2022-08-30 20:22:07 +02:00
DRON-666
d0f1060df7 Fix quadratic time complexity of Snapshot.HasPaths 2022-08-30 04:38:17 +03:00
Michael Eischer
f481ad64c8 Merge pull request #3904 from lbausch/add-newline
Add newline to keep prompt intact
2022-08-29 21:43:18 +02:00
Lorenz Bausch
7ddd803e46 Add newline to keep prompt intact 2022-08-29 17:37:49 +02:00
Michael Eischer
e5b2c4d571 b2: sniff the error that caused init retry loops 2022-08-28 17:46:03 +02:00
Michael Eischer
dc2db2de5e b2: cancel connection setup after a minute
If the connection to B2 fails, the library enters an endless loop.
2022-08-28 14:56:17 +02:00
Michael Eischer
7682149c9d repository: cleanup copy connection count check 2022-08-28 11:40:56 +02:00
Michael Eischer
b03277ead5 repository: don't hang when copying using a single connection 2022-08-28 11:40:31 +02:00
Michael Eischer
1b233c4e2e Merge pull request #2661 from creativeprojects/issue-1734
"Everything is ok" message after retrying
2022-08-28 11:04:59 +02:00
Fred
4042db5169 Add changelog 2022-08-27 22:36:19 +02:00
Fred
be6baaec12 Add success callback to the backend 2022-08-27 22:27:15 +02:00
Fred
baf58fbaa8 Add unit tests 2022-08-27 22:21:06 +02:00
Fred
d629333efe Add function to notify of success after retrying 2022-08-27 22:21:06 +02:00
Alexander Neumann
c169e37139 Merge pull request #3895 from MichaelEischer/refactor-cat-key
cat: Simplify implementation of 'cat key'
2022-08-27 18:40:46 +02:00
Michael Eischer
1b4af0c6e5 cat: Simplify implementation of 'cat key' 2022-08-26 23:21:51 +02:00
Michael Eischer
3174641ca4 add changelog for mount exit code filtering 2022-08-26 23:17:04 +02:00
Michael Eischer
5478ab22c5 mount: return exit code 0 after receiving a SIGINT 2022-08-26 23:07:07 +02:00
Michael Eischer
d768c1c3e4 Allow cleanup handlers to filter the exit code 2022-08-26 23:04:59 +02:00
Michael Eischer
908f7441fe Merge pull request #3885 from MichaelEischer/delete-fixes
Improve reliability of upload retries and B2 file deletions
2022-08-26 22:30:50 +02:00
Michael Eischer
4c90d91d4d backend: Test that failed uploads are not removed for backends with atomic replace 2022-08-26 21:20:52 +02:00
Michael Eischer
694dfa026a add changelog for reliable B2 deletes 2022-08-26 21:20:46 +02:00
MichaelEischer
582167d671 Merge pull request #3882 from MichaelEischer/sftp-init-single-connection
sftp: Only connect once to server during `init`
2022-08-26 21:13:28 +02:00
MichaelEischer
3822ded0b3 Merge pull request #3877 from MichaelEischer/no-env-in-help
Do not include the actual values of environment variables in help output
2022-08-26 20:59:54 +02:00
Michael Eischer
cf0a8d7758 sftp: Only connect once for repository creation
This is especially useful if ssh asks for a password or if closing the
initial connection could return an error due to a problematic server
implementation.
2022-08-26 20:50:40 +02:00
Michael Eischer
dd7cd5b9b3 fuse: remove unused context parameter 2022-08-26 20:48:48 +02:00
Michael Eischer
a0c1ae9f90 mount: Correctly return context.Canceled for interrupted syscalls
bazil/fuse expects us to return context.Canceled to signal that a
syscall was successfully interrupted. Returning a wrapped version of
that error however causes the fuse library to signal an EIO (input/output
error). Thus unwrap context.Canceled errors before returning them.
2022-08-26 20:48:48 +02:00
Michael Eischer
5d0649faaf Update help output in docs 2022-08-26 20:44:01 +02:00
Michael Eischer
faa4597af1 Set name for option values of cli 2022-08-26 20:42:34 +02:00
Michael Eischer
6ed157aee6 Do not include the actual values of environment variables in help output
This results in printing a `(default: $ENV) (default: value)` suffix for
the corresponding options which looks strange. In addition, some of the
environment variables might contain secrets which should not be
displayed.
2022-08-26 20:39:54 +02:00
MichaelEischer
f7808245aa Merge pull request #3878 from MichaelEischer/cheaper-cache-load
cache: Just try to open cache entry without calling stat first
2022-08-26 20:33:36 +02:00
MichaelEischer
bee15dd555 Merge pull request #3879 from MichaelEischer/mem-optimize
Some random (minor) memory-allocation optimizations
2022-08-26 20:33:02 +02:00
MichaelEischer
0e1d082b12 Merge pull request #3886 from MichaelEischer/recommend-s3-over-b2
doc: recommend usage of B2's S3 API
2022-08-26 20:29:05 +02:00
Alexander Neumann
d464543171 Update repo version table 2022-08-25 21:30:25 +02:00
Alexander Neumann
6b40456db7 Set development version for 0.14.0 2022-08-25 19:55:05 +02:00
Alexander Neumann
1bc87e1718 Add version for 0.14.0 2022-08-25 19:55:00 +02:00
Alexander Neumann
c1a5da56e3 Update manpages and auto-completion 2022-08-25 19:55:00 +02:00
Alexander Neumann
193c62dfc3 Generate CHANGELOG.md for 0.14.0 2022-08-25 19:54:02 +02:00
Alexander Neumann
a825e0d409 Prepare changelog for 0.14.0 2022-08-25 19:54:01 +02:00
rawtaz
b824d8cdcc Merge pull request #3891 from MichaelEischer/fix-secondary-repo-typo
Fix typo in the environment variable name for `--from-password-file`
2022-08-24 22:38:47 +02:00
Michael Eischer
a2e89234fc Fix typo in the environment variable name for --from-password-file 2022-08-24 22:25:18 +02:00
MichaelEischer
b4ae05627f Merge pull request #3876 from MichaelEischer/document-aws-session-token
doc: document aws session token
2022-08-24 22:20:26 +02:00
MichaelEischer
bd7bca2b51 Merge pull request #3889 from restic/prepare-0-14
Polish changelog entries
2022-08-24 21:28:44 +02:00
MichaelEischer
be90a565cc Merge pull request #3887 from MichaelEischer/rclone-permanent-error
rclone: Return a permanent error if rclone already exited
2022-08-24 21:19:00 +02:00
Michael Eischer
7a5d29ce24 Further changelog polishing 2022-08-24 21:13:14 +02:00
MichaelEischer
d198a77d86 Merge pull request #3888 from restic/doc-prepare-repo
doc: Improve/clarify preparing and versions of repositories
2022-08-24 20:40:49 +02:00
Leo R. Lundgren
cb5a61c46e doc: Improve/clarify preparing and versions of repositories 2022-08-24 02:08:46 +02:00
Leo R. Lundgren
ee6b9dc492 Polish changelog entries 2022-08-24 01:23:51 +02:00
Michael Eischer
506d92e87c rclone: Return a permanent error if rclone already exited
rclone can exit early for example when the connection to rclone is
relayed for example via ssh: `-o rclone.program='ssh user@example.org
forced-command'`
2022-08-23 22:05:04 +02:00
MichaelEischer
ad6eabbfa5 Merge pull request #3883 from MichaelEischer/update-dependencies
Update dependencies
2022-08-23 21:44:38 +02:00
Michael Eischer
7681a63fdb restic: Cleanup xattr error handling for Solaris
Since xattr 0.4.8 (https://github.com/pkg/xattr/pull/68) returns ENOTSUP
similar to Linux.
2022-08-23 21:25:15 +02:00
Michael Eischer
99e4ccbd94 remain compatible with go 1.15 2022-08-23 21:25:14 +02:00
Michael Eischer
22f46c18f9 downgrade bazil/fuse again to retain macOS support 2022-08-23 21:24:47 +02:00
Michael Eischer
6db979b3a6 update dependencies 2022-08-23 21:24:46 +02:00
MichaelEischer
9cdc8da10f Merge pull request #3881 from MichaelEischer/cleanup-release-helper
Cleanup release helper
2022-08-23 21:10:57 +02:00
MichaelEischer
b51e73e78f Merge pull request #3884 from MichaelEischer/fix-index-saving-progress
repository: Do not report ignored packs in EachByPack
2022-08-23 21:10:07 +02:00
MichaelEischer
98dcd0a887 Merge pull request #3874 from MichaelEischer/disk-wear-note
Add note that larger packs increase disk wear
2022-08-21 22:18:08 +02:00
Michael Eischer
e0d6bf525c doc: fix typo 2022-08-21 19:12:19 +02:00
Michael Eischer
5d0523e2f1 Add note that larger packs increase disk wear 2022-08-21 19:12:19 +02:00
Michael Eischer
c586a5e20f doc: recommend usage of B2's S3 API 2022-08-21 11:39:03 +02:00
Michael Eischer
623556bab6 b2: Increase list size to maximum
Just request as many files as possible in one call to reduce the number
of network roundtrips.
2022-08-21 11:20:03 +02:00
Michael Eischer
de0162ea76 backend/retry: Overwrite failed uploads instead of deleting them
For backends which are able to atomically replace files, we just can
overwrite the old copy, if it is necessary to retry an upload. This has
the benefit of issuing one operation less and might be beneficial if a
backend storage, due to bugs or similar, could mix up the order of the
upload and delete calls.
2022-08-21 11:14:53 +02:00
Michael Eischer
fc506f8538 b2: Repeat deleting until all file versions are removed
When hard deleting the latest file version on B2, this uncovers earlier
versions. If an upload required retries, multiple version might exist
for a file. Thus to reliably delete a file, we have to remove all
versions of it.
2022-08-21 11:11:00 +02:00
Michael Eischer
cc4728d287 repository: Do not report ignored packs in EachByPack
Ignored packs were reported as an empty pack by EachByPack. The most
immediate effect of this is that the progress bar for rebuilding the
index reports processing more packs than actually exist.
2022-08-21 10:38:40 +02:00
Michael Eischer
c3374b3ea5 helper: download modules as first step
There's no use in running that step in parallel.
2022-08-20 12:11:54 +02:00
Michael Eischer
7f0929e519 helper: Reduce number of parallel builds a bit
The go compiler is already parallelized. The high concurrency caused my
podman container to hit a resource limit.
2022-08-20 12:10:48 +02:00
Michael Eischer
ed94678820 helper: cleanups 2022-08-20 12:10:29 +02:00
Michael Eischer
e530d422a0 helper: don't setup cmd paths twice 2022-08-20 12:09:42 +02:00
Michael Eischer
7a992fc794 repository: Reduce buffer reallocations in ForAllIndexes
Previously the buffer was grown incrementally inside `repo.LoadUnpacked`.
But we can do better as we already know how large the index will be.
Allocate a bit more memory to increase the chance that the buffer can be
reused in the future.
2022-08-19 21:13:40 +02:00
Michael Eischer
77b1980d8e repository: MasterIndex.Packs: reduce allocations 2022-08-19 21:10:43 +02:00
Michael Eischer
6ff9517e45 repository: MasterIndex.ListPacks / Index.EachByPack allow earlier GC
Allow earlier garbage collection of some of the intermediate data
structures.
2022-08-19 21:06:33 +02:00
Michael Eischer
ce902aac67 cache: Just try to open cache entry without calling stat first
Instead of first checking whether a file is in the repository cache and
then opening it, we just can open the file. This saves one stat call. If
the file is in the cache, everything is fine and otherwise the code
follows its normal fallback path.
2022-08-19 20:59:06 +02:00
Michael Eischer
b6c86ababe doc: document aws session token 2022-08-19 20:41:15 +02:00
MichaelEischer
0d9ac78437 Merge pull request #3873 from MichaelEischer/gofmt-comments
gofmt comments
2022-08-19 19:54:30 +02:00
MichaelEischer
7e96a5af62 Merge pull request #3872 from MichaelEischer/fuse-fix
mount: Only remember successful snapshot refreshes
2022-08-19 19:21:29 +02:00
Michael Eischer
f414db987d gofmt all files
Apparently the rules for comment formatting have changed with go 1.19.
2022-08-19 19:12:26 +02:00
Michael Eischer
522406b4f0 mount: Only remember successful snapshot refreshes
If the context provided by the fuse library is canceled before the index
was loaded this could lead to missing snapshots.
2022-08-19 19:07:07 +02:00
MichaelEischer
dbca93da28 Merge pull request #3742 from MichaelEischer/from-repo
copy: replace --repo2 with --from-repo
2022-08-19 19:01:04 +02:00
MichaelEischer
b4dfab002a Merge pull request #3691 from greatroar/tag-filenames
Sanitize tags when used as filenames by restic mount
2022-08-19 18:33:44 +02:00
Michael Eischer
2758d76b77 copy: replace --repo2 with --from-repo
`init` and `copy` use `--repo2` with two different meaning which has
proven to be confusing for users. `--from-repo` now consistently marks a
source repository from which data is read. `--repo` is now always the
target/destination repository.
2022-08-19 18:33:26 +02:00
Michael Eischer
af50fe9ac0 mount: Map slashes in tags to underscores
Suggested-by: greatroar <>
2022-08-19 18:17:57 +02:00
MichaelEischer
4cccffab58 Merge pull request #3862 from restic/3861-forget-invalid-policy
forget: Error when invalid unit is given in duration policy
2022-08-18 20:45:26 +02:00
Michael Eischer
2ea6c82cf6 comment cleanup
gofmt reformatted the comment
2022-08-18 20:15:38 +02:00
Michael Eischer
bb27f7408c forget: Fail test if duration parsing error is missing 2022-08-18 20:14:09 +02:00
MichaelEischer
c4b3a154ba Merge pull request #3865 from restic/rawtaz-doc-comp-typo
doc: Fix typo in compression section
2022-08-18 19:37:44 +02:00
rawtaz
2cb2aa31cd doc: Fix typo in compression section 2022-08-12 22:17:38 +02:00
Leo R. Lundgren
6f517858e8 forget: Error when invalid unit is given in duration policy 2022-08-10 13:37:26 +02:00
rawtaz
f0bb4f8708 Merge pull request #3857 from restic/rawtaz-gcs-account
doc: Update links to GCS documentation
2022-08-08 22:52:30 +02:00
Michael Eischer
40c8755b13 doc: Update more links to GCS documentation 2022-08-08 20:00:44 +02:00
rawtaz
f673068dbb doc: Update link to GCS documentation
Updates the link to Google Cloud Storage documentation about creating a service account key.
2022-08-08 12:51:43 +02:00
rawtaz
f26231c9e6 Merge pull request #3852 from MichaelEischer/snapshots-processed
stats: Add snapshots count to json output
2022-08-08 00:05:15 +02:00
MichaelEischer
80e3efffef Merge pull request #3855 from mattxtaz/master
Fix typo with double percentage in help text
2022-08-07 22:44:36 +02:00
mattxtaz
01ab36336f Fix typo with double percentage in help text 2022-08-07 20:21:05 +01:00
Michael Eischer
6e92d852a8 stats: Add snapshots count to json output 2022-08-07 15:44:09 +02:00
MichaelEischer
9ad3ad5972 Merge pull request #3850 from lbausch/go1.19
Update tests to Go 1.19
2022-08-07 14:56:17 +02:00
MichaelEischer
2930a102de Merge pull request #3731 from metalsp0rk/feature/min-packsize-flag
Feature: min packsize flag
2022-08-07 14:54:45 +02:00
MichaelEischer
8fa64a8f99 Merge pull request #3036 from greatroar/refactor-fuse
Clean up internal/fuse
2022-08-07 14:46:30 +02:00
Michael Eischer
f3fdc66b32 restic: Use stable sorting in snapshot policy
sort.Sort is not guaranteed to be stable. Go 1.19 has changed the
sorting algorithm which resulted in changes of the sort order. When
comparing snapshots with identical timestamp but different paths and
tags lists, there is not meaningful order among them. So just keep their
order stable.
2022-08-07 14:10:40 +02:00
Lorenz Bausch
b82f4824f0 Bump golangci-lint version 2022-08-07 14:10:39 +02:00
Lorenz Bausch
0b9b4c52ad Update tests to Go 1.19 2022-08-07 14:10:39 +02:00
Michael Eischer
0b7291b8b2 mount: Fix parent inode used by snapshots dir 2022-08-07 13:03:32 +02:00
greatroar
cfa80e2c6b mount: remove unused inode field from root node 2022-08-07 13:03:26 +02:00
MichaelEischer
74ae76036f Merge pull request #2913 from aawsome/mount-snapshot-slashes
mount: Make snapshots dir structure customizable
2022-08-07 12:27:59 +02:00
MichaelEischer
09497aec02 Merge pull request #3826 from MichaelEischer/debug-log-for-release
Debug log for release build
2022-08-07 12:15:27 +02:00
Michael Eischer
83b4c50ee3 Mention --snapshot-template and --time-template in changelog 2022-08-07 12:13:06 +02:00
Michael Eischer
caa17988a3 fuse: Redesign snapshot dirstruct
Cleanly separate the directory presentation and the snapshot directory
structure. SnapshotsDir now translates the dirStruct into a format
usable by the fuse library and contains only minimal special case rules.
All decisions have moved into SnapshotsDirStructure which now creates a
fully preassembled tree data structure.
2022-08-07 12:13:06 +02:00
Michael Eischer
a3e48da3a3 Add changelog for DEBUG_LOG available in release builds 2022-08-05 23:49:39 +02:00
Michael Eischer
b3cdee66a9 update documentation to reflect DEBUG_LOG for release builds 2022-08-05 23:49:39 +02:00
Michael Eischer
1ed775e3a8 debug: support roundtripper logging also for release builds
Different from debug builds do not use the eofDetectRoundTripper if
logging is disabled.
2022-08-05 23:49:39 +02:00
Michael Eischer
38becfc436 debug: enable debug support for release builds 2022-08-05 23:49:39 +02:00
Michael Eischer
82c268c917 Remove unused hooks mechanism 2022-08-05 23:49:39 +02:00
Michael Eischer
7266f07c87 repository: StreamPack in parts if there are too large gaps
For large pack sizes we might be only interested in the first and last
blob of a pack file. Thus stream a pack file in multiple parts if the
gaps between requested blobs grow too large.
2022-08-05 23:48:36 +02:00
Michael Eischer
55a11c1396 Reword prune --repack-small description 2022-08-05 23:48:36 +02:00
Michael Eischer
eaf43607f9 Add note that pack-size is not an exact limit 2022-08-05 23:48:36 +02:00
Michael Eischer
7f3b2be1e8 s3: Disable multipart uploads below 200MB 2022-08-05 23:48:36 +02:00
Michael Eischer
176b387d98 Always repack very small pack files 2022-08-05 23:48:36 +02:00
Michael Eischer
324935cb80 Only repack small files if there are multiple of them 2022-08-05 23:48:34 +02:00
Michael Eischer
1b076cda97 rename option to --pack-size 2022-08-05 23:47:43 +02:00
Michael Eischer
d7e2892048 Add changelog for packsize option 2022-08-05 23:47:43 +02:00
Michael Eischer
8a44258b6f update restic help snippets in documentation 2022-08-05 23:47:43 +02:00
Michael Eischer
420ddc03c9 rework pack size parameter documentation 2022-08-05 23:47:43 +02:00
Kyle Brennan
e43be84eb8 document minPackSize 2022-08-05 23:47:41 +02:00
Kyle Brennan
1e3f05c3f1 repository: prevent header overfill 2022-08-05 23:47:12 +02:00
Michael Eischer
6a6d313c9a prune: reduce priority of repacking small packs 2022-08-05 23:47:12 +02:00
Kyle Brennan
0269381b8d prune: add repack-small parameter 2022-08-05 23:47:12 +02:00
Michael Eischer
0a6fa602c8 add option for setting min pack size 2022-08-05 23:47:12 +02:00
Michael Eischer
2db7733ee3 fuse: remove unused MetaDir 2022-08-05 23:46:46 +02:00
Michael Eischer
f678f7cb04 fuse: cleanup test 2022-08-05 23:46:46 +02:00
Alexander Weiss
1751afae26 Make snapshots dirs in mount command customizable 2022-08-05 23:46:46 +02:00
Alexander Weiss
57f4003f2f Generalize fuse snapshot dirs implemetation
+ allow "/" in tags and snapshot template
2022-08-05 23:46:46 +02:00
Alexander Weiss
696c18e031 Add possibility to set snapshot ID (used in test) 2022-08-05 23:46:46 +02:00
MichaelEischer
04a8ee80fb Merge pull request #3829 from MichaelEischer/prune-refactor
Split prune into slightly small functions
2022-08-05 23:29:52 +02:00
MichaelEischer
15679be858 Merge pull request #3841 from MichaelEischer/compression-env
Add environment variable RESTIC_COMPRESSION
2022-08-05 23:20:04 +02:00
MichaelEischer
7d14b1baf1 Merge pull request #3845 from greatroar/solaris-xattr
internal/restic: Handle EINVAL for xattr on Solaris
2022-08-04 22:04:52 +02:00
greatroar
ad6ac680af internal/restic: Handle EINVAL for xattr on Solaris
Also make the errors a bit less verbose by not prepending the operation,
since pkg/xattr already does that. Old errors looked like

    Listxattr: xattr.list /myfiles/.zfs/snapshot: invalid argument
2022-08-01 12:45:17 +02:00
MichaelEischer
846d021db5 Merge pull request #3840 from greatroar/sftp-init
Speed up restic init over slow SFTP links
2022-07-31 19:54:59 +02:00
greatroar
b9fa6e05bd Add changelog for #3837/#3840 2022-07-30 22:53:52 +02:00
Michael Eischer
73053674d9 repository: Test fallback to existing blobs 2022-07-30 17:37:07 +02:00
Michael Eischer
e85a21eda2 prune: move code 2022-07-30 17:37:07 +02:00
Michael Eischer
623770eebb repository: try to recover from invalid blob while repacking
If a blob that should be kept is invalid, Repack will now try to request
the blob using LoadBlob. Only return an error if that fails.
2022-07-30 17:37:07 +02:00
Michael Eischer
d0590b7841 prune: Add internal integrity check
After repacking every blob that should be kept must have been repacked.
We have seen a few cases in which a single blob went missing, which
could have been caused by a bitflip somewhere. This sanity check might
help catch some of these cases.
2022-07-30 17:37:07 +02:00
Michael Eischer
5cbde03eae prune: split into smaller functions 2022-07-30 17:37:07 +02:00
Alexander Weiss
7643237da5 prune: separate collecting/printing/pruning 2022-07-30 17:37:07 +02:00
Michael Eischer
5723c72eb1 Add environment variable RESTIC_COMPRESSION 2022-07-30 16:21:53 +02:00
greatroar
2bdc40e612 Speed up restic init over slow SFTP links
pkg/sftp.Client.MkdirAll(d) does a Stat to determine if d exists and is
a directory, then a recursive call to create the parent, so the calls
for data/?? each take three round trips. Doing a Mkdir first should
eliminate two round trips for 255/256 data directories as well as all
but one of the top-level directories.

Also, we can do all of the calls concurrently. This may reintroduce some
of the Stat calls when multiple goroutines try to create the same
parent, but at the default number of connections, that should not be
much of a problem.
2022-07-30 13:09:08 +02:00
greatroar
23ebec717c Remove stale comments from backend/sftp
The preExec and postExec functions were removed in
0bdb131521 from 2018.
2022-07-30 13:07:25 +02:00
MichaelEischer
4ffd479ba4 Merge pull request #3773 from MichaelEischer/efficient-dir-json
Reduce memory usage for large directories/files
2022-07-23 17:47:32 +02:00
Michael Eischer
2ba14160de Add changelog for the optimized tree serialization 2022-07-23 14:49:08 +02:00
Michael Eischer
4a10ebed15 archiver: reduce memory usage for large files
FutureBlob now uses a Take() method as a more memory-efficient way to
retrieve the futures result. In addition, futures are now collected
while saving the file. As only a limited number of blobs can be queued
for uploading, for a large file nearly all FutureBlobs already have
their result ready, such that the FutureBlob object just consumes
memory.
2022-07-23 14:45:07 +02:00
Michael Eischer
b817681a11 archiver: Incrementally serialize tree nodes
That way it is not necessary to keep both the Nodes forming a Tree and
the serialized JSON version in memory.
2022-07-23 14:45:07 +02:00
Michael Eischer
c206a101a3 archiver: unify FutureTree/File into futureNode
There is no real difference between the FutureTree and FutureFile
structs. However, differentiating both increases the size of the
FutureNode struct.

The FutureNode struct is now only 16 bytes large on 64bit platforms.
That way is has a very low overhead if the corresponding file/directory
was not processed yet.

There is a special case for nodes that were reused from the parent
snapshot, as a go channel seems to have 96 bytes overhead which would
result in a memory usage regression.
2022-07-23 14:45:07 +02:00
Michael Eischer
32f4997733 archiver: remove unused fileInfo from progress callback 2022-07-23 14:16:23 +02:00
Michael Eischer
dcb00fd2d1 archiver: cleanup Saver interface 2022-07-23 14:16:23 +02:00
Michael Eischer
79321a195c archiver: remove dead attribute from FutureNode 2022-07-23 14:16:23 +02:00
MichaelEischer
049f4c4144 Merge pull request #3730 from MichaelEischer/stricter-check
Let `check` warn about legacy variants of the repo format version 1
2022-07-23 14:14:50 +02:00
Michael Eischer
3bf53da672 Add changelog for stricter checks 2022-07-23 11:21:26 +02:00
Michael Eischer
5a6f2f9fa0 Fix S3 legacy layout migration 2022-07-23 11:19:32 +02:00
Michael Eischer
04e49924fb checker: Fix S3 legacy layout detection 2022-07-23 11:19:32 +02:00
Michael Eischer
768c890fcb check: Deprecate --check-unused
Unused blobs are not a problem but rather expected to exist now that
prune by default does not remove every unused blob. However, the option
has caused questions from users whether a repository is damaged or not,
so just remove that option.

Note that the remaining code is left intact as it is still useful for
our test cases.
2022-07-23 11:19:32 +02:00
Michael Eischer
fcb3ddf181 check: Complain about usage of s3 legacy layout 2022-07-23 11:19:32 +02:00
Michael Eischer
8b8bd4e8ac check: complain about mixed pack files 2022-07-23 11:19:32 +02:00
MichaelEischer
443cc49afd Merge pull request #3830 from MichaelEischer/cleanup-repo
Extract Load/SaveTree/JSONUnpacked from repository
2022-07-23 10:46:13 +02:00
MichaelEischer
1f5369e072 Merge pull request #3831 from MichaelEischer/move-code
Move code out of the restic package and consolidate backend specific code
2022-07-23 10:33:05 +02:00
MichaelEischer
827ab02eea Merge pull request #3661 from rgammans/azure_sas_support
add SAS authentication option for Azure repos
2022-07-23 10:32:03 +02:00
MichaelEischer
e9c39442fb Merge pull request #3827 from MichaelEischer/backup-doc
backup: clarify usage string
2022-07-23 10:31:33 +02:00
Michael Eischer
9729e6d7ef backend: extract readerat from restic package 2022-07-17 15:29:09 +02:00
Michael Eischer
c44b21d366 restorer: extract hardlinks index from restic package 2022-07-17 13:45:42 +02:00
Michael Eischer
8c11fc3ec9 crypto: move crypto buffer helpers 2022-07-17 13:42:23 +02:00
Michael Eischer
a0cef9f247 limiter: move to internal/backend 2022-07-17 13:40:15 +02:00
Michael Eischer
163ab9c025 mock: move to internal/backend 2022-07-17 13:40:06 +02:00
Michael Eischer
89d3ce852b repository: extract Load/StoreJSONUnpacked
A Load/Store method for each data type is much clearer. As a result the
repository no longer needs a method to load / store json.
2022-07-17 13:22:00 +02:00
Michael Eischer
fbcbd5318c repository: extract LoadTree/SaveTree
The repository has no real idea what a Tree is. So these methods never
belonged there.
2022-07-17 13:11:28 +02:00
MichaelEischer
d9ea1e9ee2 Merge pull request #3290 from aawsome/prune-handle-duplicates
prune: Handle duplicate blobs more efficiently
2022-07-17 11:51:54 +02:00
Michael Eischer
715d457aad prune: code cleanups 2022-07-17 11:41:56 +02:00
Michael Eischer
9be1bd2acc prune: handle very high duplication of some blobs
Suggested-By: Alexander Weiss <alex@weissfam.de>
2022-07-17 11:39:56 +02:00
Alexander Weiss
7478cbf70e prune: Enhance treatment of duplicates 2022-07-17 00:22:23 +02:00
Michael Eischer
b2043e8198 backup: clarify usage string
Using the `--files-from` options it is possible to run `backup` without
specifying any source paths directly on the command line.
2022-07-17 00:03:22 +02:00
Michael Eischer
5639c41b6a azure: Strip ? prefix from sas token 2022-07-16 23:55:18 +02:00
Roger Gammans
64a7ec5341 azure: add SAS authentication option 2022-07-16 23:55:18 +02:00
MichaelEischer
6cbeb4a9f9 Merge pull request #3825 from restic/rawtaz-doc-gdrive
doc: Add note about using rclone for Google Drive
2022-07-16 19:21:59 +02:00
rawtaz
f5c219f5a2 doc: Add note about using rclone for Google Drive
It wasn't clear that Google Cloud Storage and Google Drive are two different services and that one should use the rclone backend for the latter. This commit adds a note with this information.
2022-07-16 13:22:38 +02:00
MichaelEischer
d71b29221b Merge pull request #3822 from JsBergbau/doc-max-compression
Added hint for --compression max in migration process
2022-07-12 23:02:45 +02:00
Michael Eischer
71ff6b77f0 doc: Rework hint to repack with max compression 2022-07-12 21:24:40 +02:00
MichaelEischer
6970d05d47 Merge pull request #3741 from lbausch/repository-wording
Wording: change repo to repository
2022-07-12 21:15:33 +02:00
MichaelEischer
3934480da4 Merge pull request #3819 from lbausch/restore-validate-patterns
restore: validate include/exclude patterns
2022-07-12 20:58:41 +02:00
MichaelEischer
71a0157c2c Merge pull request #3820 from lbausch/patch-1
Fix wording in changelog template
2022-07-12 20:51:52 +02:00
Michael Eischer
2aad6f24b5 doc: update sample help output 2022-07-12 20:49:01 +02:00
Michael Eischer
ec4dfa3c66 Wording: replace further repo occurrences with repository 2022-07-12 20:48:01 +02:00
JsBergbau
8d3f04aefa Added hint for --compression max in migration process
Added hint for --compression max in migration process. Since this is a onetime process users should be aware of this and consider this step.
2022-07-12 00:08:26 +02:00
Lorenz Bausch
b609523582 Add changelog entry 2022-07-09 22:24:56 +02:00
lbausch
ac96a4138d Fix wording in changelog template 2022-07-09 22:13:54 +02:00
Lorenz Bausch
7e36ec279d Test restore fails when using invalid patterns 2022-07-08 20:09:26 +02:00
Lorenz Bausch
be524f0b78 Add testRunRestoreAssumeFailure function 2022-07-08 20:09:25 +02:00
Lorenz Bausch
9a7db6675c Restore: validate provided patterns 2022-07-08 20:09:25 +02:00
Lorenz Bausch
d6e3c7f28e Wording: change repo to repository 2022-07-08 20:05:35 +02:00
MichaelEischer
98a3125ce4 Merge pull request #3816 from mattxtaz/master
Fix minor typo in docs
2022-07-05 21:30:26 +02:00
mattxtaz
87d899c099 Fix minor typo in docs 2022-07-05 20:06:27 +01:00
Jerome Küttner
6f3883c9d2 add changelog 2022-07-05 08:47:48 +02:00
Alexander Neumann
545220803b Merge pull request #3813 from MichaelEischer/fix-blob-saver-data-race
Fix data race in blob_saver
2022-07-04 08:10:01 +02:00
Michael Eischer
ce89018902 Fix data race in blob_saver
After the `BlobSaver` job is submitted, the buffer can be released and
reused by another `FileSaver` even before `BlobSaver.Save` returns. That
FileSaver will the change `buf.Data` leading to wrong backup statistics.

Found by `go test -race ./...`:

WARNING: DATA RACE
Write at 0x00c0000784a0 by goroutine 41:
  github.com/restic/restic/internal/archiver.(*FileSaver).saveFile()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:176 +0x789
  github.com/restic/restic/internal/archiver.(*FileSaver).worker()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af
  github.com/restic/restic/internal/archiver.NewFileSaver.func2()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d
  golang.org/x/sync/errgroup.(*Group).Go.func1()
      /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91

Previous read at 0x00c0000784a0 by goroutine 29:
  github.com/restic/restic/internal/archiver.(*BlobSaver).Save()
      /home/michael/Projekte/restic/restic/internal/archiver/blob_saver.go:57 +0x1dd
  github.com/restic/restic/internal/archiver.(*BlobSaver).Save-fm()
      <autogenerated>:1 +0xac
  github.com/restic/restic/internal/archiver.(*FileSaver).saveFile()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:191 +0x855
  github.com/restic/restic/internal/archiver.(*FileSaver).worker()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af
  github.com/restic/restic/internal/archiver.NewFileSaver.func2()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d
  golang.org/x/sync/errgroup.(*Group).Go.func1()
      /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91
2022-07-03 14:47:53 +02:00
MichaelEischer
b6a38d43b3 Merge pull request #3611 from MichaelEischer/auto-concurrency
Adjust worker goroutines to number of backend connections
2022-07-03 12:33:32 +02:00
Michael Eischer
3af9c2cc58 Document automatic CPU/IO-concurrency 2022-07-03 12:19:26 +02:00
Michael Eischer
6f53ecc1ae adapt workers based on whether an operation is CPU or IO-bound
Use runtime.GOMAXPROCS(0) as worker count for CPU-bound tasks,
repo.Connections() for IO-bound task and a combination if a task can be
both. Streaming packs is treated as IO-bound as adding more worker
cannot provide a speedup.

Typical IO-bound tasks are download / uploading / deleting files.
Decoding / Encoding / Verifying are usually CPU-bound. Several tasks are
a combination of both, e.g. for combined download and decode functions.
In the latter case add both limits together. As the backends have their
own concurrency limits restic still won't download more than
repo.Connections() files in parallel, but the additional workers can
decode already downloaded data in parallel.
2022-07-03 12:19:26 +02:00
MichaelEischer
cd50feb66f Merge pull request #3489 from MichaelEischer/async-pack-uploads
Asynchronously upload pack files
2022-07-03 11:56:05 +02:00
Michael Eischer
74df9d5998 Add changelog for async pack uploads 2022-07-03 11:34:01 +02:00
Michael Eischer
dbb5860dc9 Document connections and compression option 2022-07-03 11:19:24 +02:00
Michael Eischer
753e56ee29 repository: Limit to a single pending pack file
Use only a single not completed pack file to keep the number of open and
active pack files low. The main change here is to defer hashing the pack
file to the upload step. This prevents the pack assembly step to become
a bottleneck as the only task is now to write data to the temporary pack
file.

The tests are cleaned up to no longer reimplement packer manager
functions.
2022-07-02 22:42:34 +02:00
Michael Eischer
fa25d6118e archiver: Reduce tree saver concurrency
Large amount of tree savers have no obvious benefit, however they can
increase the amount of (potentially large) trees kept in memory.
2022-07-02 22:42:34 +02:00
Michael Eischer
bba1e81719 archiver: Limit blob saver count to GOMAXPROCS
Now with the asynchronous uploaders there's no more benefit from using
more blob savers than we have CPUs. Thus use just one blob saver for
each CPU we are allowed to use.
2022-07-02 22:42:34 +02:00
Michael Eischer
120ccc8754 repository: Rework blob saving to use an async pack uploader
Previously, SaveAndEncrypt would assemble blobs into packs and either
return immediately if the pack is not yet full or upload the pack file
otherwise. The upload will block the current goroutine until it
finishes.

Now, the upload is done using separate goroutines. This requires changes
to the error handling. As uploads are no longer tied to a SaveAndEncrypt
call, failed uploads are signaled using an errgroup.

To count the uploaded amount of data, the pack header overhead is no
longer returned by `packer.Finalize` but rather by
`packer.HeaderOverhead`. This helper method is necessary to continue
returning the pack header overhead directly to the responsible call to
`repository.SaveBlob`. Without the method this would not be possible,
as packs are finalized asynchronously.
2022-07-02 22:42:34 +02:00
Michael Eischer
bba4c69a2a tag: Remove unnecessary flush call 2022-07-02 22:42:23 +02:00
MichaelEischer
3e1de52e0a Merge pull request #3805 from greatroar/global
cmd/restic, limiter: Move config knowledge to internal packages
2022-07-02 21:56:35 +02:00
MichaelEischer
621023a50b Merge pull request #3772 from MichaelEischer/fix-mixed-index
rebuild-index: correctly rebuild index for mixed packs
2022-07-02 20:10:02 +02:00
MichaelEischer
90e9c5c4cc Merge pull request #3729 from MichaelEischer/full-ids-in-check
Include full IDs in check output
2022-07-02 20:09:39 +02:00
MichaelEischer
7137034517 Merge pull request #3811 from restic/fix-secret-string-crash
Don't crash if SecretString is uninitialized
2022-07-02 19:56:36 +02:00
Michael Eischer
cdaf9b4f26 Don't crash if SecretString is uninitialized 2022-07-02 19:44:28 +02:00
Michael Eischer
5e0f1c3cef check: remove dead code 2022-07-02 19:28:57 +02:00
Michael Eischer
0df022fa6d check: Print full ids
The short ids are not always unique. In addition, recovering from
damages is easier when having the full ids as that makes it easier to
access the corresponding files.
2022-07-02 19:28:57 +02:00
Michael Eischer
04c23fa95d rebuild-index: correctly rebuild index for mixed packs
For mixed packs, data and tree blobs were stored in separate index
entries. This results in warning from the check command and maybe other
problems.
2022-07-02 19:24:02 +02:00
MichaelEischer
bb5f196b09 Merge pull request #3733 from restic/improve-stats
Improve stats
2022-07-02 19:07:31 +02:00
MichaelEischer
c16f989d4a Merge pull request #3470 from MichaelEischer/sanitize-debug-log
Sanitize debug log
2022-07-02 19:00:54 +02:00
Michael Eischer
00d7fcff96 extend compression feature changelog entry 2022-07-02 18:55:59 +02:00
Michael Eischer
a6e9e08034 Account for pack header overhead at each entry
This will miss the pack header crypto overhead and the length field,
which only amount to a few bytes per pack file.
2022-07-02 18:55:58 +02:00
Michael Eischer
856d5e4303 stats: return storage size for raw-data mode
raw-data summed up the size of the blob plaintexts. However, with
compression this makes little sense as the storage size in the
repository is lower due to compression. Thus sum up the actual size each
blob takes in the repository.
2022-07-02 18:55:12 +02:00
Alexander Neumann
6c4ceaf1e7 Print number of bytes added to the repo
This includes optional compression and crypto overhead.
2022-07-02 18:55:12 +02:00
Alexander Neumann
99634c0936 Return real size from SaveBlob 2022-07-02 18:55:12 +02:00
MichaelEischer
fdc53a9d32 Merge pull request #3787 from MichaelEischer/refactor-repository
repository: (Mostly) index-related cleanups
2022-07-02 18:54:04 +02:00
Michael Eischer
6923353c43 redact swift auth token in debug output 2022-07-02 18:47:35 +02:00
Michael Eischer
5a11d14082 redacted keys/token in backend config debug log 2022-07-02 18:47:35 +02:00
Michael Eischer
0936d864a4 redact http authorization header in debug log output 2022-07-02 18:47:35 +02:00
Michael Eischer
ec7c9ce88b drop unused repository.Loader interface 2022-07-02 18:39:59 +02:00
Michael Eischer
2cd7e90ad1 repository: cleanup 2022-07-02 18:39:59 +02:00
Michael Eischer
c1a8fa4290 repository: remove unused packIDToIndex field 2022-07-02 18:39:59 +02:00
Michael Eischer
e68c3a4e62 repository: simplify CreateIndexFromPacks 2022-07-02 18:39:59 +02:00
Michael Eischer
1974ad7ce2 repository: hide MasterIndex.FinalizeFullIndexes / FinalizeNotFinalIndexes 2022-07-02 18:39:59 +02:00
Michael Eischer
ef53ca4a5a repository: remove MasterIndex.All() 2022-07-02 18:39:59 +02:00
Michael Eischer
bf81bf0795 repository: Properly set id for finalized index
As MergeFinalIndex and index uploads can occur concurrently, it is
necessary for MergeFinalIndex to check whether the IDs for an index were
already set before merging it. Otherwise, we'd loose the ID of an index
which is set _after_ uploading it.
2022-07-02 18:39:59 +02:00
Michael Eischer
e0a7852b8b repository: remove unused (Master)Index.Count 2022-07-02 18:39:58 +02:00
Michael Eischer
8ef2968f28 repository: remove unused index.ListPack 2022-07-02 18:39:12 +02:00
Michael Eischer
e4f20dea61 repository: inline index.encode 2022-07-02 18:39:12 +02:00
Michael Eischer
fe5a8e137a repository: remove unused index.Store 2022-07-02 18:39:12 +02:00
Michael Eischer
628ae799ca repository: make flushPacks private 2022-07-02 18:39:12 +02:00
Michael Eischer
ed8aa15376 repository: add Save method to MasterIndex interface 2022-07-02 18:38:56 +02:00
Michael Eischer
a77d5c4d11 repository: index saving belongs into the MasterIndex 2022-07-02 18:38:56 +02:00
MichaelEischer
19641bf828 Merge pull request #3810 from greatroar/revert-3786
Revert "restic prune: Merge three loops over the index"
2022-07-01 23:13:39 +02:00
greatroar
a0fa9c6e9f Revert "restic prune: Merge three loops over the index"
This reverts commit 8bdfcf779f.
Should fix #3809. Also needed to make #3290 apply cleanly.
2022-06-30 15:27:34 +02:00
greatroar
90d2c0502b cmd/restic, limiter: Move config knowledge to internal packages
The GlobalOptions struct now embeds a backend.TransportOptions, so it
doesn't need to construct one in open and create. The upload and
download limits are similarly now a struct in internal/limiter that is
embedded in GlobalOptions.
2022-06-22 18:29:58 +02:00
MichaelEischer
bc96879d41 Merge pull request #3785 from MichaelEischer/replace-tomb-usage
Remove usage of tomb package
2022-06-19 14:42:48 +02:00
MichaelEischer
307f14604f Merge pull request #3795 from greatroar/sema
backend: Move semaphores to a dedicated package
2022-06-18 17:12:01 +02:00
MichaelEischer
19581dbc18 Merge pull request #3786 from greatroar/prune
restic prune: Merge three loops over the index
2022-06-18 16:54:50 +02:00
greatroar
8bdfcf779f restic prune: Merge three loops over the index
There were three loops over the index in restic prune, to find
duplicates, to determine sizes (in pack.Size) and to generate packInfos.
These three are now one loop. This way, prune doesn't need to construct
a set of duplicate blobs, pack.Size doesn't need to contain special
logic for prune's use case (the onlyHdr argument) and pack.Size doesn't
need to construct a map only to have it immediately transformed into a
different map.

Some quick testing on a 160GiB local repo doesn't show running time or
memory use of restic prune --dry-run changing significantly.
2022-06-18 10:40:33 +02:00
greatroar
910d917b71 backend: Move semaphores to a dedicated package
... called backend/sema. I resisted the temptation to call the main
type sema.Phore. Also, semaphores are now passed by value to skip a
level of indirection when using them.
2022-06-18 10:01:58 +02:00
MichaelEischer
2c893fe43c Merge pull request #3798 from greatroar/errors
all: Move away from pkg/errors, easy cases
2022-06-17 19:01:40 +02:00
greatroar
f92ecf13c9 all: Move away from pkg/errors, easy cases
github.com/pkg/errors is no longer getting updates, because Go 1.13
went with the more flexible errors.{As,Is} function. Use those instead:
errors from pkg/errors already support the Unwrap interface used by 1.13
error handling. Also:

* check for io.EOF with a straight ==. That value should not be wrapped,
  and the chunker (whose error is checked in the cases changed) does not
  wrap it.
* Give custom Error methods pointer receivers, so there's no ambiguity
  when type-switching since the value type will no longer implement error.
* Make restic.ErrAlreadyLocked private, and rename it to
  alreadyLockedError to match the stdlib convention that error type
  names end in Error.
* Same with rest.ErrIsNotExist => rest.notExistError.
* Make s3.Backend.IsAccessDenied a private function.
2022-06-14 08:36:38 +02:00
MichaelEischer
0c0e7b6957 Merge pull request #3776 from wjiec/bugfix/maxkeys-in-search
Limit number of key files tested while opening a repository
2022-06-12 15:51:10 +02:00
Michael Eischer
c9ef873192 tweak password test count changelog 2022-06-12 15:39:06 +02:00
Michael Eischer
d08549f0ae fix flaky key test 2022-06-12 14:19:06 +02:00
Jayson Wang
f144920ed5 fix handling of maxKeys in SearchKey 2022-06-12 14:19:06 +02:00
Alexander Neumann
1dd4b9b60e Merge pull request #3788 from greatroar/sftp-posix-rename
backend/sftp: Support atomic rename
2022-06-06 19:39:48 +02:00
Alexander Neumann
07114ccb21 Merge pull request #3789 from greatroar/fix-loadblob
internal/repository: Fix LoadBlob + fuzz test
2022-06-06 19:33:42 +02:00
greatroar
c9557b2822 internal/repository: Fix LoadBlob + fuzz test
When given a buf that is big enough for a compressed blob but not its
decompressed contents, the copy at the end of LoadBlob would skip the
last part of the contents.

Fixes #3783.
2022-06-06 17:02:28 +02:00
greatroar
fa8f02292e backend/sftp: Support atomic rename
... if the server has posix-rename@openssh.com.
OpenSSH introduced this extension in 2008:
7c29661471
2022-06-06 13:40:42 +02:00
Alexander Neumann
7d64aa7f57 Merge pull request #3784 from MichaelEischer/cleanup-migrate-check
migrate: Cleanup option to request repository check
2022-06-05 17:37:45 +02:00
Michael Eischer
853ceb3bec get rid of tomb package 2022-06-05 15:54:57 +02:00
Michael Eischer
e002b09d57 archiver: free workers once finished 2022-06-05 15:48:10 +02:00
Michael Eischer
408ac1a0c2 archiver: remove tomb usage 2022-06-05 15:47:52 +02:00
Michael Eischer
5eba1217e7 migrate: Cleanup option to request repository check 2022-06-04 23:45:00 +02:00
MichaelEischer
0cb6b3d80a Merge pull request #3778 from greatroar/ellipsis
cmd/restic: Remove trailing "..." from progress messages
2022-06-04 19:16:00 +02:00
MichaelEischer
60ca6b1418 Merge pull request #3774 from greatroar/archiver-pool
archiver: Remove cleanup goroutine from BufferPool
2022-06-04 18:50:24 +02:00
Jerome Küttner
9adaa6e240 Delete existing path before restoring a symlink 2022-06-01 17:26:25 +02:00
greatroar
b7c990871f cmd/restic: Remove trailing "..." from progress messages
These were added after message since the last refactor of the progress
printing code. Also skips an allocation in the common case.
2022-05-31 19:06:26 +02:00
greatroar
0db1d11b2e archiver: Remove cleanup goroutine from BufferPool
This isn't doing anything. Channels should get cleaned up by the GC when
the last reference to them disappears, just like all other data
structures. Also inlined BufferPool.Put in Buffer.Release, its only
caller.
2022-05-29 17:09:16 +02:00
Alexander Neumann
74f7fe2b98 Merge pull request #3767 from MichaelEischer/fix-prune-empty-snapshot
prune: Fix crash on snapshot loading error
2022-05-29 16:52:45 +02:00
Alexander Neumann
d2c5843c68 Merge pull request #3704 from MichaelEischer/compression-migrations
Support migration to repository format with compression
2022-05-29 15:52:21 +02:00
Alexander Neumann
78a21bbccf Merge pull request #3752 from MichaelEischer/fix-dir-sync-errors
local: Ignore additional errors for directory syncing
2022-05-29 12:54:51 +02:00
MichaelEischer
2ce8587598 Merge pull request #3771 from greatroar/id-marshaljson
internal/restic: Custom ID.MarshalJSON
2022-05-28 16:24:51 +02:00
Michael Eischer
a73fc31b50 Fix linter check 2022-05-28 16:13:46 +02:00
greatroar
dde8e9e296 internal/restic: Custom ID.MarshalJSON
This skips an allocation. internal/archiver benchmarks, Linux/amd64:

name                     old time/op    new time/op    delta
ArchiverSaveFileSmall-8    3.94ms ± 6%    3.91ms ± 6%    ~     (p=0.947 n=20+20)
ArchiverSaveFileLarge-8     304ms ± 3%     301ms ± 4%    ~     (p=0.265 n=18+18)

name                     old speed      new speed      delta
ArchiverSaveFileSmall-8  1.04MB/s ± 6%  1.05MB/s ± 6%    ~     (p=0.803 n=20+20)
ArchiverSaveFileLarge-8   142MB/s ± 3%   143MB/s ± 4%    ~     (p=0.421 n=18+19)

name                     old alloc/op   new alloc/op   delta
ArchiverSaveFileSmall-8    17.9MB ± 0%    17.9MB ± 0%  -0.01%  (p=0.000 n=19+19)
ArchiverSaveFileLarge-8     382MB ± 2%     382MB ± 1%    ~     (p=0.687 n=20+19)

name                     old allocs/op  new allocs/op  delta
ArchiverSaveFileSmall-8       540 ± 1%       528 ± 0%  -2.19%  (p=0.000 n=19+19)
ArchiverSaveFileLarge-8     1.93k ± 3%     1.79k ± 4%  -7.06%  (p=0.000 n=20+20)
2022-05-27 12:26:37 +02:00
Alexander Neumann
bc27c370e7 Update gopkg.in/yaml
This fixes a panic in invalid input, but I think we aren't affected.
2022-05-26 14:23:49 +02:00
Alexander Neumann
9e30152f3c Merge pull request #3770 from lbausch/update-minio-go
Update github.com/minio/minio-go/v7 to v7.0.27
2022-05-26 14:21:19 +02:00
Lorenz Bausch
cc3f8d3732 Update github.com/minio/minio-go/v7 to v7.0.27
This version adds support for Cloudflare R2, as discussed in #3757
2022-05-26 13:05:13 +02:00
Michael Eischer
c8e1ac4049 prune: Don't print stack trace if snapshot can't be loaded 2022-05-23 22:38:45 +02:00
Michael Eischer
173695104c prune: Fix crash on empty snapshot 2022-05-23 22:32:59 +02:00
MichaelEischer
ded783dd61 Merge pull request #3762 from marigbede/marigbede-gs-documentation-fix
Update 030_preparing_a_new_repo.rst
2022-05-23 22:10:38 +02:00
Arigbede Moses
2aa75d6272 Update 030_preparing_a_new_repo.rst 2022-05-18 22:03:59 +01:00
MichaelEischer
88a8701fb5 Merge pull request #3734 from lbausch/validate-patterns
Validate exclude patterns
2022-05-14 16:20:15 +02:00
MichaelEischer
b2a2e5f727 Merge pull request #3753 from greatroar/indexmap-alloc
repository: Re-tune indexmap allocation strategy
2022-05-14 15:44:08 +02:00
MichaelEischer
b52c631bd3 Merge pull request #3754 from greatroar/simplify-hashing
hashing: Fix up comments
2022-05-14 15:33:51 +02:00
Lorenz Bausch
e7df66cc91 Add changelog entry for validating exclude patterns 2022-05-11 22:41:05 +02:00
Lorenz Bausch
36bd464e8c Add tests for validating exclude patterns 2022-05-11 22:41:00 +02:00
greatroar
39a335e690 hashing: Fix up comments 2022-05-11 21:36:10 +02:00
greatroar
5141228e0c repository: Re-tune indexmap allocation strategy
fd05037e1a changed the allocation batch
size from 256 to 128 under the assumption that an indexEntry is 60 bytes
on amd64, but it's 64: structs are padded out to a multiple of 8 for
alignment reasons. That means we'd waste no space in malloc even without
the batch allocation, at least on 64-bit machines. While that strategy
cuts the overallocation down dramatically for many small indexes, it also
seems to slow allocation down (Go 1.18, Linux, amd64, -benchtime=2s):

    name                   old time/op    new time/op    delta
    DecodeIndex-8             4.67s ± 5%     4.60s ± 1%      ~     (p=0.953 n=10+5)
    DecodeIndexParallel-8     4.67s ± 3%     4.60s ± 1%      ~     (p=0.953 n=10+5)
    IndexHasUnknown-8        37.8ns ± 8%    36.5ns ±14%      ~     (p=0.841 n=5+5)
    IndexHasKnown-8          38.5ns ±12%    37.7ns ±10%      ~     (p=0.968 n=5+5)
    IndexAlloc-8              615ms ±18%     607ms ± 1%      ~     (p=1.000 n=10+5)
    IndexAllocParallel-8      245ms ±11%     285ms ± 6%   +16.40%  (p=0.001 n=10+5)
    MasterIndexAlloc-8        286ms ± 9%     275ms ± 2%      ~     (p=1.000 n=10+5)
    LoadIndex/v1-8           27.0ms ± 4%    26.8ms ± 1%      ~     (p=0.690 n=5+5)
    LoadIndex/v2-8           22.4ms ± 1%    22.8ms ± 2%    +1.48%  (p=0.016 n=5+5)

    name                   old alloc/op   new alloc/op   delta
    IndexAlloc-8              446MB ± 0%     446MB ± 0%    -0.00%  (p=0.000 n=8+4)
    IndexAllocParallel-8      446MB ± 0%     446MB ± 0%    -0.00%  (p=0.008 n=8+5)
    MasterIndexAlloc-8        213MB ± 0%     159MB ± 0%   -25.47%  (p=0.000 n=10+5)

    name                   old allocs/op  new allocs/op  delta
    IndexAlloc-8               913k ± 0%     2632k ± 0%  +188.19%  (p=0.008 n=5+5)
    IndexAllocParallel-8       913k ± 0%     2632k ± 0%  +188.21%  (p=0.008 n=5+5)
    MasterIndexAlloc-8         318k ± 0%     1172k ± 0%  +267.86%  (p=0.008 n=5+5)

Instead, this patch sets a batch size of 4, which means no space is
wasted by malloc on 64-bit and very little on 32-bit. It still gets very
close to the savings from not allocating in batches, without requiring
special code for bits.UintSize==64. Benchmark results, again for
Linux/amd64:

    name                   old time/op    new time/op    delta
    DecodeIndex-8             4.67s ± 5%     4.83s ± 9%     ~     (p=0.315 n=10+10)
    DecodeIndexParallel-8     4.67s ± 3%     4.68s ± 4%     ~     (p=0.315 n=10+10)
    IndexHasUnknown-8        37.8ns ± 8%    44.5ns ±19%     ~     (p=0.095 n=5+5)
    IndexHasKnown-8          38.5ns ±12%    36.9ns ± 8%     ~     (p=0.690 n=5+5)
    IndexAlloc-8              615ms ±18%     628ms ±18%     ~     (p=0.218 n=10+10)
    IndexAllocParallel-8      245ms ±11%     262ms ± 9%   +7.02%  (p=0.043 n=10+10)
    MasterIndexAlloc-8        286ms ± 9%     287ms ±13%     ~     (p=1.000 n=10+10)
    LoadIndex/v1-8           27.0ms ± 4%    26.8ms ± 0%     ~     (p=1.000 n=5+5)
    LoadIndex/v2-8           22.4ms ± 1%    22.5ms ± 0%     ~     (p=0.056 n=5+5)

    name                   old alloc/op   new alloc/op   delta
    IndexAlloc-8              446MB ± 0%     446MB ± 0%     ~     (p=1.000 n=8+10)
    IndexAllocParallel-8      446MB ± 0%     446MB ± 0%   -0.00%  (p=0.000 n=8+8)
    MasterIndexAlloc-8        213MB ± 0%     160MB ± 0%  -25.02%  (p=0.000 n=10+9)

    name                   old allocs/op  new allocs/op  delta
    IndexAlloc-8               913k ± 0%     1333k ± 0%  +45.94%  (p=0.000 n=8+10)
    IndexAllocParallel-8       913k ± 0%     1333k ± 0%  +45.94%  (p=0.000 n=8+8)
    MasterIndexAlloc-8         318k ± 0%      525k ± 0%  +64.99%  (p=0.000 n=10+10)

The allocation method indexmap.newEntry has also been rewritten in a
form that is a few instructions shorter.
2022-05-11 21:22:14 +02:00
Michael Eischer
48a0d83143 local: Ignore additional errors for directory syncing
Apparently SMB/CIFS on Linux/macOS returns somewhat random errnos when
trying to sync a windows share which does not support calling fsync for
a directory.
2022-05-11 20:37:59 +02:00
MichaelEischer
ac36fda155 Merge pull request #3749 from greatroar/simplify-hashing
hashing: Remove io.WriterTo implementation
2022-05-11 20:03:43 +02:00
MichaelEischer
df554e5f69 Merge pull request #3748 from greatroar/runworkers
repository: Remove RunWorkers, report ctx.Err()
2022-05-11 19:38:46 +02:00
greatroar
54b8337813 hashing: Remove io.WriterTo implementation
This functionality has gone unused since
4b3dc415ef changed hashing.Reader's only
client to use ioutil.ReadAll on a bufio.Reader wrapping the hashing
Reader.

Reverts bcb852a8d0.
2022-05-10 23:41:18 +02:00
greatroar
2e0f1f5113 repository: Remove RunWorkers, report ctx.Err()
This removes RunWorkers, which had become mere overhead by successive
refactors. It also ensures that each former user of that function
returns any context error that occurs, so failure to complete an
operation is always reported as an error.
2022-05-10 22:26:00 +02:00
MichaelEischer
47c56dea5c Merge pull request #3746 from greatroar/cache-lstat
cache: Don't Lstat before creating CACHEDIR.TAG
2022-05-10 20:31:15 +02:00
Alexander Neumann
c270ab1e08 Merge pull request #3744 from MichaelEischer/fix-windows-temp-file
Fix error on temp file deletion on windows
2022-05-10 19:53:25 +02:00
greatroar
2da377c582 cache: Don't Lstat before creating the tag file
The tag file is opened with O_CREATE|O_EXCL and ErrExist is handled, so
we don't need to check for existence first.
2022-05-10 18:52:39 +02:00
Michael Eischer
ae7e51382a Fix error on temp file deletion on windows
Apparently it can take a moment between closing a tempfile marked as
DELETE_ON_CLOSE and it actually being deleted. During that time the file
is inaccessible. Thus just skip deleting the temp file on windows.
2022-05-09 22:43:26 +02:00
Michael Eischer
5c6db534d4 Add compression migration support to changelog 2022-05-09 22:39:02 +02:00
Michael Eischer
c1bbbcd0dc migrate: Allow migrations to request a check run
This is currently only used by upgrade_repo_v2.
2022-05-09 22:31:30 +02:00
Michael Eischer
59eb132dcd check: Better differentiate between warnings and errors 2022-05-09 22:31:30 +02:00
Michael Eischer
5815f727ee checker: convert error type to use pointer-receivers 2022-05-09 22:31:30 +02:00
Michael Eischer
4faff0debe doc: Describe repository upgrade process 2022-05-09 22:31:30 +02:00
Michael Eischer
e36a40db10 upgrade_repo_v2: Use atomic replace for supported backends 2022-05-09 22:31:30 +02:00
Michael Eischer
7559d2f105 Document repository version and minimum restic version 2022-05-09 22:31:30 +02:00
Michael Eischer
381bd94c6c prune: Add option to repack uncompressed data 2022-05-09 22:31:30 +02:00
Michael Eischer
5406743102 prune: Automatically repack uncompressed trees for repo v2
Tree packs are cached locally at clients and thus benefit a lot from
being compressed. Ensure this be having prune always repack pack files
containing uncompressed trees.
2022-05-09 22:31:30 +02:00
Alexander Neumann
c8c0d659ec Add migration to compress all data 2022-05-09 22:31:30 +02:00
Alexander Neumann
8c244214bf Add tests for upgrade migration 2022-05-09 22:31:30 +02:00
Alexander Neumann
a5f1d318ac Try to make repo upgrade migration more failsafe 2022-05-09 22:31:30 +02:00
Alexander Neumann
82ed5a3a15 Add repo upgrade migration 2022-05-09 22:31:30 +02:00
Alexander Neumann
3af6c180e4 Improve migrate command 2022-05-09 22:31:30 +02:00
Michael Eischer
92816fa966 init: Enable compression support by default 2022-05-09 22:31:30 +02:00
MichaelEischer
ab49c14621 Merge pull request #3740 from MichaelEischer/fix-restore-size-stats
Fix restore size calculation for multiple snapshots
2022-05-09 21:39:45 +02:00
Michael Eischer
2c07f7fff3 stats: hardlinks only reduce restore within a snapshot
The `stats` command checks inodes to not count hardlinked files multiple
times into the restore size. This check applies across all snapshots and
not only within snapshots. As a result the result size was far too low
when calculating it for multiple snapshots and it would vary depending
on the order in which snapshots were listed.
2022-05-09 21:26:24 +02:00
Lorenz Bausch
9fb81c4246 Validate exclude patterns 2022-05-07 21:12:47 +02:00
Lorenz Bausch
e7fd200237 Keep original pattern for later use 2022-05-07 21:08:09 +02:00
MichaelEischer
cc8a03b1d0 Merge pull request #3735 from HenrikBengtsson/master
DOCS: Incorrectly used SI units when IEC units were meant (fix #3669)
2022-05-07 11:15:27 +02:00
Henrik Bengtsson
9bb532672a DOCS: Incorrectly used SI units when IEC units were meant (fix #3669) 2022-05-02 21:21:39 -07:00
MichaelEischer
26c333325c Merge pull request #3715 from ema/master
doc: specify AWS Region via AWS_DEFAULT_REGION
2022-05-02 21:46:56 +02:00
Emanuele Rocca
38c0531b52 doc: specify AWS Region via AWS_DEFAULT_REGION
If no specific Region is mentioned in RESTIC_REPOSITORY, AWS defaults to
us-east-1. For this reason, users that follow the tutorial and create
their S3 bucket in any other region get the following error:

"Fatal: create repository at [...] client.BucketExists"

Explicitly specifying the AWS region name fixes the issue.
2022-05-02 21:26:58 +02:00
Alexander Neumann
fb5b9370f3 Merge pull request #3728 from MichaelEischer/debug-examine-upload
Add support to reupload blobs in `debug examine`
2022-04-30 20:39:19 +02:00
Alexander Neumann
ffbd48c0c6 Merge pull request #3481 from MichaelEischer/recover-enospace
Recover from no free space errors
2022-04-30 20:22:26 +02:00
Michael Eischer
95bcc9ea31 debug: Support pack ID prefixes in debug examine 2022-04-30 20:20:31 +02:00
Michael Eischer
2d6a943911 debug: Add switch to upload blobs extracted by debug examine
This simplifies salvaging a damaged pack file. Reuploading tree blobs
was previously not possible.
2022-04-30 20:20:31 +02:00
Alexander Neumann
9af499d8a4 Tidy go.sum 2022-04-30 20:16:09 +02:00
Alexander Neumann
2e3d23c1d7 Update github.com/klauspost/compress 2022-04-30 20:03:21 +02:00
Michael Eischer
dbbeac7174 prune: Add unsafe option to recover from no free space
The new option allows prune to operate with nearly no scratch space by only removing
no longer necessary pack files and first deleting the index before
rebuilding it. By first deleting the index it becomes safe to just
delete no longer necessary pack files. However, as a downside there's
now the risk that the repository becomes inaccessible if prune fails.

To recover from that problem a user might have to manually delete the
repository index and then run (a full) `rebuild-index` again.
2022-04-30 19:21:07 +02:00
Michael Eischer
cf5cb673fb repository: Use existing method to collect pack ids 2022-04-30 19:14:21 +02:00
Michael Eischer
b335cb6285 repository: Refactor index IDs collection 2022-04-30 19:14:21 +02:00
MichaelEischer
9c047f170a Merge pull request #3419 from DanielG/relax-file-modes
Enable admin to decide group access to repository files
2022-04-30 16:39:37 +02:00
Daniel Gröber
f31b4f29c1 Use config file modes to derive new dir/file modes
Fixes #2351
2022-04-30 15:59:51 +02:00
Alexander Neumann
71c653f9e0 Merge pull request #3727 from MichaelEischer/changelog-3475
Add changelog for local/sftp connection limit
2022-04-30 14:05:21 +02:00
Michael Eischer
29a8f92967 Add changelog for local/sftp connection limit 2022-04-30 13:36:47 +02:00
MichaelEischer
ac9324aeaf Merge pull request #3666 from MichaelEischer/compression
Implement compression support
2022-04-30 11:49:05 +02:00
Alexander Neumann
dc5adef255 Add documentation for --repository-version 2022-04-30 11:34:10 +02:00
Michael Eischer
4b01b06f2f repository: Test compressed blobs in StreamPack 2022-04-30 11:34:10 +02:00
Michael Eischer
bcab548617 pack: slightly expand testing of compressed blobs 2022-04-30 11:34:10 +02:00
Michael Eischer
ec2b25565a repository: test uncompressedLength field and index example 2022-04-30 11:34:10 +02:00
Michael Eischer
9ffb8920f1 repository: run blackbox tests using old and new repo version 2022-04-30 11:34:10 +02:00
Michael Eischer
abe5935693 repository: unify repository version-specific initialization
Mark the master index as compressed also when initializing a new
repository. This is only relevant for testing.
2022-04-30 11:34:10 +02:00
Alexander Neumann
8776031f96 Leave allocating slices to the decompress code 2022-04-30 11:34:10 +02:00
Alexander Neumann
5eb05a0afe Configure zstd encoder/decoder 2022-04-30 11:34:10 +02:00
Michael Eischer
2f36e044db Cleanup pack header check 2022-04-30 11:34:10 +02:00
Alexander Neumann
94dc9a0fa7 Amend changelog 2022-04-30 11:34:10 +02:00
Alexander Neumann
8b11b86383 Add option global --compression 2022-04-30 11:34:10 +02:00
Michael Eischer
f38f457a64 Add basic changelog for compression support 2022-04-30 11:34:10 +02:00
Michael Eischer
ba27d29d58 Print repository version when opening a repo 2022-04-30 11:34:10 +02:00
Michael Eischer
7132df529e repository: Increase index size for repo version 2
A compressed index is only about one third the size of an uncompressed
one. Thus increase the number of entries in an index to avoid cluttering
the repository with small indexes.
2022-04-30 11:34:10 +02:00
Michael Eischer
2535524132 debug: Add support for compressed blobs 2022-04-30 11:34:10 +02:00
Michael Eischer
fda7bb0f09 debug: Reduce code duplication 2022-04-30 11:34:10 +02:00
Michael Eischer
66f9048bce repository: Alloc zstd encoder/decoder on demand 2022-04-30 11:34:10 +02:00
Michael Eischer
fd05037e1a repository: recalibrate index batch allocation size 2022-04-30 11:34:10 +02:00
Michael Eischer
6fb408d90e repository: implement pack compression 2022-04-30 11:34:10 +02:00
Michael Eischer
362ab06023 init: Add flag to specify created repository version 2022-04-30 10:07:42 +02:00
Michael Eischer
4b957e7373 repository: Implement index/snapshot/lock compression
The config file is not compressed as it should remain readable by older
restic versions such that these can return a proper error.

As the old format for unpacked data does not include a version header,
make use of a trick: The old data is always encoded as JSON. Thus it can
only start with '{' or '['. For any other value the first byte indicates
a versioned format. The version is set to 2 for now. Then the zstd
compressed data follows.
2022-04-30 10:07:42 +02:00
Michael Eischer
0957b74887 Misc design.rst cleanups 2022-04-30 10:07:42 +02:00
Alexander Neumann
270ed00d1f doc: Add repository compression support documentation
Co-authored-by: Michael Eischer <michael.eischer@fau.de>
2022-04-30 10:07:42 +02:00
Alexander Neumann
4e1ef7804a Merge pull request #3717 from MichaelEischer/fix-stuck-repack
Fix stuck repack step
2022-04-30 09:50:43 +02:00
Alexander Neumann
e4780d3956 Merge pull request #3718 from MichaelEischer/sftp-docs-fix
doc: sftp with password actually works
2022-04-23 20:27:27 +02:00
Alexander Neumann
c183e35b5a Merge pull request #3719 from MichaelEischer/read-write-order
Describe repository read/write order required by repository format
2022-04-23 15:23:00 +02:00
Michael Eischer
6f9e20a1bb doc: Describe repository read/write order 2022-04-23 15:21:02 +02:00
Michael Eischer
f9219e8608 doc: sftp with password actually works 2022-04-23 11:57:36 +02:00
Michael Eischer
3b630d9998 add missing streamPacks changelog 2022-04-23 11:50:19 +02:00
Michael Eischer
566ac11c65 fix changelog name 2022-04-23 11:37:00 +02:00
Michael Eischer
f5609d1d3c prune: Fail early if too few backend connections 2022-04-23 11:32:52 +02:00
Michael Eischer
e597b99b55 repository: Reduce repack workers to prevent deadlock
As repack streams packs these occupy one backend connection. Uploading a
new pack also requires a backend connection. To prevent a deadlock
during repack when reaching the backend connections limit, simply limit
the repackWorker count to always leave one connection for uploading.
2022-04-23 11:28:18 +02:00
Michael Eischer
ee627cd832 backend/mem: Actually enforce connection limit
This will allow tests to detect deadlocks related to the connections
limit.
2022-04-23 11:22:00 +02:00
Michael Eischer
4f97492d28 Backend: Expose connections parameter 2022-04-23 11:13:08 +02:00
rawtaz
07a565e6f7 Merge pull request #3716 from MichaelEischer/password-error-on-stderr
Print password error message on stderr
2022-04-21 01:40:54 +02:00
Michael Eischer
bf7da7ff10 Print password error message on stderr
The password prompt itself is already printed on stderr.
2022-04-20 22:22:09 +02:00
Alexander Neumann
dba47d29d5 Merge pull request #3711 from restic/doc-grouping
doc: Clarify and make grouping in forget more noticeable
2022-04-16 10:08:11 +02:00
Leo R. Lundgren
8ac7519fd5 doc: Clarify and make grouping in forget more noticeable 2022-04-16 01:13:13 +02:00
Alexander Neumann
edc1a24a90 Merge pull request #3707 from duracell/patch-1
doc: fix missing "init" in rest-server example
2022-04-12 20:39:25 +02:00
Michael
9563e2f75c doc: fix missing "init" in rest-server example 2022-04-12 18:21:53 +02:00
Alexander Neumann
7f133a28b2 Update VERSION file 2022-04-11 20:34:14 +02:00
Alexander Neumann
4f3b1f19cb Set development version for 0.13.1 2022-04-10 20:41:00 +02:00
Alexander Neumann
89ee1cf9ee Merge pull request #3610 from MichaelEischer/windows-temp-files
Improve handling of temporary files on windows
2022-04-10 20:29:04 +02:00
Alexander Neumann
a059ef90f8 Merge pull request #3702 from MichaelEischer/extend-config-error
Print used key name if config fails to load
2022-04-10 20:25:24 +02:00
Michael Eischer
4077a81b34 Add simple test for fs.TempFile on windows 2022-04-09 23:37:58 +02:00
Michael Eischer
9a3f1a9703 Simplify and comment TempFile implementation for windows 2022-04-09 23:37:58 +02:00
Michael Eischer
c2aabb2686 Print used key name if config fails to load 2022-04-09 22:38:18 +02:00
MichaelEischer
c60a5f00c9 Merge pull request #3675 from ItsMattL/update
Refactor file handing for self-update.
2022-04-09 21:55:56 +02:00
Matt LaPlante
0ba9d4ced7 Refactor file handing for self-update.
* Write new file payload to a temp file before touching the original
binary. Minimizes the possibility of failing mid-write and corrupting
the binary.
* On Windows, move the original binary out to a temp file rather than
removing it as the running binary is locked. Fixes issue #2248.
2022-04-09 21:40:33 +02:00
Alexander Neumann
04e054465a Merge pull request #3475 from MichaelEischer/local-sftp-conn-limit
Limit concurrent operations for local / sftp backend
2022-04-09 21:33:00 +02:00
Alexander Neumann
1519e9f911 Merge pull request #3570 from MichaelEischer/list-snapshots-before-index
List snapshots before index
2022-04-09 21:15:49 +02:00
Michael Eischer
ebab35581c Check in integration test that snapshots are listed before the index
As an exception prune is still allowed to load the index before
snapshots, as it uses exclusive locks. In case of problems with locking
it is also better to load snapshots created after loading the index, as
this will lead to a prune sanity check failure instead of a broken snapshot.
2022-04-09 12:27:27 +02:00
Michael Eischer
7b9ae91e04 copy: Load snapshots before indexes 2022-04-09 12:27:25 +02:00
Michael Eischer
47243176fa diff: list snapshots only once 2022-04-09 12:26:31 +02:00
Michael Eischer
5af828e3e6 add changelogs 2022-04-09 12:26:31 +02:00
Michael Eischer
4636c20397 test that TestFindListOnce calls List only once 2022-04-09 12:26:31 +02:00
Michael Eischer
9e12159230 Fix O(n) backend list calls in FindFilteredSnapshots
When resolving snapshotIDs in FindFilteredSnapshots either
FindLatestSnapshot or FindSnapshot is called. Both operations issue a
list operation to the backend. When for example passing a long list of
snapshot ids to `forget` this could lead to a large number of list
operations.
2022-04-09 12:26:31 +02:00
Michael Eischer
3d29083e60 copy/find/ls/recover/stats: Memorize snapshot listing before index
These commands filter the snapshots according to some criteria which
essentially requires loading the index before filtering the snapshots.
Thus create a copy of the snapshots list beforehand and use it later on.
2022-04-09 12:26:30 +02:00
Michael Eischer
2ec0f3303a backup/diff/dump/restore/stats: List snapshots before index
During a backup the index is written before the corresponding snapshots.
To ensure that a concurrent/later restic run can read a snapshot's data,
restic thus must first load the snapshots and only afterwards the index.
Otherwise it is not possible to ensure that the loaded index is recent
enough to cover all of the snapshot's data.
2022-04-09 12:24:09 +02:00
Michael Eischer
ece06f125e sftp: Limit concurrent backend operations 2022-04-09 12:21:38 +02:00
Michael Eischer
cd783358d3 local: Limit concurrent backend operations
Use a limit of 2 similar to the filereader concurrency in the archiver.
2022-04-09 12:21:38 +02:00
Michael Eischer
0b258cc054 backends: clean reader closing 2022-04-09 12:21:38 +02:00
Alex Duchesne
9e34c791c9 Better temp file cleanup on Windows. 2022-04-09 12:00:22 +02:00
Alexander Neumann
7d55b4f95e Merge pull request #3701 from restic/rawtaz-doc-exclude-cachedir
doc: Link to CACHEDIR.TAG specification
2022-04-08 09:35:08 +02:00
rawtaz
de4e3117eb doc: Link to CACHEDIR.TAG specification 2022-04-08 03:07:27 +02:00
MichaelEischer
500079d265 Merge pull request #3689 from brightdroid/patch-1
added documentation for zsh autocompletion
2022-04-03 21:49:47 +02:00
Alexander Neumann
192288bc9c Merge pull request #3696 from cqjjjzr/fix-win-procgrp
Fix rclone (scoop shim) and sftp issue due to detached console on Windows
2022-04-03 13:46:28 +02:00
Charlie Jiang
d9c9415cfd Fix rclone (scoop shim) and sftp issue due to detached console on Windows 2022-04-03 17:53:17 +08:00
Alexander Neumann
59370b6062 Merge pull request #3695 from jernej-9/typo-fix
Fix a typo in the docs
2022-04-01 20:10:40 +02:00
Jernej Debevc
6e5731bf2f Fix a typo in the docs 2022-04-01 17:16:55 +02:00
Alexander Neumann
305cd1e730 Merge pull request #3693 from greatroar/cast-btrfs-super-magic
Cast unix.Statfs_t.Type to int64 when checking for btrfs
2022-04-01 08:09:46 +02:00
greatroar
c23c0f7c14 Cast unix.Statfs_t.Type to int64 when checking for btrfs
Fixes #3687. Uses the cast suggested by @MichaelEischer, except that the
contant isn't cast along, because it's untyped and will be converted by
the compiler as necessary.
2022-03-31 22:30:45 +02:00
Alexander Neumann
774c2e75ca Merge pull request #3680 from restic/update-deps
Update all dependencies (except fuse), require Go 1.15
2022-03-30 21:36:08 +02:00
Alexander Neumann
66d50b72e3 Require Go 1.15 or later
The library github.com/golang-jwt/jwt/v4 requires the FillByte() method
of *big.Int, so we're raising the minimum Go version to 1.15.
2022-03-30 21:11:17 +02:00
Alexander Neumann
89d86a7933 Update all dependencies (except fuse) 2022-03-30 21:11:13 +02:00
Christoph Roeder
7510bdc247 added documentation for zsh autocompletion 2022-03-30 10:45:52 +02:00
Alexander Neumann
f190d2e60e Merge pull request #3686 from restic/fix-diff
Fix diff
2022-03-30 07:43:07 +02:00
Alexander Neumann
206550a246 Fix diff
Nodes in trees were always printed with a `+` in diff, regardless of
whether or not a dir was added or removed. Let's use the mode we were
passed in printDir().

Closes #3685
2022-03-29 21:05:11 +02:00
Alexander Neumann
db8a958991 Merge pull request #3683 from MichaelEischer/fix-golangci-lint-warnings
Fix golangci lint warnings
2022-03-29 11:45:10 +02:00
Alexander Neumann
724ace0e99 Merge pull request #3682 from MichaelEischer/refactor-code
Pack size calculation cleanup and misc other changes
2022-03-29 11:06:04 +02:00
Michael Eischer
af31266b7d golangci-lint: replace deprecated golint with revive 2022-03-28 22:33:17 +02:00
Michael Eischer
2f81af6afa bloblru: Fix comment for New function 2022-03-28 22:25:25 +02:00
Michael Eischer
61e179ee78 switch to golang.org/x/term 2022-03-28 22:24:15 +02:00
Michael Eischer
c60540b196 add go:build headers everywhere 2022-03-28 22:23:47 +02:00
Michael Eischer
fefe9f5c0e pack: Hide more implementation details 2022-03-28 22:12:16 +02:00
Michael Eischer
a773cb6527 pack: cleanup header size calculation 2022-03-28 22:09:49 +02:00
Michael Eischer
6408686973 repository: Simplify Blob equality check 2022-03-28 22:09:49 +02:00
Michael Eischer
243698680a crypto: Use helpers for size calculations 2022-03-28 22:09:49 +02:00
Michael Eischer
d6db5a1fc2 archiver: Fix test
The test relied on an undeocumented sideeffect of the LoadBlob implementation
2022-03-28 22:09:49 +02:00
Michael Eischer
f78bd14e28 repository: Remove pack implementation details from MasterIndex 2022-03-28 22:09:49 +02:00
Michael Eischer
dc3d77dacc repository: make saveAndEncrypt private 2022-03-28 22:09:49 +02:00
Michael Eischer
6877e7edbb repository: Rename LoadAndDecrypt to LoadUnpacked
The method is the complement for SaveUnpacked and not for
SaveAndEncrypt. The latter assembles blobs into pack files.
2022-03-28 22:09:49 +02:00
Michael Eischer
2e1613d4c6 errors: Ensure that errors.IsFatal(errors.Fatal("err")) == true
This fixes a few cases where restic output "Fatal: Fatal: [...]"
2022-03-28 22:09:49 +02:00
Alexander Neumann
a08b95c497 Merge pull request #3513 from MichaelEischer/fast-copy
Speed-up copy command
2022-03-28 20:18:43 +02:00
Michael Eischer
537b4c310a copy: Implement by reusing repack
The repack operation copies all selected blobs from a set of pack files
into new pack files. For prune the source and destination repositories
are identical. To implement copy, just use a different source and
destination repository.
2022-03-26 20:47:15 +01:00
Alexander Neumann
4d5db61bd0 Merge pull request #3484 from MichaelEischer/stream-check-repack
Stream packs in `check --read-data` and during repacking
2022-03-26 20:46:17 +01:00
Alexander Neumann
4ab12f59a5 Set development version for 0.13.0 2022-03-26 20:10:07 +01:00
Alexander Neumann
e682f7c0d6 Add tests for StreamPack 2022-03-21 21:15:03 +01:00
Michael Eischer
27524979e8 restorer: Remove dead code 2022-02-13 11:43:09 +01:00
Michael Eischer
bba8ba7a5b repository: cancel streampack context after error 2022-02-12 20:18:25 +01:00
Michael Eischer
47554a3428 repository: Fix error handling in repack
When storing a blob fails, this is a fatal error which must not be
retried.
2022-02-12 20:18:25 +01:00
Michael Eischer
4b3dc415ef checker: cleanup header extraction 2022-02-12 20:18:25 +01:00
Michael Eischer
930a00ad54 checker: reuse bufio reader 2022-02-12 20:18:25 +01:00
Michael Eischer
34ebafb8b6 repository: don't crash if blob size is too short 2022-02-12 20:18:25 +01:00
Michael Eischer
becebf5d88 repository: remove unused DownloadAndHash 2022-02-12 20:18:25 +01:00
Michael Eischer
f1e58e7c7f checker: rewrite ReadData to stream packs 2022-02-12 20:18:25 +01:00
Michael Eischer
f40abd92fa restorer: convert to use StreamPack 2022-02-12 20:18:25 +01:00
Michael Eischer
f00f690658 repository: stream packs during repacking 2022-02-12 20:18:25 +01:00
Michael Eischer
c4a2bfcb39 repository: Add StreamPacks function
The function supports efficiently loading a specified list of blobs from
a single pack in a streaming fashion. That is there's no need for
temporary files independent of the pack size.
2022-02-12 20:18:25 +01:00
Michael Eischer
153e2ba859 repository: Implement lisiting blobs per pack file 2022-02-12 20:18:24 +01:00
471 changed files with 17726 additions and 10980 deletions

2
.gitattributes vendored Normal file
View File

@@ -0,0 +1,2 @@
# Workaround for https://github.com/golang/go/issues/52268.
**/testdata/fuzz/*/* eol=lf

13
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,13 @@
version: 2
updates:
# Dependencies listed in go.mod
- package-ecosystem: "gomod"
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
# Dependencies listed in .github/workflows/*.yml
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"

View File

@@ -9,7 +9,7 @@ on:
pull_request:
env:
latest_go: "1.18.x"
latest_go: "1.19.x"
GO111MODULE: on
jobs:
@@ -19,47 +19,31 @@ jobs:
# list of jobs to run:
include:
- job_name: Windows
go: 1.18.x
go: 1.19.x
os: windows-latest
install_verb: install
- job_name: macOS
go: 1.18.x
go: 1.19.x
os: macOS-latest
test_fuse: false
install_verb: install
- job_name: Linux
go: 1.18.x
go: 1.19.x
os: ubuntu-latest
test_cloud_backends: true
test_fuse: true
check_changelog: true
install_verb: install
- job_name: Linux
go: 1.17.x
- job_name: Linux (race)
go: 1.19.x
os: ubuntu-latest
test_fuse: true
install_verb: install
test_opts: "-race"
- job_name: Linux
go: 1.16.x
go: 1.18.x
os: ubuntu-latest
test_fuse: true
install_verb: get
- job_name: Linux
go: 1.15.x
os: ubuntu-latest
test_fuse: true
install_verb: get
- job_name: Linux
go: 1.14.x
os: ubuntu-latest
test_fuse: true
install_verb: get
name: ${{ matrix.job_name }} Go ${{ matrix.go }}
runs-on: ${{ matrix.os }}
@@ -69,14 +53,14 @@ jobs:
steps:
- name: Set up Go ${{ matrix.go }}
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Get programs (Linux/macOS)
run: |
echo "build Go tools"
go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
go install github.com/restic/rest-server/cmd/rest-server@latest
echo "install minio server"
mkdir $HOME/bin
@@ -98,7 +82,7 @@ jobs:
chmod 755 $HOME/bin/rclone
rm -rf rclone*
# add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
# add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3)
echo $HOME/bin >> $GITHUB_PATH
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
@@ -108,7 +92,7 @@ jobs:
$ProgressPreference = 'SilentlyContinue'
echo "build Go tools"
go ${{ matrix.install_verb }} github.com/restic/rest-server/...
go install github.com/restic/rest-server/...
echo "install minio server"
mkdir $Env:USERPROFILE/bin
@@ -120,7 +104,7 @@ jobs:
unzip rclone.zip
copy rclone*/rclone.exe $Env:USERPROFILE/bin
# add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
# add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v3)
echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
echo "install tar"
@@ -142,7 +126,7 @@ jobs:
if: matrix.os == 'windows-latest'
- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Build with build.go
run: |
@@ -152,7 +136,7 @@ jobs:
env:
RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
run: |
go test -cover ./...
go test -cover ${{matrix.test_opts}} ./...
- name: Test cloud backends
env:
@@ -193,7 +177,9 @@ jobs:
# only run cloud backend tests for pull requests from and pushes to our
# own repo, otherwise the secrets are not available
if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
# Skip for Dependabot pull requests as these are run without secrets
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
- name: Check changelog files with calens
run: |
@@ -209,15 +195,16 @@ jobs:
# ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
matrix:
# run cross-compile in two batches parallel so the overall tests run faster
# run cross-compile in three batches parallel so the overall tests run faster
targets:
- "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
openbsd/386 openbsd/amd64"
- "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x"
- "freebsd/386 freebsd/amd64 freebsd/arm \
- "openbsd/386 openbsd/amd64 \
freebsd/386 freebsd/amd64 freebsd/arm \
aix/ppc64 \
darwin/amd64 darwin/arm64 \
netbsd/386 netbsd/amd64 \
darwin/amd64 darwin/arm64"
- "netbsd/386 netbsd/amd64 \
windows/386 windows/amd64 \
solaris/amd64"
@@ -230,7 +217,7 @@ jobs:
steps:
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ env.latest_go }}
@@ -239,7 +226,7 @@ jobs:
go install github.com/mitchellh/gox@latest
- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Cross-compile with gox for ${{ matrix.targets }}
env:
@@ -255,22 +242,21 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ env.latest_go }}
- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
uses: golangci/golangci-lint-action@v3
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.45
version: v1.49
# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true
args: --verbose --timeout 5m
skip-go-installation: true
# only run golangci-lint for pull requests, otherwise ALL hints get
# reported. We need to slowly address all issues until we can enable
@@ -288,11 +274,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@v4
with:
# list of Docker images to use as base name for tags
images: |
@@ -308,14 +294,14 @@ jobs:
type=sha
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
push: false
context: .

View File

@@ -24,7 +24,7 @@ linters:
- govet
# make sure names and comments are used according to the conventions
- golint
- revive
# detect when assignments to existing variables are not used
- ineffassign
@@ -51,7 +51,9 @@ issues:
# list of things to not warn about
exclude:
# golint: do not warn about missing comments for exported stuff
- exported (function|method|var|type|const) `.*` should have comment or be unexported
# golint: ignore constants in all caps
# revive: do not warn about missing comments for exported stuff
- exported (function|method|var|type|const) .* should have comment or be unexported
# revive: ignore constants in all caps
- don't use ALL_CAPS in Go names; use CamelCase
# revive: lots of packages don't have such a comment
- "package-comments: should have a package comment"

File diff suppressed because it is too large Load Diff

View File

@@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things:
Remember, the easier it is for us to reproduce the bug, the earlier it will be
corrected!
In addition, you can compile restic with debug support by running
`go run build.go -tags debug` and instructing it to create a debug
log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
In addition, you can instruct restic to create a debug log by setting the
environment variable `DEBUG_LOG` to a file, e.g. like this:
$ export DEBUG_LOG=/tmp/restic-debug.log
$ restic backup ~/work
@@ -66,8 +65,8 @@ Development Environment
The repository contains the code written for restic in the directories
`cmd/` and `internal/`.
Restic requires Go version 1.14 or later for compiling. Clone the repo (without
having `$GOPATH` set) and `cd` into the directory:
Make sure you have the minimum required Go version installed. Clone the repo
(without having `$GOPATH` set) and `cd` into the directory:
$ unset GOPATH
$ git clone https://github.com/restic/restic
@@ -77,7 +76,7 @@ Then use the `go` tool to build restic:
$ go build ./cmd/restic
$ ./restic version
restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64
You can run all tests with the following command:

View File

@@ -1 +1 @@
0.13.0
0.15.0

View File

@@ -3,8 +3,8 @@
// This program aims to make building Go programs for end users easier by just
// calling it with `go run`, without having to setup a GOPATH.
//
// This program needs Go >= 1.12. It'll use Go modules for compilation. It
// builds the package configured as Main in the Config struct.
// This program checks for a minimum Go version. It will use Go modules for
// compilation. It builds the package configured as Main in the Config struct.
// BSD 2-Clause License
//
@@ -43,7 +43,6 @@ package main
import (
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -59,7 +58,7 @@ var config = Config{
Main: "./cmd/restic", // package name for the main package
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
MinVersion: GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
}
// Config configures the build.
@@ -179,7 +178,7 @@ func test(cwd string, env map[string]string, args ...string) error {
// getVersion returns the version string from the file VERSION in the current
// directory.
func getVersionFromFile() string {
buf, err := ioutil.ReadFile("VERSION")
buf, err := os.ReadFile("VERSION")
if err != nil {
verbosePrintf("error reading file VERSION: %v\n", err)
return ""
@@ -319,12 +318,8 @@ func (v GoVersion) String() string {
}
func main() {
if !goVersion.AtLeast(GoVersion{1, 12, 0}) {
die("Go version (%v) is too old, restic requires Go >= 1.12\n", goVersion)
}
if !goVersion.AtLeast(config.MinVersion) {
fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s\n", goVersion, config.MinVersion)
os.Exit(1)
}

View File

@@ -0,0 +1,9 @@
Enhancement: Support pruning even when the disk is full
When running out of disk space it was no longer possible to add or remove
data from a repository. To help with recovering from such a deadlock, the
prune command now supports an `--unsafe-recover-no-free-space` option to
recover from these situations. Make sure to read the documentation first!
https://github.com/restic/restic/issues/1153
https://github.com/restic/restic/pull/3481

View File

@@ -0,0 +1,8 @@
Change: Support debug log creation in release builds
Creating a debug log was only possible in debug builds which required users to
manually build restic. We changed the release builds to allow creating debug
logs by simply setting the environment variable `DEBUG_LOG=logname.log`.
https://github.com/restic/restic/issues/1842
https://github.com/restic/restic/pull/3826

View File

@@ -0,0 +1,28 @@
Enhancement: Add compression support
We've added compression support to the restic repository format. To create a
repository using the new format run `init --repository-version 2`. Please note
that the repository cannot be read by restic versions prior to 0.14.0.
You can configure whether data is compressed with the option `--compression`. It
can be set to `auto` (the default, which will compress very fast), `max` (which
will trade backup speed and CPU usage for better compression), or `off` (which
disables compression). Each setting is only applied for the current run of restic
and does *not* apply to future runs. The option can also be set via the
environment variable `RESTIC_COMPRESSION`.
To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the
documentation for more details. The migration checks the repository integrity
and upgrades the repository format, but will not change any data. Afterwards,
prune will rewrite the metadata to make use of compression.
As an alternative you can use the `copy` command to migrate snapshots; First
create a new repository using
`init --repository-version 2 --copy-chunker-params --repo2 path/to/old/repo`,
and then use the `copy` command to copy all snapshots to the new repository.
https://github.com/restic/restic/issues/21
https://github.com/restic/restic/issues/3779
https://github.com/restic/restic/pull/3666
https://github.com/restic/restic/pull/3704
https://github.com/restic/restic/pull/3733

View File

@@ -0,0 +1,18 @@
Enhancement: Adaptive IO concurrency based on backend connections
Many commands used hard-coded limits for the number of concurrent operations.
This prevented speed improvements by increasing the number of connections used
by a backend.
These limits have now been replaced by using the configured number of backend
connections instead, which can be controlled using the
`-o <backend-name>.connections=5` option. Commands will then automatically
scale their parallelism accordingly.
To limit the number of CPU cores used by restic, you can set the environment
variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use
`GOMAXPROCS=1`.
https://github.com/restic/restic/issues/2162
https://github.com/restic/restic/issues/1467
https://github.com/restic/restic/pull/3611

View File

@@ -0,0 +1,8 @@
Bugfix: Support `self-update` on Windows
Restic `self-update` would fail in situations where the operating system
locks running binaries, including Windows. The new behavior works around
this by renaming the running file and swapping the updated file in place.
https://github.com/restic/restic/issues/2248
https://github.com/restic/restic/pull/3675

View File

@@ -0,0 +1,12 @@
Enhancement: Allow pack size customization
Restic now uses a target pack size of 16 MiB by default. This can be customized
using the `--pack-size size` option. Supported pack sizes range between 4 and
128 MiB.
It is possible to migrate an existing repository to _larger_ pack files using
`prune --repack-small`. This will rewrite every pack file which is
significantly smaller than the target size.
https://github.com/restic/restic/issues/2291
https://github.com/restic/restic/pull/3731

View File

@@ -0,0 +1,14 @@
Enhancement: Allow use of SAS token to authenticate to Azure
Previously restic only supported AccountKeys to authenticate to Azure
storage accounts, which necessitates giving a significant amount of
access.
We added support for Azure SAS tokens which are a more fine-grained
and time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME`
and `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for
authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it will take
precedence.
https://github.com/restic/restic/issues/2295
https://github.com/restic/restic/pull/3661

View File

@@ -0,0 +1,13 @@
Enhancement: Improve backup speed with many small files
We have restructured the backup pipeline to continue reading files while all
upload connections are busy. This allows the backup to already prepare the next
data file such that the upload can continue as soon as a connection becomes
available. This can especially improve the backup performance for high latency
backends.
The upload concurrency is now controlled using the `-o <backend-name>.connections=5`
option.
https://github.com/restic/restic/issues/2696
https://github.com/restic/restic/pull/3489

View File

@@ -0,0 +1,15 @@
Enhancement: Make snapshot directory structure of `mount` command customizable
We've added the possibility to customize the snapshot directory structure of
the `mount` command using templates passed to the `--snapshot-template` option.
The formatting of snapshots' timestamps is now controlled using `--time-template`
and supports subdirectories to for example group snapshots by year. Please
see `restic help mount` for further details.
Characters in tag names which are not allowed in a filename are replaced by
underscores `_`. For example a tag `foo/bar` will result in a directory name
of `foo_bar`.
https://github.com/restic/restic/issues/2907
https://github.com/restic/restic/pull/2913
https://github.com/restic/restic/pull/3691

View File

@@ -0,0 +1,12 @@
Enhancement: Optimize handling of duplicate blobs in `prune`
Restic `prune` always used to repack all data files containing duplicate
blobs. This effectively removed all duplicates during prune. However, as a
consequence all these data files were repacked even if the unused repository
space threshold could be reached with less work.
This is now changed and `prune` works nice and fast even when there are lots
of duplicate blobs.
https://github.com/restic/restic/issues/3114
https://github.com/restic/restic/pull/3290

View File

@@ -0,0 +1,13 @@
Change: Deprecate `check --check-unused` and add further checks
Since restic 0.12.0, it is expected to still have unused blobs after running
`prune`. This made the `--check-unused` option of the `check` command rather
useless and tended to confuse users. This option has been deprecated and is
now ignored.
The `check` command now also warns if a repository is using either the legacy
S3 layout or mixed pack files with both tree and data blobs. The latter is
known to cause performance problems.
https://github.com/restic/restic/issues/3295
https://github.com/restic/restic/pull/3730

View File

@@ -0,0 +1,14 @@
Bugfix: List snapshots in backend at most once to resolve snapshot IDs
Many commands support specifying a list of snapshot IDs which are then used to
determine the snapshots to be processed by the command. To resolve snapshot IDs
or `latest`, and check that these exist, restic previously listed all snapshots
stored in the repository. Depending on the backend this could be a slow and/or
expensive operation.
Restic now lists the snapshots only once and remembers the result in order to
resolve all further snapshot IDs swiftly.
https://github.com/restic/restic/issues/3428
https://github.com/restic/restic/pull/3570
https://github.com/restic/restic/pull/3395

View File

@@ -0,0 +1,14 @@
Bugfix: Fix rare 'not found in repository' error for `copy` command
In rare cases `copy` (and other commands) would report that `LoadTree(...)`
returned an `id [...] not found in repository` error. This could be caused by
a backup or copy command running concurrently. The error was only temporary;
running the failed restic command a second time as a workaround did resolve the
error.
This issue has now been fixed by correcting the order in which restic reads data
from the repository. It is now guaranteed that restic only loads snapshots for
which all necessary data is already available.
https://github.com/restic/restic/issues/3432
https://github.com/restic/restic/pull/3570

View File

@@ -0,0 +1,10 @@
Enhancement: Improve handling of temporary files on Windows
In some cases restic failed to delete temporary files, causing the current
command to fail. This has now been fixed by ensuring that Windows automatically
deletes the file. In addition, temporary files are only written to disk when
necessary, reducing disk writes.
https://github.com/restic/restic/issues/3465
https://github.com/restic/restic/issues/1551
https://github.com/restic/restic/pull/3610

View File

@@ -0,0 +1,7 @@
Bugfix: The `diff` command incorrectly listed some files as added
There was a bug in the `diff` command, causing it to always show files in a
removed directory as added. This has now been fixed.
https://github.com/restic/restic/issues/3685
https://github.com/restic/restic/pull/3686

View File

@@ -0,0 +1,13 @@
Bugfix: Fix rclone (shimmed by Scoop) and sftp not working on Windows
In #3602 a fix was introduced to address the problem of `rclone` prematurely
exiting when Ctrl+C is pressed on Windows. The solution was to create the
subprocess with its console detached from the restic console.
However, this solution failed when using `rclone` installed by Scoop or using
`sftp` with a passphrase-protected private key. We've now fixed this by using
a different approach to prevent Ctrl-C from passing down too early.
https://github.com/restic/restic/issues/3681
https://github.com/restic/restic/issues/3692
https://github.com/restic/restic/pull/3696

View File

@@ -0,0 +1,11 @@
Enhancement: Validate exclude patterns before backing up
Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
`--iexclude-file` previously weren't validated. As a consequence, invalid
patterns resulted in files that were meant to be excluded being backed up.
Restic now validates all patterns before running the backup and aborts with
a fatal error if an invalid pattern is detected.
https://github.com/restic/restic/issues/3709
https://github.com/restic/restic/pull/3734

View File

@@ -0,0 +1,13 @@
Bugfix: Directory sync errors for repositories accessed via SMB
On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in
restic failing to save the lock file, yielding the following errors:
Save(<lock/071fe833f0>) returned error, retrying after 552.330144ms: sync /repo/locks: no such file or directory
Save(<lock/bf789d7343>) returned error, retrying after 552.330144ms: sync /repo/locks: invalid argument
This has now been fixed by ignoring the relevant error codes.
https://github.com/restic/restic/issues/3720
https://github.com/restic/restic/issues/3751
https://github.com/restic/restic/pull/3752

View File

@@ -0,0 +1,8 @@
Bugfix: The `stats` command miscalculated restore size for multiple snapshots
Since restic 0.10.0 the restore size calculated by the `stats` command for
multiple snapshots was too low. The hardlink detection was accidentally applied
across multiple snapshots and thus ignored many files. This has now been fixed.
https://github.com/restic/restic/issues/3736
https://github.com/restic/restic/pull/3740

View File

@@ -0,0 +1,8 @@
Enhancement: Improve SFTP repository initialization over slow links
The `init` command, when used on an SFTP backend, now sends multiple `mkdir`
commands to the backend concurrently. This reduces the waiting times when
creating a repository over a very slow connection.
https://github.com/restic/restic/issues/3837
https://github.com/restic/restic/pull/3840

View File

@@ -0,0 +1,9 @@
Bugfix: Yield error on invalid policy to `forget`
The `forget` command previously silently ignored invalid/unsupported
units in the duration options, such as e.g. `--keep-within-daily 2w`.
Specifying an invalid/unsupported duration unit now results in an error.
https://github.com/restic/restic/issues/3861
https://github.com/restic/restic/pull/3862

View File

@@ -0,0 +1,21 @@
Enhancement: Use config file permissions to control file group access
Previously files in a local/SFTP repository would always end up with very
restrictive access permissions, allowing access only to the owner. This
prevented a number of valid use-cases involving groups and ACLs.
We now use the permissions of the config file in the repository to decide
whether group access should be given to newly created repository files or
not. We arrange for repository files to be created group readable exactly
when the repository config file is group readable.
To opt-in to group readable repositories, a simple `chmod -R g+r` or
equivalent on the config file can be used. For repositories that should
be writable by group members a tad more setup is required, see the docs.
Posix ACLs can also be used now that the group permissions being forced to
zero no longer masks the effect of ACL entries.
https://github.com/restic/restic/issues/2351
https://github.com/restic/restic/pull/3419
https://forum.restic.net/t/1391

View File

@@ -0,0 +1,9 @@
Enhancement: Allow limiting IO concurrency for local and SFTP backend
Restic did not support limiting the IO concurrency / number of connections for
accessing repositories stored using the local or SFTP backends. The number of
connections is now limited as for other backends, and can be configured via the
the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures
that restic does not overwhelm the backend with concurrent IO operations.
https://github.com/restic/restic/pull/3475

View File

@@ -0,0 +1,13 @@
Enhancement: Stream data in `check` and `prune` commands
The commands `check --read-data` and `prune` previously downloaded data files
into temporary files which could end up being written to disk. This could cause
a large amount of data being written to disk.
The pack files are now instead streamed, which removes the need for temporary
files. Please note that *uploads* during `backup` and `prune` still require
temporary files.
https://github.com/restic/restic/pull/3484
https://github.com/restic/restic/issues/3710
https://github.com/restic/restic/pull/3717

View File

@@ -0,0 +1,10 @@
Enhancement: Improve speed of `copy` command
The `copy` command could require a long time to copy snapshots for non-local
backends. This has been improved to provide a throughput comparable to the
`restore` command.
Additionally, `copy` now displays a progress bar.
https://github.com/restic/restic/issues/2923
https://github.com/restic/restic/pull/3513

View File

@@ -0,0 +1,8 @@
Change: Update dependencies and require Go 1.15 or newer
We've updated most dependencies. Since some libraries require newer language
features we're dropping support for Go 1.14, which means that restic now
requires at least Go 1.15 to build.
https://github.com/restic/restic/issues/3680
https://github.com/restic/restic/issues/3883

View File

@@ -0,0 +1,7 @@
Bugfix: Print "wrong password" to stderr instead of stdout
If an invalid password was entered, the error message was printed on stdout and
not on stderr as intended. This has now been fixed.
https://github.com/restic/restic/pull/3716
https://forum.restic.net/t/4965

View File

@@ -0,0 +1,8 @@
Enhancement: Display full IDs in `check` warnings
When running commands to inspect or repair a damaged repository, it is often
necessary to supply the full IDs of objects stored in the repository.
The output of `check` now includes full IDs instead of their shortened variant.
https://github.com/restic/restic/pull/3729

View File

@@ -0,0 +1,19 @@
Change: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
The `init` and `copy` commands can read data from another repository.
However, confusingly `--repo2` referred to the repository *from* which the
`init` command copies parameters, but for the `copy` command `--repo2`
referred to the copy *destination*.
We've introduced a new option, `--from-repo`, which always refers to the
source repository for both commands. The old parameter names have been
deprecated but still work. To create a new repository and copy all snapshots
to it, the commands are now as follows:
```
restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params
restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo
```
https://github.com/restic/restic/pull/3742
https://forum.restic.net/t/5017

View File

@@ -0,0 +1,14 @@
Bugfix: Correctly rebuild index for legacy repositories
After running `rebuild-index` on a legacy repository containing mixed pack
files (that is, pack files which store both metadata and file data), `check`
printed warnings like `pack 12345678 contained in several indexes: ...`.
This warning was not critical, but has now nonetheless been fixed by properly
handling mixed pack files while rebuilding the index.
Running `prune` for such legacy repositories will also fix the warning by
reorganizing the pack files which caused it.
https://github.com/restic/restic/pull/3772
https://github.com/restic/restic/pull/3884
https://forum.restic.net/t/5044/13

View File

@@ -0,0 +1,7 @@
Enhancement: Optimize memory usage for directories with many files
Backing up a directory with hundreds of thousands or more files caused restic
to require large amounts of memory. We've now optimized the `backup` command
such that it requires up to 30% less memory.
https://github.com/restic/restic/pull/3773

View File

@@ -0,0 +1,11 @@
Bugfix: Limit number of key files tested while opening a repository
Previously, restic tested the password against every key in the repository
when opening a repository. The more keys there were in the repository, the
slower this operation became.
Restic now tests the password against up to 20 key files in the repository.
Alternatively, you can use the `--key-hint=<key ID>` option to specify a
specific key file to use instead.
https://github.com/restic/restic/pull/3776

View File

@@ -0,0 +1,11 @@
Enhancement: Validate include/exclude patterns before restoring
Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include`
and `--iinclude` weren't validated before running the restore. Invalid
patterns would result in error messages being printed repeatedly, and
possibly unwanted files being restored.
Restic now validates all patterns before running the restore, and aborts
with a fatal error if an invalid pattern is detected.
https://github.com/restic/restic/pull/3819

View File

@@ -0,0 +1,8 @@
Enhancement: Implement `rewrite` command
Restic now has a `rewrite` command which allows to rewrite existing snapshots
to remove unwanted files.
https://github.com/restic/restic/issues/14
https://github.com/restic/restic/pull/2731
https://github.com/restic/restic/pull/4079

View File

@@ -0,0 +1,15 @@
Enhancement: Inform about successful retries after errors
When a recoverable error is encountered, restic shows a warning message saying
that it's retrying, e.g.:
`Save(<data/956b9ced99>) returned error, retrying after 357.131936ms: ...`
This message can be confusing in that it never clearly states whether the retry
is successful or not. This has now been fixed such that restic follows up with
a message confirming a successful retry, e.g.:
`Save(<data/956b9ced99>) operation successful after 1 retries`
https://github.com/restic/restic/issues/1734
https://github.com/restic/restic/pull/2661

View File

@@ -0,0 +1,12 @@
Enhancement: Improve handling of directories with duplicate entries
If for some reason a directory contains a duplicate entry, the `backup` command
would previously fail with a `node "path/to/file" already present` or `nodes
are not ordered got "path/to/file", last "path/to/file"` error.
The error handling has been improved to only report a warning in this case. Make
sure to check that the filesystem in question is not damaged if you see this!
https://github.com/restic/restic/issues/1866
https://github.com/restic/restic/issues/3937
https://github.com/restic/restic/pull/3880

View File

@@ -0,0 +1,10 @@
Bugfix: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT
signal to restic. This used to cause restic to exit with a non-zero exit code.
The exit code has now been changed to zero as the above is the expected way
to stop the `mount` command and should therefore be considered successful.
https://github.com/restic/restic/issues/2015
https://github.com/restic/restic/pull/3894

View File

@@ -0,0 +1,19 @@
Enhancement: Support B2 API keys restricted to hiding but not deleting files
When the B2 backend does not have the necessary permissions to permanently
delete files, it now automatically falls back to hiding files. This allows
using restic with an application key which is not allowed to delete files.
This can prevent an attacker from deleting backups with such an API key.
To use this feature create an application key without the `deleteFiles`
capability. It is recommended to restrict the key to just one bucket.
For example using the `b2` command line tool:
`b2 create-key --bucket <bucketName> <keyName> listBuckets,readFiles,writeFiles,listFiles`
Alternatively, you can use the S3 backend to access B2, as described
in the documentation. In this mode, files are also only hidden instead
of being deleted permanently.
https://github.com/restic/restic/issues/2134
https://github.com/restic/restic/pull/2398

View File

@@ -0,0 +1,11 @@
Enhancement: Make `init` open only one connection for the SFTP backend
The `init` command using the SFTP backend used to connect twice to the
repository. This could be inconvenient if the user must enter a password,
or cause `init` to fail if the server does not correctly close the first SFTP
connection.
This has now been fixed by reusing the first/initial SFTP connection opened.
https://github.com/restic/restic/issues/2152
https://github.com/restic/restic/pull/3882

View File

@@ -0,0 +1,13 @@
Enhancement: Handle cache corruption on disk and in downloads
In rare situations, like for example after a system crash, the data stored
in the cache might be corrupted. This could cause restic to fail and required
manually deleting the cache.
Restic now automatically removes broken data from the cache, allowing it
to recover from such a situation without user intervention. In addition,
restic retries downloads which return corrupt data in order to also handle
temporary download problems.
https://github.com/restic/restic/issues/2533
https://github.com/restic/restic/pull/3521

View File

@@ -0,0 +1,17 @@
Bugfix: Don't read password from stdin for `backup --stdin`
The `backup` command when used with `--stdin` previously tried to read first
the password, then the data to be backed up from standard input. This meant
it would often confuse part of the data for the password.
From now on, it will instead exit with the message `Fatal: cannot read both
password and data from stdin` unless the password is passed in some other
way (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc).
To enter the password interactively a password command has to be used. For
example on Linux, `mysqldump somedatabase | restic backup --stdin
--password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads
the password from the terminal.
https://github.com/restic/restic/issues/2591
https://github.com/restic/restic/pull/4011

View File

@@ -0,0 +1,9 @@
Enhancement: Support restoring symbolic links on Windows
The `restore` command now supports restoring symbolic links on Windows. Because
of Windows specific restrictions this is only possible when running restic with
the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator.
https://github.com/restic/restic/issues/1078
https://github.com/restic/restic/issues/2699
https://github.com/restic/restic/pull/2875

View File

@@ -0,0 +1,20 @@
Enhancement: Stricter repository lock handling
Previously, restic commands kept running even if they failed to refresh their
locks in time. This could be a problem e.g. in case the client system running
a backup entered the standby power mode while the backup was still in progress
(which would prevent the client from refreshing its lock), and after a short
delay another host successfully runs `unlock` and `prune` on the repository,
which would remove all data added by the in-progress backup. If the backup
client later continues its backup, even though its lock had expired in the
meantime, this would lead to an incomplete snapshot.
To address this, lock handling is now much stricter. Commands requiring a lock
are canceled if the lock is not refreshed successfully in time. In addition,
if a lock file is not readable restic will not allow starting a command. It may
be necessary to remove invalid lock files manually or use `unlock --remove-all`.
Please make sure that no other restic processes are running concurrently before
doing this, however.
https://github.com/restic/restic/issues/2715
https://github.com/restic/restic/pull/3569

View File

@@ -0,0 +1,9 @@
Change: Include full snapshot ID in JSON output of `backup`
We have changed the JSON output of the backup command to include the full
snapshot ID instead of just a shortened version, as the latter can be ambiguous
in some rare cases. To derive the short ID, please truncate the full ID down to
eight characters.
https://github.com/restic/restic/issues/2724
https://github.com/restic/restic/pull/3993

View File

@@ -0,0 +1,8 @@
Enhancement: Add support for `credential_process` to S3 backend
Restic now uses a newer library for the S3 backend, which adds support for the
`credential_process` option in the AWS credential configuration.
https://github.com/restic/restic/issues/3029
https://github.com/restic/restic/issues/4034
https://github.com/restic/restic/pull/4025

View File

@@ -0,0 +1,8 @@
Enhancement: Make `mount` command support macOS using macFUSE 4.x
Restic now uses a different FUSE library for mounting snapshots and making them
available as a FUSE filesystem using the `mount` command. This adds support for
macFUSE 4.x which can be used to make this work on recent macOS versions.
https://github.com/restic/restic/issues/3096
https://github.com/restic/restic/pull/4024

View File

@@ -0,0 +1,7 @@
Enhancement: Support JSON output for the `init` command
The `init` command used to ignore the `--json` option, but now outputs a JSON
message if the repository was created successfully.
https://github.com/restic/restic/issues/3124
https://github.com/restic/restic/pull/3132

View File

@@ -0,0 +1,14 @@
Bugfix: Delete files on Backblaze B2 more reliably
Restic used to only delete the latest version of files stored in B2. In most
cases this worked well as there was only a single version of the file. However,
due to retries while uploading it is possible for multiple file versions to be
stored at B2. This could lead to various problems for files that should have
been deleted but still existed.
The implementation has now been changed to delete all versions of files, which
doubles the amount of Class B transactions necessary to delete files, but
assures that no file versions are left behind.
https://github.com/restic/restic/issues/3161
https://github.com/restic/restic/pull/3885

View File

@@ -0,0 +1,12 @@
Bugfix: Make SFTP backend report no space left on device
Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when
the remote disk was full. Restic now reports "sftp: no space left on device"
and exits immediately when it detects this condition.
A fix for this issue was implemented in restic 0.12.1, but unfortunately the
fix itself contained a bug that prevented it from taking effect.
https://github.com/restic/restic/issues/3336
https://github.com/restic/restic/pull/3345
https://github.com/restic/restic/pull/4075

View File

@@ -0,0 +1,10 @@
Bugfix: Improve handling of interrupted syscalls in `mount` command
Accessing restic's FUSE mount could result in "input/output" errors when using
programs in which syscalls can be interrupted. This is for example the case for
Go programs. This has now been fixed by improved error handling of interrupted
syscalls.
https://github.com/restic/restic/issues/3567
https://github.com/restic/restic/issues/3694
https://github.com/restic/restic/pull/3875

View File

@@ -0,0 +1,7 @@
Bugfix: Fix stuck `copy` command when `-o <backend>.connections=1`
When running the `copy` command with `-o <backend>.connections=1` the
command would be infinitely stuck. This has now been fixed.
https://github.com/restic/restic/issues/3897
https://github.com/restic/restic/pull/3898

View File

@@ -0,0 +1,9 @@
Bugfix: Correct prune statistics for partially compressed repositories
In a partially compressed repository, one data blob can exist both in an
uncompressed and a compressed version. This caused the `prune` statistics to
become inaccurate and e.g. report a too high value for the unused size, such
as "unused size after prune: 16777215.991 TiB". This has now been fixed.
https://github.com/restic/restic/issues/3918
https://github.com/restic/restic/pull/3980

View File

@@ -0,0 +1,11 @@
Change: Make `unlock` display message only when locks were actually removed
The `unlock` command used to print the "successfully removed locks" message
whenever it was run, regardless of lock files having being removed or not.
This has now been changed such that it only prints the message if any lock
files were actually removed. In addition, it also reports the number of
removed lock files.
https://github.com/restic/restic/issues/3929
https://github.com/restic/restic/pull/3935

View File

@@ -0,0 +1,15 @@
Enhancement: Improve handling of ErrDot errors in rclone and sftp backends
Since Go 1.19, restic can no longer implicitly run relative executables which
are found in the current directory (e.g. `rclone` if found in `.`). This is a
security feature of Go to prevent against running unintended and possibly
harmful executables.
The error message for this was just "cannot run executable found relative to
current directory". This has now been improved to yield a more specific error
message, informing the user how to explicitly allow running the executable
using the `-o rclone.program` and `-o sftp.command` extended options with `./`.
https://github.com/restic/restic/issues/3932
https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
https://go.dev/blog/path-security

View File

@@ -0,0 +1,8 @@
Bugfix: Make `backup` no longer hang on Solaris when seeing a FIFO file
The `backup` command used to hang on Solaris whenever it encountered a FIFO
file (named pipe), due to a bug in the handling of extended attributes. This
bug has now been fixed.
https://github.com/restic/restic/issues/4003
https://github.com/restic/restic/pull/4053

View File

@@ -0,0 +1,8 @@
Bugfix: Support ExFAT-formatted local backends on macOS Ventura
ExFAT-formatted disks could not be used as local backends starting from macOS
Ventura. Restic commands would fail with an "inappropriate ioctl for device"
error. This has now been fixed.
https://github.com/restic/restic/issues/4016
https://github.com/restic/restic/pull/4021

View File

@@ -0,0 +1,11 @@
Change: Don't print skipped snapshots by default in `copy` command
The `copy` command used to print each snapshot that was skipped because it
already existed in the target repository. The amount of this output could
practically bury the list of snapshots that were actually copied.
From now on, the skipped snapshots are by default not printed at all, but
this can be re-enabled by increasing the verbosity level of the command.
https://github.com/restic/restic/issues/4033
https://github.com/restic/restic/pull/4066

View File

@@ -0,0 +1,10 @@
Bugfix: Make `init` ignore "Access Denied" errors when creating S3 buckets
In restic 0.9.0 through 0.13.0, the `init` command ignored some permission
errors from S3 backends when trying to check for bucket existence, so that
manually created buckets with custom permissions could be used for backups.
This feature became broken in 0.14.0, but has now been restored again.
https://github.com/restic/restic/issues/4085
https://github.com/restic/restic/pull/4086

View File

@@ -0,0 +1,10 @@
Bugfix: Don't generate negative UIDs and GIDs in tar files from `dump`
When using a 32-bit build of restic, the `dump` command could in some cases
create tar files containing negative UIDs and GIDs, which cannot be read by
GNU tar. This corner case especially applies to backups from stdin on Windows.
This is now fixed such that `dump` creates valid tar files in these cases too.
https://github.com/restic/restic/issues/4103
https://github.com/restic/restic/pull/4104

View File

@@ -0,0 +1,17 @@
Enhancement: Restore files with long runs of zeros as sparse files
When using `restore --sparse`, the restorer may now write files containing long
runs of zeros as sparse files (also called files with holes), where the zeros
are not actually written to disk.
How much space is saved by writing sparse files depends on the operating
system, file system and the distribution of zeros in the file.
During backup restic still reads the whole file including sparse regions, but
with optimized processing speed of sparse regions.
https://github.com/restic/restic/issues/79
https://github.com/restic/restic/issues/3903
https://github.com/restic/restic/pull/2601
https://github.com/restic/restic/pull/3854
https://forum.restic.net/t/sparse-file-support/1264

View File

@@ -0,0 +1,7 @@
Enhancement: Make backup file read concurrency configurable
The `backup` command now supports a `--read-concurrency` option which allows
tuning restic for very fast storage like NVMe disks by controlling the number
of concurrent file reads during the backup process.
https://github.com/restic/restic/pull/2750

View File

@@ -0,0 +1,8 @@
Bugfix: Make `restore` replace existing symlinks
When restoring a symlink, restic used to report an error if the target path
already existed. This has now been fixed such that the potentially existing
target path is first removed before the symlink is restored.
https://github.com/restic/restic/issues/2578
https://github.com/restic/restic/pull/3780

View File

@@ -0,0 +1,6 @@
Enhancement: Optimize prune memory usage
The `prune` command needs large amounts of memory in order to determine what to
keep and what to remove. This is now optimized to use up to 30% less memory.
https://github.com/restic/restic/pull/3899

View File

@@ -0,0 +1,6 @@
Enhancement: Improve speed of parent snapshot detection in `backup` command
Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw`
options could require a long time to find the parent snapshot. This has been improved.
https://github.com/restic/restic/pull/3905

View File

@@ -0,0 +1,12 @@
Enhancement: Add compression statistics to the `stats` command
When executed with `--mode raw-data` on a repository that supports compression,
the `stats` command now calculates and displays, for the selected repository or
snapshots: the uncompressed size of the data; the compression progress
(percentage of data that has been compressed); the compression ratio of the
compressed data; the total space saving.
It also takes into account both the compressed and uncompressed data if the
repository is only partially compressed.
https://github.com/restic/restic/pull/3915

View File

@@ -0,0 +1,6 @@
Enhancement: Provide command completion for PowerShell
Restic already provided generation of completion files for bash, fish and zsh.
Now powershell is supported, too.
https://github.com/restic/restic/pull/3925/files

View File

@@ -0,0 +1,10 @@
Enhancement: Allow `backup` file tree scanner to be disabled
The `backup` command walks the file tree in a separate scanner process to find
the total size and file/directory count, and uses this to provide an ETA. This
can slow down backups, especially of network filesystems.
The command now has a new option `--no-scan` which can be used to disable this
scanning in order to speed up backups when needed.
https://github.com/restic/restic/pull/3931

View File

@@ -0,0 +1,9 @@
Enhancement: Ignore additional/unknown files in repository
If a restic repository had additional files in it (not created by restic),
commands like `find` and `restore` could become confused and fail with an
`multiple IDs with prefix "12345678" found` error. These commands now
ignore such additional files.
https://github.com/restic/restic/pull/3943
https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17

View File

@@ -0,0 +1,7 @@
Bugfix: Make `ls` return exit code 1 if snapshot cannot be loaded
The `ls` command used to show a warning and return exit code 0 when failing
to load a snapshot. This has now been fixed such that it instead returns exit
code 1 (still showing a warning).
https://github.com/restic/restic/pull/3951

View File

@@ -0,0 +1,9 @@
Enhancement: Improve `backup` performance for small files
When backing up small files restic was slower than it could be. In particular
this affected backups using maximum compression.
This has been fixed by reworking the internal parallelism of the backup
command, making it back up small files around two times faster.
https://github.com/restic/restic/pull/3955

View File

@@ -0,0 +1,7 @@
Change: Update dependencies and require Go 1.18 or newer
Most dependencies have been updated. Since some libraries require newer language
features, support for Go 1.15-1.17 has been dropped, which means that restic now
requires at least Go 1.18 to build.
https://github.com/restic/restic/pull/4041

View File

@@ -0,0 +1,11 @@
Bugfix: Make `self-update` enabled by default only in release builds
The `self-update` command was previously included by default in all builds of
restic as opposed to only in official release builds, even if the `selfupdate`
tag was not explicitly enabled when building.
This has now been corrected, and the `self-update` command is only available
if restic was built with `-tags selfupdate` (as done for official release
builds by `build.go`).
https://github.com/restic/restic/pull/4100

View File

@@ -1,5 +1,5 @@
# The first line must start with Bugfix:, Enhancement: or Change:,
# including the colon. Use present use. Remove lines starting with '#'
# including the colon. Use present tense. Remove lines starting with '#'
# from this template.
Enhancement: Allow custom bar in the foo command

View File

@@ -11,7 +11,7 @@ import (
var cleanupHandlers struct {
sync.Mutex
list []func() error
list []func(code int) (int, error)
done bool
ch chan os.Signal
}
@@ -25,7 +25,7 @@ func init() {
// AddCleanupHandler adds the function f to the list of cleanup handlers so
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
// is received.
func AddCleanupHandler(f func() error) {
func AddCleanupHandler(f func(code int) (int, error)) {
cleanupHandlers.Lock()
defer cleanupHandlers.Unlock()
@@ -36,22 +36,24 @@ func AddCleanupHandler(f func() error) {
}
// RunCleanupHandlers runs all registered cleanup handlers
func RunCleanupHandlers() {
func RunCleanupHandlers(code int) int {
cleanupHandlers.Lock()
defer cleanupHandlers.Unlock()
if cleanupHandlers.done {
return
return code
}
cleanupHandlers.done = true
for _, f := range cleanupHandlers.list {
err := f()
var err error
code, err = f(code)
if err != nil {
Warnf("error in cleanup handler: %v\n", err)
}
}
cleanupHandlers.list = nil
return code
}
// CleanupHandler handles the SIGINT signals.
@@ -75,6 +77,6 @@ func CleanupHandler(c <-chan os.Signal) {
// Exit runs the cleanup handlers and then terminates the process with the
// given exit code.
func Exit(code int) {
RunCleanupHandlers()
code = RunCleanupHandlers(code)
os.Exit(code)
}

View File

@@ -6,16 +6,17 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/spf13/cobra"
tomb "gopkg.in/tomb.v2"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
@@ -29,7 +30,7 @@ import (
)
var cmdBackup = &cobra.Command{
Use: "backup [flags] FILE/DIR [FILE/DIR] ...",
Use: "backup [flags] [FILE/DIR] ...",
Short: "Create a new backup of files and/or directories",
Long: `
The "backup" command creates a new snapshot and saves the files and directories
@@ -54,44 +55,51 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
var t tomb.Tomb
term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
ctx := cmd.Context()
var wg sync.WaitGroup
cancelCtx, cancel := context.WithCancel(ctx)
defer func() {
// shutdown termstatus
cancel()
wg.Wait()
}()
err := runBackup(backupOptions, globalOptions, term, args)
t.Kill(nil)
if werr := t.Wait(); werr != nil {
panic(fmt.Sprintf("term.Run() returned err: %v", err))
}
return err
term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
wg.Add(1)
go func() {
defer wg.Done()
term.Run(cancelCtx)
}()
return runBackup(ctx, backupOptions, globalOptions, term, args)
},
}
// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
Parent string
Force bool
Excludes []string
InsensitiveExcludes []string
ExcludeFiles []string
InsensitiveExcludeFiles []string
ExcludeOtherFS bool
ExcludeIfPresent []string
ExcludeCaches bool
ExcludeLargerThan string
Stdin bool
StdinFilename string
Tags restic.TagLists
Host string
FilesFrom []string
FilesFromVerbatim []string
FilesFromRaw []string
TimeStamp string
WithAtime bool
IgnoreInode bool
IgnoreCtime bool
UseFsSnapshot bool
DryRun bool
excludePatternOptions
Parent string
Force bool
ExcludeOtherFS bool
ExcludeIfPresent []string
ExcludeCaches bool
ExcludeLargerThan string
Stdin bool
StdinFilename string
Tags restic.TagLists
Host string
FilesFrom []string
FilesFromVerbatim []string
FilesFromRaw []string
TimeStamp string
WithAtime bool
IgnoreInode bool
IgnoreCtime bool
UseFsSnapshot bool
DryRun bool
ReadConcurrency uint
NoScan bool
}
var backupOptions BackupOptions
@@ -103,12 +111,11 @@ func init() {
cmdRoot.AddCommand(cmdBackup)
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
@@ -116,7 +123,7 @@ func init() {
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
err := f.MarkDeprecated("hostname", "use --host")
@@ -124,7 +131,6 @@ func init() {
// MarkDeprecated only returns an error when the flag could not be found
panic(err)
}
f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
@@ -133,9 +139,14 @@ func init() {
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
if runtime.GOOS == "windows" {
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
}
// parse read concurrency from env, on error the default value will be used
readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
backupOptions.ReadConcurrency = uint(readConcurrency)
}
// filterExisting returns a slice of all existing items, or an error if no
@@ -143,7 +154,7 @@ func init() {
func filterExisting(items []string) (result []string, err error) {
for _, item := range items {
_, err := fs.Lstat(item)
if err != nil && os.IsNotExist(errors.Cause(err)) {
if errors.Is(err, os.ErrNotExist) {
Warnf("%v does not exist, skipping\n", item)
continue
}
@@ -175,7 +186,7 @@ func readLines(filename string) ([]string, error) {
)
if filename == "-" {
data, err = ioutil.ReadAll(os.Stdin)
data, err = io.ReadAll(os.Stdin)
} else {
data, err = textfile.Read(filename)
}
@@ -252,6 +263,10 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
// Check returns an error when an invalid combination of options was set.
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
if gopts.password == "" {
if opts.Stdin {
return errors.Fatal("cannot read both password and data from stdin")
}
filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
for _, filename := range filesFrom {
if filename == "-" {
@@ -292,30 +307,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
fs = append(fs, f)
}
// add patterns from file
if len(opts.ExcludeFiles) > 0 {
excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
if err != nil {
return nil, err
}
opts.Excludes = append(opts.Excludes, excludes...)
}
if len(opts.InsensitiveExcludeFiles) > 0 {
excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
if err != nil {
return nil, err
}
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
}
if len(opts.InsensitiveExcludes) > 0 {
fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
}
if len(opts.Excludes) > 0 {
fs = append(fs, rejectByPattern(opts.Excludes))
fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
if err != nil {
return nil, err
}
fs = append(fs, fsPatterns...)
if opts.ExcludeCaches {
opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
@@ -356,53 +352,6 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
return fs, nil
}
// readExcludePatternsFromFiles reads all exclude files and returns the list of
// exclude patterns. For each line, leading and trailing white space is removed
// and comment lines are ignored. For each remaining pattern, environment
// variables are resolved. For adding a literal dollar sign ($), write $$ to
// the file.
func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
getenvOrDollar := func(s string) string {
if s == "$" {
return "$"
}
return os.Getenv(s)
}
var excludes []string
for _, filename := range excludeFiles {
err := func() (err error) {
data, err := textfile.Read(filename)
if err != nil {
return err
}
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// ignore empty lines
if line == "" {
continue
}
// strip comments
if strings.HasPrefix(line, "#") {
continue
}
line = os.Expand(line, getenvOrDollar)
excludes = append(excludes, line)
}
return scanner.Err()
}()
if err != nil {
return nil, err
}
}
return excludes, nil
}
// collectTargets returns a list of target files/dirs from several sources.
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
if opts.Stdin {
@@ -425,7 +374,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
var expanded []string
expanded, err := filepath.Glob(line)
if err != nil {
return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
return nil, fmt.Errorf("pattern: %s: %w", line, err)
}
if len(expanded) == 0 {
Warnf("pattern %q does not match any files, skipping\n", line)
@@ -472,31 +421,24 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
// parent returns the ID of the parent snapshot. If there is none, nil is
// returned.
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
// Force using a parent
if !opts.Force && opts.Parent != "" {
id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
if err != nil {
return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
}
parentID = &id
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
if opts.Force {
return nil, nil
}
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentID == nil {
id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
if err == nil {
parentID = &id
} else if err != restic.ErrNoSnapshotFound {
return nil, err
}
snName := opts.Parent
if snName == "" {
snName = "latest"
}
return parentID, nil
sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName)
// Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil
}
return sn, err
}
func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
@@ -515,13 +457,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
}
}
var t tomb.Tomb
if gopts.verbosity >= 2 && !gopts.JSON {
Verbosef("open repository\n")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -532,11 +472,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
} else {
progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
}
progressReporter := backup.NewProgress(progressPrinter)
progressReporter := backup.NewProgress(progressPrinter,
calculateProgressInterval(!gopts.Quiet, gopts.JSON))
if opts.DryRun {
repo.SetDryRun()
progressReporter.SetDryRun()
}
// use the terminal for stdout/stderr
@@ -546,14 +486,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
}()
gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
wg, wgCtx := errgroup.WithContext(ctx)
cancelCtx, cancel := context.WithCancel(wgCtx)
defer cancel()
wg.Go(func() error { progressReporter.Run(cancelCtx); return nil })
if !gopts.JSON {
progressPrinter.V("lock repository")
}
lock, err := lockRepo(gopts.ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -571,30 +512,30 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
return err
}
if !gopts.JSON {
progressPrinter.V("load index files")
}
err = repo.LoadIndex(gopts.ctx)
if err != nil {
return err
}
var parentSnapshotID *restic.ID
var parentSnapshot *restic.Snapshot
if !opts.Stdin {
parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil {
return err
}
if !gopts.JSON {
if parentSnapshotID != nil {
progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
if parentSnapshot != nil {
progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
} else {
progressPrinter.P("no parent snapshot found, will read all files\n")
}
}
}
if !gopts.JSON {
progressPrinter.V("load index files")
}
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
selectByNameFilter := func(item string) bool {
for _, reject := range rejectByNameFuncs {
if reject(item) {
@@ -620,7 +561,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
}
errorHandler := func(item string, err error) error {
return progressReporter.Error(item, nil, err)
return progressReporter.Error(item, err)
}
messageHandler := func(msg string, args ...interface{}) {
@@ -647,25 +588,27 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
targets = []string{filename}
}
sc := archiver.NewScanner(targetFS)
sc.SelectByName = selectByNameFilter
sc.Select = selectFilter
sc.Error = progressReporter.ScannerError
sc.Result = progressReporter.ReportTotal
if !opts.NoScan {
sc := archiver.NewScanner(targetFS)
sc.SelectByName = selectByNameFilter
sc.Select = selectFilter
sc.Error = progressPrinter.ScannerError
sc.Result = progressReporter.ReportTotal
if !gopts.JSON {
progressPrinter.V("start scan on %v", targets)
if !gopts.JSON {
progressPrinter.V("start scan on %v", targets)
}
wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
}
t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
arch := archiver.New(repo, targetFS, archiver.Options{})
arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
arch.SelectByName = selectByNameFilter
arch.Select = selectFilter
arch.WithAtime = opts.WithAtime
success := true
arch.Error = func(item string, fi os.FileInfo, err error) error {
arch.Error = func(item string, err error) error {
success = false
return progressReporter.Error(item, fi, err)
return progressReporter.Error(item, err)
}
arch.CompleteItem = progressReporter.CompleteItem
arch.StartFile = progressReporter.StartFile
@@ -680,28 +623,24 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
}
if parentSnapshotID == nil {
parentSnapshotID = &restic.ID{}
}
snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes,
Tags: opts.Tags.Flatten(),
Time: timeStamp,
Hostname: opts.Host,
ParentSnapshot: *parentSnapshotID,
ParentSnapshot: parentSnapshot,
}
if !gopts.JSON {
progressPrinter.V("start backup on %v", targets)
}
_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines
t.Kill(nil)
cancel()
// let's see if one returned an error
werr := t.Wait()
werr := wg.Wait()
// return original error
if err != nil {
@@ -709,7 +648,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
}
// Report finished execution
progressReporter.Finish(id)
progressReporter.Finish(id, opts.DryRun)
if !gopts.JSON && !opts.DryRun {
progressPrinter.P("snapshot %s saved\n", id.Str())
}

View File

@@ -14,8 +14,7 @@ import (
)
func TestCollectTargets(t *testing.T) {
dir, cleanup := rtest.TempDir(t)
defer cleanup()
dir := rtest.TempDir(t)
fooSpace := "foo "
barStar := "bar*" // Must sort before the others, below.

View File

@@ -11,6 +11,7 @@ import (
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
)
@@ -138,7 +139,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
if err != nil {
return err
}
size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
size = fmt.Sprintf("%11s", ui.FormatBytes(uint64(bytes)))
}
name := entry.Name()

View File

@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"github.com/spf13/cobra"
@@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runCat(globalOptions, args)
return runCat(cmd.Context(), globalOptions, args)
},
}
@@ -32,40 +33,32 @@ func init() {
cmdRoot.AddCommand(cmdCat)
}
func runCat(gopts GlobalOptions, args []string) error {
func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
return errors.Fatal("type or ID not specified")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
defer unlockRepo(lock)
}
tpe := args[0]
var id restic.ID
if tpe != "masterkey" && tpe != "config" {
if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
id, err = restic.ParseID(args[1])
if err != nil {
if tpe != "snapshot" {
return errors.Fatalf("unable to parse ID: %v\n", err)
}
// find snapshot id with prefix
id, err = restic.FindSnapshot(gopts.ctx, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
return errors.Fatalf("unable to parse ID: %v\n", err)
}
}
@@ -79,7 +72,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "index":
buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil {
return err
}
@@ -87,13 +80,12 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
sn := &restic.Snapshot{}
err = repo.LoadJSONUnpacked(gopts.ctx, restic.SnapshotFile, id, sn)
sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
if err != nil {
return err
return errors.Fatalf("could not find snapshot: %v\n", err)
}
buf, err := json.MarshalIndent(&sn, "", " ")
buf, err := json.MarshalIndent(sn, "", " ")
if err != nil {
return err
}
@@ -101,19 +93,12 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "key":
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
key, err := repository.LoadKey(ctx, repo, id)
if err != nil {
return err
}
key := &repository.Key{}
err = json.Unmarshal(buf, key)
if err != nil {
return err
}
buf, err = json.MarshalIndent(&key, "", " ")
buf, err := json.MarshalIndent(&key, "", " ")
if err != nil {
return err
}
@@ -129,7 +114,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "lock":
lock, err := restic.LoadLock(gopts.ctx, repo, id)
lock, err := restic.LoadLock(ctx, repo, id)
if err != nil {
return err
}
@@ -144,7 +129,7 @@ func runCat(gopts GlobalOptions, args []string) error {
case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
return err
}
@@ -158,7 +143,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return err
case "blob":
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@@ -169,7 +154,7 @@ func runCat(gopts GlobalOptions, args []string) error {
continue
}
buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
buf, err := repo.LoadBlob(ctx, t, id, nil)
if err != nil {
return err
}

View File

@@ -1,8 +1,9 @@
package main
import (
"io/ioutil"
"context"
"math/rand"
"os"
"strconv"
"strings"
"sync"
@@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runCheck(checkOptions, globalOptions, args)
return runCheck(cmd.Context(), checkOptions, globalOptions, args)
},
PreRunE: func(cmd *cobra.Command, args []string) error {
return checkFlags(checkOptions)
@@ -57,7 +58,13 @@ func init() {
f := cmdCheck.Flags()
f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
var ignored bool
f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
if err != nil {
// MarkDeprecated only returns an error when the flag is not found
panic(err)
}
f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
}
@@ -142,10 +149,10 @@ func parsePercentage(s string) (float64, error) {
// prepareCheckCache configures a special cache directory for check.
//
// * if --with-cache is specified, the default cache is used
// * if the user explicitly requested --no-cache, we don't use any cache
// * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
// * by default, we use a cache in a temporary directory that is deleted after the check
// - if --with-cache is specified, the default cache is used
// - if the user explicitly requested --no-cache, we don't use any cache
// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
// - by default, we use a cache in a temporary directory that is deleted after the check
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
cleanup = func() {}
if opts.WithCache {
@@ -164,7 +171,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
}
// use a cache in a temporary directory
tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
if err != nil {
// if an error occurs, don't use any cache
Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
@@ -185,25 +192,26 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
return cleanup
}
func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
cleanup := prepareCheckCache(opts, &gopts)
AddCleanupHandler(func() error {
AddCleanupHandler(func(code int) (int, error) {
cleanup()
return nil
return code, nil
})
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n")
lock, err := lockRepoExclusive(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -211,20 +219,36 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
}
chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(ctx)
if err != nil {
return err
}
Verbosef("load indexes\n")
hints, errs := chkr.LoadIndex(gopts.ctx)
hints, errs := chkr.LoadIndex(ctx)
dupFound := false
errorsFound := false
suggestIndexRebuild := false
mixedFound := false
for _, hint := range hints {
Printf("%v\n", hint)
if _, ok := hint.(checker.ErrDuplicatePacks); ok {
dupFound = true
switch hint.(type) {
case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
Printf("%v\n", hint)
suggestIndexRebuild = true
case *checker.ErrMixedPack:
Printf("%v\n", hint)
mixedFound = true
default:
Warnf("error: %v\n", hint)
errorsFound = true
}
}
if dupFound {
Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
if suggestIndexRebuild {
Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
}
if mixedFound {
Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
}
if len(errs) > 0 {
@@ -234,25 +258,26 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("LoadIndex returned errors")
}
errorsFound := false
orphanedPacks := 0
errChan := make(chan error)
Verbosef("check all packs\n")
go chkr.Packs(gopts.ctx, errChan)
go chkr.Packs(ctx, errChan)
for err := range errChan {
if checker.IsOrphanedPack(err) {
orphanedPacks++
Verbosef("%v\n", err)
continue
} else if err == checker.ErrLegacyLayout {
Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
} else {
errorsFound = true
Warnf("%v\n", err)
}
errorsFound = true
Warnf("%v\n", err)
}
if orphanedPacks > 0 {
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
}
Verbosef("check snapshots, trees and blobs\n")
@@ -264,13 +289,17 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
defer wg.Done()
bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
defer bar.Done()
chkr.Structure(gopts.ctx, bar, errChan)
chkr.Structure(ctx, bar, errChan)
}()
for err := range errChan {
errorsFound = true
if e, ok := err.(checker.TreeError); ok {
Warnf("error for tree %v:\n", e.ID.Str())
if e, ok := err.(*checker.TreeError); ok {
var clean string
if stdoutCanUpdateStatus() {
clean = clearLine(0)
}
Warnf(clean+"error for tree %v:\n", e.ID.Str())
for _, treeErr := range e.Errors {
Warnf(" %v\n", treeErr)
}
@@ -285,7 +314,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
wg.Wait()
if opts.CheckUnused {
for _, id := range chkr.UnusedBlobs(gopts.ctx) {
for _, id := range chkr.UnusedBlobs(ctx) {
Verbosef("unused blob %v\n", id)
errorsFound = true
}
@@ -297,7 +326,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
p := newProgressMax(!gopts.Quiet, packCount, "packs")
errChan := make(chan error)
go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan {
errorsFound = true

View File

@@ -4,7 +4,9 @@ import (
"context"
"fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"golang.org/x/sync/errgroup"
@@ -30,16 +32,14 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
new destination repository using the "init" command.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runCopy(copyOptions, globalOptions, args)
return runCopy(cmd.Context(), copyOptions, globalOptions, args)
},
}
// CopyOptions bundles all options for the copy command.
type CopyOptions struct {
secondaryRepoOptions
Hosts []string
Tags restic.TagLists
Paths []string
snapshotFilterOptions
}
var copyOptions CopyOptions
@@ -48,45 +48,55 @@ func init() {
cmdRoot.AddCommand(cmdCopy)
f := cmdCopy.Flags()
initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots to")
f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true)
}
func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
dstGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
if err != nil {
return err
}
if isFromRepo {
// swap global options, if the secondary repo was set via from-repo
gopts, secondaryGopts = secondaryGopts, gopts
}
srcRepo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
srcRepo, err := OpenRepository(gopts)
if err != nil {
return err
}
dstRepo, err := OpenRepository(dstGopts)
dstRepo, err := OpenRepository(ctx, secondaryGopts)
if err != nil {
return err
}
if !gopts.NoLock {
srcLock, err := lockRepo(ctx, srcRepo)
var srcLock *restic.Lock
srcLock, ctx, err = lockRepo(ctx, srcRepo)
defer unlockRepo(srcLock)
if err != nil {
return err
}
}
dstLock, err := lockRepo(ctx, dstRepo)
dstLock, ctx, err := lockRepo(ctx, dstRepo)
defer unlockRepo(dstLock)
if err != nil {
return err
}
srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
debug.Log("Loading source index")
if err := srcRepo.LoadIndex(ctx); err != nil {
return err
@@ -98,7 +108,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
}
dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
if sn.Original != nil && !sn.Original.IsNull() {
dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
}
@@ -109,8 +119,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
// remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
@@ -121,7 +130,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
Verbosef("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
@@ -130,25 +140,20 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
continue
}
}
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
Verbosef(" copy started, this may take a while...\n")
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
return err
}
debug.Log("tree copied")
if err = dstRepo.Flush(ctx); err != nil {
return err
}
debug.Log("flushed packs and saved index")
// save snapshot
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
if err != nil {
return err
}
@@ -176,82 +181,61 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
return true
}
const numCopyWorkers = 8
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.IDSet, rootTreeID restic.ID) error {
visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
idChan := make(chan restic.ID)
wg, ctx := errgroup.WithContext(ctx)
wg, wgCtx := errgroup.WithContext(ctx)
treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
visited := visitedTrees.Has(treeID)
visitedTrees.Insert(treeID)
return visited
}, nil)
copyBlobs := restic.NewBlobSet()
packList := restic.NewIDSet()
enqueue := func(h restic.BlobHandle) {
pb := srcRepo.Index().Lookup(h)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
}
}
wg.Go(func() error {
defer close(idChan)
// reused buffer
var buf []byte
for tree := range treeStream {
if tree.Error != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
}
// Do we already have this tree blob?
if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
if !dstRepo.Index().Has(treeHandle) {
// copy raw tree bytes to avoid problems if the serialization changes
var err error
buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
if err != nil {
return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
}
_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
if err != nil {
return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
}
enqueue(treeHandle)
}
for _, entry := range tree.Nodes {
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range entry.Content {
select {
case idChan <- blobID:
case <-ctx.Done():
return ctx.Err()
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
if !dstRepo.Index().Has(h) {
enqueue(h)
}
}
}
}
return nil
})
for i := 0; i < numCopyWorkers; i++ {
wg.Go(func() error {
// reused buffer
var buf []byte
for blobID := range idChan {
// Do we already have this data blob?
if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
continue
}
debug.Log("Copying blob %s\n", blobID.Str())
var err error
buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
if err != nil {
return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
}
_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
if err != nil {
return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
}
}
return nil
})
err := wg.Wait()
if err != nil {
return err
}
return wg.Wait()
bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
bar.Done()
return err
}

View File

@@ -1,3 +1,4 @@
//go:build debug
// +build debug
package main
@@ -12,14 +13,17 @@ import (
"os"
"runtime"
"sort"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -44,19 +48,21 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugDump(globalOptions, args)
return runDebugDump(cmd.Context(), globalOptions, args)
},
}
var tryRepair bool
var repairByte bool
var extractPack bool
var reuploadBlobs bool
func init() {
cmdRoot.AddCommand(cmdDebug)
cmdDebug.AddCommand(cmdDebugDump)
cmdDebug.AddCommand(cmdDebugExamine)
cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
}
@@ -72,7 +78,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
}
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil {
return err
}
@@ -100,10 +106,9 @@ type Blob struct {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err)
return nil
@@ -122,12 +127,14 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
}
m.Lock()
defer m.Unlock()
return prettyPrintJSON(wr, p)
})
}
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err
@@ -137,18 +144,19 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro
})
}
func runDebugDump(gopts GlobalOptions, args []string) error {
func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -159,20 +167,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
switch tpe {
case "indexes":
return dumpIndexes(gopts.ctx, repo, gopts.stdout)
return dumpIndexes(ctx, repo, gopts.stdout)
case "snapshots":
return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
return debugPrintSnapshots(ctx, repo, gopts.stdout)
case "packs":
return printPacks(gopts.ctx, repo, gopts.stdout)
return printPacks(ctx, repo, gopts.stdout)
case "all":
Printf("snapshots:\n")
err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
err := debugPrintSnapshots(ctx, repo, gopts.stdout)
if err != nil {
return err
}
Printf("\nindexes:\n")
err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
err = dumpIndexes(ctx, repo, gopts.stdout)
if err != nil {
return err
}
@@ -188,7 +196,7 @@ var cmdDebugExamine = &cobra.Command{
Short: "Examine a pack file",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugExamine(globalOptions, args)
return runDebugExamine(cmd.Context(), globalOptions, args)
},
}
@@ -307,76 +315,104 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
return out
}
func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
be := repo.Backend()
h := restic.Handle{
Name: pack.String(),
Name: packID.String(),
Type: restic.PackFile,
}
for _, blob := range list {
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
buf := make([]byte, blob.Length)
err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
n, err := io.ReadFull(rd, buf)
if err != nil {
return fmt.Errorf("read error after %d bytes: %v", n, err)
}
return nil
})
if err != nil {
Warnf("error read: %v\n", err)
continue
}
key := repo.Key()
wg, ctx := errgroup.WithContext(ctx)
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
if err != nil {
Warnf("error decrypting blob: %v\n", err)
var plain []byte
if tryRepair || repairByte {
plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
}
var prefix string
if plain != nil {
id := restic.Hash(plain)
if !id.Equal(blob.ID) {
Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
prefix = "repaired-wrong-hash-"
} else {
Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
prefix = "repaired-"
}
} else {
plain = decryptUnsigned(ctx, key, buf)
prefix = "damaged-"
}
err = storePlainBlob(blob.ID, prefix, plain)
if err != nil {
return err
}
continue
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
prefix = "correct-"
}
if extractPack {
err = storePlainBlob(id, prefix, plaintext)
if err != nil {
return err
}
}
if reuploadBlobs {
repo.StartPackUploader(ctx, wg)
}
return nil
wg.Go(func() error {
for _, blob := range list {
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
buf := make([]byte, blob.Length)
err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
n, err := io.ReadFull(rd, buf)
if err != nil {
return fmt.Errorf("read error after %d bytes: %v", n, err)
}
return nil
})
if err != nil {
Warnf("error read: %v\n", err)
continue
}
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
Warnf("error decrypting blob: %v\n", err)
if tryRepair || repairByte {
plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(ctx, key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
Printf(" failed to decompress blob %v\n", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if extractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext)
if err != nil {
return err
}
}
if reuploadBlobs {
_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
Printf(" uploaded %v %v\n", blob.Type, id)
}
}
if reuploadBlobs {
return repo.Flush(ctx)
}
return nil
})
return wg.Wait()
}
func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
@@ -401,13 +437,21 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
return nil
}
func runDebugExamine(gopts GlobalOptions, args []string) error {
func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
ids := make([]restic.ID, 0)
for _, name := range args {
id, err := restic.ParseID(name)
if err != nil {
Warnf("error: %v\n", err)
continue
id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
if err != nil {
Warnf("error: %v\n", err)
continue
}
}
ids = append(ids, id)
}
@@ -416,26 +460,22 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
return errors.Fatal("no pack files to examine")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
for _, id := range ids {
err := examinePack(gopts.ctx, repo, id)
err := examinePack(ctx, repo, id)
if err != nil {
Warnf("error: %v\n", err)
}
@@ -475,27 +515,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
blobsLoaded := false
// examine all data the indexes have for the pack file
for _, idx := range repo.Index().(*repository.MasterIndex).All() {
idxIDs, err := idx.IDs()
if err != nil {
idxIDs = restic.IDs{}
}
blobs := idx.ListPack(id)
for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
Printf(" index %v:\n", idxIDs)
checkPackSize(blobs, fi.Size)
// convert list of blobs to []restic.Blob
var list []restic.Blob
for _, b := range blobs {
list = append(list, b.Blob)
}
checkPackSize(list, fi.Size)
err = loadBlobs(ctx, repo, id, list)
err = loadBlobs(ctx, repo, id, blobs)
if err != nil {
Warnf("error: %v\n", err)
} else {
@@ -506,7 +534,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
Printf(" ========================================\n")
Printf(" inspect the pack itself\n")
blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
blobs, _, err := repo.ListPack(ctx, id, fi.Size)
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
@@ -531,17 +559,13 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) {
if offset != uint64(pb.Offset) {
Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset)
}
offset += uint64(pb.Length)
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
// length in uint32 little endian
size += 4
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
Printf(" file sizes do not match: computed %v, file size is %v\n", size, fileSize)
} else {
Printf(" file sizes match\n")
}

View File

@@ -7,10 +7,11 @@ import (
"reflect"
"sort"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
)
@@ -35,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDiff(diffOptions, globalOptions, args)
return runDiff(cmd.Context(), diffOptions, globalOptions, args)
},
}
@@ -53,12 +54,12 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}
func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
id, err := restic.FindSnapshot(ctx, repo, desc)
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
sn, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, errors.Fatal(err.Error())
}
return restic.LoadSnapshot(ctx, repo, id)
return sn, err
}
// Comparer collects all things needed to compare two snapshots.
@@ -160,7 +161,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat)
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
debug.Log("print %v tree %v", mode, id)
tree, err := c.repo.LoadTree(ctx, id)
tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
@@ -170,7 +171,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
if node.Type == "dir" {
name += "/"
}
c.printChange(NewChange(name, "+"))
c.printChange(NewChange(name, mode))
stats.Add(node)
addBlobs(blobs, node)
@@ -187,7 +188,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
debug.Log("print tree %v", id)
tree, err := c.repo.LoadTree(ctx, id)
tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
@@ -231,12 +232,12 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
debug.Log("diffing %v to %v", id1, id2)
tree1, err := c.repo.LoadTree(ctx, id1)
tree1, err := restic.LoadTree(ctx, c.repo, id1)
if err != nil {
return err
}
tree2, err := c.repo.LoadTree(ctx, id2)
tree2, err := restic.LoadTree(ctx, c.repo, id2)
if err != nil {
return err
}
@@ -321,37 +322,36 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return nil
}
func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs")
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if err = repo.LoadIndex(ctx); err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
sn1, err := loadSnapshot(ctx, repo, args[0])
// cache snapshots listing
be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
sn1, err := loadSnapshot(ctx, be, repo, args[0])
if err != nil {
return err
}
sn2, err := loadSnapshot(ctx, repo, args[1])
sn2, err := loadSnapshot(ctx, be, repo, args[1])
if err != nil {
return err
}
@@ -360,6 +360,10 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
}
if err = repo.LoadIndex(ctx); err != nil {
return err
}
if sn1.Tree == nil {
return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
}
@@ -422,8 +426,8 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
Printf(" Added: %-5s\n", ui.FormatBytes(uint64(stats.Added.Bytes)))
Printf(" Removed: %-5s\n", ui.FormatBytes(uint64(stats.Removed.Bytes)))
}
return nil

View File

@@ -34,15 +34,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDump(dumpOptions, globalOptions, args)
return runDump(cmd.Context(), dumpOptions, globalOptions, args)
},
}
// DumpOptions collects all options for the dump command.
type DumpOptions struct {
Hosts []string
Paths []string
Tags restic.TagLists
snapshotFilterOptions
Archive string
}
@@ -52,9 +50,7 @@ func init() {
cmdRoot.AddCommand(cmdDump)
flags := cmdDump.Flags()
flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions)
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
}
@@ -87,7 +83,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
case l == 1 && dump.IsFile(node):
return d.WriteNode(ctx, node)
case l > 1 && dump.IsDir(node):
subtree, err := repo.LoadTree(ctx, *node.Subtree)
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
return errors.Wrapf(err, "cannot load subtree for %q", item)
}
@@ -96,7 +92,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
if err := checkStdoutArchive(); err != nil {
return err
}
subtree, err := repo.LoadTree(ctx, *node.Subtree)
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
return err
}
@@ -111,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
return fmt.Errorf("path %q not found in snapshot", item)
}
func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx
func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified")
}
@@ -131,52 +125,39 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
splittedPath := splitPath(path.Clean(pathToPrint))
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
var id restic.ID
if snapshotIDString == "latest" {
id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
if err != nil {
Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
}
} else {
id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
if err != nil {
Exitf(1, "invalid id %q: %v", snapshotIDString, err)
}
}
sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
if err != nil {
Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
}
tree, err := repo.LoadTree(ctx, *sn.Tree)
if err != nil {
Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
}
d := dump.New(opts.Archive, repo, os.Stdout)
err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
if err != nil {
Exitf(2, "cannot dump file: %v", err)
return errors.Fatalf("cannot dump file: %v", err)
}
return nil

View File

@@ -9,6 +9,7 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
@@ -37,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runFind(findOptions, globalOptions, args)
return runFind(cmd.Context(), findOptions, globalOptions, args)
},
}
@@ -50,9 +51,7 @@ type FindOptions struct {
PackID, ShowPackID bool
CaseInsensitive bool
ListLong bool
Hosts []string
Paths []string
Tags restic.TagLists
snapshotFilterOptions
}
var findOptions FindOptions
@@ -71,9 +70,7 @@ func init() {
f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
f.StringArrayVarP(&findOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true)
}
type findPattern struct {
@@ -470,7 +467,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
// remember which packs were found in the index
indexPackIDs := make(map[string]struct{})
for pb := range f.repo.Index().Each(wctx) {
f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
idStr := pb.PackID.String()
// keep entry in packIDs as Each() returns individual index entries
matchingID := false
@@ -488,7 +485,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
f.blobIDs[pb.ID.String()] = struct{}{}
indexPackIDs[idStr] = struct{}{}
}
}
})
for id := range indexPackIDs {
delete(packIDs, id)
@@ -537,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) {
}
}
func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("wrong number of arguments")
}
@@ -571,25 +568,28 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("cannot have several ID types")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
if err = repo.LoadIndex(gopts.ctx); err != nil {
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
if err = repo.LoadIndex(ctx); err != nil {
return err
}
f := &Finder{
repo: repo,
@@ -618,7 +618,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
}
}
for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
if f.blobIDs != nil || f.treeIDs != nil {
if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
return err

View File

@@ -14,11 +14,16 @@ var cmdForget = &cobra.Command{
Use: "forget [flags] [snapshot ID] [...]",
Short: "Remove snapshots from the repository",
Long: `
The "forget" command removes snapshots according to a policy. Please note that
this command really only deletes the snapshot object in the repository, which
is a reference to data stored there. In order to remove the unreferenced data
after "forget" was run successfully, see the "prune" command. Please also read
the documentation for "forget" to learn about important security considerations.
The "forget" command removes snapshots according to a policy. All snapshots are
first divided into groups according to "--group-by", and after that the policy
specified by the "--keep-*" options is applied to each group individually.
Please note that this command really only deletes the snapshot object in the
repository, which is a reference to data stored there. In order to remove the
unreferenced data after "forget" was run successfully, see the "prune" command.
Please also read the documentation for "forget" to learn about some important
security considerations.
EXIT STATUS
===========
@@ -27,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runForget(forgetOptions, globalOptions, args)
return runForget(cmd.Context(), forgetOptions, globalOptions, args)
},
}
@@ -47,9 +52,7 @@ type ForgetOptions struct {
WithinYearly restic.Duration
KeepTags restic.TagLists
Hosts []string
Tags restic.TagLists
Paths []string
snapshotFilterOptions
Compact bool
// Grouping
@@ -76,9 +79,9 @@ func init() {
f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false)
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
err := f.MarkDeprecated("hostname", "use --host")
if err != nil {
@@ -86,12 +89,9 @@ func init() {
panic(err)
}
f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
@@ -99,13 +99,13 @@ func init() {
addPruneOptions(cmdForget)
}
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
err := verifyPruneOptions(&pruneOptions)
if err != nil {
return err
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -115,20 +115,18 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
}
if !opts.DryRun || !gopts.NoLock {
lock, err := lockRepoExclusive(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
snapshots = append(snapshots, sn)
}
@@ -219,7 +217,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 {
if !opts.DryRun {
err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile)
err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
if err != nil {
return err
}
@@ -239,10 +237,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 && opts.Prune {
if !gopts.JSON {
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
if opts.DryRun {
Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
} else {
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
}
pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
}
return nil

View File

@@ -10,7 +10,7 @@ import (
var cmdGenerate = &cobra.Command{
Use: "generate [flags]",
Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
Long: `
The "generate" command writes automatically generated files (like the man pages
and the auto-completion files for bash, fish and zsh).
@@ -25,10 +25,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
}
type generateOptions struct {
ManDir string
BashCompletionFile string
FishCompletionFile string
ZSHCompletionFile string
ManDir string
BashCompletionFile string
FishCompletionFile string
ZSHCompletionFile string
PowerShellCompletionFile string
}
var genOpts generateOptions
@@ -40,6 +41,7 @@ func init() {
fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file`")
}
func writeManpages(dir string) error {
@@ -75,6 +77,11 @@ func writeZSHCompletion(file string) error {
return cmdRoot.GenZshCompletionFile(file)
}
func writePowerShellCompletion(file string) error {
Verbosef("writing powershell completion file to %v\n", file)
return cmdRoot.GenPowerShellCompletionFile(file)
}
func runGenerate(cmd *cobra.Command, args []string) error {
if genOpts.ManDir != "" {
err := writeManpages(genOpts.ManDir)
@@ -104,6 +111,13 @@ func runGenerate(cmd *cobra.Command, args []string) error {
}
}
if genOpts.PowerShellCompletionFile != "" {
err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
if err != nil {
return err
}
}
var empty generateOptions
if genOpts == empty {
return errors.Fatal("nothing to do, please specify at least one output file/dir")

View File

@@ -1,10 +1,15 @@
package main
import (
"context"
"encoding/json"
"strconv"
"github.com/restic/chunker"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
)
@@ -22,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runInit(initOptions, globalOptions, args)
return runInit(cmd.Context(), initOptions, globalOptions, args)
},
}
@@ -30,6 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
type InitOptions struct {
secondaryRepoOptions
CopyChunkerParameters bool
RepositoryVersion string
}
var initOptions InitOptions
@@ -40,10 +46,27 @@ func init() {
f := cmdInit.Flags()
initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from")
f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
}
func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
var version uint
if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
version = restic.MaxRepoVersion
} else if opts.RepositoryVersion == "stable" {
version = restic.StableRepoVersion
} else {
v, err := strconv.ParseUint(opts.RepositoryVersion, 10, 32)
if err != nil {
return errors.Fatal("invalid repository version")
}
version = uint(v)
}
if version < restic.MinRepoVersion || version > restic.MaxRepoVersion {
return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
}
chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
if err != nil {
return err
}
@@ -60,35 +83,51 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err
}
be, err := create(repo, gopts.extended)
be, err := create(ctx, repo, gopts.extended)
if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
s := repository.New(be)
s, err := repository.New(be, repository.Options{
Compression: gopts.Compression,
PackSize: gopts.PackSize * 1024 * 1024,
})
if err != nil {
return err
}
err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
Verbosef("\n")
Verbosef("Please note that knowledge of your password is required to access\n")
Verbosef("the repository. Losing your password means that your data is\n")
Verbosef("irrecoverably lost.\n")
if !gopts.JSON {
Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
Verbosef("\n")
Verbosef("Please note that knowledge of your password is required to access\n")
Verbosef("the repository. Losing your password means that your data is\n")
Verbosef("irrecoverably lost.\n")
} else {
status := initSuccess{
MessageType: "initialized",
ID: s.Config().ID,
Repository: location.StripPassword(gopts.Repo),
}
return json.NewEncoder(gopts.stdout).Encode(status)
}
return nil
}
func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
if opts.CopyChunkerParameters {
otherGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
if err != nil {
return nil, err
}
otherRepo, err := OpenRepository(otherGopts)
otherRepo, err := OpenRepository(ctx, otherGopts)
if err != nil {
return nil, err
}
@@ -97,8 +136,14 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker
return &pol, nil
}
if opts.Repo != "" {
if opts.Repo != "" || opts.RepositoryFile != "" || opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" {
return nil, errors.Fatal("Secondary repository must only be specified when copying the chunker parameters")
}
return nil, nil
}
type initSuccess struct {
MessageType string `json:"message_type"` // "initialized"
ID string `json:"id"`
Repository string `json:"repository"`
}

View File

@@ -3,9 +3,9 @@ package main
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"strings"
"sync"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKey(globalOptions, args)
return runKey(cmd.Context(), globalOptions, args)
},
}
@@ -56,23 +56,26 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
Created string `json:"created"`
}
var m sync.Mutex
var keys []keyInfo
err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
k, err := repository.LoadKey(ctx, s, id.String())
err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
k, err := repository.LoadKey(ctx, s, id)
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
return nil
}
key := keyInfo{
Current: id.String() == s.KeyName(),
Current: id == s.KeyID(),
ID: id.Str(),
UserName: k.Username,
HostName: k.Hostname,
Created: k.Created.Local().Format(TimeFormat),
}
m.Lock()
defer m.Unlock()
keys = append(keys, key)
return nil
})
@@ -120,18 +123,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
"enter password again: ")
}
func addKey(gopts GlobalOptions, repo *repository.Repository) error {
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key())
id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
@@ -141,40 +144,40 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
return nil
}
func deleteKey(ctx context.Context, repo *repository.Repository, name string) error {
if name == repo.KeyName() {
func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
if id == repo.KeyID() {
return errors.Fatal("refusing to remove key currently used to access repository")
}
h := restic.Handle{Type: restic.KeyFile, Name: name}
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
return err
}
Verbosef("removed key %v\n", name)
Verbosef("removed key %v\n", id)
return nil
}
func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key())
id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
oldID := repo.KeyName()
oldID := repo.KeyID()
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
h := restic.Handle{Type: restic.KeyFile, Name: oldID}
err = repo.Backend().Remove(gopts.ctx, h)
h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
err = repo.Backend().Remove(ctx, h)
if err != nil {
return err
}
@@ -187,32 +190,29 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
// Verify new key to make sure it really works. A broken key can render the
// whole repository inaccessible
err := repo.SearchKey(ctx, pw, 0, key.Name())
err := repo.SearchKey(ctx, pw, 0, key.ID().String())
if err != nil {
// the key is invalid, try to remove it
h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
_ = repo.Backend().Remove(ctx, h)
return errors.Fatalf("failed to access repository with new key: %v", err)
}
return nil
}
func runKey(gopts GlobalOptions, args []string) error {
func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
return errors.Fatal("wrong number of arguments")
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
switch args[0] {
case "list":
lock, err := lockRepo(ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -220,15 +220,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return listKeys(ctx, repo, gopts)
case "add":
lock, err := lockRepo(ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return addKey(gopts, repo)
return addKey(ctx, repo, gopts)
case "remove":
lock, err := lockRepoExclusive(ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -239,22 +239,22 @@ func runKey(gopts GlobalOptions, args []string) error {
return err
}
return deleteKey(gopts.ctx, repo, id)
return deleteKey(ctx, repo, id)
case "passwd":
lock, err := lockRepoExclusive(ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return changePassword(gopts, repo)
return changePassword(ctx, repo, gopts)
}
return nil
}
func loadPasswordFromFile(pwdFile string) (string, error) {
s, err := ioutil.ReadFile(pwdFile)
s, err := os.ReadFile(pwdFile)
if os.IsNotExist(err) {
return "", errors.Fatalf("%s does not exist", pwdFile)
}

View File

@@ -1,8 +1,10 @@
package main
import (
"context"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
@@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runList(cmd, globalOptions, args)
return runList(cmd.Context(), cmd, globalOptions, args)
},
}
@@ -29,18 +31,19 @@ func init() {
cmdRoot.AddCommand(cmdList)
}
func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified, usage: " + cmd.Use)
}
repo, err := OpenRepository(opts)
repo, err := OpenRepository(ctx, opts)
if err != nil {
return err
}
if !opts.NoLock && args[0] != "locks" {
lock, err := lockRepo(opts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -60,20 +63,20 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
case "locks":
t = restic.LockFile
case "blobs":
return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}
for blobs := range idx.Each(opts.ctx) {
idx.Each(ctx, func(blobs restic.PackedBlob) {
Printf("%v %v\n", blobs.Type, blobs.ID)
}
})
return nil
})
default:
return errors.Fatal("invalid type")
}
return repo.List(opts.ctx, t, func(id restic.ID, size int64) error {
return repo.List(ctx, t, func(id restic.ID, size int64) error {
Printf("%s\n", id)
return nil
})

View File

@@ -9,6 +9,7 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@@ -41,16 +42,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runLs(lsOptions, globalOptions, args)
return runLs(cmd.Context(), lsOptions, globalOptions, args)
},
}
// LsOptions collects all options for the ls command.
type LsOptions struct {
ListLong bool
Hosts []string
Tags restic.TagLists
Paths []string
ListLong bool
snapshotFilterOptions
Recursive bool
}
@@ -60,10 +59,8 @@ func init() {
cmdRoot.AddCommand(cmdLs)
flags := cmdLs.Flags()
initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions)
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
}
@@ -114,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n)
}
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
@@ -164,17 +161,19 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
return false
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if err = repo.LoadIndex(gopts.ctx); err != nil {
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
if err = repo.LoadIndex(ctx); err != nil {
return err
}
var (
printSnapshot func(sn *restic.Snapshot)
@@ -211,45 +210,48 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
}
}
for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
printSnapshot(sn)
sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0])
if err != nil {
return err
}
err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
if node == nil {
return false, nil
}
if withinDir(nodepath) {
// if we're within a dir, print the node
printNode(nodepath, node)
// if recursive listing is requested, signal the walker that it
// should continue walking recursively
if opts.Recursive {
return false, nil
}
}
// if there's an upcoming match deeper in the tree (but we're not
// there yet), signal the walker to descend into any subdirs
if approachingMatchingTree(nodepath) {
return false, nil
}
// otherwise, signal the walker to not walk recursively into any
// subdirs
if node.Type == "dir" {
return false, walker.ErrSkipNode
}
return false, nil
})
printSnapshot(sn)
err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return err
return false, err
}
if node == nil {
return false, nil
}
if withinDir(nodepath) {
// if we're within a dir, print the node
printNode(nodepath, node)
// if recursive listing is requested, signal the walker that it
// should continue walking recursively
if opts.Recursive {
return false, nil
}
}
// if there's an upcoming match deeper in the tree (but we're not
// there yet), signal the walker to descend into any subdirs
if approachingMatchingTree(nodepath) {
return false, nil
}
// otherwise, signal the walker to not walk recursively into any
// subdirs
if node.Type == "dir" {
return false, walker.ErrSkipNode
}
return false, nil
})
if err != nil {
return err
}
return nil

View File

@@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/restic/restic/internal/migrations"
"github.com/restic/restic/internal/restic"
@@ -8,11 +10,12 @@ import (
)
var cmdMigrate = &cobra.Command{
Use: "migrate [flags] [name]",
Use: "migrate [flags] [migration name] [...]",
Short: "Apply migrations",
Long: `
The "migrate" command applies migrations to a repository. When no migration
name is explicitly given, a list of migrations that can be applied is printed.
The "migrate" command checks which migrations can be applied for a repository
and prints a list with available migration names. If one or more migration
names are specified, these migrations are applied.
EXIT STATUS
===========
@@ -21,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runMigrate(migrateOptions, globalOptions, args)
return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
},
}
@@ -38,44 +41,64 @@ func init() {
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
}
func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
ctx := gopts.ctx
func checkMigrations(ctx context.Context, repo restic.Repository) error {
Printf("available migrations:\n")
found := false
for _, m := range migrations.All {
ok, err := m.Check(ctx, repo)
ok, _, err := m.Check(ctx, repo)
if err != nil {
return err
}
if ok {
Printf(" %v: %v\n", m.Name(), m.Desc())
Printf(" %v\t%v\n", m.Name(), m.Desc())
found = true
}
}
if !found {
Printf("no migrations found\n")
}
return nil
}
func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
ctx := gopts.ctx
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
var firsterr error
for _, name := range args {
for _, m := range migrations.All {
if m.Name() == name {
ok, err := m.Check(ctx, repo)
ok, reason, err := m.Check(ctx, repo)
if err != nil {
return err
}
if !ok {
if !opts.Force {
Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name())
if reason == "" {
reason = "check failed"
}
Warnf("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason)
continue
}
Warnf("check for migration %v failed, continuing anyway\n", m.Name())
}
if m.RepoCheck() {
Printf("checking repository integrity...\n")
checkOptions := CheckOptions{}
checkGopts := gopts
// the repository is already locked
checkGopts.NoLock = true
err = runCheck(ctx, checkOptions, checkGopts, []string{})
if err != nil {
return err
}
}
Printf("applying migration %v...\n", m.Name())
if err = m.Apply(ctx, repo); err != nil {
Warnf("migration %v failed: %v\n", m.Name(), err)
@@ -93,21 +116,21 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return firsterr
}
func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(gopts.ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
if len(args) == 0 {
return checkMigrations(opts, gopts, repo)
return checkMigrations(ctx, repo)
}
return applyMigrations(opts, gopts, repo, args)
return applyMigrations(ctx, opts, gopts, repo, args)
}

View File

@@ -1,8 +1,10 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package main
import (
"context"
"os"
"strings"
"time"
@@ -16,8 +18,8 @@ import (
resticfs "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/fuse"
systemFuse "bazil.org/fuse"
"bazil.org/fuse/fs"
systemFuse "github.com/anacrolix/fuse"
"github.com/anacrolix/fuse/fs"
)
var cmdMount = &cobra.Command{
@@ -30,10 +32,13 @@ read-only mount.
Snapshot Directories
====================
If you need a different template for all directories that contain snapshots,
you can pass a template via --snapshot-template. Example without colons:
If you need a different template for directories that contain snapshots,
you can pass a time template via --time-template and path templates via
--path-template.
--snapshot-template "2006-01-02_15-04-05"
Example time template without colons:
--time-template "2006-01-02_15-04-05"
You need to specify a sample format for exactly the following timestamp:
@@ -42,6 +47,20 @@ You need to specify a sample format for exactly the following timestamp:
For details please see the documentation for time.Format() at:
https://godoc.org/time#Time.Format
For path templates, you can use the following patterns which will be replaced:
%i by short snapshot ID
%I by long snapshot ID
%u by username
%h by hostname
%t by tags
%T by timestamp as specified by --time-template
The default path templates are:
"ids/%i"
"snapshots/%T"
"hosts/%h/%T"
"tags/%t/%T"
EXIT STATUS
===========
@@ -49,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runMount(mountOptions, globalOptions, args)
return runMount(cmd.Context(), mountOptions, globalOptions, args)
},
}
@@ -58,10 +77,9 @@ type MountOptions struct {
OwnerRoot bool
AllowOther bool
NoDefaultPermissions bool
Hosts []string
Tags restic.TagLists
Paths []string
SnapshotTemplate string
snapshotFilterOptions
TimeTemplate string
PathTemplates []string
}
var mountOptions MountOptions
@@ -74,20 +92,23 @@ func init() {
mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
mountFlags.StringArrayVarP(&mountOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`)
mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true)
mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
}
func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.SnapshotTemplate == "" {
return errors.Fatal("snapshot template string cannot be empty")
func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.TimeTemplate == "" {
return errors.Fatal("time template string cannot be empty")
}
if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") {
return errors.Fatal("time template string cannot start or end with '/'")
}
if len(args) == 0 {
return errors.Fatal("wrong number of parameters")
}
@@ -95,27 +116,28 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
debug.Log("start mount")
defer debug.Log("finish mount")
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
mountpoint := args[0]
if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
return err
}
@@ -134,13 +156,17 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
}
}
AddCleanupHandler(func() error {
AddCleanupHandler(func(code int) (int, error) {
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
err := umount(mountpoint)
if err != nil {
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
}
return nil
// replace error code of sigint
if code == 130 {
code = 0
}
return code, nil
})
c, err := systemFuse.Mount(mountpoint, mountOptions...)
@@ -153,11 +179,12 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
}
cfg := fuse.Config{
OwnerIsRoot: opts.OwnerRoot,
Hosts: opts.Hosts,
Tags: opts.Tags,
Paths: opts.Paths,
SnapshotTemplate: opts.SnapshotTemplate,
OwnerIsRoot: opts.OwnerRoot,
Hosts: opts.Hosts,
Tags: opts.Tags,
Paths: opts.Paths,
TimeTemplate: opts.TimeTemplate,
PathTemplates: opts.PathTemplates,
}
root := fuse.NewRoot(repo, cfg)

Some files were not shown because too many files have changed in this diff Show More