TheArchive/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-12-03 20:11:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

restic prune: Merge three loops over the index

Browse Source 
There were three loops over the index in restic prune, to find
duplicates, to determine sizes (in pack.Size) and to generate packInfos.
These three are now one loop. This way, prune doesn't need to construct
a set of duplicate blobs, pack.Size doesn't need to contain special
logic for prune's use case (the onlyHdr argument) and pack.Size doesn't
need to construct a map only to have it immediately transformed into a
different map.

Some quick testing on a 160GiB local repo doesn't show running time or
memory use of restic prune --dry-run changing significantly.
This commit is contained in:
greatroar
2022-06-05 10:14:32 +02:00
parent b7c990871f
commit 8bdfcf779f
4 changed files with 42 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/pack/pack.go
View File

@@ -177,8 +177,8 @@ var (
const (
// size of the header-length field at the end of the file; it is a uint32
headerLengthSize = 4
// headerSize is the header's constant overhead (independent of #entries)
headerSize = headerLengthSize + crypto.Extension
// HeaderSize is the header's constant overhead (independent of #entries)
HeaderSize = headerLengthSize + crypto.Extension
// MaxHeaderSize is the max size of header including header-length field
MaxHeaderSize = 16*1024*1024 + headerLengthSize
@@ -242,7 +242,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
// eagerly download eagerEntries header entries as part of header-length request.
// only make second request if actual number of entries is greater than eagerEntries
eagerSize := eagerEntries*int(entrySize) + headerSize
eagerSize := eagerEntries*int(entrySize) + HeaderSize
b, c, err := readRecords(rd, size, eagerSize)
if err != nil {
return nil, err
@@ -349,7 +349,7 @@ func CalculateEntrySize(blob restic.Blob) int {
}
func CalculateHeaderSize(blobs []restic.Blob) int {
size := headerSize
size := HeaderSize
for _, blob := range blobs {
size += CalculateEntrySize(blob)
}
@@ -357,20 +357,17 @@ func CalculateHeaderSize(blobs []restic.Blob) int {
}
// Size returns the size of all packs computed by index information.
// If onlyHdr is set to true, only the size of the header is returned
// Note that this function only gives correct sizes, if there are no
// duplicates in the index.
func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 {
func Size(ctx context.Context, mi restic.MasterIndex) map[restic.ID]int64 {
packSize := make(map[restic.ID]int64)
for blob := range mi.Each(ctx) {
size, ok := packSize[blob.PackID]
if !ok {
size = headerSize
}
if !onlyHdr {
size += int64(blob.Length)
size = HeaderSize
}
size += int64(blob.Length)
packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
}