mirror of
https://github.com/restic/restic.git
synced 2025-10-10 14:31:56 +00:00
move include/exclude options to filter package
This commit is contained in:
Michael Eischer
162
internal/filter/exclude.go
Normal file
162
internal/filter/exclude.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/textfile"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// RejectByNameFunc is a function that takes a filename of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectByNameFunc func(path string) bool
|
||||
|
||||
// RejectByPattern returns a RejectByNameFunc which rejects files that match
|
||||
// one of the patterns.
|
||||
func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
|
||||
parsedPatterns := ParsePatterns(patterns)
|
||||
return func(item string) bool {
|
||||
matched, err := List(parsedPatterns, item)
|
||||
if err != nil {
|
||||
warnf("error for exclude pattern: %v", err)
|
||||
}
|
||||
|
||||
if matched {
|
||||
debug.Log("path %q excluded by an exclude pattern", item)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// RejectByInsensitivePattern is like RejectByPattern but case insensitive.
|
||||
func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
|
||||
for index, path := range patterns {
|
||||
patterns[index] = strings.ToLower(path)
|
||||
}
|
||||
|
||||
rejFunc := RejectByPattern(patterns, warnf)
|
||||
return func(item string) bool {
|
||||
return rejFunc(strings.ToLower(item))
|
||||
}
|
||||
}
|
||||
|
||||
// readPatternsFromFiles reads all files and returns the list of
|
||||
// patterns. For each line, leading and trailing white space is removed
|
||||
// and comment lines are ignored. For each remaining pattern, environment
|
||||
// variables are resolved. For adding a literal dollar sign ($), write $$ to
|
||||
// the file.
|
||||
func readPatternsFromFiles(files []string) ([]string, error) {
|
||||
getenvOrDollar := func(s string) string {
|
||||
if s == "$" {
|
||||
return "$"
|
||||
}
|
||||
return os.Getenv(s)
|
||||
}
|
||||
|
||||
var patterns []string
|
||||
for _, filename := range files {
|
||||
err := func() (err error) {
|
||||
data, err := textfile.Read(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
// ignore empty lines
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// strip comments
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
line = os.Expand(line, getenvOrDollar)
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
return scanner.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
|
||||
}
|
||||
}
|
||||
return patterns, nil
|
||||
}
|
||||
|
||||
type ExcludePatternOptions struct {
|
||||
Excludes []string
|
||||
InsensitiveExcludes []string
|
||||
ExcludeFiles []string
|
||||
InsensitiveExcludeFiles []string
|
||||
}
|
||||
|
||||
func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) {
|
||||
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
|
||||
}
|
||||
|
||||
func (opts *ExcludePatternOptions) Empty() bool {
|
||||
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
|
||||
}
|
||||
|
||||
func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) {
|
||||
var fs []RejectByNameFunc
|
||||
// add patterns from file
|
||||
if len(opts.ExcludeFiles) > 0 {
|
||||
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(excludePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--exclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.Excludes = append(opts.Excludes, excludePatterns...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludeFiles) > 0 {
|
||||
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(excludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludes) > 0 {
|
||||
if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf))
|
||||
}
|
||||
|
||||
if len(opts.Excludes) > 0 {
|
||||
if err := ValidatePatterns(opts.Excludes); err != nil {
|
||||
return nil, errors.Fatalf("--exclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, RejectByPattern(opts.Excludes, warnf))
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
59
internal/filter/exclude_test.go
Normal file
59
internal/filter/exclude_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRejectByPattern(t *testing.T) {
|
||||
var tests = []struct {
|
||||
filename string
|
||||
reject bool
|
||||
}{
|
||||
{filename: "/home/user/foo.go", reject: true},
|
||||
{filename: "/home/user/foo.c", reject: false},
|
||||
{filename: "/home/user/foobar", reject: false},
|
||||
{filename: "/home/user/foobar/x", reject: true},
|
||||
{filename: "/home/user/README", reject: false},
|
||||
{filename: "/home/user/README.md", reject: true},
|
||||
}
|
||||
|
||||
patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
reject := RejectByPattern(patterns, nil)
|
||||
res := reject(tc.filename)
|
||||
if res != tc.reject {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
tc.filename, tc.reject, res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectByInsensitivePattern(t *testing.T) {
|
||||
var tests = []struct {
|
||||
filename string
|
||||
reject bool
|
||||
}{
|
||||
{filename: "/home/user/foo.GO", reject: true},
|
||||
{filename: "/home/user/foo.c", reject: false},
|
||||
{filename: "/home/user/foobar", reject: false},
|
||||
{filename: "/home/user/FOObar/x", reject: true},
|
||||
{filename: "/home/user/README", reject: false},
|
||||
{filename: "/home/user/readme.md", reject: true},
|
||||
}
|
||||
|
||||
patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
reject := RejectByInsensitivePattern(patterns, nil)
|
||||
res := reject(tc.filename)
|
||||
if res != tc.reject {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
tc.filename, tc.reject, res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
99
internal/filter/include.go
Normal file
99
internal/filter/include.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// IncludeByNameFunc is a function that takes a filename that should be included
|
||||
// in the restore process and returns whether it should be included.
|
||||
type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
|
||||
|
||||
type IncludePatternOptions struct {
|
||||
Includes []string
|
||||
InsensitiveIncludes []string
|
||||
IncludeFiles []string
|
||||
InsensitiveIncludeFiles []string
|
||||
}
|
||||
|
||||
func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
|
||||
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
|
||||
}
|
||||
|
||||
func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
|
||||
var fs []IncludeByNameFunc
|
||||
if len(opts.IncludeFiles) > 0 {
|
||||
includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(includePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--include-file: %s", err)
|
||||
}
|
||||
|
||||
opts.Includes = append(opts.Includes, includePatterns...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveIncludeFiles) > 0 {
|
||||
includePatterns, err := readPatternsFromFiles(opts.InsensitiveIncludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(includePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--iinclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.InsensitiveIncludes = append(opts.InsensitiveIncludes, includePatterns...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveIncludes) > 0 {
|
||||
if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil {
|
||||
return nil, errors.Fatalf("--iinclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf))
|
||||
}
|
||||
|
||||
if len(opts.Includes) > 0 {
|
||||
if err := ValidatePatterns(opts.Includes); err != nil {
|
||||
return nil, errors.Fatalf("--include: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, IncludeByPattern(opts.Includes, warnf))
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// IncludeByPattern returns a IncludeByNameFunc which includes files that match
|
||||
// one of the patterns.
|
||||
func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
|
||||
parsedPatterns := ParsePatterns(patterns)
|
||||
return func(item string) (matched bool, childMayMatch bool) {
|
||||
matched, childMayMatch, err := ListWithChild(parsedPatterns, item)
|
||||
if err != nil {
|
||||
warnf("error for include pattern: %v", err)
|
||||
}
|
||||
|
||||
return matched, childMayMatch
|
||||
}
|
||||
}
|
||||
|
||||
// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
|
||||
// one of the patterns, ignoring the casing of the filenames.
|
||||
func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
|
||||
for index, path := range patterns {
|
||||
patterns[index] = strings.ToLower(path)
|
||||
}
|
||||
|
||||
includeFunc := IncludeByPattern(patterns, warnf)
|
||||
return func(item string) (matched bool, childMayMatch bool) {
|
||||
return includeFunc(strings.ToLower(item))
|
||||
}
|
||||
}
|
||||
59
internal/filter/include_test.go
Normal file
59
internal/filter/include_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIncludeByPattern(t *testing.T) {
|
||||
var tests = []struct {
|
||||
filename string
|
||||
include bool
|
||||
}{
|
||||
{filename: "/home/user/foo.go", include: true},
|
||||
{filename: "/home/user/foo.c", include: false},
|
||||
{filename: "/home/user/foobar", include: false},
|
||||
{filename: "/home/user/foobar/x", include: false},
|
||||
{filename: "/home/user/README", include: false},
|
||||
{filename: "/home/user/README.md", include: true},
|
||||
}
|
||||
|
||||
patterns := []string{"*.go", "README.md"}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.filename, func(t *testing.T) {
|
||||
includeFunc := IncludeByPattern(patterns, nil)
|
||||
matched, _ := includeFunc(tc.filename)
|
||||
if matched != tc.include {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
tc.filename, tc.include, matched)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncludeByInsensitivePattern(t *testing.T) {
|
||||
var tests = []struct {
|
||||
filename string
|
||||
include bool
|
||||
}{
|
||||
{filename: "/home/user/foo.GO", include: true},
|
||||
{filename: "/home/user/foo.c", include: false},
|
||||
{filename: "/home/user/foobar", include: false},
|
||||
{filename: "/home/user/FOObar/x", include: false},
|
||||
{filename: "/home/user/README", include: false},
|
||||
{filename: "/home/user/readme.MD", include: true},
|
||||
}
|
||||
|
||||
patterns := []string{"*.go", "README.md"}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.filename, func(t *testing.T) {
|
||||
includeFunc := IncludeByInsensitivePattern(patterns, nil)
|
||||
matched, _ := includeFunc(tc.filename)
|
||||
if matched != tc.include {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
tc.filename, tc.include, matched)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user