From cfedeca55ed7fcf8411dfc343f8443fd54f7ff1d Mon Sep 17 00:00:00 2001 From: rainu Date: Wed, 19 Apr 2023 13:30:54 +0200 Subject: [PATCH] add whitelist functionality for create command --- README.md | 2 ++ backup/manager.go | 8 ++++---- backup/manager_test.go | 2 +- backup/zip.go | 30 +++++++++++++++++------------- backup/zip_test.go | 6 +++--- cli/action_create.go | 8 +++++++- config/config.go | 11 +++++++++++ 7 files changed, 45 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ba62c21..342e1a5 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ go build ## Release History +* 0.2.5 + * add whitelist functionality for CREATE command * 0.2.4 * delete the archive even if the archive id is empty * 0.2.3 diff --git a/backup/manager.go b/backup/manager.go index 77bc812..460e13c 100644 --- a/backup/manager.go +++ b/backup/manager.go @@ -25,7 +25,7 @@ type BackupResult struct { type BackupCreater interface { io.Closer - Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult + Create(files []string, blacklist, whitelist []*regexp.Regexp, description, vaultName string) *BackupResult } type BackupGetter interface { @@ -41,7 +41,7 @@ type BackupDeleter interface { type BackupManager interface { io.Closer - Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult + Create(files []string, blacklist, whitelist []*regexp.Regexp, description, vaultName string) *BackupResult Download(backupId uint, target string, fallbackPassword func() string) error Delete(backupId uint) error } @@ -94,7 +94,7 @@ func (b *backupManager) Close() error { return b.dbRepository.Close() } -func (b *backupManager) Create(files []string, blacklist []*regexp.Regexp, description, vaultName string) *BackupResult { +func (b *backupManager) Create(files []string, blacklist, whitelist []*regexp.Regexp, description, vaultName string) *BackupResult { // folder/file -> zip -> encrypt -> glacier srcZip, dstZip := io.Pipe() srcCrypt, dstCrypt := io.Pipe() @@ -112,7 +112,7 @@ func (b *backupManager) Create(files []string, blacklist []*regexp.Regexp, descr defer wg.Done() defer dstZip.Close() - Zip(files, blacklist, dstZip, contentChan) + Zip(files, blacklist, whitelist, dstZip, contentChan) }() go func() { for { diff --git a/backup/manager_test.go b/backup/manager_test.go index 1aa4a18..4477da9 100644 --- a/backup/manager_test.go +++ b/backup/manager_test.go @@ -28,7 +28,7 @@ func Test_ZipEncryptDecryptUnzip(t *testing.T) { go func() { defer dstZip.Close() - Zip([]string{"./"}, []*regexp.Regexp{}, dstZip, nil) + Zip([]string{"./"}, []*regexp.Regexp{}, []*regexp.Regexp{}, dstZip, nil) }() encodedZipFile, err := ioutil.TempFile("", ".enc") diff --git a/backup/zip.go b/backup/zip.go index 7f73ac9..0830082 100644 --- a/backup/zip.go +++ b/backup/zip.go @@ -19,8 +19,8 @@ type ZipContent struct { FileInfo os.FileInfo } -//ZIP the given file/folder and write file information out in given channel -func Zip(filePaths []string, blacklist []*regexp.Regexp, dst io.Writer, contentChan chan<- *ZipContent) { +// ZIP the given file/folder and write file information out in given channel +func Zip(filePaths []string, blacklist, whitelist []*regexp.Regexp, dst io.Writer, contentChan chan<- *ZipContent) { // Create a new zip archive. zipWriter := zip.NewWriter(dst) zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { @@ -37,10 +37,10 @@ func Zip(filePaths []string, blacklist []*regexp.Regexp, dst io.Writer, contentC } if fInfo.IsDir() { - addFiles(zipWriter, absFilePath+"/", filepath.Dir(absFilePath+"/")+"/", blacklist, contentChan) + addFiles(zipWriter, absFilePath+"/", filepath.Dir(absFilePath+"/")+"/", blacklist, whitelist, contentChan) } else { dir, name := filepath.Split(absFilePath) - addFile(zipWriter, dir, dir+"/", name, blacklist, contentChan) + addFile(zipWriter, dir, dir+"/", name, blacklist, whitelist, contentChan) } } @@ -49,7 +49,7 @@ func Zip(filePaths []string, blacklist []*regexp.Regexp, dst io.Writer, contentC } } -func addFiles(w *zip.Writer, basePath, baseInZip string, blacklist []*regexp.Regexp, contentChan chan<- *ZipContent) { +func addFiles(w *zip.Writer, basePath, baseInZip string, blacklist, whitelist []*regexp.Regexp, contentChan chan<- *ZipContent) { // Open the Directory files, err := ioutil.ReadDir(basePath) if err != nil { @@ -61,14 +61,14 @@ func addFiles(w *zip.Writer, basePath, baseInZip string, blacklist []*regexp.Reg if fileDesc.IsDir() { // recursion ahead! newBase := basePath + fileDesc.Name() + "/" - addFiles(w, newBase, baseInZip+"/"+fileDesc.Name()+"/", blacklist, contentChan) + addFiles(w, newBase, baseInZip+"/"+fileDesc.Name()+"/", blacklist, whitelist, contentChan) } else { - addFile(w, basePath, baseInZip, fileDesc.Name(), blacklist, contentChan) + addFile(w, basePath, baseInZip, fileDesc.Name(), blacklist, whitelist, contentChan) } } } -func addFile(w *zip.Writer, basePath, baseInZip, fileName string, blacklist []*regexp.Regexp, contentChan chan<- *ZipContent) int64 { +func addFile(w *zip.Writer, basePath, baseInZip, fileName string, blacklist, whitelist []*regexp.Regexp, contentChan chan<- *ZipContent) int64 { //open for reading filePath := normalizeFilePath(basePath + fileName) zipPath := normalizeZipPath(baseInZip + fileName) @@ -80,9 +80,13 @@ func addFile(w *zip.Writer, basePath, baseInZip, fileName string, blacklist []*r } defer osFile.Close() - if blacklisted, expr := isBlacklisted(filePath, blacklist); blacklisted { - LogInfo(`Ignore file because it is blacklisted: %s -> "%s"`, filePath, expr) - return 0 + if blacklisted, blExpr := isListed(filePath, blacklist); blacklisted { + if whitelisted, wlExpr := isListed(filePath, whitelist); whitelisted { + LogInfo(`Include file because it is whitelisted: %s -> "%s"`, filePath, wlExpr) + } else { + LogInfo(`Ignore file because it is blacklisted: %s -> "%s"`, filePath, blExpr) + return 0 + } } LogInfo("Add to zip: %s -> %s", osFile.Name(), zipPath) @@ -125,8 +129,8 @@ func addFile(w *zip.Writer, basePath, baseInZip, fileName string, blacklist []*r return written } -func isBlacklisted(path string, blacklist []*regexp.Regexp) (bool, *regexp.Regexp) { - for _, curExpr := range blacklist { +func isListed(path string, list []*regexp.Regexp) (bool, *regexp.Regexp) { + for _, curExpr := range list { if curExpr.MatchString(path) { return true, curExpr } diff --git a/backup/zip_test.go b/backup/zip_test.go index 2327285..8562dca 100644 --- a/backup/zip_test.go +++ b/backup/zip_test.go @@ -36,7 +36,7 @@ func Test_Zip(t *testing.T) { } }() - Zip([]string{"./"}, []*regexp.Regexp{}, tmpFile, contentChan) + Zip([]string{"./"}, []*regexp.Regexp{}, []*regexp.Regexp{}, tmpFile, contentChan) wg.Wait() assert.True(t, containsTestFile) @@ -56,7 +56,7 @@ func Test_Zip(t *testing.T) { assert.True(t, containsTestFile) } -func TestBlacklist(t *testing.T) { +func TestList(t *testing.T) { tests := []struct { name string blacklist string @@ -74,7 +74,7 @@ func TestBlacklist(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result, _ := isBlacklisted(test.testCase, []*regexp.Regexp{regexp.MustCompile(test.blacklist)}) + result, _ := isListed(test.testCase, []*regexp.Regexp{regexp.MustCompile(test.blacklist)}) assert.Equal(t, result, test.expectedResult) }) } diff --git a/cli/action_create.go b/cli/action_create.go index 8b19caf..8bbeb50 100644 --- a/cli/action_create.go +++ b/cli/action_create.go @@ -31,7 +31,7 @@ func (a *actionCreate) Do(cfg *config.Config) { } defer b.Close() - result := b.Create(cfg.Create.Files, cfg.Create.GetBlacklist(), cfg.Create.AWSArchiveDescription, cfg.Create.AWSVaultName) + result := b.Create(cfg.Create.Files, cfg.Create.GetBlacklist(), cfg.Create.GetWhitelist(), cfg.Create.AWSArchiveDescription, cfg.Create.AWSVaultName) if result.Error != nil { LogError("Could not upload backup. Error: %v", result.Error) @@ -51,6 +51,12 @@ func (a *actionCreate) Validate(cfg *config.Config) { } } + for _, curExpr := range cfg.Create.Whitelist { + if _, err := regexp.Compile(curExpr); err != nil { + cfg.Create.Fail(fmt.Sprintf(`Whitelist expression is invalid: "%s" Cause: %v`, curExpr, err)) + } + } + if !isValidPartSize(cfg.Create.AWSPartSize) { cfg.Create.Fail("The part size is not valid. Valid sizes are: %+v", validPartSizes) } diff --git a/config/config.go b/config/config.go index 1c04bd2..fa327fa 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,7 @@ type CreateConfig struct { AWSVaultName string `arg:"positional,env:AWS_VAULT_NAME,help:The name of the glacier vault."` Files []string `arg:"positional,env:FILE,help:The file or folder to backup."` Blacklist []string `arg:"-b,separate,env:BLACKLIST,help:Regular expressions of files that should be excluded."` + Whitelist []string `arg:"-w,separate,env:WHITELIST,help:Regular expressions of files that should be included even if their would be excluded by blacklist."` AWSPartSize int `arg:"--aws-part-size,env:AWS_PART_SIZE,help:The size of each part (except the last) in MiB."` AWSArchiveDescription string `arg:"-d,env:AWS_ARCHIVE_DESC,help:The description of the archive."` @@ -270,6 +271,16 @@ func (c *CreateConfig) GetBlacklist() []*regexp.Regexp { return result } +func (c *CreateConfig) GetWhitelist() []*regexp.Regexp { + var result []*regexp.Regexp + + for _, curEntry := range c.Whitelist { + result = append(result, regexp.MustCompile(curEntry)) + } + + return result +} + func (c *CreateConfig) Fail(format string, args ...interface{}) { failInternal(c.argParser, format, args...) }