eleksir/backer
0
Fork 0
Code Pull Requests Releases Activity

Implement dir_scan_timeout config option

Browse Source
This commit is contained in:
eleksir
2026-04-04 23:32:37 +03:00
parent 6e56e74df4
commit 6cea15f553
6 changed files with 82 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/backer/consts.go
View File

@@ -29,6 +29,12 @@ const (
// backupTimeoutMax is the maximum backup timeout in minutes (24 hours).
backupTimeoutMax = 1440
// dirScanTimeoutMin is the minimum directory scan timeout in minutes.
dirScanTimeoutMin = 1
// dirScanTimeoutMax is the maximum directory scan timeout in minutes (24 hours).
dirScanTimeoutMax = 1440
// compressionLevelMin is the minimum compression level.
compressionLevelMin = 1
@@ -41,6 +47,9 @@ const (
// defaultBackupTimeout is the default backup timeout in minutes.
defaultBackupTimeout = 60
// defaultDirScanTimeout is the default directory scan timeout in minutes (1 hour).
defaultDirScanTimeout = 60
// defaultCompressionLevel is the default compression level.
defaultCompressionLevel = 9

26
internal/backer/getFilesFromDirectories.go
View File

@@ -1,10 +1,12 @@
package backer
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"time"
backerlog "backer/internal/log"
)
@@ -25,7 +27,8 @@ func isExcluded(path string) bool {
// GetFilesFromDirectories makes a file list of given directories.
// Logs errors for directories that don't exist or can't be accessed but continues with valid directories.
func GetFilesFromDirectories(directories []string) ([]string, error) {
// Takes context with timeout from config dir_scan_timeout setting.
func GetFilesFromDirectories(ctx context.Context, directories []string) ([]string, error) {
var files []string
for _, dir := range directories {
@@ -41,6 +44,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
}
err := filepath.WalkDir(dir, func(path string, de fs.DirEntry, err error) error { //nolint: revive
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err != nil {
backerlog.Warnf("Skipping %s: %v", path, err)
@@ -59,6 +68,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
})
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
backerlog.Errorf("Directory scan timed out after scanning %d files", len(files))
return files, ErrDirectoryScanTimeout{ScannedFiles: len(files)}
}
backerlog.Warnf("Failed to walk directory %s: %v", dir, err)
}
}
@@ -66,4 +81,13 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
return files, nil
}
// GetFilesFromDirectoriesWithTimeout makes a file list of given directories with timeout.
// Convenience wrapper that creates context from timeout in minutes.
func GetFilesFromDirectoriesWithTimeout(timeoutMinutes int, directories []string) ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMinutes)*time.Minute)
defer cancel()
return GetFilesFromDirectories(ctx, directories)
}
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */

17
internal/backer/getFilesFromDirectories_test.go
View File

@@ -1,6 +1,7 @@
package backer
import (
"context"
"os"
"regexp"
"slices"
@@ -36,7 +37,7 @@ func TestGetFilesFromDirectories(t *testing.T) {
}
)
output, err := GetFilesFromDirectories(input.Directories)
output, err := GetFilesFromDirectories(context.Background(), input.Directories)
if err != nil {
t.Errorf("Expected no error, got %v", err)
@@ -59,7 +60,7 @@ func TestGetFilesFromDirectories(t *testing.T) {
// TestGetFilesFromDirectoriesSingleDir tests with a single directory.
func TestGetFilesFromDirectoriesSingleDir(t *testing.T) {
output, err := GetFilesFromDirectories([]string{"../../test_data/test1/foo"})
output, err := GetFilesFromDirectories(context.Background(), []string{"../../test_data/test1/foo"})
if err != nil {
t.Errorf("Expected no error, got %v", err)
@@ -80,7 +81,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) {
// Create temp empty dir
tmpDir := t.TempDir()
output, err := GetFilesFromDirectories([]string{tmpDir})
output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
if err != nil {
t.Errorf("Expected no error, got %v", err)
@@ -95,7 +96,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) {
// TestGetFilesFromDirectoriesNonExistent tests with non-existent directory.
// Should log error but not fail - returns empty list.
func TestGetFilesFromDirectoriesNonExistent(t *testing.T) {
output, err := GetFilesFromDirectories([]string{"/nonexistent/path/that/does/not/exist"})
output, err := GetFilesFromDirectories(context.Background(), []string{"/nonexistent/path/that/does/not/exist"})
// Should not return error, but logs error
if err != nil {
@@ -113,7 +114,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
// Create a temp directory that exists
tmpDir := t.TempDir()
output, err := GetFilesFromDirectories([]string{tmpDir, "/nonexistent/path/that/does/not/exist"})
output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir, "/nonexistent/path/that/does/not/exist"})
// Should not return error, but logs error
if err != nil {
@@ -128,7 +129,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
// TestGetFilesFromDirectoriesMultipleDirs tests multiple directories.
func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) {
output, err := GetFilesFromDirectories([]string{
output, err := GetFilesFromDirectories(context.Background(), []string{
"../../test_data/test1/foo",
"../../test_data/test1/bar",
})
@@ -170,7 +171,7 @@ func TestGetFilesFromDirectoriesSymlink(t *testing.T) {
t.Skip("Symlinks not supported, skipping test")
}
output, err := GetFilesFromDirectories([]string{tmpDir})
output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
if err != nil {
t.Errorf("Expected no error, got %v", err)
@@ -249,7 +250,7 @@ func TestGetFilesFromDirectoriesWithExcludes(t *testing.T) {
regexp.MustCompile(`\.tmp$`),
}
output, err := GetFilesFromDirectories([]string{tmpDir})
output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}

11
internal/backer/loadConfig.go
View File

@@ -95,6 +95,17 @@ func LoadConfig(path string) error {
return fmt.Errorf("Config option backup_timeout must be between %d and %d minutes, got %d", backupTimeoutMin, backupTimeoutMax, C.BackupTimeout)
}
// DirScanTimeout: set default, then validate range.
if C.DirScanTimeout == 0 {
C.DirScanTimeout = defaultDirScanTimeout
log.Warnf("Config option dir_scan_timeout is not set, fallback to %d minutes", C.DirScanTimeout)
}
if C.DirScanTimeout < dirScanTimeoutMin || C.DirScanTimeout > dirScanTimeoutMax {
return fmt.Errorf("Config option dir_scan_timeout must be between %d and %d minutes, got %d", dirScanTimeoutMin, dirScanTimeoutMax, C.DirScanTimeout)
}
// CompressionLevel: set default, then validate range.
if C.CompressionLevel == 0 {
C.CompressionLevel = defaultCompressionLevel

22
internal/backer/newServer.go
View File

@@ -101,14 +101,26 @@ func NewServer(configPath string) (*ServerWrapper, error) {
log.Infof("Backup started: client=%s user=%s", clientIP, username)
ctx := r.Context()
files, err := GetFilesFromDirectories(C.Directories)
// Create context with dir_scan_timeout from request context.
ctx, cancel := context.WithTimeout(r.Context(), time.Duration(C.DirScanTimeout)*time.Minute)
defer cancel()
files, err := GetFilesFromDirectories(ctx, C.Directories)
if err != nil {
log.Errorf("Failed to get files: %v", err)
switch {
case errors.Is(err, context.DeadlineExceeded):
w.Header().Set("Server", "backer")
http.Error(w, "Directory scan timed out", http.StatusRequestTimeout)
w.Header().Set("Server", "backer")
http.Error(w, "Failed to get files", http.StatusInternalServerError)
log.Errorf("Directory scan timed out: %v", err)
default:
log.Errorf("Failed to get files: %v", err)
w.Header().Set("Server", "backer")
http.Error(w, "Failed to get files", http.StatusInternalServerError)
}
return
}

11
internal/backer/types.go
View File

@@ -15,12 +15,23 @@ type (
LogLevel string `json:"loglevel"`
Directories []string `json:"directories"`
BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming.
DirScanTimeout int `json:"dir_scan_timeout"` // Timeout in minutes for directory scanning.
CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9).
ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup.
FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup").
CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm alias (deprecated: use default_compression).
DefaultCompression string `json:"default_compression"` // Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz.
}
// ErrDirectoryScanTimeout is returned when directory scan times out.
ErrDirectoryScanTimeout struct {
ScannedFiles int
}
)
// Error returns the error message for ErrDirectoryScanTimeout.
func (e ErrDirectoryScanTimeout) Error() string {
return "Directory scan timed out"
}
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */