diff --git a/internal/backer/consts.go b/internal/backer/consts.go index 2233866..04f6d89 100644 --- a/internal/backer/consts.go +++ b/internal/backer/consts.go @@ -29,6 +29,12 @@ const ( // backupTimeoutMax is the maximum backup timeout in minutes (24 hours). backupTimeoutMax = 1440 + // dirScanTimeoutMin is the minimum directory scan timeout in minutes. + dirScanTimeoutMin = 1 + + // dirScanTimeoutMax is the maximum directory scan timeout in minutes (24 hours). + dirScanTimeoutMax = 1440 + // compressionLevelMin is the minimum compression level. compressionLevelMin = 1 @@ -41,6 +47,9 @@ const ( // defaultBackupTimeout is the default backup timeout in minutes. defaultBackupTimeout = 60 + // defaultDirScanTimeout is the default directory scan timeout in minutes (1 hour). + defaultDirScanTimeout = 60 + // defaultCompressionLevel is the default compression level. defaultCompressionLevel = 9 diff --git a/internal/backer/getFilesFromDirectories.go b/internal/backer/getFilesFromDirectories.go index cf9330e..70550ec 100644 --- a/internal/backer/getFilesFromDirectories.go +++ b/internal/backer/getFilesFromDirectories.go @@ -1,10 +1,12 @@ package backer import ( + "context" "errors" "io/fs" "os" "path/filepath" + "time" backerlog "backer/internal/log" ) @@ -25,7 +27,8 @@ func isExcluded(path string) bool { // GetFilesFromDirectories makes a file list of given directories. // Logs errors for directories that don't exist or can't be accessed but continues with valid directories. -func GetFilesFromDirectories(directories []string) ([]string, error) { +// Takes context with timeout from config dir_scan_timeout setting. +func GetFilesFromDirectories(ctx context.Context, directories []string) ([]string, error) { var files []string for _, dir := range directories { @@ -41,6 +44,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) { } err := filepath.WalkDir(dir, func(path string, de fs.DirEntry, err error) error { //nolint: revive + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err != nil { backerlog.Warnf("Skipping %s: %v", path, err) @@ -59,6 +68,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) { }) if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + backerlog.Errorf("Directory scan timed out after scanning %d files", len(files)) + + return files, ErrDirectoryScanTimeout{ScannedFiles: len(files)} + } + backerlog.Warnf("Failed to walk directory %s: %v", dir, err) } } @@ -66,4 +81,13 @@ func GetFilesFromDirectories(directories []string) ([]string, error) { return files, nil } +// GetFilesFromDirectoriesWithTimeout makes a file list of given directories with timeout. +// Convenience wrapper that creates context from timeout in minutes. +func GetFilesFromDirectoriesWithTimeout(timeoutMinutes int, directories []string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMinutes)*time.Minute) + defer cancel() + + return GetFilesFromDirectories(ctx, directories) +} + /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/getFilesFromDirectories_test.go b/internal/backer/getFilesFromDirectories_test.go index 4962bbb..40c0526 100644 --- a/internal/backer/getFilesFromDirectories_test.go +++ b/internal/backer/getFilesFromDirectories_test.go @@ -1,6 +1,7 @@ package backer import ( + "context" "os" "regexp" "slices" @@ -36,7 +37,7 @@ func TestGetFilesFromDirectories(t *testing.T) { } ) - output, err := GetFilesFromDirectories(input.Directories) + output, err := GetFilesFromDirectories(context.Background(), input.Directories) if err != nil { t.Errorf("Expected no error, got %v", err) @@ -59,7 +60,7 @@ func TestGetFilesFromDirectories(t *testing.T) { // TestGetFilesFromDirectoriesSingleDir tests with a single directory. func TestGetFilesFromDirectoriesSingleDir(t *testing.T) { - output, err := GetFilesFromDirectories([]string{"../../test_data/test1/foo"}) + output, err := GetFilesFromDirectories(context.Background(), []string{"../../test_data/test1/foo"}) if err != nil { t.Errorf("Expected no error, got %v", err) @@ -80,7 +81,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) { // Create temp empty dir tmpDir := t.TempDir() - output, err := GetFilesFromDirectories([]string{tmpDir}) + output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir}) if err != nil { t.Errorf("Expected no error, got %v", err) @@ -95,7 +96,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) { // TestGetFilesFromDirectoriesNonExistent tests with non-existent directory. // Should log error but not fail - returns empty list. func TestGetFilesFromDirectoriesNonExistent(t *testing.T) { - output, err := GetFilesFromDirectories([]string{"/nonexistent/path/that/does/not/exist"}) + output, err := GetFilesFromDirectories(context.Background(), []string{"/nonexistent/path/that/does/not/exist"}) // Should not return error, but logs error if err != nil { @@ -113,7 +114,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) { // Create a temp directory that exists tmpDir := t.TempDir() - output, err := GetFilesFromDirectories([]string{tmpDir, "/nonexistent/path/that/does/not/exist"}) + output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir, "/nonexistent/path/that/does/not/exist"}) // Should not return error, but logs error if err != nil { @@ -128,7 +129,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) { // TestGetFilesFromDirectoriesMultipleDirs tests multiple directories. func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) { - output, err := GetFilesFromDirectories([]string{ + output, err := GetFilesFromDirectories(context.Background(), []string{ "../../test_data/test1/foo", "../../test_data/test1/bar", }) @@ -170,7 +171,7 @@ func TestGetFilesFromDirectoriesSymlink(t *testing.T) { t.Skip("Symlinks not supported, skipping test") } - output, err := GetFilesFromDirectories([]string{tmpDir}) + output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir}) if err != nil { t.Errorf("Expected no error, got %v", err) @@ -249,7 +250,7 @@ func TestGetFilesFromDirectoriesWithExcludes(t *testing.T) { regexp.MustCompile(`\.tmp$`), } - output, err := GetFilesFromDirectories([]string{tmpDir}) + output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir}) if err != nil { t.Fatalf("Unexpected error: %v", err) } diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index 0677526..a65d9e6 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -95,6 +95,17 @@ func LoadConfig(path string) error { return fmt.Errorf("Config option backup_timeout must be between %d and %d minutes, got %d", backupTimeoutMin, backupTimeoutMax, C.BackupTimeout) } + // DirScanTimeout: set default, then validate range. + if C.DirScanTimeout == 0 { + C.DirScanTimeout = defaultDirScanTimeout + + log.Warnf("Config option dir_scan_timeout is not set, fallback to %d minutes", C.DirScanTimeout) + } + + if C.DirScanTimeout < dirScanTimeoutMin || C.DirScanTimeout > dirScanTimeoutMax { + return fmt.Errorf("Config option dir_scan_timeout must be between %d and %d minutes, got %d", dirScanTimeoutMin, dirScanTimeoutMax, C.DirScanTimeout) + } + // CompressionLevel: set default, then validate range. if C.CompressionLevel == 0 { C.CompressionLevel = defaultCompressionLevel diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index 54b977a..7f48401 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -101,14 +101,26 @@ func NewServer(configPath string) (*ServerWrapper, error) { log.Infof("Backup started: client=%s user=%s", clientIP, username) - ctx := r.Context() - files, err := GetFilesFromDirectories(C.Directories) + // Create context with dir_scan_timeout from request context. + ctx, cancel := context.WithTimeout(r.Context(), time.Duration(C.DirScanTimeout)*time.Minute) + defer cancel() + + files, err := GetFilesFromDirectories(ctx, C.Directories) if err != nil { - log.Errorf("Failed to get files: %v", err) + switch { + case errors.Is(err, context.DeadlineExceeded): + w.Header().Set("Server", "backer") + http.Error(w, "Directory scan timed out", http.StatusRequestTimeout) - w.Header().Set("Server", "backer") - http.Error(w, "Failed to get files", http.StatusInternalServerError) + log.Errorf("Directory scan timed out: %v", err) + + default: + log.Errorf("Failed to get files: %v", err) + + w.Header().Set("Server", "backer") + http.Error(w, "Failed to get files", http.StatusInternalServerError) + } return } diff --git a/internal/backer/types.go b/internal/backer/types.go index 5f0dffe..ae37d7b 100644 --- a/internal/backer/types.go +++ b/internal/backer/types.go @@ -15,12 +15,23 @@ type ( LogLevel string `json:"loglevel"` Directories []string `json:"directories"` BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming. + DirScanTimeout int `json:"dir_scan_timeout"` // Timeout in minutes for directory scanning. CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9). ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup"). CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm alias (deprecated: use default_compression). DefaultCompression string `json:"default_compression"` // Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. } + + // ErrDirectoryScanTimeout is returned when directory scan times out. + ErrDirectoryScanTimeout struct { + ScannedFiles int + } ) +// Error returns the error message for ErrDirectoryScanTimeout. +func (e ErrDirectoryScanTimeout) Error() string { + return "Directory scan timed out" +} + /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */