Implement dir_scan_timeout config option

internal/backer/consts.go

@@ -29,6 +29,12 @@ const (
 	// backupTimeoutMax is the maximum backup timeout in minutes (24 hours).
 	backupTimeoutMax = 1440
 
+	// dirScanTimeoutMin is the minimum directory scan timeout in minutes.
+	dirScanTimeoutMin = 1
+
+	// dirScanTimeoutMax is the maximum directory scan timeout in minutes (24 hours).
+	dirScanTimeoutMax = 1440
+
 	// compressionLevelMin is the minimum compression level.
 	compressionLevelMin = 1
 
@@ -41,6 +47,9 @@ const (
 	// defaultBackupTimeout is the default backup timeout in minutes.
 	defaultBackupTimeout = 60
 
+	// defaultDirScanTimeout is the default directory scan timeout in minutes (1 hour).
+	defaultDirScanTimeout = 60
+
 	// defaultCompressionLevel is the default compression level.
 	defaultCompressionLevel = 9
 

internal/backer/getFilesFromDirectories.go

@@ -1,10 +1,12 @@
 package backer
 
 import (
+	"context"
 	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"time"
 
 	backerlog "backer/internal/log"
 )
@@ -25,7 +27,8 @@ func isExcluded(path string) bool {
 
 // GetFilesFromDirectories makes a file list of given directories.
 // Logs errors for directories that don't exist or can't be accessed but continues with valid directories.
-func GetFilesFromDirectories(directories []string) ([]string, error) {
+// Takes context with timeout from config dir_scan_timeout setting.
+func GetFilesFromDirectories(ctx context.Context, directories []string) ([]string, error) {
 	var files []string
 
 	for _, dir := range directories {
@@ -41,6 +44,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
 		}
 
 		err := filepath.WalkDir(dir, func(path string, de fs.DirEntry, err error) error { //nolint: revive
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			default:
+			}
+
 			if err != nil {
 				backerlog.Warnf("Skipping %s: %v", path, err)
 
@@ -59,6 +68,12 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
 		})
 
 		if err != nil {
+			if errors.Is(err, context.DeadlineExceeded) {
+				backerlog.Errorf("Directory scan timed out after scanning %d files", len(files))
+
+				return files, ErrDirectoryScanTimeout{ScannedFiles: len(files)}
+			}
+
 			backerlog.Warnf("Failed to walk directory %s: %v", dir, err)
 		}
 	}
@@ -66,4 +81,13 @@ func GetFilesFromDirectories(directories []string) ([]string, error) {
 	return files, nil
 }
 
+// GetFilesFromDirectoriesWithTimeout makes a file list of given directories with timeout.
+// Convenience wrapper that creates context from timeout in minutes.
+func GetFilesFromDirectoriesWithTimeout(timeoutMinutes int, directories []string) ([]string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMinutes)*time.Minute)
+	defer cancel()
+
+	return GetFilesFromDirectories(ctx, directories)
+}
+
 /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */

internal/backer/getFilesFromDirectories_test.go

@@ -1,6 +1,7 @@
 package backer
 
 import (
+	"context"
 	"os"
 	"regexp"
 	"slices"
@@ -36,7 +37,7 @@ func TestGetFilesFromDirectories(t *testing.T) {
 		}
 	)
 
-	output, err := GetFilesFromDirectories(input.Directories)
+	output, err := GetFilesFromDirectories(context.Background(), input.Directories)
 
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
@@ -59,7 +60,7 @@ func TestGetFilesFromDirectories(t *testing.T) {
 
 // TestGetFilesFromDirectoriesSingleDir tests with a single directory.
 func TestGetFilesFromDirectoriesSingleDir(t *testing.T) {
-	output, err := GetFilesFromDirectories([]string{"../../test_data/test1/foo"})
+	output, err := GetFilesFromDirectories(context.Background(), []string{"../../test_data/test1/foo"})
 
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
@@ -80,7 +81,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) {
 	// Create temp empty dir
 	tmpDir := t.TempDir()
 
-	output, err := GetFilesFromDirectories([]string{tmpDir})
+	output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
 
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
@@ -95,7 +96,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) {
 // TestGetFilesFromDirectoriesNonExistent tests with non-existent directory.
 // Should log error but not fail - returns empty list.
 func TestGetFilesFromDirectoriesNonExistent(t *testing.T) {
-	output, err := GetFilesFromDirectories([]string{"/nonexistent/path/that/does/not/exist"})
+	output, err := GetFilesFromDirectories(context.Background(), []string{"/nonexistent/path/that/does/not/exist"})
 
 	// Should not return error, but logs error
 	if err != nil {
@@ -113,7 +114,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
 	// Create a temp directory that exists
 	tmpDir := t.TempDir()
 
-	output, err := GetFilesFromDirectories([]string{tmpDir, "/nonexistent/path/that/does/not/exist"})
+	output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir, "/nonexistent/path/that/does/not/exist"})
 
 	// Should not return error, but logs error
 	if err != nil {
@@ -128,7 +129,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
 
 // TestGetFilesFromDirectoriesMultipleDirs tests multiple directories.
 func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) {
-	output, err := GetFilesFromDirectories([]string{
+	output, err := GetFilesFromDirectories(context.Background(), []string{
 		"../../test_data/test1/foo",
 		"../../test_data/test1/bar",
 	})
@@ -170,7 +171,7 @@ func TestGetFilesFromDirectoriesSymlink(t *testing.T) {
 		t.Skip("Symlinks not supported, skipping test")
 	}
 
-	output, err := GetFilesFromDirectories([]string{tmpDir})
+	output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
 
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
@@ -249,7 +250,7 @@ func TestGetFilesFromDirectoriesWithExcludes(t *testing.T) {
 		regexp.MustCompile(`\.tmp$`),
 	}
 
-	output, err := GetFilesFromDirectories([]string{tmpDir})
+	output, err := GetFilesFromDirectories(context.Background(), []string{tmpDir})
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
 	}

internal/backer/loadConfig.go

@@ -95,6 +95,17 @@ func LoadConfig(path string) error {
 		return fmt.Errorf("Config option backup_timeout must be between %d and %d minutes, got %d", backupTimeoutMin, backupTimeoutMax, C.BackupTimeout)
 	}
 
+	// DirScanTimeout: set default, then validate range.
+	if C.DirScanTimeout == 0 {
+		C.DirScanTimeout = defaultDirScanTimeout
+
+		log.Warnf("Config option dir_scan_timeout is not set, fallback to %d minutes", C.DirScanTimeout)
+	}
+
+	if C.DirScanTimeout < dirScanTimeoutMin || C.DirScanTimeout > dirScanTimeoutMax {
+		return fmt.Errorf("Config option dir_scan_timeout must be between %d and %d minutes, got %d", dirScanTimeoutMin, dirScanTimeoutMax, C.DirScanTimeout)
+	}
+
 	// CompressionLevel: set default, then validate range.
 	if C.CompressionLevel == 0 {
 		C.CompressionLevel = defaultCompressionLevel

internal/backer/newServer.go

@@ -101,14 +101,26 @@ func NewServer(configPath string) (*ServerWrapper, error) {
 
 		log.Infof("Backup started: client=%s user=%s", clientIP, username)
 
-		ctx := r.Context()
+		// Create context with dir_scan_timeout from request context.
-		files, err := GetFilesFromDirectories(C.Directories)
+		ctx, cancel := context.WithTimeout(r.Context(), time.Duration(C.DirScanTimeout)*time.Minute)
+		defer cancel()
+
+		files, err := GetFilesFromDirectories(ctx, C.Directories)
 
 		if err != nil {
-			log.Errorf("Failed to get files: %v", err)
+			switch {
+			case errors.Is(err, context.DeadlineExceeded):
+				w.Header().Set("Server", "backer")
+				http.Error(w, "Directory scan timed out", http.StatusRequestTimeout)
 
-			w.Header().Set("Server", "backer")
+				log.Errorf("Directory scan timed out: %v", err)
-			http.Error(w, "Failed to get files", http.StatusInternalServerError)
+
+			default:
+				log.Errorf("Failed to get files: %v", err)
+
+				w.Header().Set("Server", "backer")
+				http.Error(w, "Failed to get files", http.StatusInternalServerError)
+			}
 
 			return
 		}

internal/backer/types.go

@@ -15,12 +15,23 @@ type (
 		LogLevel             string   `json:"loglevel"`
 		Directories          []string `json:"directories"`
 		BackupTimeout        int      `json:"backup_timeout"`        // Timeout in minutes for backup streaming.
+		DirScanTimeout       int      `json:"dir_scan_timeout"`      // Timeout in minutes for directory scanning.
 		CompressionLevel     int      `json:"compression_level"`     // Compression level (1-9, default 9).
 		ExcludePatterns      []string `json:"exclude_patterns"`      // Regex patterns to exclude from backup.
 		FilenamePrefix       string   `json:"filename_prefix"`       // Prefix for backup filename (default: "backup").
 		CompressionAlgorithm string   `json:"compression_algorithm"` // Compression algorithm alias (deprecated: use default_compression).
 		DefaultCompression   string   `json:"default_compression"`   // Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz.
 	}
+
+	// ErrDirectoryScanTimeout is returned when directory scan times out.
+	ErrDirectoryScanTimeout struct {
+		ScannedFiles int
+	}
 )
 
+// Error returns the error message for ErrDirectoryScanTimeout.
+func (e ErrDirectoryScanTimeout) Error() string {
+	return "Directory scan timed out"
+}
+
 /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */