From 8beba095d0de159ea513410e7fcfc4a49cf09fd5 Mon Sep 17 00:00:00 2001 From: eleksir Date: Sat, 4 Apr 2026 11:47:39 +0300 Subject: [PATCH] Add ability to select compression algo by downloading location with file extension. - Also update README.md with more useful examples. --- AGENTS.md | 3 ++- README.md | 19 ++++++++++++++++--- contrib/{backer => backer.openrc} | 3 ++- data/config_example.json | 29 ++++++++++++++++------------- internal/backer/consts.go | 4 ++-- internal/backer/loadConfig.go | 19 ++++++++++++------- internal/backer/loadConfig_test.go | 24 ++++++++++++------------ internal/backer/newServer.go | 28 +++++++++++++++++++++++++++- internal/backer/types.go | 3 ++- 9 files changed, 91 insertions(+), 41 deletions(-) rename contrib/{backer => backer.openrc} (81%) diff --git a/AGENTS.md b/AGENTS.md index d7a9c47..8ade84f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,7 +198,8 @@ Config is JSON (HJSON with comments supported). Default path depends on OS: | compression_level | `9` | Gzip level (1-9) | | exclude_patterns | `[]` | Regex patterns to exclude from backup | | filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header | -| compression_algorithm | `gzip` | Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz | +| default_compression | `gzip` | Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz | +| compression_algorithm | `gzip` | Alias for default_compression (deprecated) | ## Architecture & Key Patterns diff --git a/README.md b/README.md index 1dc140f..66fee67 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,22 @@ If you running FreeBSD, use the rc.d script `contrib/backer.freebsd` and place i Build, install and configure backer. On your backup storage server you can do ```bash -curl -u username:password https://your_server:8086/archive -o "backup-$(date +%Y%m%d-%H%M%S).tar.gz" +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive ``` -The filename and extension depend on your `filename_prefix` and `compression_algorithm` settings. +The filename and extension depend on your `filename_prefix` and `default_compression` settings. + +You can also override compression by specifying the archive extension in the URL: + +```bash +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.xz +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.gz # Uses pgzip +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.bz2 +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.zst +wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.lz4 +``` + +Note: `/archive.tar.gz` always uses pgzip (parallel gzip) for better performance. Of course, something can go wrong, so you must check http status code after downloading backup. @@ -56,7 +68,8 @@ All options described in example config. | compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). | | exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. | | filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. | -| compression_algorithm | "gzip" | Compression algorithm. | Options: gzip, pgzip, bzip2, zstd, lz4, xz. | +| default_compression | "gzip" | Default compression algorithm. | Options: gzip, pgzip, bzip2, zstd, lz4, xz. | +| compression_algorithm | "gzip" | Alias for default_compression. | You should use default_compression instead. | ## Compression algorithms diff --git a/contrib/backer b/contrib/backer.openrc similarity index 81% rename from contrib/backer rename to contrib/backer.openrc index db31983..e9b8fd6 100644 --- a/contrib/backer +++ b/contrib/backer.openrc @@ -7,6 +7,7 @@ pidfile="/run/$SVCNAME/$SVCNAME.pid" depend() { need net + use dns } start_pre() { @@ -14,4 +15,4 @@ start_pre() { mkdir -p /run/$SVCNAME } -# vim: set ft=sh noet ai ts=4 sw=4 sts=4: \ No newline at end of file +# vim: set ft=sh noet ai ts=4 sw=4 sts=4: diff --git a/data/config_example.json b/data/config_example.json index 66867ae..22f63eb 100644 --- a/data/config_example.json +++ b/data/config_example.json @@ -26,27 +26,30 @@ /* Where to store logs. */ "log": "/var/log/backer.log", - /* Verbosity of logs */ + /* Verbosity of logs. */ "loglevel": "info", - /* Diretories to backup */ - "directories": [ - "/path/to/directory1", - "/path/to/directory2" - ], - /* Backup streaming timeout in minutes. */ "backup_timeout": 60, - /* Gzip compression level (1-9, where 9 is best compression). */ + /* Gzip compression level (1-9, where 9 is best compression). */ "compression_level": 9, - /* Regex patterns to exclude from backup. */ - "exclude_patterns": [], - /* Prefix for backup filename in Content-Disposition header. */ "filename_prefix": "backup", - /* Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. */ - "compression_algorithm": "gzip" + /* Compression algorithm alias (deprecated: use default_compression). */ + "compression_algorithm": "gzip", + + /* Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. */ + "default_compression": "gzip", + + /* Regex patterns to exclude from backup. */ + "exclude_patterns": [], + + /* Directories to backup. */ + "directories": [ + "/path/to/directory1", + "/path/to/directory2" + ] } diff --git a/internal/backer/consts.go b/internal/backer/consts.go index e1d9f4f..a370942 100644 --- a/internal/backer/consts.go +++ b/internal/backer/consts.go @@ -50,8 +50,8 @@ const ( // defaultFilenamePrefix is the default prefix for backup filenames. defaultFilenamePrefix = "backup" - // defaultCompressionAlgorithm is the default compression algorithm. - defaultCompressionAlgorithm = "gzip" + // defaultDefaultCompression is the default compression algorithm. + defaultDefaultCompression = "gzip" ) /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index 46ffbf9..1abbf0b 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -135,15 +135,20 @@ func LoadConfig(path string) error { log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix) } - // CompressionAlgorithm: default and validate. - if C.CompressionAlgorithm == "" { - C.CompressionAlgorithm = defaultCompressionAlgorithm - - log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm) + // DefaultCompression: default and validate. If compression_algorithm is set (deprecated), use it as fallback. + if C.DefaultCompression == "" { + if C.CompressionAlgorithm != "" { + C.DefaultCompression = C.CompressionAlgorithm + } else { + C.DefaultCompression = defaultDefaultCompression + } } - if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" && C.CompressionAlgorithm != "pgzip" { - return fmt.Errorf("Config option compression_algorithm must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.CompressionAlgorithm) + switch C.DefaultCompression { + case "gzip", "pgzip", "bzip2", "xz", "lz4", "zstd": + log.Infof("Using %s as defailt compression algorithm", C.DefaultCompression) + default: + return fmt.Errorf("Config option default_compression must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.DefaultCompression) } // Compile exclude patterns. diff --git a/internal/backer/loadConfig_test.go b/internal/backer/loadConfig_test.go index a4c4cd7..a4d1add 100644 --- a/internal/backer/loadConfig_test.go +++ b/internal/backer/loadConfig_test.go @@ -227,8 +227,8 @@ func TestLoadConfigDefaultValues(t *testing.T) { t.Errorf("C.FilenamePrefix should default to backup, got %s", C.FilenamePrefix) } - if C.CompressionAlgorithm != "gzip" { - t.Errorf("C.CompressionAlgorithm should default to gzip, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "gzip" { + t.Errorf("C.DefaultCompression should default to gzip, got %s", C.DefaultCompression) } } @@ -248,8 +248,8 @@ func TestLoadConfigCompressionAlgorithmBzip2(t *testing.T) { t.Errorf("Expected no error, got %v", err) } - if C.CompressionAlgorithm != "bzip2" { - t.Errorf("C.CompressionAlgorithm should be bzip2, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "bzip2" { + t.Errorf("C.DefaultCompression should be bzip2, got %s", C.DefaultCompression) } } @@ -269,8 +269,8 @@ func TestLoadConfigCompressionAlgorithmZstd(t *testing.T) { t.Errorf("Expected no error, got %v", err) } - if C.CompressionAlgorithm != "zstd" { - t.Errorf("C.CompressionAlgorithm should be zstd, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "zstd" { + t.Errorf("C.DefaultCompression should be zstd, got %s", C.DefaultCompression) } } @@ -290,8 +290,8 @@ func TestLoadConfigCompressionAlgorithmLz4(t *testing.T) { t.Errorf("Expected no error, got %v", err) } - if C.CompressionAlgorithm != "lz4" { - t.Errorf("C.CompressionAlgorithm should be lz4, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "lz4" { + t.Errorf("C.DefaultCompression should be lz4, got %s", C.DefaultCompression) } } @@ -311,8 +311,8 @@ func TestLoadConfigCompressionAlgorithmXz(t *testing.T) { t.Errorf("Expected no error, got %v", err) } - if C.CompressionAlgorithm != "xz" { - t.Errorf("C.CompressionAlgorithm should be xz, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "xz" { + t.Errorf("C.DefaultCompression should be xz, got %s", C.DefaultCompression) } } @@ -332,8 +332,8 @@ func TestLoadConfigCompressionAlgorithmPgzip(t *testing.T) { t.Errorf("Expected no error, got %v", err) } - if C.CompressionAlgorithm != "pgzip" { - t.Errorf("C.CompressionAlgorithm should be pgzip, got %s", C.CompressionAlgorithm) + if C.DefaultCompression != "pgzip" { + t.Errorf("C.DefaultCompression should be pgzip, got %s", C.DefaultCompression) } } diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index f865ba2..b85e3b6 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "path" "strings" "time" @@ -76,9 +77,14 @@ func NewServer(configPath string) (*http.Server, error) { var ( archive io.ReadCloser extension string + algorithm string ) - switch C.CompressionAlgorithm { + requestedExt := strings.TrimPrefix(path.Ext(r.URL.Path), ".") + + algorithm = getCompressionAlgorithm(requestedExt) + + switch algorithm { case "bzip2": archive = CreateTarBzip2Stream(ctx, files) extension = "tar.bz2" @@ -207,4 +213,24 @@ func getClientIP(r *http.Request) string { return host } +// getCompressionAlgorithm determines which compression algorithm to use based on the requested extension. +// If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default. +// For .tar.gz extension, pgzip is always used for parallel compression. +func getCompressionAlgorithm(requestedExt string) string { + switch requestedExt { + case "tar.bz2": + return "bzip2" + case "tar.zst": + return "zstd" + case "tar.lz4": + return "lz4" + case "tar.xz": + return "xz" + case "tar.gz": + return "pgzip" + default: + return C.DefaultCompression + } +} + /* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/types.go b/internal/backer/types.go index 16c6666..5f0dffe 100644 --- a/internal/backer/types.go +++ b/internal/backer/types.go @@ -18,7 +18,8 @@ type ( CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9). ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup"). - CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. + CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm alias (deprecated: use default_compression). + DefaultCompression string `json:"default_compression"` // Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. } )