From 5226688836ba0d014848921937bc5a47e9c74087 Mon Sep 17 00:00:00 2001 From: eleksir Date: Fri, 27 Mar 2026 21:28:27 +0300 Subject: [PATCH] Implemented tar+bz2 compression + unit tests for it. --- AGENTS.md | 4 +- README.md | 1 + TODO.org | 7 +- data/config_example.json | 5 +- go.mod | 2 + go.sum | 6 ++ internal/backer/loadConfig.go | 11 +++ internal/backer/loadConfig_test.go | 42 +++++++++ internal/backer/newServer.go | 21 ++++- internal/backer/types.go | 31 ++++--- internal/backer/utils.go | 141 +++++++++++++++++++++++++++++ llms.txt | 5 +- 12 files changed, 247 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 12c071a..46ace73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,6 +182,7 @@ Config is JSON (HJSON with comments supported). Default path depends on OS: | compression_level | `9` | Gzip level (1-9) | | exclude_patterns | `[]` | Regex patterns to exclude from backup | | filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header | +| compression_algorithm | `gzip` | Compression algorithm: gzip or bzip2 | ## Architecture & Key Patterns @@ -277,5 +278,4 @@ Init scripts provided in `contrib/`: ## TODO / Roadmap -See `TODO.org` for current backlog: -- Consider alternative compression algorithms (bz2, xz, lzo, lz4) +See `TODO.org` for current backlog. diff --git a/README.md b/README.md index 3746f51..f9feb52 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ AI suggests to fill this section with useful information, despite that all optio | compression_level | 9 | Gzip compression level for tar.gz archives. | 1 (fastest) to 9 (best compression). Higher values produce smaller files but take longer to compress. | | exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. Files matching any pattern are skipped. | | filename_prefix | "backup" | Prefix for backup filename in Content-Disposition header. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. | +| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2". Gzip is faster, bzip2 produces smaller files. | ## Best practices diff --git a/TODO.org b/TODO.org index 94b6422..7b90d7a 100644 --- a/TODO.org +++ b/TODO.org @@ -5,9 +5,10 @@ * TODO Backer Tasks ** TODO Consider different compression algorithms - - [ ] Evaluate alternatives to gzip: bz2, xz, lzo, lz4 - - [ ] lz4 for speed, xz for better compression - - [ ] Add compression_algorithm config option + - [X] Evaluate alternatives to gzip: bz2, xz, lzo, lz4 + - CANCELLED lzo. It has cgo implementaion which depends in liblzo, and native lid that does not support streaming. + - [X] Implemented bzip2 as alternative (pure Go, via github.com/dsnet/compress) + - [X] Add compression_algorithm config option ** DONE Configurable filename prefix - [X] Move "backup" prefix to config (currently hardcoded in newServer.go) diff --git a/data/config_example.json b/data/config_example.json index c8cdb8d..73425f4 100644 --- a/data/config_example.json +++ b/data/config_example.json @@ -45,5 +45,8 @@ "exclude_patterns": [], /* Prefix for backup filename in Content-Disposition header. */ - "filename_prefix": "backup" + "filename_prefix": "backup", + + /* Compression algorithm: gzip or bzip2. */ + "compression_algorithm": "gzip" } diff --git a/go.mod b/go.mod index 4686d0c..9c12c9c 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,5 @@ module backer go 1.25 require github.com/hjson/hjson-go v3.3.0+incompatible + +require github.com/dsnet/compress v0.0.1 // indirect diff --git a/go.sum b/go.sum index 66c2912..40dd6f4 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,8 @@ +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI= github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index 5dbbe5c..62625e9 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -135,6 +135,17 @@ func LoadConfig(path string) error { log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix) } + // CompressionAlgorithm: default and validate. + if C.CompressionAlgorithm == "" { + C.CompressionAlgorithm = "gzip" + + log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm) + } + + if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" { + return fmt.Errorf("Compression_algorithm must be gzip or bzip2, got %s", C.CompressionAlgorithm) + } + // Compile exclude patterns. excludePatterns = make([]*regexp.Regexp, 0, len(C.ExcludePatterns)) diff --git a/internal/backer/loadConfig_test.go b/internal/backer/loadConfig_test.go index 8058435..fa3de41 100644 --- a/internal/backer/loadConfig_test.go +++ b/internal/backer/loadConfig_test.go @@ -226,6 +226,48 @@ func TestLoadConfigDefaultValues(t *testing.T) { if C.FilenamePrefix != "backup" { t.Errorf("C.FilenamePrefix should default to backup, got %s", C.FilenamePrefix) } + + if C.CompressionAlgorithm != "gzip" { + t.Errorf("C.CompressionAlgorithm should default to gzip, got %s", C.CompressionAlgorithm) + } +} + +// TestLoadConfigCompressionAlgorithmBzip2 tests bzip2 compression algorithm. +func TestLoadConfigCompressionAlgorithmBzip2(t *testing.T) { + config := `{ + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": true, + "compression_algorithm": "bzip2" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if C.CompressionAlgorithm != "bzip2" { + t.Errorf("C.CompressionAlgorithm should be bzip2, got %s", C.CompressionAlgorithm) + } +} + +// TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm. +func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) { + config := `{ + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": true, + "compression_algorithm": "invalid" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err == nil { + t.Error("Expected error for invalid compression algorithm, got nil") + } } // writeTempConfig is a helper to write a temp config file for testing. diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index 19bd473..8359434 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -73,21 +73,32 @@ func NewServer(configPath string) (*http.Server, error) { return } - tarGz := CreateTarGzStream(ctx, files) + var ( + archive io.ReadCloser + extension string + ) + + if C.CompressionAlgorithm == "bzip2" { + archive = CreateTarBzip2Stream(ctx, files) + extension = "tar.bz2" + } else { + archive = CreateTarGzStream(ctx, files) + extension = "tar.gz" + } defer func() { - if err := tarGz.Close(); err != nil { - log.Errorf("Failed to close tar.gz reader: %v", err) + if err := archive.Close(); err != nil { + log.Errorf("Failed to close archive reader: %v", err) } }() timestamp := time.Now().Format("20060102-150405") - filename := fmt.Sprintf("%s-%s.tar.gz", C.FilenamePrefix, timestamp) + filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension) w.Header().Set("Content-Disposition", "attachment; filename="+filename) w.Header().Set("Content-Type", "application/octet-stream") - bytesWritten, err := io.Copy(w, tarGz) + bytesWritten, err := io.Copy(w, archive) duration := time.Since(backupStart) if err != nil { diff --git a/internal/backer/types.go b/internal/backer/types.go index 5f7390d..fc9ee4d 100644 --- a/internal/backer/types.go +++ b/internal/backer/types.go @@ -3,21 +3,22 @@ package backer type ( // Config main config struct. Config struct { - Address string `json:"address"` - Port int `json:"port"` - Cert string `json:"cert"` - Key string `json:"key"` - NoHTTPS bool `json:"nohttps"` - Location string `json:"location"` - User string `json:"user"` - Password string `json:"password"` - Log string `json:"log"` - LogLevel string `json:"loglevel"` - Directories []string `json:"directories"` - BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming. - CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9). - ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. - FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup"). + Address string `json:"address"` + Port int `json:"port"` + Cert string `json:"cert"` + Key string `json:"key"` + NoHTTPS bool `json:"nohttps"` + Location string `json:"location"` + User string `json:"user"` + Password string `json:"password"` + Log string `json:"log"` + LogLevel string `json:"loglevel"` + Directories []string `json:"directories"` + BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming. + CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9). + ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. + FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup"). + CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip or bzip2 (default: "gzip"). } ) diff --git a/internal/backer/utils.go b/internal/backer/utils.go index 4e0f3ab..7055959 100644 --- a/internal/backer/utils.go +++ b/internal/backer/utils.go @@ -11,6 +11,8 @@ import ( "strings" "backer/internal/log" + + "github.com/dsnet/compress/bzip2" ) // CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files. @@ -162,6 +164,145 @@ func CreateTarGzStream(ctx context.Context, filepaths []string) io.ReadCloser { return pr } +// CreateTarBzip2Stream takes a list of files and returns a reader that reads from a bzip2-compressed tar archive. +func CreateTarBzip2Stream(ctx context.Context, filepaths []string) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + var pipeErr error + + closePipeWithError := func(err error) { + if pipeErr != nil { + return + } + + pipeErr = err + pw.CloseWithError(err) + } + + defer func() { + if pipeErr == nil { + pw.Close() + } + }() + + bw, err := bzip2.NewWriter(pw, nil) + if err != nil { + closePipeWithError(fmt.Errorf("bzip2 writer error: %w", err)) + + return + } + + defer func() { + if pipeErr == nil { + if err := bw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close bzip2 writer: %v", err) + } + } + } + }() + + tw := tar.NewWriter(bw) + + defer func() { + if pipeErr == nil { + if err := tw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close tar writer: %v", err) + } + } + } + }() + + for _, fpath := range filepaths { + var ( + err error + linkTarget string + ) + + select { + case <-ctx.Done(): + closePipeWithError(ctx.Err()) + + return + + default: + } + + log.Debugf("Adding file: %s", fpath) + + st, err := os.Lstat(fpath) + + if err != nil { + log.Warnf("Skipping %s: %v", fpath, err) + + continue + } + + if st.Mode()&os.ModeSymlink != 0 { + linkTarget, err = os.Readlink(fpath) + + if err != nil { + log.Warnf("Skipping %s: readlink failed: %v", fpath, err) + + continue + } + } + + if st.Mode()&os.ModeSocket != 0 { + log.Debugf("Skipping socket: %s", fpath) + + continue + } + + header, err := tar.FileInfoHeader(st, linkTarget) + + if err != nil { + log.Warnf("Skipping %s: tar header failed: %v", fpath, err) + + continue + } + + header.Format = tar.FormatGNU + header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") + + mode := st.Mode() + + if err := writeWithContext(ctx, func() error { + return tw.WriteHeader(header) + }); err != nil { + closePipeWithError(err) + + return + } + + if mode.IsRegular() { + f, err := os.Open(fpath) + if err != nil { + log.Warnf("Skipping %s: open failed: %v", fpath, err) + + continue + } + + err = copyWithContext(ctx, tw, f) + + if e := f.Close(); e != nil { + log.Warnf("Failed to close file %s: %v", fpath, e) + } + + if err != nil { + log.Warnf("Skipping %s: copy failed: %v", fpath, err) + + continue + } + } + } + }() + + return pr +} + // isPipeClosedError checks if the error is due to a closed pipe. func isPipeClosedError(err error) bool { if err == nil { diff --git a/llms.txt b/llms.txt index 214b199..ea7b4a0 100644 --- a/llms.txt +++ b/llms.txt @@ -62,6 +62,7 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz | compression_level | 9 | Gzip level (1-9) | | exclude_patterns | [] | Regex patterns to exclude from backup | | filename_prefix | backup | Prefix for backup filename in Content-Disposition header | +| compression_algorithm | gzip | Compression algorithm (gzip or bzip2) | ## Key Patterns @@ -203,6 +204,4 @@ var version = "dev" // overridden by: -X main.version=$(VERSION) ``` ## Known Issues / TODO -See TODO.org for current backlog: -- Consider different compression algorithms (bz2, xz, lzo, lz4) -- Consider configurable filename prefix for content-disposition +See TODO.org for current backlog.