From 3620623321de290a71100dd6af82486d116a8fb4 Mon Sep 17 00:00:00 2001 From: eleksir Date: Fri, 27 Mar 2026 23:50:27 +0300 Subject: [PATCH] Implemented tar+xz compression + unit tests for it. --- AGENTS.md | 2 +- README.md | 2 +- TODO.org | 3 +- data/config_example.json | 2 +- go.mod | 12 +-- go.sum | 14 +++ internal/backer/loadConfig.go | 4 +- internal/backer/loadConfig_test.go | 21 +++++ internal/backer/newServer.go | 4 + internal/backer/utils.go | 140 +++++++++++++++++++++++++++++ llms.txt | 2 +- 11 files changed, 194 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b1c228..e3a5a36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,7 +182,7 @@ Config is JSON (HJSON with comments supported). Default path depends on OS: | compression_level | `9` | Gzip level (1-9) | | exclude_patterns | `[]` | Regex patterns to exclude from backup | | filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header | -| compression_algorithm | `gzip` | Compression algorithm: gzip, bzip2, zstd or lz4 | +| compression_algorithm | `gzip` | Compression algorithm: gzip, bzip2, zstd, lz4 or xz | ## Architecture & Key Patterns diff --git a/README.md b/README.md index 06774dd..1219e9b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ AI suggests to fill this section with useful information, despite that all optio | compression_level | 9 | Gzip compression level for tar.gz archives. | 1 (fastest) to 9 (best compression). Higher values produce smaller files but take longer to compress. | | exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. Files matching any pattern are skipped. | | filename_prefix | "backup" | Prefix for backup filename in Content-Disposition header. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. | -| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2", "zstd", "lz4". Gzip is fastest, bzip2 compresses best, zstd is balanced, lz4 is very fast with moderate compression. | +| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2", "zstd", "lz4", "xz". Gzip is fastest, lz4 is very fast, zstd is balanced, bzip2 and xz compress best (xz is slowest). | ## Best practices diff --git a/TODO.org b/TODO.org index 25cfae3..276014e 100644 --- a/TODO.org +++ b/TODO.org @@ -9,8 +9,9 @@ - CANCELLED lzo. It has cgo implementaion which depends in liblzo, and native lid that does not support streaming. - [X] Implemented bzip2 as alternative (pure Go, via github.com/dsnet/compress) - [X] Implemented zstd as alternative (pure Go, via github.com/klauspost/compress/zstd) + - [X] Implemented lz4 as alternative (pure Go, via github.com/pierrec/lz4) + - [X] Implemented xz as alternative (pure Go, via github.com/ulikunitz/xz) - [X] Add compression_algorithm config option - - Note: xz and lz4 can be added later if needed (pure Go libs available) ** DONE Configurable filename prefix - [X] Move "backup" prefix to config (currently hardcoded in newServer.go) diff --git a/data/config_example.json b/data/config_example.json index 2e99106..183eac9 100644 --- a/data/config_example.json +++ b/data/config_example.json @@ -47,6 +47,6 @@ /* Prefix for backup filename in Content-Disposition header. */ "filename_prefix": "backup", - /* Compression algorithm: gzip, bzip2, zstd or lz4. */ + /* Compression algorithm: gzip, bzip2, zstd, lz4 or xz. */ "compression_algorithm": "gzip" } diff --git a/go.mod b/go.mod index 5958039..ac79e3d 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,12 @@ module backer go 1.25 -require github.com/hjson/hjson-go v3.3.0+incompatible - require ( - github.com/dsnet/compress v0.0.1 // indirect - github.com/klauspost/compress v1.18.5 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/dsnet/compress v0.0.1 + github.com/hjson/hjson-go v3.3.0+incompatible + github.com/klauspost/compress v1.18.5 + github.com/pierrec/lz4 v2.6.1+incompatible + github.com/ulikunitz/xz v0.5.15 ) + +require github.com/frankban/quicktest v1.14.6 // indirect diff --git a/go.sum b/go.sum index d49380c..5b7f8a8 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,26 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI= github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index c1e6737..8fb221f 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -142,8 +142,8 @@ func LoadConfig(path string) error { log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm) } - if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" { - return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd or lz4, got %s", C.CompressionAlgorithm) + if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" { + return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd, lz4 or xz, got %s", C.CompressionAlgorithm) } // Compile exclude patterns. diff --git a/internal/backer/loadConfig_test.go b/internal/backer/loadConfig_test.go index 3d795f6..23f63ac 100644 --- a/internal/backer/loadConfig_test.go +++ b/internal/backer/loadConfig_test.go @@ -295,6 +295,27 @@ func TestLoadConfigCompressionAlgorithmLz4(t *testing.T) { } } +// TestLoadConfigCompressionAlgorithmXz tests xz compression algorithm. +func TestLoadConfigCompressionAlgorithmXz(t *testing.T) { + config := `{ + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": true, + "compression_algorithm": "xz" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if C.CompressionAlgorithm != "xz" { + t.Errorf("C.CompressionAlgorithm should be xz, got %s", C.CompressionAlgorithm) + } +} + // TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm. func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) { config := `{ diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index 6ff346e..7cd8339 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -91,6 +91,10 @@ func NewServer(configPath string) (*http.Server, error) { archive = CreateTarLz4Stream(ctx, files) extension = "tar.lz4" + case "xz": + archive = CreateTarXzStream(ctx, files) + extension = "tar.xz" + default: archive = CreateTarGzStream(ctx, files) extension = "tar.gz" diff --git a/internal/backer/utils.go b/internal/backer/utils.go index 1db1c98..66b806e 100644 --- a/internal/backer/utils.go +++ b/internal/backer/utils.go @@ -15,6 +15,7 @@ import ( "github.com/dsnet/compress/bzip2" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4" + "github.com/ulikunitz/xz" ) // CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files. @@ -590,6 +591,145 @@ func CreateTarLz4Stream(ctx context.Context, filepaths []string) io.ReadCloser { return pr } +// CreateTarXzStream takes a list of files and returns a reader that reads from an xz-compressed tar archive. +func CreateTarXzStream(ctx context.Context, filepaths []string) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + var pipeErr error + + closePipeWithError := func(err error) { + if pipeErr != nil { + return + } + + pipeErr = err + pw.CloseWithError(err) + } + + defer func() { + if pipeErr == nil { + pw.Close() + } + }() + + xw, err := xz.NewWriter(pw) + if err != nil { + closePipeWithError(fmt.Errorf("xz writer error: %w", err)) + + return + } + + defer func() { + if pipeErr == nil { + if err := xw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close xz writer: %v", err) + } + } + } + }() + + tw := tar.NewWriter(xw) + + defer func() { + if pipeErr == nil { + if err := tw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close tar writer: %v", err) + } + } + } + }() + + for _, fpath := range filepaths { + var ( + err error + linkTarget string + ) + + select { + case <-ctx.Done(): + closePipeWithError(ctx.Err()) + + return + + default: + } + + log.Debugf("Adding file: %s", fpath) + + st, err := os.Lstat(fpath) + + if err != nil { + log.Warnf("Skipping %s: %v", fpath, err) + + continue + } + + if st.Mode()&os.ModeSymlink != 0 { + linkTarget, err = os.Readlink(fpath) + + if err != nil { + log.Warnf("Skipping %s: readlink failed: %v", fpath, err) + + continue + } + } + + if st.Mode()&os.ModeSocket != 0 { + log.Debugf("Skipping socket: %s", fpath) + + continue + } + + header, err := tar.FileInfoHeader(st, linkTarget) + + if err != nil { + log.Warnf("Skipping %s: tar header failed: %v", fpath, err) + + continue + } + + header.Format = tar.FormatGNU + header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") + + mode := st.Mode() + + if err := writeWithContext(ctx, func() error { + return tw.WriteHeader(header) + }); err != nil { + closePipeWithError(err) + + return + } + + if mode.IsRegular() { + f, err := os.Open(fpath) + if err != nil { + log.Warnf("Skipping %s: open failed: %v", fpath, err) + + continue + } + + err = copyWithContext(ctx, tw, f) + + if e := f.Close(); e != nil { + log.Warnf("Failed to close file %s: %v", fpath, e) + } + + if err != nil { + log.Warnf("Skipping %s: copy failed: %v", fpath, err) + + continue + } + } + } + }() + + return pr +} + // isPipeClosedError checks if the error is due to a closed pipe. func isPipeClosedError(err error) bool { if err == nil { diff --git a/llms.txt b/llms.txt index 7066396..58b8f94 100644 --- a/llms.txt +++ b/llms.txt @@ -62,7 +62,7 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz | compression_level | 9 | Gzip level (1-9) | | exclude_patterns | [] | Regex patterns to exclude from backup | | filename_prefix | backup | Prefix for backup filename in Content-Disposition header | -| compression_algorithm | gzip | Compression algorithm (gzip, bzip2, zstd or lz4) | +| compression_algorithm | gzip | Compression algorithm (gzip, bzip2, zstd, lz4 or xz) | ## Key Patterns