From 15ac3be4109e09f1111602ea20951951795d2787 Mon Sep 17 00:00:00 2001 From: eleksir Date: Sat, 28 Mar 2026 00:19:09 +0300 Subject: [PATCH] Implemented tar+parallel gzip compression + unit tests for it. --- AGENTS.md | 2 +- README.md | 52 ++++++----- data/config_example.json | 2 +- go.mod | 1 + go.sum | 2 + internal/backer/loadConfig.go | 4 +- internal/backer/loadConfig_test.go | 21 +++++ internal/backer/newServer.go | 4 + internal/backer/utils.go | 140 +++++++++++++++++++++++++++++ llms.txt | 2 +- 10 files changed, 204 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e3a5a36..7190c66 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,7 +182,7 @@ Config is JSON (HJSON with comments supported). Default path depends on OS: | compression_level | `9` | Gzip level (1-9) | | exclude_patterns | `[]` | Regex patterns to exclude from backup | | filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header | -| compression_algorithm | `gzip` | Compression algorithm: gzip, bzip2, zstd, lz4 or xz | +| compression_algorithm | `gzip` | Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz | ## Architecture & Key Patterns diff --git a/README.md b/README.md index 1219e9b..d347a10 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Backer -Kind of backup utility. It makes tar.gz on-the-fly out of given in config.json files. +HTTP/HTTPS backup server that creates compressed archives on-the-fly from configured directories. Supports multiple compression algorithms: gzip, pgzip, bzip2, zstd, lz4, xz. ## How to build it @@ -8,9 +8,8 @@ Kind of backup utility. It makes tar.gz on-the-fly out of given in config.json f make # Build binary make build VERSION=0.1.0 # Build with version make test # Run tests -backer # Run -backer --version # Show version -backer -c /path/to/config.json # Custom config +make # Clean + build +make upgrade # Update dependencies ``` That will produce binary named `backer`. Copy it somewhere in `/usr/local/sbin/backer`, for example. In `data` @@ -32,32 +31,43 @@ Build, install and configure backer. On your backup storage server you can do curl -u username:password https://your_server:8086/archive -o "backup-$(date +%Y%m%d-%H%M%S).tar.gz" ``` -and you'll get your backup with timestamp in filename, like `backup-20260325-105100.tar.gz` +The filename and extension depend on your `filename_prefix` and `compression_algorithm` settings. Of course, something can go wrong, so you must check http status code after downloading backup. ## About config options -AI suggests to fill this section with useful information, despite that all options described in example config. +All options described in example config. | Option | Default value | Wtf | Notes | | :--- | :---- | :--- | :--- | | address | "0.0.0.0" | Address at which server listens. | In many cases 0.0.0.0 is okay. | -| port | 8086 | Port at which server binds | Number picked after intel first comercially successful cpu. | -| cert | "/path/to/ssl.crt" | Path to ssl certificate file | It is not enforced in obvious form, but you should limit access to certificate at least with file permissions. | -| key | "/path/to/ssl.key" | Path to ssl certificates key file | It is not enforced in obvious form, but you should limit access to certificates key at least with file permissions. | -| nohttps | false | ability to disable https and use plain http. | This option is for lab experiments or for development needs. In case of this utility you should consider using long enough certificate with well ciphers. | -| location | "/archive" | This is static location where curl (or more sophisticated client) should aim to download backup | /archive is okay, but you free to choose something more wild like "/backup", just for giggles. | -| user | | You should pick up some flashy username for basic auth to prevent unauthorized access to your backup. | Enough said. | -| password | | You should pick up something super-duper secretous to prevent unauthorized access to your backup. | Enough said. | -| log | stderr | Where log messages are directed. If no value set, logs drops to stderr | | -| loglevel | | Verbosity of logs. | Can be error, warn, info, debug. Pick whatever you like. | -| directories | | An array with directories. Backer can work only with directories, not single files. | Put here all the folders you'd like to backup! | -| backup_timeout | 60 | Timeout in minutes for backup streaming operations. | If a backup takes longer than this, the connection will be terminated. Useful for large backups. Range: 1-1440 minutes (1 day max). | -| compression_level | 9 | Gzip compression level for tar.gz archives. | 1 (fastest) to 9 (best compression). Higher values produce smaller files but take longer to compress. | -| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. Files matching any pattern are skipped. | -| filename_prefix | "backup" | Prefix for backup filename in Content-Disposition header. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. | -| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2", "zstd", "lz4", "xz". Gzip is fastest, lz4 is very fast, zstd is balanced, bzip2 and xz compress best (xz is slowest). | +| port | 8086 | Port at which server binds. | Number picked after intel first commercially successful cpu. | +| cert | — | Path to ssl certificate file. | Required when nohttps is false. | +| key | — | Path to ssl certificates key file. | Required when nohttps is false. | +| nohttps | false | Disable https and use plain http. | For lab experiments or development. | +| location | "/archive" | API endpoint path for backup download. | | +| user | — | Username for basic auth. | Required. | +| password | — | Password for basic auth. | Required. | +| log | stderr | Log output file path. | | +| loglevel | "info" | Verbosity of logs. | Options: error, warn, info, debug. | +| directories | — | Directories to backup. | Required. Array of paths. | +| backup_timeout | 60 | Timeout in minutes for backup streaming. | Range: 1-1440 minutes. | +| compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). | +| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. | +| filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. | +| compression_algorithm | "gzip" | Compression algorithm. | Options: gzip, pgzip, bzip2, zstd, lz4, xz. | + +## Compression algorithms + +| Algorithm | Speed | Compression | Notes | +| :--- | :--- | :--- | :--- | +| gzip | Fast | Good | Default. Standard, widely compatible. | +| pgzip | Fast | Good | Parallel gzip. Uses multiple CPU cores, slightly larger files. | +| bzip2 | Slow | Better | Pure Go. Good compression, slower than gzip. | +| zstd | Fast | Very good | Good balance of speed and compression. | +| lz4 | Fastest | Moderate | Extremely fast, larger files. | +| xz | Slowest | Best | Best compression, slowest speed. | ## Best practices diff --git a/data/config_example.json b/data/config_example.json index 183eac9..66867ae 100644 --- a/data/config_example.json +++ b/data/config_example.json @@ -47,6 +47,6 @@ /* Prefix for backup filename in Content-Disposition header. */ "filename_prefix": "backup", - /* Compression algorithm: gzip, bzip2, zstd, lz4 or xz. */ + /* Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. */ "compression_algorithm": "gzip" } diff --git a/go.mod b/go.mod index ac79e3d..6d150e4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/dsnet/compress v0.0.1 github.com/hjson/hjson-go v3.3.0+incompatible github.com/klauspost/compress v1.18.5 + github.com/klauspost/pgzip v1.2.6 github.com/pierrec/lz4 v2.6.1+incompatible github.com/ulikunitz/xz v0.5.15 ) diff --git a/go.sum b/go.sum index 5b7f8a8..686e61b 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index 8fb221f..c6da86e 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -142,8 +142,8 @@ func LoadConfig(path string) error { log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm) } - if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" { - return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd, lz4 or xz, got %s", C.CompressionAlgorithm) + if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" && C.CompressionAlgorithm != "pgzip" { + return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.CompressionAlgorithm) } // Compile exclude patterns. diff --git a/internal/backer/loadConfig_test.go b/internal/backer/loadConfig_test.go index 23f63ac..a4c4cd7 100644 --- a/internal/backer/loadConfig_test.go +++ b/internal/backer/loadConfig_test.go @@ -316,6 +316,27 @@ func TestLoadConfigCompressionAlgorithmXz(t *testing.T) { } } +// TestLoadConfigCompressionAlgorithmPgzip tests pgzip compression algorithm. +func TestLoadConfigCompressionAlgorithmPgzip(t *testing.T) { + config := `{ + "user": "test", + "password": "test", + "directories": ["../../test_data/test1/foo"], + "nohttps": true, + "compression_algorithm": "pgzip" + }` + tmpFile := writeTempConfig(t, config) + + err := LoadConfig(tmpFile) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if C.CompressionAlgorithm != "pgzip" { + t.Errorf("C.CompressionAlgorithm should be pgzip, got %s", C.CompressionAlgorithm) + } +} + // TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm. func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) { config := `{ diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index 7cd8339..ea83ea4 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -95,6 +95,10 @@ func NewServer(configPath string) (*http.Server, error) { archive = CreateTarXzStream(ctx, files) extension = "tar.xz" + case "pgzip": + archive = CreateTarPgzipStream(ctx, files) + extension = "tar.gz" + default: archive = CreateTarGzStream(ctx, files) extension = "tar.gz" diff --git a/internal/backer/utils.go b/internal/backer/utils.go index 66b806e..364aa5f 100644 --- a/internal/backer/utils.go +++ b/internal/backer/utils.go @@ -14,6 +14,7 @@ import ( "github.com/dsnet/compress/bzip2" "github.com/klauspost/compress/zstd" + "github.com/klauspost/pgzip" "github.com/pierrec/lz4" "github.com/ulikunitz/xz" ) @@ -730,6 +731,145 @@ func CreateTarXzStream(ctx context.Context, filepaths []string) io.ReadCloser { return pr } +// CreateTarPgzipStream takes a list of files and returns a reader that reads from a parallel gzip-compressed tar archive. +func CreateTarPgzipStream(ctx context.Context, filepaths []string) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + var pipeErr error + + closePipeWithError := func(err error) { + if pipeErr != nil { + return + } + + pipeErr = err + pw.CloseWithError(err) + } + + defer func() { + if pipeErr == nil { + pw.Close() + } + }() + + gw, err := pgzip.NewWriterLevel(pw, C.CompressionLevel) + if err != nil { + closePipeWithError(fmt.Errorf("pgzip writer error: %w", err)) + + return + } + + defer func() { + if pipeErr == nil { + if err := gw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close pgzip writer: %v", err) + } + } + } + }() + + tw := tar.NewWriter(gw) + + defer func() { + if pipeErr == nil { + if err := tw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close tar writer: %v", err) + } + } + } + }() + + for _, fpath := range filepaths { + var ( + err error + linkTarget string + ) + + select { + case <-ctx.Done(): + closePipeWithError(ctx.Err()) + + return + + default: + } + + log.Debugf("Adding file: %s", fpath) + + st, err := os.Lstat(fpath) + + if err != nil { + log.Warnf("Skipping %s: %v", fpath, err) + + continue + } + + if st.Mode()&os.ModeSymlink != 0 { + linkTarget, err = os.Readlink(fpath) + + if err != nil { + log.Warnf("Skipping %s: readlink failed: %v", fpath, err) + + continue + } + } + + if st.Mode()&os.ModeSocket != 0 { + log.Debugf("Skipping socket: %s", fpath) + + continue + } + + header, err := tar.FileInfoHeader(st, linkTarget) + + if err != nil { + log.Warnf("Skipping %s: tar header failed: %v", fpath, err) + + continue + } + + header.Format = tar.FormatGNU + header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") + + mode := st.Mode() + + if err := writeWithContext(ctx, func() error { + return tw.WriteHeader(header) + }); err != nil { + closePipeWithError(err) + + return + } + + if mode.IsRegular() { + f, err := os.Open(fpath) + if err != nil { + log.Warnf("Skipping %s: open failed: %v", fpath, err) + + continue + } + + err = copyWithContext(ctx, tw, f) + + if e := f.Close(); e != nil { + log.Warnf("Failed to close file %s: %v", fpath, e) + } + + if err != nil { + log.Warnf("Skipping %s: copy failed: %v", fpath, err) + + continue + } + } + } + }() + + return pr +} + // isPipeClosedError checks if the error is due to a closed pipe. func isPipeClosedError(err error) bool { if err == nil { diff --git a/llms.txt b/llms.txt index 58b8f94..116e110 100644 --- a/llms.txt +++ b/llms.txt @@ -62,7 +62,7 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz | compression_level | 9 | Gzip level (1-9) | | exclude_patterns | [] | Regex patterns to exclude from backup | | filename_prefix | backup | Prefix for backup filename in Content-Disposition header | -| compression_algorithm | gzip | Compression algorithm (gzip, bzip2, zstd, lz4 or xz) | +| compression_algorithm | gzip | Compression algorithm (gzip, pgzip, bzip2, zstd, lz4 or xz) | ## Key Patterns