eleksir/backer
0
Fork 0
Code Pull Requests Releases Activity

Implemented tar+zstd compression + unit tests for it.

Browse Source
This commit is contained in:
eleksir
2026-03-27 23:03:44 +03:00
parent 5226688836
commit a73354b59d
11 changed files with 196 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
AGENTS.md
View File

@@ -182,7 +182,7 @@ Config is JSON (HJSON with comments supported). Default path depends on OS:
| compression_level | `9` | Gzip level (1-9) |
| exclude_patterns | `[]` | Regex patterns to exclude from backup |
| filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header |
| compression_algorithm | `gzip` | Compression algorithm: gzip or bzip2 |
| compression_algorithm | `gzip` | Compression algorithm: gzip, bzip2 or zstd |
## Architecture & Key Patterns

2
README.md
View File

@@ -57,7 +57,7 @@ AI suggests to fill this section with useful information, despite that all optio
| compression_level | 9 | Gzip compression level for tar.gz archives. | 1 (fastest) to 9 (best compression). Higher values produce smaller files but take longer to compress. |
| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. Files matching any pattern are skipped. |
| filename_prefix | "backup" | Prefix for backup filename in Content-Disposition header. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. |
| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2". Gzip is faster, bzip2 produces smaller files. |
| compression_algorithm | "gzip" | Compression algorithm for archive. | Options: "gzip", "bzip2", "zstd". Gzip is fastest, bzip2 compresses better, zstd is a good balance. |
## Best practices

3
TODO.org
View File

@@ -8,6 +8,9 @@
- [X] Evaluate alternatives to gzip: bz2, xz, lzo, lz4
- CANCELLED lzo. It has cgo implementaion which depends in liblzo, and native lid that does not support streaming.
- [X] Implemented bzip2 as alternative (pure Go, via github.com/dsnet/compress)
- [ ] Implement xz as another alternative (pure Go, via github.com/ulikunitz/xz)
- [X] Implement zstd as another alternative (pure Go, via github.com/klauspost/compress/zstd)
- [ ] Implement lz4 as another alternative (pure Go, via github.com/pierrec/lz4)
- [X] Add compression_algorithm config option
** DONE Configurable filename prefix

2
data/config_example.json
View File

@@ -47,6 +47,6 @@
/* Prefix for backup filename in Content-Disposition header. */
"filename_prefix": "backup",
/* Compression algorithm: gzip or bzip2. */
/* Compression algorithm: gzip, bzip2 or zstd. */
"compression_algorithm": "gzip"
}

5
go.mod
View File

@@ -4,4 +4,7 @@ go 1.25
require github.com/hjson/hjson-go v3.3.0+incompatible
require github.com/dsnet/compress v0.0.1 // indirect
require (
github.com/dsnet/compress v0.0.1 // indirect
github.com/klauspost/compress v1.18.5 // indirect
)

2
go.sum
View File

@@ -4,5 +4,7 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=

4
internal/backer/loadConfig.go
View File

@@ -142,8 +142,8 @@ func LoadConfig(path string) error {
log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm)
}
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" {
return fmt.Errorf("Compression_algorithm must be gzip or bzip2, got %s", C.CompressionAlgorithm)
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" {
return fmt.Errorf("Compression_algorithm must be gzip, bzip2 or zstd, got %s", C.CompressionAlgorithm)
}
// Compile exclude patterns.

21
internal/backer/loadConfig_test.go
View File

@@ -253,6 +253,27 @@ func TestLoadConfigCompressionAlgorithmBzip2(t *testing.T) {
}
}
// TestLoadConfigCompressionAlgorithmZstd tests zstd compression algorithm.
func TestLoadConfigCompressionAlgorithmZstd(t *testing.T) {
config := `{
"user": "test",
"password": "test",
"directories": ["../../test_data/test1/foo"],
"nohttps": true,
"compression_algorithm": "zstd"
}`
tmpFile := writeTempConfig(t, config)
err := LoadConfig(tmpFile)
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
if C.CompressionAlgorithm != "zstd" {
t.Errorf("C.CompressionAlgorithm should be zstd, got %s", C.CompressionAlgorithm)
}
}
// TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm.
func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) {
config := `{

10
internal/backer/newServer.go
View File

@@ -78,10 +78,16 @@ func NewServer(configPath string) (*http.Server, error) {
extension string
)
if C.CompressionAlgorithm == "bzip2" {
switch C.CompressionAlgorithm {
case "bzip2":
archive = CreateTarBzip2Stream(ctx, files)
extension = "tar.bz2"
} else {
case "zstd":
archive = CreateTarZstdStream(ctx, files)
extension = "tar.zst"
default:
archive = CreateTarGzStream(ctx, files)
extension = "tar.gz"
}

152
internal/backer/utils.go
View File

@@ -13,6 +13,7 @@ import (
"backer/internal/log"
"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
)
// CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files.
@@ -303,6 +304,157 @@ func CreateTarBzip2Stream(ctx context.Context, filepaths []string) io.ReadCloser
return pr
}
// zstdLevel returns the zstd encoder level based on the 1-9 compression level.
func zstdLevel(level int) zstd.EncoderLevel {
switch {
case level <= 3:
return zstd.SpeedFastest
case level <= 6:
return zstd.SpeedDefault
default:
return zstd.SpeedBetterCompression
}
}
// CreateTarZstdStream takes a list of files and returns a reader that reads from a zstd-compressed tar archive.
func CreateTarZstdStream(ctx context.Context, filepaths []string) io.ReadCloser {
pr, pw := io.Pipe()
go func() {
var pipeErr error
closePipeWithError := func(err error) {
if pipeErr != nil {
return
}
pipeErr = err
pw.CloseWithError(err)
}
defer func() {
if pipeErr == nil {
pw.Close()
}
}()
zw, err := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstdLevel(C.CompressionLevel)))
if err != nil {
closePipeWithError(fmt.Errorf("zstd writer error: %w", err))
return
}
defer func() {
if pipeErr == nil {
if err := zw.Close(); err != nil {
if !isPipeClosedError(err) {
log.Errorf("Failed to close zstd writer: %v", err)
}
}
}
}()
tw := tar.NewWriter(zw)
defer func() {
if pipeErr == nil {
if err := tw.Close(); err != nil {
if !isPipeClosedError(err) {
log.Errorf("Failed to close tar writer: %v", err)
}
}
}
}()
for _, fpath := range filepaths {
var (
err error
linkTarget string
)
select {
case <-ctx.Done():
closePipeWithError(ctx.Err())
return
default:
}
log.Debugf("Adding file: %s", fpath)
st, err := os.Lstat(fpath)
if err != nil {
log.Warnf("Skipping %s: %v", fpath, err)
continue
}
if st.Mode()&os.ModeSymlink != 0 {
linkTarget, err = os.Readlink(fpath)
if err != nil {
log.Warnf("Skipping %s: readlink failed: %v", fpath, err)
continue
}
}
if st.Mode()&os.ModeSocket != 0 {
log.Debugf("Skipping socket: %s", fpath)
continue
}
header, err := tar.FileInfoHeader(st, linkTarget)
if err != nil {
log.Warnf("Skipping %s: tar header failed: %v", fpath, err)
continue
}
header.Format = tar.FormatGNU
header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/")
mode := st.Mode()
if err := writeWithContext(ctx, func() error {
return tw.WriteHeader(header)
}); err != nil {
closePipeWithError(err)
return
}
if mode.IsRegular() {
f, err := os.Open(fpath)
if err != nil {
log.Warnf("Skipping %s: open failed: %v", fpath, err)
continue
}
err = copyWithContext(ctx, tw, f)
if e := f.Close(); e != nil {
log.Warnf("Failed to close file %s: %v", fpath, e)
}
if err != nil {
log.Warnf("Skipping %s: copy failed: %v", fpath, err)
continue
}
}
}
}()
return pr
}
// isPipeClosedError checks if the error is due to a closed pipe.
func isPipeClosedError(err error) bool {
if err == nil {

2
llms.txt
View File

@@ -62,7 +62,7 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz
| compression_level | 9 | Gzip level (1-9) |
| exclude_patterns | [] | Regex patterns to exclude from backup |
| filename_prefix | backup | Prefix for backup filename in Content-Disposition header |
| compression_algorithm | gzip | Compression algorithm (gzip or bzip2) |
| compression_algorithm | gzip | Compression algorithm (gzip, bzip2 or zstd) |
## Key Patterns