eleksir/backer
0
Fork 0
Code Pull Requests Releases Activity

Implemented tar+bz2 compression + unit tests for it.

Browse Source
This commit is contained in:
eleksir
2026-03-27 21:28:27 +03:00
parent 05c9eb6669
commit 5226688836
12 changed files with 247 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/backer/loadConfig.go
View File

@@ -135,6 +135,17 @@ func LoadConfig(path string) error {
log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix)
}
// CompressionAlgorithm: default and validate.
if C.CompressionAlgorithm == "" {
C.CompressionAlgorithm = "gzip"
log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm)
}
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" {
return fmt.Errorf("Compression_algorithm must be gzip or bzip2, got %s", C.CompressionAlgorithm)
}
// Compile exclude patterns.
excludePatterns = make([]*regexp.Regexp, 0, len(C.ExcludePatterns))

42
internal/backer/loadConfig_test.go
View File

@@ -226,6 +226,48 @@ func TestLoadConfigDefaultValues(t *testing.T) {
if C.FilenamePrefix != "backup" {
t.Errorf("C.FilenamePrefix should default to backup, got %s", C.FilenamePrefix)
}
if C.CompressionAlgorithm != "gzip" {
t.Errorf("C.CompressionAlgorithm should default to gzip, got %s", C.CompressionAlgorithm)
}
}
// TestLoadConfigCompressionAlgorithmBzip2 tests bzip2 compression algorithm.
func TestLoadConfigCompressionAlgorithmBzip2(t *testing.T) {
config := `{
"user": "test",
"password": "test",
"directories": ["../../test_data/test1/foo"],
"nohttps": true,
"compression_algorithm": "bzip2"
}`
tmpFile := writeTempConfig(t, config)
err := LoadConfig(tmpFile)
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
if C.CompressionAlgorithm != "bzip2" {
t.Errorf("C.CompressionAlgorithm should be bzip2, got %s", C.CompressionAlgorithm)
}
}
// TestLoadConfigCompressionAlgorithmInvalid tests invalid compression algorithm.
func TestLoadConfigCompressionAlgorithmInvalid(t *testing.T) {
config := `{
"user": "test",
"password": "test",
"directories": ["../../test_data/test1/foo"],
"nohttps": true,
"compression_algorithm": "invalid"
}`
tmpFile := writeTempConfig(t, config)
err := LoadConfig(tmpFile)
if err == nil {
t.Error("Expected error for invalid compression algorithm, got nil")
}
}
// writeTempConfig is a helper to write a temp config file for testing.

21
internal/backer/newServer.go
View File

@@ -73,21 +73,32 @@ func NewServer(configPath string) (*http.Server, error) {
return
}
tarGz := CreateTarGzStream(ctx, files)
var (
archive io.ReadCloser
extension string
)
if C.CompressionAlgorithm == "bzip2" {
archive = CreateTarBzip2Stream(ctx, files)
extension = "tar.bz2"
} else {
archive = CreateTarGzStream(ctx, files)
extension = "tar.gz"
}
defer func() {
if err := tarGz.Close(); err != nil {
log.Errorf("Failed to close tar.gz reader: %v", err)
if err := archive.Close(); err != nil {
log.Errorf("Failed to close archive reader: %v", err)
}
}()
timestamp := time.Now().Format("20060102-150405")
filename := fmt.Sprintf("%s-%s.tar.gz", C.FilenamePrefix, timestamp)
filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension)
w.Header().Set("Content-Disposition", "attachment; filename="+filename)
w.Header().Set("Content-Type", "application/octet-stream")
bytesWritten, err := io.Copy(w, tarGz)
bytesWritten, err := io.Copy(w, archive)
duration := time.Since(backupStart)
if err != nil {

31
internal/backer/types.go
View File

@@ -3,21 +3,22 @@ package backer
type (
// Config main config struct.
Config struct {
Address string `json:"address"`
Port int `json:"port"`
Cert string `json:"cert"`
Key string `json:"key"`
NoHTTPS bool `json:"nohttps"`
Location string `json:"location"`
User string `json:"user"`
Password string `json:"password"`
Log string `json:"log"`
LogLevel string `json:"loglevel"`
Directories []string `json:"directories"`
BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming.
CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9).
ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup.
FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup").
Address string `json:"address"`
Port int `json:"port"`
Cert string `json:"cert"`
Key string `json:"key"`
NoHTTPS bool `json:"nohttps"`
Location string `json:"location"`
User string `json:"user"`
Password string `json:"password"`
Log string `json:"log"`
LogLevel string `json:"loglevel"`
Directories []string `json:"directories"`
BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming.
CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9).
ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup.
FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup").
CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip or bzip2 (default: "gzip").
}
)

141
internal/backer/utils.go
View File

@@ -11,6 +11,8 @@ import (
"strings"
"backer/internal/log"
"github.com/dsnet/compress/bzip2"
)
// CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files.
@@ -162,6 +164,145 @@ func CreateTarGzStream(ctx context.Context, filepaths []string) io.ReadCloser {
return pr
}
// CreateTarBzip2Stream takes a list of files and returns a reader that reads from a bzip2-compressed tar archive.
func CreateTarBzip2Stream(ctx context.Context, filepaths []string) io.ReadCloser {
pr, pw := io.Pipe()
go func() {
var pipeErr error
closePipeWithError := func(err error) {
if pipeErr != nil {
return
}
pipeErr = err
pw.CloseWithError(err)
}
defer func() {
if pipeErr == nil {
pw.Close()
}
}()
bw, err := bzip2.NewWriter(pw, nil)
if err != nil {
closePipeWithError(fmt.Errorf("bzip2 writer error: %w", err))
return
}
defer func() {
if pipeErr == nil {
if err := bw.Close(); err != nil {
if !isPipeClosedError(err) {
log.Errorf("Failed to close bzip2 writer: %v", err)
}
}
}
}()
tw := tar.NewWriter(bw)
defer func() {
if pipeErr == nil {
if err := tw.Close(); err != nil {
if !isPipeClosedError(err) {
log.Errorf("Failed to close tar writer: %v", err)
}
}
}
}()
for _, fpath := range filepaths {
var (
err error
linkTarget string
)
select {
case <-ctx.Done():
closePipeWithError(ctx.Err())
return
default:
}
log.Debugf("Adding file: %s", fpath)
st, err := os.Lstat(fpath)
if err != nil {
log.Warnf("Skipping %s: %v", fpath, err)
continue
}
if st.Mode()&os.ModeSymlink != 0 {
linkTarget, err = os.Readlink(fpath)
if err != nil {
log.Warnf("Skipping %s: readlink failed: %v", fpath, err)
continue
}
}
if st.Mode()&os.ModeSocket != 0 {
log.Debugf("Skipping socket: %s", fpath)
continue
}
header, err := tar.FileInfoHeader(st, linkTarget)
if err != nil {
log.Warnf("Skipping %s: tar header failed: %v", fpath, err)
continue
}
header.Format = tar.FormatGNU
header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/")
mode := st.Mode()
if err := writeWithContext(ctx, func() error {
return tw.WriteHeader(header)
}); err != nil {
closePipeWithError(err)
return
}
if mode.IsRegular() {
f, err := os.Open(fpath)
if err != nil {
log.Warnf("Skipping %s: open failed: %v", fpath, err)
continue
}
err = copyWithContext(ctx, tw, f)
if e := f.Close(); e != nil {
log.Warnf("Failed to close file %s: %v", fpath, e)
}
if err != nil {
log.Warnf("Skipping %s: copy failed: %v", fpath, err)
continue
}
}
}
}()
return pr
}
// isPipeClosedError checks if the error is due to a closed pipe.
func isPipeClosedError(err error) bool {
if err == nil {