From 722620945dc1d3835c5a462bd2dd308400ec6a21 Mon Sep 17 00:00:00 2001 From: eleksir Date: Sat, 28 Mar 2026 01:38:17 +0300 Subject: [PATCH] Move code to different places, add comments here and there, no actual changes --- AGENTS.md | 35 +- TODO.org | 18 - internal/backer/archive.go | 175 ++++ internal/backer/bzip2.go | 17 + internal/backer/consts.go | 57 ++ internal/backer/getFilesFromDirectories.go | 1 - internal/backer/globals.go | 3 + internal/backer/gzip.go | 25 + internal/backer/loadConfig.go | 34 +- internal/backer/lz4.go | 17 + internal/backer/newServer.go | 15 +- internal/backer/types.go | 6 +- internal/backer/utils.go | 884 --------------------- internal/backer/xz.go | 17 + internal/backer/zstd.go | 29 + llms.txt | 21 +- 16 files changed, 406 insertions(+), 948 deletions(-) delete mode 100644 TODO.org create mode 100644 internal/backer/archive.go create mode 100644 internal/backer/bzip2.go create mode 100644 internal/backer/consts.go create mode 100644 internal/backer/gzip.go create mode 100644 internal/backer/lz4.go delete mode 100644 internal/backer/utils.go create mode 100644 internal/backer/xz.go create mode 100644 internal/backer/zstd.go diff --git a/AGENTS.md b/AGENTS.md index 7190c66..b70270d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project Overview -Backer is a Go-based HTTP/HTTPS backup server that creates `tar.gz` archives on-the-fly from configured directories. It serves a single endpoint that clients (e.g., `curl`) can query to download a timestamped backup archive. Licensed under MIT. +Backer is a Go-based HTTP/HTTPS backup server that creates compressed archives on-the-fly from configured directories. Supports multiple compression algorithms: gzip, pgzip, bzip2, zstd, lz4, xz. It serves a single endpoint that clients (e.g., `curl`) can query to download a timestamped backup archive. Licensed under MIT. ## Workflow Rules @@ -11,7 +11,7 @@ Backer is a Go-based HTTP/HTTPS backup server that creates `tar.gz` archives on- - **When stuck, ask a clarification question, or propose a short plan, or both.** Do not guess intent or make assumptions — ask. - **Always make a TODO plan before acting. And try to follow this plan.** - **If you notice defects, bugs, or issues, always report it.** -- **Track new features, known issues, bugs, and other defects in `TODO.org`.** Update it as you implement features, fix issues, fix bugs, or discover new ones — check off completed items, add new findings. +- **Track new features, known issues, bugs, and other defects.** Update documentation as you implement features, fix issues, fix bugs, or discover new ones. - **Update `AGENTS.md` as the project evolves.** Keep it comprehensive but dense. - **Don't use `context` package unless absolutely necessary.** Try to find another solution first. - **Create unit tests if possible/necessary** to verify changes that are not covered by unit tests. @@ -76,8 +76,8 @@ if C.Port == 0 { log.Warnf("Config option port is not set, fallback to %d", C.Port) } -if C.Port < 1 || C.Port > 65535 { - return fmt.Errorf("Port must be between 1 and 65535, got %d", C.Port) +if C.Port < portMin || C.Port > portMax { + return fmt.Errorf("Config option port must be between %d and %d, got %d", portMin, portMax, C.Port) } ``` @@ -123,15 +123,21 @@ cmd/ internal/ backer/ types.go # Config struct definition + consts.go # Named constants for magic numbers globals.go # Global C Config variable, compiled excludePatterns + archive.go # Common archive logic (pipe, tar, file iteration) + gzip.go # CreateTarGzStream, CreateTarPgzipStream + bzip2.go # CreateTarBzip2Stream + zstd.go # CreateTarZstdStream, zstdLevel helper + lz4.go # CreateTarLz4Stream + xz.go # CreateTarXzStream loadConfig.go # HJSON/JSON config parsing with validation loadConfig_test.go # Config validation tests newServer.go # HTTP server creation, auth handler, context helpers newServer_test.go # Server, auth, and context helper tests - utils.go # CreateTarGzStream (tar.gz streaming via io.Pipe) - utils_test.go # Tar.gz stream tests getFilesFromDirectories.go # Directory walking, exclusion filtering (isExcluded) getFilesFromDirectories_test.go # File traversal and exclude pattern tests + utils_test.go # Archive stream tests log/ main.go # Logging wrapper around stdlib slog main_test.go # Log package tests @@ -150,7 +156,6 @@ test_data/ test1/ # File/directory fixtures for tests Makefile # Build targets -TODO.org # Project backlog (Org mode) README.md # User documentation LICENSE # MIT License llms.txt # LLM-friendly project reference @@ -201,9 +206,10 @@ json.Unmarshal(buf, &C) // decode into Config struct ### Streaming Archive Creation -No temp files on disk. Uses `io.Pipe` with a goroutine to stream tar.gz on-the-fly: -- `CreateTarGzStream()` returns an `io.ReadCloser` -- Goroutine writes tar entries → gzip writer → pipe writer +No temp files on disk. Uses `io.Pipe` with a goroutine to stream archives on-the-fly: +- Common logic in `archive.go`: `createArchiveStream()` handles pipe, goroutine, tar writing, file iteration +- Each compression algorithm has its own file: `gzip.go`, `bzip2.go`, `zstd.go`, `lz4.go`, `xz.go` +- All return an `io.ReadCloser` - HTTP handler copies from pipe reader directly to HTTP response ### Context-Aware I/O @@ -242,6 +248,11 @@ Checks `X-Forwarded-For` header first (for proxied requests), then falls back to ## Dependencies - `github.com/hjson/hjson-go` v3.3.0 — HJSON config parsing +- `github.com/dsnet/compress` v0.0.1 — bzip2 compression +- `github.com/klauspost/compress` v1.18.5 — zstd compression +- `github.com/klauspost/pgzip` v1.2.6 — parallel gzip compression +- `github.com/pierrec/lz4` v2.6.1 — lz4 compression +- `github.com/ulikunitz/xz` v0.5.15 — xz compression ## Testing @@ -275,7 +286,3 @@ Init scripts provided in `contrib/`: - **systemd** (`backer.service`): `ExecStart=/usr/local/sbin/backer`, restart on failure - **OpenRC** (`backer`): uses `supervise-daemon`, creates log/pid dirs in `start_pre` - **FreeBSD rc.d** (`backer.freebsd`): daemonizes via `/usr/sbin/daemon` - -## TODO / Roadmap - -See `TODO.org` for current backlog. diff --git a/TODO.org b/TODO.org deleted file mode 100644 index 276014e..0000000 --- a/TODO.org +++ /dev/null @@ -1,18 +0,0 @@ -#+TITLE: Backer TODO List -#+AUTHOR: Backer Project -#+DATE: <2026-03-25 Wed> - -* TODO Backer Tasks - -** DONE Consider different compression algorithms - - [X] Evaluate alternatives to gzip: bz2, xz, lzo, lz4 - - CANCELLED lzo. It has cgo implementaion which depends in liblzo, and native lid that does not support streaming. - - [X] Implemented bzip2 as alternative (pure Go, via github.com/dsnet/compress) - - [X] Implemented zstd as alternative (pure Go, via github.com/klauspost/compress/zstd) - - [X] Implemented lz4 as alternative (pure Go, via github.com/pierrec/lz4) - - [X] Implemented xz as alternative (pure Go, via github.com/ulikunitz/xz) - - [X] Add compression_algorithm config option - -** DONE Configurable filename prefix - - [X] Move "backup" prefix to config (currently hardcoded in newServer.go) - - [X] Default: "backup", e.g., "backup-20260325-092341.tar.gz" diff --git a/internal/backer/archive.go b/internal/backer/archive.go new file mode 100644 index 0000000..964896d --- /dev/null +++ b/internal/backer/archive.go @@ -0,0 +1,175 @@ +// Package backer implements HTTP/HTTPS backup server. +package backer + +import ( + "archive/tar" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "backer/internal/log" +) + +// createArchiveStream creates a pipe and streams a tar archive compressed by the provided writer. +// The writerFactory creates the compression writer wrapping the pipe writer. +// This is the core of all CreateTar*Stream functions. +func createArchiveStream(ctx context.Context, filepaths []string, writerFactory func(io.Writer) (io.WriteCloser, error)) io.ReadCloser { + pr, pw := io.Pipe() + + go func() { + var pipeErr error + + closePipeWithError := func(err error) { + if pipeErr != nil { + return + } + + pipeErr = err + pw.CloseWithError(err) + } + + defer func() { + if pipeErr == nil { + pw.Close() + } + }() + + cw, err := writerFactory(pw) + if err != nil { + closePipeWithError(fmt.Errorf("compression writer error: %w", err)) + + return + } + + defer func() { + if pipeErr == nil { + if err := cw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close compression writer: %v", err) + } + } + } + }() + + tw := tar.NewWriter(cw) + + defer func() { + if pipeErr == nil { + if err := tw.Close(); err != nil { + if !isPipeClosedError(err) { + log.Errorf("Failed to close tar writer: %v", err) + } + } + } + }() + + writeFilesToTar(ctx, filepaths, tw, closePipeWithError) + }() + + return pr +} + +// writeFilesToTar iterates over filepaths and writes each file to the tar archive. +func writeFilesToTar(ctx context.Context, filepaths []string, tw *tar.Writer, closePipeWithError func(error)) { + for _, fpath := range filepaths { + var ( + err error + linkTarget string + ) + + select { + case <-ctx.Done(): + closePipeWithError(ctx.Err()) + + return + + default: + } + + log.Debugf("Adding file: %s", fpath) + + st, err := os.Lstat(fpath) + + if err != nil { + log.Warnf("Skipping %s: %v", fpath, err) + + continue + } + + if st.Mode()&os.ModeSymlink != 0 { + linkTarget, err = os.Readlink(fpath) + + if err != nil { + log.Warnf("Skipping %s: readlink failed: %v", fpath, err) + + continue + } + } + + // Skip sockets explicitly (they cannot be backed up). + if st.Mode()&os.ModeSocket != 0 { + log.Debugf("Skipping socket: %s", fpath) + + continue + } + + header, err := tar.FileInfoHeader(st, linkTarget) + + if err != nil { + log.Warnf("Skipping %s: tar header failed: %v", fpath, err) + + continue + } + + header.Format = tar.FormatGNU + header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") + + mode := st.Mode() + + if err := writeWithContext(ctx, func() error { + return tw.WriteHeader(header) + }); err != nil { + closePipeWithError(err) + + return + } + + // Only stream file contents for regular files. + if mode.IsRegular() { + f, err := os.Open(fpath) + if err != nil { + log.Warnf("Skipping %s: open failed: %v", fpath, err) + + continue + } + + err = copyWithContext(ctx, tw, f) + + if e := f.Close(); e != nil { + log.Warnf("Failed to close file %s: %v", fpath, e) + } + + if err != nil { + log.Warnf("Skipping %s: copy failed: %v", fpath, err) + + continue + } + } + } +} + +// isPipeClosedError checks if the error is due to a closed pipe. +func isPipeClosedError(err error) bool { + if err == nil { + return false + } + + // Check for the specific pipe closed error message. + return err.Error() == "io: read/write on closed pipe" || + err.Error() == "io: read on closed pipe" +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/bzip2.go b/internal/backer/bzip2.go new file mode 100644 index 0000000..45ef460 --- /dev/null +++ b/internal/backer/bzip2.go @@ -0,0 +1,17 @@ +package backer + +import ( + "context" + "io" + + "github.com/dsnet/compress/bzip2" +) + +// CreateTarBzip2Stream takes a list of files and returns a reader that reads from a bzip2-compressed tar archive. +func CreateTarBzip2Stream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return bzip2.NewWriter(w, nil) + }) +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/consts.go b/internal/backer/consts.go new file mode 100644 index 0000000..e1d9f4f --- /dev/null +++ b/internal/backer/consts.go @@ -0,0 +1,57 @@ +package backer + +import ( + "time" +) + +const ( + // readHeaderTimeout is the maximum duration for reading request headers. + readHeaderTimeout = 5 * time.Second + + // copyBufferSize is the buffer size used for streaming file contents. + copyBufferSize = 32 * 1024 + + // portMin is the minimum valid port number. + portMin = 1 + + // portMax is the maximum valid port number. + portMax = 65535 + + // backupTimeoutMin is the minimum backup timeout in minutes. + backupTimeoutMin = 1 + + // backupTimeoutMax is the maximum backup timeout in minutes (24 hours). + backupTimeoutMax = 1440 + + // compressionLevelMin is the minimum compression level. + compressionLevelMin = 1 + + // compressionLevelMax is the maximum compression level. + compressionLevelMax = 9 + + // defaultPort is the default listen port. + defaultPort = 8086 + + // defaultBackupTimeout is the default backup timeout in minutes. + defaultBackupTimeout = 60 + + // defaultCompressionLevel is the default compression level. + defaultCompressionLevel = 9 + + // defaultLocation is the default API endpoint path. + defaultLocation = "/archive" + + // defaultAddress is the default listen address. + defaultAddress = "0.0.0.0" + + // defaultLogLevel is the default log verbosity. + defaultLogLevel = "info" + + // defaultFilenamePrefix is the default prefix for backup filenames. + defaultFilenamePrefix = "backup" + + // defaultCompressionAlgorithm is the default compression algorithm. + defaultCompressionAlgorithm = "gzip" +) + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/getFilesFromDirectories.go b/internal/backer/getFilesFromDirectories.go index d0ab6cd..bd93cd1 100644 --- a/internal/backer/getFilesFromDirectories.go +++ b/internal/backer/getFilesFromDirectories.go @@ -1,4 +1,3 @@ -// Package backer implements simple backup server. package backer import ( diff --git a/internal/backer/globals.go b/internal/backer/globals.go index 1012969..03fc2f4 100644 --- a/internal/backer/globals.go +++ b/internal/backer/globals.go @@ -6,8 +6,11 @@ import ( var ( // C contains the global configuration loaded from config file. + // This is intentional — config is loaded once at startup and never modified concurrently. C Config + // excludePatterns contains compiled regex patterns from C.ExcludePatterns. + // Populated by LoadConfig, used by isExcluded during directory walking. excludePatterns []*regexp.Regexp ) diff --git a/internal/backer/gzip.go b/internal/backer/gzip.go new file mode 100644 index 0000000..3136746 --- /dev/null +++ b/internal/backer/gzip.go @@ -0,0 +1,25 @@ +package backer + +import ( + "compress/gzip" + "context" + "io" + + "github.com/klauspost/pgzip" +) + +// CreateTarGzStream takes a list of files and returns a reader that reads from a gzip-compressed tar archive. +func CreateTarGzStream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return gzip.NewWriterLevel(w, C.CompressionLevel) + }) +} + +// CreateTarPgzipStream takes a list of files and returns a reader that reads from a parallel gzip-compressed tar archive. +func CreateTarPgzipStream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return pgzip.NewWriterLevel(w, C.CompressionLevel) + }) +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/loadConfig.go b/internal/backer/loadConfig.go index c6da86e..46ffbf9 100644 --- a/internal/backer/loadConfig.go +++ b/internal/backer/loadConfig.go @@ -38,25 +38,25 @@ func LoadConfig(path string) error { // Address: default and no range validation needed. if C.Address == "" { - C.Address = "0.0.0.0" + C.Address = defaultAddress log.Warnf("Config option address is not set, fallback to %s", C.Address) } // Port: set default, then validate range. if C.Port == 0 { - C.Port = 8086 + C.Port = defaultPort log.Warnf("Config option port is not set, fallback to %d", C.Port) } - if C.Port < 1 || C.Port > 65535 { - return fmt.Errorf("Port must be between 1 and 65535, got %d", C.Port) + if C.Port < portMin || C.Port > portMax { + return fmt.Errorf("Config option port must be between %d and %d, got %d", portMin, portMax, C.Port) } // Location: default and no range validation needed. if C.Location == "" { - C.Location = "/archive" + C.Location = defaultLocation log.Warnf("Config option location is not set, fallback to %s", C.Location) } @@ -68,7 +68,7 @@ func LoadConfig(path string) error { // LogLevel: default and no range validation needed. if C.LogLevel == "" { - C.LogLevel = "info" + C.LogLevel = defaultLogLevel log.Warnf("Config option loglevel is not set, fallback to %s", C.LogLevel) } @@ -86,24 +86,24 @@ func LoadConfig(path string) error { // BackupTimeout: set default, then validate range. if C.BackupTimeout == 0 { - C.BackupTimeout = 60 + C.BackupTimeout = defaultBackupTimeout log.Warnf("Config option backup_timeout is not set, fallback to %d minutes", C.BackupTimeout) } - if C.BackupTimeout < 1 || C.BackupTimeout > 1440 { - return fmt.Errorf("Backup_timeout must be between 1 and 1440 minutes, got %d", C.BackupTimeout) + if C.BackupTimeout < backupTimeoutMin || C.BackupTimeout > backupTimeoutMax { + return fmt.Errorf("Config option backup_timeout must be between %d and %d minutes, got %d", backupTimeoutMin, backupTimeoutMax, C.BackupTimeout) } // CompressionLevel: set default, then validate range. if C.CompressionLevel == 0 { - C.CompressionLevel = 9 + C.CompressionLevel = defaultCompressionLevel log.Warnf("Config option compression_level is not set, fallback to %d", C.CompressionLevel) } - if C.CompressionLevel < 1 || C.CompressionLevel > 9 { - return fmt.Errorf("Compression_level must be between 1 and 9, got %d", C.CompressionLevel) + if C.CompressionLevel < compressionLevelMin || C.CompressionLevel > compressionLevelMax { + return fmt.Errorf("Config option compression_level must be between %d and %d, got %d", compressionLevelMin, compressionLevelMax, C.CompressionLevel) } // HTTPS: validate cert/key when HTTPS enabled. @@ -120,30 +120,30 @@ func LoadConfig(path string) error { // User: required. if C.User == "" { - return errors.New("Option user is not set") + return errors.New("Config option user is not set") } // Password: required. if C.Password == "" { - return errors.New("Option password is not set") + return errors.New("Config option password is not set") } // FilenamePrefix: default. if C.FilenamePrefix == "" { - C.FilenamePrefix = "backup" + C.FilenamePrefix = defaultFilenamePrefix log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix) } // CompressionAlgorithm: default and validate. if C.CompressionAlgorithm == "" { - C.CompressionAlgorithm = "gzip" + C.CompressionAlgorithm = defaultCompressionAlgorithm log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm) } if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" && C.CompressionAlgorithm != "pgzip" { - return fmt.Errorf("Compression_algorithm must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.CompressionAlgorithm) + return fmt.Errorf("Config option compression_algorithm must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.CompressionAlgorithm) } // Compile exclude patterns. diff --git a/internal/backer/lz4.go b/internal/backer/lz4.go new file mode 100644 index 0000000..8417cc5 --- /dev/null +++ b/internal/backer/lz4.go @@ -0,0 +1,17 @@ +package backer + +import ( + "context" + "io" + + "github.com/pierrec/lz4" +) + +// CreateTarLz4Stream takes a list of files and returns a reader that reads from an lz4-compressed tar archive. +func CreateTarLz4Stream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return lz4.NewWriter(w), nil + }) +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/newServer.go b/internal/backer/newServer.go index ea83ea4..095cf66 100644 --- a/internal/backer/newServer.go +++ b/internal/backer/newServer.go @@ -134,7 +134,18 @@ func NewServer(configPath string) (*http.Server, error) { return &http.Server{ Addr: fmt.Sprintf("%s:%d", C.Address, C.Port), Handler: withServerHeader(mux), - ReadHeaderTimeout: 5 * time.Second, + ReadHeaderTimeout: readHeaderTimeout, + WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute, + }, nil + } + + // Decision about which mode (http or https) to run made in cmd/backer/main.go, where we run server.ListenAndServe() + // or server.ListenAndServeTLS() depending on value of C.NoHTTPS. + if C.NoHTTPS { + return &http.Server{ + Addr: fmt.Sprintf("%s:%d", C.Address, C.Port), + Handler: withServerHeader(mux), + ReadHeaderTimeout: readHeaderTimeout, WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute, }, nil } @@ -166,7 +177,7 @@ func writeWithContext(ctx context.Context, fn func() error) error { // context cancellation. If the context is canceled, the copy stops and returns // ctx.Err(). func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { - buf := make([]byte, 32*1024) + buf := make([]byte, copyBufferSize) for { select { diff --git a/internal/backer/types.go b/internal/backer/types.go index fc9ee4d..16c6666 100644 --- a/internal/backer/types.go +++ b/internal/backer/types.go @@ -1,7 +1,7 @@ package backer +// Config main config struct. type ( - // Config main config struct. Config struct { Address string `json:"address"` Port int `json:"port"` @@ -15,10 +15,10 @@ type ( LogLevel string `json:"loglevel"` Directories []string `json:"directories"` BackupTimeout int `json:"backup_timeout"` // Timeout in minutes for backup streaming. - CompressionLevel int `json:"compression_level"` // Gzip compression level (1-9, default 9). + CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9). ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup. FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup"). - CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip or bzip2 (default: "gzip"). + CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. } ) diff --git a/internal/backer/utils.go b/internal/backer/utils.go deleted file mode 100644 index 364aa5f..0000000 --- a/internal/backer/utils.go +++ /dev/null @@ -1,884 +0,0 @@ -package backer - -import ( - "archive/tar" - "compress/gzip" - "context" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "backer/internal/log" - - "github.com/dsnet/compress/bzip2" - "github.com/klauspost/compress/zstd" - "github.com/klauspost/pgzip" - "github.com/pierrec/lz4" - "github.com/ulikunitz/xz" -) - -// CreateTarGzStream takes a list of files and returns a reader that reads from a tar archive containing these files. -func CreateTarGzStream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error // Track if we close pipe with error. - - // Helper to close pipe with error, only once. - closePipeWithError := func(err error) { - if pipeErr != nil { - return // Already closed with error. - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - // Only do normal close if we haven't closed with error. - if pipeErr == nil { - pw.Close() - } - }() - - gw, err := gzip.NewWriterLevel(pw, C.CompressionLevel) - if err != nil { - closePipeWithError(fmt.Errorf("gzip writer error: %w", err)) - - return - } - - defer func() { - // Only close gzip writer if pipe is still open. - if pipeErr == nil { - if err := gw.Close(); err != nil { - // Ignore pipe closed errors - reader may have closed early. - if !isPipeClosedError(err) { - log.Errorf("Failed to close gzip writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(gw) - - defer func() { - // Only close tar writer if pipe is still open. - if pipeErr == nil { - if err := tw.Close(); err != nil { - // Ignore pipe closed errors - reader may have closed early. - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - // Check for cancellation early. - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - // Skip sockets explicitly (they cannot be backed up). - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - // Write header with cancellation awareness. - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - // Only stream file contents for regular files. - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// CreateTarBzip2Stream takes a list of files and returns a reader that reads from a bzip2-compressed tar archive. -func CreateTarBzip2Stream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error - - closePipeWithError := func(err error) { - if pipeErr != nil { - return - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - if pipeErr == nil { - pw.Close() - } - }() - - bw, err := bzip2.NewWriter(pw, nil) - if err != nil { - closePipeWithError(fmt.Errorf("bzip2 writer error: %w", err)) - - return - } - - defer func() { - if pipeErr == nil { - if err := bw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close bzip2 writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(bw) - - defer func() { - if pipeErr == nil { - if err := tw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// zstdLevel returns the zstd encoder level based on the 1-9 compression level. -func zstdLevel(level int) zstd.EncoderLevel { - switch { - case level <= 3: - return zstd.SpeedFastest - case level <= 6: - return zstd.SpeedDefault - default: - return zstd.SpeedBetterCompression - } -} - -// CreateTarZstdStream takes a list of files and returns a reader that reads from a zstd-compressed tar archive. -func CreateTarZstdStream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error - - closePipeWithError := func(err error) { - if pipeErr != nil { - return - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - if pipeErr == nil { - pw.Close() - } - }() - - zw, err := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstdLevel(C.CompressionLevel))) - if err != nil { - closePipeWithError(fmt.Errorf("zstd writer error: %w", err)) - - return - } - - defer func() { - if pipeErr == nil { - if err := zw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close zstd writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(zw) - - defer func() { - if pipeErr == nil { - if err := tw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// CreateTarLz4Stream takes a list of files and returns a reader that reads from an lz4-compressed tar archive. -func CreateTarLz4Stream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error - - closePipeWithError := func(err error) { - if pipeErr != nil { - return - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - if pipeErr == nil { - pw.Close() - } - }() - - lw := lz4.NewWriter(pw) - - defer func() { - if pipeErr == nil { - if err := lw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close lz4 writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(lw) - - defer func() { - if pipeErr == nil { - if err := tw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// CreateTarXzStream takes a list of files and returns a reader that reads from an xz-compressed tar archive. -func CreateTarXzStream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error - - closePipeWithError := func(err error) { - if pipeErr != nil { - return - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - if pipeErr == nil { - pw.Close() - } - }() - - xw, err := xz.NewWriter(pw) - if err != nil { - closePipeWithError(fmt.Errorf("xz writer error: %w", err)) - - return - } - - defer func() { - if pipeErr == nil { - if err := xw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close xz writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(xw) - - defer func() { - if pipeErr == nil { - if err := tw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// CreateTarPgzipStream takes a list of files and returns a reader that reads from a parallel gzip-compressed tar archive. -func CreateTarPgzipStream(ctx context.Context, filepaths []string) io.ReadCloser { - pr, pw := io.Pipe() - - go func() { - var pipeErr error - - closePipeWithError := func(err error) { - if pipeErr != nil { - return - } - - pipeErr = err - pw.CloseWithError(err) - } - - defer func() { - if pipeErr == nil { - pw.Close() - } - }() - - gw, err := pgzip.NewWriterLevel(pw, C.CompressionLevel) - if err != nil { - closePipeWithError(fmt.Errorf("pgzip writer error: %w", err)) - - return - } - - defer func() { - if pipeErr == nil { - if err := gw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close pgzip writer: %v", err) - } - } - } - }() - - tw := tar.NewWriter(gw) - - defer func() { - if pipeErr == nil { - if err := tw.Close(); err != nil { - if !isPipeClosedError(err) { - log.Errorf("Failed to close tar writer: %v", err) - } - } - } - }() - - for _, fpath := range filepaths { - var ( - err error - linkTarget string - ) - - select { - case <-ctx.Done(): - closePipeWithError(ctx.Err()) - - return - - default: - } - - log.Debugf("Adding file: %s", fpath) - - st, err := os.Lstat(fpath) - - if err != nil { - log.Warnf("Skipping %s: %v", fpath, err) - - continue - } - - if st.Mode()&os.ModeSymlink != 0 { - linkTarget, err = os.Readlink(fpath) - - if err != nil { - log.Warnf("Skipping %s: readlink failed: %v", fpath, err) - - continue - } - } - - if st.Mode()&os.ModeSocket != 0 { - log.Debugf("Skipping socket: %s", fpath) - - continue - } - - header, err := tar.FileInfoHeader(st, linkTarget) - - if err != nil { - log.Warnf("Skipping %s: tar header failed: %v", fpath, err) - - continue - } - - header.Format = tar.FormatGNU - header.Name = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(fpath)), "/") - - mode := st.Mode() - - if err := writeWithContext(ctx, func() error { - return tw.WriteHeader(header) - }); err != nil { - closePipeWithError(err) - - return - } - - if mode.IsRegular() { - f, err := os.Open(fpath) - if err != nil { - log.Warnf("Skipping %s: open failed: %v", fpath, err) - - continue - } - - err = copyWithContext(ctx, tw, f) - - if e := f.Close(); e != nil { - log.Warnf("Failed to close file %s: %v", fpath, e) - } - - if err != nil { - log.Warnf("Skipping %s: copy failed: %v", fpath, err) - - continue - } - } - } - }() - - return pr -} - -// isPipeClosedError checks if the error is due to a closed pipe. -func isPipeClosedError(err error) bool { - if err == nil { - return false - } - - // Check for the specific pipe closed error message. - return err.Error() == "io: read/write on closed pipe" || - err.Error() == "io: read on closed pipe" -} - -/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/xz.go b/internal/backer/xz.go new file mode 100644 index 0000000..e7b8411 --- /dev/null +++ b/internal/backer/xz.go @@ -0,0 +1,17 @@ +package backer + +import ( + "context" + "io" + + "github.com/ulikunitz/xz" +) + +// CreateTarXzStream takes a list of files and returns a reader that reads from an xz-compressed tar archive. +func CreateTarXzStream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return xz.NewWriter(w) + }) +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/internal/backer/zstd.go b/internal/backer/zstd.go new file mode 100644 index 0000000..0846cd1 --- /dev/null +++ b/internal/backer/zstd.go @@ -0,0 +1,29 @@ +package backer + +import ( + "context" + "io" + + "github.com/klauspost/compress/zstd" +) + +// CreateTarZstdStream takes a list of files and returns a reader that reads from a zstd-compressed tar archive. +func CreateTarZstdStream(ctx context.Context, filepaths []string) io.ReadCloser { + return createArchiveStream(ctx, filepaths, func(w io.Writer) (io.WriteCloser, error) { + return zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel(C.CompressionLevel))) + }) +} + +// zstdLevel returns the zstd encoder level based on the 1-9 compression level. +func zstdLevel(level int) zstd.EncoderLevel { + switch { + case level <= 3: + return zstd.SpeedFastest + case level <= 6: + return zstd.SpeedDefault + default: + return zstd.SpeedBetterCompression + } +} + +/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/llms.txt b/llms.txt index 116e110..e336148 100644 --- a/llms.txt +++ b/llms.txt @@ -1,7 +1,7 @@ # llms.txt for Backer ## Overview -Backer is a Go-based backup utility that creates tar.gz archives on-the-fly and serves them via HTTP/HTTPS with basic authentication. +Backer is a Go-based backup utility that creates compressed archives on-the-fly and serves them via HTTP/HTTPS with basic authentication. Supports gzip, pgzip, bzip2, zstd, lz4, xz compression. ## Quick Start ```bash @@ -26,10 +26,16 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz ├── internal/ │ ├── backer/ # Core logic │ │ ├── types.go # Config struct definition +│ │ ├── consts.go # Named constants for magic numbers │ │ ├── globals.go # Global C Config variable +│ │ ├── archive.go # Common archive logic (createArchiveStream, writeFilesToTar) +│ │ ├── gzip.go # CreateTarGzStream, CreateTarPgzipStream +│ │ ├── bzip2.go # CreateTarBzip2Stream +│ │ ├── zstd.go # CreateTarZstdStream +│ │ ├── lz4.go # CreateTarLz4Stream +│ │ ├── xz.go # CreateTarXzStream │ │ ├── loadConfig.go # HJSON/JSON config parsing with validation │ │ ├── newServer.go # HTTP server, auth handler, context helpers -│ │ ├── utils.go # CreateTarGzStream, isPipeClosedError │ │ ├── getFilesFromDirectories.go # Directory walking, exclusion filtering │ │ └── *_test.go # Test files │ └── log/main.go # Logging wrapper around stdlib slog @@ -40,7 +46,6 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz ├── test_data/ # Test fixtures ├── data/config_example.json # Example config ├── Makefile # Build targets -├── TODO.org # Project backlog ├── README.md # User documentation └── LICENSE # MIT License ``` @@ -92,7 +97,7 @@ Uses `pipeErr` tracking to avoid noisy errors when reader closes early. Skips files that can't be read (logs warning, continues backup). ### Content-Disposition Filename -Backup filenames include timestamp: `backup-20060102-150405.tar.gz` +Backup filenames include timestamp and configured prefix: `backup-20060102-150405.tar.gz` (extension depends on compression_algorithm). ### Special File Types - **Device files** (character/block) are included with correct major/minor numbers. @@ -102,7 +107,8 @@ Backup filenames include timestamp: `backup-20060102-150405.tar.gz` - **Regular files** are compressed; other file types have no data section. ### Error Recovery -- `writeWithContext` and `copyWithContext` recover from panics in callbacks and return an error. +- `writeWithContext` checks context before calling fn. +- `copyWithContext` does buffered copy with cancellation checks. ### Performance Notes - Exclude patterns are precompiled after config load for faster matching. @@ -154,7 +160,7 @@ if C.Port == 0 { } if C.Port < 1 || C.Port > 65535 { - return fmt.Errorf("Port must be between 1 and 65535, got %d", C.Port) + return fmt.Errorf("Config option port must be between 1 and 65535, got %d", C.Port) } ``` @@ -202,6 +208,3 @@ Version is injected at build time via ldflags: ```go var version = "dev" // overridden by: -X main.version=$(VERSION) ``` - -## Known Issues / TODO -See TODO.org for current backlog.