Add ability to select compression algo by downloading location with file extension.
- Also update README.md with more useful examples.
This commit is contained in:
@@ -198,7 +198,8 @@ Config is JSON (HJSON with comments supported). Default path depends on OS:
|
|||||||
| compression_level | `9` | Gzip level (1-9) |
|
| compression_level | `9` | Gzip level (1-9) |
|
||||||
| exclude_patterns | `[]` | Regex patterns to exclude from backup |
|
| exclude_patterns | `[]` | Regex patterns to exclude from backup |
|
||||||
| filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header |
|
| filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header |
|
||||||
| compression_algorithm | `gzip` | Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz |
|
| default_compression | `gzip` | Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz |
|
||||||
|
| compression_algorithm | `gzip` | Alias for default_compression (deprecated) |
|
||||||
|
|
||||||
## Architecture & Key Patterns
|
## Architecture & Key Patterns
|
||||||
|
|
||||||
|
|||||||
19
README.md
19
README.md
@@ -28,10 +28,22 @@ If you running FreeBSD, use the rc.d script `contrib/backer.freebsd` and place i
|
|||||||
Build, install and configure backer. On your backup storage server you can do
|
Build, install and configure backer. On your backup storage server you can do
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -u username:password https://your_server:8086/archive -o "backup-$(date +%Y%m%d-%H%M%S).tar.gz"
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive
|
||||||
```
|
```
|
||||||
|
|
||||||
The filename and extension depend on your `filename_prefix` and `compression_algorithm` settings.
|
The filename and extension depend on your `filename_prefix` and `default_compression` settings.
|
||||||
|
|
||||||
|
You can also override compression by specifying the archive extension in the URL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.xz
|
||||||
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.gz # Uses pgzip
|
||||||
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.bz2
|
||||||
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.zst
|
||||||
|
wget --quiet --no-check-certificate --content-disposition --http-user=username --http-password=password https://your_server:8086/archive.lz4
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: `/archive.tar.gz` always uses pgzip (parallel gzip) for better performance.
|
||||||
|
|
||||||
Of course, something can go wrong, so you must check http status code after downloading backup.
|
Of course, something can go wrong, so you must check http status code after downloading backup.
|
||||||
|
|
||||||
@@ -56,7 +68,8 @@ All options described in example config.
|
|||||||
| compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). |
|
| compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). |
|
||||||
| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. |
|
| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. |
|
||||||
| filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. |
|
| filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. |
|
||||||
| compression_algorithm | "gzip" | Compression algorithm. | Options: gzip, pgzip, bzip2, zstd, lz4, xz. |
|
| default_compression | "gzip" | Default compression algorithm. | Options: gzip, pgzip, bzip2, zstd, lz4, xz. |
|
||||||
|
| compression_algorithm | "gzip" | Alias for default_compression. | You should use default_compression instead. |
|
||||||
|
|
||||||
## Compression algorithms
|
## Compression algorithms
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ pidfile="/run/$SVCNAME/$SVCNAME.pid"
|
|||||||
|
|
||||||
depend() {
|
depend() {
|
||||||
need net
|
need net
|
||||||
|
use dns
|
||||||
}
|
}
|
||||||
|
|
||||||
start_pre() {
|
start_pre() {
|
||||||
@@ -26,27 +26,30 @@
|
|||||||
/* Where to store logs. */
|
/* Where to store logs. */
|
||||||
"log": "/var/log/backer.log",
|
"log": "/var/log/backer.log",
|
||||||
|
|
||||||
/* Verbosity of logs */
|
/* Verbosity of logs. */
|
||||||
"loglevel": "info",
|
"loglevel": "info",
|
||||||
|
|
||||||
/* Diretories to backup */
|
|
||||||
"directories": [
|
|
||||||
"/path/to/directory1",
|
|
||||||
"/path/to/directory2"
|
|
||||||
],
|
|
||||||
|
|
||||||
/* Backup streaming timeout in minutes. */
|
/* Backup streaming timeout in minutes. */
|
||||||
"backup_timeout": 60,
|
"backup_timeout": 60,
|
||||||
|
|
||||||
/* Gzip compression level (1-9, where 9 is best compression). */
|
/* Gzip compression level (1-9, where 9 is best compression). */
|
||||||
"compression_level": 9,
|
"compression_level": 9,
|
||||||
|
|
||||||
/* Regex patterns to exclude from backup. */
|
|
||||||
"exclude_patterns": [],
|
|
||||||
|
|
||||||
/* Prefix for backup filename in Content-Disposition header. */
|
/* Prefix for backup filename in Content-Disposition header. */
|
||||||
"filename_prefix": "backup",
|
"filename_prefix": "backup",
|
||||||
|
|
||||||
/* Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. */
|
/* Compression algorithm alias (deprecated: use default_compression). */
|
||||||
"compression_algorithm": "gzip"
|
"compression_algorithm": "gzip",
|
||||||
|
|
||||||
|
/* Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz. */
|
||||||
|
"default_compression": "gzip",
|
||||||
|
|
||||||
|
/* Regex patterns to exclude from backup. */
|
||||||
|
"exclude_patterns": [],
|
||||||
|
|
||||||
|
/* Directories to backup. */
|
||||||
|
"directories": [
|
||||||
|
"/path/to/directory1",
|
||||||
|
"/path/to/directory2"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,8 +50,8 @@ const (
|
|||||||
// defaultFilenamePrefix is the default prefix for backup filenames.
|
// defaultFilenamePrefix is the default prefix for backup filenames.
|
||||||
defaultFilenamePrefix = "backup"
|
defaultFilenamePrefix = "backup"
|
||||||
|
|
||||||
// defaultCompressionAlgorithm is the default compression algorithm.
|
// defaultDefaultCompression is the default compression algorithm.
|
||||||
defaultCompressionAlgorithm = "gzip"
|
defaultDefaultCompression = "gzip"
|
||||||
)
|
)
|
||||||
|
|
||||||
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
||||||
|
|||||||
@@ -135,15 +135,20 @@ func LoadConfig(path string) error {
|
|||||||
log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix)
|
log.Warnf("Config option filename_prefix is not set, fallback to %s", C.FilenamePrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompressionAlgorithm: default and validate.
|
// DefaultCompression: default and validate. If compression_algorithm is set (deprecated), use it as fallback.
|
||||||
if C.CompressionAlgorithm == "" {
|
if C.DefaultCompression == "" {
|
||||||
C.CompressionAlgorithm = defaultCompressionAlgorithm
|
if C.CompressionAlgorithm != "" {
|
||||||
|
C.DefaultCompression = C.CompressionAlgorithm
|
||||||
log.Warnf("Config option compression_algorithm is not set, fallback to %s", C.CompressionAlgorithm)
|
} else {
|
||||||
|
C.DefaultCompression = defaultDefaultCompression
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "gzip" && C.CompressionAlgorithm != "bzip2" && C.CompressionAlgorithm != "zstd" && C.CompressionAlgorithm != "lz4" && C.CompressionAlgorithm != "xz" && C.CompressionAlgorithm != "pgzip" {
|
switch C.DefaultCompression {
|
||||||
return fmt.Errorf("Config option compression_algorithm must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.CompressionAlgorithm)
|
case "gzip", "pgzip", "bzip2", "xz", "lz4", "zstd":
|
||||||
|
log.Infof("Using %s as defailt compression algorithm", C.DefaultCompression)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("Config option default_compression must be gzip, bzip2, zstd, lz4, xz or pgzip, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compile exclude patterns.
|
// Compile exclude patterns.
|
||||||
|
|||||||
@@ -227,8 +227,8 @@ func TestLoadConfigDefaultValues(t *testing.T) {
|
|||||||
t.Errorf("C.FilenamePrefix should default to backup, got %s", C.FilenamePrefix)
|
t.Errorf("C.FilenamePrefix should default to backup, got %s", C.FilenamePrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "gzip" {
|
if C.DefaultCompression != "gzip" {
|
||||||
t.Errorf("C.CompressionAlgorithm should default to gzip, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should default to gzip, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -248,8 +248,8 @@ func TestLoadConfigCompressionAlgorithmBzip2(t *testing.T) {
|
|||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "bzip2" {
|
if C.DefaultCompression != "bzip2" {
|
||||||
t.Errorf("C.CompressionAlgorithm should be bzip2, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should be bzip2, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,8 +269,8 @@ func TestLoadConfigCompressionAlgorithmZstd(t *testing.T) {
|
|||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "zstd" {
|
if C.DefaultCompression != "zstd" {
|
||||||
t.Errorf("C.CompressionAlgorithm should be zstd, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should be zstd, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -290,8 +290,8 @@ func TestLoadConfigCompressionAlgorithmLz4(t *testing.T) {
|
|||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "lz4" {
|
if C.DefaultCompression != "lz4" {
|
||||||
t.Errorf("C.CompressionAlgorithm should be lz4, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should be lz4, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -311,8 +311,8 @@ func TestLoadConfigCompressionAlgorithmXz(t *testing.T) {
|
|||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "xz" {
|
if C.DefaultCompression != "xz" {
|
||||||
t.Errorf("C.CompressionAlgorithm should be xz, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should be xz, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,8 +332,8 @@ func TestLoadConfigCompressionAlgorithmPgzip(t *testing.T) {
|
|||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if C.CompressionAlgorithm != "pgzip" {
|
if C.DefaultCompression != "pgzip" {
|
||||||
t.Errorf("C.CompressionAlgorithm should be pgzip, got %s", C.CompressionAlgorithm)
|
t.Errorf("C.DefaultCompression should be pgzip, got %s", C.DefaultCompression)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -76,9 +77,14 @@ func NewServer(configPath string) (*http.Server, error) {
|
|||||||
var (
|
var (
|
||||||
archive io.ReadCloser
|
archive io.ReadCloser
|
||||||
extension string
|
extension string
|
||||||
|
algorithm string
|
||||||
)
|
)
|
||||||
|
|
||||||
switch C.CompressionAlgorithm {
|
requestedExt := strings.TrimPrefix(path.Ext(r.URL.Path), ".")
|
||||||
|
|
||||||
|
algorithm = getCompressionAlgorithm(requestedExt)
|
||||||
|
|
||||||
|
switch algorithm {
|
||||||
case "bzip2":
|
case "bzip2":
|
||||||
archive = CreateTarBzip2Stream(ctx, files)
|
archive = CreateTarBzip2Stream(ctx, files)
|
||||||
extension = "tar.bz2"
|
extension = "tar.bz2"
|
||||||
@@ -207,4 +213,24 @@ func getClientIP(r *http.Request) string {
|
|||||||
return host
|
return host
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getCompressionAlgorithm determines which compression algorithm to use based on the requested extension.
|
||||||
|
// If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default.
|
||||||
|
// For .tar.gz extension, pgzip is always used for parallel compression.
|
||||||
|
func getCompressionAlgorithm(requestedExt string) string {
|
||||||
|
switch requestedExt {
|
||||||
|
case "tar.bz2":
|
||||||
|
return "bzip2"
|
||||||
|
case "tar.zst":
|
||||||
|
return "zstd"
|
||||||
|
case "tar.lz4":
|
||||||
|
return "lz4"
|
||||||
|
case "tar.xz":
|
||||||
|
return "xz"
|
||||||
|
case "tar.gz":
|
||||||
|
return "pgzip"
|
||||||
|
default:
|
||||||
|
return C.DefaultCompression
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ type (
|
|||||||
CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9).
|
CompressionLevel int `json:"compression_level"` // Compression level (1-9, default 9).
|
||||||
ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup.
|
ExcludePatterns []string `json:"exclude_patterns"` // Regex patterns to exclude from backup.
|
||||||
FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup").
|
FilenamePrefix string `json:"filename_prefix"` // Prefix for backup filename (default: "backup").
|
||||||
CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz.
|
CompressionAlgorithm string `json:"compression_algorithm"` // Compression algorithm alias (deprecated: use default_compression).
|
||||||
|
DefaultCompression string `json:"default_compression"` // Default compression algorithm: gzip, pgzip, bzip2, zstd, lz4 or xz.
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user