Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b87d520b61 | ||
|
|
cc83c61561 | ||
|
|
4fc69bcf3a | ||
|
|
e4b6a84c2f | ||
|
|
0581d7430c | ||
|
|
b35851567c |
10
AGENTS.md
10
AGENTS.md
@@ -207,6 +207,7 @@ Config is JSON (HJSON with comments supported). Default path depends on OS:
|
|||||||
| loglevel | `info` | Log verbosity: error/warn/info/debug |
|
| loglevel | `info` | Log verbosity: error/warn/info/debug |
|
||||||
| directories | — | Directories to backup (required, validated for existence) |
|
| directories | — | Directories to backup (required, validated for existence) |
|
||||||
| backup_timeout | `60` | Stream timeout in minutes (1-1440) |
|
| backup_timeout | `60` | Stream timeout in minutes (1-1440) |
|
||||||
|
| dir_scan_timeout | `60` | Directory scan timeout in minutes (1-1440) |
|
||||||
| compression_level | `9` | Gzip level (1-9) |
|
| compression_level | `9` | Gzip level (1-9) |
|
||||||
| exclude_patterns | `[]` | Regex patterns to exclude from backup |
|
| exclude_patterns | `[]` | Regex patterns to exclude from backup |
|
||||||
| filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header |
|
| filename_prefix | `backup` | Prefix for backup filename in Content-Disposition header |
|
||||||
@@ -302,7 +303,6 @@ The server uses `http.NewServeMux()` for routing. Multiple routes are registered
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
mux.Handle(C.Location, http.HandlerFunc(backupHandler)) // /archive
|
mux.Handle(C.Location, http.HandlerFunc(backupHandler)) // /archive
|
||||||
mux.Handle(C.Location+"/", http.HandlerFunc(backupHandler)) // /archive/
|
|
||||||
mux.Handle(C.Location+".tar.gz", http.HandlerFunc(backupHandler)) // /archive.tar.gz
|
mux.Handle(C.Location+".tar.gz", http.HandlerFunc(backupHandler)) // /archive.tar.gz
|
||||||
mux.Handle(C.Location+".tar.xz", http.HandlerFunc(backupHandler)) // /archive.tar.xz
|
mux.Handle(C.Location+".tar.xz", http.HandlerFunc(backupHandler)) // /archive.tar.xz
|
||||||
mux.Handle(C.Location+".tar.bz2", http.HandlerFunc(backupHandler)) // /archive.tar.bz2
|
mux.Handle(C.Location+".tar.bz2", http.HandlerFunc(backupHandler)) // /archive.tar.bz2
|
||||||
@@ -318,6 +318,14 @@ The `getCompressionAlgorithm()` function handles both full paths and bare extens
|
|||||||
- Bare extension: `tar.xz` → `.tar.xz` → xz
|
- Bare extension: `tar.xz` → `.tar.xz` → xz
|
||||||
- Single ext: `xz` → `.xz` → xz
|
- Single ext: `xz` → `.xz` → xz
|
||||||
|
|
||||||
|
### Unmatched Route Logging
|
||||||
|
|
||||||
|
The server wraps the `http.ResponseWriter` to capture status codes and log unmatched routes:
|
||||||
|
- **404 Not Found**: Logged as `"Not Found from {clientIP} to {path}"`
|
||||||
|
- **405 Method Not Allowed**: Logged as `"Method Not Allowed from {clientIP} for {method} {path}"`
|
||||||
|
|
||||||
|
This uses a `responseWriterWrapper` that tracks the status code via the `WriteHeader()` method.
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
- `github.com/hjson/hjson-go` v3.3.0 — HJSON config parsing
|
- `github.com/hjson/hjson-go` v3.3.0 — HJSON config parsing
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ All options described in example config.
|
|||||||
| loglevel | "info" | Verbosity of logs. | Options: error, warn, info, debug. |
|
| loglevel | "info" | Verbosity of logs. | Options: error, warn, info, debug. |
|
||||||
| directories | — | Directories to backup. | Required. Array of paths. |
|
| directories | — | Directories to backup. | Required. Array of paths. |
|
||||||
| backup_timeout | 60 | Timeout in minutes for backup streaming. | Range: 1-1440 minutes. |
|
| backup_timeout | 60 | Timeout in minutes for backup streaming. | Range: 1-1440 minutes. |
|
||||||
|
| dir_scan_timeout | 60 | Timeout in minutes for getting file list from all backupped directories. | Range: 1-1440 minutes. |
|
||||||
| compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). |
|
| compression_level | 9 | Compression level. | 1 (fastest) to 9 (best compression). |
|
||||||
| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. |
|
| exclude_patterns | [] | Regex patterns to exclude from backup. | E.g., `[".*\\.tmp$", "/node_modules/"]`. |
|
||||||
| filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. |
|
| filename_prefix | "backup" | Prefix for backup filename. | E.g., "mybackup" produces `mybackup-20260325-092341.tar.gz`. |
|
||||||
|
|||||||
@@ -47,6 +47,9 @@
|
|||||||
/* Regex patterns to exclude from backup. */
|
/* Regex patterns to exclude from backup. */
|
||||||
"exclude_patterns": [],
|
"exclude_patterns": [],
|
||||||
|
|
||||||
|
/* Maximum allowed time, spent for getting file list from backupped directories. */
|
||||||
|
"dir_scan_timeout": 60,
|
||||||
|
|
||||||
/* Directories to backup. */
|
/* Directories to backup. */
|
||||||
"directories": [
|
"directories": [
|
||||||
"/path/to/directory1",
|
"/path/to/directory1",
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
|
||||||
|
|
||||||
backerlog "backer/internal/log"
|
backerlog "backer/internal/log"
|
||||||
)
|
)
|
||||||
@@ -81,13 +80,4 @@ func GetFilesFromDirectories(ctx context.Context, directories []string) ([]strin
|
|||||||
return files, nil
|
return files, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetFilesFromDirectoriesWithTimeout makes a file list of given directories with timeout.
|
|
||||||
// Convenience wrapper that creates context from timeout in minutes.
|
|
||||||
func GetFilesFromDirectoriesWithTimeout(timeoutMinutes int, directories []string) ([]string, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMinutes)*time.Minute)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
return GetFilesFromDirectories(ctx, directories)
|
|
||||||
}
|
|
||||||
|
|
||||||
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
||||||
|
|||||||
@@ -15,9 +15,18 @@ func normalizePath(p string) string {
|
|||||||
return strings.ReplaceAll(p, "\\", "/")
|
return strings.ReplaceAll(p, "\\", "/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// normalizePathSlice normalizes all paths in a slice.
|
||||||
|
func normalizePathSlice(paths []string) []string {
|
||||||
|
result := make([]string, len(paths))
|
||||||
|
for i, p := range paths {
|
||||||
|
result[i] = normalizePath(p)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// TestGetFilesFromDirectories checks if GetFilesFromDirectories returns the proper amount of strings, correct strings and no error.
|
// TestGetFilesFromDirectories checks if GetFilesFromDirectories returns the proper amount of strings, correct strings and no error.
|
||||||
func TestGetFilesFromDirectories(t *testing.T) {
|
func TestGetFilesFromDirectories(t *testing.T) {
|
||||||
testDataPath := filepath.Join("..", "test_data")
|
testDataPath := "../../test_data"
|
||||||
|
|
||||||
// Log the paths we're checking for debugging
|
// Log the paths we're checking for debugging
|
||||||
t.Logf("Checking directories: %v", []string{
|
t.Logf("Checking directories: %v", []string{
|
||||||
@@ -61,7 +70,8 @@ func TestGetFilesFromDirectories(t *testing.T) {
|
|||||||
|
|
||||||
for _, filepath := range output {
|
for _, filepath := range output {
|
||||||
normalized := normalizePath(filepath)
|
normalized := normalizePath(filepath)
|
||||||
if !slices.Contains(expectedData, normalized) {
|
normalizedExpected := normalizePathSlice(expectedData)
|
||||||
|
if !slices.Contains(normalizedExpected, normalized) {
|
||||||
t.Errorf("expectedData data does not contain string from output: %s", filepath)
|
t.Errorf("expectedData data does not contain string from output: %s", filepath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -69,7 +79,7 @@ func TestGetFilesFromDirectories(t *testing.T) {
|
|||||||
|
|
||||||
// TestGetFilesFromDirectoriesSingleDir tests with a single directory.
|
// TestGetFilesFromDirectoriesSingleDir tests with a single directory.
|
||||||
func TestGetFilesFromDirectoriesSingleDir(t *testing.T) {
|
func TestGetFilesFromDirectoriesSingleDir(t *testing.T) {
|
||||||
testDataPath := filepath.Join("test_data")
|
testDataPath := filepath.Join("../../test_data")
|
||||||
output, err := GetFilesFromDirectories(context.Background(), []string{filepath.Join(testDataPath, "test1", "foo")})
|
output, err := GetFilesFromDirectories(context.Background(), []string{filepath.Join(testDataPath, "test1", "foo")})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -106,7 +116,7 @@ func TestGetFilesFromDirectoriesEmptyDir(t *testing.T) {
|
|||||||
// TestGetFilesFromDirectoriesNonExistent tests with non-existent directory.
|
// TestGetFilesFromDirectoriesNonExistent tests with non-existent directory.
|
||||||
// Should log error but not fail - returns empty list.
|
// Should log error but not fail - returns empty list.
|
||||||
func TestGetFilesFromDirectoriesNonExistent(t *testing.T) {
|
func TestGetFilesFromDirectoriesNonExistent(t *testing.T) {
|
||||||
testDataPath := filepath.Join("test_data")
|
testDataPath := filepath.Join("../../test_data")
|
||||||
nonExistentPath := filepath.Join(testDataPath, "nonexistent", "path", "that", "does", "not", "exist")
|
nonExistentPath := filepath.Join(testDataPath, "nonexistent", "path", "that", "does", "not", "exist")
|
||||||
output, err := GetFilesFromDirectories(context.Background(), []string{nonExistentPath})
|
output, err := GetFilesFromDirectories(context.Background(), []string{nonExistentPath})
|
||||||
|
|
||||||
@@ -123,7 +133,7 @@ func TestGetFilesFromDirectoriesNonExistent(t *testing.T) {
|
|||||||
|
|
||||||
// TestGetFilesFromDirectoriesPartialFailure tests with mix of valid and invalid directories.
|
// TestGetFilesFromDirectoriesPartialFailure tests with mix of valid and invalid directories.
|
||||||
func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
|
func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
|
||||||
testDataPath := filepath.Join("test_data")
|
testDataPath := filepath.Join("../../test_data")
|
||||||
// Create a temp directory that exists
|
// Create a temp directory that exists
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
@@ -142,7 +152,7 @@ func TestGetFilesFromDirectoriesPartialFailure(t *testing.T) {
|
|||||||
|
|
||||||
// TestGetFilesFromDirectoriesMultipleDirs tests multiple directories.
|
// TestGetFilesFromDirectoriesMultipleDirs tests multiple directories.
|
||||||
func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) {
|
func TestGetFilesFromDirectoriesMultipleDirs(t *testing.T) {
|
||||||
testDataPath := filepath.Join("test_data")
|
testDataPath := filepath.Join("../../test_data")
|
||||||
output, err := GetFilesFromDirectories(context.Background(), []string{
|
output, err := GetFilesFromDirectories(context.Background(), []string{
|
||||||
filepath.Join(testDataPath, "test1", "foo"),
|
filepath.Join(testDataPath, "test1", "foo"),
|
||||||
filepath.Join(testDataPath, "test1", "bar"),
|
filepath.Join(testDataPath, "test1", "bar"),
|
||||||
|
|||||||
@@ -48,6 +48,23 @@ func isTLSError(err error) bool {
|
|||||||
return strings.HasPrefix(err.Error(), "tls:")
|
return strings.HasPrefix(err.Error(), "tls:")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withStatusLogging wraps a handler to log 404 and 405 responses.
|
||||||
|
func withStatusLogging(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
rw := &responseWriterWrapper{ResponseWriter: w, statusCode: http.StatusOK}
|
||||||
|
next.ServeHTTP(rw, r)
|
||||||
|
|
||||||
|
clientIP := getClientIP(r)
|
||||||
|
|
||||||
|
switch rw.statusCode {
|
||||||
|
case http.StatusNotFound:
|
||||||
|
log.Infof("Not Found from %s to %s", clientIP, r.RequestURI)
|
||||||
|
case http.StatusMethodNotAllowed:
|
||||||
|
log.Infof("Method Not Allowed from %s for %s %s", clientIP, r.Method, r.RequestURI)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// withServerHeader wraps a handler to add Server header to all responses.
|
// withServerHeader wraps a handler to add Server header to all responses.
|
||||||
func withServerHeader(next http.Handler) http.Handler {
|
func withServerHeader(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -74,6 +91,13 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
backupHandler := func(w http.ResponseWriter, r *http.Request) {
|
backupHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Only GET method is allowed.
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
username, password, ok := r.BasicAuth()
|
username, password, ok := r.BasicAuth()
|
||||||
|
|
||||||
// No auth? Good bye.
|
// No auth? Good bye.
|
||||||
@@ -219,7 +243,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
srv := &ServerWrapper{
|
srv := &ServerWrapper{
|
||||||
Server: &http.Server{
|
Server: &http.Server{
|
||||||
Addr: fmt.Sprintf("%s:%d", C.Address, C.Port),
|
Addr: fmt.Sprintf("%s:%d", C.Address, C.Port),
|
||||||
Handler: withServerHeader(mux),
|
Handler: withServerHeader(withStatusLogging(mux)),
|
||||||
ReadHeaderTimeout: readHeaderTimeout,
|
ReadHeaderTimeout: readHeaderTimeout,
|
||||||
WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute,
|
WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute,
|
||||||
IdleTimeout: idleTimeout,
|
IdleTimeout: idleTimeout,
|
||||||
@@ -235,7 +259,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
srv := &ServerWrapper{
|
srv := &ServerWrapper{
|
||||||
Server: &http.Server{
|
Server: &http.Server{
|
||||||
Addr: fmt.Sprintf("%s:%d", C.Address, C.Port),
|
Addr: fmt.Sprintf("%s:%d", C.Address, C.Port),
|
||||||
Handler: withServerHeader(mux),
|
Handler: withServerHeader(withStatusLogging(mux)),
|
||||||
ReadHeaderTimeout: readHeaderTimeout,
|
ReadHeaderTimeout: readHeaderTimeout,
|
||||||
WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute,
|
WriteTimeout: time.Duration(C.BackupTimeout) * time.Minute,
|
||||||
IdleTimeout: idleTimeout,
|
IdleTimeout: idleTimeout,
|
||||||
@@ -374,8 +398,20 @@ type (
|
|||||||
ServerWrapper struct {
|
ServerWrapper struct {
|
||||||
*http.Server
|
*http.Server
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// responseWriterWrapper wraps http.ResponseWriter to capture status code.
|
||||||
|
responseWriterWrapper struct {
|
||||||
|
http.ResponseWriter
|
||||||
|
statusCode int
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// WriteHeader captures the status code for logging.
|
||||||
|
func (rw *responseWriterWrapper) WriteHeader(code int) {
|
||||||
|
rw.statusCode = code
|
||||||
|
rw.ResponseWriter.WriteHeader(code)
|
||||||
|
}
|
||||||
|
|
||||||
// Serve starts the server and classifies any errors returned.
|
// Serve starts the server and classifies any errors returned.
|
||||||
// TLS-related errors are logged at debug level, other errors at warn level.
|
// TLS-related errors are logged at debug level, other errors at warn level.
|
||||||
func (sw *ServerWrapper) Serve() error {
|
func (sw *ServerWrapper) Serve() error {
|
||||||
|
|||||||
@@ -523,4 +523,99 @@ func TestCopyWithContextCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestStatusLogging404 tests that 404 responses are logged.
|
||||||
|
func TestStatusLogging404(t *testing.T) {
|
||||||
|
C = Config{} // Reset global config.
|
||||||
|
excludePatterns = nil
|
||||||
|
|
||||||
|
// Create a minimal mux that returns 404 for all paths
|
||||||
|
testMux := http.NewServeMux()
|
||||||
|
testMux.HandleFunc("/archive", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedMux := withStatusLogging(testMux)
|
||||||
|
|
||||||
|
// Use a ResponseRecorder to capture the response
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
req, _ := http.NewRequest("GET", "/nonexistent", nil)
|
||||||
|
req.RemoteAddr = "127.0.0.1:12345"
|
||||||
|
|
||||||
|
wrappedMux.ServeHTTP(rr, req)
|
||||||
|
|
||||||
|
if rr.Code != http.StatusNotFound {
|
||||||
|
t.Errorf("Expected status 404, got %d", rr.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatusLogging405 tests that 405 responses are logged.
|
||||||
|
func TestStatusLogging405(t *testing.T) {
|
||||||
|
C = Config{} // Reset global config.
|
||||||
|
excludePatterns = nil
|
||||||
|
|
||||||
|
// Create a minimal mux that returns 405 for wrong method
|
||||||
|
testMux := http.NewServeMux()
|
||||||
|
testMux.HandleFunc("/archive", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||||
|
})
|
||||||
|
|
||||||
|
wrappedMux := withStatusLogging(testMux)
|
||||||
|
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
req, _ := http.NewRequest("GET", "/archive", nil)
|
||||||
|
req.RemoteAddr = "127.0.0.1:12345"
|
||||||
|
|
||||||
|
wrappedMux.ServeHTTP(rr, req)
|
||||||
|
|
||||||
|
if rr.Code != http.StatusMethodNotAllowed {
|
||||||
|
t.Errorf("Expected status 405, got %d", rr.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMethodNotAllowedHandler tests that non-GET methods return 405.
|
||||||
|
func TestMethodNotAllowedHandler(t *testing.T) {
|
||||||
|
C = Config{
|
||||||
|
Address: "0.0.0.0",
|
||||||
|
Port: 8086,
|
||||||
|
Location: "/archive",
|
||||||
|
User: "testuser",
|
||||||
|
Password: "testpass",
|
||||||
|
Directories: []string{"../../test_data/test1/foo"},
|
||||||
|
}
|
||||||
|
excludePatterns = nil
|
||||||
|
|
||||||
|
// Create test server
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
username, password, ok := r.BasicAuth()
|
||||||
|
if !ok || username != C.User || password != C.Password {
|
||||||
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
w.Header().Set("Server", "backer")
|
||||||
|
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
// Test POST method
|
||||||
|
req, _ := http.NewRequest("POST", server.URL+"/archive", nil)
|
||||||
|
req.SetBasicAuth("testuser", "testpass")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusMethodNotAllowed {
|
||||||
|
t.Errorf("Expected status 405, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
||||||
|
|||||||
107
llms.txt
107
llms.txt
@@ -30,13 +30,13 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz
|
|||||||
│ │ ├── globals.go # Global C Config variable
|
│ │ ├── globals.go # Global C Config variable
|
||||||
│ │ ├── archive.go # Common archive logic (createArchiveStream, writeFilesToTar)
|
│ │ ├── archive.go # Common archive logic (createArchiveStream, writeFilesToTar)
|
||||||
│ │ ├── gzip.go # CreateTarGzStream, CreateTarPgzipStream
|
│ │ ├── gzip.go # CreateTarGzStream, CreateTarPgzipStream
|
||||||
│ │ ├── bzip2.go # CreateTarBzip2Stream
|
│ │ ├── bzip2.go # CreateTarBzip2Stream, mapCompressionLevelToBzip2
|
||||||
│ │ ├── zstd.go # CreateTarZstdStream
|
│ │ ├── zstd.go # CreateTarZstdStream, zstdLevel
|
||||||
│ │ ├── lz4.go # CreateTarLz4Stream
|
│ │ ├── lz4.go # CreateTarLz4Stream, mapCompressionLevelToLz4
|
||||||
│ │ ├── xz.go # CreateTarXzStream
|
│ │ ├── xz.go # CreateTarXzStream
|
||||||
│ │ ├── loadConfig.go # HJSON/JSON config parsing with validation
|
│ │ ├── loadConfig.go # HJSON/JSON config parsing with validation
|
||||||
│ │ ├── newServer.go # HTTP server, auth handler, context helpers
|
│ │ ├── newServer.go # HTTP server, auth handler, context helpers
|
||||||
│ │ ├── getFilesFromDirectories.go # Directory walking, exclusion filtering
|
│ │ ├── getFilesFromDirectories.go # Directory walking, exclusion filtering, isExcluded
|
||||||
│ │ └── *_test.go # Test files
|
│ │ └── *_test.go # Test files
|
||||||
│ └── log/main.go # Logging wrapper around stdlib slog
|
│ └── log/main.go # Logging wrapper around stdlib slog
|
||||||
├── contrib/ # Init scripts
|
├── contrib/ # Init scripts
|
||||||
@@ -55,7 +55,7 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz
|
|||||||
|--------|---------|-------------|
|
|--------|---------|-------------|
|
||||||
| address | 0.0.0.0 | Listen address |
|
| address | 0.0.0.0 | Listen address |
|
||||||
| port | 8086 | Listen port |
|
| port | 8086 | Listen port |
|
||||||
| cert/key | - | TLS cert/key paths |
|
| cert/key | - | TLS cert/key paths (required when nohttps=false) |
|
||||||
| nohttps | false | Disable HTTPS |
|
| nohttps | false | Disable HTTPS |
|
||||||
| location | /archive | API endpoint path |
|
| location | /archive | API endpoint path |
|
||||||
| user | - | Basic auth username (required) |
|
| user | - | Basic auth username (required) |
|
||||||
@@ -64,10 +64,23 @@ curl -u user:pass https://server:8086/archive -o backup.tar.gz
|
|||||||
| loglevel | info | Log verbosity (debug/info/warn/error) |
|
| loglevel | info | Log verbosity (debug/info/warn/error) |
|
||||||
| directories | - | Directories to backup (required) |
|
| directories | - | Directories to backup (required) |
|
||||||
| backup_timeout | 60 | Stream timeout in minutes (1-1440) |
|
| backup_timeout | 60 | Stream timeout in minutes (1-1440) |
|
||||||
| compression_level | 9 | Gzip level (1-9) |
|
| dir_scan_timeout | 60 | Directory scan timeout in minutes (1-1440) |
|
||||||
|
| compression_level | 9 | Compression level (1-9) |
|
||||||
| exclude_patterns | [] | Regex patterns to exclude from backup |
|
| exclude_patterns | [] | Regex patterns to exclude from backup |
|
||||||
| filename_prefix | backup | Prefix for backup filename in Content-Disposition header |
|
| filename_prefix | backup | Prefix for backup filename in Content-Disposition header |
|
||||||
| compression_algorithm | gzip | Compression algorithm (gzip, pgzip, bzip2, zstd, lz4 or xz) |
|
| default_compression | gzip | Default compression algorithm (gzip, pgzip, bzip2, zstd, lz4, xz) |
|
||||||
|
| compression_algorithm | gzip | Alias for default_compression (deprecated) |
|
||||||
|
|
||||||
|
## Compression Level Mapping
|
||||||
|
|
||||||
|
The `compression_level` config option (1-9) is mapped differently for each algorithm:
|
||||||
|
|
||||||
|
| Algorithm | Level Mapping |
|
||||||
|
|-----------|---------------|
|
||||||
|
| gzip/pgzip/bzip2 | 1-9 directly maps to compression levels |
|
||||||
|
| lz4 | 1-3 → Fastest (0), 4-6 → Default (1), 7-9 → Best (2) |
|
||||||
|
| zstd | 1-3 → Fastest, 4-6 → Default, 7-9 → BetterCompression |
|
||||||
|
| xz | Uses default compression (no configurable level) |
|
||||||
|
|
||||||
## Key Patterns
|
## Key Patterns
|
||||||
|
|
||||||
@@ -85,16 +98,36 @@ json.Marshal(tmp)
|
|||||||
json.Unmarshal(buf, &C)
|
json.Unmarshal(buf, &C)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Stream Cancellation
|
### Context-Aware I/O
|
||||||
Context-aware write and copy functions:
|
Two helper functions in newServer.go handle context cancellation during streaming:
|
||||||
|
- `writeWithContext(ctx, fn)` — checks context before calling fn
|
||||||
|
- `copyWithContext(ctx, dst, src)` — buffered copy (32KB) with cancellation checks, computes MD5 hash
|
||||||
|
|
||||||
```go
|
```go
|
||||||
func writeWithContext(ctx context.Context, fn func() error) error
|
func writeWithContext(ctx context.Context, fn func() error) error
|
||||||
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error
|
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, string, error)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### HTTP Server Middleware
|
||||||
|
Two middleware functions wrap the handler:
|
||||||
|
- `withServerHeader` — adds "Server: backer" header to all responses
|
||||||
|
- `withStatusLogging` — logs 404 and 405 responses
|
||||||
|
|
||||||
|
```go
|
||||||
|
Handler: withServerHeader(withStatusLogging(mux))
|
||||||
|
```
|
||||||
|
|
||||||
|
### Unmatched Route Logging
|
||||||
|
The server wraps http.ResponseWriter to capture status codes:
|
||||||
|
- **404 Not Found**: `"Not Found from {clientIP} to {path}"`
|
||||||
|
- **405 Method Not Allowed**: `"Method Not Allowed from {clientIP} for {method} {path}"`
|
||||||
|
|
||||||
|
Uses `responseWriterWrapper` that tracks status code via WriteHeader() method.
|
||||||
|
|
||||||
### Pipe Error Handling
|
### Pipe Error Handling
|
||||||
Uses `pipeErr` tracking to avoid noisy errors when reader closes early.
|
Uses `pipeErr` tracking to avoid noisy errors when reader closes early.
|
||||||
Skips files that can't be read (logs warning, continues backup).
|
Skips files that can't be read (logs warning, continues backup).
|
||||||
|
- `isPipeClosedError()` checks for "io: read/write on closed pipe" errors
|
||||||
|
|
||||||
### Content-Disposition Filename
|
### Content-Disposition Filename
|
||||||
Backup filenames include timestamp and configured prefix: `backup-20060102-150405.tar.gz` (extension depends on compression_algorithm).
|
Backup filenames include timestamp and configured prefix: `backup-20060102-150405.tar.gz` (extension depends on compression_algorithm).
|
||||||
@@ -102,18 +135,43 @@ Backup filenames include timestamp and configured prefix: `backup-20060102-15040
|
|||||||
### Special File Types
|
### Special File Types
|
||||||
- **Device files** (character/block) are included with correct major/minor numbers.
|
- **Device files** (character/block) are included with correct major/minor numbers.
|
||||||
- **Sockets** are skipped (no entry created).
|
- **Sockets** are skipped (no entry created).
|
||||||
- **Symlinks** are stored as symlinks (target path stored).
|
- **Symlinks** are stored as symlinks (target path stored) - uses os.Lstat, no cycles possible.
|
||||||
- **Named pipes** and other special files are included (tar header created).
|
- **Named pipes** and other special files are included (tar header created).
|
||||||
|
- **Hard links** are deduplicated - first occurrence stored with content, subsequent as link entries.
|
||||||
- **Regular files** are compressed; other file types have no data section.
|
- **Regular files** are compressed; other file types have no data section.
|
||||||
|
|
||||||
### Error Recovery
|
### Error Recovery
|
||||||
- `writeWithContext` checks context before calling fn.
|
- `writeWithContext` checks context before calling fn.
|
||||||
- `copyWithContext` does buffered copy with cancellation checks.
|
- `copyWithContext` does buffered copy with cancellation checks.
|
||||||
|
|
||||||
|
### Client IP Detection
|
||||||
|
Checks `X-Forwarded-For` header first (for proxied requests), then falls back to `RemoteAddr`.
|
||||||
|
|
||||||
|
### TLS Error Handling
|
||||||
|
Uses `ServerWrapper` to intercept and classify errors from Serve() and ServeTLS():
|
||||||
|
- TLS-specific errors logged at debug level to reduce log clutter
|
||||||
|
- Uses errors.As to detect: tls.AlertError, tls.CertificateVerificationError, tls.ECHRejectionError, tls.RecordHeaderError
|
||||||
|
|
||||||
|
### HTTP Routing & Compression Selection
|
||||||
|
The server uses http.NewServeMux() for routing. Multiple routes support different compression formats:
|
||||||
|
```go
|
||||||
|
mux.Handle(C.Location, http.HandlerFunc(backupHandler)) // /archive
|
||||||
|
mux.Handle(C.Location+".tar.gz", http.HandlerFunc(backupHandler)) // archive.tar.gz
|
||||||
|
mux.Handle(C.Location+".tar.xz", http.HandlerFunc(backupHandler)) // archive.tar.xz
|
||||||
|
mux.Handle(C.Location+".tar.bz2", http.HandlerFunc(backupHandler)) // archive.tar.bz2
|
||||||
|
// ... etc (.tar.lz4, .tar.zst, .tar.zstd)
|
||||||
|
```
|
||||||
|
|
||||||
|
Compression algorithm determined by path extension using `path.Ext()`:
|
||||||
|
- Full path: `/archive.tar.xz` → `.xz` → xz
|
||||||
|
- Bare extension: `tar.xz` → `.xz` → xz
|
||||||
|
- Default when no extension matches
|
||||||
|
|
||||||
### Performance Notes
|
### Performance Notes
|
||||||
- Exclude patterns are precompiled after config load for faster matching.
|
- Exclude patterns are precompiled after config load for faster matching.
|
||||||
- Compression level is configurable (1-9); default 9 uses more CPU.
|
- Compression level is configurable (1-9); default 9 uses more CPU.
|
||||||
- Large directory walks use `filepath.WalkDir` (efficient).
|
- Large directory walks use `filepath.WalkDir` (efficient).
|
||||||
|
- Context cancellation checked on each file during directory scan.
|
||||||
|
|
||||||
## Coding Style & Conventions
|
## Coding Style & Conventions
|
||||||
|
|
||||||
@@ -128,7 +186,8 @@ GetFilesFromDirectories()
|
|||||||
// Unexported (private): verbNoun or isNoun pattern
|
// Unexported (private): verbNoun or isNoun pattern
|
||||||
writeWithContext()
|
writeWithContext()
|
||||||
copyWithContext()
|
copyWithContext()
|
||||||
isPipeClosedError() // predicate helper
|
isPipeClosedError()
|
||||||
|
isExcluded()
|
||||||
defaultConfigPath()
|
defaultConfigPath()
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -137,6 +196,7 @@ Struct fields use PascalCase, json tags use snake_case:
|
|||||||
```go
|
```go
|
||||||
BackupTimeout int `json:"backup_timeout"`
|
BackupTimeout int `json:"backup_timeout"`
|
||||||
CompressionLevel int `json:"compression_level"`
|
CompressionLevel int `json:"compression_level"`
|
||||||
|
DirScanTimeout int `json:"dir_scan_timeout"`
|
||||||
```
|
```
|
||||||
|
|
||||||
### Error Messages
|
### Error Messages
|
||||||
@@ -149,6 +209,7 @@ CompressionLevel int `json:"compression_level"`
|
|||||||
- Past tense: `"Failed to close gzip writer: %v", err`
|
- Past tense: `"Failed to close gzip writer: %v", err`
|
||||||
- Context included: `"Skipping %s: %v", filename, err`
|
- Context included: `"Skipping %s: %v", filename, err`
|
||||||
- Appropriate level: Warn for recoverable, Error for issues, Fatal for startup failures
|
- Appropriate level: Warn for recoverable, Error for issues, Fatal for startup failures
|
||||||
|
- Log levels: error, warn, info, debug
|
||||||
|
|
||||||
### Config Validation Pattern
|
### Config Validation Pattern
|
||||||
For each config option: set default first, then immediately validate:
|
For each config option: set default first, then immediately validate:
|
||||||
@@ -166,7 +227,7 @@ if C.Port < 1 || C.Port > 65535 {
|
|||||||
|
|
||||||
### Go Formatting
|
### Go Formatting
|
||||||
- Tabs for indentation
|
- Tabs for indentation
|
||||||
- Vim modeline at EOF: `/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */`
|
- Vim modeline at EOF: `/* vim: setlocal ft=go noet ai ts=4 sw=4 sts=4: */
|
||||||
|
|
||||||
### Linter Suppressions (intentional)
|
### Linter Suppressions (intentional)
|
||||||
| Rule | Reason |
|
| Rule | Reason |
|
||||||
@@ -186,10 +247,11 @@ go test -v -run TestName ./... # Run specific test
|
|||||||
golangci-lint run ./... # Lint
|
golangci-lint run ./... # Lint
|
||||||
```
|
```
|
||||||
|
|
||||||
Coverage: ~69% on internal/backer (main and log packages not covered by design)
|
Coverage: ~69% on internal/backer (~100% on internal/log)
|
||||||
|
|
||||||
### Test Conventions
|
### Test Conventions
|
||||||
- Reset global config: `C = Config{}`
|
- Reset global config: `C = Config{}`
|
||||||
|
- Reset exclude patterns: `excludePatterns = nil`
|
||||||
- Use `t.TempDir()` for temporary files
|
- Use `t.TempDir()` for temporary files
|
||||||
- Helper function `writeTempConfig()` for config tests
|
- Helper function `writeTempConfig()` for config tests
|
||||||
|
|
||||||
@@ -208,3 +270,20 @@ Version is injected at build time via ldflags:
|
|||||||
```go
|
```go
|
||||||
var version = "dev" // overridden by: -X main.version=$(VERSION)
|
var version = "dev" // overridden by: -X main.version=$(VERSION)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Request Flow (Good Path)
|
||||||
|
```
|
||||||
|
Request → Method Check (GET) → Auth Check → Get Files (with timeout) → Create Archive → Stream → Complete
|
||||||
|
```
|
||||||
|
|
||||||
|
## Request Flow (Error Paths)
|
||||||
|
| Request | Response | Logged |
|
||||||
|
|---------|----------|--------|
|
||||||
|
| Non-GET method | 405 Method Not Allowed | Yes (info) |
|
||||||
|
| No auth | 401 Unauthorized | Yes (info) |
|
||||||
|
| Wrong credentials | 401 Unauthorized | Yes (info) |
|
||||||
|
| Unknown path | 404 Not Found | Yes (info) |
|
||||||
|
| Dir scan timeout | 504 Request Timeout | Yes (error) |
|
||||||
|
| Failed to get files | 500 Internal Server Error | Yes (error) |
|
||||||
|
| Client disconnect | - | Yes (info) |
|
||||||
|
| Backup completed | 200 OK | Yes (info) |
|
||||||
|
|||||||
@@ -5,10 +5,10 @@
|
|||||||
/* By default listen on port 8086, like first intel commercially successful cpu model. */
|
/* By default listen on port 8086, like first intel commercially successful cpu model. */
|
||||||
"port": 8086,
|
"port": 8086,
|
||||||
|
|
||||||
/* Path location of certificate file */
|
/* Path location of test certificate file */
|
||||||
"cert": "../../test_data/example.crt",
|
"cert": "../../test_data/example.crt",
|
||||||
|
|
||||||
/* Path location of certificate key */
|
/* Path location of test certificate key */
|
||||||
"key": "../../test_data/example.key",
|
"key": "../../test_data/example.key",
|
||||||
|
|
||||||
/* We really want to use https for backup archive tranferring. */
|
/* We really want to use https for backup archive tranferring. */
|
||||||
|
|||||||
Reference in New Issue
Block a user