Fixed bug with file extentions, enhance logging

- Fix 404 on downloading archive with extension
- Add downloaded archive length to log
- Add md5 hash to log
This commit is contained in:
eleksir
2026-04-04 21:45:33 +03:00
parent e063352252
commit 65a57a6af6
3 changed files with 113 additions and 30 deletions

View File

@@ -173,7 +173,7 @@ func writeFilesToTar(ctx context.Context, filepaths []string, tw *tar.Writer, cl
continue continue
} }
err = copyWithContext(ctx, tw, f) _, _, err = copyWithContext(ctx, tw, f)
if e := f.Close(); e != nil { if e := f.Close(); e != nil {
log.Warnf("Failed to close file %s: %v", fpath, e) log.Warnf("Failed to close file %s: %v", fpath, e)
@@ -188,7 +188,7 @@ func writeFilesToTar(ctx context.Context, filepaths []string, tw *tar.Writer, cl
} }
} }
// getInode extracts the inode number from os.FileInfo. // getInode extracts the inode number from [os.FileInfo].
// Returns 0 if unable to determine (e.g., on Windows or with certain file systems). // Returns 0 if unable to determine (e.g., on Windows or with certain file systems).
func getInode(fi os.FileInfo) uint64 { func getInode(fi os.FileInfo) uint64 {
if fi.Sys() == nil { if fi.Sys() == nil {

View File

@@ -2,8 +2,10 @@ package backer
import ( import (
"context" "context"
"crypto/md5"
"crypto/subtle" "crypto/subtle"
"crypto/tls" "crypto/tls"
"encoding/hex"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -71,7 +73,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc(C.Location, func(w http.ResponseWriter, r *http.Request) { backupHandler := func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth() username, password, ok := r.BasicAuth()
// No auth? Good bye. // No auth? Good bye.
@@ -105,7 +107,6 @@ func NewServer(configPath string) (*ServerWrapper, error) {
if err != nil { if err != nil {
log.Errorf("Failed to get files: %v", err) log.Errorf("Failed to get files: %v", err)
w.Header().Set("Server", "backer")
http.Error(w, "Failed to get files", http.StatusInternalServerError) http.Error(w, "Failed to get files", http.StatusInternalServerError)
return return
@@ -121,7 +122,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
algorithm string algorithm string
) )
requestedExt := strings.TrimPrefix(path.Ext(r.URL.Path), ".") requestedExt := r.URL.Path
algorithm = getCompressionAlgorithm(requestedExt) algorithm = getCompressionAlgorithm(requestedExt)
@@ -132,7 +133,11 @@ func NewServer(configPath string) (*ServerWrapper, error) {
case "zstd": case "zstd":
archive = CreateTarZstdStream(ctx, files) archive = CreateTarZstdStream(ctx, files)
if strings.HasSuffix(requestedExt, ".zst") {
extension = "tar.zst" extension = "tar.zst"
} else {
extension = "tar.zstd"
}
case "lz4": case "lz4":
archive = CreateTarLz4Stream(ctx, files) archive = CreateTarLz4Stream(ctx, files)
@@ -157,23 +162,43 @@ func NewServer(configPath string) (*ServerWrapper, error) {
} }
}() }()
timestamp := time.Now().Format("20060102-150405") timestamp := backupStart.Format("20060102-150405")
filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension) filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension)
w.Header().Set("Content-Disposition", "attachment; filename="+filename) w.Header().Set("Content-Disposition", "attachment; filename="+filename)
w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Type", "application/octet-stream")
bytesWritten, err := io.Copy(w, archive) bytesWritten, md5Hash, err := copyWithContext(ctx, w, archive)
if err != nil {
duration := time.Since(backupStart) duration := time.Since(backupStart)
if err != nil { if errors.Is(err, context.Canceled) {
log.Infof("Backup canceled: client=%s user=%s duration=%s", clientIP, username, duration)
return
}
if isPipeClosedError(err) {
log.Infof("Client disconnected during backup: client=%s user=%s", clientIP, username)
return
}
log.Errorf("Backup failed: client=%s user=%s duration=%s error=%v", clientIP, username, duration, err) log.Errorf("Backup failed: client=%s user=%s duration=%s error=%v", clientIP, username, duration, err)
return return
} }
log.Infof("Backup completed: client=%s user=%s files=%d bytes=%d duration=%s", clientIP, username, len(files), bytesWritten, duration) log.Infof("Backup completed: client=%s user=%s files=%d bytes=%d md5=%s duration=%s", clientIP, username, len(files), bytesWritten, md5Hash, time.Since(backupStart))
}) }
mux.Handle(C.Location, http.HandlerFunc(backupHandler))
extensions := []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.lz4", ".tar.zst", ".tar.zstd"}
for _, ext := range extensions {
muxPath := C.Location + ext
mux.Handle(muxPath, http.HandlerFunc(backupHandler))
}
// HTTP mode if nohttps is enabled in config. // HTTP mode if nohttps is enabled in config.
// HTTPS mode if nohttps is disabled in config. // HTTPS mode if nohttps is disabled in config.
@@ -226,15 +251,20 @@ func writeWithContext(ctx context.Context, fn func() error) error {
} }
// copyWithContext copies data from src to dst using a 32KB buffer, respecting // copyWithContext copies data from src to dst using a 32KB buffer, respecting
// context cancellation. If the context is canceled, the copy stops and returns // context cancellation. Returns bytes written, MD5 hash (hex string), and any error encountered.
// ctx.Err(). //
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { //nolint:gosec
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, string, error) {
buf := make([]byte, copyBufferSize) buf := make([]byte, copyBufferSize)
var bytesWritten int64
hasher := md5.New()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return bytesWritten, "", ctx.Err()
default: default:
} }
@@ -242,21 +272,29 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error {
if n > 0 { if n > 0 {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return bytesWritten, "", ctx.Err()
default: default:
} }
if _, werr := dst.Write(buf[:n]); werr != nil { if _, werr := hasher.Write(buf[:n]); werr != nil {
return werr return bytesWritten, "", werr
} }
if _, werr := dst.Write(buf[:n]); werr != nil {
return bytesWritten, "", werr
}
bytesWritten += int64(n)
} }
if readErr != nil { if readErr != nil {
if readErr == io.EOF { if readErr == io.EOF {
return nil hash := hex.EncodeToString(hasher.Sum(nil))
return bytesWritten, hash, nil
} }
return readErr return bytesWritten, "", readErr
} }
} }
} }
@@ -279,17 +317,39 @@ func getClientIP(r *http.Request) string {
// getCompressionAlgorithm determines which compression algorithm to use based on the requested extension. // getCompressionAlgorithm determines which compression algorithm to use based on the requested extension.
// If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default. // If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default.
// For .tar.gz extension, pgzip is always used for parallel compression. // For .tar.gz extension, pgzip is always used for parallel compression.
func getCompressionAlgorithm(requestedExt string) string { // The input can be either a full path like "/archive.tar.xz" or just the extension like "tar.xz" or "xz".
switch requestedExt { func getCompressionAlgorithm(input string) string {
case "tar.bz2": // Extract extension from full path (e.g., "/archive.tar.xz" -> ".xz").
// or use the input directly if it's just an extension (e.g., "tar.xz" -> ".xz").
ext := path.Ext(input)
if ext == "" {
ext = "." + input
}
switch ext {
case ".tar.bz2":
return "bzip2" return "bzip2"
case "tar.zst": case ".tar.zstd":
return "zstd" return "zstd"
case "tar.lz4": case ".tar.zst":
return "zstd"
case ".tar.lz4":
return "lz4" return "lz4"
case "tar.xz": case ".tar.xz":
return "xz" return "xz"
case "tar.gz": case ".tar.gz":
return "pgzip"
case ".xz":
return "xz"
case ".bz2":
return "bzip2"
case ".zst":
return "zstd"
case ".zstd":
return "zstd"
case ".lz4":
return "lz4"
case ".gz":
return "pgzip" return "pgzip"
default: default:
return C.DefaultCompression return C.DefaultCompression

View File

@@ -463,7 +463,7 @@ func TestCopyWithContextSuccess(t *testing.T) {
src := strings.NewReader("hello world") src := strings.NewReader("hello world")
dst := &strings.Builder{} dst := &strings.Builder{}
err := copyWithContext(ctx, dst, src) _, _, err := copyWithContext(ctx, dst, src)
if err != nil { if err != nil {
t.Errorf("Expected no error, got %v", err) t.Errorf("Expected no error, got %v", err)
} }
@@ -473,6 +473,25 @@ func TestCopyWithContextSuccess(t *testing.T) {
} }
} }
// TestCopyWithContextMD5Hash tests that MD5 hash is correctly computed.
func TestCopyWithContextMD5Hash(t *testing.T) {
ctx := context.Background()
data := "hello world"
src := strings.NewReader(data)
dst := &strings.Builder{}
_, md5Hash, err := copyWithContext(ctx, dst, src)
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
// MD5 of "hello world" is 5eb63bbbe01eeed093cb22bb8f5acdc3
expectedMD5 := "5eb63bbbe01eeed093cb22bb8f5acdc3"
if md5Hash != expectedMD5 {
t.Errorf("Expected MD5 %s, got %s", expectedMD5, md5Hash)
}
}
// TestCopyWithContextCancellation tests copy with cancelled context. // TestCopyWithContextCancellation tests copy with cancelled context.
func TestCopyWithContextCancellation(t *testing.T) { func TestCopyWithContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -487,14 +506,18 @@ func TestCopyWithContextCancellation(t *testing.T) {
// Start copy in goroutine // Start copy in goroutine
errCh := make(chan error, 1) errCh := make(chan error, 1)
bytesCh := make(chan int64, 1)
go func() { go func() {
errCh <- copyWithContext(ctx, dst, src) n, _, err := copyWithContext(ctx, dst, src)
bytesCh <- n
errCh <- err
}() }()
// Cancel context immediately // Cancel context immediately
cancel() cancel()
err := <-errCh err := <-errCh
<-bytesCh
if err != context.Canceled { if err != context.Canceled {
t.Errorf("Expected context.Canceled, got %v", err) t.Errorf("Expected context.Canceled, got %v", err)
} }