Fixed bug with file extentions, enhance logging
- Fix 404 on downloading archive with extension - Add downloaded archive length to log - Add md5 hash to log
This commit is contained in:
@@ -173,7 +173,7 @@ func writeFilesToTar(ctx context.Context, filepaths []string, tw *tar.Writer, cl
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err = copyWithContext(ctx, tw, f)
|
_, _, err = copyWithContext(ctx, tw, f)
|
||||||
|
|
||||||
if e := f.Close(); e != nil {
|
if e := f.Close(); e != nil {
|
||||||
log.Warnf("Failed to close file %s: %v", fpath, e)
|
log.Warnf("Failed to close file %s: %v", fpath, e)
|
||||||
@@ -188,7 +188,7 @@ func writeFilesToTar(ctx context.Context, filepaths []string, tw *tar.Writer, cl
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// getInode extracts the inode number from os.FileInfo.
|
// getInode extracts the inode number from [os.FileInfo].
|
||||||
// Returns 0 if unable to determine (e.g., on Windows or with certain file systems).
|
// Returns 0 if unable to determine (e.g., on Windows or with certain file systems).
|
||||||
func getInode(fi os.FileInfo) uint64 {
|
func getInode(fi os.FileInfo) uint64 {
|
||||||
if fi.Sys() == nil {
|
if fi.Sys() == nil {
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ package backer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
"crypto/subtle"
|
"crypto/subtle"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -71,7 +73,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
mux.HandleFunc(C.Location, func(w http.ResponseWriter, r *http.Request) {
|
backupHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||||
username, password, ok := r.BasicAuth()
|
username, password, ok := r.BasicAuth()
|
||||||
|
|
||||||
// No auth? Good bye.
|
// No auth? Good bye.
|
||||||
@@ -105,7 +107,6 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Failed to get files: %v", err)
|
log.Errorf("Failed to get files: %v", err)
|
||||||
|
|
||||||
w.Header().Set("Server", "backer")
|
|
||||||
http.Error(w, "Failed to get files", http.StatusInternalServerError)
|
http.Error(w, "Failed to get files", http.StatusInternalServerError)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -121,7 +122,7 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
algorithm string
|
algorithm string
|
||||||
)
|
)
|
||||||
|
|
||||||
requestedExt := strings.TrimPrefix(path.Ext(r.URL.Path), ".")
|
requestedExt := r.URL.Path
|
||||||
|
|
||||||
algorithm = getCompressionAlgorithm(requestedExt)
|
algorithm = getCompressionAlgorithm(requestedExt)
|
||||||
|
|
||||||
@@ -132,7 +133,11 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
|
|
||||||
case "zstd":
|
case "zstd":
|
||||||
archive = CreateTarZstdStream(ctx, files)
|
archive = CreateTarZstdStream(ctx, files)
|
||||||
|
if strings.HasSuffix(requestedExt, ".zst") {
|
||||||
extension = "tar.zst"
|
extension = "tar.zst"
|
||||||
|
} else {
|
||||||
|
extension = "tar.zstd"
|
||||||
|
}
|
||||||
|
|
||||||
case "lz4":
|
case "lz4":
|
||||||
archive = CreateTarLz4Stream(ctx, files)
|
archive = CreateTarLz4Stream(ctx, files)
|
||||||
@@ -157,23 +162,43 @@ func NewServer(configPath string) (*ServerWrapper, error) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
timestamp := time.Now().Format("20060102-150405")
|
timestamp := backupStart.Format("20060102-150405")
|
||||||
filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension)
|
filename := fmt.Sprintf("%s-%s.%s", C.FilenamePrefix, timestamp, extension)
|
||||||
|
|
||||||
w.Header().Set("Content-Disposition", "attachment; filename="+filename)
|
w.Header().Set("Content-Disposition", "attachment; filename="+filename)
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
|
||||||
bytesWritten, err := io.Copy(w, archive)
|
bytesWritten, md5Hash, err := copyWithContext(ctx, w, archive)
|
||||||
|
if err != nil {
|
||||||
duration := time.Since(backupStart)
|
duration := time.Since(backupStart)
|
||||||
|
|
||||||
if err != nil {
|
if errors.Is(err, context.Canceled) {
|
||||||
|
log.Infof("Backup canceled: client=%s user=%s duration=%s", clientIP, username, duration)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if isPipeClosedError(err) {
|
||||||
|
log.Infof("Client disconnected during backup: client=%s user=%s", clientIP, username)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
log.Errorf("Backup failed: client=%s user=%s duration=%s error=%v", clientIP, username, duration, err)
|
log.Errorf("Backup failed: client=%s user=%s duration=%s error=%v", clientIP, username, duration, err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Backup completed: client=%s user=%s files=%d bytes=%d duration=%s", clientIP, username, len(files), bytesWritten, duration)
|
log.Infof("Backup completed: client=%s user=%s files=%d bytes=%d md5=%s duration=%s", clientIP, username, len(files), bytesWritten, md5Hash, time.Since(backupStart))
|
||||||
})
|
}
|
||||||
|
|
||||||
|
mux.Handle(C.Location, http.HandlerFunc(backupHandler))
|
||||||
|
|
||||||
|
extensions := []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.lz4", ".tar.zst", ".tar.zstd"}
|
||||||
|
for _, ext := range extensions {
|
||||||
|
muxPath := C.Location + ext
|
||||||
|
mux.Handle(muxPath, http.HandlerFunc(backupHandler))
|
||||||
|
}
|
||||||
|
|
||||||
// HTTP mode if nohttps is enabled in config.
|
// HTTP mode if nohttps is enabled in config.
|
||||||
// HTTPS mode if nohttps is disabled in config.
|
// HTTPS mode if nohttps is disabled in config.
|
||||||
@@ -226,15 +251,20 @@ func writeWithContext(ctx context.Context, fn func() error) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// copyWithContext copies data from src to dst using a 32KB buffer, respecting
|
// copyWithContext copies data from src to dst using a 32KB buffer, respecting
|
||||||
// context cancellation. If the context is canceled, the copy stops and returns
|
// context cancellation. Returns bytes written, MD5 hash (hex string), and any error encountered.
|
||||||
// ctx.Err().
|
//
|
||||||
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error {
|
//nolint:gosec
|
||||||
|
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, string, error) {
|
||||||
buf := make([]byte, copyBufferSize)
|
buf := make([]byte, copyBufferSize)
|
||||||
|
|
||||||
|
var bytesWritten int64
|
||||||
|
|
||||||
|
hasher := md5.New()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return bytesWritten, "", ctx.Err()
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,21 +272,29 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error {
|
|||||||
if n > 0 {
|
if n > 0 {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return bytesWritten, "", ctx.Err()
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, werr := dst.Write(buf[:n]); werr != nil {
|
if _, werr := hasher.Write(buf[:n]); werr != nil {
|
||||||
return werr
|
return bytesWritten, "", werr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if _, werr := dst.Write(buf[:n]); werr != nil {
|
||||||
|
return bytesWritten, "", werr
|
||||||
|
}
|
||||||
|
|
||||||
|
bytesWritten += int64(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
if readErr == io.EOF {
|
if readErr == io.EOF {
|
||||||
return nil
|
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||||
|
|
||||||
|
return bytesWritten, hash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return readErr
|
return bytesWritten, "", readErr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -279,17 +317,39 @@ func getClientIP(r *http.Request) string {
|
|||||||
// getCompressionAlgorithm determines which compression algorithm to use based on the requested extension.
|
// getCompressionAlgorithm determines which compression algorithm to use based on the requested extension.
|
||||||
// If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default.
|
// If the extension is recognized, it returns the corresponding algorithm; otherwise, it returns the default.
|
||||||
// For .tar.gz extension, pgzip is always used for parallel compression.
|
// For .tar.gz extension, pgzip is always used for parallel compression.
|
||||||
func getCompressionAlgorithm(requestedExt string) string {
|
// The input can be either a full path like "/archive.tar.xz" or just the extension like "tar.xz" or "xz".
|
||||||
switch requestedExt {
|
func getCompressionAlgorithm(input string) string {
|
||||||
case "tar.bz2":
|
// Extract extension from full path (e.g., "/archive.tar.xz" -> ".xz").
|
||||||
|
// or use the input directly if it's just an extension (e.g., "tar.xz" -> ".xz").
|
||||||
|
ext := path.Ext(input)
|
||||||
|
if ext == "" {
|
||||||
|
ext = "." + input
|
||||||
|
}
|
||||||
|
|
||||||
|
switch ext {
|
||||||
|
case ".tar.bz2":
|
||||||
return "bzip2"
|
return "bzip2"
|
||||||
case "tar.zst":
|
case ".tar.zstd":
|
||||||
return "zstd"
|
return "zstd"
|
||||||
case "tar.lz4":
|
case ".tar.zst":
|
||||||
|
return "zstd"
|
||||||
|
case ".tar.lz4":
|
||||||
return "lz4"
|
return "lz4"
|
||||||
case "tar.xz":
|
case ".tar.xz":
|
||||||
return "xz"
|
return "xz"
|
||||||
case "tar.gz":
|
case ".tar.gz":
|
||||||
|
return "pgzip"
|
||||||
|
case ".xz":
|
||||||
|
return "xz"
|
||||||
|
case ".bz2":
|
||||||
|
return "bzip2"
|
||||||
|
case ".zst":
|
||||||
|
return "zstd"
|
||||||
|
case ".zstd":
|
||||||
|
return "zstd"
|
||||||
|
case ".lz4":
|
||||||
|
return "lz4"
|
||||||
|
case ".gz":
|
||||||
return "pgzip"
|
return "pgzip"
|
||||||
default:
|
default:
|
||||||
return C.DefaultCompression
|
return C.DefaultCompression
|
||||||
|
|||||||
@@ -463,7 +463,7 @@ func TestCopyWithContextSuccess(t *testing.T) {
|
|||||||
src := strings.NewReader("hello world")
|
src := strings.NewReader("hello world")
|
||||||
dst := &strings.Builder{}
|
dst := &strings.Builder{}
|
||||||
|
|
||||||
err := copyWithContext(ctx, dst, src)
|
_, _, err := copyWithContext(ctx, dst, src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Expected no error, got %v", err)
|
t.Errorf("Expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -473,6 +473,25 @@ func TestCopyWithContextSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCopyWithContextMD5Hash tests that MD5 hash is correctly computed.
|
||||||
|
func TestCopyWithContextMD5Hash(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
data := "hello world"
|
||||||
|
src := strings.NewReader(data)
|
||||||
|
dst := &strings.Builder{}
|
||||||
|
|
||||||
|
_, md5Hash, err := copyWithContext(ctx, dst, src)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MD5 of "hello world" is 5eb63bbbe01eeed093cb22bb8f5acdc3
|
||||||
|
expectedMD5 := "5eb63bbbe01eeed093cb22bb8f5acdc3"
|
||||||
|
if md5Hash != expectedMD5 {
|
||||||
|
t.Errorf("Expected MD5 %s, got %s", expectedMD5, md5Hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestCopyWithContextCancellation tests copy with cancelled context.
|
// TestCopyWithContextCancellation tests copy with cancelled context.
|
||||||
func TestCopyWithContextCancellation(t *testing.T) {
|
func TestCopyWithContextCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -487,14 +506,18 @@ func TestCopyWithContextCancellation(t *testing.T) {
|
|||||||
|
|
||||||
// Start copy in goroutine
|
// Start copy in goroutine
|
||||||
errCh := make(chan error, 1)
|
errCh := make(chan error, 1)
|
||||||
|
bytesCh := make(chan int64, 1)
|
||||||
go func() {
|
go func() {
|
||||||
errCh <- copyWithContext(ctx, dst, src)
|
n, _, err := copyWithContext(ctx, dst, src)
|
||||||
|
bytesCh <- n
|
||||||
|
errCh <- err
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Cancel context immediately
|
// Cancel context immediately
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
err := <-errCh
|
err := <-errCh
|
||||||
|
<-bytesCh
|
||||||
if err != context.Canceled {
|
if err != context.Canceled {
|
||||||
t.Errorf("Expected context.Canceled, got %v", err)
|
t.Errorf("Expected context.Canceled, got %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user